def LABVisualization(bins, slices, project=True,color=False,fullOpacity=True, data = "8bit"):
startTime = time.time()
colorMax = 255
if data == "8bitsegmented":
histogram = DataParse.loadStack(300, 200, 3000, mode="8bit", segmented=True)
elif data == "8bit":
histogram = DataParse.loadStack(300, 200, 3000, mode="8bit", segmented=False)
elif data == "12bit":
histogram = DataParse.loadStack(300, 200, 3000, mode="12bit", segmented=False)
colorMax = 4095
ColorPlots.histogramToLAB(histogram, colorMax, slices, project=project, color=color, fullOpacity=True)
endTime = time.time()
print "Total Time: %g Seconds" % (endTime-startTime)
def main():
arg_parser = ap.ArgumentParser(description='')
arg_parser.add_argument(
'--mode',
'-m',
choices=['gather', 'parse', 'analyze', 'all'],
default='all',
nargs='?',
help='gather, or parse, or analyze data, or do all three',
action="store_true")
arg_parser.add_argument('--help',
'-h',
help='usage options',
action=arg_parser.print_help)
args = arg_parser.parse_args()
mode = args.mode
if mode == 'all':
dg = DataGather()
dg.run()
dp = DataParse()
dp.run()
da = DataAnalyze()
da.run()
elif mode == 'gather':
dg = DataGather()
dg.run()
elif mode == 'parse':
dp = DataParse()
dp.run()
elif mode == 'analyze':
da = DataAnalyze()
da.run()
print(args)
def saveABPlot(slices, bins, floor, ceiling):
startTime = time.time()
histogram = DataParse.loadStack(slices, bins, floor, ceiling, bits=12, segmented=False, colorSpace="LAB")
filename = "AB Plot" + str(bins) + "Bins" + "Floor" + str(floor) + "Ceiling" + str(ceiling) + ".png"
ColorPlots.ABPlot(histogram, bins, filename)
endTime = time.time()
print "Total Time: %g Seconds" % (endTime-startTime)
print "For %g Bins" % bins
def __init__(self, name, jsonData, game):
self.geometry = []
self.name = name
# Load the json file
self.json = jsonData
print("Processing geometries...")
# print(json.dumps(self.json, indent=4, sort_keys=True))
if "[Male]" in name:
# Parse all the geometry indices for male items and parse the geometries
for geometryIndex in self.json["content"][0]["male_index_set"]["geometry"]:
geometryFile = self.json["content"][0]["geometry"][geometryIndex]
path = bungieUrlPrefix+bungieGeometryPrefix[game]+geometryFile
print("Geometry file: "+path)
request = urllib.request.Request(path, headers=headers)
response = urllib.request.urlopen(request)
data = DataParse.DataParse(response.read())
self.geometry.append(DestinyGeometry.parse(data))
elif "[Female]" in name:
# Parse all the geometry indices for female items and parse the geometries
for geometryIndex in self.json["content"][0]["female_index_set"]["geometry"]:
geometryFile = self.json["content"][0]["geometry"][geometryIndex]
path = bungieUrlPrefix+bungieGeometryPrefix[game]+geometryFile
print("Geometry file: "+path)
request = urllib.request.Request(path, headers=headers)
response = urllib.request.urlopen(request)
data = DataParse.DataParse(response.read())
self.geometry.append(DestinyGeometry.parse(data))
else:
# Get the geometry file names from the json and parse the geometries
for geometryFile in self.json["content"][0]["geometry"]:
path = bungieUrlPrefix+bungieGeometryPrefix[game]+geometryFile
print("Geometry file: "+path)
request = urllib.request.Request(path, headers=headers)
response = urllib.request.urlopen(request)
data = DataParse.DataParse(response.read())
self.geometry.append(DestinyGeometry.parse(data))
print("Done processing geometries...")
return
def setUp(self):
self.subreddit1 = SubredditClass('Food1')
self.subreddit2 = SubredditClass('Food2')
self.subreddit3 = SubredditClass('Food3')
self.Food_Nov = open('Food-Nov.json', 'r+').read()
self.Food_Dec = open('Food-Dec.json', 'r+').read()
self.Food_Jan = open('Food-Jan.json', 'r+').read()
self.food1 = json.loads(self.Food_Nov)
self.food2 = json.loads(self.Food_Dec)
self.food3 = json.loads(self.Food_Jan)
DataParse.json_data_extract(self.food1, self.subreddit1, 'Food1')
DataParse.json_data_extract(self.food2, self.subreddit2, 'Food2')
DataParse.json_data_extract(self.food3, self.subreddit3, 'Food3')
def run(self):
Thread.__init__(self)
def getInterface(expression):
ifs = findalldevs()
input = int(
expression) # turns into an integer for the function to accept
return ifs[input] # returns the inputted interface
def recv_pkts(hdr, data):
try:
currentInt = getInterface(self.expression)
if str(parser.learn_mode) == str(False):
eth = EthDecoder().decode(data)
ethChild = eth.child()
ethChild2 = ethChild.child()
if ethChild2.get_type() == 134:
parser.sniffSlaac(data, currentInt)
elif ethChild2.get_type() == 135:
parser.sniffSlaac(data, currentInt)
elif ethChild2.get_type() == 136:
parser.sniffSlaac(data, currentInt)
elif str(parser.learn_mode) == str(True):
parser.activateLearningMode(data)
except:
pass
pcapy.findalldevs()
max_bytes = 1024
promiscuous = False
read_timeout = 100
parser = DataParse.Dataparse(self.mode)
pc = pcapy.open_live(getInterface(self.expression), max_bytes,
promiscuous, read_timeout)
pc.setfilter('icmp6')
while self.isRunning is True:
pc.loop(1,
recv_pkts) # capture packets while the thread is running
def histogram():
histogram = DataParse.loadStack()
Draw.draw(histogram)
def test_json_data_extract(self):
DataParse.json_data_extract(self.animals1, self.Animals, 'Animals')
DataParse.json_data_extract(self.animals2, self.Animals, 'Animals')
DataParse.json_data_extract(self.animals3, self.Animals, 'Animals')
DataParse.json_data_extract(self.food1, self.Food, 'Food')
DataParse.json_data_extract(self.food2, self.Food, 'Food')
DataParse.json_data_extract(self.food3, self.Food, 'Food')
assert len(self.Food.posts) == 30
assert self.Food.posts[0].getClass() == 'Food'
assert self.Food.posts[29].getClass() == 'Food'
assert self.Food.posts[0].getTitle() == 'banana'
assert self.Food.posts[29].getTitle() == 'cracker'
assert self.Food.posts[0].selftext == 'apple'
assert self.Food.posts[29].selftext == 'apple'
assert len(self.Animals.posts) == 30
assert self.Animals.posts[0].getClass() == 'Animals'
assert self.Animals.posts[29].getClass() == 'Animals'
assert self.Animals.posts[0].getTitle() == 'cat'
assert self.Animals.posts[29].getTitle() == 'octopus'
assert self.Animals.posts[0].selftext == 'dog'
assert self.Animals.posts[29].selftext == 'dog'
def fourChannelDraw():
histogram = DataParse.fourChannelTest()
Draw.fourChannelDraw(histogram)
def sliceDraw():
histogram = DataParse.loadStack()
data, centroidList, distance = Cluster.kMeans(histogram, main.CLUSTERS)
Draw.drawSlice(data, centroidList)
def EM():
histogram = DataParse.loadStack()
MG, obs = Cluster.expectationMaximization(histogram, main.CLUSTERS)
Draw.EMDraw(obs, MG, histogram)
def volumeRender():
histogram = DataParse.loadStack(300, 200, 3000, mode="8bit", segmented=False)
Draw.volumeRender(histogram, mode="fullcolor", prune=0)
with open('Daily_Stats.txt', 'w') as outText:
for d in data:
outText.write(d.stat + '\n')
for player in d._players:
overallRank[player.name] += rankWeighting(player.rank)
playerTeamDict[player.name] = player.team
outStrings = [str(value).strip() for value in player]
outText.write(' | '.join(outStrings) + '\n')
outText.write('\n\n')
sortedRank = Counter(overallRank)
outText.write('Overall Statistical Standings\n')
for standing, player in enumerate(sortedRank.most_common(10), 1):
outText.write(
f'{standing} | {player[0]} | {playerTeamDict[player[0]]} | {player[1]} \n'
)
if __name__ == '__main__':
_year = datetime.datetime.now().strftime('%Y')
outDir = askdirectory(title='Select File to Output Text File to:')
rawData = dp.getData(
f'https://www.baseball-reference.com/leagues/MLB/{_year}-batting-leaders.shtml'
)
data = dp.findStats(rawData)
writeToFile(data, outDir)
import DataParse
# Required lists
seed_Users = []
used_Ids = []
user_Ids = []
used_Games = []
match_List = []
top_Seeds = []
# Handling command line arguments
clargs = sys.argv
# Api initialisation
api = RiotAPIv2.RiotAPI(clargs[1])
parse = DataParse.DataParse('countergg', 'winrates3')
# Rate limit Setting
rate_Limit = float(clargs[2])
# Patch date in unix time
patch_Date = int(clargs[3])
# Get starting point
seed_Users.append(21999355)
# if toplist not there or time has not passed
# Generate more people in the traditional way
# otherwise, go through the list for games
# increase people's score
def kMeans():
histogram = DataParse.loadStack()
data, centroidList, distance = Cluster.kMeans(histogram, main.CLUSTERS)
Draw.clusterDraw(data, centroidList, histogram)
def twelveBitVolumeRender():
histogram = DataParse.loadStack(300, 200, 3000, mode="12bit")
Draw.volumeRender(histogram, mode="greyscale", prune=0)
def SVM():
histogram = DataParse.loadStack()
data, classifier = Classifier.train(histogram)
Draw.SVMDraw(data, classifier, histogram)
def volumeRenderLowerCorner():
histogram = DataParse.loadStack()
histogram = Utils.magnifyLowerLeft(histogram, .25)
Draw.volumeRender(histogram, mode="greyscale", prune=0)
def dirichelet():
histogram = DataParse.loadStack()
MG, obs = Cluster.expectationMaximization(histogram, main.CLUSTERS, mode="Dirichelet")
Draw.EMDraw(obs, MG, histogram)
def axonColorComparison():
axons = DataParse.readExcelColorSamples()
Draw.axonComparison(axons, 2)
def howManyClusters():
histogram = DataParse.loadStack()
Cluster.howManyClusters(histogram, 2, 45, 5)
def histogramToPNG():
histogram = DataParse.loadStack(300, 200, 3000, mode="12bit", segmented=False)
Utils.saveHistogramAsPNG(histogram)
def run(self):
Thread.__init__(self)
def getInterface(expression):
ifs = findalldevs()
if 0 == len(ifs):
print "You don't have enough permissions to open any interface on this system."
sys.exit(1)
# Only one interface available, use it.
elif 1 == len(ifs):
print 'Only one interface present, defaulting to it.'
return ifs[0]
input = int(
expression) # turns into an integer for the function to accept
return ifs[input] # returns the inputted interface
def recv_pkts(hdr, data):
try:
currentInt = getInterface(self.expression)
#print "parser.mode is "+ str(parser.learn_mode)
if str(parser.learn_mode) == str(False):
eth = EthDecoder().decode(data)
ethChild = eth.child()
ethChild2 = ethChild.child()
if ethChild2.get_type() == 134:
# ------------Time Start------------
#test_open = open("../TestFiles/realtime_test1_sniff", 'a')
#test_start = datetime.now()
#sum = Decimal(test_start.strftime(("%s"))) + Decimal(test_start.strftime(("%f"))) / 1000000
#test_open.write(str(sum))
#test_open.write('\n')
#test_open.close()
# -----------------------------------
parser.sniffSlaac(data, currentInt)
# ------------Time Start------------
#test_open = open("../TestFiles/realtime_test1_detect", 'a')
#test_start = datetime.now()
#sum = Decimal(test_start.strftime(("%s"))) + Decimal(test_start.strftime(("%f"))) / 1000000
#test_open.write(str(sum))
#test_open.write('\n')
#test_open.close()
# -----------------------------------
# print "Hello"
elif ethChild2.get_type() == 135:
# ------------Time Start------------
#test_open = open("../TestFiles/realtime_test1_sniff", 'a')
#test_start = datetime.now()
#sum = Decimal(test_start.strftime(("%s"))) + Decimal(test_start.strftime(("%f"))) / 1000000
#test_open.write(str(sum))
#test_open.write('\n')
#test_open.close()
# -----------------------------------
parser.sniffSlaac(data, currentInt)
#------------Time Start------------
#test_open = open("../TestFiles/realtime_test1_detect", 'a')
#test_start = datetime.now()
#sum = Decimal(test_start.strftime(("%s"))) + Decimal(test_start.strftime(("%f"))) / 1000000
#test_open.write(str(sum))
#test_open.write('\n')
#test_open.close()
# -----------------------------------
#print "Hello"
elif ethChild2.get_type() == 136:
# ------------Time Start------------
#test_open = open("../TestFiles/realtime_test1_sniff", 'a')
#test_start = datetime.now()
#sum = Decimal(test_start.strftime(("%s"))) + Decimal(test_start.strftime(("%f"))) / 1000000
#test_open.write(str(sum))
#test_open.write('\n')
#test_open.close()
# ----------0-------------------------
parser.sniffSlaac(data, currentInt)
#------------Time Start------------
#test_open = open("../TestFiles/realtime_test1_detect", 'a')
#test_start = datetime.now()
#sum = Decimal(test_start.strftime(("%s"))) + Decimal(test_start.strftime(("%f"))) / 1000000
#test_open.write(str(sum))
#test_open.write('\n')
#test_open.close()
# -----------------------------------
#print "Hello"
elif str(parser.learn_mode) == str(True):
#print "activating learning mode"
parser.activateLearningMode(data)
except:
x = 1
print "error"
pcapy.findalldevs()
max_bytes = 1024
promiscuous = False
read_timeout = 100 # in milliseconds
#print "self.mode " + str(self.mode)
parser = DataParse.Dataparse(self.mode)
# callback for received packets
#print "---------------"
#print self.expression
pc = pcapy.open_live(getInterface(self.expression), max_bytes,
promiscuous, read_timeout)
pc.setfilter('icmp6')
# list all the network devices
while self.isRunning is True:
pc.loop(1,
recv_pkts) # capture packets while the thread is running
def setUp(self):
self.Animals_Nov = open('Animals-Nov.json', 'r+').read()
self.Animals_Dec = open('Animals-Dec.json', 'r+').read()
self.Animals_Jan = open('Animals-Jan.json', 'r+').read()
self.Food_Nov = open('Food-Nov.json', 'r+').read()
self.Food_Dec = open('Food-Dec.json', 'r+').read()
self.Food_Jan = open('Food-Jan.json', 'r+').read()
self.animals1 = json.loads(self.Animals_Nov)
self.animals2 = json.loads(self.Animals_Dec)
self.animals3 = json.loads(self.Animals_Jan)
self.food1 = json.loads(self.Food_Nov)
self.food2 = json.loads(self.Food_Dec)
self.food3 = json.loads(self.Food_Jan)
self.Animals = SubredditClass('Animals')
self.Food = SubredditClass('Food')
self.AnimalsTestClass = SubredditClass('Animals')
self.FoodTestClass = SubredditClass('Food')
DataParse.json_data_extract(self.animals1, self.Animals, 'Animals')
DataParse.json_data_extract(self.animals2, self.Animals, 'Animals')
DataParse.json_data_extract(self.animals3, self.AnimalsTestClass,
'Animals')
DataParse.json_data_extract(self.food1, self.Food, 'Food')
DataParse.json_data_extract(self.food2, self.Food, 'Food')
DataParse.json_data_extract(self.food3, self.FoodTestClass, 'Food')
self.Food.splitData('A')
self.Food.updateWords()
self.Animals.splitData('A')
self.Animals.updateWords()
self.FoodTestClass.splitData('T')
self.AnimalsTestClass.splitData('T')
self.FoodControl = self.Food.copy()
self.AnimalControl = self.Animals.copy()
self.Classifier = StatisticSet(self.Food, self.Animals)