def LABVisualization(bins, slices, project=True,color=False,fullOpacity=True, data = "8bit"): startTime = time.time() colorMax = 255 if data == "8bitsegmented": histogram = DataParse.loadStack(300, 200, 3000, mode="8bit", segmented=True) elif data == "8bit": histogram = DataParse.loadStack(300, 200, 3000, mode="8bit", segmented=False) elif data == "12bit": histogram = DataParse.loadStack(300, 200, 3000, mode="12bit", segmented=False) colorMax = 4095 ColorPlots.histogramToLAB(histogram, colorMax, slices, project=project, color=color, fullOpacity=True) endTime = time.time() print "Total Time: %g Seconds" % (endTime-startTime)
def main(): arg_parser = ap.ArgumentParser(description='') arg_parser.add_argument( '--mode', '-m', choices=['gather', 'parse', 'analyze', 'all'], default='all', nargs='?', help='gather, or parse, or analyze data, or do all three', action="store_true") arg_parser.add_argument('--help', '-h', help='usage options', action=arg_parser.print_help) args = arg_parser.parse_args() mode = args.mode if mode == 'all': dg = DataGather() dg.run() dp = DataParse() dp.run() da = DataAnalyze() da.run() elif mode == 'gather': dg = DataGather() dg.run() elif mode == 'parse': dp = DataParse() dp.run() elif mode == 'analyze': da = DataAnalyze() da.run() print(args)
def saveABPlot(slices, bins, floor, ceiling): startTime = time.time() histogram = DataParse.loadStack(slices, bins, floor, ceiling, bits=12, segmented=False, colorSpace="LAB") filename = "AB Plot" + str(bins) + "Bins" + "Floor" + str(floor) + "Ceiling" + str(ceiling) + ".png" ColorPlots.ABPlot(histogram, bins, filename) endTime = time.time() print "Total Time: %g Seconds" % (endTime-startTime) print "For %g Bins" % bins
def __init__(self, name, jsonData, game): self.geometry = [] self.name = name # Load the json file self.json = jsonData print("Processing geometries...") # print(json.dumps(self.json, indent=4, sort_keys=True)) if "[Male]" in name: # Parse all the geometry indices for male items and parse the geometries for geometryIndex in self.json["content"][0]["male_index_set"]["geometry"]: geometryFile = self.json["content"][0]["geometry"][geometryIndex] path = bungieUrlPrefix+bungieGeometryPrefix[game]+geometryFile print("Geometry file: "+path) request = urllib.request.Request(path, headers=headers) response = urllib.request.urlopen(request) data = DataParse.DataParse(response.read()) self.geometry.append(DestinyGeometry.parse(data)) elif "[Female]" in name: # Parse all the geometry indices for female items and parse the geometries for geometryIndex in self.json["content"][0]["female_index_set"]["geometry"]: geometryFile = self.json["content"][0]["geometry"][geometryIndex] path = bungieUrlPrefix+bungieGeometryPrefix[game]+geometryFile print("Geometry file: "+path) request = urllib.request.Request(path, headers=headers) response = urllib.request.urlopen(request) data = DataParse.DataParse(response.read()) self.geometry.append(DestinyGeometry.parse(data)) else: # Get the geometry file names from the json and parse the geometries for geometryFile in self.json["content"][0]["geometry"]: path = bungieUrlPrefix+bungieGeometryPrefix[game]+geometryFile print("Geometry file: "+path) request = urllib.request.Request(path, headers=headers) response = urllib.request.urlopen(request) data = DataParse.DataParse(response.read()) self.geometry.append(DestinyGeometry.parse(data)) print("Done processing geometries...") return
def setUp(self): self.subreddit1 = SubredditClass('Food1') self.subreddit2 = SubredditClass('Food2') self.subreddit3 = SubredditClass('Food3') self.Food_Nov = open('Food-Nov.json', 'r+').read() self.Food_Dec = open('Food-Dec.json', 'r+').read() self.Food_Jan = open('Food-Jan.json', 'r+').read() self.food1 = json.loads(self.Food_Nov) self.food2 = json.loads(self.Food_Dec) self.food3 = json.loads(self.Food_Jan) DataParse.json_data_extract(self.food1, self.subreddit1, 'Food1') DataParse.json_data_extract(self.food2, self.subreddit2, 'Food2') DataParse.json_data_extract(self.food3, self.subreddit3, 'Food3')
def run(self): Thread.__init__(self) def getInterface(expression): ifs = findalldevs() input = int( expression) # turns into an integer for the function to accept return ifs[input] # returns the inputted interface def recv_pkts(hdr, data): try: currentInt = getInterface(self.expression) if str(parser.learn_mode) == str(False): eth = EthDecoder().decode(data) ethChild = eth.child() ethChild2 = ethChild.child() if ethChild2.get_type() == 134: parser.sniffSlaac(data, currentInt) elif ethChild2.get_type() == 135: parser.sniffSlaac(data, currentInt) elif ethChild2.get_type() == 136: parser.sniffSlaac(data, currentInt) elif str(parser.learn_mode) == str(True): parser.activateLearningMode(data) except: pass pcapy.findalldevs() max_bytes = 1024 promiscuous = False read_timeout = 100 parser = DataParse.Dataparse(self.mode) pc = pcapy.open_live(getInterface(self.expression), max_bytes, promiscuous, read_timeout) pc.setfilter('icmp6') while self.isRunning is True: pc.loop(1, recv_pkts) # capture packets while the thread is running
def histogram(): histogram = DataParse.loadStack() Draw.draw(histogram)
def test_json_data_extract(self): DataParse.json_data_extract(self.animals1, self.Animals, 'Animals') DataParse.json_data_extract(self.animals2, self.Animals, 'Animals') DataParse.json_data_extract(self.animals3, self.Animals, 'Animals') DataParse.json_data_extract(self.food1, self.Food, 'Food') DataParse.json_data_extract(self.food2, self.Food, 'Food') DataParse.json_data_extract(self.food3, self.Food, 'Food') assert len(self.Food.posts) == 30 assert self.Food.posts[0].getClass() == 'Food' assert self.Food.posts[29].getClass() == 'Food' assert self.Food.posts[0].getTitle() == 'banana' assert self.Food.posts[29].getTitle() == 'cracker' assert self.Food.posts[0].selftext == 'apple' assert self.Food.posts[29].selftext == 'apple' assert len(self.Animals.posts) == 30 assert self.Animals.posts[0].getClass() == 'Animals' assert self.Animals.posts[29].getClass() == 'Animals' assert self.Animals.posts[0].getTitle() == 'cat' assert self.Animals.posts[29].getTitle() == 'octopus' assert self.Animals.posts[0].selftext == 'dog' assert self.Animals.posts[29].selftext == 'dog'
def fourChannelDraw(): histogram = DataParse.fourChannelTest() Draw.fourChannelDraw(histogram)
def sliceDraw(): histogram = DataParse.loadStack() data, centroidList, distance = Cluster.kMeans(histogram, main.CLUSTERS) Draw.drawSlice(data, centroidList)
def EM(): histogram = DataParse.loadStack() MG, obs = Cluster.expectationMaximization(histogram, main.CLUSTERS) Draw.EMDraw(obs, MG, histogram)
def volumeRender(): histogram = DataParse.loadStack(300, 200, 3000, mode="8bit", segmented=False) Draw.volumeRender(histogram, mode="fullcolor", prune=0)
with open('Daily_Stats.txt', 'w') as outText: for d in data: outText.write(d.stat + '\n') for player in d._players: overallRank[player.name] += rankWeighting(player.rank) playerTeamDict[player.name] = player.team outStrings = [str(value).strip() for value in player] outText.write(' | '.join(outStrings) + '\n') outText.write('\n\n') sortedRank = Counter(overallRank) outText.write('Overall Statistical Standings\n') for standing, player in enumerate(sortedRank.most_common(10), 1): outText.write( f'{standing} | {player[0]} | {playerTeamDict[player[0]]} | {player[1]} \n' ) if __name__ == '__main__': _year = datetime.datetime.now().strftime('%Y') outDir = askdirectory(title='Select File to Output Text File to:') rawData = dp.getData( f'https://www.baseball-reference.com/leagues/MLB/{_year}-batting-leaders.shtml' ) data = dp.findStats(rawData) writeToFile(data, outDir)
import DataParse # Required lists seed_Users = [] used_Ids = [] user_Ids = [] used_Games = [] match_List = [] top_Seeds = [] # Handling command line arguments clargs = sys.argv # Api initialisation api = RiotAPIv2.RiotAPI(clargs[1]) parse = DataParse.DataParse('countergg', 'winrates3') # Rate limit Setting rate_Limit = float(clargs[2]) # Patch date in unix time patch_Date = int(clargs[3]) # Get starting point seed_Users.append(21999355) # if toplist not there or time has not passed # Generate more people in the traditional way # otherwise, go through the list for games # increase people's score
def kMeans(): histogram = DataParse.loadStack() data, centroidList, distance = Cluster.kMeans(histogram, main.CLUSTERS) Draw.clusterDraw(data, centroidList, histogram)
def twelveBitVolumeRender(): histogram = DataParse.loadStack(300, 200, 3000, mode="12bit") Draw.volumeRender(histogram, mode="greyscale", prune=0)
def SVM(): histogram = DataParse.loadStack() data, classifier = Classifier.train(histogram) Draw.SVMDraw(data, classifier, histogram)
def volumeRenderLowerCorner(): histogram = DataParse.loadStack() histogram = Utils.magnifyLowerLeft(histogram, .25) Draw.volumeRender(histogram, mode="greyscale", prune=0)
def dirichelet(): histogram = DataParse.loadStack() MG, obs = Cluster.expectationMaximization(histogram, main.CLUSTERS, mode="Dirichelet") Draw.EMDraw(obs, MG, histogram)
def axonColorComparison(): axons = DataParse.readExcelColorSamples() Draw.axonComparison(axons, 2)
def howManyClusters(): histogram = DataParse.loadStack() Cluster.howManyClusters(histogram, 2, 45, 5)
def histogramToPNG(): histogram = DataParse.loadStack(300, 200, 3000, mode="12bit", segmented=False) Utils.saveHistogramAsPNG(histogram)
def run(self): Thread.__init__(self) def getInterface(expression): ifs = findalldevs() if 0 == len(ifs): print "You don't have enough permissions to open any interface on this system." sys.exit(1) # Only one interface available, use it. elif 1 == len(ifs): print 'Only one interface present, defaulting to it.' return ifs[0] input = int( expression) # turns into an integer for the function to accept return ifs[input] # returns the inputted interface def recv_pkts(hdr, data): try: currentInt = getInterface(self.expression) #print "parser.mode is "+ str(parser.learn_mode) if str(parser.learn_mode) == str(False): eth = EthDecoder().decode(data) ethChild = eth.child() ethChild2 = ethChild.child() if ethChild2.get_type() == 134: # ------------Time Start------------ #test_open = open("../TestFiles/realtime_test1_sniff", 'a') #test_start = datetime.now() #sum = Decimal(test_start.strftime(("%s"))) + Decimal(test_start.strftime(("%f"))) / 1000000 #test_open.write(str(sum)) #test_open.write('\n') #test_open.close() # ----------------------------------- parser.sniffSlaac(data, currentInt) # ------------Time Start------------ #test_open = open("../TestFiles/realtime_test1_detect", 'a') #test_start = datetime.now() #sum = Decimal(test_start.strftime(("%s"))) + Decimal(test_start.strftime(("%f"))) / 1000000 #test_open.write(str(sum)) #test_open.write('\n') #test_open.close() # ----------------------------------- # print "Hello" elif ethChild2.get_type() == 135: # ------------Time Start------------ #test_open = open("../TestFiles/realtime_test1_sniff", 'a') #test_start = datetime.now() #sum = Decimal(test_start.strftime(("%s"))) + Decimal(test_start.strftime(("%f"))) / 1000000 #test_open.write(str(sum)) #test_open.write('\n') #test_open.close() # ----------------------------------- parser.sniffSlaac(data, currentInt) #------------Time Start------------ #test_open = open("../TestFiles/realtime_test1_detect", 'a') #test_start = datetime.now() #sum = Decimal(test_start.strftime(("%s"))) + Decimal(test_start.strftime(("%f"))) / 1000000 #test_open.write(str(sum)) #test_open.write('\n') #test_open.close() # ----------------------------------- #print "Hello" elif ethChild2.get_type() == 136: # ------------Time Start------------ #test_open = open("../TestFiles/realtime_test1_sniff", 'a') #test_start = datetime.now() #sum = Decimal(test_start.strftime(("%s"))) + Decimal(test_start.strftime(("%f"))) / 1000000 #test_open.write(str(sum)) #test_open.write('\n') #test_open.close() # ----------0------------------------- parser.sniffSlaac(data, currentInt) #------------Time Start------------ #test_open = open("../TestFiles/realtime_test1_detect", 'a') #test_start = datetime.now() #sum = Decimal(test_start.strftime(("%s"))) + Decimal(test_start.strftime(("%f"))) / 1000000 #test_open.write(str(sum)) #test_open.write('\n') #test_open.close() # ----------------------------------- #print "Hello" elif str(parser.learn_mode) == str(True): #print "activating learning mode" parser.activateLearningMode(data) except: x = 1 print "error" pcapy.findalldevs() max_bytes = 1024 promiscuous = False read_timeout = 100 # in milliseconds #print "self.mode " + str(self.mode) parser = DataParse.Dataparse(self.mode) # callback for received packets #print "---------------" #print self.expression pc = pcapy.open_live(getInterface(self.expression), max_bytes, promiscuous, read_timeout) pc.setfilter('icmp6') # list all the network devices while self.isRunning is True: pc.loop(1, recv_pkts) # capture packets while the thread is running
def setUp(self): self.Animals_Nov = open('Animals-Nov.json', 'r+').read() self.Animals_Dec = open('Animals-Dec.json', 'r+').read() self.Animals_Jan = open('Animals-Jan.json', 'r+').read() self.Food_Nov = open('Food-Nov.json', 'r+').read() self.Food_Dec = open('Food-Dec.json', 'r+').read() self.Food_Jan = open('Food-Jan.json', 'r+').read() self.animals1 = json.loads(self.Animals_Nov) self.animals2 = json.loads(self.Animals_Dec) self.animals3 = json.loads(self.Animals_Jan) self.food1 = json.loads(self.Food_Nov) self.food2 = json.loads(self.Food_Dec) self.food3 = json.loads(self.Food_Jan) self.Animals = SubredditClass('Animals') self.Food = SubredditClass('Food') self.AnimalsTestClass = SubredditClass('Animals') self.FoodTestClass = SubredditClass('Food') DataParse.json_data_extract(self.animals1, self.Animals, 'Animals') DataParse.json_data_extract(self.animals2, self.Animals, 'Animals') DataParse.json_data_extract(self.animals3, self.AnimalsTestClass, 'Animals') DataParse.json_data_extract(self.food1, self.Food, 'Food') DataParse.json_data_extract(self.food2, self.Food, 'Food') DataParse.json_data_extract(self.food3, self.FoodTestClass, 'Food') self.Food.splitData('A') self.Food.updateWords() self.Animals.splitData('A') self.Animals.updateWords() self.FoodTestClass.splitData('T') self.AnimalsTestClass.splitData('T') self.FoodControl = self.Food.copy() self.AnimalControl = self.Animals.copy() self.Classifier = StatisticSet(self.Food, self.Animals)