def main(): if not have_required_permissions(): logger.error("Missing some required permissions, exiting") TCMConstants.exit_gracefully(TCMConstants.SPECIAL_EXIT_CODE, None) if TCMConstants.MULTI_CAR: for car in TCMConstants.CAR_LIST: setup_video_paths(f"{car}/") else: setup_video_paths("") while True: for share in TCMConstants.SHARE_PATHS: for folder in TCMConstants.FOOTAGE_FOLDERS: for directory in next(os.walk(f"{share}{folder}"))[1]: if os.listdir(f"{share}{folder}/{directory}"): logger.debug( f"Directory {share}{folder}/{directory} not empty, skipping" ) else: remove_empty_old_directory(f"{share}{folder}/", directory) for path in VIDEO_PATHS: for file in os.listdir(path): remove_old_file(path, file) if datetime.datetime.now().minute in TCMConstants.STATS_FREQUENCY: Stats.generate_stats_image() time.sleep(TCMConstants.SLEEP_DURATION)
def create(self): super(default, self).create() player_id = None for id in self.stat_dict.keys(): if self.seat == self.stat_dict[id]['seat']: player_id = id if player_id is None: self.destroy_pop() self.lab = QLabel() self.setLayout(QVBoxLayout()) self.layout().addWidget(self.lab) text, tip_text = "", "" for stat in self.pop.pu_stats: number = Stats.do_stat(self.stat_dict, player=int(player_id), stat=stat, hand_instance=self.hand_instance) if number: text += number[3] + "\n" tip_text += number[5] + " " + number[4] + "\n" else: text += "xxx" + "\n" tip_text += "xxx" + " " + "xxx" + "\n" #trim final \n tip_text = tip_text[:-1] text = text[:-1] self.lab.setText(text) Stats.do_tip(self.lab, tip_text)
def __call__(self, track, slice=None): if track == self.mMaster: return [] columns = Stats.Summary().getHeaders() x = CoverageByTranscripts.__call__(self, track, slice) stats = Stats.Summary(x["pover1"]) data = stats.items() data.append( ("100% covered", self.getValue( """SELECT COUNT(*) FROM %s_vs_%s WHERE pover1>= 100""" % (self.mMaster, track)))) data.append( ("90% covered", self.getValue( """SELECT COUNT(*) FROM %s_vs_%s WHERE pover1>= 90""" % (self.mMaster, track)))) data.append(( "1to1", len( list( self.execute( """SELECT gene_id1 FROM %s_vs_%s_ovl GROUP BY gene_id1 HAVING COUNT (gene_id2) = 1""" % (self.mMaster, track)))))) data.append(( "1toM", len( list( self.execute( """SELECT gene_id1 FROM %s_vs_%s_ovl GROUP BY gene_id1 HAVING COUNT (gene_id2) > 1""" % (self.mMaster, track)))))) return odict(data)
def create(self): super(default, self).create() player_id = None for id in self.stat_dict.keys(): if self.seat == self.stat_dict[id]['seat']: player_id = id if player_id is None: self.destroy_pop() self.lab = gtk.Label() self.eb.add(self.lab) text, tip_text = "", "" for stat in self.pop.pu_stats: number = Stats.do_stat(self.stat_dict, player=int(player_id), stat=stat, hand_instance=self.hand_instance) text += number[3] + "\n" tip_text += number[5] + " " + number[4] + "\n" #trim final \n tip_text = tip_text[:-1] text = text[:-1] self.lab.set_text(text) Stats.do_tip(self.lab, tip_text) self.lab.modify_bg(gtk.STATE_NORMAL, self.win.aw.bgcolor) self.lab.modify_fg(gtk.STATE_NORMAL, self.win.aw.fgcolor) self.eb.connect("button_press_event", self.button_press_cb) self.show_all()
def stat_calc(): 'Takes input from the entry box and stores it in userData' userData = str(entry.get()) 'Puts userData in Stats.calc. Then the result of Stats.calc is given to Stats.output as input. Stats.output displays' 'the result of the calculations' return Stats.output(Stats.calc(userData))
def getData(Config, fields): t = TimeSeries(Config) s = Stats(Config) r = s.sget("stats.hostlist") for i in r: print "Hostname: %s" % i line = "when\t\t\t\t" for f in fields: if f != 'when': line += f +"\t" print "%s" % line x = t.zget(i) for y in x: line = "%s" % asctime( localtime( y['when'] )) line += "\t" for f in fields: if f != 'when': try: if type(y[f]) == 'int': line += y[f] + "\t" elif type(y[f]) == 'float': line += "%.2f" % y[f] line += "\t" else: line += str(y[f]) + "\t" except: line += "-\t" print "%s" % line print '\n'
def run_game(): pygame.init() clock = pygame.time.Clock() g_set = Settings(boardwidth=600, boardheight=600, message_board_height=50) g_stats = Stats(g_set) game_screen = pygame.display.set_mode((g_set.game_screen_width, g_set.game_screen_height + g_set.message_board_height)) messageBoard = MessageBoard(game_screen, g_set, g_stats) areasList = createClickAreas(game_screen, g_set, g_stats) updateScreen(game_screen, g_set, g_stats, areasList, messageBoard) while True: clock.tick(120) if g_stats.waiting_game_input and not g_stats.drawing_click_area: checkEvents(game_screen, g_set, g_stats, areasList, messageBoard) elif not g_stats.waiting_game_input and g_stats.drawing_click_area: updateScreen(game_screen, g_set, g_stats, areasList, messageBoard) elif not g_stats.waiting_game_input and not g_stats.drawing_click_area: # This code comes into play once animation for X or O is done. # Checking the board to winner and turning event checker back ON! if checkBoard(g_stats, areasList) == False: g_stats.switch_turns() messageBoard.assignTurnMessage() pygame.event.clear() g_stats.waiting_game_input = True updateScreen(game_screen, g_set, g_stats, areasList, messageBoard)
def create(self): super(default, self).create() player_id = None for id in self.stat_dict.keys(): if self.seat == self.stat_dict[id]['seat']: player_id = id if player_id is None: self.destroy_pop() self.lab = gtk.Label() self.eb.add(self.lab) text,tip_text = "","" for stat in self.pop.pu_stats: number = Stats.do_stat( self.stat_dict, player = int(player_id),stat = stat, hand_instance = self.hand_instance) if number: text += number[3] + "\n" tip_text += number[5] + " " + number[4] + "\n" else: text += "xxx" + "\n" tip_text += "xxx" + " " + "xxx" + "\n" #trim final \n tip_text = tip_text[:-1] text = text[:-1] self.lab.set_text(text) Stats.do_tip(self.lab, tip_text) self.lab.modify_bg(gtk.STATE_NORMAL, self.win.aw.bgcolor) self.lab.modify_fg(gtk.STATE_NORMAL, self.win.aw.fgcolor) self.eb.connect("button_press_event", self.button_press_cb) self.show_all()
def update(self, player_id, stat_dict): super(Classic_stat, self).update(player_id, stat_dict) if not self.number: #stat did not create, so exit now return False fg = self.hudcolor if self.stat_loth != "": try: # number[1] might not be a numeric (e.g. NA) if float(self.number[1]) < float(self.stat_loth): fg = self.stat_locolor except: pass if self.stat_hith != "": try: # number[1] might not be a numeric (e.g. NA) if float(self.number[1]) > float(self.stat_hith): fg = self.stat_hicolor except: pass self.set_color(fg=fg, bg=None) statstring = "%s%s%s" % (self.hudprefix, str( self.number[1]), self.hudsuffix) self.lab.set_text(statstring) tip = "%s\n%s\n%s, %s" % (stat_dict[player_id]['screen_name'], self.number[5], self.number[3], self.number[4]) Stats.do_tip(self.widget, tip)
def snp_in_duplicate_region(snp, bam_file, reference_genome_file): sites = dict() left = snp['POS']-misc_params["snp_dist_limit"] if snp['POS']-misc_params["snp_dist_limit"] > 0 else 0 right = snp['POS']+misc_params["snp_dist_limit"] for read in bam_file.fetch(snp['CHROM'], left, right): if read.mapping_quality >= stats_params["mapping_quality"] and read.is_paired and read.is_proper_pair: r = Stats.Read(read.query_name, None, read.query_sequence, read.get_aligned_pairs(), read.reference_start, read.reference_end-1, read.query_qualities, read.mapping_quality, False) for pos in r.bases.keys(): if pos >= left and pos <= right and r.base_quality[pos] > stats_params["base_quality"]: if pos not in sites.keys(): sites[pos] = {'A':0, 'C':0, 'G':0, 'T':0} sites[pos][r.bases[pos].upper()] += 1 reference = Stats.get_references(snp['CHROM'], left, right, reference_genome_file) pos_list = list(sites.keys()) for pos in pos_list: ref = reference[pos] T = sum(sites[pos].values()) if ref not in acceptable_bases or float(sites[pos][ref])/T >= stats_params["bulk_ref_limit"]: sites.pop(pos) pos_list = sorted(list(sites.keys())) in_duplicate_region = False if len(pos_list) > misc_params["snp_nr_limit"]: for i in range(len(pos_list)-misc_params["snp_nr_limit"] + 1): interval = pos_list[i:i+misc_params["snp_nr_limit"]] if max(interval) - min(interval) <= misc_params["snp_dist_limit"]: in_duplicate_region = True break return in_duplicate_region
def ProcessCentroidVals(files,basename,StartCut,clone): Section = 'Centroid' try: xVals = FetchDataArray2(files,'Observables/'+Section+'/xVals',StartCut) yzVals = FetchDataArray2(files,'Observables/'+Section+'/yzVals',StartCut) N = len(xVals[0][0][0]) # Particles xValStats,yzValStats = [],[] for i in range(0,N): # particle xValStats.append([]) yzValStats.append([]) for d in range(0,2): # in between and outside xValStats[i].append([]) for f in range(0,len(xVals)): # file for m in range(0,len(xVals[f])): # measurement xValStats[i][d].append(xVals[f][m][i][d]) xValStats[i][d] = Stats.stats(xValStats[i][d]) for f in range(0,len(yzVals)): # file for m in range(0,len(yzVals[f])): # measurement yzValStats[i].append(yzVals[f][m][i]) yzValStats[i] = Stats.stats(yzValStats[i]) g = open(basename+'/'+Section+'xVals'+clone+'.dat','w') for i in range(0,N): for d in range(0,2): g.write('%i%i %e %e \n' % (i, d, xValStats[i][d][0], xValStats[i][d][3])) g.close() g = open(basename+'/'+Section+'yzVals'+clone+'.dat','w') for i in range(0,N): g.write('%i %e %e \n' % (i, yzValStats[i][0], yzValStats[i][3])) g.close() except: print 'Error in', Section
def ProcessCentroidVals(files, basename, StartCut, clone): Section = 'Centroid' try: xVals = FetchDataArray2(files, 'Observables/' + Section + '/xVals', StartCut) yzVals = FetchDataArray2(files, 'Observables/' + Section + '/yzVals', StartCut) N = len(xVals[0][0][0]) # Particles xValStats, yzValStats = [], [] for i in range(0, N): # particle xValStats.append([]) yzValStats.append([]) for d in range(0, 2): # in between and outside xValStats[i].append([]) for f in range(0, len(xVals)): # file for m in range(0, len(xVals[f])): # measurement xValStats[i][d].append(xVals[f][m][i][d]) xValStats[i][d] = Stats.stats(xValStats[i][d]) for f in range(0, len(yzVals)): # file for m in range(0, len(yzVals[f])): # measurement yzValStats[i].append(yzVals[f][m][i]) yzValStats[i] = Stats.stats(yzValStats[i]) g = open(basename + '/' + Section + 'xVals' + clone + '.dat', 'w') for i in range(0, N): for d in range(0, 2): g.write('%i%i %e %e \n' % (i, d, xValStats[i][d][0], xValStats[i][d][3])) g.close() g = open(basename + '/' + Section + 'yzVals' + clone + '.dat', 'w') for i in range(0, N): g.write('%i %e %e \n' % (i, yzValStats[i][0], yzValStats[i][3])) g.close() except: print 'Error in', Section
def ProcessCentroidSpread(files,basename,StartCut,clone): Section = 'Centroid' try: vals = FetchDataArray2(files,'Observables/'+Section+'/SpreadVals',StartCut) vecs = FetchDataArray2(files,'Observables/'+Section+'/SpreadVecs',StartCut) N = len(vals[0][0]) # Particles D = len(vals[0][0][0]) # Dimensions ValStats,VecStats = [],[] for i in range(0,N): # particle ValStats.append([]) VecStats.append([]) for d in range(0,D): # dimension ValStats[i].append([]) for f in range(0,len(vals)): # file for m in range(0,len(vals[f])): # measurement ValStats[i][d].append(vals[f][m][i][d]) ValStats[i][d] = Stats.stats(ValStats[i][d]) VecStats[i].append([]) for d2 in range(0,D): VecStats[i][d].append([]) for f in range(0,len(vecs)): # file for m in range(0,len(vecs[f])): # measurement VecStats[i][d][d2].append(vecs[f][m][i][d][d2]) VecStats[i][d][d2] = Stats.stats(VecStats[i][d][d2]) g = open(basename+'/'+Section+'Spread'+clone+'.dat','w') for i in range(0,N): for d in range(0,D): g.write('%i%i %e %e \n' % (i, d, ValStats[i][d][0], ValStats[i][d][3])) for d1 in range(0,D): for d2 in range(0,D): g.write('%i%i%i %e %e \n' % (i, d1, d2, VecStats[i][d1][d2][0], VecStats[i][d1][d2][3])) g.close() except: print 'Error in', Section
def getData(Config, fields): t = TimeSeries(Config) s = Stats(Config) r = s.sget("stats.hostlist") for i in r: print "Hostname: %s" % i line = "when\t\t\t\t" for f in fields: if f != 'when': line += f + "\t" print "%s" % line x = t.zget(i) for y in x: line = "%s" % asctime(localtime(y['when'])) line += "\t" for f in fields: if f != 'when': try: if type(y[f]) == 'int': line += y[f] + "\t" elif type(y[f]) == 'float': line += "%.2f" % y[f] line += "\t" else: line += str(y[f]) + "\t" except: line += "-\t" print "%s" % line print '\n'
def calcCounters(self, dpid, ports): Globals.STATSLOG.write(' ====== Update Stats ====== \n') Stats.updateInstalledCounters(dpid, ports) Stats.updatePathCounters(dpid, ports) # Stats.updateInstallTable() # Stats.printStats() Globals.STATSUPDATE = True
def euro_2016(): input_color = colored("Please choose from the following options:\n" "Stats 'S', Highlights 'H', Live Streams 'L', " "Exit 'E'\n", 'red', attrs=['bold']) # prompt user for input choose_menu = input(input_color).lower() if (choose_menu == "s"): Stats.choose_menu() euro_2016() elif (choose_menu == "h"): Streams.footballHighlights() euro_2016() elif (choose_menu == "l"): Streams.sportLinks() euro_2016() elif (choose_menu == "e"): Stats.Logout() # user must have entered invalid option else: euro_2016()
def euro_2016(): input_color = colored( "Please choose from the following options:\n" "Stats 'S', Highlights 'H', Live Streams 'L', " "Exit 'E'\n", 'red', attrs=['bold']) # prompt user for input choose_menu = input(input_color).lower() if (choose_menu == "s"): Stats.choose_menu() euro_2016() elif (choose_menu == "h"): Streams.footballHighlights() euro_2016() elif (choose_menu == "l"): Streams.sportLinks() euro_2016() elif (choose_menu == "e"): Stats.Logout() # user must have entered invalid option else: euro_2016()
class SkynetDetector: def __init__(self, classifier, features_processors): self._features = features_processors self._data_classes = [] self._features_data = [] self.stats = None self.classifier = classifier def train(self, file_path): splitter = DatasetSplitter(line_callback=self._process_sentence) splitter.split(file_path) # provide data to train svm self.classifier.train(self._features_data, self._data_classes) def evaluate_file(self, file_path): splitter = DatasetSplitter(line_callback=self.predict, parse_class=False) splitter.split(file_path) def predict(self, sentence, print_to_console=True): feature_vector = self._process_features(sentence.lower()) prediction = self.classifier.predict([feature_vector])[0] if print_to_console: print str(prediction) + "\t" + sentence.strip("\n") return prediction def accuracy(self, test_file_path): self.stats = Stats() splitter = DatasetSplitter(line_callback=self._evaluate) splitter.split(test_file_path) #check http://stats.stackexchange.com/questions/92101/prediction-with-scikit-and-an-precomputed-kernel-svm return self.stats.accuracy() def _process_sentence(self, class_sentence, sentence): self._data_classes.append(class_sentence) feature_vector = self._process_features(sentence) self._features_data.append(feature_vector) def _process_features(self, sentence): feature_vector = [] for feature in self._features: len_sentence = len(sentence.split(" ")) value = feature.process(sentence, len_sentence) if isinstance(value, list): feature_vector.extend(value) else: feature_vector.append(value) return feature_vector def _evaluate(self, expected_class_sentence, sentence): predicted_class = self.predict(sentence, print_to_console=False) self.stats.add(expected_class_sentence, predicted_class)
def run(self, grid): solvedCells = 0 for x, y in iter(grid): if self.block_hidden_single(grid, x, y) or self.horizontal_hidden_single(grid, x, y) or self.vertical_hidden_single(grid, x, y): logging.info("Hidden Single resolution : found value for ({},{}) : {}".format(x, y, grid.get_solution(x, y))) Stats.increment(self.__class__.__name__) solvedCells += 1 return solvedCells
def DoStatistics(self, xs, ys): stats = [] for (x, y) in zip(xs, ys): statsx = [] for yi in y: statsx.append(Stats.stats(yi)) stats.append(Stats.UnweightedAvg(statsx)) return stats
def run(self, grid): solvedCells = 0 for xBlock in [0, 3, 6]: for yBlock in [0, 3, 6]: solvedCells += self.row_block_reduction(grid, xBlock, yBlock) + self.column_block_reduction(grid, xBlock, yBlock) + \ self.block_row_reduction(grid, xBlock, yBlock) + self.block_column_reduction(grid, xBlock, yBlock) Stats.increment(self.__class__.__name__, solvedCells) return solvedCells
def getKDE(df_estu, df_all, n = 10000): kde_est = {} kde_est['estu_xgrid'] = np.linspace(-0.1, 0.1, n) kde_est['estu'] = stat.kde(df_estu.values, kde_est['estu_xgrid'].ravel()).ravel() kde_est['all_xgrid'] = np.linspace(-0.1, 0.1, n) kde_est['all'] = stat.kde(df_all.values, kde_est['all_xgrid'].ravel()).ravel() kde_est = pd.DataFrame(kde_est, dtype=float) return kde_est
def accuracy(self, test_file_path): self.stats = Stats() splitter = DatasetSplitter(line_callback=self._evaluate) splitter.split(test_file_path) #check http://stats.stackexchange.com/questions/92101/prediction-with-scikit-and-an-precomputed-kernel-svm return self.stats.accuracy()
def run(self, grid): solvedCells = 0 for x, y in iter(grid): if self.vertical_naked_pair(grid, x, y): logging.info("Naked Pairs resolution : found value for ({},{}) : {}".format(x, y, grid.get_solution(x, y))) Stats.increment(self.__class__.__name__) solvedCells += 1 return solvedCells
def __init__(self, width, height, color): displayInfo = pygame.display.Info() self.FullScreenSize = [displayInfo.current_w, displayInfo.current_h] self.Width = width self.Height = height self.WindowedSize = [self.Width, self.Height] self.Color = color self.MakeWindowed() self.Status = Stats((0,0,0), color)
def __init__(self, *args, **kwargs): global reg, enum, bdd tk.Frame.__init__(self, *args, **kwargs) # Setup 4 frames for the 4 pages of the application reg = Register(self) enum = List(self) stat = Stats(self) admin = Admin(self) call.id_call.enum = enum bdd = Page.get_bdd(enum) call.id_call.bdd = bdd button_frame = tk.Frame(self) container = tk.Frame(self) button_frame.pack(side="top", fill='x', expand=False) container.pack(side="top", fill="both", expand=True) # Place all the 4 frames on the main windows, they are superimposed reg.place(in_=container, x=0, y=0, relwidth=1, relheight=1) enum.place(in_=container, x=0, y=0, relwidth=1, relheight=1) stat.place(in_=container, x=0, y=0, relwidth=1, relheight=1) admin.place(in_=container, x=0, y=0, relwidth=1, relheight=1) # Setup all the 4 buttons to switch between the 4 pages reg_b = tk.Button(button_frame, text="Inscription", width=19, height=1, command=reg.lift, font=BIG_FONT) enum_b = tk.Button(button_frame, text="Liste", width=19, height=1, command=enum.lift_list, font=BIG_FONT) stat_b = tk.Button(button_frame, text="Statistiques", width=20, height=1, command=stat.lift_stats, font=BIG_FONT) admin_b = tk.Button(button_frame, text="Administration", width=20, height=1, command=admin.ask_password, font=BIG_FONT) # Place all the buttons on the main windows reg_b.grid(row=0, column=0) enum_b.grid(row=0, column=1) stat_b.grid(row=0, column=2) admin_b.grid(row=0, column=3) reg.show()
def setStatistic(self, s): if s == 'Top 10': self.currentStat = Stats.top10Stat(self.myCollection) if s == 'Bottom 10': self.currentStat = Stats.bottom10Stat(self.myCollection) if s == 'Unique Songs': self.currentStat = Stats.uniqueSongsSeen(self.myCollection) if s == 'Unique Shows': self.currentStat = Stats.uniqueShowsSeen(self.myCollection)
def __burstiness(self): # Parse current_key = "" count = 0 self.burstiness = {} for line in self.lines: line = line.strip() tokens = line.split(' ') tokens = self.__clean(tokens) if '.' not in tokens[1]: continue key = self.__get_key(tokens) if "SYN" in line: self.burstiness[key] = {} continue # Including length == 0 packets because it is a packet that must be processed # if "Len=0" in line: # continue if current_key is not "" and current_key != key: if self.burstiness[current_key].has_key(count): num = self.burstiness[current_key][count] num += 1 self.burstiness[current_key][count] = num else: self.burstiness[current_key][count] = 1 if current_key == key: count += 1 else: current_key = key count = 1 if self.burstiness[current_key].has_key(count): num = self.burstiness[current_key][count] num += 1 self.burstiness[current_key][count] = num else: self.burstiness[current_key][count] = 1 s = Stats() for key in self.burstiness: print key array = [] for lines in self.burstiness[key].keys(): count = self.burstiness[key][lines] for i in range(0, count): array.append(lines) print "Min, 25th, 50th, 75th, and Max values:" print float(min(array)), "\t", s.get_25th_percentile(array), "\t", s.median(array), "\t", s.get_75th_percentile(array), "\t", float(max(array))
def dailyValuePerStock( self ): ''' Return a DataFrame showing the daily fluctuations in each asset. ''' dates = pd.date_range( self.startDate, self.endDate ) df = Stats.getDailyStockPrices( self.symbols, dates ) df = Stats.normalizeData( df ) df = df * self.allocs portfolio = df * self.startVal return portfolio
def counterTimer(self): Globals.log.info('Stats Request') for i, switch in enumerate(Globals.SWITCHES): # This would work if we could measure flow stats # for j, rule in enumerate(Globals.INSTALLEDRULES): # self.ctxt.send_port_stats_request(switch['mac']) self.ctxt.send_port_stats_request(switch['mac']) Stats.updateInstallTable() Stats.printStats() self.post_callback(Globals.PORT_STATS_PERIOD, lambda : self.counterTimer())
def calculateLI(self, orderID, yiDev, nhVals, weights): """Calculates Local I for a given feature. INPUTS: orderID (int): order in corresponding numpy value arrays yiDev (float): value for given feature nhVals (array, nn): values for neighboring features (1) weights (array, nn): weight values for neighboring features (1) NOTES: (1) nn is equal to the number of neighboring features """ sumW = weights.sum() sumWSquared = sumW**2.0 sumW2 = (weights**2.0).sum() totalVal = yiDev / self.yDev2NormSum lagVal = (nhVals * weights).sum() liVal = totalVal * lagVal eiVal = -1. * (sumW / self.nm1) eiVal2 = eiVal**2 #### Variance, Randomization #### v1 = (sumW2 * (self.numObs - self.b2i)) / self.nm1 v2 = sumWSquared / (self.nm1**2.) v3 = (sumWSquared - sumW2) * ((2. * self.b2i) - self.numObs) viVal = v1 + v3 / self.nm12 - v2 ziVal = (liVal - eiVal) / (viVal**.5) pVal = STATS.zProb(ziVal, type = 2) #### Assign To Result Vectors #### self.li[orderID] = liVal self.ei[orderID] = eiVal self.vi[orderID] = viVal self.zi[orderID] = ziVal self.pVals[orderID] = pVal #### Store Info For Binning #### clusterBool = ziVal > 0 localGlobalBool = lagVal >= 0 featureGlobalBool = yiDev >= 0 self.moranInfo[orderID] = (clusterBool, localGlobalBool, featureGlobalBool) #### Do Permutations #### if self.permutations: numNHS = len(nhVals) randomInts = RAND.random_integers(0, self.numObs-1, (self.permutations, numNHS)) nhValsPerm = self.yDev[randomInts] lagValsPerm = (nhValsPerm * weights).sum(1) liValsPerm = totalVal * lagValsPerm pseudoP = STATS.pseudoPValue(liVal, liValsPerm) self.pseudoPVals[orderID] = pseudoP
def main(): for i in range(len(Stats.n)): for c in range(Stats.count[i]): print("i: " + str(i) + ", c: " + str(c)) max_circle_matrix = create_max_circle_matrix(Stats.n[i]) timer = datetime.now() result_max_circle_1 = tmg.make_relation_matrix( np.copy(max_circle_matrix)) Stats.max_circle_tmg_time[i] += datetime.now() - timer print( str(datetime.now()) + " max 1 " + str(Stats.max_circle_tmg_time[i])) #timer = datetime.now() #result_max_circle_2 = bilp.make_relation_matrix(np.copy(max_circle_matrix)) #Stats.max_circle_bilp_time[i] += datetime.now() - timer #print(str(datetime.now()) + " max 2 " + str(Stats.max_circle_bilp_time[i])) no_circle_matrix = create_no_circle_matrix(result_max_circle_1) timer = datetime.now() result_no_circle_1 = tmg.make_relation_matrix( np.copy(no_circle_matrix)) Stats.no_circle_tmg_time[i] += datetime.now() - timer print( str(datetime.now()) + " no 1 " + str(Stats.no_circle_tmg_time[i])) #timer = datetime.now() #result_no_circle_2 = bilp.make_relation_matrix(np.copy(no_circle_matrix)) #Stats.no_circle_bilp_time[i] += datetime.now() - timer #print(str(datetime.now()) + " no 2 " + str(Stats.no_circle_bilp_time[i])) single_circle_matrix = create_single_circle_matrix(Stats.n[i]) timer = datetime.now() result_single_circle_1 = tmg.make_relation_matrix( np.copy(single_circle_matrix)) Stats.single_circle_tmg_time[i] += datetime.now() - timer print( str(datetime.now()) + " single 1 " + str(Stats.single_circle_tmg_time[i])) #timer = datetime.now() #result_single_circle_2 = bilp.make_relation_matrix(np.copy(single_circle_matrix)) #Stats.single_circle_bilp_time[i] += datetime.now() - timer #print(str(datetime.now()) + " single 2 " + str(Stats.single_circle_bilp_time[i])) util.init_logging(logging.DEBUG, "GenerateTreeTiming_" + time.strftime("%Y%m%d-%H%M%S")) Stats.log_results() exit(0)
def __init__(self, name: str, play_type: PlayType, play_level_scoring_factor: float, initial_points=0.0): self._name = name.lower() self._play_type = play_type self._stats = Stats(initial_points, play_level_scoring_factor) self._stats.reset_data('match_points') self._stats.reset_data('ranking')
def test_set_points(self): stats = Stats(initial_points=0.0, initial_level=1.0) stats.set_match_results(6, 3, LeagueIndex(5)) with self.assertRaises(TypeError): stats.set_data('match_points', 6.0, 'a') with self.assertRaises(TypeError): stats.set_data('match_points', 6.0, '1.0') # Set points earned for match stats.set_data('match_points', 3.0, LeagueIndex(5))
def calculateLI(self, orderID, yiDev, nhVals, weights): """Calculates Local I for a given feature. INPUTS: orderID (int): order in corresponding numpy value arrays yiDev (float): value for given feature nhVals (array, nn): values for neighboring features (1) weights (array, nn): weight values for neighboring features (1) NOTES: (1) nn is equal to the number of neighboring features """ sumW = weights.sum() sumWSquared = sumW**2.0 sumW2 = (weights**2.0).sum() totalVal = yiDev / self.yDev2NormSum lagVal = (nhVals * weights).sum() liVal = totalVal * lagVal eiVal = -1. * (sumW / self.nm1) eiVal2 = eiVal**2 #### Variance, Randomization #### v1 = (sumW2 * (self.numObs - self.b2i)) / self.nm1 v2 = sumWSquared / (self.nm1**2.) v3 = (sumWSquared - sumW2) * ((2. * self.b2i) - self.numObs) viVal = v1 + v3 / self.nm12 - v2 ziVal = (liVal - eiVal) / (viVal**.5) pVal = STATS.zProb(ziVal, type=2) #### Assign To Result Vectors #### self.li[orderID] = liVal self.ei[orderID] = eiVal self.vi[orderID] = viVal self.zi[orderID] = ziVal self.pVals[orderID] = pVal #### Store Info For Binning #### clusterBool = ziVal > 0 localGlobalBool = lagVal >= 0 featureGlobalBool = yiDev >= 0 self.moranInfo[orderID] = (clusterBool, localGlobalBool, featureGlobalBool) #### Do Permutations #### if self.permutations: numNHS = len(nhVals) randomInts = RAND.random_integers(0, self.numObs - 1, (self.permutations, numNHS)) nhValsPerm = self.yDev[randomInts] lagValsPerm = (nhValsPerm * weights).sum(1) liValsPerm = totalVal * lagValsPerm pseudoP = STATS.pseudoPValue(liVal, liValsPerm) self.pseudoPVals[orderID] = pseudoP
def main(): # logging util.init_logging(logging.INFO, "PredictRelationTiming_" + time.strftime("%Y%m%d-%H%M%S")) # command line interface parser = argparse.ArgumentParser() parser.add_argument('MajorClaim', type=str, help='topic of the discussion') parser.add_argument('--search', type=int, help='number of sentences to further process from the search results (if not given all sentences retrieved are used)') parser.add_argument('--classify', nargs='+', type=str, help='multiple sentences (group sentences with ""), a text or an url to be used as a source to collect arguments from') parser.add_argument('-svm', action='store_true', help='change classifier for estimating relation probabilities from BERT to SVM') parser.add_argument('-bilp', action='store_true', help='change from generate tree approach tmg (Traversing and Modifying Graphs) to bilp (Binary Linear Integer Programming)') parser.add_argument('--cluster', nargs=2, type=float, help='cluster arguments before processing them (relation only possible within cluster) -> first arg: similarity threshold, second arg: min_cluster_size') args = parser.parse_args() logging.info(args) # search engine search_engine = ArgumenText("userID", "apiKey") if args.classify is None: sentences = search_engine.query_search_api(args.MajorClaim) if args.search is not None and args.search < len(sentences): stance_pro = [a for a in sentences if a["stanceLabel"] == 'pro'] stance_con = [a for a in sentences if a["stanceLabel"] == 'contra'] stance_pro.sort(key=lambda s: s["argumentConfidence"]*s["stanceConfidence"], reverse=True) stance_con.sort(key=lambda s: s["argumentConfidence"]*s["stanceConfidence"], reverse=True) pro_len = min(int(args.search/2), len(stance_pro)) con_len = min(args.search - pro_len, len(stance_con)) diff = args.search - pro_len - con_len pro_len += diff sentences = stance_pro[:pro_len] sentences.extend(stance_con[:con_len]) else: if len(args.classify) is 1: args.classify = args.classify[0] sentences = search_engine.query_classify_api(args.MajorClaim, args.classify) arguments = ArgumentList(args.MajorClaim, sentences) # clustering if args.cluster is not None: clusters = search_engine.query_cluster_api([s["sentenceOriginal"] for s in sentences], args.cluster[0], args.cluster[1]) logging.debug(clusters) arguments.apply_clusters(clusters) # relation processing relation_processing = RelationProcessor(args.svm, args.bilp) relation_processing.generate_relation_matrix(arguments) Stats.log_results() exit(0)
def __init__(self, map_size_rows, map_size_cols, inital_index, previousMemory): #self.currentMapObservations = [Observation()]*map_size_cols*map_size_rows #NO Walls self.currentMapObservations = [None] * (map_size_cols + 2) * (map_size_rows + 2) self.currentIndex = inital_index self.initalIndex = inital_index self.previousMemory = previousMemory self.currentMemory = Memory() self.currentUnknownIndexes = [] self.previousIndex = [] self.currentSteps = 0 self.currentActioNumber = 0 self.previousActions = [] self.rewardForAction = [] self.stats = Stats() self.hasFallenIndexes = [] self.killedGolemAtIndex = [] self.cummulativeReward = 0 self.lastAction = None self.rewardForLastAction = 0 self.lastCurrentIndex = -1 self.lastObservationalCollection = ObservationCollection() self.currentObservationCollection = ObservationCollection() self.currentActionHistory = ActionHistory() self.gamma = 0.2 self.siFactor = 0.2 self.randomness = 0.35 self.risk = 0.75 self.exploration = 1.0 self._fill_map(map_size_cols, map_size_rows) self.num_rows = map_size_rows self.num_cols = map_size_cols self.totalTiles = self.num_rows * self.num_cols self.orientation = [-map_size_cols, 1, map_size_cols, -1] #(delta row, delta col) for move_forward
def processAgentResponse(resp): if resp is not None and resp['result'] is not None: d = json.loads(resp['result']) for outgoing in d['outLinks']: tryAddUrlToQueue(outgoing) crawlRecord = { "url": d['url'], "renderTime": d['renderTime'], "serverErrors": d['serverErrors'], "browserErrors": d['browserErrors'], "date": datetime.datetime.utcnow() } updateCrawlRecordErrors(crawlRecord) DB.addCrawlRecord(crawlRecord) Stats.updateStats(crawlRecord)
def test_add_match_results(self): stats = Stats(initial_points=0.0, initial_level=1.0) with self.assertRaises(TypeError): stats.set_match_results(6, 3, 'a') with self.assertRaises(TypeError): stats.set_match_results(6, 3, '1.0') stats.set_match_results(6, 3, LeagueIndex(5))
def check( self, method ): '''check for length equality and elementwise equality.''' a = R['p.adjust'](self.pvalues, method = method ) b = Stats.adjustPValues( self.pvalues, method = method ) self.assertEqual( len(a), len(b)) for x,y in zip( a,b): self.assertAlmostEqual( x,y )
def testLRT(self): """test that the false positive rate is in the same order as mSignificance. Sample from a normal distribution and compare two models: 1. mean estimated = complex model (1 df) 2. mean given = simple model (0 df) Likelihood = P(model | data) """ simple_np = 0 complex_np = 1 npassed = 0 for replicate in range(0,self.mNumReplicates): sample = scipy.stats.norm.rvs(size=self.mNumSamples, loc = 0.0, scale = 1.0) mean = scipy.mean( sample ) complex_ll = numpy.sum( numpy.log( scipy.stats.norm.pdf( sample, loc = mean, scale = 1.0 ) ) ) simple_ll = numpy.sum( numpy.log( scipy.stats.norm.pdf( sample, loc = 0.0, scale = 1.0 ) ) ) a = Stats.doLogLikelihoodTest( complex_ll, complex_np, simple_ll, simple_np, significance_threshold = self.mSignificance ) if a.mPassed: npassed += 1 r = float(npassed) / self.mNumReplicates self.assertAlmostEqual( self.mSignificance, r, places = self.nplaces )
def main(): parser = argparse.ArgumentParser(description="A sudoku resolver") parser.add_argument("-v", "--verbose", action="count", default=0, help="increase output verbosity (-vv or --verbose=2 for even more verbosity)") parser.add_argument("-g", "--grid", help="a grid directly on the command line (9x9 digits and dots, with or without \\n)") parser.add_argument("gridAsFile", type=argparse.FileType('r'), nargs='?', help="a file containing the grid to solve") args = parser.parse_args() if not args.gridAsFile and not args.grid: parser.error("A grid must be provided.") logLevel = [logging.WARNING, logging.INFO, logging.DEBUG][args.verbose] logging.basicConfig(format='%(levelname)s: %(message)s', level=logLevel) grid = prepareGrid(args.gridAsFile, args.grid) print "Loaded grid :" print grid.display() gridResolution = GridResolution(grid) if gridResolution.solve(): logging.info("Grid solved completely !") else: logging.info("Solving stopped without being able to finish the grid.") print "Final grid :" print grid.display() logging.info("Distribution of resolution strategies used :") for key, value in Stats.results().iteritems(): logging.info(" {}\t : {} cell(s)".format(key, value))
def loop(self): """ Begin the polling loop for Arduino data :param object conn: Connection :return None """ while True: data = self.arduino.get_data() print data if self.store: self.store.insert_new_data(data) days_left = Stats.calculate_ttnw(data[4]) if days_left > 0: self.emailer.send_mail(str(days_left) + ' until next watering!') else: self.emailer.send_mail('Water your plant TODAY!') self.graph.upload_data(data) time.sleep(float(self.opts['time']))
def __init__(self,name,stats,resists,money): self.name = name self.defence = 0 # 2 defence защищают от 1 урона self.stats = Stats() self.stats.addStats(stats) self.resists = resists self.money = money
def __call__(self, track, slice = None): data = [] # obtain evalue distribution evalues = self.getValues( "SELECT evalue FROM %(track)s_mast WHERE motif = '%(slice)s'" % locals() ) bin_edges, with_motifs, explained = Motifs.computeMastCurve( evalues ) # determine the e-value cutoff as the maximum of "explained" cutoff = bin_edges[numpy.argmax( explained )] # retrieve values of interest together with e-value rocs = [] values = self.get( """SELECT d.closest_dist, m.evalue FROM %(track)s_mast as m, %(track)s_intervals as i, %(track)s_tss AS d WHERE i.interval_id = m.id AND motif = '%(slice)s' AND d.gene_id = m.id AND d.closest_dist > 0 ORDER BY d.closest_dist""" % locals() ) rocs.append( Stats.computeROC( [ (x[0], x[1] <= cutoff) for x in values ] )) d = Histogram.Combine( rocs ) bins = [ x[0] for x in d ] values = zip( *[ x[1] for x in d ] ) result = odict() for f,v in zip(self.mFields + ("dist",), values): result[f] = odict( (("FPR", bins), (f,v)) ) return result
def GetDataStats(self, files): xs,yStats = [],[] count = 0 for j in range(len(files)): file_not_read = True while (file_not_read): try: f = h5.File(files[j],'r') xs = np.array(f[self.prefix+self.name+"/x"]) ys = np.transpose(f[self.prefix+self.name+"/y"][self.startCut:]) f.flush() f.close() yStats.append([]) for i in range(len(xs)): yStats[j].append(Stats.stats(np.array(ys[i]))) file_not_read = False except IOError as e: print 'Trouble reading', self.prefix+self.name, 'in', file except KeyError as e: print 'Data not found for', self.prefix+self.name, 'in', file file_not_read = False stats = [] for i in range(len(xs)): yStatsi = [x[i] for x in yStats] stats.append(Stats.UnweightedAvg(np.array(yStatsi))) return (xs, stats)
def __call__(self, track, slice = None ): result = odict() merged = None rocs = [] for field in self.mFields: data = [] for replicate in EXPERIMENTS.getTracks( track ): statement = "SELECT contig, start, end,%(field)s FROM %(replicate)s_intervals" % locals() data.append( self.get( statement) ) idx = [] for x in range(len(data)): i = IndexedGenome.IndexedGenome() for contig, start, end, peakval in data[x]: i.add( contig, start, end, peakval ) idx.append( i ) def _iter( all ): all.sort() last_contig, first_start, last_end, last_value = all[0] for contig, start, end, value in all[1:]: if contig != last_contig or last_end < start: yield (last_contig, first_start, last_end) last_contig, first_start, last_end = contig, start, end else: last_end = max(last_end, end ) yield (last_contig, first_start, last_end) if not merged: all = [ x for x in itertools.chain( *data ) ] merged = list( _iter(all) ) roc_data = [] for contig, start, end in merged: intervals = [] for i in idx: try: intervals.append( list(i.get( contig, start, end )) ) except KeyError: continue if len(intervals) == 0: continue is_repro = len( [ x for x in intervals if x != [] ] ) == len(data) value = max( [ x[2] for x in itertools.chain( *intervals )] ) # fpr, tpr roc_data.append( (value, is_repro) ) roc_data.sort() roc_data.reverse() roc = list(zip(*Stats.computeROC( roc_data ))) result[field] = odict( (("FPR", roc[0]), (field,roc[1])) ) return result
def duplicate_regions(snps_path, bam_path, reference_path, nodes=1, output_name="duplicate_regions"): if not os.path.exists("./.conbase"): os.makedirs("./.conbase") if not os.path.exists("../results"): os.makedirs("../results") # os.system("rm ./.conbase/duplicate_region_*") # os.system("rm ./.conbase/" + output_name + "_snp_chunk_*") snps_chunks_path, _ = Stats.snps_to_chunks(snps_path, int(nodes), output_name) jobs = [] queue = mp.Queue() for snps_chunk_path in snps_chunks_path: p = mp.Process(target=SNP_duplicate_region, args=(snps_chunk_path, bam_path, reference_path, queue)) jobs.append(p) p.start() for job in jobs: job.join() while not queue.empty(): queue.get() print('all done') f = open( '../results/' + output_name + '.tsv', 'w') f.write('CHROM' + '\t' + 'POS' + '\t' + 'REF' + '\t' + 'ALT' + '\n') f.close() for snps_chunk_path in snps_chunks_path: f = snps_chunk_path[:-4] + '_not_duplicate_region_.tsv' os.system('cat '+f+' >> ../results/' + output_name + '.tsv') os.system("rm ./.conbase/duplicate_region_*") os.system("rm ./.conbase/" + output_name + "_snp_chunk_*")
def GetDataStats(self, files): data = {} for file in files: file_not_read = True while (file_not_read): try: f = h5.File(file,'r') matrices = np.array(f[self.prefix+self.name+"/y"][self.startCut:]) shape = np.array(f[self.prefix+self.name+"/shape"]) for i in range(shape[0]): for j in range(shape[1]): data_ij = matrices[:,i,j].copy() file_stats = Stats.stats(data_ij) try: data[i,j].append(file_stats) except: data[i,j] = [file_stats] f.flush() f.close() file_not_read = False except IOError as e: print 'Trouble reading', self.prefix+self.name, 'in', file except KeyError as e: print 'Data not found for', self.prefix+self.name, 'in', file file_not_read = False stats = {} for key,val in data.iteritems(): stats[key] = Stats.UnweightedAvg(np.array(val)) return stats
def DoStatistics(self, xs, ys): stats = [] for (x,y) in zip(xs,ys): statsx = [] for yi in y: statsx.append(Stats.stats(yi)) stats.append(Stats.UnweightedAvg(statsx)) return stats
def checkFDR( self, **kwargs ): old = Stats.doFDR( self.pvalues, **kwargs ) #print old.mQValues[:10] #print old.mPi0 new = Stats.doFDRPython( self.pvalues, **kwargs ) #print new.mQValues[:10] #print new.mPi0 # self.assertAlmostEqual( old.mPi0, new.mPi0, places=3) self.assertTrue( getRelativeError( old.mPi0, new.mPi0 ) < self.max_error ) for pvalue, a,b in zip( self.pvalues, old.mQValues, new.mQValues ): self.assertTrue( getRelativeError( a,b ) < self.max_error, "qvalues: relative error %f > %f (pvalue=%f, %f, %f)" % \ (getRelativeError( a,b), self.max_error, pvalue, a, b ) )
def calculate(self): """Calculates the nearest neighbor statistic.""" #### Attribute Shortcuts #### ssdo = self.ssdo gaTable = ssdo.gaTable N = ssdo.numObs studyArea = self.studyArea ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84007), 0, N, 1) #### Create k-Nearest Neighbor Search Type #### gaSearch = GAPY.ga_nsearch(gaTable) gaConcept = self.concept.lower() gaSearch.init_nearest(0.0, 1, gaConcept) neighDist = ARC._ss.NeighborDistances(gaTable, gaSearch) distances = NUM.empty((N,), float) #### Add All NN Distances #### for row in xrange(N): distances[row] = neighDist[row][-1][0] ARCPY.SetProgressorPosition() maxDist = distances.max() if ssdo.useChordal: hardMaxExtent = ARC._ss.get_max_gcs_distance(ssdo.spatialRef) if maxDist > hardMaxExtent: ARCPY.AddIDMessage("ERROR", 1609) raise SystemExit() #### Calculate Mean Nearest Neighbor Distance #### ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84007)) observedMeanDist = distances.mean() #### Calculate Expected Mean Nearest Neighbor Distance #### expectedMeanDist = 1.0 / (2.0 * ((N / studyArea)**0.5)) #### Calculate the Z-Score #### standardError = 0.26136 / ((N**2.0 / studyArea)**0.5) #### Verify Results #### check1 = abs(expectedMeanDist) > 0.0 check2 = abs(standardError) > 0.0 if not (check1 and check2): ARCPY.AddIDMessage("Error", 907) raise SystemExit() #### Calculate Statistic #### ratio = observedMeanDist / expectedMeanDist zScore = (observedMeanDist - expectedMeanDist) / standardError pVal = STATS.zProb(zScore, type = 2) #### Set Attributes #### self.nn = observedMeanDist self.en = expectedMeanDist self.ratio = ratio self.zn = zScore self.pVal = pVal
def __init__(self, traceFile, configFile): """Search Engine Constructor""" self.stats = Stats() self.width = configFile.getFrameW() self.height = configFile.getFrameH() self.mmu = MMU(configFile, self.stats) self._initMbArrayOrder(configFile) self._initMbMatrix(traceFile) self.logger = LogFile() self.logger.setEnableLog(False)
def calculateGI(self, orderID, yiVal, nhVals, weights): """Calculates Local Gi* for a given feature. INPUTS: orderID (int): order in corresponding numpy value arrays yiVal (float): value for given feature nhVals (array, nn): values for neighboring features (1) weights (array, nn): weight values for neighboring features (1) NOTES: (1) nn is equal to the number of neighboring features """ sumW = weights.sum() sumW2 = (weights ** 2.0).sum() lagVal = (nhVals * weights).sum() ei = sumW * self.yBar dev = lagVal - ei denomNum = (self.floatN * sumW2) - sumW ** 2.0 denomG = self.S * NUM.sqrt(denomNum / self.nm1) giVal = dev / denomG pVal = STATS.zProb(giVal, type=2) #### Assign To Result Vectors #### self.gi[orderID] = giVal self.pVals[orderID] = pVal #### Do Permutations #### if self.permutations: numNHS = len(nhVals) if self.pType == "BOOT": randomInts = RAND.random_integers(0, self.numObs - 1, (self.permutations, numNHS)) else: randomInts = NUM.zeros((self.permutations, numNHS), int) for perm in xrange(self.permutations): randomInts[perm] = PYRAND.sample(self.intRange, numNHS) nhValsPerm = self.y[randomInts] lagValsPerm = (nhValsPerm * weights).sum(1) devs = lagValsPerm - ei giValsPerm = devs / denomG pseudoP = STATS.pseudoPValue(giVal, giValsPerm) self.pseudoPVals[orderID] = pseudoP
def __call__(self, track, slice = None ): statement = ''' SELECT distinct i.peakval, i.interval_id, m.motif IS NOT NULL FROM %(track)s_intervals AS i LEFT JOIN %(track)s_nubiscan AS m ON m.id = i.interval_id AND m.motif = '%(slice)s' ORDER BY i.peakval DESC''' % locals() data = self.get( statement ) result = Stats.getSensitivityRecall( [(int(x[0]), x[2] > 0) for x in data ] ) return odict( zip( ("peakval", "proportion with motif", "recall" ), zip( *result ) ) )