def generate_tree(df, root_taxon): """ Convert NCBI taxonomy node structure into a tree using a depth-first search """ logger.info( f"Generating tree from root taxon {root_taxon['rank']}={root_taxon['name']}" ) max_estimate = ESTIMATES.get(root_taxon['tax_id'], 1500000) bar = Bar('Processing', max=max_estimate, suffix=BAR_SUFFIX) def find_children(taxon): """ Get a row and all its children as a dict """ # If the taxon is blacklisted, ignore it and all its children if _is_ignored(taxon): return {} child_taxa = df[df['parent_tax_id'] == taxon['tax_id']] child_nodes = {} bar.next() # Base case: no children; Recursive case: update tree with all children for _, child_taxon in child_taxa.iterrows(): child_nodes.update(find_children(child_taxon)) # Skip over node if it's 'no rank' (typically meaning unclassified) if taxon['rank'] == 'no rank': return child_nodes else: return {f"taxonomy:{taxon['rank']}={taxon['name']}": child_nodes} tree = find_children(root_taxon) bar.finish() return tree
def gibbs_sampling(self, predict=dict(), given=dict(), n=10000, skip = 50): bar = Bar('Sampling', max=n) nodes = list(self.node.keys()) sample = self.random_sample(preset=given) count = 0 sum = 0 for i in range(n): last = None bar.next() node = None last = node while node is None or node in given.keys() or node == last: node = nodes[randint(0,len(nodes)-1)] parents = self.node[node]['parents'] if parents[0] is None: sample[node] = self.sample(node) else: given = {key: sample[key] for key in parents} sample[node] = self.sample(node, given=given) if count == skip: evidence = {key: sample[key] for key in predict.keys()} if not predict == evidence: continue sum += 1 else: count += 1 bar.finish() return sum/(n-count)
def gibbs_sampling(self, bn, predict=dict(), given=dict(), n=10000, skip = 50): bar = Bar('Sampling', max=n) nodes = list(self.node.keys()) sample = self.random_sample(bn, preset=given) count = 0 sum = 0 for i in range(n): last = None bar.next() node = None last = node while node is None or node in given.keys() or node == last: node = nodes[randint(0,len(nodes)-1)] parents = self.node[node]['parents'] if parents[0] is None: sample[node] = self.sample(node) else: given = {key: sample[key] for key in parents} sample[node] = self.sample(node, given=given) if count == skip: evidence = {key: sample[key] for key in predict.keys()} if not predict == evidence: continue sum += 1 else: count += 1 bar.finish() return sum/(n-count)
def import_profiles(self): response = requests.get(self.path) headers = response.headers total_pages = headers.get('x-wp-totalpages') if total_pages is not None: self.pages = int(total_pages) total_results = headers.get('x-wp-total') if total_results is not None: self.degrees_found = int(total_results) self.progress_bar = ChargingBar( 'Processing', max=self.degrees_found ) programs = response.json() self.process_page(programs) if self.pages > 1: for page in range(2, self.pages + 1): path = "{0}?page={1}".format(self.path, page) response = requests.get(path) programs = response.json() self.process_page(programs)
def crawl(self,current=False): """ The main function - crawl the league and mine some data. """ logging.info('Starting crawl') self.driver.get(self.league_link) self.team_names = set([unidecode(thr.text) for thr in \ self.driver.find_element_by_class_name("stat-table").find_elements_by_class_name("team-link")]) self.driver.find_element(By.XPATH, '//*[@id="sub-navigation"]').find_element(By.PARTIAL_LINK_TEXT, 'Fixtures').click() self.played_months = self.get_played_months() self.load_previous_data(current) prog_bar = ChargingBar('Progress of %s crawling:'%' '.join([self.league,str(self.year)]),max=sum([len(self.fixtures[month]) for month in self.played_months[self.played_months.index(self.start_month)::-1]])) for month in self.played_months[self.played_months.index(self.start_month)::-1]: for game in self.fixtures[month]: logging.info('Starting to parse game') if game: self.parse_game(game) prog_bar.next() logging.info('Finished game, moving to the next one') else: logging.info('Finished month, saving to disk') self.save_month(month) if current: pass # DBHandler(args_parser.LEAGUE_NAME).update_db(self.all_teams_dict,str(self.year)) else: #we're done - we can save to the DB now # if not current: DBHandler(args_parser.LEAGUE_NAME).insert_to_db(self.all_teams_dict,str(self.year)) self.driver.quit() prog_bar.finish()
def train(self, inputExamples, expected, iterations=BARSIZE): if self.initialized: currentIteration = 1 barIteration = 0 statusBar = ChargingBar("\x1b[4;36m"+"\t>> Training:", max=BARSIZE) interval = int(iterations/BARSIZE) if iterations > 100 else 1 errorMedia = 0 statusBar.start() while(currentIteration <= iterations): errorMedia = 0 prediction = self.__backPropagation(inputExamples, expected) errorMedia = self.cost_fun(prediction, expected) currentIteration += 1 if barIteration % interval == 0: statusBar.next() barIteration = 0 barIteration += 1 while(currentIteration < BARSIZE): currentIteration += 1 statusBar.next() statusBar.finish() self.trainingStatus = str(round(errorMedia,4)) + "TE" print("\x1b[1;33m"+"\t>> Error (Media) Cost: ", round(errorMedia,4)) print("\x1b[0;37m"+"=-"*35 + "=") else: print("<Error>: Empty Neural Network, use reset() or loadFromFile(file)") exit(1)
def UpdateRoutine(): addrinfo = socket.getaddrinfo(MYGROUP_4, None)[0] Socket = socket.socket(addrinfo[0], socket.SOCK_DGRAM) # Set Time-to-live (optional) ttl_bin = struct.pack('@i', MYTTL) Socket.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl_bin) SecureFirmwareName = 'SecureFirmware.bin' SecureFirmwareLen = GetFirmwareSize(SecureFirmwareName) print("Firmware file has load.") ChunkSize = 1012 ChunkCount = ceil( SecureFirmwareLen / ChunkSize ) SecureFirmwareFile = open(SecureFirmwareName,"rb") print("Firmware file size is %s bytes" % SecureFirmwareLen) datafile = SecureFirmwareFile.read( ChunkSize ) Progress = ChargingBar('Updating', max=ChunkCount, suffix = '%(index)d/%(max)d [%(percent)d%%]') while datafile: Socket.sendto(datafile, (addrinfo[4][0], MYPORT)) datafile = SecureFirmwareFile.read( ChunkSize ) Progress.next( ) time.sleep(0.2)
def apt_get(pkg_name_list): cache = apt.cache.Cache() cache.update() cache.open() print('Telechargement de ', *pkg_name_list) bar = ChargingBar('Telechargement & Installation', max=len(pkg_name_list) + 1) bar.next() for pkg_name in pkg_name_list: pkg = cache[pkg_name] if pkg.is_installed: print("\n{pkg_name} est deja installer".format(pkg_name=pkg_name), file=log_file) bar.next() else: pkg.mark_install() try: cache.commit(install_progress=LogInstallProgress()) print("{pkg_name} c\'est installer".format(pkg_name=pkg_name), file=log_file) bar.next() except (Exception, arg): print(colored( "l\'installation du packet a echouer [{err}]".format( err=str(arg)), "red"), file=log_file) bar.finish()
def run(directory, output): """ Compiles a directory of HTML into a single pdf. :param directory: Directory to search. :param output: filename to write to. """ files = [f for f in os.listdir(directory) if f[-5:] == '.html'] files.sort(key=lambda x: int(x[:4])) with ChargingBar('Converting', max=len(files)) as progress: for file in files: html = weasyprint.HTML(filename=f'{directory}/{file}') html.write_pdf(target=f"{directory}/{file[:-5]}.pdf") progress.next() merger = PdfFileMerger() with ChargingBar('Merging', max=len(files) + 1) as progress: for file in files: reader = open(f"{directory}/{file[:-5]}.pdf") merger.append(reader, bookmark=f"{file[5:-5]}", import_bookmarks=False) progress.next() merger.write(output) progress.next() for file in files: os.remove(f"{directory}/{file[:-5]}.html") os.remove(f"{directory}/{file[:-5]}.pdf")
def key_gen(): # shared - n, open_key - e,secret_key - d global shared_key global open_key global secret_key bar = ChargingBar('Генерация ключей', max=3) while True: p = rnd.randint(50, 1000) if prost(p) == True: break bar.next() time.sleep(1) while True: q = rnd.randint(50, 1000) if prost(q) == True: break shared_key = p * q # общая часть ключа pq = (p - 1) * (q - 1) bar.next() time.sleep(1) while True: d = rnd.randint(50, 10000) if vz_prost(d, pq) == True: break secret_key = d # часть секретного ключа sootv_usl = [e for e in range(50, 10000000) if (e * d) % pq == 1] open_key = rnd.choice(sootv_usl) # часть открытого ключа bar.next() time.sleep(1) bar.finish() print("Открытый ключ: {", open_key, ",", shared_key, "}") print("Секретный ключ: {", secret_key, ",", shared_key, "}")
def update_progress_bar(queue, length): bar = ChargingBar('Downloading ------------>', max=length) while True: try: data = queue.get() bar.next(data) except EOFError: pass
def write_ml_csv(self): """ Writes the machine learning / info file containing user behavior :return: Nothing """ # 2D list, 1st row contains variable title and each row after contains data for 1 user out = [] bar = ChargingBar('Writing data file', max=2600, stream=sys.stdout) # calculate and add all variables in the 2D list self.add_user_metadata(out) self.var1_nb_im_visited(out, bar) self.var2_total_nb_positions(out, bar) self.var3_average_nb_positions(out, bar) self.var4_median_nb_positions(out, bar) self.var5_total_time(out, bar) self.var6_avg_time(out, bar) self.var7_median_time(out, bar) self.var8_position_zooms(out, bar) self.var9_avg_position_zooms(out, bar) self.var10_median_position_zooms(out, bar) self.var11_total_annotation_actions(out, bar) self.var12_avg_annotation_actions(out, bar) self.var13_median_annotation_actions(out, bar) self.var14_avg_position_by_zoom(out, bar) self.var15_median_position_by_zoom(out, bar) self.var16_user_image_score_per_image(out, bar) self.var17_per_image_info(out, bar) self.var18_module_variables(out, bar) self.var19_relative_time_worked(out, bar) self.var20_nb_of_days_worked(out, bar) self.var21_module_percent_time(out, bar) self.var22_annotation_visit_order(out, bar) self.var_results(out) # output file if self.ml_out_dir is None: csv_out_filename = 'learning_data.csv' stats_file = os.path.join( config.WORKING_DIRECTORY + self.project_name + "/", csv_out_filename) else: stats_file = self.ml_out_dir f = open(stats_file, "wb") csv_out = csv.writer(f) # write data into file print len(out[0]) csv_out.writerow(out[0]) for i in range(len(self.user_list) + 1): csv_out.writerow(out[i + 1]) f.close() bar.finish()
def calc_inter_layer_covariance(self, model_wrapper, use_training_data=True, batch_size=-1, **options): is_chainer = model_wrapper.model_type == "chainer" train, test = model_wrapper.dataset data_x = train if use_training_data else test if is_chainer: data_x = np.moveaxis(data_x, -1, 0)[0] else: data_x = data_x[0] data_x = np.stack(data_x, axis=0) data_size = n = len(data_x) if batch_size > 0: perm = np.random.permutation(data_size) data_x = data_x[perm[0:batch_size]] n = batch_size n_layers = len(model_wrapper.layers()) bar = ChargingBar("Calculating inter layer covariance", max=n_layers) layer_outputs = model_wrapper.get_layer_outputs(data_x) to_save = {} for l, layer_output in enumerate(layer_outputs): if is_chainer: layer_output = layer_output.data flat_shape = layer_output[0].flatten().shape[0] sigma = tf.zeros(shape=(flat_shape, flat_shape), dtype=tf.float32, name="sigma%d" % l) for output in layer_output: g = tf.constant(output.flatten()) sigma += tf.einsum('i,j->ij', g, g) sigma = tf.Variable(1 / (n - 1) * sigma, name="sigma%d" % l) eigen_values = tf.self_adjoint_eigvals(sigma, name="eigen_values%d" % l) to_save["sigma%d" % l] = sigma to_save["eigen_values%d" % l] = tf.Variable(eigen_values) bar.next() self.store_elements(group_name="inter_layer_covariance", elements=to_save, model_wrapper=model_wrapper)
def __process_research(self): self.progress_bar = ChargingBar( 'Processing research...', max=self.researchers_to_process.qsize()) self.mt_lock = threading.Lock() for _ in range(self.max_threads): threading.Thread(target=self.get_researcher_research, daemon=True).start() self.researchers_to_process.join()
def recursive_combat(player_1, player_2, level=1, bar: ChargingBar = None): print_wrapper(f"=== Game {level} ===") visited_games = set() round = 0 while player_1 and player_2: round += 1 if (player_1, player_2) in visited_games: return 1, player_1 visited_games.add((player_1, player_2)) print_wrapper(f"Player 1's deck: {player_1}") print_wrapper(f"Player 2's deck: {player_2}") card_1, player_1 = player_1[0], player_1[1:] card_2, player_2 = player_2[0], player_2[1:] print_wrapper(f"Player 1 plays: {card_1}") print_wrapper(f"Player 2 plays: {card_2}") # breakpoint() if len(player_1) >= card_1 and len(player_2) >= card_2: print_wrapper("Playing a sub-game to determine the winner...") winner, _ = recursive_combat( copy_deck(player_1, card_1), copy_deck(player_2, card_2), level=level + 1, ) print_wrapper("...anyway, back to game 1.") if winner == 1: print_wrapper(f"Player 1 wins round {round} of game {level}!") player_1 = (*player_1, card_1, card_2) else: print_wrapper(f"Player 2 wins round {round} of game {level}!") player_2 = (*player_2, card_2, card_1) else: if card_1 > card_2: print_wrapper(f"Player 1 wins round {round} of game {level}!") player_1 = (*player_1, card_1, card_2) else: print_wrapper(f"Player 2 wins round {round} of game {level}!") player_2 = (*player_2, card_2, card_1) if bar and max(len(player_1), len(player_2)) > bar.index: bar.goto(max(len(player_1), len(player_2))) winner = max(player_1, player_2) if winner == player_1: print_wrapper(f"The winner of game {level} is player 1!") else: print_wrapper(f"The winner of game {level} is player 2!") return (1 if winner == player_1 else 2), winner
def generate_gif(input_file): if not os.path.exists(folder): os.makedirs(folder) video_duration = (int( float( subprocess.check_output([ "ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", input_file ], universal_newlines=True).strip()))) # creating vars for easy reading current_time = args.begin random_start = args.randstart random_end = args.randend # workaround for videos in source root folder if len(input_file.split("/")) == 1: print("\nCurrent file: " + input_file) else: print("\nCurrent file: " + input_file.split("/")[1]) bar = ChargingBar('Processing', max=video_duration) # initialize bar bar.goto(0) while current_time <= video_duration: current_time = current_time + (random.randrange( random_start, random_end)) if current_time > video_duration: # set progress bar to 100% before breaking bar.goto(video_duration) break # creating vars for easy reading filename_generator = args.destination + '%s-%s.gif' gif_length = args.length # you can increase the fps=12 and scale=w=480 values with a higher number for smoother/bigger gifs, # increases the file size. subprocess.check_output([ 'ffmpeg', '-y', '-ss', str(current_time), '-t', gif_length, '-i', input_file, '-filter_complex', '[0:v] fps=12,scale=w=480:h=-1,split [a][b];[a] palettegen=stats_mode=single [p];[' 'b][p] paletteuse=new=1', filename_generator % (input_file, current_time) ], stderr=subprocess.STDOUT, universal_newlines=True).strip() bar.goto(current_time) bar.finish()
def __setup_description_processing(self): """ Sets up multiple threads for processing catalog descriptions. """ self.catalog_description_progress = ChargingBar( 'Processing descriptions...', max=self.catalog_description_queue.qsize()) for _ in range(self.max_threads): Thread(target=self.__process_descriptions, daemon=True).start() self.catalog_description_queue.join()
def __setup_curriculum_processing(self): """ Sets up multiple threads for processing catalog curriculum data. """ self.catalog_curriculum_progress = ChargingBar( 'Processing curriculums...', max=self.catalog_curriculum_queue.qsize()) for _ in range(self.max_threads): Thread(target=self.__process_curriculums, daemon=True).start() self.catalog_curriculum_queue.join()
def __init__(self, path, language, outpath=None): self.path = path self.language = language self.translator = Translator() with open(self.path) as f: self.lines = f.readlines() self.parts_to_skip = [ 'equation', 'array', 'figure', 'algorithm', 'hyp', 'thm', 'table', 'tabular' ] self.list_commands = ['itemize', 'enumerate'] self.outpath = outpath self.bar = ChargingBar('Translating Tex File...', max=len(self.lines))
def train(self, inputs, outputs, iterations): if self.initialized: statusBar = ChargingBar('\t>> Training:', max=100) status = False iteration = 0 interval = int(iterations/100) if iterations > 100 else 1 passed = 0 nonPassed = 0 while(status != True and iteration < iterations): status = True passed = 0 nonPassed = 0 for i in range(len(inputs)): currentStatus = self.fit(inputs[i], outputs[i]) status = status and currentStatus if currentStatus: passed += 1 else: nonPassed += 1 iteration += 1 if iteration % interval == 0: statusBar.next() while(iteration < 100): iteration += 1 statusBar.next() statusBar.finish() print("\t>> Tests Passed:", passed, "- Test Non passed:", nonPassed) self.trainingStatus = str(int((float(passed)/float(passed + nonPassed)) * 100)) + "%" print("<>"*20) return print("<Error>: Empty Neural Network, use overrideNewNN() or loadFromFile(file)") exit(1)
def PREPARE_DATASET(trainingArticleCount, preprocessor: Preprocessor, maxArticles, allowedCategories=[], dtype='reuters') -> List[DataSet]: #create Array with two datasets. One training, one test dataSet = [DataSet(), DataSet()] #check datatype and initialize provider if dtype == 'reuters': soupLoader = SoupLoader(-1) provider = ReutersProvider(soupLoader) else: provider = TwentyNewsProvider('../TwentyNews/') #start nice percentage bar. Good to have visuals ;) bar = ChargingBar("Preparing dataset: ", max=maxArticles) while bar.index <= maxArticles: try: #try to create a new article article = ArticleFactory.GET_NEXT_ARTICLE( provider, allowedCategories) article.process(preprocessor) #append the article to the dataset dataSet[int(bar.index / trainingArticleCount)].append(article) bar.next() except OutOfArticlesError: #bar would stop at 99% if not increased once more bar.next() break bar.finish() return dataSet
def upload_to_ya(self, upload_list): ya_obj = YaUpLoader(self.ya_token) ya_load_to = input('\nВведите путь до папки на ya_disk: ') print(ya_obj.check_folder(ya_load_to)) print(f'\nЗагружаем файлы на YaDisk') bar = ChargingBar('Countdown', max=len(upload_list[1])) hash_map = {} for photo in upload_list[1]: bar.start() file_name = photo['file_name'] if file_name in hash_map.keys(): last_name = file_name value = hash_map[last_name] + 1 file_name = file_name.split('.')[0] + '_' + str(value) + '.jpg' hash_map[last_name] = value else: hash_map[file_name] = 1 ya_file_to = ya_load_to + '/' + file_name url = photo['url'] res = requests.get(url).content ya_obj.upload(ya_file_to, res) bar.next() bar.finish()
def decrypt_file(self, ciphertext_file_name, plaintext_file_name): # filename, chunk_size, padding_size, base_destination): ct_chunk_reader = self.decryption_chunk_reader(ciphertext_file_name, self.padding_block_size_bytes, self.base_num) # ct_chunk_reader = self.decryption_chunk_reader(ciphertext_file_name) #, self.padding_block_size_bytes, self.base_num, self.max_block_size_after_encrypt_bytes, self.block_size_bytes ) pt_file = open(plaintext_file_name, 'wb+') file_size = os.stat(ciphertext_file_name).st_size bar = ChargingBar('[*] Decrypting ', max=(file_size // (self.max_block_size_after_encrypt_bytes * 1000)) + 1) for index, (pt_chunk, block_size) in enumerate(ct_chunk_reader): if index % 1000 == 0: bar.next() nums_after_lookup_table = '' for num in pt_chunk: nums_after_lookup_table = nums_after_lookup_table + self.look_up_table_decryption.get(num) nums_after_lookup_table = int(nums_after_lookup_table, self.base_num) # if len(bin(nums_after_lookup_table)[2:])/8 > block_size: # someshit=1 ct_bytes = nums_after_lookup_table.to_bytes(block_size, byteorder='little') # ct_bytes = nums_after_lookup_table.to_bytes(self.block_size_bytes, byteorder='little') pt_file.write(ct_bytes) # except Exception as e: # print(e) # pass bar.next(bar.max - bar.index) bar.finish() l("Decryption done") l("Saved at %s" % os.path.abspath(pt_file.name))
def handle(self, *args, **options): """ Main entry function for the command. Execution logic handled here. """ # Remove all existing Units before proceeding. # Unfortunately, the logic in this importer currently # is not consistent on imports against existing Units # vs a fresh db, so to have the most accurate data we can, # we have to start fresh with each run. Unit.objects.all().delete() # Perform mapping. # NOTE: order is important here! Particularly, # in order for teledata and Program Departments to # map properly, Colleges must be mapped first. self.full_name_replacements = settings.UNIT_NAME_FULL_REPLACEMENTS self.basic_replacements = settings.UNIT_NAME_PARTIAL_REPLACEMENTS self.lowercase_replacements = settings.UNIT_NAME_LOWERCASE_REPLACEMENTS self.uppercase_replacements = settings.UNIT_NAME_UPPERCASE_REPLACEMENTS self.colleges_processed = College.objects.all() # Teledata Organizations that look like they could align to a College # should be prioritized for processing, hence the ordering here: self.teledata_orgs_processed = TeledataOrg.objects.annotate( college_index=StrIndex('name', V('college'))).order_by( '-college_index') self.program_depts_processed = ProgramDept.objects.all() self.teledata_depts_processed = TeledataDept.objects.all() self.mapping_progress_bar = ChargingBar( 'Mapping data...', max=self.colleges_processed.count() + self.teledata_orgs_processed.count() + self.program_depts_processed.count() + self.teledata_depts_processed.count()) self.map_orgs_colleges() self.map_orgs_teledata() self.map_depts_programs() self.map_depts_teledata() # Consolidate duplicate Units as best as we can. self.consolidatable_unit_names = Unit.objects.values('name').annotate( name_count=Count('pk')).filter(name_count=2) self.cleanup_progress_bar = ChargingBar( 'Cleaning up...', max=self.consolidatable_unit_names.count()) self.consolidate_duplicate_units() # Done. self.print_stats()
def rejection_sample(self, predict=dict(), given=dict(), n=10000): sum = 0 bar = Bar('Sampling', max=n) for i in range(n): bar.next() sample = self.compute_sample() evidence = {key: sample[key] for key in given.keys()} if not given == evidence: continue evidence = {key: sample[key] for key in predict.keys()} if not predict == evidence: continue sum += 1 bar.finish() return sum/n
def rejection_sample(self, predict=dict(), given=dict(), n=10000): sum = 0 bar = Bar('Sampling', max=n) for i in range(n): bar.next() sample = self.compute_sample() evidence = {key: sample[key] for key in given.keys()} if not given == evidence: continue evidence = {key: sample[key] for key in predict.keys()} if not predict == evidence: continue sum += 1 bar.finish() return sum/n
def __match_programs(self): """ Loops through the catalog entries and attempts to match them to existing programs. """ self.catalog_match_progress = ChargingBar( 'Matching existing programs to catalog entries...', max=len(self.matchable_programs)) for mp in self.matchable_programs: self.catalog_match_progress.next() # Create a list of CatalogEntry's to match against that # share the same career and level as the program, and # that share a college: filtered_entries = [ x for x in self.catalog_entries if x.level_pk == mp.level_pk and x.career_pk == mp.career_pk and (x.college_short in mp.program.colleges.values_list('short_name', flat=True) if x.college_short is not None else True) ] # Determine all potential catalog entry matches for the program: for entry in filtered_entries: mp.match(entry) if mp.has_matches: # Send the MatchableProgram off for further processing self.catalog_description_queue.put(mp) # Get the best match and save it for later match = mp.get_best_match() mp.best_match = match[1] # Increment match counts self.program_match_count += 1 mp.best_match.match_count += 1 logging.log( logging.INFO, f"MATCH \n Matched program name: {mp.program.name} \n Cleaned program name: {mp.name_clean} \n Catalog entry full name: {mp.best_match.data['title']} \n Cleaned catalog entry name: {mp.best_match.name_clean} \n Match score: {match[0]} \n" ) else: logging.log( logging.INFO, f"FAILURE \n Matched program name: {mp.program.name} \n Cleaned program name: {mp.name_clean} \n" )
def likelihood_weighting(self, predict=dict(), given=dict(), n=10000): num = den = 0 bar = Bar('Sampling', max=n) for i in range(n): bar.next() sample = self.compute_sample(preset=predict) for node in predict.keys(): parents = self.node[node]['parents'] given_pa = {key: sample[key] for key in parents} weight = float(self.get_probability(node, evidence=given_pa, value=predict[node])) evidence = {key: sample[key] for key in given.keys()} if given == evidence: num += weight den += weight bar.finish() return num/den
def _align_reads(): consolidated_results_list = [] with open(CSV_FILE) as csv_file: csv_reader = csv.reader(csv_file, delimiter=',') output_file_count = 1 for row in ChargingBar('Processing', max=length(CSV_FILE)).iter(csv_reader): index_name = _get_substr_before('.', row[REF_SEQ_INDEX]) run_args = [ ALIGNER, REF_SEQ_FLAG, index_name, READ_1_FLAG, row[READ_1_INDEX], READ_2_FLAG, row[READ_2_INDEX], OUTPUT_FILE_FLAG, OUTPUT_FILE_PREFIX + str(output_file_count) + OUTPUT_FILE_EXT ] proc = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) overall_alignment_rate = _get_overall_alignment_rate( str(proc.communicate()[PROC_STDOUT_INDEX])) consolidated_results_list.append([ row[REF_SEQ_INDEX], row[READ_1_INDEX], row[READ_2_INDEX], overall_alignment_rate ]) output_file_count += 1 return consolidated_results_list
def likelihood_weighting(self, predict=dict(), given=dict(), n=10000): num = den = 0 bar = Bar('Sampling', max=n) for i in range(n): bar.next() sample = self.compute_sample(preset=predict) for node in predict.keys(): parents = self.node[node]['parents'] given_pa = {key: sample[key] for key in parents} weight = float(self.get_probability(node, evidence=given_pa, value=predict[node])) evidence = {key: sample[key] for key in given.keys()} if given == evidence: num += weight den += weight bar.finish() return num/den
def tag_dir(dir_path): ''' Takes a directory at <dir_path> Tags its content (audio files only) with metadata retrieved through the AcoustID API ''' lth = 0 for _file in listdir(dir_path): if is_audio_file(dir_path + '/' + _file): lth += 1 bar = ChargingBar(max=lth, suffix='[%(index)d/%(max)d]') for _file in listdir(dir_path): song_path = dir_path + '/' + _file if is_audio_file(song_path): bar.message = INDENT + cyan_fg(_file) + '\n' tag(song_path) bar.next()
def doList(taskList): print("Do tasks on list") print(f"""Here are your tasks. Enter the number of the task when you complete it:""") for task in taskList: print(f"{taskList.index(task) + 1}. {task[0]}") completedTasks = [] os.chdir('c:\\Users\\tomas\\Desktop\\Anti-Procrastination') now = datetime.datetime.now() path = f'Todo list - quicklist - {now.year}-{now.month}-{now.day}.txt' todo = open(path, 'a') todo.write(f"\n\nThe date is {now.month} and you have {len(taskList)} tasks to finish!\n") #bar = ChargingBar('Status', max=len(taskList)) print('\n') with ChargingBar(max=len(taskList)) as bar: for i in range(len(taskList)): update = int(input("Enter number of completed task: \n")) - 1 now = microSecSlicer(datetime.datetime.now()) timeTaken = now - taskList[update][1] todo.write(f"{taskList[update][0]} - TASK COMPLETE\nIt took you(hours:minutes:seconds): {timeTaken}\n") completedTasks.append(taskList[update]) bar.next() print("\nRemaining tasks:") for task in taskList: if task not in completedTasks: print(f"{taskList.index(task) + 1}. {task[0]}") if len(completedTasks) == len(taskList): print("all tasks completed") else: print("something went wrong, length of tasklist and length of completedTasks do not match") todo.close() print("closing file...")
def subList(task): print( f"You have selected {task[0]} to work on. Let\'s break this task down into managable sub-tasks." ) time.sleep(1) subTasks = [] entry = "" while entry.lower() != "stop": entry = (input("what task do you want to add?\n")) if entry.lower() == "stop": continue else: subTasks.append(entry) print(f"{entry} is added to your sublist for {task[0]}.") with ChargingBar(max=len(subTasks)) as bar: while len(subTasks) > 0: print("\n") for subTask in subTasks: print(f"{subTasks.index(subTask) + 1}.{subTask}") update = input('Enter the number of sub-task completed.') if update.isdecimal() == True: if int(update) - 1 in range(len(subTasks)): subTasks.pop(int(update) - 1) bar.next() time.sleep(1) bar.finish() time.sleep(1) print("All done! Now you can take this tricky task off your list.") time.sleep(1)
def discover_net(self, ip_range=24): protocol = self.protocol base_ip = self.my_ip # print_figlet() if not protocol: protocol = "ICMP" else: if protocol != "ICMP": logging.warning( f"Warning: {protocol} is not supported by discover_net function! Changed to ICMP" ) if protocol == "ICMP": logging.info("Starting - Discover Hosts Scan") base_ip = base_ip.split('.') base_ip = f"{str(base_ip[0])}.{str(base_ip[1])}.{str(base_ip[2])}.0/{str(ip_range)}" hosts = list(ipaddress.ip_network(base_ip)) bar = ChargingBar("Scanning...", max=len(hosts)) sys.stdout = None bar.start() threads = [None] * len(hosts) results = [None] * len(hosts) for i in range(len(threads)): threads[i] = Thread(target=self.send_icmp, args=(hosts[i], results, i)) threads[i].start() for i in range(len(threads)): threads[i].join() bar.next() bar.finish() sys.stdout = sys.__stdout__ hosts_found = [i for i in results if i is not None] if not hosts_found: logging.warn('[[red]-[/red]]Not found any host') else: print("") logging.info(f'{len(hosts_found)} hosts founded') for host in hosts_found: logging.info(f'Host found: {host}') return True else: logging.critical("[[red]-[/red]]Invalid protocol for this scan") return False
def create_examples(self,year,lookback=15,current=False): """ This function creates all the examples for self.league, year. The examples are created using the given lookback. """ def update_all_teams_dict(res,all_teams_dict,team,first): for fix in sorted(res): if fix == 1 and res[fix] == {}: all_teams_dict[team][fix] = [] continue if first: all_teams_dict[team][fix] = [res[fix][k] for k in sorted(res[fix])] else: all_teams_dict[team][fix] += [res[fix][k] for k in sorted(res[fix])] def relative_features(arr1,arr2,fn): combined_list_all_1 = [value for (value,key) in zip(arr1,fn) if key.split("all_pos")>1 ] combined_list_att_1 = [value for (value,key) in zip(arr1,fn) if key.split("att_pos")>1 ] combined_list_def_1 = [value for (value,key) in zip(arr1,fn) if key.split("def_pos")>1 ] combined_list_all_2 = [value for (value,key) in zip(arr2,fn) if key.split("all_pos")>1 ] combined_list_att_2 = [value for (value,key) in zip(arr2,fn) if key.split("att_pos")>1 ] combined_list_def_2 = [value for (value,key) in zip(arr2,fn) if key.split("def_pos")>1 ] all_rel = [1 for (val1,val2) in zip (combined_list_all_1,combined_list_all_2) if val1 > val2] att_rel = [1 for (val1,val2) in zip (combined_list_att_1,combined_list_att_2) if val1 > val2] def_rel = [1 for (val1,val2) in zip (combined_list_def_1,combined_list_def_2) if val1 > val2] return float(len(all_rel))/len(combined_list_all_1), float(len(att_rel))/len(combined_list_att_1), float(len(def_rel))/len(combined_list_def_1) from features.features import Features temp_DB = self.temp_DB all_teams_names = [g['_id'] for g in temp_DB[self.league].aggregate([{"$match":{"Year":int(year)}},{"$group":{"_id":"$GName"}}])] all_teams_dict = {name:{} for name in all_teams_names} features = Features(temp_DB[self.league],year,self.league) features_names = [] prog_bar = ChargingBar('Creating examples for %s-%s'%(self.league,year),max=len(all_teams_dict)) for team in all_teams_dict: res_by_all, res_by_non_avg = features.create_features(team,lookback) if not features_names: features_names = features.features_names update_all_teams_dict(res_by_all, all_teams_dict, team, True) update_all_teams_dict(res_by_non_avg, all_teams_dict, team, False) prog_bar.next() examples = [] tags = [] curr_examples = [] prog_bar.finish() for team in all_teams_names: for fix in sorted(all_teams_dict[team]): if fix == 1 and all_teams_dict[team][fix]==[]: continue curr_game = temp_DB[self.league].find_one({"GName":team,"Fix":fix,"Year":int(year)}) if curr_game is None: continue if curr_game["HA"]=="home": vs_curr_game = temp_DB[self.league].find_one({"GName":curr_game["VS"],"VS":team,"HA":"away","Year":int(year)}) try: vs_curr_fix = vs_curr_game["Fix"] except TypeError as e: vs_curr_fix = fix+1 all_teams_dict[curr_game["VS"]][vs_curr_fix] = [] if all_teams_dict[curr_game["VS"]][vs_curr_fix] == []: continue rel_all, rel_att, rel_def = relative_features(all_teams_dict[team][fix], all_teams_dict[curr_game["VS"]][vs_curr_fix], features_names) examples += [np.array(all_teams_dict[team][fix])-np.array(all_teams_dict[curr_game["VS"]][vs_curr_fix])] examples[-1] = np.concatenate((examples[-1],[rel_all, rel_att, rel_def])) temp_dict = {"Ex":examples[-1],"Fix":curr_game["Fix"],"Res":curr_game["Result"],"Home":team,"Away":curr_game["VS"],"League":self.league} curr_examples += [temp_dict] tags += [curr_game["Tag"]] if not current: return examples,tags else: return curr_examples,tags
def main(args=sys.argv[1:]): parser = argparse.ArgumentParser( prog='calbum', add_help=True, description='calbum is an unattended calendar-based photo organiser. ' 'It is meant to allow easy management of pictures based ' 'on their location, date and calendar events without ' 'the need of a database or a special browser to retrieve ' 'them.') parser.add_argument('--link-only', help='Keep files where they are in the inbox folder.', action='store_true') parser.add_argument('--inbox', help='The path of the inbox directory. ' '(default: ./inbox)', metavar='path', default='./inbox') parser.add_argument('--timeline', help='The path of the timeline directory. ' '(default: ./timeline)', metavar='path', default='./timeline') parser.add_argument('--album', help='The path of the album directory. ' '(default: ./album)', metavar='path', default='./album') parser.add_argument('--calendar', help='The url of the album calendar. (ical)', metavar='url') parser.add_argument('--date-format', help='The format to use for timestamps.', metavar='format', default=model.TimeLine.media_path_format) parser.add_argument('--save-events', help='Keep the calendar event in the album.', action='store_true') parser.add_argument('--time-zone', metavar='tz', help='Pictures timezone (default to local time).') settings = vars(parser.parse_args(args)) # Configure data model model.TimeLine.media_path_format = settings['date_format'] model.Media.time_zone = gettz(settings['time_zone']) # Create filters timeline_filter = timeline.TimelineFilter(settings['timeline']) album_filter = NoopMediaFilter() if settings['calendar']: events = calendar.CalendarEvent.load_from_url(url=settings['calendar']) album_filter = album.CalendarAlbumFilter( albums_path=settings['album'], events=events, save_events=settings['save_events'] ) filter_actions = [ timeline_filter.link if settings['link_only'] else timeline_filter.move, album_filter.link ] # Perform actions suffix = '%(index)d/%(max)d [eta: %(eta)ds]' bar = ChargingBar('Processing inbox:', suffix=suffix) for picture in bar.iter(list(model.MediaCollection(settings['inbox']))): for action in filter_actions: action(picture)
cmd = "istat "+fs+" "+inode+" | awk '/Direct/,/^$/'" ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE) grep_o = ps.communicate()[0] blocks = grep_o.split('\n')[1:-1] index = 1 for i in range(1, len(blocks)): if blocks[-i] != "": index = i break last_block = blocks[-index].split()[-1] # store file info f = {"inode":inode, "name":filename, "size":size, "block": last_block} files.append(f) bar_max = len(files) bar = ChargingBar('Progress:', max=bar_max) # loop through files and read last block for f in files: cmd = "dd if="+fs+" bs="+bsize+" skip="+f["block"]+" count=1 > "+tmpfile+" 2> /dev/null" ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE) output = ps.communicate()[0] # compute how many bytes to expect in last block nblocks = int(math.ceil(int(f['size'])/float(bsize))) slack = nblocks*int(bsize) - int(f['size']) actualbytes = int(bsize) - slack # read slack space tmp = open(tmpfile, "rb") bcount = 0 hidden="" try: byte = tmp.read(1)