def download_model(name, url): """ Downloads large model file returns the hash of the newly downloded model and location of the model in temp folder :param url: string of url location of the model :param name: string name of model """ # https://stackoverflow.com/questions/16694907/download-large-file-in-python-with-requests local_filename = url.split('/')[-1] local_filename = TEMP_LOCATION + local_filename full_hash = sha3_256() with requests.get(url, stream=True) as r: size = r.headers['content-length'] if size: p = ShadyBar(local_filename, max=int(size)) else: p = Spinner(local_filename) with open(local_filename, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): if chunk: # filter out keep-alive new chunks p.next(len(chunk)) f.write(chunk) full_hash.update(chunk) # f.flush() unique_filename = MODEL_LOCATION + name + ".h5" os.rename(local_filename, unique_filename) return full_hash.hexdigest(), unique_filename
def compute_edit_distance_variability(event_log: Log) -> float: """ Computes the edit distance variability of the log using the Levenstein distance formula Args: event_log: an instance of Log class Returns: The computed edit distance of the input log """ traces = event_log.trace_list distance: int = 0 number_of_comparisons: int = 0 bar = ShadyBar("Edit distance computation", max=len(traces) - 1) trace_1: Trace = traces[0] for trace_2 in traces: if trace_1 != trace_2: bar.next() distance += _levenshtein_distance( trace_1, trace_2) * trace_1.frequency * trace_2.frequency number_of_comparisons += 1 bar.finish() return distance / number_of_comparisons
def main(): logging.basicConfig(level=logging.INFO, filename="pynmj.log") #_LOGGER.setLevel(logging.INFO) options, arguments = parse_options() try: try: get_lock(arguments[0]) updater = NMJUpdater(arguments[0], "local_directory") if options.clean_name: updater.clean_names() medias = updater.scan_dir() _LOGGER.info("Found %s medias", len(medias)) bar = ShadyBar('Updating database', max=len(medias), suffix='[ETA %(eta_td)s] (%(percent)d%%)') for rank, media in enumerate(medias): _LOGGER.info("Media %s/%s", rank + 1, len(medias)) updater.search_media_and_add(media) bar.next() _LOGGER.info("Cleaning DB...") updater.clean() _LOGGER.info("Done") bar.finish() except: import traceback traceback.print_exc() finally: release_lock(arguments[0])
def run(self): if os.path.exists(self.results_file): # There's something in the cache, read it. print('Using cached result in working directory') regions = self.read_reduce_result(self.results_file) max_id = int(open(self.max_id_file).read()) return regions, max_id print('No cached result found in working directory') self.reset() cpus = mp.cpu_count() pool = mp.Pool(processes=cpus) print('Starting mapreduce using %d cores' % cpus) qfiles = os.listdir(self.qdir) bar = ShadyBar('Mapping', max=len(qfiles), suffix='%(index)d/%(max)d - %(elapsed)ds') for _ in pool.imap_unordered(self.mapper, qfiles): bar.next() bar.finish() pool.terminate() pool = mp.Pool(processes=cpus) bar = ShadyBar('Reducing', max=self.num_map_outputs, suffix='%(index)d/%(max)d - %(elapsed)ds') for _ in pool.imap_unordered(self.reducer, range(self.num_map_outputs)): bar.next() bar.finish() pool.terminate() return self.aggregate(), self.max_id
def compute_my_variability(event_log: Log) -> float: """ Computes the prefix entropy of the input Log Args: event_log (Log): the input log Returns: the prefix-block entropy """ prefixes: List[List[Event]] = [] bar: Bar = IncrementalBar("Prefix generation", max=len(event_log.trace_list)) for trace in event_log.trace_list: trace_prefixes: List[List[Event]] = trace.get_all_prefixes() for prefix in trace_prefixes: if prefix not in prefixes: prefixes.append(prefix) bar.next() bar.finish() entropy: float = 0 bar = ShadyBar("Prefix likelihood estimation", max=len(prefixes)) for prefix in prefixes: p: float = _prefix_likelihood_estimator(event_log, prefix) entropy += p * logarithm(p, 10) bar.next() bar.finish() entropy *= -1 return entropy
def _extract_sentums(self, article, **kwargs): ref = [article.get_summary_string()] doc_strings = [] bad_idxs = [] for i in range(len(article)): doc_strings.append(article.get_doc_sent_string(i)) if len(article.doc_sents[i]) < 4: # 4 word sents minimum bad_idxs.append(i) best_sents = [] best_score = -1 combos = list(combinations(range(len(article)), NUM_SENTS_EXTRACT)) bar = ShadyBar('Exhaustive', max=len(combos)) for sent_idxs in combos: # if our sent_idxs to test contains a bad idx, skip it if list(filter(lambda idx: idx in bad_idxs, sent_idxs)): continue # test the Extraction extr = [doc_strings[idx] for idx in sent_idxs] score = get_score([' '.join(extr)], ref, option=self.opt_option) if score > best_score: best_sents = sent_idxs best_score = score bar.next() bar.finish() return list(sorted(best_sents))
def create_chain(self): tweets = self.get_clean_tweets() markov_chain = {} bar = ShadyBar('Populating chain', max=self.get_total_tweets_words(tweets)) for tweet in tweets: for index, word in enumerate(tweet): if word not in markov_chain: markov_chain[word] = {NODES_KEY: {}, IS_INIT_KEY: False} if markov_chain[word][IS_INIT_KEY] is False: markov_chain[word][IS_INIT_KEY] = index == 0 if index != len(tweet) - 1: following_word = tweet[index + 1] if following_word not in markov_chain[word][NODES_KEY]: markov_chain[word][NODES_KEY][following_word] = { N_OCCURR_KEY: 0, W_TYPE_KEY: { N_END_KEY: 0, N_MID_KEY: 0 } } markov_chain[word][NODES_KEY][following_word][ N_OCCURR_KEY] += 1 w_type_n = N_END_KEY if index + 1 != len(tweet) - 1: w_type_n = N_MID_KEY markov_chain[word][NODES_KEY][following_word][W_TYPE_KEY][ w_type_n] += 1 bar.next() bar.finish() return self.calculate_rates(markov_chain)
def remove_rows(color_index): mass = [] zz = [] # xlsFileName clear() print('Обработка файла: %s' % xlsFileName) barr = ShadyBar('Определяю цвета ', max=getNumRows+1) for i in range(7, getNumRows+1): barr.next() this_is = sheet.Cells(i, 2).Font.ColorIndex if this_is != color_index: zz.append(i) if len(zz)>0: c = [zz[0]] mass.append(c) for i in zz[1:]: if i == c[-1] + 1: c.append(i) else: c = [i] mass.append(c) clear() print('Обработка файла: %s' % xlsFileName) bar = ShadyBar('Удаляю строки ', max=len(mass)) for i in reversed(mass): # print('Удаляю строки с %s по %s' % (min(i), max(i))) bar.next() delstr = 'A%s:A%s' % (min(i), max(i)) sheet.Range(delstr).EntireRow.Delete(Shift=color_index)
def get_studio_dict(): studio_dict = { # 'studio': 'lastest id number' } full_page = '' page_bar = ShadyBar('Get entire site info', max=total_page) for page in range(1, total_page+1): url = page_url.format(page=str(page)) req = sess.get(url, headers=header, timeout=timeout) full_page += req.text # sleep(1) page_bar.next() page_bar.finish() soup = BeautifulSoup(full_page, 'html.parser') movie_boxes = soup.find_all(class_='movie-box') for movie in movie_boxes: tmp = movie['href'].split('/')[-1].split('-') if tmp[0] not in studio_dict: studio_dict[tmp[0]] = tmp[1] else: number_in_movie = int(re.search(r'(\d+)', tmp[1]).group(1)) number_in_dict = int(re.search(r'(\d+)', studio_dict[tmp[0]]).group(1)) if number_in_dict < number_in_movie: studio_dict[tmp[0]] = calculate_id(number_in_movie, studio_dict[tmp[0]]) return studio_dict
def load_tweets_from_csv(fname='../data/annotated_tweets.csv', preprocess=True, serialize=True): # Load the data into memory print('Loading MTSA csv...') ids_to_content = defaultdict(lambda: []) with open(fname) as f: csv_reader = DictReader(f) for i, row in enumerate(csv_reader): ids_to_content[row[ID_KEY]].append(row) # construct the tweets and labels bar = ShadyBar('Labelling MTSA', max=len(ids_to_content)) tweets = [] for sample in ids_to_content.values(): bar.next() csv_twt = sample[0] # skip the test questions used for crowdflower! if csv_twt[IS_GOLD_KEY] == 'true': continue # build up the tweet statistics of labels tweet_stats = {s: 0 for s in LABELS} for labelling in sample: if labelling[HAS_SENTIMENT_KEY] == 'no': tweet_stats['obj'] += 1 for key in CSV_LABELS: if labelling[POS_NEG_COM_KEY] == key: tweet_stats[key[0:3]] += 1 # Skipping tweet that had < 5 annotations if sum(tweet_stats.values()) < 5: continue # extract the necessary data tweet = Tweet(csv_twt[TWEET_ID], csv_twt['text'], csv_twt['topic']) tweet.labelling = tweet_stats tweets.append(tweet) bar.finish() """ The preprocessing pipeline is: (see preprocessing.py) (tokenize), (filter_tokens), (remove_mentions), (split_hashtags), (autocorrect), (lemmatize) """ print('Removed {} tweets that had < 5 annotations.'.format(len(ids_to_content) - len(tweets))) print('We now have a total of {} tweets in the MTSA.'.format(len(tweets))) if preprocess: preprocess_tweets(tweets) # save data if desired if serialize: np.save('../data/processed_annotated_tweets.npy', np.array(tweets)) return tweets
def main(): bar = ShadyBar('Preparing update', max=100) for i in range(100): time.sleep(.01) bar.next() bar.finish() subprocess.check_call([sys.executable, "-m", "pip", "install", UPDATE_URL])
def progress_bar(): bar = ShadyBar('Processing Crypto-analysis', max=1000, suffix='%(percent)d%% | %(eta)d seconds remaining |') for i in range(1000): bar.next() time.sleep(.015) bar.finish()
def main(self): print("Welcome to the coin flipping simulator!") # Infinite loop while True: # Catch exception if user enters letters or nothing try: times = input("How many coins do you want to flip?: ") # Check if user entered `exit` or `quit` before converting to int # (So exception wouldn't trigger) if times == "exit" or times == "quit": print("Exiting...") # Exit the loop which quits the program break else: times = int(times) if times <= 0: raise ZeroDivisionError # Create progress bar for counting flips bar = Bar('Flipping {} coin(s)...'.format(times), max=times, suffix='%(index)d/%(max)d - %(percent).1f%% - %(eta)ds') # Define vars for possible options heads, tails = 0, 0 # Loop for the amount of times for x in range(times): # Random flip flip = randint(0, 1) # Check results if flip == 0: heads += 1 else: tails += 1 # Progress bar next bar.next() # Finish the progress bar bar.finish() # Output results table = PrettyTable(["Side", "Amount", "Percent"]) # Define the rows rows = [["Heads", heads, f"%{round((heads / times * 100), 4)}"], ["Tails", tails, f"%{round((tails / times * 100), 4)}"]] # Loop over the rows instead of calling `add_row()` multiple times # Allows for easy expandability for row in rows: table.add_row(row) # Output the Table print(table) # Set the config vars for the table graphics.data = [heads, tails] graphics.labels = ["Heads", "Tails"] graphics.title = "Result of {} flipped coin(s)".format(times) # Display a vertical bar graph print("Generating graph...") graphics.plot_x() except ValueError: print("Please only enter numbers") except ZeroDivisionError: print("You must flip at least 1 coin")
def rescale_images(directory, height, width): bar = ShadyBar('Image Resize Processing', max=len(os.listdir(directory))) for img in os.listdir(directory): size = (height, width) im = Image.open(directory + '/' + img) im_resized = im.resize(size, Image.ANTIALIAS) im_resized.save(directory + '/' + img) time.sleep(0.005) bar.next() bar.finish()
def Pb9(): from progress.bar import ShadyBar import time bar = ShadyBar('进度条9', max=100) #max的值100,可调节 for i in range(100): #这个也需要适当调节 bar.next() time.sleep(0.1) #延迟时间,可调节,0.1~1之间最佳 bar.finish()
def RenameFiles(dir, file_name): os.getcwd() bar = ShadyBar('Rename Processing', max=len(os.listdir(dir))) for i, filename in enumerate(os.listdir(dir)): path = pathlib.Path(dir + "/" + filename) if os.path.isfile(path): os.rename(dir + "/" + filename, dir + "/" + file_name + str(i) + ".jpg") time.sleep(0.005) bar.next() bar.finish()
def load_data(self) -> List[object]: """ Loaded all data with file and return their. If file empty or exist return empty list """ file_manager.check_db_file() bar = ShadyBar('Load data', suffix='%(percent)d%%', max=1) if self._file_path.stat().st_size == 0: return [] with self._file_path.open() as f: bar.next() return ObjectDeserializer().decode_data(constants.json_format, f)
def liste_ver(self, combo_dosya, cikti) -> None: with open(combo_dosya, 'r+') as combo: combo_liste = combo.readlines() bar = ShadyBar('Taranıyor..', max=len(combo_liste)) for user in combo_liste: kullanici, sifre = user.rstrip('\n').split(':') self.giris_yap(kullanici, sifre, cikti) bar.next() bar.finish()
def compute_cost(query_dir, block_size): cost = 0 files = [os.path.join(query_dir, f) for f in os.listdir(query_dir)] args = zip(files, [block_size] * len(files)) cpus = mp.cpu_count() bar = ShadyBar('Scanning Queries (%d cores)' % cpus, max=len(files)) pool = mp.Pool(processes=cpus) for c in pool.imap_unordered(blocks_for_query, args): cost += c bar.next() bar.finish() print('Total blocks accessed: %d' % cost)
def dl_file(url, **kwargs): """ Download the file from the given url Open file object in binary mode since get.content returns a byte object. Written file will have CRLF line endings. """ # using carriage return to escape incompatible ANSI chars bar = ShadyBar('\rDownloading', max=10, suffix='%(percent)d%%') try: response = get(url) # byte-like object # Added progress intervals for visual consistency. for i in range(10): sleep(0.05) bar.next() sys.stdout.flush() bar.finish() sys.stdout.flush() # errors related to internet connectivity except(NewConnectionError, MaxRetryError, ConnectionError): error_msg = "Failed to send request to URL." + \ " Check your internet connection and try again." print(colored('\rFatal:', 'red'), error_msg, file=sys.stderr) raise SystemExit(1) # error related to HTTP responses if response.status_code != 200: print(colored("\rFatal:", 'red'), "HTTP Error ({})." .format(response.status_code), file=sys.stderr) print("Try setting up your repo manually or contact repo admin.", file=sys.stderr) raise SystemExit(1) # Write response to file if kwargs['file'] == 'pre-commit': file_name = os.path.join(get_hook_dir(), "pre-commit") # in odrer to write 'Done' inline sys.stdout.write('\nWriting response to file ... ') sys.stdout.flush() with open(file_name, "wb") as file: file.write(response.content) elif kwargs['file'] == '.editorconfig': file_name = os.path.join(os.getcwd(), ".editorconfig") # in odrer to write 'Done' inline sys.stdout.write('\nWriting response to file ... ') sys.stdout.flush() with open(file_name, "wb") as file: file.write(response.content) # delays for visual consistency sleep(0.3) sys.stdout.write(colored("Done", 'green')) sys.stdout.flush() print('\n')
def __iter__(self): progress_bar = ShadyBar( "Solving Phase II", max=self.timeslots / 10, width=60, suffix="%(percent).1f%% - ETA: %(eta_td)s", ) count = 0 for k in OmniscientSTWPredictor.__iter__(self): yield k count = (count + 1) % 10 if count == 0: progress_bar.next() progress_bar.finish()
def scanne_data_dir(src_folder,save_folder,N,R): file_list=os.listdir(src_folder) file_num=len(file_list) bar = ShadyBar('Processing', max=file_num) for root,dirs,files in os.walk(src_folder): for file in files: bar.next() name_str=file.split('.') if name_str[1]=='JPG' or name_str[1]=='jpg' or name_str[1]=='png' or name_str[1]=='PNG' or name_str[1]=='bmp': path=os.path.join(root,file) img=getImg(path) code,result_img=CCT_extract1(img,N,R) cv2.imwrite(save_folder+file,img) bar.finish()
def prim(self, coords, distance_matrix, start_node=0): """Algorithm of Prim to calculate minimum spanning tree Takes coords as a list of tuplse in the form of '(x,y)' Returns a list of edges of the MST in the form of tuples "(node1, node2)" """ # initialize the priority queue as a list of dicts for each coordinate with # the vertex with its index and x- and y-coordinate # a dist value which saves the minimum distance connecting the vertex to the MST # and the parent vertex which is the closest vertex already in the MST # Format: [[index, distance, parent, (x-coord, y-coord)],...] priority_queue = [[v, sys.maxsize, math.inf, coords[v]] for v in range(len(coords))] # initialize a MST list which holds the vertices in the order they are visited # initialize a edges list which holds the edges of the MST mst = [] edges = [] bar = ShadyBar('Constructing MST', max=len(coords)) # initialize distance of first vertex as 0 priority_queue[0][1] = 0 # remove first vertex of priority queue and add to MST mst.append(priority_queue.pop(0)) while priority_queue: for v in priority_queue: time_v = -time.process_time() for u in mst: # if the current closest distance of v is bigger than the distance between u and v if v[1] > distance_matrix[v[0]][u[0]]: # update distance of v to the distance of u and v v[1] = distance_matrix[v[0]][u[0]] # and add u as the parent of v for the closest distance v[2] = u[0] time_v += time.process_time() # get the vertex that has least dist in the priority queue new_vertex = min(priority_queue, key=lambda x: x[1]) # removes new vertex from priority queue and adds to MST mst.append(priority_queue.pop(priority_queue.index(new_vertex))) # appends the edge connecting the new vertex and its parent vertex to the edges list edges.append([(new_vertex[0], new_vertex[2]), distance_matrix[new_vertex[0]][new_vertex[2]]]) bar.next() bar.finish() return mst, edges
def aggregate(self): # Do the same aggregation over the reducer output region_size = defaultdict(int) bar = ShadyBar('Aggregating', max=self.num_map_outputs, suffix='%(index)d/%(max)d - %(elapsed)ds') for ix in range(self.num_map_outputs): region_size_tmp = self.read_reduce_result( self.get_reduce_output_filename(ix)) for qs, c in region_size_tmp.items(): region_size[qs] += c bar.next() bar.finish() self.write_reduce_result(self.results_file, region_size) return region_size
def liste_ver(self, combo_dosya, cikti) -> None: if not cikti: cikti = "KekikAkademi" with open(combo_dosya, 'r+') as combo: combo_liste = combo.readlines() bar = ShadyBar('Taranıyor..', max=len(combo_liste)) for user in combo_liste: ayristir = user.rstrip('\n').split(':') self.giris_yap(ayristir[0], ayristir[1], cikti) bar.next() bar.finish()
def PrepareImage(dir, file_name=None): print(text) bar = ShadyBar('Image Processing', max=len(os.listdir(dir))) for i, file in enumerate(os.listdir(dir)): path = pathlib.Path(dir + "/" + file) if os.path.isfile(path): if (path.suffix != '.jpg' and path.suffix != '.jpeg'): path.unlink() time.sleep(0.005) bar.next() bar.finish() time.sleep(2) if file_name is None: file_name = time.strftime("%Y-%m-%d-%H-%M-%S") RenameFiles(dir, file_name) else: RenameFiles(dir, file_name)
def opt2(self, improvement_threshold=0.05): """Run 2-opt algorithm to improve the initial solution improvement_threshold: specifies a threshold of improvement for the opt-2 algorithm to run again e.g. improvement_threshold=0.05 stop running 2-opt on the improved solution if the improvement of the solution value of the previous tour is less than 5% Replacing of two edges with two new edges is achieved by using the tour as a sequence of vertices and reversing a subsequence within the tour sequence (i.e. 0 1 2 3 4 5 0 -> 0 1 4 3 2 5 0: removes (1,2), (2,3)) """ # initial best value and tour (shift tour -1) tour = list(map(lambda x: x - 1, self.get_tour())) best_value = self.get_val() best_tour = tour improvement_factor = 1 previous_best_value = best_value previous_best_tour = best_tour # animation of loading bar, length is only rough estimate bar = ShadyBar('Running 2-opt', max=len(tour)**2) # only run 2-opt if improvement is higher than specified improvement threshold while improvement_factor > improvement_threshold: for first_index in range(1, len(best_tour) - 2): for last_index in range(first_index + 1, len(best_tour) - 1): # replace two existing edges with two new edges (more in docstring of this method) new_tour = previous_best_tour[:first_index] + previous_best_tour[ last_index - 1:first_index - 1:-1] + previous_best_tour[last_index:] new_value = self.calculate_tour_distance(new_tour) # update the previous best value and tour if the new one is better if previous_best_value > new_value: previous_best_value = new_value previous_best_tour = new_tour bar.next() # animation of loading bar improvement_factor = 1 - (previous_best_value / best_value) best_value = previous_best_value best_tour = previous_best_tour self.set_tour(best_tour) self.set_val(best_value, improved=True) self.set_path(best_tour, construct_edges=True) bar.finish() # animation of loading bar return best_value, best_tour
def scaler(env, sample_size=500): observations = [] bar = ShadyBar('Learning Scaler', max=sample_size) for e in range(sample_size): bar.next() state = env.reset() observations.append(state) done = False while not done: state, _, done, _ = env.step(env.action_space.sample()) observations.append(state) scaler = StandardScaler() scaler.fit(observations) bar.finish() return scaler
class ProgressBar: def __init__(self): self.current = 0 self.max = 100 self.bar = None def start(self): self.bar = ShadyBar('Downloading:', max=self.max) def set(self, percentage): percentage = int(percentage) if self.current != percentage: self.bar.next() self.current = percentage def finish(self): self.bar.finish() print('')
def calculate_rates(self, markov_chain): bar = ShadyBar('Calculating chain rates', max=self.get_total_chain_nodes(markov_chain)) for word in markov_chain: total_occurr = self.get_total_occurrences(markov_chain[word]) for node in markov_chain[word][NODES_KEY]: n_occurr = markov_chain[word][NODES_KEY][node][N_OCCURR_KEY] n_mid = markov_chain[word][NODES_KEY][node][W_TYPE_KEY][ N_MID_KEY] n_end = markov_chain[word][NODES_KEY][node][W_TYPE_KEY][ N_END_KEY] markov_chain[word][NODES_KEY][node][R_OCCURR_KEY] = round( n_occurr / total_occurr, N_DIGITS_ROUND) markov_chain[word][NODES_KEY][node][W_TYPE_KEY][ R_MID_KEY] = round(n_mid / n_occurr, N_DIGITS_ROUND) markov_chain[word][NODES_KEY][node][W_TYPE_KEY][ R_END_KEY] = round(n_end / n_occurr, N_DIGITS_ROUND) bar.next() bar.finish() return markov_chain