def test_track() -> None: _time = 1.0 def get_time(): nonlocal _time try: return _time finally: _time += 1 console = Console(file=io.StringIO(), force_terminal=True, width=40, color_system="truecolor") test = ["foo", "bar", "baz"] expected_values = iter(test) for value in track(test, "test", console=console, auto_refresh=False, get_time=get_time): assert value == next(expected_values) result = console.file.getvalue() print(repr(result)) expected = "test \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 0%\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[?25l\r\x1b[2Ktest \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 0%\x1b[0m \x1b[36m-:--:--\x1b[0m \r\x1b[2Ktest \x1b[38;2;249;38;114m━━━━━━━\x1b[0m\x1b[38;5;237m╺\x1b[0m\x1b[38;5;237m━━━━━━━━━━━━━\x1b[0m \x1b[35m 33%\x1b[0m \x1b[36m-:--:--\x1b[0m \r\x1b[2Ktest \x1b[38;2;249;38;114m━━━━━━━━━━━━━━\x1b[0m\x1b[38;5;237m╺\x1b[0m\x1b[38;5;237m━━━━━━\x1b[0m \x1b[35m 67%\x1b[0m \x1b[36m0:00:06\x1b[0m \r\x1b[2Ktest \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m \r\x1b[2Ktest \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m \n\x1b[?25h" assert result == expected with pytest.raises(ValueError): for n in track(5): pass
def test_track() -> None: console = Console( file=io.StringIO(), force_terminal=True, width=60, color_system="truecolor", legacy_windows=False, ) test = ["foo", "bar", "baz"] expected_values = iter(test) for value in track(test, "test", console=console, auto_refresh=False, get_time=MockClock(auto=True)): assert value == next(expected_values) result = console.file.getvalue() print(repr(result)) expected = "\x1b[?25ltest \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 0%\x1b[0m \x1b[36m-:--:--\x1b[0m\r\x1b[2Ktest \x1b[38;2;249;38;114m━━━━━━━━━━━━━\x1b[0m\x1b[38;5;237m╺\x1b[0m\x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 33%\x1b[0m \x1b[36m-:--:--\x1b[0m\r\x1b[2Ktest \x1b[38;2;249;38;114m━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m\x1b[38;2;249;38;114m╸\x1b[0m\x1b[38;5;237m━━━━━━━━━━━━━\x1b[0m \x1b[35m 67%\x1b[0m \x1b[36m0:00:06\x1b[0m\r\x1b[2Ktest \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m\r\x1b[2Ktest \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m\n\x1b[?25h" assert result == expected with pytest.raises(ValueError): for n in track(5): pass
def save( self, output="result", ): output_dir = Path(output) fieldnames = list(self.courses[0].keys()) with open(output_dir.joinpath("grade.csv"), "w", newline="") as f: writer = csv.DictWriter(f, fieldnames) writer.writeheader() for grade in track(self.courses, description="Fetching grades..."): writer.writerow( {key: val["display"] for key, val in grade.items()}) with open(output_dir.joinpath("course.json"), "w", encoding="utf8") as f: for course in track(self.origin_data, description="Fetching courses data..."): res = self.courses_analysis(course=course["JXBID"], semester=course["XNXQDM"], stu=course["XH"]) added_name = { "class_name": course["KCM"], **res, } f.write(json.dumps(added_name, ensure_ascii=False) + "\n")
def polygonize(load_dict, id_index=''): if id_index: for intersection in track(load_dict['intersections']): point = intersection['point'] ix = point['x'] iy = point['y'] dis = 0 for road in intersection['roads']: _ = load_dict['roads'][id_index[road]] mx = _['midpoint']['x'] my = _['midpoint']['y'] length = math.sqrt((ix - mx)**2 + ((iy - my)**2)) dis = dis + length intersection['distanceToMidpoints'] = dis else: for intersection in track(load_dict['intersections']): point = intersection['point'] ix = point['x'] iy = point['y'] dis = 0 for road in intersection['roads']: for _ in load_dict['roads']: if _['id'] == road: mx = _['midpoint']['x'] my = _['midpoint']['y'] length = math.sqrt((ix - mx)**2 + ((iy - my)**2)) dis = dis + length intersection['distanceToMidpoints'] = dis
def solve_it(input_data): lines = input_data.split('\n') parts = lines[0].split() customer_count = int(parts[0]) vehicle_count = int(parts[1]) vehicle_capacity = int(parts[2]) console.print("Reading data: ....", style = "bold red") customers = [] for i in track(range(1, customer_count+1)): line = lines[i] parts = line.split() customers.append(Customer(i-1, int(parts[0]), Point(float(parts[1]), float(parts[2])))) vehicles = [] for i in track(range(vehicle_count)): vehicle = Vehicle() vehicle.index = i vehicle.capacity = sys.maxsize vehicle.available = vehicle_capacity vehicles.append(vehicle) app = GuildLocalSearch(customers, vehicles) # app.search(max_step=10000, max_repeat= 100) app.search() best_cost = app.best_cost best_vehicles = app.best_vehicles output = str(best_cost) + " 0\n" for vehicle in best_vehicles: output += " ".join(str(node.index) for node in vehicle.tour) + "\n" return output
def features_extractor(adversarial_vector: str, feature_mapping: str): info("\t[*] Loading feature vector mapping from pickle file ...") feature_vector_mapping = pickle.load(open(feature_mapping, "rb")) info("\t[*] Loading adversarially generated feature vectors from pickle file ...") adversarial_feature_vector = pickle.load(open(adversarial_vector, "rb")) feature_vector_mapping = [import_lib for import_lib in feature_vector_mapping] debug(f"\t[+] adversarial_feature_vector length : {len(adversarial_feature_vector)}") adversarial_imports_set = [] adversarial_sections_set = [] count = 0 info("\t[*] Generating imports and section set from adversarially generated feature vectors ...") for index in track(range(len(adversarial_feature_vector)), description=" Mapping imports and sections ...", transient=True): info( f"\t\t[+] Mapping imports and sections from batch {index} with {len(adversarial_feature_vector[index])} adversarial feature vectors ...", ) for i in track( range(len(adversarial_feature_vector[index])), description=" Extracting ...", transient=True ): sample = adversarial_feature_vector[index][i] sample = sample.tolist() adversial_imports = [] adversial_section = [] adversial_features = [] debug(f"\t\t[+] Sample lenght : {len(sample)}") debug(f"\t\t[+] Adv Feature Vector : {(sample)}") for i in range(len(sample)): if sample[i] > 0: adversial_features.append(feature_vector_mapping[i]) debug(f"\t\t\t[+] feature : [bold green]{feature_vector_mapping[i]}", extra={"markup":True}) for feature in adversial_features: if ":" in feature: adversial_imports.append(feature) else: adversial_section.append(feature) pass adversarial_imports_set.append(adversial_imports) adversarial_sections_set.append(adversial_section) debug( f"Feature mapping for adversarial feature vector [{count}] completed with {len(adversial_imports)} imports and {len(adversial_section)} sections ...", ) # debug("Import mapping for adversarial feature vector [" + str(count) + "] completed with " + str(len(adversial_imports)) + " imports ...\n") count = count + 1 info("\t[+] Adversarial Imports and sections have been extracted ...") return adversarial_imports_set, adversarial_sections_set
def build_vocabulary(max_doc, min_df, min_freq, word_tokenizer, w2v, w2v_num_epoch, w2v_skip_gram, w2v_size): """Build vocabulary & optionally word embeddings""" total = sum(1 for _ in sentence_iter(word_tokenizer, max_doc)) counter_cs = VocabularyCounter(word_tokenizer, case_sensitive=True, min_tf=min_freq, min_df=min_df) counter = VocabularyCounter(word_tokenizer, case_sensitive=False, min_tf=min_freq, min_df=min_df) if w2v: model = get_w2v_model(w2v_size, min_freq, w2v_skip_gram) console.log("Building vocabulary...") model.build_vocab(sentence_iter(word_tokenizer, max_doc)) for _ in track(range(w2v_num_epoch), total=w2v_num_epoch, description="Building word vectors..."): model.train(sentences=sentence_iter(word_tokenizer, max_doc), epochs=1, total_examples=total, report_delay=1) click.secho( click.style(f"...Frequency calculation over extended dataset", fg="blue")) with tokenizer_context(word_tokenizer) as Doc: if max_doc is None: corpus = load_extended_sents_corpus() else: corpus = islice(load_extended_sents_corpus(), max_doc) corpus, X = tee(corpus, 2) for i, d in track(enumerate(corpus), total=sum(1 for _ in X), description="Building vocabulary..."): doc = Doc.from_sentences(d['sentences']) for sent in doc: for word in sent.tokens: counter.inc(word, i, 1) counter_cs.inc(word, i, 1) counter = counter.prune() counter_cs = counter_cs.prune() if w2v: counter.to_hdf5(model) else: counter.to_hdf5() counter_cs.to_hdf5()
def ps(ip, output, data, ps_mode): threads = [] result = {} try: print('\n\n' + Y + '[!]' + Y + ' Starting Port Scan...' + W + '\n') if ps_mode == 'fast': print(G + '[+]' + C + ' Testing Top 1000 Ports...' + W + '\n') port_list = [1,3,4,6,7,9,13,17,19,20,21,22,23,24,25,26,30,32,33,37,42,43,49,53,70,79,80,81,82,83,84,85,88,89,90,99,100,106,109,110,111,113,119,125,135,139,143,144,146,161,163,179,199,211,212,222,254,255,256,259,264,280,301,306,311,340,366,389,406,407,416,417,425,427,443,444,445,458,464,465,481,497,500,512,513,514,515,524,541,543,544,545,548,554,555,563,587,593,616,617,625,631,636,646,648,666,667,668,683,687,691,700,705,711,714,720,722,726,749,765,777,783,787,800,801,808,843,873,880,888,898,900,901,902,903,911,912,981,987,990,992,993,995,999,1000,1001,1002,1007,1009,1010,1011,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065,1066,1067,1068,1069,1070,1071,1072,1073,1074,1075,1076,1077,1078,1079,1080,1081,1082,1083,1084,1085,1086,1087,1088,1089,1090,1091,1092,1093,1094,1095,1096,1097,1098,1099,1100,1102,1104,1105,1106,1107,1108,1110,1111,1112,1113,1114,1117,1119,1121,1122,1123,1124,1126,1130,1131,1132,1137,1138,1141,1145,1147,1148,1149,1151,1152,1154,1163,1164,1165,1166,1169,1174,1175,1183,1185,1186,1187,1192,1198,1199,1201,1213,1216,1217,1218,1233,1234,1236,1244,1247,1248,1259,1271,1272,1277,1287,1296,1300,1301,1309,1310,1311,1322,1328,1334,1352,1417,1433,1434,1443,1455,1461,1494,1500,1501,1503,1521,1524,1533,1556,1580,1583,1594,1600,1641,1658,1666,1687,1688,1700,1717,1718,1719,1720,1721,1723,1755,1761,1782,1783,1801,1805,1812,1839,1840,1862,1863,1864,1875,1900,1914,1935,1947,1971,1972,1974,1984,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2013,2020,2021,2022,2030,2033,2034,2035,2038,2040,2041,2042,2043,2045,2046,2047,2048,2049,2065,2068,2099,2100,2103,2105,2106,2107,2111,2119,2121,2126,2135,2144,2160,2161,2170,2179,2190,2191,2196,2200,2222,2251,2260,2288,2301,2323,2366,2381,2382,2383,2393,2394,2399,2401,2492,2500,2522,2525,2557,2601,2602,2604,2605,2607,2608,2638,2701,2702,2710,2717,2718,2725,2800,2809,2811,2869,2875,2909,2910,2920,2967,2968,2998,3000,3001,3003,3005,3006,3007,3011,3013,3017,3030,3031,3052,3071,3077,3128,3168,3211,3221,3260,3261,3268,3269,3283,3300,3301,3306,3322,3323,3324,3325,3333,3351,3367,3369,3370,3371,3372,3389,3390,3404,3476,3493,3517,3527,3546,3551,3580,3659,3689,3690,3703,3737,3766,3784,3800,3801,3809,3814,3826,3827,3828,3851,3869,3871,3878,3880,3889,3905,3914,3918,3920,3945,3971,3986,3995,3998,4000,4001,4002,4003,4004,4005,4006,4045,4111,4125,4126,4129,4224,4242,4279,4321,4343,4443,4444,4445,4446,4449,4550,4567,4662,4848,4899,4900,4998,5000,5001,5002,5003,5004,5009,5030,5033,5050,5051,5054,5060,5061,5080,5087,5100,5101,5102,5120,5190,5200,5214,5221,5222,5225,5226,5269,5280,5298,5357,5405,5414,5431,5432,5440,5500,5510,5544,5550,5555,5560,5566,5631,5633,5666,5678,5679,5718,5730,5800,5801,5802,5810,5811,5815,5822,5825,5850,5859,5862,5877,5900,5901,5902,5903,5904,5906,5907,5910,5911,5915,5922,5925,5950,5952,5959,5960,5961,5962,5963,5987,5988,5989,5998,5999,6000,6001,6002,6003,6004,6005,6006,6007,6009,6025,6059,6100,6101,6106,6112,6123,6129,6156,6346,6389,6502,6510,6543,6547,6565,6566,6567,6580,6646,6666,6667,6668,6669,6689,6692,6699,6779,6788,6789,6792,6839,6881,6901,6969,7000,7001,7002,7004,7007,7019,7025,7070,7100,7103,7106,7200,7201,7402,7435,7443,7496,7512,7625,7627,7676,7741,7777,7778,7800,7911,7920,7921,7937,7938,7999,8000,8001,8002,8007,8008,8009,8010,8011,8021,8022,8031,8042,8045,8080,8081,8082,8083,8084,8085,8086,8087,8088,8089,8090,8093,8099,8100,8180,8181,8192,8193,8194,8200,8222,8254,8290,8291,8292,8300,8333,8383,8400,8402,8443,8500,8600,8649,8651,8652,8654,8701,8800,8873,8888,8899,8994,9000,9001,9002,9003,9009,9010,9011,9040,9050,9071,9080,9081,9090,9091,9099,9100,9101,9102,9103,9110,9111,9200,9207,9220,9290,9415,9418,9485,9500,9502,9503,9535,9575,9593,9594,9595,9618,9666,9876,9877,9878,9898,9900,9917,9929,9943,9944,9968,9998,9999,10000,10001,10002,10003,10004,10009,10010,10012,10024,10025,10082,10180,10215,10243,10566,10616,10617,10621,10626,10628,10629,10778,11110,11111,11967,12000,12174,12265,12345,13456,13722,13782,13783,14000,14238,14441,14442,15000,15002,15003,15004,15660,15742,16000,16001,16012,16016,16018,16080,16113,16992,16993,17877,17988,18040,18101,18988,19101,19283,19315,19350,19780,19801,19842,20000,20005,20031,20221,20222,20828,21571,22939,23502,24444,24800,25734,25735,26214,27000,27352,27353,27355,27356,27715,28201,30000,30718,30951,31038,31337,32768,32769,32770,32771,32772,32773,32774,32775,32776,32777,32778,32779,32780,32781,32782,32783,32784,32785,33354,33899,34571,34572,34573,35500,38292,40193,40911,41511,42510,44176,44442,44443,44501,45100,48080,49152,49153,49154,49155,49156,49157,49158,49159,49160,49161,49163,49165,49167,49175,49176,49400,49999,50000,50001,50002,50003,50006,50300,50389,50500,50636,50800,51103,51493,52673,52822,52848,52869,54045,54328,55055,55056,55555,55600,56737,56738,57294,57797,58080,60020,60443,61532,61900,62078,63331,64623,64680,65000,65129,65389] for port in track(port_list): t = threading.Thread(target=sock_conn, args=[ip, port, output, result, 3]) t.daemon = True t.start() threads.append(t) time.sleep(0.001) for thread in threads: thread.join() elif ps_mode == 'full': set_open_file_cmd = 'ulimit -Sn 66000' print(G + '[+]' + C + ' Testing All Ports...' + W + '\n') print(Y + '[!]' + C + ' Please make sure proper open files value is set for full scan mode\n' + W) # 65535 os.system(set_open_file_cmd) try: for port in track(range(0, 65535)): t = threading.Thread(target=sock_conn, args=[ip, port, output, result, 3]) t.daemon = True t.start() time.sleep(0.0005) for thread in threads: thread.join() except Exception: pass else: print(R + "\n[-]" + C + " \'" + ps_mode + "\' port scan mode is not supported please recheck\n" + W) return except Exception as e: print('\n' + R + '[-]' + C + ' Exception : ' + W + str(e) + '\n') if output != 'None': result.update({'Exception':str(e)}) if output != 'None': ps_output(output, data, result) print()
def _fetch_data(self, odds_input): # Main train data params = pd.merge(pd.DataFrame(self.main_leagues_param_grid_), pd.DataFrame(self.param_grid_)).to_records(False) data_train_main = [] for _, division, league, url, year in track(params, description='Football-Data.co.uk | Download training data of main leagues:'): url = urljoin(URL, url) names = pd.read_csv(url, nrows=0).columns try: data = pd.read_csv(url, names=names, skiprows=1) except (UnicodeDecodeError, pd.errors.ParserError): data = pd.read_csv(url, names=names, skiprows=1, encoding='ISO-8859-1') # Extract columns data.replace('#REF!', np.nan, inplace=True) data.drop(columns=[col for col in data.columns if 'Unnamed' in col], inplace=True) data = data.assign(league=league, division=division, year=year, test=False) data.rename(columns=self._cols_mapping, inplace=True) # Append data to main data data_train_main.append(data) data_train_main = pd.concat(data_train_main, ignore_index=True) if data_train_main else pd.DataFrame() # Extra train data params = pd.merge(pd.DataFrame(self.extra_leagues_param_grid_), pd.DataFrame(self.param_grid_)).drop(columns='year').drop_duplicates().to_records(False) data_train_extra = [] for division, league, url in track(params, description='Football-Data.co.uk | Download training data of extra leagues:'): data = pd.read_csv(urljoin(URL, url)) data = data.assign(league=league, division=division, year=lambda s: s.Season if type(s.Season) is not str else s.Season.split('/')[-1], test=False) data.rename(columns=self._cols_mapping, inplace=True) data_train_extra.append(data) data_train_extra = pd.concat(data_train_extra, ignore_index=True) if data_train_extra else pd.DataFrame() # Test data params = pd.DataFrame(self.main_leagues_param_grid_).drop(columns='url').groupby(['league', 'division'], as_index=False).max() data_test = pd.read_csv(join(URL, 'fixtures.csv')) data_test.drop(columns=[col for col in data_test.columns if 'Unnamed' in col], inplace=True) data_test = pd.merge(data_test, params[['Div', 'league', 'division', 'year']].dropna().drop_duplicates(), how='left') data_test['test'] = True data_test.rename(columns=self._cols_mapping, inplace=True) # Combine data self.data_ = pd.concat([data_train_main, data_train_extra, data_test], ignore_index=True) # Select odds types cols_odds = [] for col in self.data_.columns: for odds_type in odds_input: if col.startswith(odds_type) or col.startswith(f'{self.odds_type_}_'): cols_odds.append(col) return self
def test_track() -> None: console = Console(file=io.StringIO(), force_terminal=True, width=40) test = ["foo", "bar", "baz"] expected_values = iter(test) for value in track(test, "test", console=console, auto_refresh=False): assert value == next(expected_values) result = console.file.getvalue() expected = "test \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 0%\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[?25l\r\x1b[2Ktest \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 0%\x1b[0m \x1b[36m-:--:--\x1b[0m \r\x1b[2Ktest \x1b[38;5;13m━━━━━━━\x1b[0m\x1b[38;5;237m╺\x1b[0m\x1b[38;5;237m━━━━━━━━━━━━━\x1b[0m \x1b[35m 33%\x1b[0m \x1b[36m-:--:--\x1b[0m \r\x1b[2Ktest \x1b[38;5;13m━━━━━━━━━━━━━━\x1b[0m\x1b[38;5;237m╺\x1b[0m\x1b[38;5;237m━━━━━━\x1b[0m \x1b[35m 67%\x1b[0m \x1b[36m0:00:01\x1b[0m \r\x1b[2Ktest \x1b[38;5;10m━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m \r\x1b[2Ktest \x1b[38;5;10m━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m \n\x1b[?25h" assert result == expected with pytest.raises(ValueError): for n in track(5): pass
def import_extractor(adversarial_vector: str, feature_mapping: str): debug(f"[+] feature mapping type : {(feature_mapping)}") info("[*] Loading import feature vector mapping from pickle file ...") feature_vector_mapping = pickle.load(open(str(feature_mapping), "rb")) info("[*] Loading adversarially generated import feature vectors from pickle file ...") adversarial_feature_vector = pickle.load(open(adversarial_vector, "rb")) feature_vector_mapping = [import_lib for import_lib in feature_vector_mapping] debug(f"[+] adversarial_feature_vector length : {len(adversarial_feature_vector)}") adversarial_imports_set = [] count = 0 info("[*] Generating imports set from adversarially generated feature vectors ...") for index in track(range(len(adversarial_feature_vector)), description="Mapping imports ...", transient=True): info( f"[+] Mapping imports from batch {index} with {len(adversarial_feature_vector[index])} adversarial feature vectors ..." ) for i in track( range(len(adversarial_feature_vector[index])), description=" Extracting ... ", transient=True ): sample = adversarial_feature_vector[index][i] sample = sample.tolist() adversial_imports = [] unfiltered_adversial_imports = [] debug("[+] Sample lenght : {len(sample)}") for i in range(len(sample)): if sample[i] > 0: unfiltered_adversial_imports.append(feature_vector_mapping[i]) for imports in unfiltered_adversial_imports: if "32" in imports: adversial_imports.append(imports) debug("\t[+] Filtered Imports : " + str(imports)) adversarial_imports_set.append(adversial_imports) debug( f"Import mapping for adversarial feature vector [{count}] completed with {len(adversial_imports)} imports ...\n" ) count = count + 1 # info("%d adversarial feature vectors have been mapped ...", count) debug(f"[+] Number of feature vectors : {len(adversarial_imports_set)}") return adversarial_imports_set
def generate_dataset(path, parts = 5, max_length = 100): dataset = [] with open(path, 'r') as data: lines = data.readlines() for l in track(lines, description = "processing..."): strokes = [] d = json.loads(l) drawing = d['drawing'] x, y = drawing[0][0][0], drawing[0][1][0] for stroke in drawing: xs, ys = stroke # add the beginning pen up movement to the start of the next stroke strokes.append([ xs[0] - x, ys[0] - y, 0 ]) x, y = xs[0], ys[0] for t in range (1, len(xs)): strokes.append([ xs[t] - x, ys[t] - y, 1 ]) x, y = xs[t], ys[t] strokes.append([0, 0, 2]) # end drawing del strokes[0] if len(strokes) < max_length: dataset.append(strokes) # go through dataset and pad for d in track(range(len(dataset)), description = 'padding... '): if len(dataset[d]) < max_length: dataset[d] = dataset[d] + [[0, 0, 2]] * (max_length - len(dataset[d])) # save each section of the dataset split = len(dataset) // parts print("Splitting into %s sections of %s examples each" % (parts, split)) for p in track(range (parts), description = 'splitting... '): section = dataset[ : split] del dataset[: split] ds = np.array(section) np.save('data/cloud_p%s.npy' % p, ds)
def to_adjacency_list(intersections, roads, road_id_index, intersection_id_index): adjacenccy_list = [] for intersection in track(intersections): cur_list = [] for road in intersection['roads']: _ = roads[road_id_index[road]] # if _['startIntersection'] is not intersection['id']: # cur_list.append( # intersection_id_index[_['startIntersection']]) # elif _['endIntersection'] is not intersection['id']: # cur_list.append( # intersection_id_index[_['endIntersection']]) # else: # raise Exception( # 'start- and endIntersection both the same as ', intersection['id']) if _['endIntersection'] == intersection['id']: pass else: cur_list.append(intersection_id_index[_['endIntersection']]) # if len(cur_list) == 8: # adjacenccy_list.append(np.array(cur_list[:4])) # if len(cur_list) == 2: # adjacenccy_list.append(np.array(cur_list[:1])) adjacenccy_list.append(np.array(cur_list)) return adjacenccy_list
def _generate_paginated_index(write_html, url_prefix, items, items_per_page, description): pages = list(_paginate(items, items_per_page)) def get_page_name(start_index, end_index): return (f"{url_prefix}/index.html" if start_index == 1 else f"{url_prefix}/index-{start_index}-{end_index}.html") for (prev_page, ((start_index, end_index), page_items), next_page) in track( zip( [None] + pages, pages, pages[1:] + [None], ), description=description, total=len(pages), ): write_html( page_items, template_file=f"{url_prefix}/index.html", output_file=get_page_name(start_index, end_index), start_index=start_index, end_index=end_index, prev_page=get_page_name(*prev_page[0]) if prev_page else None, next_page=get_page_name(*next_page[0]) if next_page else None, )
def generate_mutated_malware(file, model, args): pe = pefeatures2.PEFeatureExtractor2() rn = RangeNormalize(-0.5,0.5) info("[*] Reading file : " + str(file)) bytez = [] with open(str(file), 'rb') as infile: bytez = infile.read() for t in track(range(1, args.rl_mutations) , description="Generating mutation ...", transient=True): state = pe.extract( bytez ) state_norm = rn(state) state_norm = torch.from_numpy(state_norm).float().unsqueeze(0).to(device) actions = model.forward(state_norm) action = torch.argmax(actions).item() action = ACTION_LOOKUP[action] bytez = bytes(manipulate.modify_without_breaking(bytez, [action])) new_score = interface.get_score_local( bytez ) if(new_score < interface.local_model_threshold): break if not os.path.exists(args.o): os.mkdir(args.o) info("[*] output directory has been created at : " + str(args.o)) output_file = os.path.join(args.o, "mutated_" + str(os.path.basename(file))) info("[*] Writing mutated file to : " + str(output_file) + "\n\n") with open(str(output_file), mode='wb') as file1: file1.write(bytes(bytez)) return
def daily_task(): """Main workhorse function. Support functions defined below""" global CATEGORIES_PAGES global BROWSER global DATE global OBSERVATION log.info('Scraper started') # Refresh date DATE = str(datetime.date.today()) OBSERVATION = 0 # Initiate headless web browser log.info('Initializing browser') BROWSER = webdriver.Chrome(executable_path=CHROME_DRIVER, options=OPTIONS) # Download topsite and get categories directories base_file_name = "All_cat_" + DATE + ".html" fetch_html(BASE_URL, base_file_name, PATH_HTML, attempts_limit=1000) html_file = open(PATH_HTML + base_file_name).read() CATEGORIES_PAGES = get_category_list(html_file) log.info('Found ' + str(len(CATEGORIES_PAGES)) + ' categories') # Read each categories pages and scrape for data for cat in track(CATEGORIES_PAGES, description="[green]Scraping...", total=len(CATEGORIES_PAGES)): cat_file = "cat_" + cat['name'] + "_" + DATE + ".html" download = fetch_html(cat['directlink'], cat_file, PATH_HTML) if download: scrap_data(cat) # Close browser BROWSER.close() BROWSER.service.process.send_signal(signal.SIGTERM) BROWSER.quit()
def gen_multi(function, args, noise=False): with Pool() as p: results = [p.apply_async(function, args=(a, noise)) for a in args] dataset = [ p.get() for p in track(results, description='Generating sounds ...') ]
def website(full_table): # Enters into the info page and parses out the info for f in track(full_table, description="Parsing..."): source = requests.get(f['News Media Info']) soup = BeautifulSoup(source.content, 'lxml') try: # getting the website link to news source locate_html_class = soup.find('div', {'class': 'dynamic-grid'}) locate_paragraph = locate_html_class.find('a')['href'] f['News Source Site'] = locate_paragraph except TypeError: pass try: # getting the creation date of the news source locate__html_class = soup.find('div', {'class': 'dynamic-grid'}) locate_paragraph = locate__html_class.find_all('p')[1].text.split( '.')[-1].strip() f['Established'] = locate_paragraph except IndexError: pass try: # Who the news source owned by locate__html_class = soup.find('div', {'class': 'dynamic-grid'}) locate_paragraph = locate__html_class.find_all('p')[2].text.split( ':')[-1].strip() f['Owned by'] = locate_paragraph except IndexError: pass sleep(10) return full_table
def sentences(dataset_dir: str, tokenizer: str, force: bool): dataset = os.path.dirname(__file__) if dataset_dir is None else dataset_dir raws = glob.glob(pjoin(dataset, 'raw', '*.txt')) sents_dir = pjoin(dataset, 'sents') os.makedirs(sents_dir, exist_ok=True) toker = get_tokenizer(tokenizer) n = 0 logger.info("|raw documents|: {}".format(len(raws))) logger.info("Sentence tokenizer: {}".format(tokenizer)) for file in track(raws, description="Building sentence corpus..."): basename = os.path.basename(file) name, _ = os.path.splitext(basename) sentences_json_file = pjoin(sents_dir, "{}.json".format(name)) if not os.path.exists(sentences_json_file) or force: with open(sentences_json_file, 'w') as wp, open(file) as fp: json.dump(dict(sentences=toker(fp.read())), wp) n += 1 logger.info("Total number of files dumped is {}".format(n))
def test_progress_track() -> None: console = Console( file=io.StringIO(), force_terminal=True, width=60, color_system="truecolor", legacy_windows=False, _environ={}, ) progress = Progress( console=console, auto_refresh=False, get_time=MockClock(auto=True) ) test = ["foo", "bar", "baz"] expected_values = iter(test) with progress: for value in progress.track(test, description="test"): assert value == next(expected_values) result = console.file.getvalue() print(repr(result)) expected = "\x1b[?25l\r\x1b[2Ktest \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 0%\x1b[0m \x1b[36m-:--:--\x1b[0m\r\x1b[2Ktest \x1b[38;2;249;38;114m━━━━━━━━━━━━━\x1b[0m\x1b[38;5;237m╺\x1b[0m\x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 33%\x1b[0m \x1b[36m-:--:--\x1b[0m\r\x1b[2Ktest \x1b[38;2;249;38;114m━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m\x1b[38;2;249;38;114m╸\x1b[0m\x1b[38;5;237m━━━━━━━━━━━━━\x1b[0m \x1b[35m 67%\x1b[0m \x1b[36m0:00:06\x1b[0m\r\x1b[2Ktest \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m\r\x1b[2Ktest \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m\n\x1b[?25h" print(expected) print(repr(expected)) print(result) print(repr(result)) assert result == expected
def generate_representative_snapshots( transformations: List[TransformationAnalysis], project_dir: str, project_data_dir: str, output_dir: str, max_binding_free_energy: Optional[float], cache_dir: Optional[str], num_procs: Optional[int], ) -> None: from rich.progress import track with multiprocessing.Pool(num_procs) as pool: result_iter = pool.imap_unordered( partial( generate_representative_snapshot, project_dir=project_dir, project_data_dir=project_data_dir, output_dir=output_dir, cache_dir=cache_dir, max_binding_free_energy=max_binding_free_energy, ), transformations, ) for _ in track( result_iter, total=len(transformations), description="Generating representative snapshots", ): pass
def build_vocabulary(max_doc, min_df, word_tokenizer): """Build vocabulary""" if max_doc > 0: corpus = islice(load_extended_sents_corpus(), max_doc) else: corpus = load_extended_sents_corpus() corpus, replica = tee(corpus, 2) total = sum(1 for _ in replica) vocab = Vocabulary.factory(word_tokenizer) click.secho(click.style(f"...Frequency calculation over extended dataset", fg="blue")) with tokenizer_context(word_tokenizer) as Doc: for i, d in track(enumerate(corpus), total=max_doc if max_doc > 0 else total, description="Building vocabulary..."): doc = Doc.from_sentences(d['sentences']) for sent in doc: for word in sent.tokens: vocab.add_word_to_doc(word, i) vocab.build(min_df) vocab.save() click.secho(click.style(f"Total documents {vocab.document_count}", fg="blue")) click.secho(click.style(f"Vocabulary size {len(vocab)} (words occurred more than {min_df} documents)", fg="blue"))
def track_markets(repeat): get_marketplace = lambda x: pd.DataFrame(core.Game().location(x). marketplace()) user = core.get_user("JimHawkins") trackers = get_trackers(user.get_ships()) tracker_locations = [tracker.location for tracker in trackers] for x in range(repeat): for loc in tracker_locations: print("Adding Market Records for: " + loc) write_marketplace_to_db(get_marketplace(loc), loc) logging.info("Premptive pause for throttle") for n in track(range(5), description="Pausing..."): time.sleep(1) logging.info("Sleeping") for n in track(range(60), description="Sleeping..."): time.sleep(1)
def build_candidates(args, queried_region, selectedOligos, oligoDB): logging.info("Build probe candidates.") args.threads = ap.check_threads(args.threads) if args.threads != 1: candidateList = Parallel(n_jobs=args.threads, backend="threading", verbose=1)( delayed(query.OligoProbe)( queried_region[0], selectedOligos.iloc[ii : (ii + args.n_oligo), :], oligoDB, ) for ii in range(selectedOligos.shape[0] - args.n_oligo + 1) ) else: candidateList = [ query.OligoProbe( queried_region[0], selectedOligos.iloc[i : (i + args.n_oligo), :], oligoDB, ) for i in track(range(selectedOligos.shape[0] - args.n_oligo + 1)) ] logging.info(f"Found {len(candidateList)} probe candidates.") return candidateList
def workflow(func, args, n_jobs=4, unit="sample(s)", progress=True): """Run analyses for several samples in parallel. This will analyze several samples in parallel. Includes a workaround for optlang memory leak. Arguments --------- func : function A function that takes a single argument (can be any object) and that performs your analysis for a single sample. args : array-like object An array-like object (list, tuple, numpy array, pandas Series, etc.) that contains the arguments for each sample. n_jobs : positive int How many samples to analyze in parallel at once. unit : str The unit used for the progress bar. progress : bool Whether to show a progress bar. """ if not isinstance(args, Sized): ValueError("`args` must have a length.") with Pool(processes=n_jobs, maxtasksperchild=1) as pool: it = pool.imap_unordered(func, args) if progress: it = track(it, total=len(args), description="Running") results = list(it) return results
def time(model, example, models, n): """ Benchmark a model on an example. Time n iterations of a model's call on an example. """ service = _configure_from_cli_arguments(models, [model], {"lazy_loading": True}) console = Console() t0 = perf_counter() model = service.get(model) console.print(f"{f'Loaded model `{model.configuration_key}` in':50} " f"... {f'{perf_counter()-t0:.2f} s':>10}") example_deserialized = json.loads(example) console.print(f"Calling `predict` {n} times on example:") console.print(f"{json.dumps(example_deserialized, indent = 2)}") times = [] for _ in track(range(n)): t0 = perf_counter() model(example_deserialized) times.append(perf_counter() - t0) console.print(f"Finished in {sum(times):.1f} s, " f"approximately {sum(times)/n*1e3:.2f} ms per call") t0 = perf_counter() model([example_deserialized] * n) batch_time = perf_counter() - t0 console.print(f"Finished batching in {batch_time:.1f} s, approximately" f" {batch_time/n*1e3:.2f} ms per call")
def sync(d, source, long_journal_name, max_workers): if source == "crossref": source = crossref.Crossref(long_journal_name) else: assert source == "dblp", "Illegal source." source = dblp.Dblp() num_success = 0 with concurrent.futures.ThreadPoolExecutor( max_workers=max_workers) as executor: responses = { executor.submit(source.find_unique, entry): (bib_id, tools.decode(entry)) for bib_id, entry in d.items() } for future in track( concurrent.futures.as_completed(responses), total=len(responses), description="Syncing...", ): bib_id, entry = responses[future] try: data = future.result() except (errors.NotFoundError, errors.UniqueError): pass except errors.HttpError as e: print(e.args[0]) else: num_success += 1 d[bib_id] = tools.update(entry, data) print(f"\n\nTotal number of entries: {len(d)}") print(f"Found: {num_success}") return d
def transform(self, X, y=None): if isinstance(X, list): check_doc_type(X) n_total = len(X) else: X1, X2, X = tee(X, 3) check_doc_type(X1) n_total = sum((1 for _ in X2)) if n_total == 0: raise ValueError(f"Ensure that X contains at least one valid document. Found {n_total}") indptr = [0] indices = [] data = [] for doc in track(X, total=n_total, description="Transforming document(s)", update_period=1, disable=not self.show_progress): if self.lowercase: n_vocabulary = doc.builder.tokenizer.vocabulary.size else: n_vocabulary = doc.builder.tokenizer.vocabulary.size_cs tfidf = doc.get_tfidf(self.tf_method, self.idf_method, drop_stopwords=self.drop_stopwords, lowercase=self.lowercase, drop_suffix=self.drop_suffix, drop_punct=self.drop_punct) for idx in tfidf.nonzero()[0]: indices.append(idx) data.append(tfidf[idx]) indptr.append(len(indices)) return csr_matrix((data, indices, indptr), dtype=np.float32, shape=(n_total, n_vocabulary))
def run(config: Dict, args: Namespace): config_dir = args.config_dir matches = list(match(config, config_dir, hide_progress=True)) hide_progress = args.dry or args.interactive for config in track(matches, description=f'Updating Configuration Directory: {config_dir}', disable=hide_progress): if config.status not in ALLOWED_MATCH_STATUSES: logger.warning('%s had status %s. Skipping.', config.key, config.status) continue if config.disabled: logger.warning('%s is disabled. Skipping.', config.key) continue if args.filter and (config.key not in args.filter): logger.debug('%s is not in explicit filter list %s. Skipping.', config.key, args.filter) continue if config.source_type == ConfigurationFileType.FILE: logger.info('Copying File %s => %s', config.target, config.source) directory = os.path.dirname(config.source) call(os.makedirs, args.dry, args.interactive, directory, exist_ok=True) call(copyfile, args.dry, args.interactive, config.target, config.source) elif config.source_type == ConfigurationFileType.DIRECTORY: logger.info('Copying Directory %s => %s', config.target, config.source) call(copy_tree, args.dry, args.interactive, config.target, config.source)
def retrieve_youtube_subscriptions(args): # In order to retrieve the YouTube subscriptions for the current user, # the user needs to authenticate and authorize access to their YouTube # subscriptions. youtube_authorization = get_authenticated_service(args) try: # init all_channels = [] next_page_token = '' subscriptions_response = youtube_subscriptions( youtube_authorization, next_page_token) total_results = subscriptions_response['pageInfo']['totalResults'] results_per_page = subscriptions_response['pageInfo']['resultsPerPage'] total_iterations = math.ceil(total_results / results_per_page) for _ in track(range(total_iterations), description='Fetching subscriptions'): # retrieve the YouTube subscriptions for the authorized user subscriptions_response = youtube_subscriptions( youtube_authorization, next_page_token) next_page_token = get_next_page(subscriptions_response) # extract the required subscription information channels = parse_youtube_subscriptions(subscriptions_response) # add the channels relieved to the main channel list all_channels.extend(channels) return all_channels except HttpError as err: print("An HTTP error {} occurred:\n{}".format( err.resp.status, err.content))