def test_track() -> None:
_time = 1.0
def get_time():
nonlocal _time
try:
return _time
finally:
_time += 1
console = Console(file=io.StringIO(),
force_terminal=True,
width=40,
color_system="truecolor")
test = ["foo", "bar", "baz"]
expected_values = iter(test)
for value in track(test,
"test",
console=console,
auto_refresh=False,
get_time=get_time):
assert value == next(expected_values)
result = console.file.getvalue()
print(repr(result))
expected = "test \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 0%\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[?25l\r\x1b[2Ktest \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 0%\x1b[0m \x1b[36m-:--:--\x1b[0m \r\x1b[2Ktest \x1b[38;2;249;38;114m━━━━━━━\x1b[0m\x1b[38;5;237m╺\x1b[0m\x1b[38;5;237m━━━━━━━━━━━━━\x1b[0m \x1b[35m 33%\x1b[0m \x1b[36m-:--:--\x1b[0m \r\x1b[2Ktest \x1b[38;2;249;38;114m━━━━━━━━━━━━━━\x1b[0m\x1b[38;5;237m╺\x1b[0m\x1b[38;5;237m━━━━━━\x1b[0m \x1b[35m 67%\x1b[0m \x1b[36m0:00:06\x1b[0m \r\x1b[2Ktest \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m \r\x1b[2Ktest \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m \n\x1b[?25h"
assert result == expected
with pytest.raises(ValueError):
for n in track(5):
pass
def test_track() -> None:
console = Console(
file=io.StringIO(),
force_terminal=True,
width=60,
color_system="truecolor",
legacy_windows=False,
)
test = ["foo", "bar", "baz"]
expected_values = iter(test)
for value in track(test,
"test",
console=console,
auto_refresh=False,
get_time=MockClock(auto=True)):
assert value == next(expected_values)
result = console.file.getvalue()
print(repr(result))
expected = "\x1b[?25ltest \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 0%\x1b[0m \x1b[36m-:--:--\x1b[0m\r\x1b[2Ktest \x1b[38;2;249;38;114m━━━━━━━━━━━━━\x1b[0m\x1b[38;5;237m╺\x1b[0m\x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 33%\x1b[0m \x1b[36m-:--:--\x1b[0m\r\x1b[2Ktest \x1b[38;2;249;38;114m━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m\x1b[38;2;249;38;114m╸\x1b[0m\x1b[38;5;237m━━━━━━━━━━━━━\x1b[0m \x1b[35m 67%\x1b[0m \x1b[36m0:00:06\x1b[0m\r\x1b[2Ktest \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m\r\x1b[2Ktest \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m\n\x1b[?25h"
assert result == expected
with pytest.raises(ValueError):
for n in track(5):
pass
def save(
self,
output="result",
):
output_dir = Path(output)
fieldnames = list(self.courses[0].keys())
with open(output_dir.joinpath("grade.csv"), "w", newline="") as f:
writer = csv.DictWriter(f, fieldnames)
writer.writeheader()
for grade in track(self.courses, description="Fetching grades..."):
writer.writerow(
{key: val["display"]
for key, val in grade.items()})
with open(output_dir.joinpath("course.json"), "w",
encoding="utf8") as f:
for course in track(self.origin_data,
description="Fetching courses data..."):
res = self.courses_analysis(course=course["JXBID"],
semester=course["XNXQDM"],
stu=course["XH"])
added_name = {
"class_name": course["KCM"],
**res,
}
f.write(json.dumps(added_name, ensure_ascii=False) + "\n")
def polygonize(load_dict, id_index=''):
if id_index:
for intersection in track(load_dict['intersections']):
point = intersection['point']
ix = point['x']
iy = point['y']
dis = 0
for road in intersection['roads']:
_ = load_dict['roads'][id_index[road]]
mx = _['midpoint']['x']
my = _['midpoint']['y']
length = math.sqrt((ix - mx)**2 + ((iy - my)**2))
dis = dis + length
intersection['distanceToMidpoints'] = dis
else:
for intersection in track(load_dict['intersections']):
point = intersection['point']
ix = point['x']
iy = point['y']
dis = 0
for road in intersection['roads']:
for _ in load_dict['roads']:
if _['id'] == road:
mx = _['midpoint']['x']
my = _['midpoint']['y']
length = math.sqrt((ix - mx)**2 + ((iy - my)**2))
dis = dis + length
intersection['distanceToMidpoints'] = dis
def solve_it(input_data):
lines = input_data.split('\n')
parts = lines[0].split()
customer_count = int(parts[0])
vehicle_count = int(parts[1])
vehicle_capacity = int(parts[2])
console.print("Reading data: ....", style = "bold red")
customers = []
for i in track(range(1, customer_count+1)):
line = lines[i]
parts = line.split()
customers.append(Customer(i-1, int(parts[0]), Point(float(parts[1]), float(parts[2]))))
vehicles = []
for i in track(range(vehicle_count)):
vehicle = Vehicle()
vehicle.index = i
vehicle.capacity = sys.maxsize
vehicle.available = vehicle_capacity
vehicles.append(vehicle)
app = GuildLocalSearch(customers, vehicles)
# app.search(max_step=10000, max_repeat= 100)
app.search()
best_cost = app.best_cost
best_vehicles = app.best_vehicles
output = str(best_cost) + " 0\n"
for vehicle in best_vehicles:
output += " ".join(str(node.index) for node in vehicle.tour) + "\n"
return output
def features_extractor(adversarial_vector: str, feature_mapping: str):
info("\t[*] Loading feature vector mapping from pickle file ...")
feature_vector_mapping = pickle.load(open(feature_mapping, "rb"))
info("\t[*] Loading adversarially generated feature vectors from pickle file ...")
adversarial_feature_vector = pickle.load(open(adversarial_vector, "rb"))
feature_vector_mapping = [import_lib for import_lib in feature_vector_mapping]
debug(f"\t[+] adversarial_feature_vector length : {len(adversarial_feature_vector)}")
adversarial_imports_set = []
adversarial_sections_set = []
count = 0
info("\t[*] Generating imports and section set from adversarially generated feature vectors ...")
for index in track(range(len(adversarial_feature_vector)), description=" Mapping imports and sections ...", transient=True):
info(
f"\t\t[+] Mapping imports and sections from batch {index} with {len(adversarial_feature_vector[index])} adversarial feature vectors ...",
)
for i in track(
range(len(adversarial_feature_vector[index])), description=" Extracting ...", transient=True
):
sample = adversarial_feature_vector[index][i]
sample = sample.tolist()
adversial_imports = []
adversial_section = []
adversial_features = []
debug(f"\t\t[+] Sample lenght : {len(sample)}")
debug(f"\t\t[+] Adv Feature Vector : {(sample)}")
for i in range(len(sample)):
if sample[i] > 0:
adversial_features.append(feature_vector_mapping[i])
debug(f"\t\t\t[+] feature : [bold green]{feature_vector_mapping[i]}", extra={"markup":True})
for feature in adversial_features:
if ":" in feature:
adversial_imports.append(feature)
else:
adversial_section.append(feature)
pass
adversarial_imports_set.append(adversial_imports)
adversarial_sections_set.append(adversial_section)
debug(
f"Feature mapping for adversarial feature vector [{count}] completed with {len(adversial_imports)} imports and {len(adversial_section)} sections ...",
)
# debug("Import mapping for adversarial feature vector [" + str(count) + "] completed with " + str(len(adversial_imports)) + " imports ...\n")
count = count + 1
info("\t[+] Adversarial Imports and sections have been extracted ...")
return adversarial_imports_set, adversarial_sections_set
def build_vocabulary(max_doc, min_df, min_freq, word_tokenizer, w2v,
w2v_num_epoch, w2v_skip_gram, w2v_size):
"""Build vocabulary & optionally word embeddings"""
total = sum(1 for _ in sentence_iter(word_tokenizer, max_doc))
counter_cs = VocabularyCounter(word_tokenizer,
case_sensitive=True,
min_tf=min_freq,
min_df=min_df)
counter = VocabularyCounter(word_tokenizer,
case_sensitive=False,
min_tf=min_freq,
min_df=min_df)
if w2v:
model = get_w2v_model(w2v_size, min_freq, w2v_skip_gram)
console.log("Building vocabulary...")
model.build_vocab(sentence_iter(word_tokenizer, max_doc))
for _ in track(range(w2v_num_epoch),
total=w2v_num_epoch,
description="Building word vectors..."):
model.train(sentences=sentence_iter(word_tokenizer, max_doc),
epochs=1,
total_examples=total,
report_delay=1)
click.secho(
click.style(f"...Frequency calculation over extended dataset",
fg="blue"))
with tokenizer_context(word_tokenizer) as Doc:
if max_doc is None:
corpus = load_extended_sents_corpus()
else:
corpus = islice(load_extended_sents_corpus(), max_doc)
corpus, X = tee(corpus, 2)
for i, d in track(enumerate(corpus),
total=sum(1 for _ in X),
description="Building vocabulary..."):
doc = Doc.from_sentences(d['sentences'])
for sent in doc:
for word in sent.tokens:
counter.inc(word, i, 1)
counter_cs.inc(word, i, 1)
counter = counter.prune()
counter_cs = counter_cs.prune()
if w2v:
counter.to_hdf5(model)
else:
counter.to_hdf5()
counter_cs.to_hdf5()
def ps(ip, output, data, ps_mode):
threads = []
result = {}
try:
print('\n\n' + Y + '[!]' + Y + ' Starting Port Scan...' + W + '\n')
if ps_mode == 'fast':
print(G + '[+]' + C + ' Testing Top 1000 Ports...' + W + '\n')
port_list = [1,3,4,6,7,9,13,17,19,20,21,22,23,24,25,26,30,32,33,37,42,43,49,53,70,79,80,81,82,83,84,85,88,89,90,99,100,106,109,110,111,113,119,125,135,139,143,144,146,161,163,179,199,211,212,222,254,255,256,259,264,280,301,306,311,340,366,389,406,407,416,417,425,427,443,444,445,458,464,465,481,497,500,512,513,514,515,524,541,543,544,545,548,554,555,563,587,593,616,617,625,631,636,646,648,666,667,668,683,687,691,700,705,711,714,720,722,726,749,765,777,783,787,800,801,808,843,873,880,888,898,900,901,902,903,911,912,981,987,990,992,993,995,999,1000,1001,1002,1007,1009,1010,1011,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065,1066,1067,1068,1069,1070,1071,1072,1073,1074,1075,1076,1077,1078,1079,1080,1081,1082,1083,1084,1085,1086,1087,1088,1089,1090,1091,1092,1093,1094,1095,1096,1097,1098,1099,1100,1102,1104,1105,1106,1107,1108,1110,1111,1112,1113,1114,1117,1119,1121,1122,1123,1124,1126,1130,1131,1132,1137,1138,1141,1145,1147,1148,1149,1151,1152,1154,1163,1164,1165,1166,1169,1174,1175,1183,1185,1186,1187,1192,1198,1199,1201,1213,1216,1217,1218,1233,1234,1236,1244,1247,1248,1259,1271,1272,1277,1287,1296,1300,1301,1309,1310,1311,1322,1328,1334,1352,1417,1433,1434,1443,1455,1461,1494,1500,1501,1503,1521,1524,1533,1556,1580,1583,1594,1600,1641,1658,1666,1687,1688,1700,1717,1718,1719,1720,1721,1723,1755,1761,1782,1783,1801,1805,1812,1839,1840,1862,1863,1864,1875,1900,1914,1935,1947,1971,1972,1974,1984,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2013,2020,2021,2022,2030,2033,2034,2035,2038,2040,2041,2042,2043,2045,2046,2047,2048,2049,2065,2068,2099,2100,2103,2105,2106,2107,2111,2119,2121,2126,2135,2144,2160,2161,2170,2179,2190,2191,2196,2200,2222,2251,2260,2288,2301,2323,2366,2381,2382,2383,2393,2394,2399,2401,2492,2500,2522,2525,2557,2601,2602,2604,2605,2607,2608,2638,2701,2702,2710,2717,2718,2725,2800,2809,2811,2869,2875,2909,2910,2920,2967,2968,2998,3000,3001,3003,3005,3006,3007,3011,3013,3017,3030,3031,3052,3071,3077,3128,3168,3211,3221,3260,3261,3268,3269,3283,3300,3301,3306,3322,3323,3324,3325,3333,3351,3367,3369,3370,3371,3372,3389,3390,3404,3476,3493,3517,3527,3546,3551,3580,3659,3689,3690,3703,3737,3766,3784,3800,3801,3809,3814,3826,3827,3828,3851,3869,3871,3878,3880,3889,3905,3914,3918,3920,3945,3971,3986,3995,3998,4000,4001,4002,4003,4004,4005,4006,4045,4111,4125,4126,4129,4224,4242,4279,4321,4343,4443,4444,4445,4446,4449,4550,4567,4662,4848,4899,4900,4998,5000,5001,5002,5003,5004,5009,5030,5033,5050,5051,5054,5060,5061,5080,5087,5100,5101,5102,5120,5190,5200,5214,5221,5222,5225,5226,5269,5280,5298,5357,5405,5414,5431,5432,5440,5500,5510,5544,5550,5555,5560,5566,5631,5633,5666,5678,5679,5718,5730,5800,5801,5802,5810,5811,5815,5822,5825,5850,5859,5862,5877,5900,5901,5902,5903,5904,5906,5907,5910,5911,5915,5922,5925,5950,5952,5959,5960,5961,5962,5963,5987,5988,5989,5998,5999,6000,6001,6002,6003,6004,6005,6006,6007,6009,6025,6059,6100,6101,6106,6112,6123,6129,6156,6346,6389,6502,6510,6543,6547,6565,6566,6567,6580,6646,6666,6667,6668,6669,6689,6692,6699,6779,6788,6789,6792,6839,6881,6901,6969,7000,7001,7002,7004,7007,7019,7025,7070,7100,7103,7106,7200,7201,7402,7435,7443,7496,7512,7625,7627,7676,7741,7777,7778,7800,7911,7920,7921,7937,7938,7999,8000,8001,8002,8007,8008,8009,8010,8011,8021,8022,8031,8042,8045,8080,8081,8082,8083,8084,8085,8086,8087,8088,8089,8090,8093,8099,8100,8180,8181,8192,8193,8194,8200,8222,8254,8290,8291,8292,8300,8333,8383,8400,8402,8443,8500,8600,8649,8651,8652,8654,8701,8800,8873,8888,8899,8994,9000,9001,9002,9003,9009,9010,9011,9040,9050,9071,9080,9081,9090,9091,9099,9100,9101,9102,9103,9110,9111,9200,9207,9220,9290,9415,9418,9485,9500,9502,9503,9535,9575,9593,9594,9595,9618,9666,9876,9877,9878,9898,9900,9917,9929,9943,9944,9968,9998,9999,10000,10001,10002,10003,10004,10009,10010,10012,10024,10025,10082,10180,10215,10243,10566,10616,10617,10621,10626,10628,10629,10778,11110,11111,11967,12000,12174,12265,12345,13456,13722,13782,13783,14000,14238,14441,14442,15000,15002,15003,15004,15660,15742,16000,16001,16012,16016,16018,16080,16113,16992,16993,17877,17988,18040,18101,18988,19101,19283,19315,19350,19780,19801,19842,20000,20005,20031,20221,20222,20828,21571,22939,23502,24444,24800,25734,25735,26214,27000,27352,27353,27355,27356,27715,28201,30000,30718,30951,31038,31337,32768,32769,32770,32771,32772,32773,32774,32775,32776,32777,32778,32779,32780,32781,32782,32783,32784,32785,33354,33899,34571,34572,34573,35500,38292,40193,40911,41511,42510,44176,44442,44443,44501,45100,48080,49152,49153,49154,49155,49156,49157,49158,49159,49160,49161,49163,49165,49167,49175,49176,49400,49999,50000,50001,50002,50003,50006,50300,50389,50500,50636,50800,51103,51493,52673,52822,52848,52869,54045,54328,55055,55056,55555,55600,56737,56738,57294,57797,58080,60020,60443,61532,61900,62078,63331,64623,64680,65000,65129,65389]
for port in track(port_list):
t = threading.Thread(target=sock_conn, args=[ip, port, output, result, 3])
t.daemon = True
t.start()
threads.append(t)
time.sleep(0.001)
for thread in threads:
thread.join()
elif ps_mode == 'full':
set_open_file_cmd = 'ulimit -Sn 66000'
print(G + '[+]' + C + ' Testing All Ports...' + W + '\n')
print(Y + '[!]' + C + ' Please make sure proper open files value is set for full scan mode\n' + W)
# 65535
os.system(set_open_file_cmd)
try:
for port in track(range(0, 65535)):
t = threading.Thread(target=sock_conn, args=[ip, port, output, result, 3])
t.daemon = True
t.start()
time.sleep(0.0005)
for thread in threads:
thread.join()
except Exception:
pass
else:
print(R + "\n[-]" + C + " \'" + ps_mode + "\' port scan mode is not supported please recheck\n" + W)
return
except Exception as e:
print('\n' + R + '[-]' + C + ' Exception : ' + W + str(e) + '\n')
if output != 'None':
result.update({'Exception':str(e)})
if output != 'None':
ps_output(output, data, result)
print()
def _fetch_data(self, odds_input):
# Main train data
params = pd.merge(pd.DataFrame(self.main_leagues_param_grid_), pd.DataFrame(self.param_grid_)).to_records(False)
data_train_main = []
for _, division, league, url, year in track(params, description='Football-Data.co.uk | Download training data of main leagues:'):
url = urljoin(URL, url)
names = pd.read_csv(url, nrows=0).columns
try:
data = pd.read_csv(url, names=names, skiprows=1)
except (UnicodeDecodeError, pd.errors.ParserError):
data = pd.read_csv(url, names=names, skiprows=1, encoding='ISO-8859-1')
# Extract columns
data.replace('#REF!', np.nan, inplace=True)
data.drop(columns=[col for col in data.columns if 'Unnamed' in col], inplace=True)
data = data.assign(league=league, division=division, year=year, test=False)
data.rename(columns=self._cols_mapping, inplace=True)
# Append data to main data
data_train_main.append(data)
data_train_main = pd.concat(data_train_main, ignore_index=True) if data_train_main else pd.DataFrame()
# Extra train data
params = pd.merge(pd.DataFrame(self.extra_leagues_param_grid_), pd.DataFrame(self.param_grid_)).drop(columns='year').drop_duplicates().to_records(False)
data_train_extra = []
for division, league, url in track(params, description='Football-Data.co.uk | Download training data of extra leagues:'):
data = pd.read_csv(urljoin(URL, url))
data = data.assign(league=league, division=division, year=lambda s: s.Season if type(s.Season) is not str else s.Season.split('/')[-1], test=False)
data.rename(columns=self._cols_mapping, inplace=True)
data_train_extra.append(data)
data_train_extra = pd.concat(data_train_extra, ignore_index=True) if data_train_extra else pd.DataFrame()
# Test data
params = pd.DataFrame(self.main_leagues_param_grid_).drop(columns='url').groupby(['league', 'division'], as_index=False).max()
data_test = pd.read_csv(join(URL, 'fixtures.csv'))
data_test.drop(columns=[col for col in data_test.columns if 'Unnamed' in col], inplace=True)
data_test = pd.merge(data_test, params[['Div', 'league', 'division', 'year']].dropna().drop_duplicates(), how='left')
data_test['test'] = True
data_test.rename(columns=self._cols_mapping, inplace=True)
# Combine data
self.data_ = pd.concat([data_train_main, data_train_extra, data_test], ignore_index=True)
# Select odds types
cols_odds = []
for col in self.data_.columns:
for odds_type in odds_input:
if col.startswith(odds_type) or col.startswith(f'{self.odds_type_}_'):
cols_odds.append(col)
return self
def test_track() -> None:
console = Console(file=io.StringIO(), force_terminal=True, width=40)
test = ["foo", "bar", "baz"]
expected_values = iter(test)
for value in track(test, "test", console=console, auto_refresh=False):
assert value == next(expected_values)
result = console.file.getvalue()
expected = "test \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 0%\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[?25l\r\x1b[2Ktest \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 0%\x1b[0m \x1b[36m-:--:--\x1b[0m \r\x1b[2Ktest \x1b[38;5;13m━━━━━━━\x1b[0m\x1b[38;5;237m╺\x1b[0m\x1b[38;5;237m━━━━━━━━━━━━━\x1b[0m \x1b[35m 33%\x1b[0m \x1b[36m-:--:--\x1b[0m \r\x1b[2Ktest \x1b[38;5;13m━━━━━━━━━━━━━━\x1b[0m\x1b[38;5;237m╺\x1b[0m\x1b[38;5;237m━━━━━━\x1b[0m \x1b[35m 67%\x1b[0m \x1b[36m0:00:01\x1b[0m \r\x1b[2Ktest \x1b[38;5;10m━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m \r\x1b[2Ktest \x1b[38;5;10m━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m \n\x1b[?25h"
assert result == expected
with pytest.raises(ValueError):
for n in track(5):
pass
def import_extractor(adversarial_vector: str, feature_mapping: str):
debug(f"[+] feature mapping type : {(feature_mapping)}")
info("[*] Loading import feature vector mapping from pickle file ...")
feature_vector_mapping = pickle.load(open(str(feature_mapping), "rb"))
info("[*] Loading adversarially generated import feature vectors from pickle file ...")
adversarial_feature_vector = pickle.load(open(adversarial_vector, "rb"))
feature_vector_mapping = [import_lib for import_lib in feature_vector_mapping]
debug(f"[+] adversarial_feature_vector length : {len(adversarial_feature_vector)}")
adversarial_imports_set = []
count = 0
info("[*] Generating imports set from adversarially generated feature vectors ...")
for index in track(range(len(adversarial_feature_vector)), description="Mapping imports ...", transient=True):
info(
f"[+] Mapping imports from batch {index} with {len(adversarial_feature_vector[index])} adversarial feature vectors ..."
)
for i in track(
range(len(adversarial_feature_vector[index])), description=" Extracting ... ", transient=True
):
sample = adversarial_feature_vector[index][i]
sample = sample.tolist()
adversial_imports = []
unfiltered_adversial_imports = []
debug("[+] Sample lenght : {len(sample)}")
for i in range(len(sample)):
if sample[i] > 0:
unfiltered_adversial_imports.append(feature_vector_mapping[i])
for imports in unfiltered_adversial_imports:
if "32" in imports:
adversial_imports.append(imports)
debug("\t[+] Filtered Imports : " + str(imports))
adversarial_imports_set.append(adversial_imports)
debug(
f"Import mapping for adversarial feature vector [{count}] completed with {len(adversial_imports)} imports ...\n"
)
count = count + 1
# info("%d adversarial feature vectors have been mapped ...", count)
debug(f"[+] Number of feature vectors : {len(adversarial_imports_set)}")
return adversarial_imports_set
def generate_dataset(path, parts = 5, max_length = 100):
dataset = []
with open(path, 'r') as data:
lines = data.readlines()
for l in track(lines, description = "processing..."):
strokes = []
d = json.loads(l)
drawing = d['drawing']
x, y = drawing[0][0][0], drawing[0][1][0]
for stroke in drawing:
xs, ys = stroke
# add the beginning pen up movement to the start of the next stroke
strokes.append([
xs[0] - x, ys[0] - y, 0
])
x, y = xs[0], ys[0]
for t in range (1, len(xs)):
strokes.append([
xs[t] - x, ys[t] - y, 1
])
x, y = xs[t], ys[t]
strokes.append([0, 0, 2]) # end drawing
del strokes[0]
if len(strokes) < max_length:
dataset.append(strokes)
# go through dataset and pad
for d in track(range(len(dataset)), description = 'padding... '):
if len(dataset[d]) < max_length:
dataset[d] = dataset[d] + [[0, 0, 2]] * (max_length - len(dataset[d]))
# save each section of the dataset
split = len(dataset) // parts
print("Splitting into %s sections of %s examples each" % (parts, split))
for p in track(range (parts), description = 'splitting... '):
section = dataset[ : split]
del dataset[: split]
ds = np.array(section)
np.save('data/cloud_p%s.npy' % p, ds)
def to_adjacency_list(intersections, roads, road_id_index,
intersection_id_index):
adjacenccy_list = []
for intersection in track(intersections):
cur_list = []
for road in intersection['roads']:
_ = roads[road_id_index[road]]
# if _['startIntersection'] is not intersection['id']:
# cur_list.append(
# intersection_id_index[_['startIntersection']])
# elif _['endIntersection'] is not intersection['id']:
# cur_list.append(
# intersection_id_index[_['endIntersection']])
# else:
# raise Exception(
# 'start- and endIntersection both the same as ', intersection['id'])
if _['endIntersection'] == intersection['id']:
pass
else:
cur_list.append(intersection_id_index[_['endIntersection']])
# if len(cur_list) == 8:
# adjacenccy_list.append(np.array(cur_list[:4]))
# if len(cur_list) == 2:
# adjacenccy_list.append(np.array(cur_list[:1]))
adjacenccy_list.append(np.array(cur_list))
return adjacenccy_list
def _generate_paginated_index(write_html, url_prefix, items, items_per_page,
description):
pages = list(_paginate(items, items_per_page))
def get_page_name(start_index, end_index):
return (f"{url_prefix}/index.html" if start_index == 1 else
f"{url_prefix}/index-{start_index}-{end_index}.html")
for (prev_page, ((start_index, end_index), page_items),
next_page) in track(
zip(
[None] + pages,
pages,
pages[1:] + [None],
),
description=description,
total=len(pages),
):
write_html(
page_items,
template_file=f"{url_prefix}/index.html",
output_file=get_page_name(start_index, end_index),
start_index=start_index,
end_index=end_index,
prev_page=get_page_name(*prev_page[0]) if prev_page else None,
next_page=get_page_name(*next_page[0]) if next_page else None,
)
def generate_mutated_malware(file, model, args):
pe = pefeatures2.PEFeatureExtractor2()
rn = RangeNormalize(-0.5,0.5)
info("[*] Reading file : " + str(file))
bytez = []
with open(str(file), 'rb') as infile:
bytez = infile.read()
for t in track(range(1, args.rl_mutations) , description="Generating mutation ...", transient=True):
state = pe.extract( bytez )
state_norm = rn(state)
state_norm = torch.from_numpy(state_norm).float().unsqueeze(0).to(device)
actions = model.forward(state_norm)
action = torch.argmax(actions).item()
action = ACTION_LOOKUP[action]
bytez = bytes(manipulate.modify_without_breaking(bytez, [action]))
new_score = interface.get_score_local( bytez )
if(new_score < interface.local_model_threshold):
break
if not os.path.exists(args.o):
os.mkdir(args.o)
info("[*] output directory has been created at : " + str(args.o))
output_file = os.path.join(args.o, "mutated_" + str(os.path.basename(file)))
info("[*] Writing mutated file to : " + str(output_file) + "\n\n")
with open(str(output_file), mode='wb') as file1:
file1.write(bytes(bytez))
return
def daily_task():
"""Main workhorse function. Support functions defined below"""
global CATEGORIES_PAGES
global BROWSER
global DATE
global OBSERVATION
log.info('Scraper started')
# Refresh date
DATE = str(datetime.date.today())
OBSERVATION = 0
# Initiate headless web browser
log.info('Initializing browser')
BROWSER = webdriver.Chrome(executable_path=CHROME_DRIVER, options=OPTIONS)
# Download topsite and get categories directories
base_file_name = "All_cat_" + DATE + ".html"
fetch_html(BASE_URL, base_file_name, PATH_HTML, attempts_limit=1000)
html_file = open(PATH_HTML + base_file_name).read()
CATEGORIES_PAGES = get_category_list(html_file)
log.info('Found ' + str(len(CATEGORIES_PAGES)) + ' categories')
# Read each categories pages and scrape for data
for cat in track(CATEGORIES_PAGES,
description="[green]Scraping...",
total=len(CATEGORIES_PAGES)):
cat_file = "cat_" + cat['name'] + "_" + DATE + ".html"
download = fetch_html(cat['directlink'], cat_file, PATH_HTML)
if download:
scrap_data(cat)
# Close browser
BROWSER.close()
BROWSER.service.process.send_signal(signal.SIGTERM)
BROWSER.quit()
def gen_multi(function, args, noise=False):
with Pool() as p:
results = [p.apply_async(function, args=(a, noise)) for a in args]
dataset = [
p.get()
for p in track(results, description='Generating sounds ...')
]
def website(full_table):
# Enters into the info page and parses out the info
for f in track(full_table, description="Parsing..."):
source = requests.get(f['News Media Info'])
soup = BeautifulSoup(source.content, 'lxml')
try:
# getting the website link to news source
locate_html_class = soup.find('div', {'class': 'dynamic-grid'})
locate_paragraph = locate_html_class.find('a')['href']
f['News Source Site'] = locate_paragraph
except TypeError:
pass
try:
# getting the creation date of the news source
locate__html_class = soup.find('div', {'class': 'dynamic-grid'})
locate_paragraph = locate__html_class.find_all('p')[1].text.split(
'.')[-1].strip()
f['Established'] = locate_paragraph
except IndexError:
pass
try:
# Who the news source owned by
locate__html_class = soup.find('div', {'class': 'dynamic-grid'})
locate_paragraph = locate__html_class.find_all('p')[2].text.split(
':')[-1].strip()
f['Owned by'] = locate_paragraph
except IndexError:
pass
sleep(10)
return full_table
def sentences(dataset_dir: str, tokenizer: str, force: bool):
dataset = os.path.dirname(__file__) if dataset_dir is None else dataset_dir
raws = glob.glob(pjoin(dataset, 'raw', '*.txt'))
sents_dir = pjoin(dataset, 'sents')
os.makedirs(sents_dir, exist_ok=True)
toker = get_tokenizer(tokenizer)
n = 0
logger.info("|raw documents|: {}".format(len(raws)))
logger.info("Sentence tokenizer: {}".format(tokenizer))
for file in track(raws, description="Building sentence corpus..."):
basename = os.path.basename(file)
name, _ = os.path.splitext(basename)
sentences_json_file = pjoin(sents_dir, "{}.json".format(name))
if not os.path.exists(sentences_json_file) or force:
with open(sentences_json_file, 'w') as wp, open(file) as fp:
json.dump(dict(sentences=toker(fp.read())), wp)
n += 1
logger.info("Total number of files dumped is {}".format(n))
def test_progress_track() -> None:
console = Console(
file=io.StringIO(),
force_terminal=True,
width=60,
color_system="truecolor",
legacy_windows=False,
_environ={},
)
progress = Progress(
console=console, auto_refresh=False, get_time=MockClock(auto=True)
)
test = ["foo", "bar", "baz"]
expected_values = iter(test)
with progress:
for value in progress.track(test, description="test"):
assert value == next(expected_values)
result = console.file.getvalue()
print(repr(result))
expected = "\x1b[?25l\r\x1b[2Ktest \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 0%\x1b[0m \x1b[36m-:--:--\x1b[0m\r\x1b[2Ktest \x1b[38;2;249;38;114m━━━━━━━━━━━━━\x1b[0m\x1b[38;5;237m╺\x1b[0m\x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 33%\x1b[0m \x1b[36m-:--:--\x1b[0m\r\x1b[2Ktest \x1b[38;2;249;38;114m━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m\x1b[38;2;249;38;114m╸\x1b[0m\x1b[38;5;237m━━━━━━━━━━━━━\x1b[0m \x1b[35m 67%\x1b[0m \x1b[36m0:00:06\x1b[0m\r\x1b[2Ktest \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m\r\x1b[2Ktest \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m\n\x1b[?25h"
print(expected)
print(repr(expected))
print(result)
print(repr(result))
assert result == expected
def generate_representative_snapshots(
transformations: List[TransformationAnalysis],
project_dir: str,
project_data_dir: str,
output_dir: str,
max_binding_free_energy: Optional[float],
cache_dir: Optional[str],
num_procs: Optional[int],
) -> None:
from rich.progress import track
with multiprocessing.Pool(num_procs) as pool:
result_iter = pool.imap_unordered(
partial(
generate_representative_snapshot,
project_dir=project_dir,
project_data_dir=project_data_dir,
output_dir=output_dir,
cache_dir=cache_dir,
max_binding_free_energy=max_binding_free_energy,
),
transformations,
)
for _ in track(
result_iter,
total=len(transformations),
description="Generating representative snapshots",
):
pass
def build_vocabulary(max_doc, min_df, word_tokenizer):
"""Build vocabulary"""
if max_doc > 0:
corpus = islice(load_extended_sents_corpus(), max_doc)
else:
corpus = load_extended_sents_corpus()
corpus, replica = tee(corpus, 2)
total = sum(1 for _ in replica)
vocab = Vocabulary.factory(word_tokenizer)
click.secho(click.style(f"...Frequency calculation over extended dataset", fg="blue"))
with tokenizer_context(word_tokenizer) as Doc:
for i, d in track(enumerate(corpus), total=max_doc if max_doc > 0 else total,
description="Building vocabulary..."):
doc = Doc.from_sentences(d['sentences'])
for sent in doc:
for word in sent.tokens:
vocab.add_word_to_doc(word, i)
vocab.build(min_df)
vocab.save()
click.secho(click.style(f"Total documents {vocab.document_count}", fg="blue"))
click.secho(click.style(f"Vocabulary size {len(vocab)} (words occurred more than {min_df} documents)", fg="blue"))
def track_markets(repeat):
get_marketplace = lambda x: pd.DataFrame(core.Game().location(x).
marketplace())
user = core.get_user("JimHawkins")
trackers = get_trackers(user.get_ships())
tracker_locations = [tracker.location for tracker in trackers]
for x in range(repeat):
for loc in tracker_locations:
print("Adding Market Records for: " + loc)
write_marketplace_to_db(get_marketplace(loc), loc)
logging.info("Premptive pause for throttle")
for n in track(range(5), description="Pausing..."):
time.sleep(1)
logging.info("Sleeping")
for n in track(range(60), description="Sleeping..."):
time.sleep(1)
def build_candidates(args, queried_region, selectedOligos, oligoDB):
logging.info("Build probe candidates.")
args.threads = ap.check_threads(args.threads)
if args.threads != 1:
candidateList = Parallel(n_jobs=args.threads, backend="threading", verbose=1)(
delayed(query.OligoProbe)(
queried_region[0],
selectedOligos.iloc[ii : (ii + args.n_oligo), :],
oligoDB,
)
for ii in range(selectedOligos.shape[0] - args.n_oligo + 1)
)
else:
candidateList = [
query.OligoProbe(
queried_region[0],
selectedOligos.iloc[i : (i + args.n_oligo), :],
oligoDB,
)
for i in track(range(selectedOligos.shape[0] - args.n_oligo + 1))
]
logging.info(f"Found {len(candidateList)} probe candidates.")
return candidateList
def workflow(func, args, n_jobs=4, unit="sample(s)", progress=True):
"""Run analyses for several samples in parallel.
This will analyze several samples in parallel. Includes a workaround for
optlang memory leak.
Arguments
---------
func : function
A function that takes a single argument (can be any object) and
that performs your analysis for a single sample.
args : array-like object
An array-like object (list, tuple, numpy array, pandas Series, etc.)
that contains the arguments for each sample.
n_jobs : positive int
How many samples to analyze in parallel at once.
unit : str
The unit used for the progress bar.
progress : bool
Whether to show a progress bar.
"""
if not isinstance(args, Sized):
ValueError("`args` must have a length.")
with Pool(processes=n_jobs, maxtasksperchild=1) as pool:
it = pool.imap_unordered(func, args)
if progress:
it = track(it, total=len(args), description="Running")
results = list(it)
return results
def time(model, example, models, n):
"""
Benchmark a model on an example.
Time n iterations of a model's call on an example.
"""
service = _configure_from_cli_arguments(models, [model],
{"lazy_loading": True})
console = Console()
t0 = perf_counter()
model = service.get(model)
console.print(f"{f'Loaded model `{model.configuration_key}` in':50} "
f"... {f'{perf_counter()-t0:.2f} s':>10}")
example_deserialized = json.loads(example)
console.print(f"Calling `predict` {n} times on example:")
console.print(f"{json.dumps(example_deserialized, indent = 2)}")
times = []
for _ in track(range(n)):
t0 = perf_counter()
model(example_deserialized)
times.append(perf_counter() - t0)
console.print(f"Finished in {sum(times):.1f} s, "
f"approximately {sum(times)/n*1e3:.2f} ms per call")
t0 = perf_counter()
model([example_deserialized] * n)
batch_time = perf_counter() - t0
console.print(f"Finished batching in {batch_time:.1f} s, approximately"
f" {batch_time/n*1e3:.2f} ms per call")
def sync(d, source, long_journal_name, max_workers):
if source == "crossref":
source = crossref.Crossref(long_journal_name)
else:
assert source == "dblp", "Illegal source."
source = dblp.Dblp()
num_success = 0
with concurrent.futures.ThreadPoolExecutor(
max_workers=max_workers) as executor:
responses = {
executor.submit(source.find_unique, entry):
(bib_id, tools.decode(entry))
for bib_id, entry in d.items()
}
for future in track(
concurrent.futures.as_completed(responses),
total=len(responses),
description="Syncing...",
):
bib_id, entry = responses[future]
try:
data = future.result()
except (errors.NotFoundError, errors.UniqueError):
pass
except errors.HttpError as e:
print(e.args[0])
else:
num_success += 1
d[bib_id] = tools.update(entry, data)
print(f"\n\nTotal number of entries: {len(d)}")
print(f"Found: {num_success}")
return d
def transform(self, X, y=None):
if isinstance(X, list):
check_doc_type(X)
n_total = len(X)
else:
X1, X2, X = tee(X, 3)
check_doc_type(X1)
n_total = sum((1 for _ in X2))
if n_total == 0:
raise ValueError(f"Ensure that X contains at least one valid document. Found {n_total}")
indptr = [0]
indices = []
data = []
for doc in track(X, total=n_total, description="Transforming document(s)", update_period=1,
disable=not self.show_progress):
if self.lowercase:
n_vocabulary = doc.builder.tokenizer.vocabulary.size
else:
n_vocabulary = doc.builder.tokenizer.vocabulary.size_cs
tfidf = doc.get_tfidf(self.tf_method, self.idf_method, drop_stopwords=self.drop_stopwords,
lowercase=self.lowercase,
drop_suffix=self.drop_suffix,
drop_punct=self.drop_punct)
for idx in tfidf.nonzero()[0]:
indices.append(idx)
data.append(tfidf[idx])
indptr.append(len(indices))
return csr_matrix((data, indices, indptr), dtype=np.float32, shape=(n_total, n_vocabulary))
def run(config: Dict, args: Namespace):
config_dir = args.config_dir
matches = list(match(config, config_dir, hide_progress=True))
hide_progress = args.dry or args.interactive
for config in track(matches, description=f'Updating Configuration Directory: {config_dir}', disable=hide_progress):
if config.status not in ALLOWED_MATCH_STATUSES:
logger.warning('%s had status %s. Skipping.', config.key, config.status)
continue
if config.disabled:
logger.warning('%s is disabled. Skipping.', config.key)
continue
if args.filter and (config.key not in args.filter):
logger.debug('%s is not in explicit filter list %s. Skipping.', config.key, args.filter)
continue
if config.source_type == ConfigurationFileType.FILE:
logger.info('Copying File %s => %s', config.target, config.source)
directory = os.path.dirname(config.source)
call(os.makedirs, args.dry, args.interactive, directory, exist_ok=True)
call(copyfile, args.dry, args.interactive, config.target, config.source)
elif config.source_type == ConfigurationFileType.DIRECTORY:
logger.info('Copying Directory %s => %s', config.target, config.source)
call(copy_tree, args.dry, args.interactive, config.target, config.source)
def retrieve_youtube_subscriptions(args):
# In order to retrieve the YouTube subscriptions for the current user,
# the user needs to authenticate and authorize access to their YouTube
# subscriptions.
youtube_authorization = get_authenticated_service(args)
try:
# init
all_channels = []
next_page_token = ''
subscriptions_response = youtube_subscriptions(
youtube_authorization, next_page_token)
total_results = subscriptions_response['pageInfo']['totalResults']
results_per_page = subscriptions_response['pageInfo']['resultsPerPage']
total_iterations = math.ceil(total_results / results_per_page)
for _ in track(range(total_iterations), description='Fetching subscriptions'):
# retrieve the YouTube subscriptions for the authorized user
subscriptions_response = youtube_subscriptions(
youtube_authorization, next_page_token)
next_page_token = get_next_page(subscriptions_response)
# extract the required subscription information
channels = parse_youtube_subscriptions(subscriptions_response)
# add the channels relieved to the main channel list
all_channels.extend(channels)
return all_channels
except HttpError as err:
print("An HTTP error {} occurred:\n{}".format(
err.resp.status, err.content))
