def generate_tree(df, root_taxon):
"""
Convert NCBI taxonomy node structure into a tree using a depth-first search
"""
logger.info(
f"Generating tree from root taxon {root_taxon['rank']}={root_taxon['name']}"
)
max_estimate = ESTIMATES.get(root_taxon['tax_id'], 1500000)
bar = Bar('Processing', max=max_estimate, suffix=BAR_SUFFIX)
def find_children(taxon):
""" Get a row and all its children as a dict """
# If the taxon is blacklisted, ignore it and all its children
if _is_ignored(taxon):
return {}
child_taxa = df[df['parent_tax_id'] == taxon['tax_id']]
child_nodes = {}
bar.next()
# Base case: no children; Recursive case: update tree with all children
for _, child_taxon in child_taxa.iterrows():
child_nodes.update(find_children(child_taxon))
# Skip over node if it's 'no rank' (typically meaning unclassified)
if taxon['rank'] == 'no rank':
return child_nodes
else:
return {f"taxonomy:{taxon['rank']}={taxon['name']}": child_nodes}
tree = find_children(root_taxon)
bar.finish()
return tree
def gibbs_sampling(self, predict=dict(), given=dict(), n=10000, skip = 50):
bar = Bar('Sampling', max=n)
nodes = list(self.node.keys())
sample = self.random_sample(preset=given)
count = 0
sum = 0
for i in range(n):
last = None
bar.next()
node = None
last = node
while node is None or node in given.keys() or node == last:
node = nodes[randint(0,len(nodes)-1)]
parents = self.node[node]['parents']
if parents[0] is None:
sample[node] = self.sample(node)
else:
given = {key: sample[key] for key in parents}
sample[node] = self.sample(node, given=given)
if count == skip:
evidence = {key: sample[key] for key in predict.keys()}
if not predict == evidence:
continue
sum += 1
else:
count += 1
bar.finish()
return sum/(n-count)
def gibbs_sampling(self, bn, predict=dict(), given=dict(), n=10000, skip = 50):
bar = Bar('Sampling', max=n)
nodes = list(self.node.keys())
sample = self.random_sample(bn, preset=given)
count = 0
sum = 0
for i in range(n):
last = None
bar.next()
node = None
last = node
while node is None or node in given.keys() or node == last:
node = nodes[randint(0,len(nodes)-1)]
parents = self.node[node]['parents']
if parents[0] is None:
sample[node] = self.sample(node)
else:
given = {key: sample[key] for key in parents}
sample[node] = self.sample(node, given=given)
if count == skip:
evidence = {key: sample[key] for key in predict.keys()}
if not predict == evidence:
continue
sum += 1
else:
count += 1
bar.finish()
return sum/(n-count)
def import_profiles(self):
response = requests.get(self.path)
headers = response.headers
total_pages = headers.get('x-wp-totalpages')
if total_pages is not None:
self.pages = int(total_pages)
total_results = headers.get('x-wp-total')
if total_results is not None:
self.degrees_found = int(total_results)
self.progress_bar = ChargingBar(
'Processing',
max=self.degrees_found
)
programs = response.json()
self.process_page(programs)
if self.pages > 1:
for page in range(2, self.pages + 1):
path = "{0}?page={1}".format(self.path, page)
response = requests.get(path)
programs = response.json()
self.process_page(programs)
def crawl(self,current=False):
"""
The main function - crawl the league and mine some data.
"""
logging.info('Starting crawl')
self.driver.get(self.league_link)
self.team_names = set([unidecode(thr.text) for thr in \
self.driver.find_element_by_class_name("stat-table").find_elements_by_class_name("team-link")])
self.driver.find_element(By.XPATH, '//*[@id="sub-navigation"]').find_element(By.PARTIAL_LINK_TEXT, 'Fixtures').click()
self.played_months = self.get_played_months()
self.load_previous_data(current)
prog_bar = ChargingBar('Progress of %s crawling:'%' '.join([self.league,str(self.year)]),max=sum([len(self.fixtures[month]) for month in self.played_months[self.played_months.index(self.start_month)::-1]]))
for month in self.played_months[self.played_months.index(self.start_month)::-1]:
for game in self.fixtures[month]:
logging.info('Starting to parse game')
if game:
self.parse_game(game)
prog_bar.next()
logging.info('Finished game, moving to the next one')
else:
logging.info('Finished month, saving to disk')
self.save_month(month)
if current:
pass
# DBHandler(args_parser.LEAGUE_NAME).update_db(self.all_teams_dict,str(self.year))
else: #we're done - we can save to the DB now
# if not current:
DBHandler(args_parser.LEAGUE_NAME).insert_to_db(self.all_teams_dict,str(self.year))
self.driver.quit()
prog_bar.finish()
def train(self, inputExamples, expected, iterations=BARSIZE):
if self.initialized:
currentIteration = 1
barIteration = 0
statusBar = ChargingBar("\x1b[4;36m"+"\t>> Training:", max=BARSIZE)
interval = int(iterations/BARSIZE) if iterations > 100 else 1
errorMedia = 0
statusBar.start()
while(currentIteration <= iterations):
errorMedia = 0
prediction = self.__backPropagation(inputExamples, expected)
errorMedia = self.cost_fun(prediction, expected)
currentIteration += 1
if barIteration % interval == 0:
statusBar.next()
barIteration = 0
barIteration += 1
while(currentIteration < BARSIZE):
currentIteration += 1
statusBar.next()
statusBar.finish()
self.trainingStatus = str(round(errorMedia,4)) + "TE"
print("\x1b[1;33m"+"\t>> Error (Media) Cost: ", round(errorMedia,4))
print("\x1b[0;37m"+"=-"*35 + "=")
else:
print("<Error>: Empty Neural Network, use reset() or loadFromFile(file)")
exit(1)
def UpdateRoutine():
addrinfo = socket.getaddrinfo(MYGROUP_4, None)[0]
Socket = socket.socket(addrinfo[0], socket.SOCK_DGRAM)
# Set Time-to-live (optional)
ttl_bin = struct.pack('@i', MYTTL)
Socket.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl_bin)
SecureFirmwareName = 'SecureFirmware.bin'
SecureFirmwareLen = GetFirmwareSize(SecureFirmwareName)
print("Firmware file has load.")
ChunkSize = 1012
ChunkCount = ceil( SecureFirmwareLen / ChunkSize )
SecureFirmwareFile = open(SecureFirmwareName,"rb")
print("Firmware file size is %s bytes" % SecureFirmwareLen)
datafile = SecureFirmwareFile.read( ChunkSize )
Progress = ChargingBar('Updating', max=ChunkCount, suffix = '%(index)d/%(max)d [%(percent)d%%]')
while datafile:
Socket.sendto(datafile, (addrinfo[4][0], MYPORT))
datafile = SecureFirmwareFile.read( ChunkSize )
Progress.next( )
time.sleep(0.2)
def apt_get(pkg_name_list):
cache = apt.cache.Cache()
cache.update()
cache.open()
print('Telechargement de ', *pkg_name_list)
bar = ChargingBar('Telechargement & Installation',
max=len(pkg_name_list) + 1)
bar.next()
for pkg_name in pkg_name_list:
pkg = cache[pkg_name]
if pkg.is_installed:
print("\n{pkg_name} est deja installer".format(pkg_name=pkg_name),
file=log_file)
bar.next()
else:
pkg.mark_install()
try:
cache.commit(install_progress=LogInstallProgress())
print("{pkg_name} c\'est installer".format(pkg_name=pkg_name),
file=log_file)
bar.next()
except (Exception, arg):
print(colored(
"l\'installation du packet a echouer [{err}]".format(
err=str(arg)), "red"),
file=log_file)
bar.finish()
def run(directory, output):
"""
Compiles a directory of HTML into a single pdf.
:param directory: Directory to search.
:param output: filename to write to.
"""
files = [f for f in os.listdir(directory) if f[-5:] == '.html']
files.sort(key=lambda x: int(x[:4]))
with ChargingBar('Converting', max=len(files)) as progress:
for file in files:
html = weasyprint.HTML(filename=f'{directory}/{file}')
html.write_pdf(target=f"{directory}/{file[:-5]}.pdf")
progress.next()
merger = PdfFileMerger()
with ChargingBar('Merging', max=len(files) + 1) as progress:
for file in files:
reader = open(f"{directory}/{file[:-5]}.pdf")
merger.append(reader,
bookmark=f"{file[5:-5]}",
import_bookmarks=False)
progress.next()
merger.write(output)
progress.next()
for file in files:
os.remove(f"{directory}/{file[:-5]}.html")
os.remove(f"{directory}/{file[:-5]}.pdf")
def key_gen(): # shared - n, open_key - e,secret_key - d
global shared_key
global open_key
global secret_key
bar = ChargingBar('Генерация ключей', max=3)
while True:
p = rnd.randint(50, 1000)
if prost(p) == True:
break
bar.next()
time.sleep(1)
while True:
q = rnd.randint(50, 1000)
if prost(q) == True:
break
shared_key = p * q # общая часть ключа
pq = (p - 1) * (q - 1)
bar.next()
time.sleep(1)
while True:
d = rnd.randint(50, 10000)
if vz_prost(d, pq) == True:
break
secret_key = d # часть секретного ключа
sootv_usl = [e for e in range(50, 10000000) if (e * d) % pq == 1]
open_key = rnd.choice(sootv_usl) # часть открытого ключа
bar.next()
time.sleep(1)
bar.finish()
print("Открытый ключ: {", open_key, ",", shared_key, "}")
print("Секретный ключ: {", secret_key, ",", shared_key, "}")
def update_progress_bar(queue, length):
bar = ChargingBar('Downloading ------------>', max=length)
while True:
try:
data = queue.get()
bar.next(data)
except EOFError:
pass
def write_ml_csv(self):
"""
Writes the machine learning / info file containing user behavior
:return: Nothing
"""
# 2D list, 1st row contains variable title and each row after contains data for 1 user
out = []
bar = ChargingBar('Writing data file', max=2600, stream=sys.stdout)
# calculate and add all variables in the 2D list
self.add_user_metadata(out)
self.var1_nb_im_visited(out, bar)
self.var2_total_nb_positions(out, bar)
self.var3_average_nb_positions(out, bar)
self.var4_median_nb_positions(out, bar)
self.var5_total_time(out, bar)
self.var6_avg_time(out, bar)
self.var7_median_time(out, bar)
self.var8_position_zooms(out, bar)
self.var9_avg_position_zooms(out, bar)
self.var10_median_position_zooms(out, bar)
self.var11_total_annotation_actions(out, bar)
self.var12_avg_annotation_actions(out, bar)
self.var13_median_annotation_actions(out, bar)
self.var14_avg_position_by_zoom(out, bar)
self.var15_median_position_by_zoom(out, bar)
self.var16_user_image_score_per_image(out, bar)
self.var17_per_image_info(out, bar)
self.var18_module_variables(out, bar)
self.var19_relative_time_worked(out, bar)
self.var20_nb_of_days_worked(out, bar)
self.var21_module_percent_time(out, bar)
self.var22_annotation_visit_order(out, bar)
self.var_results(out)
# output file
if self.ml_out_dir is None:
csv_out_filename = 'learning_data.csv'
stats_file = os.path.join(
config.WORKING_DIRECTORY + self.project_name + "/",
csv_out_filename)
else:
stats_file = self.ml_out_dir
f = open(stats_file, "wb")
csv_out = csv.writer(f)
# write data into file
print len(out[0])
csv_out.writerow(out[0])
for i in range(len(self.user_list) + 1):
csv_out.writerow(out[i + 1])
f.close()
bar.finish()
def calc_inter_layer_covariance(self,
model_wrapper,
use_training_data=True,
batch_size=-1,
**options):
is_chainer = model_wrapper.model_type == "chainer"
train, test = model_wrapper.dataset
data_x = train if use_training_data else test
if is_chainer:
data_x = np.moveaxis(data_x, -1, 0)[0]
else:
data_x = data_x[0]
data_x = np.stack(data_x, axis=0)
data_size = n = len(data_x)
if batch_size > 0:
perm = np.random.permutation(data_size)
data_x = data_x[perm[0:batch_size]]
n = batch_size
n_layers = len(model_wrapper.layers())
bar = ChargingBar("Calculating inter layer covariance", max=n_layers)
layer_outputs = model_wrapper.get_layer_outputs(data_x)
to_save = {}
for l, layer_output in enumerate(layer_outputs):
if is_chainer:
layer_output = layer_output.data
flat_shape = layer_output[0].flatten().shape[0]
sigma = tf.zeros(shape=(flat_shape, flat_shape),
dtype=tf.float32,
name="sigma%d" % l)
for output in layer_output:
g = tf.constant(output.flatten())
sigma += tf.einsum('i,j->ij', g, g)
sigma = tf.Variable(1 / (n - 1) * sigma, name="sigma%d" % l)
eigen_values = tf.self_adjoint_eigvals(sigma,
name="eigen_values%d" % l)
to_save["sigma%d" % l] = sigma
to_save["eigen_values%d" % l] = tf.Variable(eigen_values)
bar.next()
self.store_elements(group_name="inter_layer_covariance",
elements=to_save,
model_wrapper=model_wrapper)
def __process_research(self):
self.progress_bar = ChargingBar(
'Processing research...', max=self.researchers_to_process.qsize())
self.mt_lock = threading.Lock()
for _ in range(self.max_threads):
threading.Thread(target=self.get_researcher_research,
daemon=True).start()
self.researchers_to_process.join()
def recursive_combat(player_1, player_2, level=1, bar: ChargingBar = None):
print_wrapper(f"=== Game {level} ===")
visited_games = set()
round = 0
while player_1 and player_2:
round += 1
if (player_1, player_2) in visited_games:
return 1, player_1
visited_games.add((player_1, player_2))
print_wrapper(f"Player 1's deck: {player_1}")
print_wrapper(f"Player 2's deck: {player_2}")
card_1, player_1 = player_1[0], player_1[1:]
card_2, player_2 = player_2[0], player_2[1:]
print_wrapper(f"Player 1 plays: {card_1}")
print_wrapper(f"Player 2 plays: {card_2}")
# breakpoint()
if len(player_1) >= card_1 and len(player_2) >= card_2:
print_wrapper("Playing a sub-game to determine the winner...")
winner, _ = recursive_combat(
copy_deck(player_1, card_1),
copy_deck(player_2, card_2),
level=level + 1,
)
print_wrapper("...anyway, back to game 1.")
if winner == 1:
print_wrapper(f"Player 1 wins round {round} of game {level}!")
player_1 = (*player_1, card_1, card_2)
else:
print_wrapper(f"Player 2 wins round {round} of game {level}!")
player_2 = (*player_2, card_2, card_1)
else:
if card_1 > card_2:
print_wrapper(f"Player 1 wins round {round} of game {level}!")
player_1 = (*player_1, card_1, card_2)
else:
print_wrapper(f"Player 2 wins round {round} of game {level}!")
player_2 = (*player_2, card_2, card_1)
if bar and max(len(player_1), len(player_2)) > bar.index:
bar.goto(max(len(player_1), len(player_2)))
winner = max(player_1, player_2)
if winner == player_1:
print_wrapper(f"The winner of game {level} is player 1!")
else:
print_wrapper(f"The winner of game {level} is player 2!")
return (1 if winner == player_1 else 2), winner
def generate_gif(input_file):
if not os.path.exists(folder):
os.makedirs(folder)
video_duration = (int(
float(
subprocess.check_output([
"ffprobe", "-v", "error", "-show_entries", "format=duration",
"-of", "default=noprint_wrappers=1:nokey=1", input_file
],
universal_newlines=True).strip())))
# creating vars for easy reading
current_time = args.begin
random_start = args.randstart
random_end = args.randend
# workaround for videos in source root folder
if len(input_file.split("/")) == 1:
print("\nCurrent file: " + input_file)
else:
print("\nCurrent file: " + input_file.split("/")[1])
bar = ChargingBar('Processing', max=video_duration)
# initialize bar
bar.goto(0)
while current_time <= video_duration:
current_time = current_time + (random.randrange(
random_start, random_end))
if current_time > video_duration:
# set progress bar to 100% before breaking
bar.goto(video_duration)
break
# creating vars for easy reading
filename_generator = args.destination + '%s-%s.gif'
gif_length = args.length
# you can increase the fps=12 and scale=w=480 values with a higher number for smoother/bigger gifs,
# increases the file size.
subprocess.check_output([
'ffmpeg', '-y', '-ss',
str(current_time), '-t', gif_length, '-i', input_file,
'-filter_complex',
'[0:v] fps=12,scale=w=480:h=-1,split [a][b];[a] palettegen=stats_mode=single [p];['
'b][p] paletteuse=new=1', filename_generator %
(input_file, current_time)
],
stderr=subprocess.STDOUT,
universal_newlines=True).strip()
bar.goto(current_time)
bar.finish()
def __setup_description_processing(self):
"""
Sets up multiple threads for processing
catalog descriptions.
"""
self.catalog_description_progress = ChargingBar(
'Processing descriptions...',
max=self.catalog_description_queue.qsize())
for _ in range(self.max_threads):
Thread(target=self.__process_descriptions, daemon=True).start()
self.catalog_description_queue.join()
def __setup_curriculum_processing(self):
"""
Sets up multiple threads for processing
catalog curriculum data.
"""
self.catalog_curriculum_progress = ChargingBar(
'Processing curriculums...',
max=self.catalog_curriculum_queue.qsize())
for _ in range(self.max_threads):
Thread(target=self.__process_curriculums, daemon=True).start()
self.catalog_curriculum_queue.join()
def __init__(self, path, language, outpath=None):
self.path = path
self.language = language
self.translator = Translator()
with open(self.path) as f:
self.lines = f.readlines()
self.parts_to_skip = [
'equation', 'array', 'figure', 'algorithm', 'hyp', 'thm', 'table',
'tabular'
]
self.list_commands = ['itemize', 'enumerate']
self.outpath = outpath
self.bar = ChargingBar('Translating Tex File...', max=len(self.lines))
def train(self, inputs, outputs, iterations):
if self.initialized:
statusBar = ChargingBar('\t>> Training:', max=100)
status = False
iteration = 0
interval = int(iterations/100) if iterations > 100 else 1
passed = 0
nonPassed = 0
while(status != True and iteration < iterations):
status = True
passed = 0
nonPassed = 0
for i in range(len(inputs)):
currentStatus = self.fit(inputs[i], outputs[i])
status = status and currentStatus
if currentStatus:
passed += 1
else:
nonPassed += 1
iteration += 1
if iteration % interval == 0:
statusBar.next()
while(iteration < 100):
iteration += 1
statusBar.next()
statusBar.finish()
print("\t>> Tests Passed:", passed, "- Test Non passed:", nonPassed)
self.trainingStatus = str(int((float(passed)/float(passed + nonPassed)) * 100)) + "%"
print("<>"*20)
return
print("<Error>: Empty Neural Network, use overrideNewNN() or loadFromFile(file)")
exit(1)
def PREPARE_DATASET(trainingArticleCount,
preprocessor: Preprocessor,
maxArticles,
allowedCategories=[],
dtype='reuters') -> List[DataSet]:
#create Array with two datasets. One training, one test
dataSet = [DataSet(), DataSet()]
#check datatype and initialize provider
if dtype == 'reuters':
soupLoader = SoupLoader(-1)
provider = ReutersProvider(soupLoader)
else:
provider = TwentyNewsProvider('../TwentyNews/')
#start nice percentage bar. Good to have visuals ;)
bar = ChargingBar("Preparing dataset: ", max=maxArticles)
while bar.index <= maxArticles:
try:
#try to create a new article
article = ArticleFactory.GET_NEXT_ARTICLE(
provider, allowedCategories)
article.process(preprocessor)
#append the article to the dataset
dataSet[int(bar.index / trainingArticleCount)].append(article)
bar.next()
except OutOfArticlesError:
#bar would stop at 99% if not increased once more
bar.next()
break
bar.finish()
return dataSet
def upload_to_ya(self, upload_list):
ya_obj = YaUpLoader(self.ya_token)
ya_load_to = input('\nВведите путь до папки на ya_disk: ')
print(ya_obj.check_folder(ya_load_to))
print(f'\nЗагружаем файлы на YaDisk')
bar = ChargingBar('Countdown', max=len(upload_list[1]))
hash_map = {}
for photo in upload_list[1]:
bar.start()
file_name = photo['file_name']
if file_name in hash_map.keys():
last_name = file_name
value = hash_map[last_name] + 1
file_name = file_name.split('.')[0] + '_' + str(value) + '.jpg'
hash_map[last_name] = value
else:
hash_map[file_name] = 1
ya_file_to = ya_load_to + '/' + file_name
url = photo['url']
res = requests.get(url).content
ya_obj.upload(ya_file_to, res)
bar.next()
bar.finish()
def decrypt_file(self, ciphertext_file_name, plaintext_file_name):
# filename, chunk_size, padding_size, base_destination):
ct_chunk_reader = self.decryption_chunk_reader(ciphertext_file_name, self.padding_block_size_bytes,
self.base_num)
# ct_chunk_reader = self.decryption_chunk_reader(ciphertext_file_name) #, self.padding_block_size_bytes, self.base_num, self.max_block_size_after_encrypt_bytes, self.block_size_bytes )
pt_file = open(plaintext_file_name, 'wb+')
file_size = os.stat(ciphertext_file_name).st_size
bar = ChargingBar('[*] Decrypting ', max=(file_size // (self.max_block_size_after_encrypt_bytes * 1000)) + 1)
for index, (pt_chunk, block_size) in enumerate(ct_chunk_reader):
if index % 1000 == 0:
bar.next()
nums_after_lookup_table = ''
for num in pt_chunk:
nums_after_lookup_table = nums_after_lookup_table + self.look_up_table_decryption.get(num)
nums_after_lookup_table = int(nums_after_lookup_table, self.base_num)
# if len(bin(nums_after_lookup_table)[2:])/8 > block_size:
# someshit=1
ct_bytes = nums_after_lookup_table.to_bytes(block_size, byteorder='little')
# ct_bytes = nums_after_lookup_table.to_bytes(self.block_size_bytes, byteorder='little')
pt_file.write(ct_bytes)
# except Exception as e:
# print(e)
# pass
bar.next(bar.max - bar.index)
bar.finish()
l("Decryption done")
l("Saved at %s" % os.path.abspath(pt_file.name))
def handle(self, *args, **options):
"""
Main entry function for the command.
Execution logic handled here.
"""
# Remove all existing Units before proceeding.
# Unfortunately, the logic in this importer currently
# is not consistent on imports against existing Units
# vs a fresh db, so to have the most accurate data we can,
# we have to start fresh with each run.
Unit.objects.all().delete()
# Perform mapping.
# NOTE: order is important here! Particularly,
# in order for teledata and Program Departments to
# map properly, Colleges must be mapped first.
self.full_name_replacements = settings.UNIT_NAME_FULL_REPLACEMENTS
self.basic_replacements = settings.UNIT_NAME_PARTIAL_REPLACEMENTS
self.lowercase_replacements = settings.UNIT_NAME_LOWERCASE_REPLACEMENTS
self.uppercase_replacements = settings.UNIT_NAME_UPPERCASE_REPLACEMENTS
self.colleges_processed = College.objects.all()
# Teledata Organizations that look like they could align to a College
# should be prioritized for processing, hence the ordering here:
self.teledata_orgs_processed = TeledataOrg.objects.annotate(
college_index=StrIndex('name', V('college'))).order_by(
'-college_index')
self.program_depts_processed = ProgramDept.objects.all()
self.teledata_depts_processed = TeledataDept.objects.all()
self.mapping_progress_bar = ChargingBar(
'Mapping data...',
max=self.colleges_processed.count() +
self.teledata_orgs_processed.count() +
self.program_depts_processed.count() +
self.teledata_depts_processed.count())
self.map_orgs_colleges()
self.map_orgs_teledata()
self.map_depts_programs()
self.map_depts_teledata()
# Consolidate duplicate Units as best as we can.
self.consolidatable_unit_names = Unit.objects.values('name').annotate(
name_count=Count('pk')).filter(name_count=2)
self.cleanup_progress_bar = ChargingBar(
'Cleaning up...', max=self.consolidatable_unit_names.count())
self.consolidate_duplicate_units()
# Done.
self.print_stats()
def rejection_sample(self, predict=dict(), given=dict(), n=10000):
sum = 0
bar = Bar('Sampling', max=n)
for i in range(n):
bar.next()
sample = self.compute_sample()
evidence = {key: sample[key] for key in given.keys()}
if not given == evidence:
continue
evidence = {key: sample[key] for key in predict.keys()}
if not predict == evidence:
continue
sum += 1
bar.finish()
return sum/n
def rejection_sample(self, predict=dict(), given=dict(), n=10000):
sum = 0
bar = Bar('Sampling', max=n)
for i in range(n):
bar.next()
sample = self.compute_sample()
evidence = {key: sample[key] for key in given.keys()}
if not given == evidence:
continue
evidence = {key: sample[key] for key in predict.keys()}
if not predict == evidence:
continue
sum += 1
bar.finish()
return sum/n
def __match_programs(self):
"""
Loops through the catalog entries and
attempts to match them to existing programs.
"""
self.catalog_match_progress = ChargingBar(
'Matching existing programs to catalog entries...',
max=len(self.matchable_programs))
for mp in self.matchable_programs:
self.catalog_match_progress.next()
# Create a list of CatalogEntry's to match against that
# share the same career and level as the program, and
# that share a college:
filtered_entries = [
x for x in self.catalog_entries
if x.level_pk == mp.level_pk and x.career_pk == mp.career_pk
and (x.college_short in
mp.program.colleges.values_list('short_name', flat=True)
if x.college_short is not None else True)
]
# Determine all potential catalog entry matches for the program:
for entry in filtered_entries:
mp.match(entry)
if mp.has_matches:
# Send the MatchableProgram off for further processing
self.catalog_description_queue.put(mp)
# Get the best match and save it for later
match = mp.get_best_match()
mp.best_match = match[1]
# Increment match counts
self.program_match_count += 1
mp.best_match.match_count += 1
logging.log(
logging.INFO,
f"MATCH \n Matched program name: {mp.program.name} \n Cleaned program name: {mp.name_clean} \n Catalog entry full name: {mp.best_match.data['title']} \n Cleaned catalog entry name: {mp.best_match.name_clean} \n Match score: {match[0]} \n"
)
else:
logging.log(
logging.INFO,
f"FAILURE \n Matched program name: {mp.program.name} \n Cleaned program name: {mp.name_clean} \n"
)
def likelihood_weighting(self, predict=dict(), given=dict(), n=10000):
num = den = 0
bar = Bar('Sampling', max=n)
for i in range(n):
bar.next()
sample = self.compute_sample(preset=predict)
for node in predict.keys():
parents = self.node[node]['parents']
given_pa = {key: sample[key] for key in parents}
weight = float(self.get_probability(node, evidence=given_pa, value=predict[node]))
evidence = {key: sample[key] for key in given.keys()}
if given == evidence:
num += weight
den += weight
bar.finish()
return num/den
def _align_reads():
consolidated_results_list = []
with open(CSV_FILE) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
output_file_count = 1
for row in ChargingBar('Processing',
max=length(CSV_FILE)).iter(csv_reader):
index_name = _get_substr_before('.', row[REF_SEQ_INDEX])
run_args = [
ALIGNER, REF_SEQ_FLAG, index_name, READ_1_FLAG,
row[READ_1_INDEX], READ_2_FLAG, row[READ_2_INDEX],
OUTPUT_FILE_FLAG,
OUTPUT_FILE_PREFIX + str(output_file_count) + OUTPUT_FILE_EXT
]
proc = subprocess.Popen(run_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
overall_alignment_rate = _get_overall_alignment_rate(
str(proc.communicate()[PROC_STDOUT_INDEX]))
consolidated_results_list.append([
row[REF_SEQ_INDEX], row[READ_1_INDEX], row[READ_2_INDEX],
overall_alignment_rate
])
output_file_count += 1
return consolidated_results_list
def likelihood_weighting(self, predict=dict(), given=dict(), n=10000):
num = den = 0
bar = Bar('Sampling', max=n)
for i in range(n):
bar.next()
sample = self.compute_sample(preset=predict)
for node in predict.keys():
parents = self.node[node]['parents']
given_pa = {key: sample[key] for key in parents}
weight = float(self.get_probability(node, evidence=given_pa, value=predict[node]))
evidence = {key: sample[key] for key in given.keys()}
if given == evidence:
num += weight
den += weight
bar.finish()
return num/den
def tag_dir(dir_path):
'''
Takes a directory at <dir_path>
Tags its content (audio files only) with metadata retrieved through the AcoustID API
'''
lth = 0
for _file in listdir(dir_path):
if is_audio_file(dir_path + '/' + _file):
lth += 1
bar = ChargingBar(max=lth, suffix='[%(index)d/%(max)d]')
for _file in listdir(dir_path):
song_path = dir_path + '/' + _file
if is_audio_file(song_path):
bar.message = INDENT + cyan_fg(_file) + '\n'
tag(song_path)
bar.next()
def doList(taskList):
print("Do tasks on list")
print(f"""Here are your tasks. Enter the number of the task when you complete it:""")
for task in taskList:
print(f"{taskList.index(task) + 1}. {task[0]}")
completedTasks = []
os.chdir('c:\\Users\\tomas\\Desktop\\Anti-Procrastination')
now = datetime.datetime.now()
path = f'Todo list - quicklist - {now.year}-{now.month}-{now.day}.txt'
todo = open(path, 'a')
todo.write(f"\n\nThe date is {now.month} and you have {len(taskList)} tasks to finish!\n")
#bar = ChargingBar('Status', max=len(taskList))
print('\n')
with ChargingBar(max=len(taskList)) as bar:
for i in range(len(taskList)):
update = int(input("Enter number of completed task: \n")) - 1
now = microSecSlicer(datetime.datetime.now())
timeTaken = now - taskList[update][1]
todo.write(f"{taskList[update][0]} - TASK COMPLETE\nIt took you(hours:minutes:seconds): {timeTaken}\n")
completedTasks.append(taskList[update])
bar.next()
print("\nRemaining tasks:")
for task in taskList:
if task not in completedTasks:
print(f"{taskList.index(task) + 1}. {task[0]}")
if len(completedTasks) == len(taskList):
print("all tasks completed")
else:
print("something went wrong, length of tasklist and length of completedTasks do not match")
todo.close()
print("closing file...")
def subList(task):
print(
f"You have selected {task[0]} to work on. Let\'s break this task down into managable sub-tasks."
)
time.sleep(1)
subTasks = []
entry = ""
while entry.lower() != "stop":
entry = (input("what task do you want to add?\n"))
if entry.lower() == "stop":
continue
else:
subTasks.append(entry)
print(f"{entry} is added to your sublist for {task[0]}.")
with ChargingBar(max=len(subTasks)) as bar:
while len(subTasks) > 0:
print("\n")
for subTask in subTasks:
print(f"{subTasks.index(subTask) + 1}.{subTask}")
update = input('Enter the number of sub-task completed.')
if update.isdecimal() == True:
if int(update) - 1 in range(len(subTasks)):
subTasks.pop(int(update) - 1)
bar.next()
time.sleep(1)
bar.finish()
time.sleep(1)
print("All done! Now you can take this tricky task off your list.")
time.sleep(1)
def discover_net(self, ip_range=24):
protocol = self.protocol
base_ip = self.my_ip
# print_figlet()
if not protocol:
protocol = "ICMP"
else:
if protocol != "ICMP":
logging.warning(
f"Warning: {protocol} is not supported by discover_net function! Changed to ICMP"
)
if protocol == "ICMP":
logging.info("Starting - Discover Hosts Scan")
base_ip = base_ip.split('.')
base_ip = f"{str(base_ip[0])}.{str(base_ip[1])}.{str(base_ip[2])}.0/{str(ip_range)}"
hosts = list(ipaddress.ip_network(base_ip))
bar = ChargingBar("Scanning...", max=len(hosts))
sys.stdout = None
bar.start()
threads = [None] * len(hosts)
results = [None] * len(hosts)
for i in range(len(threads)):
threads[i] = Thread(target=self.send_icmp,
args=(hosts[i], results, i))
threads[i].start()
for i in range(len(threads)):
threads[i].join()
bar.next()
bar.finish()
sys.stdout = sys.__stdout__
hosts_found = [i for i in results if i is not None]
if not hosts_found:
logging.warn('[[red]-[/red]]Not found any host')
else:
print("")
logging.info(f'{len(hosts_found)} hosts founded')
for host in hosts_found:
logging.info(f'Host found: {host}')
return True
else:
logging.critical("[[red]-[/red]]Invalid protocol for this scan")
return False
def create_examples(self,year,lookback=15,current=False):
"""
This function creates all the examples for self.league, year.
The examples are created using the given lookback.
"""
def update_all_teams_dict(res,all_teams_dict,team,first):
for fix in sorted(res):
if fix == 1 and res[fix] == {}:
all_teams_dict[team][fix] = []
continue
if first:
all_teams_dict[team][fix] = [res[fix][k] for k in sorted(res[fix])]
else:
all_teams_dict[team][fix] += [res[fix][k] for k in sorted(res[fix])]
def relative_features(arr1,arr2,fn):
combined_list_all_1 = [value for (value,key) in zip(arr1,fn) if key.split("all_pos")>1 ]
combined_list_att_1 = [value for (value,key) in zip(arr1,fn) if key.split("att_pos")>1 ]
combined_list_def_1 = [value for (value,key) in zip(arr1,fn) if key.split("def_pos")>1 ]
combined_list_all_2 = [value for (value,key) in zip(arr2,fn) if key.split("all_pos")>1 ]
combined_list_att_2 = [value for (value,key) in zip(arr2,fn) if key.split("att_pos")>1 ]
combined_list_def_2 = [value for (value,key) in zip(arr2,fn) if key.split("def_pos")>1 ]
all_rel = [1 for (val1,val2) in zip (combined_list_all_1,combined_list_all_2) if val1 > val2]
att_rel = [1 for (val1,val2) in zip (combined_list_att_1,combined_list_att_2) if val1 > val2]
def_rel = [1 for (val1,val2) in zip (combined_list_def_1,combined_list_def_2) if val1 > val2]
return float(len(all_rel))/len(combined_list_all_1), float(len(att_rel))/len(combined_list_att_1), float(len(def_rel))/len(combined_list_def_1)
from features.features import Features
temp_DB = self.temp_DB
all_teams_names = [g['_id'] for g in temp_DB[self.league].aggregate([{"$match":{"Year":int(year)}},{"$group":{"_id":"$GName"}}])]
all_teams_dict = {name:{} for name in all_teams_names}
features = Features(temp_DB[self.league],year,self.league)
features_names = []
prog_bar = ChargingBar('Creating examples for %s-%s'%(self.league,year),max=len(all_teams_dict))
for team in all_teams_dict:
res_by_all, res_by_non_avg = features.create_features(team,lookback)
if not features_names: features_names = features.features_names
update_all_teams_dict(res_by_all, all_teams_dict, team, True)
update_all_teams_dict(res_by_non_avg, all_teams_dict, team, False)
prog_bar.next()
examples = []
tags = []
curr_examples = []
prog_bar.finish()
for team in all_teams_names:
for fix in sorted(all_teams_dict[team]):
if fix == 1 and all_teams_dict[team][fix]==[]:
continue
curr_game = temp_DB[self.league].find_one({"GName":team,"Fix":fix,"Year":int(year)})
if curr_game is None:
continue
if curr_game["HA"]=="home":
vs_curr_game = temp_DB[self.league].find_one({"GName":curr_game["VS"],"VS":team,"HA":"away","Year":int(year)})
try:
vs_curr_fix = vs_curr_game["Fix"]
except TypeError as e:
vs_curr_fix = fix+1
all_teams_dict[curr_game["VS"]][vs_curr_fix] = []
if all_teams_dict[curr_game["VS"]][vs_curr_fix] == []:
continue
rel_all, rel_att, rel_def = relative_features(all_teams_dict[team][fix], all_teams_dict[curr_game["VS"]][vs_curr_fix], features_names)
examples += [np.array(all_teams_dict[team][fix])-np.array(all_teams_dict[curr_game["VS"]][vs_curr_fix])]
examples[-1] = np.concatenate((examples[-1],[rel_all, rel_att, rel_def]))
temp_dict = {"Ex":examples[-1],"Fix":curr_game["Fix"],"Res":curr_game["Result"],"Home":team,"Away":curr_game["VS"],"League":self.league}
curr_examples += [temp_dict]
tags += [curr_game["Tag"]]
if not current:
return examples,tags
else:
return curr_examples,tags
def main(args=sys.argv[1:]):
parser = argparse.ArgumentParser(
prog='calbum',
add_help=True,
description='calbum is an unattended calendar-based photo organiser. '
'It is meant to allow easy management of pictures based '
'on their location, date and calendar events without '
'the need of a database or a special browser to retrieve '
'them.')
parser.add_argument('--link-only',
help='Keep files where they are in the inbox folder.',
action='store_true')
parser.add_argument('--inbox',
help='The path of the inbox directory. '
'(default: ./inbox)',
metavar='path',
default='./inbox')
parser.add_argument('--timeline',
help='The path of the timeline directory. '
'(default: ./timeline)',
metavar='path',
default='./timeline')
parser.add_argument('--album',
help='The path of the album directory. '
'(default: ./album)',
metavar='path',
default='./album')
parser.add_argument('--calendar',
help='The url of the album calendar. (ical)',
metavar='url')
parser.add_argument('--date-format',
help='The format to use for timestamps.',
metavar='format',
default=model.TimeLine.media_path_format)
parser.add_argument('--save-events',
help='Keep the calendar event in the album.',
action='store_true')
parser.add_argument('--time-zone',
metavar='tz',
help='Pictures timezone (default to local time).')
settings = vars(parser.parse_args(args))
# Configure data model
model.TimeLine.media_path_format = settings['date_format']
model.Media.time_zone = gettz(settings['time_zone'])
# Create filters
timeline_filter = timeline.TimelineFilter(settings['timeline'])
album_filter = NoopMediaFilter()
if settings['calendar']:
events = calendar.CalendarEvent.load_from_url(url=settings['calendar'])
album_filter = album.CalendarAlbumFilter(
albums_path=settings['album'],
events=events,
save_events=settings['save_events']
)
filter_actions = [
timeline_filter.link if settings['link_only'] else timeline_filter.move,
album_filter.link
]
# Perform actions
suffix = '%(index)d/%(max)d [eta: %(eta)ds]'
bar = ChargingBar('Processing inbox:', suffix=suffix)
for picture in bar.iter(list(model.MediaCollection(settings['inbox']))):
for action in filter_actions:
action(picture)
cmd = "istat "+fs+" "+inode+" | awk '/Direct/,/^$/'"
ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE)
grep_o = ps.communicate()[0]
blocks = grep_o.split('\n')[1:-1]
index = 1
for i in range(1, len(blocks)):
if blocks[-i] != "":
index = i
break
last_block = blocks[-index].split()[-1]
# store file info
f = {"inode":inode, "name":filename, "size":size, "block": last_block}
files.append(f)
bar_max = len(files)
bar = ChargingBar('Progress:', max=bar_max)
# loop through files and read last block
for f in files:
cmd = "dd if="+fs+" bs="+bsize+" skip="+f["block"]+" count=1 > "+tmpfile+" 2> /dev/null"
ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE)
output = ps.communicate()[0]
# compute how many bytes to expect in last block
nblocks = int(math.ceil(int(f['size'])/float(bsize)))
slack = nblocks*int(bsize) - int(f['size'])
actualbytes = int(bsize) - slack
# read slack space
tmp = open(tmpfile, "rb")
bcount = 0
hidden=""
try:
byte = tmp.read(1)
