def test_no_count(self):
"""
Test for an empty counter
"""
ctr = enlighten.Counter(total=10, desc='Test', unit='ticks')
formatted = ctr.format(width=80)
self.assertEqual(len(formatted), 80)
self.assertRegex(formatted, r'Test 0%\|[ ]+ \| 0/10 \[00:0\d<\?, 0.00 ticks/s\]')
# No unit, no description
ctr = enlighten.Counter(total=10)
formatted = ctr.format(width=80)
self.assertEqual(len(formatted), 80)
self.assertRegex(formatted, r' 0%\|[ ]+ \| 0/10 \[00:0\d<\?, 0.00/s\]')
def pack(self, files):
if not self.path:
return False
Print.info('\tRepacking to NSP...')
hd = self.generateHeader(files)
totalSize = len(hd) + sum(os.path.getsize(file) for file in files)
if os.path.exists(self.path) and os.path.getsize(self.path) == totalSize:
Print.info('\t\tRepack %s is already complete!' % self.path)
return
t = enlighten.Counter(total=totalSize, unit='B', desc=os.path.basename(self.path), leave=False)
t.write('\t\tWriting header...')
outf = open(self.path, 'wb')
outf.write(hd)
t.update(len(hd))
done = 0
for file in files:
t.write('\t\tAppending %s...' % os.path.basename(file))
with open(file, 'rb') as inf:
while True:
buf = inf.read(4096)
if not buf:
break
outf.write(buf)
t.update(len(buf))
t.close()
Print.info('\t\tRepacked to %s!' % outf.name)
outf.close()
def call_extract_multible_xml(args):
'''
A specify extractor for multible xml files that keep their text in the 's' mark.
The function extracts text form mulitble files and exports .txt files that have the names of the
subfolders in rootfolder.
'''
list_of_file_paths = get_file_directories(args)
pbar = enlighten.Counter(total=len(list_of_file_paths),
desc='Extracting files')
dir_info = directory_info(args)
root_name = dir_info.get_root_name()
name = get_subfolder_name(list_of_file_paths[0], root_name)
list_to_create = []
for path in list_of_file_paths:
current_subfolder = get_subfolder_name(path, root_name)
#Compare current subfolder to name. Given that the list_of_file_paths
#is linear this will create files with the names of the subfolder and
#then reset the list_to_create
if name != current_subfolder or len(list_to_create) > 500:
filename = '.\data\\05 - raw\\' + name + '.txt'
create_file(list_to_create, filename, mode='a')
list_to_create = []
name = current_subfolder
list_to_create.extend(xml_extractor(path))
pbar.update()
filename = '.\data\\05 - raw\\' + name + '.txt'
create_file(list_to_create, filename, mode='a')
def load_demo(path_to_json_data, mongo_db, convert_to_ensembl=False):
"""Inserts demo patient data into database
Demo data consists of a set of 50 patients from this paper: http://onlinelibrary.wiley.com/doi/10.1002/humu.22850
Args:
path_to_demo_data(str): absolute path to json file containing the demo patients.
mongo_db(pymongo.database.Database)
Returns:
inserted_ids(list): the database ID of the inserted patients
"""
patients = [] # a list of dictionaries
inserted_ids = []
#open json file and try to insert one patient at the time
try:
LOG.info('reading patients file')
with open(path_to_json_data) as json_data:
patients = json.load(json_data)
# create a progress bar
pbar = enlighten.Counter(total=len(patients), desc='', unit='patients')
for json_patient in patients:
#parse patient into format accepted by database
patient = mme_patient(json_patient, convert_to_ensembl)
inserted_id = backend_add_patient(mongo_db=mongo_db, patient=patient)[1]
if inserted_id:
inserted_ids.append(inserted_id)
pbar.update()
except Exception as err:
LOG.fatal("An error occurred while importing benchmarking patients: {}".format(err))
return inserted_ids
def augment_data(args, data):
# Augmenta data
augmented_data = pd.DataFrame(columns=['wav_filename', 'wav_filesize', 'transcript', 'duration'])
data_augment = sample(args, data, True)
random.seed(args.augment_seed)
shifts = [random.randint(1, 5) for _ in range (len(data_augment))]
pbar = enlighten.Counter(total=len(data_augment), desc='Augmenting train set')
for i, row in data_augment.iterrows():
pbar.update()
# Augmenta þetta sýni og vista undir augmented
y, sr = manipulate(row['wav_filename'], shifts[i])
_, filename = os.path.split(row['wav_filename'])
filepath = os.path.join(args.malromur_dir, 'augmented', filename)
sf.write(filepath, y, sr, 'PCM_16')
# Búa til new row
new_row = row
new_row['wav_filename'] = filepath
new_row['wav_filesize'] = os.path.getsize(filepath)
# Bæta new row við augmented_data
augmented_data = augmented_data.append(row, ignore_index=True)
# Sameina við data
new_data = data.append(augmented_data, ignore_index=True)
return new_data, augmented_data['duration'].sum()
def remove_variants(conn, dataset, list_of_var_tuples):
"""
Deletes variants from beacon
"""
delete_counter = 0
LOG.info('Deleting variants from database..')
#loop over each sample(key) of the dictionary:
click.echo("variants to remove:%s (it might take some time!)" %
len(list_of_var_tuples))
pbar = enlighten.Counter(total=len(list_of_var_tuples),
desc='',
unit='ticks')
for var_tuple in list_of_var_tuples:
try:
unique_key = dataset + "_" + str(var_tuple[0]) + "_" + str(
var_tuple[1]) + "_" + var_tuple[2]
# Remove 1 from the occurrence field if this is not the last occurrence
sql = "update beacon_data_table set occurrence = occurrence -1 where chr_pos_alt_dset=%s"
result = conn.execute(sql, unique_key)
if result.rowcount > 0:
delete_counter += 1
pbar.update()
except Exception as ex:
print('Unexpected error:', ex)
# delete all records with no samples associated:
if delete_counter > 0:
sql = "delete from beacon_data_table where occurrence = 0;"
conn.execute(sql)
return delete_counter
def setUp(self):
os.environ['TERM'] = 'vt100'
self.tty = MockTTY()
self.manager = MockManager(stream=self.tty.stdout)
self.ctr = enlighten.Counter(total=10, desc='Test', unit='ticks', manager=self.manager)
self.manager.counters[self.ctr] = 3
self.output = r'Test 0%\|[ ]+ \| 0/10 \[00:0\d<\?, 0.00 ticks/s\]'
def ___test_model___(self, clusters, model):
self.logger.debug(f'Testing model: modelSize = {self.modelSize}, clusters = {self.clustersSize}, tskip|tsize = {self.trainSkip}|{self.trainSize}')
pbar = enlighten.Counter(total=self.trainSize, desc='Tested', unit='matches')
testResult = {'matches': 0, 'correct': 0, 'nodata': 0}
for x in self.con["data"].find(skip=self.trainSkip, limit = self.trainSize):
if any(player is {} or 'hero_id' not in player or player['hero_id'] in [0,None] for player in x['players']):
pbar.update()
continue
key = []
hkey = []
for player in x['players']:
heroId = player['hero_id']
hkey.append(self.originalHeroes[heroId])
key.append(clusters[self.originalHeroes[heroId]])
radiant = sorted(key[0:5])
dire = sorted(key[5:10])
rd = repr(radiant+dire)
dr = repr(dire+radiant)
if dr in model:
key = dr
elif rd in model:
key = rd
else:
testResult['nodata']+=1
pbar.update()
continue
adv = self.___evaluate_advantage___(hkey)
evaluation = model[key]['radiantwin'] / model[key]['matches'] + adv*self.multiplier
isRadiant = evaluation > self.activator
testResult['matches']+=1
testResult['correct']+=int(isRadiant == x['radiant_win'])
pbar.update()
self.logger.debug(f'Testing is done!')
self.logger.debug(f'Accuracy: {testResult["correct"]/testResult["matches"]*100}; correct|tested: {testResult["correct"]}|{testResult["matches"]}; activator|mult: {self.activator}|{self.multiplier}; modelsize|cl = {self.modelSize}|{self.clustersSize}; tskip|tsize = {self.trainSkip}|{self.trainSize}.')
return testResult
def ___train_model___(self, clusters):
self.logger.debug(f'Training model: modelSize = {self.modelSize}, clusters = {self.clustersSize}.')
path = os.path.join(os.path.dirname(__file__), '{}/{}_{}.pickle'.format('models',self.modelSize, self.clustersSize))
if os.path.exists(path):
self.logger.debug(f'Found cached data: MODEL/{self.modelSize}_{self.clustersSize} - {path}')
return self.___deserialize___(path)
else:
pbar = enlighten.Counter(total=self.modelSize, desc='Training', unit='matches')
model = {}
for x in self.con["data"].find(limit = self.modelSize):
if any(player is {} or 'hero_id' not in player or player['hero_id'] in [0,None] for player in x['players']):
pbar.update()
continue
key = []
for player in x['players']:
heroId = player['hero_id']
key.append(clusters[self.originalHeroes[heroId]])
radiant = sorted(key[0:5])
dire = sorted(key[5:10])
rd = repr(radiant+dire)
dr = repr(dire+radiant)
key = rd
if dr in model:
key = dr
elif rd in model:
key = rd
else:
model[key] = {'matches' : 0, 'radiantwin': 0}
model[key]['matches']+=1
model[key]['radiantwin']+=int(x['radiant_win'])
pbar.update()
self.___serialize___('models','{}_{}'.format(self.modelSize, self.clustersSize), model)
self.logger.debug(f'Done training model: modelSize = {self.modelSize}, clusters = {self.clustersSize}.')
return model
def testRange(activatorStart = 0.4, activatorEnd = 0.6, activatorStep = 0.005,
multiplierStart = 0.0, multiplierEnd = 2.0, multiplierStep = 0.02):
fig = plt.figure()
ax = fig.gca(projection='3d')
activator = np.arange(activatorStart, activatorEnd, activatorStep, dtype=np.dtype(float))
multiplier = np.arange(multiplierStart, multiplierEnd, multiplierStep, dtype=np.dtype(float))
X, Y = np.meshgrid(activator, multiplier)
Z = np.copy(X)
pbar = enlighten.Counter(total=np.size(Z,1)*np.size(Z,0), desc='Progress', unit='tests')
for i in range(np.size(Z,1)):
for j in range(np.size(Z,0)):
model.set_params(X[0,i].item(), Y[j,i].item())
trained = model.clusterize_train(2)
Z[j,i] = accuracy(trained)
pbar.update()
surf = ax.plot_surface(X, Y, Z, cmap= cm.get_cmap("coolwarm"), rstride=1, cstride=1,
linewidth=0, antialiased=True)
ax.set_zlim(46, 61)
ax.set_xlabel('Активатор')
ax.set_xticks(activator)
ax.set_ylabel('Множитель')
ax.set_yticks(multiplier)
ax.set_zlabel('Точность предсказания')
ax.zaxis.set_major_locator(LinearLocator(10))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))
fig.colorbar(surf, shrink=0.3, aspect=5)
plt.show()
def _hero_matrix_(self):
path = os.path.join(os.path.dirname(__file__), '{}/{}.pickle'.format('matrices',self.trainSkip))
if os.path.exists(path):
self.logger.debug(f'Found cached data: HEROMATRIX: {self.trainSkip} - {path}')
self.matrix = self.___deserialize___(path)
else:
self.logger.debug(f'Building HEROMATRIX: {self.trainSkip} matches.')
pbar = enlighten.Counter(total=self.trainSkip, desc='Heromatrix', unit='matches')
matrix = [[{'matches': 0, 'won':0, 'winrate' : 0} for x in range(len(self.heroesArray))] for y in range(len(self.heroesArray))]
for x in self.con["data"].find(limit = self.trainSkip):
if any(player is {} or 'hero_id' not in player or player['hero_id'] in [None,0] for player in x['players']):
pbar.update()
continue
for i in range(0,5):
for j in range (i+1, 5):
self.___eval_matrix_match___(matrix, x, i, j, False)
for j in range (5,10):
a,b = self.originalHeroes[x['players'][i]['hero_id']], self.originalHeroes[x['players'][j]['hero_id']]
r,c = max(a,b), min(a,b)
matrix[r][c]['matches']+=1
matrix[r][c]['won']+= (a == r) == x['radiant_win']
for i in range(5,9):
for j in range (i+1, 10):
self.___eval_matrix_match___(matrix, x, i, j, True)
pbar.update()
for i in range(0,len(self.heroesArray)):
for j in range (i+1, len(self.heroesArray)):
matrix[i][j]['winrate'] = 0 if matrix[i][j]['matches'] == 0 else matrix[i][j]['won'] / matrix[i][j]['matches']
matrix[j][i]['winrate'] = 0 if matrix[j][i]['matches'] == 0 else matrix[j][i]['won'] / matrix[j][i]['matches']
self.___serialize___('matrices', self.trainSkip, matrix)
self.matrix = matrix
self.logger.debug(f'Done building HEROMATRIX: {self.trainSkip} matches.')
return self.matrix
def eliminateDuplicateVectors(database):
'''
Since it requires a lot of time, importing directory does not call
eliminate duplicate vectors from database.
It can be done by manually calling this function.
It eliminates for every class...
'''
all_processes = []
allCatIds = list(database.keys())
print('Optimizing database, printing approximate progress..')
total_classes = len(self._database)
serial_cat_id = allCatIds[np.argmax(
[len(self._database[xx]) for xx in self._database])]
parallel_cat_ids = list(set(allCatIds) - set([serial_cat_id]))
bar = enlighten.Counter(total=len(self._database[serial_cat_id]))
p = Pool(total_classes)
# print(list(map(lambda catId : (database, catId, self._new_cat_threshold), allCatIds[:-1])))
# a,b,c=list(map(lambda catId: [self, catId, bar], allCatIds))[0]
# print(b)
# pdb.set_trace()
eliminateList = p.map_async(
Identifier._eliminateDuplicateVectors_parallel,
list(
map(
lambda catId: [
catId, self._database[catId], self.
_new_cat_threshold
], parallel_cat_ids)))
catId = serial_cat_id
eliminateThose = []
for index1, vector1 in enumerate(database[catId]):
bar.update()
for index2, vector2 in enumerate(database[catId]):
if index1 != index2 and np.linalg.norm(
vector1 - vector2
) < self._new_cat_threshold: # TODO : note this is static!
eliminateThose.append(index1)
eliminateThose = list(np.unique(eliminateThose))
p.close()
p.join()
eliminateList = eliminateList.get()
eliminateList.append((serial_cat_id, eliminateThose))
for identity_and_cat in eliminateList:
catId, identity = identity_and_cat
identity.sort(reverse=True)
for el in identity:
database[catId].pop(el)
def test_full_bar(self):
ctr = enlighten.Counter(total=10, desc='Test', unit='ticks')
ctr.count = 10
ctr.start = time.time() - 10
formatted = ctr.format(width=80)
self.assertEqual(len(formatted), 80)
self.assertRegex(formatted,
r'Test 100%\|' + u'█+' + r'\| 10/10 \[00:\d\d<00:00, \d.\d\d ticks/s\]')
def test_zero_total(self):
"""
If the total is 0, the bar should be full
"""
ctr = enlighten.Counter(total=0, desc='Test', unit='ticks')
formatted = ctr.format(width=80)
self.assertEqual(len(formatted), 80)
self.assertRegex(formatted, r'Test 100%\|' u'█+' + r'\| 0/0 \[00:0\d<00:00, 0.00 ticks/s\]')
def process_files():
"""
Process files with a single progress bar
"""
with enlighten.Counter(total=100, desc='Simple', unit='ticks') as pbar:
for _ in range(100):
time.sleep(0.05)
pbar.update()
def process_files():
"""
Process files with a single progress bar
"""
with enlighten.Counter(total=100, desc='Simple', unit='ticks') as pbar:
for num in range(100): # pylint: disable=unused-variable
time.sleep(0.05)
pbar.update()
def test_format_no_total(self):
# No unit, No desc
ctr = enlighten.Counter()
self.assertRegex(ctr.format(width=80), r'0 \[00:0\d, 0.00/s\]')
ctr.count = 50
ctr.start = time.time() - 50
self.assertRegex(ctr.format(width=80), r'50 \[00:5\d, \d.\d\d/s\]')
# With unit and description
ctr = enlighten.Counter(desc='Test', unit='ticks')
rtn = ctr.format(width=80)
self.assertEqual(len(rtn), 80)
self.assertRegex(rtn, r'Test 0 ticks \[00:0\d, 0.00 ticks/s\]')
ctr.count = 50
ctr.start = time.time() - 50
rtn = ctr.format(width=80)
self.assertEqual(len(rtn), 80)
self.assertRegex(rtn, r'Test 50 ticks \[00:5\d, \d.\d\d ticks/s\]')
def PerformutationFeatureImportance(self):
if self.data.shape[0]>100000:
data = self.data.sample(n=100000,axis=0)
else:
data = self.data
inputsLL = np.hsplit(data[self.inputs].astype(np.float32).values,len(self.inputs))
has_LBN = any([l.__class__.__name__ == 'LBNLayer' for l in self.model.layers])
if has_LBN:
inputsLBN = data[self.LBN_inputs].astype(np.float32).values.reshape(-1,4,len(self.LBN_inputs)//4)
inputs = (inputsLL,inputsLBN)
else:
inputs = inputsLL
logging.info("Producing output for true score")
outputs = self.model.predict(inputs,batch_size=10000,verbose=0)
targets = data[self.outputs].values
true_F1_score = f1_score(targets.argmax(1),outputs.argmax(1),average='macro')
permutations = 10
f1_scores = np.zeros(len(self.inputs))
f1_scores_err = np.zeros(len(self.inputs))
logging.info("Producing output for permutation scores")
pbar = enlighten.Counter(total=len(self.inputs), desc='Permutations', unit='Input')
for idxPerm,inputName in enumerate(self.inputs):
pbar.update()
logging.info("Looking at input %d/%d"%(idxPerm,len(self.inputs)))
inputs_perm = copy.deepcopy(inputsLL)
perm_f1_scores = []
for perm in range(permutations):
logging.info("... Permutation %d/%d"%(perm,permutations))
np.random.shuffle(inputs_perm[idxPerm])
if has_LBN:
outputs_perm = self.model.predict((inputs_perm,inputsLBN),batch_size=10000,verbose=0)
else:
outputs_perm = self.model.predict(inputs_perm,batch_size=10000,verbose=0)
perm_f1_scores.append(f1_score(targets.argmax(1),outputs_perm.argmax(1),average='macro'))
perm_f1_scores = np.array(perm_f1_scores)
f1_scores[idxPerm] = abs(perm_f1_scores.mean()-true_F1_score)/true_F1_score
f1_scores_err[idxPerm] = perm_f1_scores.std()
idxSort = np.flip(np.argsort(f1_scores))
inputNames = np.array(self.inputs,dtype=object)[idxSort]
f1_scores = f1_scores[idxSort]
f1_scores_err = f1_scores_err[idxSort]
fig, ax = plt.subplots(figsize=(8,15))
plt.subplots_adjust(left=0.15, right=0.95, top=0.95, bottom=0.1)
y_pos = np.arange(len(self.inputs))
ax.barh(y_pos,f1_scores,xerr=f1_scores_err,align='center')
ax.set_yticks(y_pos)
ax.set_yticklabels(inputNames,size=int(80/math.sqrt(len(self.inputs))))
ax.invert_yaxis() # labels read top-to-bottom
ax.set_xlabel('Importance',fontsize=18)
name = os.path.join(self.path_out,'feature_importance.png')
fig.savefig(name)
logging.info("... saved as %s"%name)
def test_direct(self):
ctr = enlighten.Counter(stream=self.tty.stdout, total=100, desc='Test', unit='ticks')
self.assertIsInstance(ctr.manager, enlighten.Manager)
ctr.start = time.time() - 50
ctr.update(50, force=True)
self.tty.stdout.write('X\n')
value = self.tty.stdread.readline()
if enlighten.NEEDS_UNICODE_HELP:
value = value.decode('utf-8')
self.assertRegex(value, r'Test 50%\|' + u'█+[▏▎▍▌▋▊▉]?' +
r'[ ]+\| 50/100 \[00:5\d<00:5\d, \d.\d\d ticks/s\]X\n')
with mock.patch.object(self.tty, 'stdout', wraps=self.tty.stdout) as mockstdout:
ctr = enlighten.Counter(stream=self.tty.stdout, total=100, desc='Test', unit='ticks')
ctr.refresh(flush=False)
self.assertFalse(mockstdout.flush.called)
ctr.refresh(flush=True)
self.assertTrue(mockstdout.flush.called)
def test_custom_series(self):
ctr = enlighten.Counter(total=100, desc='Test', unit='ticks', series=[' ', '>', '-'])
ctr.count = 50
formatted = ctr.format(elapsed=50, width=80)
self.assertEqual(len(formatted), 80)
self.assertRegex(formatted, r'Test 50%\|' + u'-+[>]?' +
r'[ ]+\| 50/100 \[00:5\d<00:5\d, \d.\d\d ticks/s\]')
ctr.count = 13
formatted = ctr.format(elapsed=13, width=80)
self.assertEqual(len(formatted), 80)
self.assertRegex(formatted, r'Test 13%\|' + u'---->' +
r'[ ]+\| 13/100 \[00:1\d<01:\d\d, \d.\d\d ticks/s\]')
ctr = enlighten.Counter(total=100, desc='Test', unit='ticks', series=[u'⭘', u'⬤'])
ctr.count = 50
formatted = ctr.format(elapsed=50, width=80)
self.assertEqual(len(formatted), 80)
self.assertRegex(formatted, r'Test 50%\|' + u'⬤+⭘+' +
r'\| 50/100 \[00:5\d<00:5\d, \d.\d\d ticks/s\]')
def test_format_count_gt_total(self):
"""
Counter should fall back to no-total format if count is greater than total
"""
ctr = enlighten.Counter(total=10, desc='Test', unit='ticks')
ctr.count = 50
ctr.start = time.time() - 50
rtn = ctr.format(width=80)
self.assertEqual(len(rtn), 80)
self.assertRegex(rtn, r'Test 50 ticks \[00:5\d, \d.\d\d ticks/s\]')
def process_files(count=None):
"""
Process files with a single progress bar
"""
pbar = enlighten.Counter(total=count, desc='Simple', unit='ticks',
bar_format=BAR_FMT, counter_format=COUNTER_FMT)
for _ in range(100):
time.sleep(0.05)
pbar.update(1.1)
def fetch_urls(browser, max_scrolls, scroll_wait_time, logger):
"""Catch all the pictures links of the Instagram profile."""
links = []
links.extend(re.findall("/p/([^/]+)/", browser.page_source))
logger.info("Scrolling the Instagram, scraping pictures URLs ... ")
pbar = enlighten.Counter(total=max_scrolls,
desc='Scrolling',
unit='scrolls')
try:
prev_link_batch = None
scroll_count = 0
while scroll_count < max_scrolls:
page_source = browser.page_source
# check if browser has Loading icon, i.e. there is still new images to load
if not "Loading..." in page_source:
logger.info(
f"`Loading...` string not found, I suppose that the we reached the end of page. Stopping scrolling. (scroll count: {scroll_count})"
)
break
# scroll
browser.execute_script(
"window.scrollTo(0, document.body.scrollHeight)")
link_batch = re.findall("/p/([^/]+)/", page_source)
if link_batch == prev_link_batch:
logger.warning("Ineffective scroll")
else:
links.extend(link_batch)
scroll_count += 1
pbar.update()
prev_link_batch = link_batch
# sleep between scrolls
time.sleep(scroll_wait_time)
if max_scrolls == scroll_count:
logger.info(
f"Reached max_scrolls. Stop scrolling. (scroll count: {max_scrolls})"
)
except KeyboardInterrupt as e:
logger.warning(
'KeyboardInterrupt. Stopping scrolling and will continue with the rest.'
)
except Exception as e:
logger.warning('Error occured while scrolling: %s', e)
logger.warning(
"But we will continue with what we have scraped so far ;)")
logger.info("Pictures links collected")
return list(set(links)) # remove duplicates
def add_baitset(file, db_uri, email, panel_name, version, genomic_build):
LOG.info("saving baitset {}.{} to database".format(panel_name, version))
client = MongoClient(db_uri)
db_name = db_uri.split('/')[-1]
db = client[db_name]
adapter = PanelAdapter(client=client, db_name=db_name)
created_baitset_id = adapter.add_baitset(name=panel_name,
version=version,
build=genomic_build)
inserted_baits = 0
# if baitset creation was successful, insert baits
if created_baitset_id:
# obtain all baits from this baitset, formatted as objects
baits_list = baits(path_to_file=file,
created_baitset_id=created_baitset_id)
# insert one bait at the time into db
pbar = enlighten.Counter(total=len(baits_list), desc='', unit='ticks')
for bait in baits_list:
try:
# add the bait
inserted_bait_id = adapter.add_bait(bait)
inserted_baits += 1
except pymongo.errors.DuplicateKeyError:
# if bait exists do nothing
LOG.info('pymongo DuplicateKeyError')
finally:
# update baitset with list of baits contained in it
updated_baitset = db.baitset.find_one_and_update(
{'_id': created_baitset_id},
{'$push': {
'baits': bait['_id']
}},
upsert=True)
pbar.update()
LOG.info(
'created baitset with ID {0}. Inserted {1} out of {2} new available baits into db'
.format(created_baitset_id, inserted_baits, len(baits_list)))
else:
LOG.error(
"Something went wrong and the baitset coudn't be saved to db.")
def dump_collection(db, collection):
f = open(f'{db.name}({collection})-{datetime.now()}.json', 'w')
print(f'{db.name}({collection})-{datetime.now()}')
collection = db[collection]
pbar = enlighten.Counter(total=collection.find({}).count(),
desc='Basic',
unit='documents')
cursor = collection.find({})
for document in cursor:
f.write(f'{json.dumps(document, default=str, ensure_ascii=False)}\n')
pbar.update()
pbar.close()
def main(argv):
# Process args.
parser = argparse.ArgumentParser(
prog="check_dif_statuses",
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument(
"repo_top",
help=
"""Relative path to where the OpenTitan repository is checked out.""")
parser.add_argument(
"--show-unimplemented",
action="store_true",
help="""Show unimplemented functions for each incomplete DIF.""")
parser.add_argument("--table-format",
type=str,
choices=["grid", "github", "pipe"],
default="grid",
help="""Format to print status tables in.""")
args = parser.parse_args(argv)
# Define root path of DIFs.
difs_root_path = os.path.join("sw", "device", "lib", "dif")
# Get list of all DIF basenames.
# TODO: automatically get the list below by cross referencing DIF names
# with IP block names. Hardcoded for now.
shared_headers = ["dif_warn_unused_result"]
difs = get_list_of_difs(difs_root_path, shared_headers)
# Get DIF statuses (while displaying a progress bar).
dif_statuses = []
progress_bar = enlighten.Counter(total=len(difs),
desc="Analyzing statuses of DIFs ...",
unit="DIFs")
for dif in difs:
dif_statuses.append(DIFStatus(args.repo_top, difs_root_path, dif))
progress_bar.update()
# Build table and print it to STDOUT.
print_status_table(dif_statuses, args.table_format)
if args.show_unimplemented:
print_unimplemented_functions(dif_statuses, args.table_format)
def run_task_loop(sqs, sns):
first_attempt = True
notified = False
counter = enlighten.Counter(desc='Batches processed', unit='batches')
while True:
found_msgs = False # flag to check if any msgs were read in an attempt
for task_body, task_handle in get_tasks(sqs=sqs):
process(task_body)
delete_task(sqs, task_handle)
found_msgs = True
notified = False
first_attempt = False
if not found_msgs and not notified and not first_attempt:
# if no more tasks present, i.e a batch is completely processed, send notification
notify(sns=sns)
notified = True
counter.update()
time.sleep(0.1)
def _download_file(link, filename, chunk_size, progress):
with session.get(link, stream=True,
allow_redirects=True) as resp, open(filename,
"wb") as out:
resp.raise_for_status()
if resp.url != link:
logger.info(f"Original link: {link}")
logger.info(f"Resolved link: {resp.url}")
total_size = int(resp.headers.get("content-length", "0"))
pbar = enlighten.Counter(total=total_size,
desc=filename,
enabled=progress,
unit="B",
min_delta=0.5)
for chunk in resp.iter_content(chunk_size=chunk_size):
out.write(chunk)
pbar.update(len(chunk))
return total_size
def test_floats(self):
"""
Using floats for total and count is supported by the logic, but not by the
default format strings
"""
ctr = enlighten.Counter(total=100.2, desc='Test', unit='ticks', min_delta=500)
ctr.update(50.1)
self.assertEqual(ctr.count, 50.1)
# Won't work with default formatting
with self.assertRaises(ValueError):
formatted = ctr.format(elapsed=50.1)
ctr.bar_format = u'{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:.1f}/{total:.1f} ' + \
u'[{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]'
formatted = ctr.format(elapsed=50.1, width=80)
self.assertEqual(len(formatted), 80)
self.assertRegex(formatted, r'Test 50%\|' + u'█+' +
r'[ ]+\| 50.1/100.2 \[00:5\d<00:5\d, \d.\d\d ticks/s\]')
def ___find_averages___(self, data):
logmsg = 'evaluating average hero statistics.'
self.logger.debug(f'Starting: {logmsg}')
pbar = enlighten.Counter(total=len(data['values']), desc='Averages', unit='hero')
properties = playerOnly.__len__()
averages = [{ 'data': np.zeros(properties), 'amount' : 0 } for f in repeat(None, self.heroesArray.__len__())]
finalData = [np.zeros(properties) for f in repeat(None, self.heroesArray.__len__())]
for i in range(len(data['values'])):
heroId = data['heroes'][i]
if not (heroId is None):
entry = averages[heroId]
for k in range(properties):
entry['data'][k] += data['values'][i, k]
entry['amount']+=1
pbar.update()
for i in range(len(averages)):
entry = averages[i]
if entry['amount'] != 0:
for k in range(properties):
finalData[i][k] = entry['data'][k] / entry['amount']
self.logger.debug(f'Done: {logmsg}')
return finalData
