def Evaluate_upperbound_rdmuser(self):
F1s = 0
n_notselected_seq = 0
widgets = [' -- [ ',progressbar.Counter(), '|', str(self.dataset_size), ' ] ',
progressbar.Bar(), ' name: ', progressbar.FormatLabel(''),
' F1s: ', progressbar.FormatLabel(''),
' (', progressbar.ETA(), ' ) ']
pbar = progressbar.ProgressBar(max_value=self.dataset_size, widgets=widgets)
pbar.start()
#FIXME This process is problematic and needs update!
for video_idx, (s_name, s_groundtruth) in enumerate(zip(self.videonames, self.groundtruthscores)):
n_frames = s_groundtruth.shape[0]
n_users = s_groundtruth.shape[1]
select_user_id = np.random.choice(n_users)
pred_scores = s_groundtruth[:,select_user_id]
s_F1, _, _ = sum_tools.evaluate_summary(pred_scores, s_groundtruth.transpose(), self.eval_metrics)
F1s += s_F1
widgets[-6] = progressbar.FormatLabel('{:s}'.format(s_name))
widgets[-4] = progressbar.FormatLabel('{:.4f}'.format(s_F1))
pbar.update(video_idx)
# print(s_F1)
if n_notselected_seq > 0:
print("not selected sequence:{:d}".format(n_notselected_seq))
return F1s/self.dataset_size
def main(args):
global es, MAX_USERS
es = Elasticsearch(args.elasticsearch)
MAX_USERS = args.max_users
# Extraer usuarios del primer índice
print("Recuperando usuarios de /r/lonely...")
users = get_users(args.source_users)
# Buscar posibles usuarios "gemelos"
print("Obteniendo posibles gemelos...")
# Una barra de progreso que muestra el último usuario procesado además de la información habitual
widgets = [
pb.Percentage(), " (",
pb.SimpleProgress(), ") ",
pb.Bar(), " ",
pb.FormatLabel(""), " ",
pb.Timer(), " ",
pb.ETA(), " "
]
bar = pb.ProgressBar(max_value=len(users), widgets=widgets)
for username in bar(users):
widgets[6] = pb.FormatLabel("User: "******" ")
find_twins(username, users[username], args.user_index)
print("Filtrando usuarios que hayan posteado en el subreddit...")
filter_subreddit_posters(users)
# Se guarda el diccionario resultante en un .pickle, formato de serialización de Python
print("Serializando los resultados...")
with open(args.output, "wb") as f:
pickle.dump(users, f)
def processReplay(self, infile: Path):
# FIXME: Sinks need to support progress bar.
widgets = [
progressbar.FormatLabel(
f'Converting to {self.args.format.upper()}'),
progressbar.BouncingBar(),
progressbar.FormatLabel(' Elapsed: %(elapsed)s'),
]
with progressbar.ProgressBar(widgets=widgets) as progress:
print(f"[*] Converting '{infile}' to {self.args.format.upper()}")
outfile = self.prefix + infile.stem
sink, outfile = getSink(self.args.format,
outfile,
progress=lambda: progress.update(0))
if not sink:
print(
"The input file is already a replay file. Nothing to do.")
sys.exit(1)
fd = open(infile, "rb")
replay = Replay(fd, handler=sink)
print(f"\n[+] Succesfully wrote '{outfile}'")
sink.cleanup()
fd.close()
def write_to_csv(self):
if self.num_results > 0:
self.num_results = sum(1 for line in open(self.tmp_file, 'r'))
if self.num_results > 0:
timer = 0
widgets = [
'Write to csv ',
progressbar.Bar(left='[', marker='#', right=']'),
progressbar.FormatLabel(' [%(value)i/%(max)i] ['),
progressbar.Percentage(),
progressbar.FormatLabel('] [%(elapsed)s] ['),
progressbar.ETA(), '] [',
progressbar.FileTransferSpeed('lines'), ']'
]
bar = progressbar.ProgressBar(widgets=widgets,
maxval=self.num_results).start()
for line in gzip.open(self.tmp_file, 'r'):
timer += 1
bar.update(timer)
bar.finish()
else:
print('There is no docs with selected field(s): %s.' %
','.join(self.opts.fields))
self.tmp_file.close()
def translate_all_docs(self):
#tranlated_comments = [translate(comment, "es") for comment in comments]
total_comments = len(self.df["Comments"])
first_trasnlation = False
save_csv = 500
widgets = [
"Translating: ",
progressbar.Counter(), "/{}".format(total_comments),
progressbar.Percentage(), " ",
progressbar.Bar(), " ",
progressbar.AdaptiveETA(), " Saved comment: ",
progressbar.FormatLabel(' ')
]
pbar = progressbar.ProgressBar(maxval=total_comments,
widgets=widgets).start()
for i, comment in enumerate(self.df["Comments"]):
if self.df["Translated"][i] == "Untranslated":
first_trasnlation = True
#print("{}/{}".format(i+1,total))
#print(comment)
for num_t in range(self.num_translations):
comment = self.translate_single_doc(comment, self.language)
#print(comment_trans)
self.translated_comments[i] = comment
if i % save_csv == 0 and first_trasnlation:
self.save()
widgets[9] = progressbar.FormatLabel('{0}'.format(i))
pbar.update(i)
def write_to_csv(self):
if self.num_results > 0:
self.num_results = sum(1 for line in open(self.tmp_file, 'r'))
if self.num_results > 0:
output_file = open(self.opts.output_file, 'a')
csv_writer = csv.DictWriter(output_file, fieldnames=self.csv_headers, delimiter=self.opts.delimiter, quoting=csv.QUOTE_ALL)
csv_writer.writeheader()
timer = 0
widgets = ['Write to csv ',
progressbar.Bar(left='[', marker='#', right=']'),
progressbar.FormatLabel(' [%(value)i/%(max)i] ['),
progressbar.Percentage(),
progressbar.FormatLabel('] [%(elapsed)s] ['),
progressbar.ETA(), '] [',
progressbar.FileTransferSpeed(unit='lines'), ']'
]
bar = progressbar.ProgressBar(widgets=widgets, maxval=self.num_results).start()
for line in open(self.tmp_file, 'r'):
timer += 1
bar.update(timer)
line_as_dict = json.loads(line)
line_dict_utf8 = {k: v.encode('utf8') if isinstance(v, unicode) else v for k, v in line_as_dict.items()}
csv_writer.writerow(line_dict_utf8)
output_file.close()
bar.finish()
else:
print('There is no docs with selected field(s): %s.' % ','.join(self.opts.fields))
os.remove(self.tmp_file)
def write_to_csv(self):
if self.num_results > 0:
self.num_results = sum(1 for line in open(self.tmp_file, 'r'))
if self.num_results > 0:
self.csv_headers.sort()
output_file = open(self.opts.output_file, 'a')
csv_writer = csv.DictWriter(output_file, fieldnames=self.csv_headers, delimiter=self.opts.delimiter)
csv_writer.writeheader()
timer = 0
widgets = ['Write to csv ',
progressbar.Bar(left='[', marker='#', right=']'),
progressbar.FormatLabel(' [%(value)i/%(max)i] ['),
progressbar.Percentage(),
progressbar.FormatLabel('] [%(elapsed)s] ['),
progressbar.ETA(), '] [',
progressbar.FileTransferSpeed(), ']'
]
bar = progressbar.ProgressBar(widgets=widgets, maxval=self.num_results).start()
for line in open(self.tmp_file, 'r'):
timer += 1
bar.update(timer)
csv_writer.writerow(json.loads(line))
output_file.close()
bar.finish()
else:
print('There is no docs with selected field(s): %s.' % ','.join(self.opts.fields))
os.remove(self.tmp_file)
def _finished_widget(self, code):
"""
Just a helper function to say our our progress went.
"""
if code == 0:
return progressbar.FormatLabel('[OK]')
else:
return progressbar.FormatLabel('[FAILED]')
def train_model(self):
psnr, mse = 0.0, 0.0
data, label = self.read_data(self.dataset_path)
testdata, testlabel = self.read_test(self.testdataset_path)
widget = [
' [',
progressbar.Timer(),
'] ',
progressbar.Percentage(),
' (',
progressbar.SimpleProgress(),
') ',
' (',
progressbar.ETA(),
') ',
progressbar.FormatLabel(
str.format('psnr:{:.3f} mse:{:.5f}', psnr, mse)),
]
bar = progressbar.ProgressBar(widgets=widget,
maxval=self.epoch * len(data) //
self.batch_size)
bar.start()
for i in range(self.epoch):
batch_data, batch_label = self.get_batchDataAndLabel(
data, label, self.batch_size)
for j in range(len(batch_data)):
one_batch_data, one_batch_label = batch_data[j], batch_label[j]
self.session.run(self.train,
feed_dict={
self.data: one_batch_data,
self.label: one_batch_label
})
# print('epoch:', str(i), 'batch:', str(j), "process:", str.format("{:.3f}", j / len(batch_data) * 100),
# "%")
bar.update(i * len(data) // self.batch_size + j)
if j % 50 == 0 or j + 1 == len(batch_data):
result = self.session.run(self.merged,
feed_dict={
self.data: one_batch_data,
self.label: one_batch_label
}) # merged也是需要run的
self.writer.add_summary(result, i * len(batch_data) + j)
psnr, mse = self.testPSNR(testdata, testlabel)
widget[10] = progressbar.FormatLabel(
str.format('psnr:{:.3f} mse:{:.5f}', psnr, mse))
# print('psnr:',psnr,'mse:',mse)
result_mse = self.session.run(
self.test_merged_loss,
feed_dict={self.tep: np.array([mse])})
self.writer.add_summary(result_mse,
i * len(batch_data) + j)
result_psnr = self.session.run(
self.test_merged_psnr,
feed_dict={self.tep: np.array([psnr])})
self.writer.add_summary(result_psnr,
i * len(batch_data) + j)
bar.finish()
def download_data_by_time(self,
index,
td=timedelta(minutes=5),
time_from=None,
time_to=datetime.now(pytz.utc),
verbose=False):
if time_from is None:
time_from = time_to - td
page = self.es.search(index='{}.{}'.format(self.index_prefix, index),
scroll=self.scroll_time,
size=1000,
body={
"query": {
"range": {
"timestamp": {
"gte": time_from,
"lt": time_to,
}
}
}
},
sort='_id')
sid = page['_scroll_id']
scrolled = len(page['hits']['hits'])
scroll_size = page['hits']['total']
if verbose:
widgets = [
'Downloading {index}'.format(index=str(index)),
progressbar.Bar(left='[', marker='#', right=']'),
progressbar.FormatLabel(' [%(value)i/%(max)i] ['),
progressbar.Percentage(),
progressbar.FormatLabel('] [%(elapsed)s] ['),
progressbar.ETA(), '] [',
progressbar.FileTransferSpeed(unit='docs'), ']'
]
bar = progressbar.ProgressBar(widgets=widgets,
maxval=scroll_size).start()
bar.update(scrolled)
data = [item['_source'] for item in page['hits']['hits']]
df = pd.DataFrame(data).set_index('timestamp')
while (scrolled < scroll_size):
page = self.es.scroll(scroll_id=sid, scroll=self.scroll_time)
# Update the scroll ID
sid = page['_scroll_id']
# Get the number of results that we returned in the last scroll
scrolled += len(page['hits']['hits'])
if verbose:
bar.update(scrolled)
data = [item['_source'] for item in page['hits']['hits']]
df = df.append(pd.DataFrame(data).set_index('timestamp'))
df.index = pd.to_datetime(df.index, format='%Y-%m-%d %H:%M:%S.%f')
return df
def get_train_pbar(epoch):
widgets = [progressbar.FormatLabel(f'Epoch {epoch:3d} | Batch '),
progressbar.SimpleProgress(), ' | ',
progressbar.Percentage(), ' | ',
progressbar.FormatLabel(f'Loss N/A'), ' | ',
progressbar.Timer(), ' | ',
progressbar.ETA()]
return progressbar.ProgressBar(widgets=widgets, fd=sys.stdout)
def __init__(self,
name,
max_value=100,
history_len=5,
display=True,
display_data={
'train': ['loss', 'accuracy'],
'test': ['loss', 'accuracy']
},
level=logging.INFO,
train_log_mode='TRAIN_PROGRESS',
test_log_mode='TEST_PROGRESS'):
super(ProgressbarLogger, self).__init__(name,
level=level,
display=display,
logfile=None,
train_log_mode=train_log_mode,
test_log_mode=test_log_mode)
self.train_log_data = {}
self.test_log_data = {}
self.max_value = max_value
self.history_len = history_len
self.display_data = display_data
self.mode['TRAIN_PROGRESS'] = self.log_train_progress
self.mode['TEST_PROGRESS'] = self.log_test_progress
# create logging format
self.widgets = [
progressbar.FormatLabel('(%(value)d of %(max)s)'), ' ',
progressbar.Percentage(), ' ',
progressbar.Bar()
]
self.dynamic_data = {
k + '_' + kk: 0.0
for k in display_data.keys() for kk in display_data[k]
}
diff_data = {
'diff_' + k + '_' + kk: 0.0
for k in display_data.keys() for kk in display_data[k]
}
self.dynamic_data.update(diff_data)
for t in display_data.keys():
ddstr = ' [' + t + ']'
for s in display_data[t]:
value_name = t + '_' + s
ddstr = ddstr + ' ' + s + ':' + '%(' + value_name + ').3f (%(diff_' + value_name + ').3f)'
self.widgets.append(progressbar.FormatLabel(ddstr))
self.widgets.extend([
'|',
progressbar.FormatLabel('Time: %(elapsed)s'), '|',
progressbar.AdaptiveETA()
])
def format_label():
widgets = [
progressbar.FormatLabel('Processed: %(value)d lines (in: %(elapsed)s)')
]
bar = progressbar.ProgressBar(widgets=widgets)
for i in bar((i for i in range(15))):
time.sleep(0.1)
def buildDT(s):
s.addCol("DTrho")
from scipy.spatial import Delaunay
tmp=np.zeros((s.n,2))
tmp[:,0]=s.data['y']
tmp[:,1]=s.data['x']
d=Delaunay(tmp)
tmp=np.zeros(s.n)
m=np.median(s.mass)
#For searching
flat=d.vertices.flatten()
widgets = [progressbar.Percentage(), progressbar.Bar(),' ',progressbar.FormatLabel('Time elapsed: %(elapsed)s'),' ',progressbar.ETA()]
pbar = progressbar.ProgressBar(widgets=widgets, maxval=s.n).start()
#For each point, need to calculate the contiguous Voronoi cell...
for i in xrange(s.n):
pbar.update(i)
pts=np.where(flat==i)[0]/3
#Calculate the area of each, add it to total
area=0.
for pt in pts:
nodes=d.points[d.vertices[pt]]
a=nodes[0]
b=nodes[1]
c=nodes[2]
area=area+np.abs((a[0]-c[0])*(b[1]-a[1])-(a[0]-b[0])*(c[1]-a[1]))/2.
tmp[i]=(3.*m)/area
s.data['DTrho']=tmp
def interpolateMap(s,grid=False):
"""Interpolates the surface density to surface using either the grid or
the 10% column calculation."""
s.addCol("cdTotInt")
#Create it if it hasn't been...
if not grid and not "cdTot" in s.data.dtype.names:
s.seedMap()
if grid and not hasattr(s,"gridArray"):
s.gridMap()
#Now we have to interpolate the missing points...
rr=s.subset
if rr is None:
rr=np.arange(s.n)
#Again, far faster to use a different data structure...
tmp=np.zeros((rr.shape[0],3))
tmp[:,0]=s.data['x'][rr]
tmp[:,1]=s.data['y'][rr]
widgets = [progressbar.Percentage(), progressbar.Bar(),' ',progressbar.FormatLabel('Time elapsed: %(elapsed)s'),' ',progressbar.ETA()]
pbar = progressbar.ProgressBar(widgets=widgets, maxval=s.N).start()
if grid:
gtree=sp.KDTree(s.gridArray[:,:2])
tt=s.gridArray[:,2]
else:
src=np.isfinite(s.data['cdTot'])
gtree=sp.KDTree(np.column_stack((s.data['x'][src],s.data['y'][src])))
tt=s.data['cdTot'][src]
for i in xrange(rr.shape[0]):
pbar.update(i)
pts=gtree.query(tmp[i,:2],1)
tmp[i,2]=tt[pts[1]]
s.data['cdTotInt'][rr]=tmp[:,2]
def seedMap(s,subset=False,nn=50,subsetFrac=.1,maxR=5.):
"""Calculate the column density at a subset of particles. If subset is
set to true then we calculate it at the points specified in subset,
otherwise a random set is chosen. nn is the number of neighbours to
use to calculate the surface density at each point and subsetFrac is
the fraction of total particles to calculate it for."""
s.init_tree()
if subset and s.subset is not None:
npts=s.N
rr=s.subset
else:
npts=int(s.n*subsetFrac)
rr=np.random.permutation(s.n)[:npts]
s.addCol("cdTot")
#Make some temporary lists to make calculation (a lot) faster
ghost=np.zeros((npts,3))
ghost[:,0]=s.data['x'][rr]
ghost[:,1]=s.data['y'][rr]
mm=s.data['m']
widgets = [progressbar.Percentage(), progressbar.Bar(),' ',progressbar.FormatLabel('Time elapsed: %(elapsed)s'),' ',progressbar.ETA()]
pbar = progressbar.ProgressBar(widgets=widgets, maxval=npts).start()
for i in xrange(npts):
pbar.update(i)
pts=s.tree2.query(ghost[i,:2],nn)
ghost[i,2]=np.sum(mm[pts[1]]/(2.*math.pi*np.max(pts[0])**2))
s.data['cdTot'][rr]=ghost[:,2]
def calcPotential(s):
"""Calculate the potential energy for a subset of particles..."""
s.addCol("Pot")
widgets = [
progressbar.Percentage(),
progressbar.Bar(), ' ',
progressbar.FormatLabel('Time elapsed: %(elapsed)s'), ' ',
progressbar.ETA()
]
pbar = progressbar.ProgressBar(widgets=widgets, maxval=s.N).start()
x = s.fetch(s.data['x'])
y = s.fetch(s.data['y'])
z = s.fetch(s.data['z'])
xall = s.data['x']
yall = s.data['y']
zall = s.data['z']
pot = np.zeros(s.N)
mass = np.median(s.data['m'])
for i in xrange(s.N):
pbar.update(i)
tmp = np.sqrt((xall - x[i])**2 + (yall - y[i])**2 +
(zall - z[i])**2)
tmp[tmp == 0] = np.inf
pot[i] = -mass * s.G * np.sum(1. / tmp)
s.data["Pot"][s.subset] = pot
def __zone_clear_tags(self,
zone_ids,
batch_size=100,
show_progressbar=False,
widget=None):
batches = list(self.__chunks(zone_ids, batch_size))
if show_progressbar:
widget[0] = progressbar.FormatLabel('Removing existing tags')
bar = progressbar.ProgressBar(max_value=len(batches),
widgets=widget)
count = 0
for batch in batches:
count += 1
bar.update(count) if show_progressbar else False
i = 0
params = {}
for id in batch:
i += 1
params['param' + str(i)] = id
bind = [':' + v for v in params.keys()]
sql = "DELETE FROM dns_zone_tags WHERE dns_zone_id IN({0})".format(
', '.join(bind))
db.session.execute(sql, params)
db.session.commit()
return True
def test_all_widgets_large_values(max_value):
widgets = [
progressbar.Timer(),
progressbar.ETA(),
progressbar.AdaptiveETA(),
progressbar.AbsoluteETA(),
progressbar.DataSize(),
progressbar.FileTransferSpeed(),
progressbar.AdaptiveTransferSpeed(),
progressbar.AnimatedMarker(),
progressbar.Counter(),
progressbar.Percentage(),
progressbar.FormatLabel('%(value)d/%(max_value)d'),
progressbar.SimpleProgress(),
progressbar.Bar(fill=lambda progress, data, width: '#'),
progressbar.ReverseBar(),
progressbar.BouncingBar(),
progressbar.FormatCustomText('Custom %(text)s', dict(text='text')),
]
p = progressbar.ProgressBar(widgets=widgets, max_value=max_value)
p.update()
time.sleep(1)
p.update()
for i in range(0, 10**6, 10**4):
time.sleep(1)
p.update(i)
def _common_progress_bar(text, unit, tot_size=None):
if tot_size:
widgets = [
progressbar.FormatLabel(f"{text}: %(value)d of %(max_value)d {unit} "),
progressbar.Percentage(),
progressbar.Bar(),
]
return progressbar.ProgressBar(widgets=widgets, max_value=tot_size)
else:
widgets = [
progressbar.FormatLabel(f"{text}: %(value)d {unit}"),
progressbar.RotatingMarker(),
]
return progressbar.ProgressBar(
widgets=widgets, max_value=progressbar.UnknownLength
)
def format_label_rotating_bouncer():
widgets = [progressbar.FormatLabel('Animated Bouncer: value %(value)d - '),
progressbar.BouncingBar(marker=progressbar.RotatingMarker())]
bar = progressbar.ProgressBar(widgets=widgets)
for i in bar((i for i in range(18))):
time.sleep(0.1)
def processReplay(self, infile: Path):
widgets = [
progressbar.FormatLabel('Encoding MP4 '),
progressbar.BouncingBar(),
progressbar.FormatLabel(' Elapsed: %(elapsed)s'),
]
with progressbar.ProgressBar(widgets=widgets) as progress:
print(f"[*] Converting '{infile}' to MP4.")
outfile = self.prefix + infile.stem + '.mp4'
sink = Mp4EventHandler(outfile, progress=lambda: progress.update(0))
fd = open(infile, "rb")
replay = Replay(fd, handler=sink)
print(f"\n[+] Succesfully wrote '{outfile}'")
sink.cleanup()
fd.close()
def step(self, pbar):
"""Make a single gradient update.
This is called by train()
"""
self.model.train()
self.train_evaluator.reset()
for data, targets in pbar(self.train_loader):
data, targets = data.numpy(), targets.numpy()
targets = onehot(targets, self.num_classes)
# Forward pass, compute loss
scores = self.model.forward(data)
loss = self.loss.forward(scores, targets)
# Backward pass, compute gradient
self.optimizer.zero_grad()
delta = self.loss.backward(scores, targets)
self.model.backward(delta)
# Take step with optimizer
self.optimizer.step()
# Update evaluator
self.train_evaluator.update(scores, targets, loss)
pbar.widgets[5] = progressbar.FormatLabel(
f'Loss (E/B) {self.train_evaluator.loss:4.2f} / {loss.mean():4.2f}'
)
def test_all_widgets_max_width(max_width, term_width):
widgets = [
progressbar.Timer(max_width=max_width),
progressbar.ETA(max_width=max_width),
progressbar.AdaptiveETA(max_width=max_width),
progressbar.AbsoluteETA(max_width=max_width),
progressbar.DataSize(max_width=max_width),
progressbar.FileTransferSpeed(max_width=max_width),
progressbar.AdaptiveTransferSpeed(max_width=max_width),
progressbar.AnimatedMarker(max_width=max_width),
progressbar.Counter(max_width=max_width),
progressbar.Percentage(max_width=max_width),
progressbar.FormatLabel('%(value)d', max_width=max_width),
progressbar.SimpleProgress(max_width=max_width),
progressbar.Bar(max_width=max_width),
progressbar.ReverseBar(max_width=max_width),
progressbar.BouncingBar(max_width=max_width),
progressbar.FormatCustomText('Custom %(text)s',
dict(text='text'),
max_width=max_width),
progressbar.DynamicMessage('custom', max_width=max_width),
progressbar.CurrentTime(max_width=max_width),
]
p = progressbar.ProgressBar(widgets=widgets, term_width=term_width)
p.update(0)
p.update()
for widget in p._format_widgets():
if max_width and max_width < term_width:
assert widget == ''
else:
assert widget != ''
def test_all_widgets_small_values(max_value):
widgets = [
progressbar.Timer(),
progressbar.ETA(),
progressbar.AdaptiveETA(),
progressbar.AbsoluteETA(),
progressbar.DataSize(),
progressbar.FileTransferSpeed(),
progressbar.AdaptiveTransferSpeed(),
progressbar.AnimatedMarker(),
progressbar.Counter(),
progressbar.Percentage(),
progressbar.FormatLabel('%(value)d'),
progressbar.SimpleProgress(),
progressbar.Bar(),
progressbar.ReverseBar(),
progressbar.BouncingBar(),
progressbar.CurrentTime(),
progressbar.CurrentTime(microseconds=False),
progressbar.CurrentTime(microseconds=True),
]
p = progressbar.ProgressBar(widgets=widgets, max_value=max_value)
for i in range(10):
time.sleep(1)
p.update(i + 1)
p.finish()
def ProgressIndicator(label, maxval=1000000):
widgets = [
pb.FormatLabel(label),
pb.BouncingBar(marker=pb.RotatingMarker())
]
pind = pb.ProgressBar(widgets=widgets, maxval=maxval).start()
return pind
def upload(self,
locales=None,
force=True,
verbose=False,
save_stats=True,
download=False):
source_lang = self.pod.podspec.default_locale
locales = locales or self.pod.catalogs.list_locales()
stats = []
num_files = len(locales)
if not locales:
self.pod.logger.info('No locales to upload.')
return
if download and self.pod.file_exists(Translator.TRANSLATOR_STATS_PATH):
self.download(locales=locales, save_stats=save_stats)
if not force:
if (self.has_immutable_translation_resources and
self.pod.file_exists(Translator.TRANSLATOR_STATS_PATH)):
text = 'Found existing translator data in: {}'
self.pod.logger.info(
text.format(Translator.TRANSLATOR_STATS_PATH))
text = 'This will be updated with new data after the upload is complete.'
self.pod.logger.info(text)
text = 'Proceed to upload {} translation catalogs?'
text = text.format(num_files)
if not utils.interactive_confirm(text):
self.pod.logger.info('Aborted.')
return
text = 'Uploading translations: %(value)d/{} (in %(elapsed)s)'
widgets = [progressbar.FormatLabel(text.format(num_files))]
bar = progressbar.ProgressBar(widgets=widgets, maxval=num_files)
bar.start()
threads = []
def _do_upload(locale):
catalog = self.pod.catalogs.get(locale)
stat = self._upload_catalog(catalog, source_lang)
stats.append(stat)
for i, locale in enumerate(locales):
thread = utils.ProgressBarThread(bar,
True,
target=_do_upload,
args=(locale, ))
threads.append(thread)
thread.start()
# Perform the first operation synchronously to avoid oauth2 refresh
# locking issues.
if i == 0:
thread.join()
for i, thread in enumerate(threads):
if i > 0:
thread.join()
bar.finish()
stats = sorted(stats, key=lambda stat: stat.lang)
if verbose:
self.pretty_print_stats(stats)
if save_stats:
self.save_stats(stats)
return stats
def calcsumgz(self,recalculate=False):
"""Calculates direct sum gravity and inserts it into the table"""
s=self
x=s.data['x']
y=s.data['y']
z=s.data['z']
ptx=self.fetch(x)
pty=self.fetch(y)
ptz=self.fetch(z)
#If we've calculated the values previously, no need to redo them...
index=self.fetch(np.arange(self.n))
s.addCol('sumgz')
if not recalculate:
o=np.where(np.isnan(self.data['sumgz'][index]))
ptx=ptx[o]
pty=pty[o]
ptz=ptz[o]
index=index[o]
widgets = [progressbar.Percentage(), progressbar.Bar(),' ',progressbar.FormatLabel('Time elapsed: %(elapsed)s'),' ',progressbar.ETA()]
pbar = progressbar.ProgressBar(widgets=widgets, maxval=len(index)).start()
for count,i in enumerate(index):
rtmp=np.sqrt((ptx[count]-s.data['x'])**2+(pty[count]-s.data['y'])**2+(ptz[count]-s.data['z'])**2)
rtmp[rtmp==0]=np.Infinity
s.data['sumgz'][i]=np.sum((s.G*s.data['m']*(s.data['z']-ptz[count]))/rtmp**3)
pbar.update(count)
def dispatch_ip_tick_hook(
self,
ipaddr,
node, # pylint: disable=w0613
message,
timestamp, # pylint: disable=w0613
duration): # pylint: disable=w0613
# start progressbar
if self.pbar is None:
widgets = [
'Collecting image : ',
# progressbar.BouncingBar(marker=progressbar.RotatingMarker()),
progressbar.BouncingBar(marker='*'),
progressbar.FormatLabel(' %(seconds).2fs'),
]
self.pbar = \
progressbar.ProgressBar(widgets=widgets,
maxval=progressbar.UnknownLength)
self.pbar.start()
# progressbar is willing to work as expected here
# with maxval=UnknownLength
# but still insists to get a real value apparently
self.value = 0
self.value += 1
self.pbar.update(self.value)
# hack way to finish the progressbar
# since we have no other way to figure it out
if message['tick'] == 'END':
self.pbar.finish()
def load(fileName, linesToLoad=sys.maxsize, verbose=True):
import progressbar
fileName = os.path.expanduser(fileName)
content = []
i = 0
widgets = [
progressbar.Bar('>'), ' ',
progressbar.ETA(),
progressbar.FormatLabel('; Total: %(value)d sents (in: %(elapsed)s)')
]
with open(fileName) as file:
if verbose is True:
loadProgressBar =\
progressbar.ProgressBar(widgets=widgets,
maxval=min(
sum(1 for line in file),
linesToLoad)).start()
file.seek(0)
for line in file:
i += 1
if verbose is True:
loadProgressBar.update(i)
content.append(constructTreeFromStr(line))
if i == linesToLoad:
break
if verbose is True:
loadProgressBar.finish()
return content
