def get_progress_bar(max_value=100):
widgets = [Percentage(), ' (', SimpleProgress(), ')', Bar(), ETA()]
bar = ProgressBar(max_value=max_value,
redirect_stdout=True,
widgets=widgets)
bar.start()
return bar
def _interpolate_epochs(epochs, bad_log):
"""Interpolate channels epochwise.
Parameters
----------
epochs : instance of mne.Epochs
The epochs to be interpolated.
bad_log : array, shape (n_epochs, n_channels)
"""
from utils import interpolate_bads
from progressbar import ProgressBar, SimpleProgress
if len(epochs) != bad_log.shape[0]:
raise ValueError('Shape of bad_log and epochs do not match')
if len(epochs.info['ch_names']) != bad_log.shape[1]:
raise ValueError('Shape of bad_log and ch_names do not match')
pbar = ProgressBar(widgets=[SimpleProgress()])
# XXX: last epoch may be smaller than window size
print('Repairing epochs: ')
for epoch_idx in pbar(range(len(epochs))):
bad_idx = np.where(bad_log[epoch_idx] == 1)[0]
bad_chs = [epochs.info['ch_names'][p] for p in bad_idx]
epoch = epochs[epoch_idx].copy()
epoch.info['bads'] = bad_chs
interpolate_bads(epoch, reset_bads=True)
epochs._data[epoch_idx] = epoch._data
def __init__(self, analysis, sample, **kwargs):
# default to access via sample/analysis
self.analysis = analysis
self.sample = sample
self.shift = kwargs.pop('shift', '')
logging.debug('Initializing {0} {1} {2}'.format(
self.analysis, self.sample, self.shift))
# backup passing custom parameters
self.ntupleDirectory = kwargs.pop(
'ntupleDirectory', '{0}/{1}'.format(
getNtupleDirectory(self.analysis, shift=self.shift),
self.sample))
self.inputFileList = kwargs.pop('inputFileList', '')
self.outputFile = kwargs.pop('outputFile', '')
self.json = kwargs.pop('json', getSkimJson(self.analysis, self.sample))
self.pickle = kwargs.pop('pickle',
getSkimPickle(self.analysis, self.sample))
self.treeName = kwargs.pop('treeName', getTreeName(self.analysis))
if hasProgress:
self.pbar = kwargs.pop(
'progressbar',
ProgressBar(widgets=[
'{0}: '.format(sample), ' ',
SimpleProgress(), ' events ',
Percentage(), ' ',
Bar(), ' ',
ETA()
]))
else:
self.pbar = None
# get stuff needed to flatten
self.infile = 0
self.tchain = 0
self.initialized = False
self.counts = {}
def tag_json():
"""
Tags all the files in the JSON directory
"""
records = db_util.get_all_records()
pbar = ProgressBar(widgets=[SimpleProgress()], maxval=len(records)).start()
for idx, record in enumerate(records):
record = db_util.open_NASARecord(record)
# it's OK to concatenate all the text since we're using a
# bag of words approach
record_words = set() # set of all the words & phrases in the record
for i in range(1, tag_analyzer.MAX_PHRASE_LENGTH + 1):
word_hash = {}
tag_analyzer.hash_string(record, i, word_hash)
record_words = record_words.union(set(word_hash.keys()))
# now, you have a hash of all the word combinations
#print "Description:"
#print record.description
#print record_words
found_tags = list(canonical_set.intersection(record_words))
#print "\n------------------------------------------------"
#print "Tags found:"
indicies = [tag_index[canonical_tags[x]] for x in found_tags]
record.tags = indicies
record.save()
# TODO: save back into the file
# TODO: create a tag_list.txt file which has all the possible tags
# in a list, so that you don't have to write the whole string into
# the file
pbar.update(idx + 1)
pbar.finish()
def __init__(self, name="", total=UnknownLength, period=1, verbose=3):
"""
Args:
verbose: 0: no output, do nothing
1: only output in start
2: only output in start and end
3: output in progresss
"""
self.verbose = verbose
self.period = period
if verbose >= 2:
widgets = ["...", name]
if total is not UnknownLength:
widgets.extend([
" ",
Bar(), " ",
SimpleProgress("/"), " (",
Percentage(), ") ",
PreciseETA()
])
else:
widgets.extend([" (", PreciseTimer(), ")"])
self.pbar = ProgressBar(widgets=widgets, maxval=total)
elif verbose >= 1:
print("..." + name)
def example5():
pbar = ProgressBar(widgets=[SimpleProgress()], maxval=17).start()
for i in range(17):
time.sleep(0.2)
pbar.update(i + 1)
pbar.finish()
sys.stdout.write('\n')
def migrate(apps, schema_editor, direction):
pbar_widgets = [
SimpleProgress(), ' ',
Percentage(), ' ',
Bar(), ' ',
Timer(), ' ',
AdaptiveETA()
]
for model in ['Comment', 'Newsitem']:
queryset = globals()[model].objects.filter(text__isnull=True)
if queryset.count() == 0:
continue
print "Processing " + model
pbar = ProgressBar(widgets=pbar_widgets, maxval=queryset.count())
done = 0
pbar.start()
tokenizer = RegexpTokenizer(r'\w+')
for item in queryset_iterator(queryset, chunksize=20):
if item.content:
if direction == "forward":
item.text = Text.objects.create(
text=item.content,
wordcount=len(tokenizer.tokenize(item.content)))
elif direction == "backwards":
item.content = item.text.text
item.save()
done += 1
pbar.update(done)
pbar.finish()
def __init__(self, maxval, alternate=False):
self.done = 0
if alternate:
pbar_widgets = [
FormatLabel(
'Processed: %(value)d/%(max)d items (in: %(elapsed)s)'),
' -=- ',
Percentage(), ' -=- ',
ETA()
]
self.pbar = ProgressBar(widgets=pbar_widgets,
maxval=maxval,
endline_character="\n")
else:
pbar_widgets = [
SimpleProgress(), ' ',
Percentage(), ' ',
Bar(), ' ',
Timer(), ' ',
AdaptiveETA()
]
self.pbar = ProgressBar(widgets=pbar_widgets, maxval=maxval)
if alternate:
self.pbar.update_interval = maxval / 50
self.pbar.start()
def doSearch(output='names', **kwargs):
r = doSearchFlex(**kwargs)
if len(re.findall("jquery.js", r.text.split('</script>')[0])) > 0:
return getCard(str(re.findall("cardid=(\d+)", r.text)[0]))
try:
maxpages = int(
re.findall('changepage\(\$\(this\)\);}">of (\d+)', r.text)[0])
except:
maxpages = 1
allresults = []
print "query returned %s pages." % str(maxpages)
pbar = ProgressBar(widgets=[SimpleProgress()], maxval=maxpages).start()
for page in range(0, maxpages):
pbar.update(page + 1)
for each in doSearchByPage(page=str(page + 1), **kwargs):
if output == "names":
allresults.append(str(each[0]).replace('•', '-'))
elif output == "ids":
allresults.append(
[str(each[0]).replace('•', '-'),
str(each[1])])
elif output == "info":
allresults.append(getCard(each[1]))
pbar.finish()
return allresults
def progress_bar_widgets(unit='items'):
return [
SimpleProgress(sep='/'), ' ',
Bar(), ' ',
FileTransferSpeed(unit=unit), ', ',
AdaptiveETA()
]
def _progress(iterable):
if ProgressBar:
pbar = ProgressBar(
widgets=[SimpleProgress(), Bar(), ETA()])
else:
pbar = iter
return pbar(iterable)
def flatten(analysis, sample, **kwargs):
inputFileList = kwargs.pop('inputFileList', '')
outputFile = kwargs.pop('outputFile', '')
shift = kwargs.pop('shift', '')
njobs = kwargs.pop('njobs', 1)
job = kwargs.pop('job', 0)
multi = kwargs.pop('multi', False)
if hasProgress:
pbar = kwargs.pop(
'progressbar',
ProgressBar(widgets=[
'{0}: '.format(sample), ' ',
SimpleProgress(), ' ',
Percentage(), ' ',
Bar(), ' ',
ETA()
]))
else:
pbar = None
if outputFile:
flattener = flatteners[analysis](sample,
inputFileList=inputFileList,
outputFile=outputFile,
shift=shift,
progressbar=pbar)
else:
flattener = flatteners[analysis](sample,
inputFileList=inputFileList,
shift=shift,
progressbar=pbar)
flattener.flatten()
def validate_links(data):
widgets = [Bar(), SimpleProgress()]
pbar = ProgressBar(widgets=widgets, maxval=len(data)).start()
for i, element in enumerate(data):
url = element['url']
if url == '':
continue
scheme = urlparse.urlsplit(url).scheme
host = urlparse.urlsplit(url).netloc
if scheme in ('http', 'https') and \
url_status_cache.get(url) is not True:
try:
request = head(url, timeout=10)
# some web sites cannot into head requests
if request.status_code in (403, 405, 500) or \
host in ('mobil.morgenpost.de'):
request = get(url)
except Timeout as e:
stderr.write('Connection to <%s> timeouted.\n' % url)
exit(1)
except ConnectionError as e:
stderr.write('Connection to <%s> failed.\n' % url)
stderr.write(str(e) + '\n')
exit(1)
if request.ok:
url_status_cache.set(url, request.ok)
else:
stderr.write('<%s> is unreachable.\n' % url)
exit(1)
pbar.update(i + 1)
def create_progress_bar(self):
"""
Initialize the progress bar
"""
if self.total is None:
self.total = progressbar.UnknownLength
widgets = [
DynamicStringMessage('imagename'), ' ',
Counter(), ' ',
BouncingBar(marker=u'\u2588', left=u'\u2502', right=u'\u2502'),
' ',
Timer()
]
else:
widgets = [
DynamicStringMessage('imagename'), ' ',
Percentage(), ' (',
SimpleProgress(), ')', ' ',
Bar(marker=u'\u2588', left=u'\u2502', right=u'\u2502'), ' ',
Timer()
]
self.progress_bar = progressbar.ProgressBar(max_value=self.total,
redirect_stdout=True,
redirect_stderr=True,
widgets=widgets)
self.progress_bar.start()
self.progress_bar.update(self.step, imagename=self.imagename)
def average_time_series(df):
import pandas as pd
from progressbar import ProgressBar,Percentage,Bar,ETA,SimpleProgress
averaged_df = pd.DataFrame(columns = ['x','y','u','v','w',
'u_rms','v_rms','w_rms'])
grouped_df = df.groupby(['x','y'])
progress = ProgressBar(
widgets=[
Bar(),' ',
Percentage(),' ',
ETA(), ' (file ',
SimpleProgress(),')'],
maxval=len(grouped_df)
).start()
cnt = 0
for (x, y), data in grouped_df:
result = {
'x':x,
'y':y,
}
for v in ['u','v','w']:
result[v] = data[v].mean()
result[v+'_rms'] = data[v].std()
averaged_df = averaged_df.append( pd.DataFrame( result , index = [0]),
ignore_index = True)
cnt += 1
progress.update(cnt)
progress.finish()
return averaged_df
def __init__(self, title, end=100):
widgets = [
title + ': ',
Percentage(), ' ',
Bar(marker='#', left='[', right=']'), ' ',
SimpleProgress()
]
self.pbar = ProgressBar(widgets=widgets, maxval=end).start()
def init_widgets(counter, t):
return [
'Progress: ',
SimpleProgress('/') if counter else Percentage(), ' ',
Bar(marker='>'), ' ',
ETA(), ' ',
FileTransferSpeed() if t is None else EventSpeed(t)
]
def get_ged_plus_scores(decomposition_graphs,
gold_graphs,
exclude_thr=None,
debug=False,
num_processes=5):
samples = list(zip(decomposition_graphs, gold_graphs))
pool = Pool(num_processes)
pbar = ProgressBar(widgets=[SimpleProgress()], maxval=len(samples)).start()
results = []
_ = [
pool.apply_async(get_ged_plus_score,
args=(i, samples[i][0], samples[i][1], exclude_thr,
debug),
callback=results.append) for i in range(len(samples))
]
while len(results) < len(samples):
pbar.update(len(results))
pbar.finish()
pool.close()
pool.join()
edit_op_counts = {
"insertion": 0,
"deletion": 0,
"substitution": 0,
"merging": 0,
"splitting": 0
}
idxs, scores_tmp, num_paths, num_ops, ops_ratio = [], [], [], [], []
for result in results:
idx, curr_edit_path, curr_cost, curr_num_paths, curr_num_ops, curr_ops_ratio, curr_edit_op_counts = result
idxs.append(idx)
scores_tmp.append(curr_cost)
if not curr_cost:
continue
num_paths.append(float(curr_num_paths))
num_ops.append(float(curr_num_ops))
if debug:
ops_ratio.append(curr_ops_ratio)
for op in curr_edit_op_counts:
edit_op_counts[op] += curr_edit_op_counts[op]
scores = [score for (idx, score) in sorted(zip(idxs, scores_tmp))]
print("edit op statistics:", edit_op_counts)
print("number of explored paths: mean {:.2}, min {:.2}, max {:.2}".format(
np.mean(num_paths), np.min(num_paths), np.max(num_paths)))
print("number of edit ops: mean {:.2}, min {:.2}, max {:.2}".format(
np.mean(num_ops), np.min(num_ops), np.max(num_ops)))
if debug:
print("explored ops ratio: mean {:.2}, min {:.2}, max {:.2}".format(
np.mean(ops_ratio), np.min(ops_ratio), np.max(ops_ratio)))
return scores
def begin(self, tests):
from progressbar import ProgressBar, Percentage, ETA, SimpleProgress
widgets = ['[', Percentage(), '] ', SimpleProgress(), ' ', ETA()]
self.counter = 0
self.progress = ProgressBar(maxval=len(tests), widgets=widgets)
if tests:
self.progress.start()
self.passes = []
self.failures = []
def get_progress_bar(num_reads):
bar_format = [
RotatingMarker(), " ",
SimpleProgress(),
Bar(),
Percentage(), " ",
ETA()
]
return ProgressBar(maxval=num_reads, widgets=bar_format).start()
def fixed_samples_prog(n_samples):
spike_counter = spike_counter_widget()
pbar = ProgressBar(widgets=[
SimpleProgress(), " (",
Percentage(), " )", " samples processed. ",
Bar(">"), " ", spike_counter, " spikes found."
],
maxval=n_samples).start()
return pbar, spike_counter
def console_scan_loop(scanners, scan_titles, verbose):
""" Scan all the AsyncScanner object printing status to console.
Inputs:
- scanners -- List of AsyncScanner objects to scan.
- scan_titles -- List of string with the names of the world/regionsets in the same
order as in scanners.
- verbose -- Boolean, if true it will print a line per scanned region file.
"""
try:
for scanner, title in zip(scanners, scan_titles):
print("\n{0:-^60}".format(title))
if not len(scanner):
print("Info: No files to scan.")
else:
total = len(scanner)
if not verbose:
pbar = ProgressBar(
widgets=[SimpleProgress(),
Bar(), AdaptiveETA()],
maxval=total).start()
try:
scanner.scan()
counter = 0
while not scanner.finished:
scanner.sleep()
result = scanner.get_last_result()
if result:
logging.debug(
"\nNew result: {0}\n\nOneliner: {1}\n".format(
result, result.oneliner_status))
counter += 1
if not verbose:
pbar.update(counter)
else:
status = "(" + result.oneliner_status + ")"
fn = result.filename
fol = result.folder
print(
"Scanned {0: <12} {1:.<43} {2}/{3}".format(
join(fol, fn), status, counter, total))
if not verbose:
pbar.finish()
except KeyboardInterrupt as e:
# If not, dead processes will accumulate in windows
scanner.terminate()
raise e
except ChildProcessException as e:
# print "\n\nSomething went really wrong scanning a file."
# print ("This is probably a bug! If you have the time, please report "
# "it to the region-fixer github or in the region fixer post "
# "in minecraft forums")
# print e.printable_traceback
raise e
def __init__(self, cursor, conn):
xml.sax.handler.ContentHandler.__init__(self)
self._db_cursor = cursor
self._db_conn = conn
self._count = 0
self._pbar = ProgressBar(
widgets=[Bar(), SimpleProgress(),
AdaptiveETA()],
maxval=UnknownLength)
self.reset()
def gen_wgt(msg):
bar_wdg = [
"( ", SimpleProgress(), " ) ",
Bar(),
Percentage(),
" [", Timer(), "] ",
]
return bar_wdg
def progress(number, **kwargs):
return ProgressBar(max_value=number,
widgets=[
Percentage(), ' (',
SimpleProgress(), ') ',
Bar(), ' ',
Timer(), ' ',
AbsoluteETABrief()
],
**kwargs).start()
def find_word_occurences(word_len, out_file):
json_list = get_input_file_list()
word_hash = {}
pbar = ProgressBar(widgets=[SimpleProgress()], maxval=NUM_ENTRIES).start()
for idx, json in enumerate(json_list):
json_record = db_util.open_NASARecord(json)
hash_string(json_record, word_len, word_hash)
pbar.update(idx + 1)
pbar.finish()
output_file(word_hash, out_file)
def add_periodic_images(self, rx = [-1,0], ry = [-1,0], rz = [-1,0] ):
a = self.atoms
p = self.positions
f = self.forces
c = self.basis
nx = len(rx)
ny = len(ry)
nz = len(rz)
nsteps = p.shape[0]
natoms = p.shape[1]
natoms2 = natoms*nx*ny*nz
# we may run out of memory here, but better now than later
# forces may be commented out if not used
# (or better, there should be an option to generally don't include forces to save mem)
p2 = np.empty((nsteps,natoms2,3))
#f2 = np.empty((nsteps,natoms2,3))
f2 = np.repeat(f,nx*ny*nz,axis=1)
#print p2.shape
#print f2.shape
print "%.2f MB allocated." % (p2.nbytes/(1024.**2))
c2 = np.empty(c.shape)
a2 = np.tile(a,nx*ny*nz)
# This may take some time
pbar = ProgressBar(widgets=[u'Calculating images... frame ',SimpleProgress()], maxval = nsteps).start()
for s in range(nsteps):
pbar.update(s)
for i,x in enumerate(rx):
for j,y in enumerate(ry):
for k,z in enumerate(rz):
idx = ny*nz*i + nz*j + k
#print "Adding cell %d of %d" % (f+1,nx*ny*nz)
ids = idx*natoms
ide = (idx+1)*natoms
p2[s,ids:ide,0] = (x + p[s,:,0]) / nx
p2[s,ids:ide,1] = (y + p[s,:,1]) / ny
p2[s,ids:ide,2] = (z + p[s,:,2]) / nz
#f2[s,ids:ide,0] = (x + f[s,:,0]) / nx
#f2[s,ids:ide,1] = (y + f[s,:,1]) / ny
#f2[s,ids:ide,2] = (z + f[s,:,2]) / nz
c2[s,0] = c[s,0] * nx
c2[s,1] = c[s,1] * ny
c2[s,2] = c[s,2] * nz
self.atoms = a2
self.positions = p2
self.forces = f2
self.basis = c2
pbar.finish()
print "Added %d images of the original unit cell. The number of atoms is now %d." % (nx*ny*nz-1, self.num_atoms)
def __init__(self, generations):
widgets = [
'Generation ',
SimpleProgress(), ' (',
Percentage(), ') ',
Bar(), ' ',
AdaptiveETA(), ' ',
FileTransferSpeed(unit='Gens')
]
self.pbar = ProgressBar(widgets=widgets, maxval=generations)
self.current = 1
def iterator_progress_bar(iterator, maxval=None):
""" Returns an iterator for an iterator that renders a progress bar with a
countdown timer. """
from progressbar import ProgressBar, SimpleProgress, Bar, ETA
pbar = ProgressBar(
maxval=maxval,
widgets=[SimpleProgress(sep='/'), ' ',
Bar(), ' ', ETA()],
)
return pbar(iterator)
def start_to_copy(src, dest, msg):
global file_copy_progress_count
file_copy_progress_count = 0
tempCount = 0
for dirPath, dirNames, fileNames in os.walk(src):
for f in fileNames:
tempCount += 1
print msg
pbar = ProgressBar(widgets=[SimpleProgress()], maxval=tempCount).start()
recursive_overwrite(src, dest, pbar)
pbar.finish()
