def save_video(url, directory, filename, extension):
path = os.path.join(directory, "%s.%s" % (filename, extension))
temp = "%s.tmp" % path
# Temp file already exists, so we'll bin it and try again
if os.path.exists(temp):
os.remove(temp)
os.remove(path)
if not os.path.exists(path):
response = session.get(url, stream=True)
with open(temp, 'w') as f:
f.write('\n')
with open(path, 'wb') as f:
total_length = int(response.headers.get('Content-Length'))
widgets = ['Saving %s: ' % path, Percentage(), ' ', Bar(), ' ', ETA(), ' ', FileTransferSpeed()]
pbar = ProgressBar(widgets=widgets, maxval=total_length).start()
bytes_downloaded = 0
for chunk in response.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
bytes_downloaded += len(chunk)
pbar.update(bytes_downloaded)
pbar.finish()
# Delete temp file
os.remove(temp)
def sentence_similarity(idx, ob, mode):
s_list = list()
pbar = ProgressBar(widgets=['%s: image ' % mode, SimpleProgress()],
maxval=len(sentences)).start()
for im_idx, sentence_group in enumerate(np.array(sentences)[idx, :]):
pbar.update(im_idx + 1)
for sent in sentence_group:
words = analyze(sent)
sim = list()
for w in words:
syn1 = wn.synsets(w)
syn2 = wn.synsets(ob)
if syn1 and syn2:
sim.append(max(s1.path_similarity(s2) for (s1, s2)
in product(syn1, syn2)))
else:
# ignore word if no synset combination was found on wordnet
sim.append(None)
if max(sim):
s_list.append(max(sim))
else:
# ignore sentence if no word was similar enough
s_list.append(float('nan'))
pbar.finish()
return s_list
def formalize(sarc):
all_words = defaultdict(lambda: 0)
for line in range(0, len(CORPUS)):
for word in range(0, len(CORPUS[line])):
all_words[CORPUS[line][word]] += 1
all_words_sorted = sorted(all_words.items(), key=operator.itemgetter(1), reverse=True)
# print all_words_sorted
with open("contexts.cols", "w") as raw:
for word in range(0, len(all_words_sorted)):
# print all_words_sorted[word][0]
raw.write(all_words_sorted[word][0] + "\n")
with open("targets.rows", "w") as raw:
for word in range(0, 1):
raw.write(all_words_sorted[word][0] + "\n")
with open("target_context_count.sm", "a") as raw:
# raw.write('target_word\tcontext_word\tcount\n')
for target in range(0, 1):
pbar = ProgressBar(maxval=len(all_words_sorted)).start()
for context in range(0, len(all_words_sorted)):
# print all_words_sorted[target]
pbar.update(context + 1)
context_ = getContext(all_words_sorted[target][0], CORPUS, WINDOW_SIZE)[all_words_sorted[context][0]]
if context_ > 2:
# print all_words_sorted[context]
# raw.write('{}\t{}\t{}\t{}\n'.format(all_words_sorted[target][0],all_words_sorted[context][0],context_, ppmi(getContext(all_words_sorted[context][0], CORPUS, WINDOW_SIZE), getContext(all_words_sorted[target][0], CORPUS, WINDOW_SIZE), all_words_sorted[context][0], all_words_sorted[target][0])))
raw.write(
"{}\t{}\t{}\t{}\n".format(
all_words_sorted[target][0], all_words_sorted[context][0], context_, sarc
)
)
# print('\n')
pbar.finish()
def main(url,User_agent,thread = 5):
regex = re.compile(r'^(?:http|ftp)s?://'r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'r'localhost|'r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'r'(?::\d+)?'r'(?:/?|[/?]\S+)$', re.IGNORECASE)
if not regex.match(url):
print "%s is not a effective url"%url
return
filename = urllib2.unquote(url).decode('utf8').split('/')[-1]
req = urllib2.Request(url)
req.headers['User-Agent'] =User_agent
f=urllib2.urlopen(req)
size = int(f.info().getheaders('Content-Length')[0])
widgets = ['Test: ', Percentage(), ' ',Bar(marker='>',left='[',right=']'), ' ', ETA(), ' ', FileTransferSpeed()]
pbar = ProgressBar(widgets=widgets, maxval=size)
fb = open(filename,"wb+")
block_size = size % thread + 1
thread_list = []
for i in range(thread):
start_file_seek = i*block_size
end_file_seek = start_file_seek + block_size
if i == thread - 1:
end_file_seek = ''
t = downFileMul(url,User_agent,pbar,start_file_seek,end_file_seek,fb)
thread_list.append(t)
pbar.start()
for i in thread_list:
i.start()
for i in thread_list:
i.join()
fb.close()
pbar.finish()
def __iter__(self):
#size = float()
p = ProgressBar(max_value = len(self.text_list), min_value = 1)
for i, line in enumerate(self.text_list):
#if i%100 == 0:
p.update((i+1))
sentence = line.rstrip()
tagger = MeCab.Tagger('') # 別のTaggerを使ってもいい
#print sentence
node = tagger.parseToNode(sentence)
words = []
while node:
# macabで分けると、文の最後に’’が、その手前に'EOS'が来る
info = node.feature.split(",")
try:
word = node.surface
except:
pass
if word != 'EOS' and word != '' and info[0] != "記号":
if info[6] != '*':
words.append(info[6])
else:
words.append(word)
node = node.next
#print(words)
yield words
p.finish()
def get_data(limit=None):
"""Read csv file of training data and put it in the database"""
if limit is not None:
limit = min(ROW_COUNT, int(limit))
else:
limit = ROW_COUNT
pbar = ProgressBar(maxval=limit).start()
with open(PATH, 'rb') as csvfile:
csvreader = csv.reader(csvfile)
next(csvreader) # to skip the headings
final_data = []
for counter, row in enumerate(csvreader):
if counter == limit - 1: break
datapoint = Row()
datapoint.id_ = int(row[0])
datapoint.target = int(row[-1])
for x in range(1, 1934 + 1):
if x in ABSENT_ATTRIBUTES: continue
x_temp = x
if 218 < x < 240: x_temp -= 1
if x > 240: x_temp -= 2
setattr(datapoint, 'VAR_' + _make_equal_length(x), row[x_temp])
yield datapoint
pbar.update(counter)
pbar.finish()
def vectorizePass(didout_fname, vecout_fname, docset):
print "processing to output file %s" % vecout_fname
did_out = open(didout_fname, "w")
vector_out = open(vecout_fname, "w")
pbar = ProgressBar(widgets=[Percentage(), Bar()], maxval=len(docset)).start()
for row, doc in enumerate(docset):
did, vec = doc
# vec is [word1, word2, word3, ...]
# want to convert to {word1: cnt, word2: cnt, ...}
# after the conversion, we replace word with its index
# index is added one because matlab index starts from one.
index_cnt = {}
for word, cnt in Counter(vec).iteritems():
try:
index_cnt[words_index[word]] = cnt
except KeyError: # thrown when meeting non-popular words
continue
# we normalize the vector, and output
cnt_sum = float(sum(index_cnt.values()))
# output to file
did_out.write("%s\n" % did)
for col, value in index_cnt.iteritems():
vector_out.write("%d\t%d\t%f\n" % (row + 1, col + 1, value / cnt_sum))
pbar.update(row)
pbar.finish()
did_out.close()
vector_out.close()
def train_imdb(self):
f_train_loss=open('./train_loss.txt','w')
f_valid_acc=open('./valid_acc.txt','w')
f_train_loss.close()
f_valid_acc.close()
self.session = tf.Session()
init_op = tf.initialize_all_variables()
self.session.run(init_op)
print("Training...")
for epoch in range(self.num_epoch):
pbar = ProgressBar(widgets=[Percentage(), Bar(), ETA()], maxval=len(self.X_batch)).start()
loss=0.0
for i in range(len(self.X_batch)):
results = self.session.run([self.cross_entropy,self.optimizer], feed_dict={self.input: self.X_batch[i], self.target: self.y_batch[i], self.keep_prob: 0.5})
loss+=results[0]
pbar.update(i+1)
print('\nloss: {}\n'.format(loss/len(self.X_batch)))
f_train_loss=open('./train_loss.txt','a')
f_train_loss.write(str(loss/len(self.X_batch))+'\n')
f_train_loss.close()
for i in range(len(self.X_valid_batch)):
acc = self.session.run(self.accuracy, feed_dict={self.input:self.X_valid_batch[i], self.target: self.y_valid_batch[i], self.keep_prob: 1.0})
f_train_loss=open('./valid_acc.txt','a')
f_train_loss.write(str(acc/len(self.X_valid_batch))+'\n')
f_train_loss.close()
print('\naccuracy: {}\n'.format(acc/len(self.X_valid_batch)))
print('Training completed!')
def main(parser, options, args):
app_globs = app_globals._current_obj()
app_id = app_globals.settings['facebook_appid']
if not app_id:
print 'No Facebook app_id configured, exiting'
sys.exit(3)
app_secret = options.app_secret
fb = FacebookAPI(app_id, app_secret)
from mediadrop.model import DBSession, Media
# eager loading of 'meta' to speed up later check.
all_media = Media.query.options(joinedload('_meta')).all()
print 'Checking all media for existing Facebook comments'
progress = ProgressBar(maxval=len(all_media)).start()
for i, media in enumerate(all_media):
progress.update(i+1)
if 'facebook-comment-xid' not in media.meta:
continue
if not fb.has_xid_comments(media):
continue
media.meta[u'facebook-comment-xid'] = unicode(media.id)
DBSession.add(media)
DBSession.commit()
progress.finish()
def example3():
widgets = [Bar('>'), ' ', ETA(), ' ', ReverseBar('<')]
pbar = ProgressBar(widgets=widgets, maxval=10000000).start()
for i in range(1000000):
# do something
pbar.update(10*i+1)
pbar.finish()
def train(train_dl, N, model, optimizer, trans, args, input_q, data_q):
pbar = ProgressBar(N)
perm = np.random.permutation(N)
sum_loss = 0
# putting all data
for i in range(0, N, args.batchsize):
x_batch = train_dl[perm[i:i + args.batchsize]]
input_q.put(x_batch)
# training
for i in range(0, N, args.batchsize):
input_data, label = data_q.get()
if args.gpu >= 0:
input_data = cuda.to_gpu(input_data.astype(np.float32))
label = cuda.to_gpu(label.astype(np.float32))
optimizer.zero_grads()
loss, pred = model.forward(input_data, label, train=True)
loss.backward()
optimizer.update()
sum_loss += float(cuda.to_cpu(loss.data)) * batchsize
pbar.update(i + args.batchsize if (i + args.batchsize) < N else N)
return sum_loss
def train_epoch(train_data, train_labels, model, optimizer, batchsize, transformations, silent, gpu=0, finetune=False):
N = train_data.shape[0]
pbar = ProgressBar(0, N)
perm = np.random.permutation(N)
sum_accuracy = 0
sum_loss = 0
for i in range(0, N, batchsize):
x_batch = train_data[perm[i : i + batchsize]]
y_batch = train_labels[perm[i : i + batchsize]]
if transformations is not None:
if "rotation" == transformations:
x_batch = rotate_transform_batch(x_batch, rotation=2 * np.pi)
if gpu >= 0:
x_batch = cuda.to_gpu(x_batch.astype(np.float32))
y_batch = cuda.to_gpu(y_batch.astype(np.int32))
optimizer.zero_grads()
x = Variable(x_batch)
t = Variable(y_batch)
loss, acc = model(x, t, train=True, finetune=finetune)
if not finetune:
loss.backward()
optimizer.update()
sum_loss += float(cuda.to_cpu(loss.data)) * y_batch.size
sum_accuracy += float(cuda.to_cpu(acc.data)) * y_batch.size
if not silent:
pbar.update(i + y_batch.size)
return sum_loss, sum_accuracy
def writeTweets2file(self, filePath):
if len(filePath) <= 0:
print "No tweets are collected!"
return
tweet_list = self.getText()
pbar = ProgressBar(widgets=[Percentage(), Bar()], maxval = len(tweet_list)).start()
print "Begin to write tweets into file..."
index = 0
filewrite = open(filePath, 'w')
for tweet in tweet_list:
text = tweet.replace('\n', ' ') # 替換掉所有換行符
text = text.replace(';', '.') # 所有分號替換成句號
text = text.replace('\t', ' ') # 替換掉所有tab
filewrite.write(text)
filewrite.write('\t' + 'surprise' + '\t' + 'sadness') # 每行後面添加一個制表符，為了符合JAVA的simpleClassifier代碼
filewrite.write('\r\n')
pbar.update(index+1)
index += 1
pbar.finish()
filewrite.close()
print "Finishing writing tweets!"
print str(len(self._text)) + ' tweets have been written into the file: ' + filePath
def gan_test(self, test_set, n_batch=None, verbose=False):
if n_batch is None:
n_batch = self.n_batch
n_x = test_set[0].shape[0]
nbatches = n_x // n_batch
z_dim = (n_batch,) + self.hidden_dim
loss_all = []
if verbose:
pbar = ProgressBar(maxval=nbatches).start()
for i in range(nbatches):
start = i * n_batch
end = start + n_batch
batch_x = [_x[start:end] for _x in test_set]
batch_z =\
self.rng.uniform(-1., 1.,
size=z_dim).astype(batch_x[0].dtype)
_x = [batch_z] + batch_x
loss = self.test(*_x)
loss_all.append(np.array(loss))
if verbose:
pbar.update(i)
loss_all = np.mean(loss_all, axis=0)
return loss_all
def train(self, train_set, freq=1, verbose=False):
n_x = len(train_set[0])
nbatches = n_x // self.n_batch
z_dim = (self.n_batch,) + self.hidden_dim
loss_all = []
if verbose:
pbar = ProgressBar(maxval=nbatches).start()
for i in range(nbatches):
start = i * self.n_batch
end = start + self.n_batch
batch_x = [_x[start:end] for _x in train_set]
batch_z =\
self.rng.uniform(-1., 1.,
size=z_dim).astype(batch_x[0].dtype)
_x = [batch_z] + batch_x
loss = self.p_train(*_x)
loss = self.d_train(*_x)
loss_all.append(np.array(loss))
if verbose:
pbar.update(i)
loss_all = np.mean(loss_all, axis=0)
return loss_all
def remove_padding():
"""
Remove paddings
:return: nothing
"""
problem_folder = 'lamellarity'
out_dir = '/home/sergii/Documents/microscopic_data/' + problem_folder + '/images/without_padding/'
in_dir = '/home/sergii/Documents/microscopic_data/' + problem_folder + '/images/particles/'
path_to_json = '/home/sergii/Documents/microscopic_data/' + problem_folder + '/particles_repaired_2.json'
print '...Reading dataframe at %s' % path_to_json
df = pd.read_json(path_to_json)
print '...Starting processing images'
if os.path.exists(out_dir):
shutil.rmtree(out_dir)
os.makedirs(out_dir)
total_count = df.shape[0]
pbar = ProgressBar(widgets=[Percentage(), Bar()], maxval=total_count).start()
counter = 0
for _, row in df.iterrows():
img_name = row['Image']
padding = row['Padding']
_remove_padding(in_dir + img_name,
out_dir + img_name,
padding)
counter += 1
pbar.update(counter)
pbar.finish()
def create_zip_files(self, dirs):
"""Create zip files."""
logmsg.header('Creating zip files...', self.logger)
# Create output directory to hold our zips
output_dir = 'Zips'
if not os.path.exists(output_dir):
os.mkdir(output_dir)
num_zips = 0
# Create a progress bar
pbar = ProgressBar(term_width=80, maxval=len(dirs)).start()
for d in dirs:
self.logger.debug('Zipping: "%s"' % d)
parent_dir = os.path.join(os.path.dirname(d).split(os.path.sep)[-1], '')
if parent_dir == self.input_dir:
parent_dir = ''
output_file = os.path.join(output_dir, parent_dir, os.path.basename(d))
shutil.make_archive(output_file, format="zip", root_dir=d)
num_zips += 1
# Update progress bar
pbar.update(num_zips)
# Ensure progress bar is finished
pbar.finish()
time_elapsed = "(Time Elapsed: {0})".format(secs_to_mins(pbar.seconds_elapsed))
logmsg.success('Created {0} zip files {1}'.format(num_zips,
time_elapsed), self.logger)
def dir_walk(target_dir=None, quiet=None):
'''recursively walk a directory containing cti and return the stats'''
files = find_files('*.xml', resolve_path(target_dir))
if not quiet:
widgets = ['Directory Walk: ', Percentage(), ' ', Bar(marker=RotatingMarker()),
' ', ETA()]
progress = ProgressBar(widgets=widgets, maxval=len(files)).start()
cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
'exploit_targets': set(), 'incidents': set(), \
'indicators': set(), 'threat_actors': set(), \
'ttps': set()}
cooked_cybox_objs = dict()
for file_ in files:
try:
stix_package = file_to_stix(file_)
(raw_stix_objs, raw_cybox_objs) = \
process_stix_pkg(stix_package)
for k in raw_stix_objs.keys():
cooked_stix_objs[k].update(raw_stix_objs[k])
for k in raw_cybox_objs.keys():
if not k in cooked_cybox_objs.keys():
cooked_cybox_objs[k] = set()
cooked_cybox_objs[k].update(raw_cybox_objs[k])
if not quiet:
progress.update(i)
except:
next
if not quiet:
progress.finish()
return (cooked_stix_objs, cooked_cybox_objs)
class Euler:
def __init__(self, solver, tf, shift=False):
self.solver = solver
self.tf = tf
self.shift = shift
self.pbar = ProgressBar(maxval=self.tf)
def integrate(self):
pass
def savedata(self):
# filename format EQUATION_ORDER_SCHEME_N_TIME.npz
filename = "./data/%s_%s_%s_%s_%s.npz" % (
self.solver.f.__name__,
self.solver.order,
self.solver.scheme,
self.solver.N,
self.tf,
)
np.savez(filename, x=self.solver.xc, u=self.solver.u, uactual=Complex().init(self.solver.xc, self.tf))
print filename
def run(self):
t = 0.0
while t < self.tf:
dt = self.solver.calc_dt()
residue = self.solver.calc_residue()
self.solver.u += residue * dt
self.pbar.update(t)
t += dt
self.savedata()
def retrieve(self):
'''
Execute the jobs
'''
numleft = self.total
writer = Writer((0,self.numCores))
width = int(math.floor(term.width/3))
name = 'Total'
writer.write('{0:3} {1}: Queued'.format('',name[:width]+' '*max(0,width-len(name))))
pbar = ProgressBar(widgets=['{0:3} {1}: '.format('',name[:width]+' '*max(0,width-len(name))),' ',SimpleProgress(),' ',Percentage(),' ',Bar(),' ',Timer()],fd=writer,term_width=term.width, maxval=self.total).start()
try:
while True:
numleft_new = self.jobsRemaining()
if numleft_new < numleft:
pbar.update(self.total-numleft_new)
numleft = numleft_new
if numleft==0: break
time.sleep(0.1)
theResult = [r.get(9999999999) for r in self.results]
except:
e = sys.exc_info()[0]
self.pool.terminate()
print term.exit_fullscreen
print e
theResult = []
else:
print term.exit_fullscreen
self.pool.close()
self.pool.join()
return theResult
def mongodb2shape(mongodb_server, mongodb_port, mongodb_db, mongodb_collection, output_shape, query_filter):
"""Convert a mongodb collection (all elements must have same attributes) to a shapefile"""
print ' Converting a mongodb collection to a shapefile '
connection = Connection(mongodb_server, mongodb_port)
print 'Getting database MongoDB %s...' % mongodb_db
db = connection[mongodb_db]
print 'Getting the collection %s...' % mongodb_collection
collection = db[mongodb_collection]
print 'Exporting %s elements in collection to shapefile...' % collection.count()
drv = ogr.GetDriverByName("ESRI Shapefile")
ds = drv.CreateDataSource(output_shape)
lyr = ds.CreateLayer('test', None, ogr.wkbPolygon)
print 'Shapefile %s created...' % ds.name
cursor = collection.find(query_filter)
# define the progressbar
pbar = ProgressBar(collection.count()).start()
k=0
# iterate the features in the collection and copy them to the shapefile
# for simplicity we export only the geometry to the shapefile
# if we would like to store also the other fields we should have created a metadata element with fields datatype info
for element in cursor:
element_geom = element['geom']
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetGeometry(ogr.CreateGeometryFromWkt(element_geom))
lyr.CreateFeature(feat)
feat.Destroy()
k = k + 1
pbar.update(k)
pbar.finish()
print '%s features loaded in shapefile from MongoDb.' % lyr.GetFeatureCount()
def write_wav(self):
"""
Synthesizes the analysis to a WAV file in the synthesis directory
"""
print("Performing Resynthesis...")
progress = ProgressBar(
widgets=[Percentage(), Bar()],
maxval=self.num_samples
).start()
sines = []
for i in range(len(self.bins)):
data = [sin(2 * pi * self.bins[i][0] * (x / self.frate))
for x in range(self.num_samples)]
for s in data:
s = s * self.bins[i][1]
sines.append(data)
samples = []
for i in range(len(sines[0])):
s = 0
for j in range(len(sines)):
s += sines[j][i]
samples.append(s)
progress.update(i+1)
samples = scale(samples, -1.0, 1.0)
wavfile.write(
"../synthesis/" + self.wav_name + "_resynth.wav",
self.sample_rate,
array(samples)
)
progress.finish()
def extractFeatures(data, n):
logging.info('Features: extracting {0}...'.format(n))
# create DF
columns = []
col_names = ['open', 'high', 'low', 'close', 'volume']
for col_name in col_names:
for m in xrange(1, n+1):
columns.append('{0}_{1}'.format(col_name, m))
# pprint(columns)
df = pd.DataFrame(dtype=float, columns=columns)
pb = ProgressBar(maxval=len(data)).start()
for i in xrange(n, len(data)+1):
pb.update(i)
slice = data.ix[i-n:i]
# print slice
scale(slice, axis=0, copy=False)
# print slice
cntr = 0
item = {}
for slice_index, slice_row in slice.iterrows():
cntr += 1
# print slice_index
# print slice_row
for col in slice.columns:
item['{0}_{1}'.format(col, cntr)] = slice_row[col]
# pprint(item)
df.loc[i] = item
# break
pb.finish()
logging.info('Features: extracted')
return df
def import_stack(self, file, isImport, isForce, rbundles, isUseMajor):
try:
if isImport:
printer.out("Importing template from [" + file + "] archive ...")
else:
if constants.TMP_WORKING_DIR in str(file):
printer.out("Creating template from temporary [" + file + "] archive ...")
else:
printer.out("Creating template from [" + file + "] archive ...")
file = open(file, "r")
# The following code could not be used for the moment
# appImport = applianceImport()
# appImport.imported = isImport
# appImport.forceRw = isForce
# appImport.reuseBundles = rbundles
# appImport.useMajor = isUseMajor
# appImport = self.api.Users(self.login).Imports.Import(appImport)
appImport = self.api.Users(self.login).Imports.Import(
Imported=isImport, Force=isForce, Reusebundles=rbundles, Usemajor=isUseMajor
)
if appImport is None:
if isImport:
printer.out("error importing appliance", printer.ERROR)
else:
printer.out("error creating appliance", printer.ERROR)
return 2
else:
status = self.api.Users(self.login).Imports(appImport.dbId).Uploads.Upload(file)
progress = ProgressBar(widgets=[Percentage(), Bar()], maxval=100).start()
while not (status.complete or status.error):
progress.update(status.percentage)
status = self.api.Users(self.login).Imports(appImport.dbId).Status.Get()
time.sleep(2)
progress.finish()
if status.error:
if isImport:
printer.out("Template import: " + status.message + "\n" + status.errorMessage, printer.ERROR)
if status.detailedError:
printer.out(status.detailedErrorMsg)
else:
printer.out("Template create: " + status.message + "\n" + status.errorMessage, printer.ERROR)
else:
if isImport:
printer.out("Template import: DONE", printer.OK)
else:
printer.out("Template create: DONE", printer.OK)
# get appliance import
appImport = self.api.Users(self.login).Imports(appImport.dbId).Get()
printer.out("Template URI: " + appImport.referenceUri)
printer.out("Template Id : " + generics_utils.extract_id(appImport.referenceUri))
return 0
except IOError as e:
printer.out("File error: " + str(e), printer.ERROR)
return 2
except Exception as e:
return handle_uforge_exception(e)
def _load_data(self, strpcons, dt_start, dt_end, n, spec_date):
all_data = OrderedDict()
max_window = -1
logger.info("loading data...")
pbar = ProgressBar().start()
for i, pcon in enumerate(strpcons):
#print "load data: %s" % pcon
if pcon in spec_date:
dt_start = spec_date[pcon][0]
dt_end = spec_date[pcon][1]
assert(dt_start < dt_end)
if n:
wrapper = self._data_manager.get_last_bars(pcon, n)
else:
wrapper = self._data_manager.get_bars(pcon, dt_start, dt_end)
if len(wrapper) == 0:
continue
all_data[pcon] = DataContext(wrapper)
max_window = max(max_window, len(wrapper))
pbar.update(i*100.0/len(strpcons))
#progressbar.log('')
if n:
assert(max_window <= n)
pbar.finish()
if len(all_data) == 0:
assert(False)
## @TODO raise
return all_data, max_window
def EvaluateGenomeList_Serial(genome_list, evaluator, display=True):
fitnesses = []
count = 0
if prbar_installed and display:
widg = ['Individuals: ', Counter(), ' of ' + str(len(genome_list)),
' ', ETA(), ' ', AnimatedMarker()]
progress = ProgressBar(maxval=len(genome_list), widgets=widg).start()
for g in genome_list:
f = evaluator(g)
fitnesses.append(f)
if display:
if prbar_installed:
progress.update(count+1)
else:
print 'Individuals: (%s/%s)' % (count, len(genome_list))
count += 1
if prbar_installed and display:
progress.finish()
return fitnesses
def EvaluateGenomeList_Parallel(genome_list, evaluator, cores=4, display=True):
fitnesses = []
pool = mpc.Pool(processes=cores)
curtime = time.time()
if prbar_installed and display:
widg = ['Individuals: ', Counter(),
' of ' + str(len(genome_list)), ' ', ETA(), ' ',
AnimatedMarker()]
progress = ProgressBar(maxval=len(genome_list), widgets=widg).start()
for i, fitness in enumerate(pool.imap(evaluator, genome_list)):
if prbar_installed and display:
progress.update(i)
else:
if display:
print 'Individuals: (%s/%s)' % (i, len(genome_list))
if cvnumpy_installed:
cv2.waitKey(1)
fitnesses.append(fitness)
if prbar_installed and display:
progress.finish()
elapsed = time.time() - curtime
if display:
print 'seconds elapsed: %s' % elapsed
pool.close()
pool.join()
return fitnesses
def dark_spot_gridsearch(observer, stars, min_altitude, alt_resolution=0.5, az_resolution=4):
azs = np.deg2rad(np.arange(0, 360, az_resolution))
alts = np.deg2rad(np.arange(min_altitude, 91, 0.5))
light = []
coords = []
prog = ProgressBar(maxval=len(azs) * len(alts)).start()
for i, az in enumerate(azs):
for j, alt in enumerate(alts):
coords.append((az, alt))
light.append(light_in_fov(az, alt, stars, observer))
prog.update(i * len(alts) + j)
light = np.array(light)
coords = np.array(coords)
azs = coords[:, 0]
alts = coords[:, 1]
min_index = np.argmin(light)
az = azs[min_index]
alt = alts[min_index]
ra, dec = observer.radec_of(az, alt)
return az, alt, ra, dec
def sample_many(self, num_samples = 2000):
"""
Generates a given number of trajectories, using the method sample().
Returns a fixmat with the generated data.
Parameters:
num_samples : int, optional
The number of trajectories that shall be generated.
"""
x = []
y = []
fix = []
sample = []
# XXX: Delete ProgressBar
pbar = ProgressBar(widgets=[Percentage(),Bar()], maxval=num_samples).start()
for s in xrange(0, num_samples):
for i, (xs, ys) in enumerate(self.sample()):
x.append(xs)
y.append(ys)
fix.append(i+1)
sample.append(s)
pbar.update(s+1)
fields = {'fix':np.array(fix), 'y':np.array(y), 'x':np.array(x)}
param = {'pixels_per_degree':self.fm.pixels_per_degree}
out = fixmat.VectorFixmatFactory(fields, param)
return out
def EvaluateGenomeList_Serial(genome_list, evaluator):
fitnesses = []
curtime = time.time()
if prbar_installed:
widg = ['Individuals: ', Counter(), ' of ' + str(len(genome_list)), ' ', ETA(), ' ', AnimatedMarker()]
progress = ProgressBar(maxval=len(genome_list), widgets=widg).start()
count = 0
for g in genome_list:
f = evaluator(g)
fitnesses.append(f)
if prbar_installed:
progress.update(count)
else:
print ('Individuals: (',count,'/',len(genome_list),')')
count += 1
if prbar_installed:
progress.finish()
elapsed = time.time() - curtime
print ('seconds elapsed:', elapsed)
return (fitnesses, elapsed)
def setUp(self):
self.p = ProgressBar()
fourcc = cv2.VideoWriter_fourcc(*'MJPG')
video_path = os.path.join(args.vis_path,
'%s.avi' % eval_path.replace('/', '.'))
image_path = os.path.join(args.vis_path,
'%s' % eval_path.replace('/', '.'))
print('Saving video to', video_path)
print('Saving image to', image_path)
video_shape = (1280, 480) if fancy else (1200, 600)
out = cv2.VideoWriter(video_path, fourcc, 14., video_shape)
os.system('mkdir -p ' + image_path)
print('min: %.4f, max: %.4f' % (np.min(result), np.max(result)))
lim_low = np.min(result)
lim_high = np.max(result) * 1. / 2.
bar = ProgressBar(max_value=touch_rec.shape[0])
for i in bar(range(touch_rec.shape[0])):
touch_raw_cur = touch_raw_rec[i]
touch_cur = touch_rec[i, args.obs_window // 2]
touch_cal = result[i, 0]
touch_raw_render = vis.render(
touch_raw_cur,
lim_low=np.min(np.median(touch_raw_rec, 0)) + 10,
lim_high=800,
fancy=fancy)
'''
touch_render = vis.render(
touch_cur, lim_low=0., lim_high=24, text='raw', fancy=fancy)
'''
result_render = vis.render(touch_cal,
def process_file(self, input_file):
block = Block(default_compress_level=self.config.compress_level)
resultados = []
self.input_file = input_file
size_test_file = os.path.getsize(self.input_file)
self.spool_types = {
"fixed":
SpoolFixedRecordLength(self.input_file,
buffer_size=self.config.buffer_size,
encoding=self.config.encoding,
newpage_code=self.config.EOP),
"fcfc":
SpoolHostReprint(self.input_file,
buffer_size=self.config.buffer_size,
encoding=self.config.encoding)
}
compresiones = [
e for e in block.compressor.available_types
if e[0] == self.config.compress_type
]
encriptados = [
e for e in block.cipher.available_types
if e[0] == self.config.cipher_type
]
mode = "ab"
r = ReportMatcher(self.config.report_cfg)
reports = []
for encriptado in encriptados:
for compress in compresiones:
start = time.time()
paginas = 0
# file_name = "{0}.{1}.oerm".format(self.config.output_path, slugify("{0}.{1}".format(compress[1], encriptado[1]), "_"))
file_name = os.path.join(
self.config.output_path,
generate_filename(self.config.file_mask) + ".oerm")
db = Database(
file=file_name,
mode=mode,
default_compress_method=compress[0],
default_compress_level=self.config.compress_level,
default_encription_method=encriptado[0],
pages_in_container=self.config.pages_in_group)
file_size = os.path.getsize(file_name)
reportname_anterior = ""
widgets = [
os.path.basename(self.input_file), ': ',
FormatLabel(
'%(value)d bytes de %(max_value)d (%(percentage)0.2f)'
),
Bar(marker='#', left='[', right=']'), ' ',
ETA(), ' ',
FileTransferSpeed()
]
p_size = 0
with ProgressBar(max_value=size_test_file,
widgets=widgets) as bar:
spool = self.spool_types[self.config.file_type]
with spool as s:
for page in s:
p_size += len(page)
bar.update(p_size)
data = r.match(page)
reportname = data[0]
if reportname not in reports:
reports.append(reportname)
if reportname != reportname_anterior:
rpt_id = db.get_report(reportname)
if rpt_id:
db.set_report(reportname)
else:
db.add_report(reporte=reportname,
sistema=data[1],
departamento=data[2],
fecha=data[3])
reportname_anterior = reportname
paginas = paginas + 1
db.add_page(page)
db.close()
compress_time = time.time() - start
compress_size = os.path.getsize(file_name) - file_size
resultados.append([
"[{0}] {1} ({2}p/cont.)".format(
compress[0], compress[1], self.config.pages_in_group),
("" if encriptado[0] == 0 else encriptado[1]),
float(size_test_file),
float(compress_size),
(compress_size / size_test_file) * 100,
paginas / compress_time,
len(reports)
])
tablestr = tabulate(tabular_data=resultados,
headers=[
"Algoritmo", "Encript.", "Real (bytes)",
"Compr. (bytes)", "Ratio", "Compr. Pg/Seg",
"Reportes"
],
floatfmt="8.2f",
tablefmt="psql",
numalign="right",
stralign="left",
override_cols_fmt=[
None, None, ",.0f", ",.0f", ",.2f", ",.2f",
",.2f"
])
return tablestr
original_images = []
for root, dirs, filenames in os.walk(path):
for filename in filenames:
original_images.append(os.path.join(root, filename))
original_images = sorted(original_images)
print('num:', len(original_images))
f = open('check_error.txt', 'w+')
error_images = []
progress = ProgressBar()
for filename in progress(original_images):
check = imghdr.what(filename)
if check == None:
f.write(filename)
f.write('\n')
error_images.append(filename)
print(len(error_images))
f.seek(0)
s = len(time_series.keys()) - 2
#Get number of simulations
n_simul = len(np.unique(time_series.n_simulation))
#Filter networks
keys = list(networks.keys())
junk = [i for i in keys if not (i.startswith('sub') or i.startswith('prod'))]
#Get rid of junk
networks = networks.drop(columns=junk)
#Initialize storing object
names_strains = ['s' + str(i) for i in range(s)]
column_names = ["n_simulation", "t"] + names_strains
inter_evol_df = pd.DataFrame(columns=column_names)
c_evol_df = pd.DataFrame(columns=column_names)
f_evol_df = pd.DataFrame(columns=column_names)
surv_vector = np.array([])
pbar = ProgressBar()
print('Calculating evolution of interactions during community assembly')
for i in pbar(range(n_simul)):
time_series_i = time_series[time_series.n_simulation == i].reset_index(
drop=True)
networks_i = networks.iloc[i * s:(i + 1) * s, :].reset_index(drop=True)
#Get evolution of richness during community assembly
richness = evol_richness(abundances_t=time_series_i)
#Get positions where extinctions happen
indx = np.where(richness[:-1] != richness[1:])[0]
#Get values of time where extinctions happen
t_ext = time_series_i.t[indx]
t_points = len(t_ext)
t_ext_ind = np.array(t_ext.index)
f_evol, c_evol, inter_evol = interaction_evolution(time_series_i,
networks_i,
def setUp(self):
self.p = ProgressBar(**self.custom)
class CustomizedTestCase(unittest.TestCase):
"""
ProgressBar custom:
start = 0
end = 100
width = 20
fill = '#'
blank = '.'
format = '%(progress)s%% [%(fill)s%(blank)s]'
incremental = True
"""
custom = {
'end': 100,
'width': 20,
'fill': '#',
'format': '%(progress)s%% [%(fill)s%(blank)s]'
}
def setUp(self):
self.p = ProgressBar(**self.custom)
def tearDown(self):
del (self.p)
def test_initialization(self):
"""
>>> p = ProgressBar(custom)
>>> p
0% [....................]
"""
self.assertEqual(str(self.p), '0% [....................]')
def test_increment(self):
"""
>>> p = ProgressBar(custom)
>>> p + 1
1% [....................]
"""
self.p + 1
self.assertEqual(str(self.p), '1% [....................]')
self.p + 4
self.assertEqual(str(self.p), '5% [#...................]')
def test_reset(self):
"""
>>> p = ProgressBar(custom)
>>> p += 8
>>> p.reset()
0% [....................]
"""
self.p += 8
self.p.reset()
self.assertEqual(str(self.p), '0% [....................]')
def test_full_progress(self):
"""
>>> p = ProgressBar()
>>> p + 10
100% [####################]
"""
self.p + 100
self.assertEqual(str(self.p), '100% [####################]')
self.p + 100
self.assertEqual(str(self.p), '100% [####################]')
def getlistoffileitems(rootpath):
'''
scan the root and return a list of file items
-these are not validated show or non show yet
-make assumption shows have Season directory in them
-non shows do not
'''
shows = []
nonshows = []
dirtodel = []
fileitems = os.listdir(rootpath)
fileitems.sort()
max = int(len(fileitems))
pbar = ProgressBar()
print("Scanning root directory")
quickrun = 0 # remove later
fh = open("filestodelete.txt", "w")
fh2 = open("showfiles.txt", "w")
#looping through root directory
for item in pbar(fileitems):
if item[0:8] != "_UNPACK_":
if os.path.isdir(rootpath + "\\" + item):
isshow = False
numfiles = 0
subfileitems = os.listdir(rootpath + "\\" + item)
if numfiles == 0:
numfiles = len(subfileitems)
if numfiles == 0:
dirtodel.append(rootpath + item)
#loop through directory and check all files
#for "Season" indicating a show dir
for subfileitem in subfileitems:
if "season" in subfileitem.lower():
isshow = True
if (isshow):
shows.append(item)
fh2.write(item)
fh2.write("\n")
else:
nonshows.append(item)
fh.write(rootpath + item + "\\\\" + subfileitem)
fh.write("\n")
fh.write(str(numfiles))
fh.write("\n")
if numfiles == 0:
fh2.write("DEL: " + rootpath + item)
subfileitem = ""
#some code to shorten test time
quickrun += 1
if quickrun > 10000:
break
fh2.close()
fh.close()
return shows, nonshows, dirtodel
def insert_documents(data):
""" Insert all provided documents. Checks if the document has been manually
changed before - if it has, and the new document does not match, it is
marked as a conflict """
factory = ModelFactory.get_instance()
factory.set_db()
temp_col = Config.get_mongo_collection("temp_scraped")
manual_col = Config.get_mongo_collection("manual")
unknown_col = Config.get_mongo_collection("unknown")
prod_col = Config.get_mongo_collection("prod")
conflict_col = Config.get_mongo_collection("conflicts")
print("Starting insertion of {} documents".format(len(data)))
pbar = ProgressBar()
for i, doc in enumerate(pbar(data)):
factory.post_document(doc, temp_col)
print("Successfully inserted {} documents".format(i + 1))
manual_docs = factory.get_collection(manual_col).find()
conflicts = []
for manual_doc in manual_docs:
if "id" in manual_doc:
idx = manual_doc["id"]
else:
continue
# Mark corresponding entry in temp collection as manually changed
factory.get_database() \
.get_collection(temp_col) \
.update_one({"id": idx}, {"$set": {"manually_changed": True}})
prod_doc = next(
factory.get_collection(prod_col).find({"id": idx}), None)
temp_doc = next(
factory.get_collection(temp_col).find({"id": idx}), None)
if prod_doc and temp_doc:
if not temp_doc["content"] == prod_doc["content"]:
title = temp_doc["content"]["title"]
conflicts.append({"id": idx, "title": title})
print("Conflicts: {}".format(conflicts))
factory.get_collection(conflict_col).create_index([("id", 1)], unique=True)
for conflict in conflicts:
try:
factory.post_document(conflict, conflict_col)
except pymongo.errors.DuplicateKeyError:
# In case there are dupliacte, unsolved conflicts
pass
# Update production collection
db = factory.get_database()
try:
db.get_collection(prod_col).rename("old_prod")
except pymongo.errors.OperationFailure:
# If the prod collection does not exist
pass
try:
db.get_collection(temp_col).rename(prod_col)
except Exception as e:
print("Failed to update production db collection")
print(e)
db.get_collection("old_prod").rename(prod_col)
finally:
db.get_collection("old_prod").drop()
db.get_collection(temp_col).drop()
# Update all indexes
factory.set_index(prod_col)
factory.set_index(manual_col)
factory.set_index(temp_col)
# Removes duplicates
factory.get_collection(unknown_col).create_index([("query_text", 1)],
unique=True)
return conflicts
def run(self):
self.initialize()
self.iters = 0
for epoch in range(self.args.epoch_size):
data_style_A, data_style_B = shuffle_data(self.data_style_A,
self.data_style_B)
widgets = ['epoch #%d|' % epoch, Percentage(), Bar(), ETA()]
pbar = ProgressBar(maxval=self.n_batches, widgets=widgets)
pbar.start()
for i in range(self.n_batches):
pbar.update(i)
self.generator_A.zero_grad()
self.generator_B.zero_grad()
self.discriminator_A.zero_grad()
self.discriminator_B.zero_grad()
self.A_path = data_style_A[i * self.args.batch_size:(i + 1) *
self.args.batch_size]
self.B_path = data_style_B[i * self.args.batch_size:(i + 1) *
self.args.batch_size]
A, B = self.get_images()
A = Variable(torch.FloatTensor(A))
B = Variable(torch.FloatTensor(B))
if self.cuda:
A = A.cuda()
B = B.cuda()
AB = self.generator_B(A)
BA = self.generator_A(B)
ABA = self.generator_A(AB)
BAB = self.generator_B(BA)
# Reconstruction Loss
self.recon_loss_A = self.recon_criterion(ABA, A)
self.recon_loss_B = self.recon_criterion(BAB, B)
# Real/Fake GAN Loss (A)
A_dis_real, A_feats_real = self.discriminator_A(A)
A_dis_fake, A_feats_fake = self.discriminator_A(BA)
self.dis_loss_A, self.gen_loss_A = self.get_gan_loss(
A_dis_real, A_dis_fake)
self.fm_loss_A = self.get_fm_loss(A_feats_real, A_feats_fake)
# Real/Fake GAN Loss (B)
B_dis_real, B_feats_real = self.discriminator_B(B)
B_dis_fake, B_feats_fake = self.discriminator_B(AB)
self.dis_loss_B, self.gen_loss_B = self.get_gan_loss(
B_dis_real, B_dis_fake)
self.fm_loss_B = self.get_fm_loss(B_feats_real, B_feats_fake)
# Total Loss
if self.iters < self.args.gan_curriculum:
rate = self.args.starting_rate
else:
rate = self.args.default_rate
self.gen_loss_A_total = (
self.gen_loss_B * 0.1 + self.fm_loss_B * 0.9) * (
1. - rate) + self.recon_loss_A * rate
self.gen_loss_B_total = (
self.gen_loss_A * 0.1 + self.fm_loss_A * 0.9) * (
1. - rate) + self.recon_loss_B * rate
if self.args.model_arch == 'discogan':
self.gen_loss = self.gen_loss_A_total + self.gen_loss_B_total
self.dis_loss = self.dis_loss_A + self.dis_loss_B
elif self.args.model_arch == 'recongan':
self.gen_loss = self.gen_loss_A_total
self.dis_loss = self.dis_loss_B
elif self.args.model_arch == 'recongan_reverse':
self.gen_loss = self.gen_loss_B_total
self.dis_loss = self.dis_loss_A
elif self.args.model_arch == 'gan':
self.gen_loss = (self.gen_loss_B * 0.1 +
self.fm_loss_B * 0.9)
self.dis_loss = self.dis_loss_B
self.finish_iteration()
self.iters += 1
def color_images_full(model, b_size=32):
"""
Function that colors images with approaches on full images.
Function is used on reg-full and reg-full-vgg approaches.
Parameters
----------
model : keras.engine.training.Model
Model for image colorization
b_size : int
Size of bach of images
"""
abs_file_path = get_abs_path(data_origin)
images = get_image_list(abs_file_path)
# get list of images to color
num_of_images = len(images)
#progress bar
pbar = ProgressBar(maxval=num_of_images,
widgets=[Percentage(), ' ',
Bar(), ' ',
ETA()])
pbar.start()
# for each batch
for batch_n in range(int(math.ceil(num_of_images / b_size))):
_b_size = b_size if (
batch_n + 1) * b_size < num_of_images else num_of_images % b_size
# load images
original_size_images = []
all_images_l = np.zeros((_b_size, 224, 224, 1))
for i in range(_b_size):
# get image
image_lab = load_images(
os.path.join(abs_file_path, images[batch_n * b_size + i]))
original_size_images.append(image_lab[:, :, 0])
image_lab_resized = resize_image_lab(image_lab, (224, 224))
all_images_l[i, :, :, :] = image_lab_resized[:, :, 0][:, :,
np.newaxis]
# prepare images for a global network
all_vgg = np.zeros((_b_size, 224, 224, 3))
for i in range(_b_size):
all_vgg[i, :, :, :] = np.tile(all_images_l[i], (1, 1, 1, 3))
# color
if model.name == "reg_full_vgg": # vgg has no global network
color_im = model.predict(all_vgg, batch_size=b_size)
else:
color_im = model.predict([all_images_l, all_vgg],
batch_size=b_size)
# save all images
abs_save_path = get_abs_path(data_destination)
for i in range(_b_size):
# to rgb
original_im_bw = original_size_images[i]
h, w = original_im_bw.shape
# workaround for not suitable shape while resizing
small_images = np.concatenate((all_images_l[i], color_im[i]),
axis=2)
colored_im = resize_image_lab(small_images, (h, w))
lab_im = np.concatenate(
(original_im_bw[:, :, np.newaxis], colored_im[:, :, 1:]),
axis=2)
im_rgb = color.lab2rgb(lab_im)
# save
scipy.misc.toimage(im_rgb, cmin=0.0,
cmax=1.0).save(abs_save_path + model.name +
"_" +
images[batch_n * b_size + i])
# update progress bar
pbar.update(min((batch_n + 1) * b_size, num_of_images))
# stop progress bar
pbar.finish()
def train(self, epochs, train_loader, val_loader):
"""
Método para entrenar en modelo
:param epochs: (int) numero de épocas que se entrenará el modelo
:param train_loader: (Data.Loader) Iterador con los datos provenientes del data_set
ref: http://pytorch.org/docs/0.3.1/data.html#torch.utils.data.DataLoader
:param val_loader: (Data.Loader) Iterador con los datos que se utilizarán para validar el entrenamiento
ref: http://pytorch.org/docs/0.3.1/data.html#torch.utils.data.DataLoader
"""
total_batch_number = len(train_loader)
val_loss_prev = 1e5
for epoch_idx in range(1, epochs + 1):
sys.stdout.write("epoch {}/{}\n".format(epoch_idx, epochs))
with ProgressBar(widgets=[Percentage(), Bar()],
maxval=total_batch_number) as pbar:
for batch_idx, (x, y) in enumerate(train_loader):
x, y = Variable(x), Variable(y)
if self.use_cuda:
x, y = x.cuda(), y.cuda()
y_pred = self.model(x)
loss = self.loss_metric(y_pred, y)
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
pbar.update(batch_idx + 1)
sys.stdout.write("epoch: {}, train loss: {:.6f}\n".format(
epoch_idx, loss.data[0]))
# Calculamos la perdida y accuracy en el conjunto de validación
correct_cnt = 0
total_cnt = 0
for batch_idx, (x, y) in enumerate(val_loader):
if self.use_cuda:
x, y = x.cuda(), y.cuda()
x, target = Variable(x, volatile=True), Variable(y,
volatile=True)
y_pred = self.model(x)
loss = self.loss_metric(y_pred, target)
_, pred_label = torch.max(y_pred.data, 1)
total_cnt += x.data.size()[0]
correct_cnt += (pred_label == target.data).sum()
sys.stdout.write(
"\nepoch: {}, validation loss: {:.6f}, acc: {:.3f}\n\n".format(
epoch_idx, loss.data[0], correct_cnt * 1.0 / total_cnt))
## Guardamos el modelo
val_loss = loss.data[0]
is_best = val_loss_prev > val_loss
## Podríamos no guardar en cada época sino que cada K epocas ...
self.save_checkpoint(
self.model.state_dict(), is_best,
self.root_models + self.model.__name__() +
"_epoch{}_val_loss{:.6f}.pt".format(epoch_idx + 1, val_loss))
val_loss_prev = val_loss if val_loss < val_loss_prev else val_loss_prev
sys.stdout.write("\nTerminó el entrenamiento ...\n")
## Guardamos el ultimo modelo, sea mejor o no, por si se quiere continuar el entrenamiento
self.save_checkpoint(
self.model.state_dict(), False,
self.root_models + self.model.__name__() + "_finalmodel.pt")
def run():
import sys
if sys.argv[1] == 'dhfr':
[system, positions, testsystem_name] = dhfr()
elif sys.argv[1] == 'tip3p':
[system, positions, testsystem_name] = tip3p()
precision = sys.argv[2]
platform_name = sys.argv[3]
print('%s %s : contains %d particles' %
(testsystem_name, precision, system.getNumParticles()))
print('')
# Remove CMMotionRemover and barostat
indices_to_remove = list()
for index in range(system.getNumForces()):
force = system.getForce(index)
force_name = force.__class__.__name__
print(force_name)
if force_name in ['MonteCarloBarostat', 'CMMotionRemover']:
print('Removing %s (force index %d)' % (force_name, index))
indices_to_remove.append(index)
indices_to_remove.reverse()
for index in indices_to_remove:
system.removeForce(index)
print('')
# Add barostat
barostat = openmm.MonteCarloBarostat(pressure, temperature, frequency)
# Create OpenMM context
print('Creating context...')
from openmmtools import integrators
integrator = integrators.VelocityVerletIntegrator(timestep)
integrator.setConstraintTolerance(1.0e-8)
platform = openmm.Platform.getPlatformByName(platform_name)
platform.setPropertyDefaultValue('Precision', precision)
if platform_name == 'CUDA':
platform.setPropertyDefaultValue('DeterministicForces', 'true')
print('Using deterministic forces...')
context = openmm.Context(system, integrator, platform)
context.setPositions(positions)
print('')
# Get PME parameters
print('Retrieving PME parameters...')
for force in system.getForces():
if force.__class__.__name__ == 'NonbondedForce':
nbforce = force
break
pme_parameters = nbforce.getPMEParametersInContext(context)
print(pme_parameters)
# Flush
sys.stdout.flush()
# Equilibrate with barostat
print('equilibrating...')
barostat.setFrequency(frequency)
from progressbar import Percentage, Bar, ETA, RotatingMarker
widgets = [
'equilibration: ',
Percentage(), ' ',
Bar(marker=RotatingMarker()), ' ',
ETA()
]
progress = ProgressBar(widgets=widgets)
for iteration in progress(range(nequil)):
context.setVelocitiesToTemperature(temperature)
integrator.step(nequilsteps)
# Get positions, velocities, and box vectors
print('')
state = context.getState(getPositions=True, getVelocities=True)
box_vectors = state.getPeriodicBoxVectors()
positions = state.getPositions(asNumpy=True)
velocities = state.getVelocities(asNumpy=True)
del context, integrator
# Remove CMMotionRemover and barostat
indices_to_remove = list()
for index in range(system.getNumForces()):
force = system.getForce(index)
force_name = force.__class__.__name__
print(force_name)
if force_name in ['MonteCarloBarostat', 'CMMotionRemover']:
print('Removing %s (force index %d)' % (force_name, index))
indices_to_remove.append(index)
indices_to_remove.reverse()
for index in indices_to_remove:
system.removeForce(index)
#
integrator = integrators.VelocityVerletIntegrator(timestep)
integrator.setConstraintTolerance(1.0e-8)
context = openmm.Context(system, integrator, platform)
context.setPeriodicBoxVectors(*box_vectors)
context.setPositions(positions)
context.setVelocities(velocities)
# Open NetCDF file for writing.
ncfile = netcdf.Dataset(
'work-%s-%s-%s.nc' % (testsystem_name, precision, platform_name), 'w')
ncfile.createDimension('nwork', 0) # extensible dimension
ncfile.createDimension('nworkvals', nworkvals + 1)
ncfile.createVariable('work', np.float32, ('nwork', 'nworkvals'))
work = np.zeros([nwork, nworkvals + 1], np.float32)
for i in range(nwork):
context.setVelocitiesToTemperature(temperature)
integrator.step(nequilsteps) # equilibrate
state = context.getState(getEnergy=True)
initial_energy = state.getPotentialEnergy() + state.getKineticEnergy()
widgets = [
'Work %5d / %5d: ' % (i, nwork),
Percentage(), ' ',
Bar(marker=RotatingMarker()), ' ',
ETA()
]
progress = ProgressBar(widgets=widgets)
for workval in progress(range(nworkvals)):
integrator.step(nworksteps)
state = context.getState(getEnergy=True)
current_energy = state.getPotentialEnergy(
) + state.getKineticEnergy()
work[i, workval + 1] = (current_energy - initial_energy) / kT
ncfile.variables['work'][i, workval + 1] = work[i, workval + 1]
print(work[i, :])
ncfile.sync()
def zippy_attack(url):
global zippy_secret_attempts
global zippy_conn_attempts
zippy_req = urllib.request.Request(url)
try:
zippy_data = retry_urlopen(zippy_req)
if zippy_data.status == 200:
zippy_html = str(zippy_data.read())
zippy_soup = BeautifulSoup(zippy_html)
if not zippy_soup.title.text == "Zippyshare.com - ":
zippy_dl = zippy_soup.find('a', id="dlbutton")
if not zippy_dl is None:
zippy_js = zippy_soup.find_all('script')
for js in zippy_js:
if re.match('\\\\n var somffunction',
js.text) or re.match(
'\\\\n var otfunction', js.text):
a = re.search('var a = (\d*)\;', js.text)
if a.group(1):
if args.v:
print(colors.OKGREEN +
"Attemping to break secret" +
colors.ENDC)
secret = int(a.group(1))
download_secret = str(
int((secret % 78956) * (secret % 3) + 18))
url_info = url.split('/')
download_server = str(
url_info[2].split('.')[0])
download_file = str(url_info[4])
zippy_title = zippy_soup.title.text.split(
' - ')
zippy_title.pop(0)
download_name = " ".join(zippy_title)
download_name = urllib.parse.quote(
download_name)
url = "http://" + download_server + ".zippyshare.com/d/" + download_file + "/" + download_secret + "/" + download_name
test_req = urllib.request.Request(
url=url, method='HEAD')
test_data = urllib.request.urlopen(test_req)
content_type = test_data.headers[
'content-type'].split(';')
if content_type[0] == "application/x-download":
if args.v:
print(colors.OKBLUE + "\tSuccess" +
colors.ENDC)
widgets = [
" " + " ".join(zippy_title) + " ",
Percentage(), ' ',
Bar(), ' ',
ETA(), ' ',
FileTransferSpeed()
]
pbar = ProgressBar(widgets=widgets)
def dlProgress(count, blockSize,
totalSize):
if pbar.maxval is None:
pbar.maxval = totalSize
pbar.start()
pbar.update(
min(count * blockSize, totalSize))
dl, headers = urllib.request.urlretrieve(
url,
" ".join(zippy_title),
reporthook=dlProgress)
pbar.finish()
elif zippy_secret_attempts <= zippy_secret_attempts_max:
if args.v:
print(colors.WARNING + "\tFailed" +
colors.ENDC)
zippy_secret_attempts += 1
zippy_attack(url)
else:
print(
colors.FAIL +
"Reached max secret attempts, exiting"
+ colors.ENDC)
exit(0)
else:
print(colors.WARNING + "Can't find download button..." +
colors.ENDC)
else:
print(colors.WARNING + "Dead link" + colors.ENDC)
else:
print(colors.WARNING + "Bad status code: " +
str(zippy_data.status) + colors.ENDC)
except URLError:
if zippy_conn_attempts <= zippy_conn_attempts_max:
if args.v:
print(colors.WARNING +
"Connection refused, let's wait 5 seconds and retry" +
colors.ENDC)
zippy_conn_attempts += 1
time.sleep(5)
zippy_attack(url)
else:
print(colors.FAIL + "Reached connection retry limit, exiting" +
colors.ENDC)
exit(0)
def progress_bar(self, total):
widget = ["AutoCutMovie: ", Percentage(), Bar("#"), Timer(), " ", ETA()]
bar = ProgressBar(widgets=widget, maxval=total).start()
return bar
def color_images_part(model):
"""
Function that colors images with approaches on part of images.
Function is used on reg-parts, class-wo-weights and class-with-weights approaches.
Parameters
----------
model : keras.engine.training.Model
Model for image colorization
"""
# get images to color
test_set_dir_path = get_abs_path(data_origin)
image_list = get_image_list(test_set_dir_path)
num_of_images = len(image_list)
# init progress bar
pbar = ProgressBar(maxval=num_of_images,
widgets=[Percentage(), ' ',
Bar(), ' ',
ETA()])
pbar.start()
# repeat for each image
for i in range(num_of_images):
# get image
image_lab = load_images(os.path.join(test_set_dir_path, image_list[i]))
image_l = image_lab[:, :, 0]
h, w = image_l.shape
# split images to list of images
slices_dim_h = int(math.ceil(h / 32))
slices_dim_w = int(math.ceil(w / 32))
slices = np.zeros((slices_dim_h * slices_dim_w * 4, 32, 32, 1))
for a in range(slices_dim_h * 2 - 1):
for b in range(slices_dim_w * 2 - 1):
part = image_l[a * 32 // 2:a * 32 // 2 + 32,
b * 32 // 2:b * 32 // 2 + 32]
# fill with zero on edges
_part = np.zeros((32, 32))
_part[:part.shape[0], :part.shape[1]] = part
slices[a * slices_dim_w * 2 + b] = _part[:, :, np.newaxis]
# lover originals dimension to 224x224 to feed vgg and increase dim
image_lab_224_b = resize_image_lab(image_lab, (224, 224))
image_l_224 = np.repeat(image_lab_224_b[:, :, 0, np.newaxis],
3,
axis=2).astype(float)
# append together booth lists
input_data = [
slices,
np.array([
image_l_224,
] * slices_dim_h * slices_dim_w * 4)
]
# predict
predictions_ab = model.predict(input_data, batch_size=32)
# for histograms -> transformation from hist to ab
if model.name == "class_wo_weights" or model.name == "class_with_weights":
indices = np.argmax(predictions_ab[:, :, :, :], axis=3)
predictions_a = indices // 20 * 10 - 100 + 5
predictions_b = indices % 20 * 10 - 100 + 5 # +5 to set in the middle box
predictions_ab = np.stack((predictions_a, predictions_b), axis=3)
# reshape back to original size
original_size_im = np.zeros((slices_dim_h * 32, slices_dim_w * 32, 2))
o_h, o_w = original_size_im.shape[:2]
for n in range(predictions_ab.shape[0]):
a, b = n // (slices_dim_w * 2) * 16, n % (slices_dim_w * 2) * 16
if a + 32 > o_h or b + 32 > o_w:
continue # it is empty edge
# weight decision
if a == 0 and b == 0:
weight = weight_top_left
elif a == 0 and b == o_w - 32:
weight = weight_top_right
elif a == 0:
weight = weight_top
elif a == o_h - 32 and b == 0:
weight = weight_bottom_left
elif b == 0:
weight = weight_left
elif a == o_h - 32 and b == o_w - 32:
weight = weight_bottom_right
elif a == o_h - 32:
weight = weight_bottom
elif b == o_w - 32:
weight = weight_right
else:
weight = weight_m
im_a = predictions_ab[n, :, :, 0] * weight
im_b = predictions_ab[n, :, :, 1] * weight
original_size_im[a:a + 32, b:b + 32, :] += np.stack((im_a, im_b),
axis=2)
# make original shape image
original_size_im = original_size_im[:h, :w]
# to rgb
color_im = np.concatenate(
(image_l[:, :, np.newaxis], original_size_im), axis=2)
im_rgb = color.lab2rgb(color_im)
# save
abs_svave_path = get_abs_path(data_destination)
scipy.misc.toimage(im_rgb, cmin=0.0,
cmax=1.0).save(abs_svave_path + model.name + "_" +
image_list[i])
# update progress bar
pbar.update(i + 1)
# finish progress bar
pbar.finish()
def rest_allspec(overwrite=False):
"""Load and interpolate *ALL* HST FOS/GHRS starburst spectra
on to the same rest-frame wavelength grid
"""
path = join(datapath.hstfos_path(), _subpath, 'corrected')
# check output files
bands = _allinone_rest_bands
for thisband in bands:
# check outfiles
outfile = allinone_rest_filename(thisband)
if isfile(outfile) and not overwrite:
print "File {0} exists. Use overwrite to overwrite it.".format(outfile)
return -1
# print "Will write into these files: {0}".format(outfile)
# read in the starburst catalog
objs_ori = starburst_readin()
nobj = objs_ori.size
# make a temporary new catalog
objs_dtype = [('RA', 'f8'),
('DEC', 'f8'),
('Z', 'f8'),
('gal', 'S15')]
objs = np.zeros(nobj, dtype=objs_dtype)
objs['RA'] = 0.
objs['DEC'] = 0.
objs['Z'] = 0.
objs['gal'] = objs_ori['gal']
# read in master wavelength grid
master_wave = (aio.allinone_wave_readin())[0]['WAVE']
master_loglam = np.log10(master_wave)
nwave = master_wave.size
# initialization, nobj second dimension because of NMF traditions
rest_allflux = np.zeros((nwave, nobj))
rest_allivar = np.zeros((nwave, nobj))
# Wavelength
wave_pos = np.array([1000., 3300.])
rest_loc = np.searchsorted(master_wave, wave_pos)
newloglam = master_loglam[rest_loc[0]:rest_loc[1]]
flux = np.zeros((objs.size, newloglam.size))
ivar = np.zeros((objs.size, newloglam.size))
pbar = ProgressBar(maxval=nobj).start()
# Progress bar
for (iobj, thisobj) in zip(np.arange(objs.size), objs):
pbar.update(iobj)
thisdata = readspec_rest(thisobj)
inloglam = np.log10(thisdata['wave'])
influx = thisdata['flux']
inivar = 1./np.power(thisdata['error'], 2)
(rest_allflux[rest_loc[0]:rest_loc[1], iobj], rest_allivar[rest_loc[0]:rest_loc[1], iobj]) = specutils.interpol_spec(inloglam, influx, inivar, newloglam)
#Progress bar
pbar.finish()
# write out
print "Now I am writing everything out..."
allinone_rest_writeout(objs, master_wave, rest_allflux, rest_allivar, overwrite=overwrite)
def scan(self,
clock_en1=True,
pixels=512,
clock_en2=True,
trigger_en=True,
measure_direction=True,
offset=15,
mask_steps=4,
PrmpVbpDac=80,
vthin2Dac=0,
columns=[True] * 16,
vthin1Dac=80,
preCompVbnDac=50,
mask_filename='',
**kwargs):
'''Scan loop
Parameters
----------
mask : int
Number of mask steps.
repeat : int
Number of injections.
'''
inj_factor = 1.0
INJ_LO = 0.0
#try:
#dut = Dut(ScanBase.get_basil_dir(self)+'/examples/lab_devices/agilent33250a_pyserial.yaml')
#dut.init()
#logging.info('Connected to '+str(dut['Pulser'].get_info()))
#except RuntimeError:
#INJ_LO = 0.0#0.2
#inj_factor = 2.0
#logging.info('External injector not connected. Switch to internal one')
#self.dut['INJ_LO'].set_voltage(INJ_LO, unit='V')
offset = offset - 1
vthin1Dac = vthin1Dac + 1
self.dut['global_conf']['PrmpVbpDac'] = 80
self.dut['global_conf']['vthin1Dac'] = 255
self.dut['global_conf']['vthin2Dac'] = 0
self.dut['global_conf']['vffDac'] = 24
self.dut['global_conf']['PrmpVbnFolDac'] = 51
self.dut['global_conf']['vbnLccDac'] = 1
self.dut['global_conf']['compVbnDac'] = 25
self.dut['global_conf']['preCompVbnDac'] = 50
self.dut.write_global()
self.dut['control']['RESET'] = 0b01
self.dut['control']['DISABLE_LD'] = 0
self.dut['control']['PIX_D_CONF'] = 0
self.dut['control'].write()
self.dut['control']['CLK_OUT_GATE'] = 1
self.dut['control']['CLK_BX_GATE'] = 1
self.dut['control'].write()
time.sleep(0.1)
self.dut['control']['RESET'] = 0b11
self.dut['control'].write()
self.dut['global_conf']['OneSr'] = 1
self.dut['global_conf']['TestHit'] = 0
self.dut['global_conf']['SignLd'] = 0
self.dut['global_conf']['InjEnLd'] = 0
self.dut['global_conf']['TDacLd'] = 0
self.dut['global_conf']['PixConfLd'] = 0
self.dut.write_global()
#self.dut['global_conf']['OneSr'] = 0 #all multi columns in parallel
self.dut['global_conf']['ColEn'][:] = bitarray.bitarray([True] *
16) #(columns)
self.dut['global_conf']['ColSrEn'][:] = bitarray.bitarray([True] * 16)
self.dut.write_global()
self.dut['pixel_conf'].setall(False)
self.dut.write_pixel()
self.dut['global_conf']['InjEnLd'] = 1
self.dut.write_global()
self.dut['global_conf']['InjEnLd'] = 0
mask_en = np.full([64, 64], False, dtype=np.bool)
mask_tdac = np.full([64, 64], 16, dtype=np.uint8)
###
if pixels > 1 and pixels <= 64:
mask_en[1:2, :] = True
###
if pixels == 1:
mask_en[1][1] = True
if mask_filename:
logging.info('Using pixel mask from file: %s', mask_filename)
with tb.open_file(mask_filename, 'r') as in_file_h5:
mask_tdac = in_file_h5.root.scan_results.tdac_mask[:]
if pixels > 64:
mask_en = in_file_h5.root.scan_results.en_mask[:]
self.dut.write_en_mask(mask_en)
self.dut.write_tune_mask(mask_tdac)
self.dut.write_global()
self.dut['global_conf']['OneSr'] = 0
self.dut.write_global()
self.dut['trigger'].set_delay(10000) #trigger for no injection 10000
self.dut['trigger'].set_width(16) #16
self.dut['trigger'].set_repeat(1)
self.dut['trigger'].set_en(False)
logging.debug('Configure TDC')
self.dut['tdc']['RESET'] = True
self.dut['tdc']['EN_TRIGGER_DIST'] = True
self.dut['tdc']['ENABLE_EXTERN'] = False
self.dut['tdc']['EN_ARMING'] = False
self.dut['tdc']['EN_INVERT_TRIGGER'] = False
self.dut['tdc']['EN_INVERT_TDC'] = False
self.dut['tdc']['EN_WRITE_TIMESTAMP'] = True
lmask = [1] + ([0] * (mask_steps - 1))
lmask = lmask * ((64 * 64) / mask_steps + 1)
lmask = lmask[:64 * 64]
ranges = np.arange(0, (vthin1Dac - offset), 1)
n = 0
for ni in ranges:
time.sleep(0.5)
bv_mask = bitarray.bitarray(lmask)
if measure_direction:
vthin1Dac1 = vthin1Dac - n
else:
vthin1Dac1 = n + offset
with self.readout(scan_param_id=vthin1Dac1): #vthin1Dac-n):
logging.info('Scan Parameter: %f (%d of %d)', vthin1Dac1,
n + 1, vthin1Dac - offset)
pbar = ProgressBar(maxval=mask_steps).start()
self.dut['global_conf']['vthin1Dac'] = 255
self.dut['global_conf']['preCompVbnDac'] = 50
self.dut['global_conf']['vthin2Dac'] = 0
self.dut['global_conf']['PrmpVbpDac'] = 80
self.dut.write_global()
time.sleep(0.1)
self.dut['pixel_conf'][:] = bv_mask
self.dut.write_pixel_col()
self.dut['global_conf']['InjEnLd'] = 0 #1
#self.dut['global_conf']['PixConfLd'] = 0b11
self.dut.write_global()
bv_mask[1:] = bv_mask[0:-1]
bv_mask[0] = 0
self.dut['global_conf']['vthin1Dac'] = vthin1Dac1
self.dut['global_conf']['preCompVbnDac'] = preCompVbnDac
self.dut['global_conf']['vthin2Dac'] = vthin2Dac
self.dut['global_conf']['PrmpVbpDac'] = PrmpVbpDac
self.dut.write_global()
time.sleep(0.1)
#while not self.dut['inj'].is_done():
#pass
if trigger_en == True:
self.dut['trigger'].set_repeat(0)
if clock_en1 == False:
self.dut['control']['CLK_BX_GATE'] = 0
self.dut['control'].write()
if clock_en2 == False:
self.dut['control']['CLK_OUT_GATE'] = 0
self.dut['control'].write()
if trigger_en == True:
self.dut['trigger'].start()
self.dut['tdc'].ENABLE = True
time.sleep(5) #10
self.dut['tdc'].ENABLE = False
#n=0
#if trigger_en == True:
#while not self.dut['trigger'].is_done():
#time.sleep(1)
#n=n+1
#print self.dut['trigger'].is_done() , n, "sekunden"
if clock_en1 == False:
self.dut['control']['CLK_BX_GATE'] = 1
self.dut['control'].write()
if clock_en2 == False:
self.dut['control']['CLK_OUT_GATE'] = 1
self.dut['control'].write()
#while not self.dut['trigger'].is_done():
#pass
n = n + 1
scan_results = self.h5_file.create_group("/", 'scan_results',
'Scan Masks')
self.h5_file.createCArray(scan_results, 'tdac_mask', obj=mask_tdac)
self.h5_file.createCArray(scan_results, 'en_mask', obj=mask_en)
class Installer(object):
def __init__(self,
install_config,
maxy=0,
maxx=0,
iso_installer=False,
rpm_path="../stage/RPMS",
log_path="../stage/LOGS",
ks_config=None):
self.install_config = install_config
self.ks_config = ks_config
self.iso_installer = iso_installer
self.rpm_path = rpm_path
self.log_path = log_path
self.mount_command = "./mk-mount-disk.sh"
self.prepare_command = "./mk-prepare-system.sh"
self.finalize_command = "./mk-finalize-system.sh"
self.install_package_command = "./mk-install-package.sh"
self.chroot_command = "./mk-run-chroot.sh"
self.setup_grub_command = "./mk-setup-grub.sh"
self.unmount_disk_command = "./mk-unmount-disk.sh"
if self.iso_installer:
self.working_directory = "/mnt/photon-root"
elif 'working_directory' in self.install_config:
self.working_directory = self.install_config['working_directory']
else:
self.working_directory = "/mnt/photon-root"
self.photon_root = self.working_directory + "/photon-chroot"
self.restart_command = "shutdown"
if self.iso_installer:
self.output = open(os.devnull, 'w')
else:
self.output = None
if self.iso_installer:
#initializing windows
self.maxy = maxy
self.maxx = maxx
self.height = 10
self.width = 75
self.progress_padding = 5
self.progress_width = self.width - self.progress_padding
self.starty = (self.maxy - self.height) / 2
self.startx = (self.maxx - self.width) / 2
self.window = Window(self.height,
self.width,
self.maxy,
self.maxx,
'Installing Photon',
False,
items=[])
self.progress_bar = ProgressBar(
self.starty + 3, self.startx + self.progress_padding / 2,
self.progress_width)
signal.signal(signal.SIGINT, self.exit_gracefully)
# This will be called if the installer interrupted by Ctrl+C or exception
def exit_gracefully(self, signal, frame):
if self.iso_installer:
self.progress_bar.hide()
self.window.addstr(
0, 0,
'Opps, Installer got interrupted.\n\nPress any key to get to the bash...'
)
self.window.content_window().getch()
modules.commons.dump(modules.commons.LOG_FILE_NAME)
sys.exit(1)
def install(self, params):
try:
return self.unsafe_install(params)
except Exception as inst:
if self.iso_installer:
modules.commons.log(modules.commons.LOG_ERROR, repr(inst))
self.exit_gracefully(None, None)
else:
raise
def unsafe_install(self, params):
if self.iso_installer:
self.window.show_window()
self.progress_bar.initialize('Initializing installation...')
self.progress_bar.show()
#self.rpm_path = "https://dl.bintray.com/vmware/photon_release_1.0_TP2_x86_64"
if self.rpm_path.startswith(
"https://") or self.rpm_path.startswith("http://"):
cmdoption = 's/baseurl.*/baseurl={}/g'.format(
self.rpm_path.replace('/', '\/'))
process = subprocess.Popen([
'sed', '-i', cmdoption, '/etc/yum.repos.d/photon-iso.repo'
])
retval = process.wait()
if retval != 0:
modules.commons.log(modules.commons.LOG_INFO,
"Failed to reset repo")
self.exit_gracefully(None, None)
cmdoption = 's/cachedir=\/var/cachedir={}/g'.format(
self.photon_root.replace('/', '\/'))
process = subprocess.Popen(
['sed', '-i', cmdoption, '/etc/tdnf/tdnf.conf'])
retval = process.wait()
if retval != 0:
modules.commons.log(modules.commons.LOG_INFO,
"Failed to reset tdnf cachedir")
self.exit_gracefully(None, None)
self.execute_modules(modules.commons.PRE_INSTALL)
self.initialize_system()
if self.iso_installer:
self.get_size_of_packages()
selected_packages = self.install_config['packages']
for package in selected_packages:
self.progress_bar.update_message(
'Installing {0}...'.format(package))
process = subprocess.Popen([
'tdnf', 'install', package, '--installroot',
self.photon_root, '--nogpgcheck', '--assumeyes'
],
stdout=self.output,
stderr=subprocess.STDOUT)
retval = process.wait()
# 0 : succeed; 137 : package already installed; 65 : package not found in repo.
if retval != 0 and retval != 137:
modules.commons.log(
modules.commons.LOG_ERROR,
"Failed install: {} with error code {}".format(
package, retval))
self.exit_gracefully(None, None)
self.progress_bar.increment(self.size_of_packages[package])
else:
#install packages
for rpm in self.rpms_tobeinstalled:
# We already installed the filesystem in the preparation
if rpm['package'] == 'filesystem':
continue
return_value = self.install_package(rpm['filename'])
if return_value != 0:
self.exit_gracefully(None, None)
if self.iso_installer:
self.progress_bar.show_loading('Finalizing installation')
self.finalize_system()
if not self.install_config['iso_system']:
# Execute post installation modules
self.execute_modules(modules.commons.POST_INSTALL)
# install grub
try:
if self.install_config['boot'] == 'bios':
process = subprocess.Popen([
self.setup_grub_command, '-w', self.photon_root,
"bios", self.install_config['disk']['disk'],
self.install_config['disk']['root'],
self.install_config['disk']['boot'],
self.install_config['disk']['bootdirectory']
],
stdout=self.output)
elif self.install_config['boot'] == 'efi':
process = subprocess.Popen([
self.setup_grub_command, '-w', self.photon_root, "efi",
self.install_config['disk']['disk'],
self.install_config['disk']['root'],
self.install_config['disk']['boot'],
self.install_config['disk']['bootdirectory']
],
stdout=self.output)
except:
#install bios if variable is not set.
process = subprocess.Popen([
self.setup_grub_command, '-w', self.photon_root, "bios",
self.install_config['disk']['disk'],
self.install_config['disk']['root'],
self.install_config['disk']['boot'],
self.install_config['disk']['bootdirectory']
],
stdout=self.output)
retval = process.wait()
self.update_fstab()
command = [self.unmount_disk_command, '-w', self.photon_root]
if not self.install_config['iso_system']:
command.extend(self.generate_partitions_param(reverse=True))
process = subprocess.Popen(command, stdout=self.output)
retval = process.wait()
if self.iso_installer:
self.progress_bar.hide()
self.window.addstr(
0, 0,
'Congratulations, Photon has been installed in {0} secs.\n\nPress any key to continue to boot...'
.format(self.progress_bar.time_elapsed))
if self.ks_config == None:
self.window.content_window().getch()
return ActionResult(True, None)
def copy_rpms(self):
# prepare the RPMs list
rpms = []
for root, dirs, files in os.walk(self.rpm_path):
for name in files:
file = os.path.join(root, name)
size = os.path.getsize(file)
rpms.append({'filename': name, 'path': file, 'size': size})
self.rpms_tobeinstalled = []
selected_packages = self.install_config['packages']
for package in selected_packages:
pattern = package + '-[0-9]*.rpm'
if (package == 'glibc'):
pattern2 = pattern
else:
pattern2 = package + '-[a-z][0-9]*.rpm'
for rpm in rpms:
if fnmatch.fnmatch(rpm['filename'],
pattern) or fnmatch.fnmatch(
rpm['filename'], pattern2):
rpm['package'] = package
self.rpms_tobeinstalled.append(rpm)
break
# Copy the rpms
for rpm in self.rpms_tobeinstalled:
shutil.copy(rpm['path'], self.photon_root + '/RPMS/')
def copy_files(self):
# Make the photon_root directory if not exits
process = subprocess.Popen(['mkdir', '-p', self.photon_root],
stdout=self.output)
retval = process.wait()
# Copy the installer files
process = subprocess.Popen(
['cp', '-r', "../installer", self.photon_root], stdout=self.output)
retval = process.wait()
# Create the rpms directory
process = subprocess.Popen(['mkdir', '-p', self.photon_root + '/RPMS'],
stdout=self.output)
retval = process.wait()
self.copy_rpms()
def bind_installer(self):
# Make the photon_root/installer directory if not exits
process = subprocess.Popen(
['mkdir', '-p',
os.path.join(self.photon_root, "installer")],
stdout=self.output)
retval = process.wait()
# The function finalize_system will access the file /installer/mk-finalize-system.sh after chroot to photon_root.
# Bind the /installer folder to self.photon_root/installer, so that after chroot to photon_root,
# the file can still be accessed as /installer/mk-finalize-system.sh.
process = subprocess.Popen([
'mount', '--bind', '/installer',
os.path.join(self.photon_root, "installer")
],
stdout=self.output)
retval = process.wait()
def update_fstab(self):
fstab_file = open(os.path.join(self.photon_root, "etc/fstab"), "w")
fstab_file.write("#system\tmnt-pt\ttype\toptions\tdump\tfsck\n")
for partition in self.install_config['disk']['partitions']:
options = 'defaults'
dump = 1
fsck = 2
if 'mountpoint' in partition and partition['mountpoint'] == '/':
options = options + ',barrier,noatime,noacl,data=ordered'
fsck = 1
if partition['filesystem'] == 'swap':
mountpoint = 'swap'
dump = 0
fsck = 0
else:
mountpoint = partition['mountpoint']
fstab_file.write("{}\t{}\t{}\t{}\t{}\t{}\n".format(
partition['path'], mountpoint, partition['filesystem'],
options, dump, fsck))
# Add the cdrom entry
fstab_file.write("/dev/cdrom\t/mnt/cdrom\tiso9660\tro,noauto\t0\t0\n")
fstab_file.close()
def generate_partitions_param(self, reverse=False):
if reverse:
step = -1
else:
step = 1
params = []
for partition in self.install_config['disk']['partitions'][::step]:
if partition["filesystem"] == "swap":
continue
params.extend([
'--partitionmountpoint', partition["path"],
partition["mountpoint"]
])
return params
def initialize_system(self):
#Setup the disk
if (not self.install_config['iso_system']):
command = [self.mount_command, '-w', self.photon_root]
command.extend(self.generate_partitions_param())
process = subprocess.Popen(command, stdout=self.output)
retval = process.wait()
if self.iso_installer:
self.bind_installer()
process = subprocess.Popen(
[self.prepare_command, '-w', self.photon_root, 'install'],
stdout=self.output)
retval = process.wait()
else:
self.copy_files()
#Setup the filesystem basics
process = subprocess.Popen(
[self.prepare_command, '-w', self.photon_root],
stdout=self.output)
retval = process.wait()
def finalize_system(self):
#Setup the disk
shutil.copy("/etc/resolv.conf", self.photon_root + '/etc/.')
process = subprocess.Popen([
self.chroot_command, '-w', self.photon_root, self.finalize_command,
'-w', self.photon_root
],
stdout=self.output)
retval = process.wait()
if self.iso_installer:
modules.commons.dump(modules.commons.LOG_FILE_NAME)
shutil.copy(modules.commons.LOG_FILE_NAME,
self.photon_root + '/var/log/')
# unmount the installer directory
process = subprocess.Popen(
['umount',
os.path.join(self.photon_root, "installer")],
stdout=self.output)
retval = process.wait()
# remove the installer directory
process = subprocess.Popen(
['rm', '-rf',
os.path.join(self.photon_root, "installer")],
stdout=self.output)
retval = process.wait()
# Disable the swap file
process = subprocess.Popen(['swapoff', '-a'], stdout=self.output)
retval = process.wait()
# remove the tdnf cache directory and the swapfile.
process = subprocess.Popen(
['rm', '-rf',
os.path.join(self.photon_root, "cache")],
stdout=self.output)
def install_package(self, package_name):
rpm_params = ''
os.environ["RPMROOT"] = self.rpm_path
rpm_params = rpm_params + ' --force '
rpm_params = rpm_params + ' --root ' + self.photon_root
rpm_params = rpm_params + ' --dbpath /var/lib/rpm '
if ('type' in self.install_config and
(self.install_config['type']
in ['micro', 'minimal'])) or self.install_config['iso_system']:
rpm_params = rpm_params + ' --excludedocs '
process = subprocess.Popen([
self.install_package_command, '-w', self.photon_root, package_name,
rpm_params
],
stdout=self.output)
return process.wait()
def execute_modules(self, phase):
modules_paths = glob.glob('modules/m_*.py')
for mod_path in modules_paths:
module = mod_path.replace('/', '.', 1)
module = os.path.splitext(module)[0]
try:
__import__(module)
mod = sys.modules[module]
except ImportError:
modules.commons.log(modules.commons.LOG_ERROR,
'Error importing module {}'.format(module))
continue
# the module default is disabled
if not hasattr(mod, 'enabled') or mod.enabled == False:
modules.commons.log(modules.commons.LOG_INFO,
"module {} is not enabled".format(module))
continue
# check for the install phase
if not hasattr(mod, 'install_phase'):
modules.commons.log(
modules.commons.LOG_ERROR,
"Error: can not defind module {} phase".format(module))
continue
if mod.install_phase != phase:
modules.commons.log(
modules.commons.LOG_INFO,
"Skipping module {0} for phase {1}".format(module, phase))
continue
if not hasattr(mod, 'execute'):
modules.commons.log(
modules.commons.LOG_ERROR,
"Error: not able to execute module {}".format(module))
continue
mod.execute(module, self.ks_config, self.install_config,
self.photon_root)
def get_install_size_of_a_package(self, name_size_pairs, package):
modules.commons.log(modules.commons.LOG_INFO,
"Find the install size of: {} ".format(package))
for index, name in enumerate(name_size_pairs, start=0):
if name[name.find(":") + 1:].strip() == package.strip():
item = name_size_pairs[index + 1]
size = item[item.find("(") + 1:item.find(")")]
return int(size)
raise LookupError(
"Cannot find package {} in the repo.".format(package))
def get_size_of_packages(self):
#call tdnf info to get the install size of all the packages.
process = subprocess.Popen(
['tdnf', 'info', '--installroot', self.photon_root],
stdout=subprocess.PIPE)
out, err = process.communicate()
if err != None and err != 0:
modules.commons.log(
modules.commons.LOG_ERROR,
"Failed to get infomation from : {} with error code {}".format(
package, err))
name_size_pairs = re.findall("(?:^Name.*$)|(?:^.*Install Size.*$)",
out, re.M)
selected_packages = self.install_config['packages']
self.size_of_packages = {}
progressbar_num_items = 0
for package in selected_packages:
size = self.get_install_size_of_a_package(name_size_pairs, package)
progressbar_num_items += size
self.size_of_packages[package] = size
self.progress_bar.update_num_items(progressbar_num_items)
def run(self, command, comment=None):
if comment != None:
modules.commons.log(modules.commons.LOG_INFO,
"Installer: {} ".format(comment))
self.progress_bar.update_loading_message(comment)
modules.commons.log(modules.commons.LOG_INFO,
"Installer: {} ".format(command))
process = subprocess.Popen([command],
shell=True,
stdout=subprocess.PIPE)
out, err = process.communicate()
if err != None and err != 0 and "systemd-tmpfiles" not in command:
modules.commons.log(
modules.commons.LOG_ERROR,
"Installer: failed in {} with error code {}".format(
command, err))
modules.commons.log(modules.commons.LOG_ERROR, out)
self.exit_gracefully(None, None)
return err
shutil.copy2(
Path(project_dir) / 'libcblite.so.sym', f'./libcblite-{args.version}')
os.chdir(workspace)
package_name = f'{args.product}-{args.edition}-{args.version}-{args.bld_num}-{args.os}.tar.gz'
print()
print(f"=== Creating {workspace}/{package_name} package ===")
print()
os.chdir(str(workspace_path / 'build_release'))
shutil.copy2(
workspace_path / 'product-texts' / 'mobile' / 'couchbase-lite' /
'license' / f'LICENSE_{args.edition}.txt',
f'libcblite-{args.version}/LICENSE.txt')
pbar = ProgressBar(maxval=3)
pbar.start()
with tarfile.open(f'{workspace}/{package_name}', 'w:gz') as tar:
tar.add(f'libcblite-{args.version}/include', recursive=True)
pbar.update(1)
tar.add(f'libcblite-{args.version}/lib', recursive=True)
pbar.update(2)
tar.add(f'libcblite-{args.version}/LICENSE.txt')
pbar.update(3)
pbar.finish()
symbols_package_name = f'{args.product}-{args.edition}-{args.version}-{args.bld_num}-{args.os}-symbols.tar.gz'
with tarfile.open(f'{workspace}/{symbols_package_name}', 'w:gz') as tar:
tar.add(f'libcblite-{args.version}/libcblite.so.sym')
os.chdir(workspace)
def upload_files(binary_filename, updown_client):
"""Upload a binary file to the Store.
Submit a file to the Store upload service and return the
corresponding upload_id.
"""
result = {'success': False, 'errors': []}
try:
binary_file_size = os.path.getsize(binary_filename)
binary_file = open(binary_filename, 'rb')
encoder = MultipartEncoder(
fields={
'binary': ('filename', binary_file, 'application/octet-stream')
}
)
# Create a progress bar that looks like: Uploading foo [== ] 50%
progress_bar = ProgressBar(
widgets=['Uploading {} '.format(binary_filename),
Bar(marker='=', left='[', right=']'), ' ', Percentage()],
maxval=os.path.getsize(binary_filename))
progress_bar.start()
# Print a newline so the progress bar has some breathing room.
logger.info('')
# Create a monitor for this upload, so that progress can be displayed
monitor = MultipartEncoderMonitor(
encoder, functools.partial(_update_progress_bar, progress_bar,
binary_file_size))
# Begin upload
response = updown_client.upload(monitor)
# Make sure progress bar shows 100% complete
progress_bar.finish()
if response.ok:
response_data = response.json()
result.update({
'success': response_data.get('successful', True),
'upload_id': response_data['upload_id'],
'binary_filesize': os.path.getsize(binary_filename),
'source_uploaded': False,
})
else:
logger.error(
'There was an error uploading the package.\n'
'Reason: %s\n'
'Text: %s',
response.reason, response.text)
result['errors'] = [response.text]
except Exception as err:
logger.exception(
'An unexpected error was found while uploading files.')
result['errors'] = [str(err)]
finally:
# Close the open file
binary_file.close()
return result
def setup(self):
"""
Checks the location of the jar files.
Spawns the server as a process.
"""
jars = [
"stanford-corenlp-3.2.0.jar", "stanford-corenlp-3.2.0-models.jar",
"joda-time.jar", "xom.jar", "jollyday.jar"
]
# if CoreNLP libraries are in a different directory,
# change the corenlp_path variable to point to them
corenlp_path = os.path.relpath(__file__).split(
'/')[0] + "/stanford-corenlp-full-2013-06-20/"
#corenlp_path = "stanford-corenlp-full-2013-06-20/"
java_path = "java"
classname = "edu.stanford.nlp.pipeline.StanfordCoreNLP"
# include the properties file, so you can change defaults
# but any changes in output format will break parse_parser_results()
props = "-props " + os.path.relpath(__file__).split(
'/')[0] + "/default.properties"
# add and check classpaths
jars = [corenlp_path + jar for jar in jars]
for jar in jars:
if not os.path.exists(jar):
print "Error! Cannot locate %s" % jar
sys.exit(1)
#Change from ':' to ';'
# spawn the server
start_corenlp = "%s -Xmx2500m -cp %s %s %s" % (
java_path, ':'.join(jars), classname, props)
if VERBOSE: print start_corenlp
self.corenlp = pexpect.spawn(start_corenlp)
# show progress bar while loading the models
widgets = ['Loading Models: ', Fraction()]
pbar = ProgressBar(widgets=widgets, maxval=4,
force_update=True).start()
self.corenlp.expect("done.",
timeout=20) # Load pos tagger model (~5sec)
pbar.update(1)
self.corenlp.expect("done.",
timeout=200) # Load NER-all classifier (~33sec)
pbar.update(2)
self.corenlp.expect("done.",
timeout=600) # Load NER-muc classifier (~60sec)
pbar.update(3)
self.corenlp.expect("done.",
timeout=600) # Load CoNLL classifier (~50sec)
pbar.update(4)
# self.corenlp.expect("done.", timeout=200) # Loading PCFG (~3sec)
# pbar.update(5)
self.corenlp.expect("Entering interactive shell.")
pbar.finish()
else:
fname = sys.argv[1]
if len(sys.argv) >= 4:
acctid = sys.argv[2]
secret = sys.argv[3]
if acctid is None:
acctid = raw_input("AWS_ACCESS_KEY_ID: ").strip()
if secret is None:
secret = raw_input("AWS_SECRET_ACCESS_KEY: ").strip()
bucket = "kroll.appcelerator.com"
key = os.path.basename(fname)
conn = S3Connection(acctid, secret)
bucket = conn.get_bucket(bucket)
k = bucket.new_key(key)
pbar = ProgressBar().start()
try:
def progress_callback(current, total):
pbar.update(int(100 * (float(current) / float(total))))
k.set_contents_from_filename(fname,
cb=progress_callback,
num_cb=100,
policy='public-read')
finally:
pbar.finish()
# fill inbox
done = s.query(tables.clangs.program_id) \
.filter(tables.clangs.clang == args.clang)
todo = s.query(tables.programs) \
.filter(~tables.programs.id.in_(done)) \
.order_by(tables.programs.date_added) \
.limit(BATCH_SIZE)
for program in todo:
inbox.append(program)
for tables in tablesets:
if args.recheck:
q = s.query(tables.clang_stderrs)
for stderr in ProgressBar(max_value=q.count())(q):
assertion_ = util.get_assertion(s, tables.clang_assertions,
stderr.stderr)
unreachable_ = util.get_unreachable(
s, tables.clang_unreachables, stderr.stderr)
terminate_ = util.get_terminate(s, tables.clang_terminates,
stderr.stderr)
errs = sum(1 if x else 0
for x in [assertion_, unreachable_, terminate_])
if errs > 1:
raise LookupError(
f"Multiple errors types found in: {stderr}\n\n" +
f"Assertion: {assertion_}\n" +
f"Unreachable: {unreachable_}\n" +
f"Terminate: {terminate_}")
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--category',
'-c',
help='category name',
required=True)
parser.add_argument('--level',
'-l',
type=int,
help='level id',
required=True)
parser.add_argument('--load_ckpt',
'-k',
help='Path to a check point file for load',
required=True)
parser.add_argument('--model', '-m', help='Model to use', required=True)
parser.add_argument('--setting',
'-x',
help='Setting to use',
required=True)
parser.add_argument('--batch_size',
'-b',
help='Batch size during testing',
default=8,
type=int)
parser.add_argument('--save_ply',
'-s',
help='Save results as ply',
action='store_true')
parser.add_argument('--save_dir',
'-o',
help='The output directory',
type=str,
default=None)
parser.add_argument('--save_num_shapes',
'-u',
help='how many shapes to visualize',
default=20,
type=int)
args = parser.parse_args()
print(args)
if args.save_ply:
if os.path.exists(args.save_dir):
print('ERROR: folder %s exists! Please check and delete!' %
args.save_dir)
exit(1)
os.mkdir(args.save_dir)
model = importlib.import_module(args.model)
setting_path = os.path.join(os.path.dirname(__file__), args.model)
sys.path.append(setting_path)
setting = importlib.import_module(args.setting)
sample_num = setting.sample_num
batch_size = args.batch_size
args.data_folder = '../../data/sem_seg_h5/'
# Load all test data
args.filelist = os.path.join(args.data_folder,
'%s-%d' % (args.category, args.level),
'test_files.txt')
data_test, _, label_gt = data_utils.load_seg(args.filelist)
num_shape = data_test.shape[0]
print('Loaded data: %s shapes in total to test.' % num_shape)
# Load current category + level statistics
with open(
'../../stats/after_merging_label_ids/%s-level-%d.txt' %
(args.category, args.level), 'r') as fin:
setting.num_class = len(fin.readlines()) + 1 # with "other"
print('NUM CLASS: %d' % setting.num_class)
######################################################################
# Placeholders
is_training = tf.placeholder(tf.bool, name='is_training')
pts_fts = tf.placeholder(tf.float32,
shape=(batch_size, sample_num, setting.data_dim),
name='points')
######################################################################
######################################################################
pts_fts_sampled = pts_fts
points_sampled = pts_fts_sampled
features_sampled = None
net = model.Net(points_sampled, features_sampled, is_training, setting)
seg_probs_op = tf.nn.softmax(net.logits, name='seg_probs')
# for restore model
saver = tf.train.Saver()
parameter_num = np.sum(
[np.prod(v.shape.as_list()) for v in tf.trainable_variables()])
print('{}-Parameter number: {:d}.'.format(datetime.now(), parameter_num))
# Create a session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.log_device_placement = False
sess = tf.Session(config=config)
# Load the model
ckptstate = tf.train.get_checkpoint_state(args.load_ckpt)
if ckptstate is not None:
LOAD_MODEL_FILE = os.path.join(
args.load_ckpt, os.path.basename(ckptstate.model_checkpoint_path))
saver.restore(sess, LOAD_MODEL_FILE)
print("Model loaded in file: %s" % LOAD_MODEL_FILE)
else:
print("Fail to load modelfile: %s" % args.load_ckpt)
# Start the testing
print('{}-Testing...'.format(datetime.now()))
num_batch = (num_shape - 1) // batch_size + 1
pts_batch = np.zeros((batch_size, sample_num, 3), dtype=np.float32)
avg_acc = 0.0
avg_cnt = 0
shape_iou_tot = 0.0
shape_iou_cnt = 0
part_intersect = np.zeros((setting.num_class), dtype=np.float32)
part_union = np.zeros((setting.num_class), dtype=np.float32)
bar = ProgressBar()
all_seg_probs = []
for batch_idx in bar(range(num_batch)):
start_idx = batch_idx * batch_size
end_idx = min((batch_idx + 1) * batch_size, num_shape)
pts_batch[:end_idx - start_idx, ...] = data_test[start_idx:end_idx]
seg_probs = sess.run(seg_probs_op,
feed_dict={
pts_fts: pts_batch,
is_training: False
})
seg_probs = seg_probs[:end_idx - start_idx]
all_seg_probs.append(seg_probs)
seg_res = np.argmax(seg_probs[:, :, 1:], axis=-1) + 1
avg_acc += np.sum(
np.mean((seg_res == label_gt[start_idx:end_idx]) |
(label_gt[start_idx:end_idx] == 0),
axis=-1))
avg_cnt += end_idx - start_idx
seg_gt = label_gt[start_idx:end_idx]
seg_res[seg_gt == 0] = 0
for i in range(end_idx - start_idx):
cur_pred = seg_res[i]
cur_gt = seg_gt[i]
cur_shape_iou_tot = 0.0
cur_shape_iou_cnt = 0
for j in range(1, setting.num_class):
cur_gt_mask = (cur_gt == j)
cur_pred_mask = (cur_pred == j)
has_gt = (np.sum(cur_gt_mask) > 0)
has_pred = (np.sum(cur_pred_mask) > 0)
if has_gt or has_pred:
intersect = np.sum(cur_gt_mask & cur_pred_mask)
union = np.sum(cur_gt_mask | cur_pred_mask)
iou = intersect / union
cur_shape_iou_tot += iou
cur_shape_iou_cnt += 1
part_intersect[j] += intersect
part_union[j] += union
if cur_shape_iou_cnt > 0:
cur_shape_miou = cur_shape_iou_tot / cur_shape_iou_cnt
shape_iou_tot += cur_shape_miou
shape_iou_cnt += 1
if args.save_ply and start_idx < args.save_num_shapes:
for i in range(start_idx, min(end_idx, args.save_num_shapes)):
out_fn = os.path.join(args.save_dir, 'shape-%02d-pred.ply' % i)
data_utils.save_ply_property(data_test[i],
seg_res[i - start_idx],
setting.num_class, out_fn)
out_fn = os.path.join(args.save_dir, 'shape-%02d-gt.ply' % i)
data_utils.save_ply_property(data_test[i], label_gt[i],
setting.num_class, out_fn)
all_seg_probs = np.vstack(all_seg_probs)
np.save('out.npy', all_seg_probs)
print('{}-Done!'.format(datetime.now()))
print('Average Accuracy: %f' % (avg_acc / avg_cnt))
print('Shape mean IoU: %f' % (shape_iou_tot / shape_iou_cnt))
part_iou = np.divide(part_intersect[1:], part_union[1:])
mean_part_iou = np.mean(part_iou)
print('Category mean IoU: %f, %s' % (mean_part_iou, str(part_iou)))
out_list = ['%3.1f' % (item * 100) for item in part_iou.tolist()]
print('%3.1f;%3.1f;%3.1f;%s' %
(avg_acc * 100 / avg_cnt, shape_iou_tot * 100 / shape_iou_cnt,
mean_part_iou * 100, '[' + ', '.join(out_list) + ']'))
def _upload_files(sca_client, name, data, result):
data['name'] = name
response = sca_client.snap_upload(data)
if response.ok:
response_data = response.json()
status_url = response_data['status_url']
# This is just a waiting game, so we'll show an indeterminate
# AnimatedMarker for it.
progress_indicator = ProgressBar(
widgets=['Checking package status... ', AnimatedMarker()],
maxval=7)
progress_indicator.start()
# Execute the package scan in another thread so we can update the
# progress indicator.
with ThreadPoolExecutor(max_workers=1) as executor:
future = executor.submit(get_scan_data, sca_client, status_url)
count = 0
while not future.done():
# Annoyingly, there doesn't seem to be a way to actually
# make a progress indicator that will go on forever, so we
# need to restart this one each time we reach the end of
# its animation.
if count >= 7:
progress_indicator.start()
count = 0
# Actually update the progress indicator
progress_indicator.update(count)
count += 1
time.sleep(0.15)
# Grab the results from the package scan
completed, data = future.result()
progress_indicator.finish()
if completed:
message = data.get('message', '')
if not message:
result['success'] = True
result['revision'] = data.get('revision')
else:
result['errors'] = [message]
else:
result['errors'] = [
'Package scan took too long.',
]
status_web_url = response_data.get('web_status_url')
if status_web_url:
result['errors'].append(
'Please check the status later at: {}.'.format(
status_web_url),
)
result['application_url'] = data.get('application_url', '')
else:
logger.error(
'There was an error uploading the application.\n'
'Reason: {}\n'
'Text: {}'.format(response.reason, response.text))
result['errors'] = [response.text]
return result
def generate_subtitles( # pylint: disable=too-many-locals,too-many-arguments
source_path,
output=None,
concurrency=DEFAULT_CONCURRENCY,
src_language=DEFAULT_SRC_LANGUAGE,
dst_language=DEFAULT_DST_LANGUAGE,
subtitle_file_format=DEFAULT_SUBTITLE_FORMAT,
api_key=None,
):
"""
Given an input audio/video file, generate subtitles in the specified language and format.
"""
audio_filename, audio_rate = extract_audio(source_path)
regions = find_speech_regions(audio_filename)
pool = multiprocessing.Pool(concurrency)
converter = FLACConverter(source_path=audio_filename)
recognizer = SpeechRecognizer(language=src_language,
rate=audio_rate,
api_key=GOOGLE_SPEECH_API_KEY)
transcripts = []
if regions:
try:
widgets = [
"Converting speech regions to FLAC files: ",
Percentage(), ' ',
Bar(), ' ',
ETA()
]
pbar = ProgressBar(widgets=widgets, maxval=len(regions)).start()
extracted_regions = []
for i, extracted_region in enumerate(pool.imap(converter,
regions)):
extracted_regions.append(extracted_region)
pbar.update(i)
pbar.finish()
widgets = [
"Performing speech recognition: ",
Percentage(), ' ',
Bar(), ' ',
ETA()
]
pbar = ProgressBar(widgets=widgets, maxval=len(regions)).start()
for i, transcript in enumerate(
pool.imap(recognizer, extracted_regions)):
transcripts.append(transcript)
pbar.update(i)
pbar.finish()
if src_language.split("-")[0] != dst_language.split("-")[0]:
if api_key:
google_translate_api_key = api_key
translator = Translator(dst_language,
google_translate_api_key,
dst=dst_language,
src=src_language)
prompt = "Translating from {0} to {1}: ".format(
src_language, dst_language)
widgets = [prompt, Percentage(), ' ', Bar(), ' ', ETA()]
pbar = ProgressBar(widgets=widgets,
maxval=len(regions)).start()
translated_transcripts = []
for i, transcript in enumerate(
pool.imap(translator, transcripts)):
translated_transcripts.append(transcript)
pbar.update(i)
pbar.finish()
transcripts = translated_transcripts
else:
print(
"Error: Subtitle translation requires specified Google Translate API key. "
"See --help for further information.")
return 1
except KeyboardInterrupt:
pbar.finish()
pool.terminate()
pool.join()
print("Cancelling transcription")
raise
timed_subtitles = [(r, t) for r, t in zip(regions, transcripts) if t]
formatter = FORMATTERS.get(subtitle_file_format)
formatted_subtitles = formatter(timed_subtitles)
dest = output
if not dest:
base = os.path.splitext(source_path)[0]
dest = "{base}.{format}".format(base=base, format=subtitle_file_format)
with open(dest, 'wb') as output_file:
output_file.write(formatted_subtitles.encode("utf-8"))
os.remove(audio_filename)
return dest
organization_id) + '&y=' + str(year_range)
organizations_url = 'http://api.aiddata.org/data/origin/organizations?'
json_orgs = json.load(urllib2.urlopen(organizations_url))
donating_org = ''
# Finds the organization based on the id
for org in json_orgs['hits']:
donating_org = org['name']
print 'Creating map for ' + donating_org
json_result = json.load(urllib2.urlopen(url))
num_projects = json_result['project_count']
count = 0
totamt = 0
country_dict = {}
pbar = ProgressBar(maxval=num_projects).start()
# Iterates over the projects from the AidData api in chunks of 50, the max size allowed by the api
while (count < num_projects):
project_info = getProjectData(count, organization_id, year_range)
for project in project_info['items']:
# Only looks at projects that have transaction values
if 'transactions' in project:
for transactions in project['transactions']:
# Ignores projects that don't indicate a recipient country
if 'tr_receiver_country' in transactions and transactions[
'tr_receiver_country']['iso3'] != '':
donor = transactions['tr_funding_org']['name']
receiver = transactions['tr_receiver_country']['iso3']
amount = transactions['tr_constant_value']
totamt += amount
def with_example22():
try:
with ProgressBar(maxval=-1) as progress:
progress.start()
except ValueError:
pass
def svg_heatmap(data, filename, row_labels=None, box_size=4,
index=None,
cmap=ISH, norm_rows_by=None, draw_row_labels=False,
col_sep='', box_height=None, total_width=None,
draw_box=False, draw_name=False, data_names=None,
progress_bar = False,
max_width=np.inf,
spacers=None,
cmap_by_prefix=None,
split_columns=False,
vspacer=30,
hatch_nan=True, hatch_size=20,
first_col='', last_col=''):
"""
Draw heatmap as an SVG file stored in filename
*data* can be either a 2D array-like type (list of lists, numpy array,
pandas DataFrame, etc), or a tuple of 2D array-likes, in which case a
separator will be added between each one in the output
*cmap* is a matplotlib-like colormap (i.e. a callable that expects floats
in the range 0.0-1.0.), or an iterable of the same length as the tuple
*data* containing colormaps
*row_labels* can be supplied, otherwise they will detected from the first
item in *data*, if available, and if not they will be blank.
If *total_width* is supplied, width of each dataset in *data* will be
scaled to that constant. If *box_height* is supplied, the height of each
row will be *box_height*, otherwise it will be equal to the width of each
element. If neither are supplied, elements will be squares equal to
*box_size*. IT IS STRONGLY RECOMMENDED that if if supplying *total_width*,
*box_height* also be specified, but this is not enforced.
*draw_row_labels*, if True, will label the rows on the right hand side. As
of 2013/09/03, this won't scale the SVG properly, so including the
resulting file in an html element won't display properly.
*spacers* is the distance between adjacent datasets. Can either be a
number, in which case it will apply to all datasets, or an interable for
different distances. If the iterable is shorter than the number of
datasets, the last value will be repeated.
"""
import svgwrite as svg
import pandas as pd
if split_columns and isinstance(data, pd.DataFrame):
from Utils import sel_startswith
colnames = list(sorted(
{col.split(col_sep)[0] for col in data.columns}))
data = tuple(
data.select(**sel_startswith(colname)) for colname in colnames
)
elif not isinstance(data, tuple):
data = (data,)
rows, cols = np.shape(data[0])
if index is not None:
rows = len(index)
if box_height is None:
box_height = box_size
if total_width is not None and max_width is not np.inf:
dwg = svg.Drawing(filename,
size=(max_width,
np.ceil((len(data) * total_width)/max_width)
* (box_height+vspacer)))
else:
dwg = svg.Drawing(filename)
dwg.add(svg.base.Title(path.basename(filename)))
pat = dwg.pattern(id='hatch', insert=(0, 0), size=(hatch_size, hatch_size),
patternUnits='userSpaceOnUse')
g = pat.add(dwg.g(style="fill:none; stroke:#B0B0B0; stroke-width:1"))
g.add(dwg.path(('M0,0', 'l{hatch},{hatch}'.format(hatch=hatch_size))))
g.add(dwg.path(('M{hatch2},0 l{hatch2},{hatch2}'.format(hatch2=hatch_size/2).split())))
g.add(dwg.path(('M0,{hatch2} l{hatch2},{hatch2}'.format(hatch2=hatch_size/2).split())))
dwg.add(pat)
if row_labels is None:
if index is not None:
row_labels = index
elif hasattr(data[0], 'index'):
row_labels = data[0].index
else:
row_labels = ['' for row in range(rows)]
if box_height is None:
box_height = box_size
if not hasattr(cmap, "__len__"):
cmap = [cmap for frame in data]
if data_names is None:
data_names = ["" for frame in data]
if len(cmap) != len(data):
raise ValueError("cmap and data should be the same length")
if not hasattr(spacers, "__len__"):
spacers = [spacers]
else:
spacers = list(spacers)
while len(spacers) < len(data):
spacers.append(spacers[-1])
if not isinstance(norm_rows_by, tuple):
norm_rows_by = repeat(norm_rows_by)
x_start = 0
y_start = 0
y_diff = 0
if progress_bar:
from progressbar import ProgressBar
iterator = zip(data, cmap, data_names, norm_rows_by, spacers)
pbar = ProgressBar(maxval=len(iterator)*rows).start()
pbar_val = 0
else:
iterator = zip(data, cmap, data_names, norm_rows_by, spacers)
for frame, c_cmap, name, normer, spacer in iterator:
if frame is None:
if total_width is not None:
if spacer is None:
x_start += total_width * 1.1
else:
x_start += total_width + spacer
else:
if spacer is None:
x_start += box_size
else:
x_start += spacer
if x_start > max_width:
x_start = 0
y_start += y_diff
continue
frame = pd.DataFrame(frame)
if normer is None:
norm_data = frame.copy()
elif normer is 'mean':
norm_data = frame.divide(frame.dropna(axis=1).mean(axis=1)+10, axis=0)
elif normer is 'max':
norm_data = frame.divide(frame.dropna(axis=1).max(axis=1)+10, axis=0)
elif normer is 'center0':
norm_data = (0.5 +
0.5 * frame.divide(frame.dropna(axis=1).abs().max(axis=1),
axis=0)
)
elif index is not None and hasattr(normer, "ix"):
norm_data = frame.divide(normer.ix[index], axis=0)
elif hasattr(normer, "__len__") and len(normer) == rows:
norm_data = frame.divide(normer, axis=0)
elif hasattr(normer, "__len__"):
raise TypeError("norm_rows_by should be the same shape "
"as the number of rows")
else:
norm_data = frame.divide(normer, axis=0)
if not c_cmap or str(c_cmap).lower() == 'default':
c_cmap = ISH
new_rows, new_cols = np.shape(frame)
if hasattr(frame, 'index'):
col_labels = frame.columns
else:
col_labels = ['' for col in range(new_cols)]
if new_rows != rows:
raise ValueError("All input elements must have the same number of"
" rows (and same row meanings --unchecked)")
if total_width is not None:
box_size = total_width / float(new_cols)
for i in range(rows):
if progress_bar:
pbar.update(pbar_val)
pbar_val += 1
prefix = col_labels[0][:col_labels[0].find(col_sep)]
if cmap_by_prefix:
c_cmap = cmap_by_prefix(prefix)
for j in range(new_cols):
g = dwg.g()
g.add(svg.base.Title("{}, {}: {:.2f}".format(row_labels[i],
col_labels[j],
frame.ix[i, j])))
hatch = not isfinite(norm_data.ix[i, j])
if hatch:
n = 0
norm_data.ix[i, j] = 0
if j > 0:
norm_data.ix[i, j] += norm_data.ix[i, j-1]
n += 1
if j + 1 < len(norm_data.columns):
norm_data.ix[i, j] += norm_data.ix[i, j+1]
n += 1
norm_data.ix[i, j] /= n
g.add(dwg.rect((x_start + box_size*j, y_start + i*box_height),
(box_size, box_height),
style="fill:#{:02x}{:02x}{:02x}"
.format(*[int(255*x) for x in
c_cmap(norm_data.ix[i, j])])))
dwg.add(g)
if hatch_nan and hatch:
g.add(dwg.rect((x_start + box_size*j,
y_start + i*box_height),
(box_size, box_height),
style="fill:url(#hatch)"
)
)
col_base = col_labels[j][:col_labels[j].find(col_sep)]
if col_base != prefix:
prefix = col_base
if cmap_by_prefix:
c_cmap = cmap_by_prefix(prefix)
g.add(dwg.line((x_start + box_size * j,
y_start + i * box_height),
(x_start + box_size * j,
y_start + (i + 1) * box_height),
style="stroke-width:{}; stroke:#000000"
.format(.1 * box_size)))
dwg.add(dwg.text(first_col, (x_start,
y_start + (i + 1) * box_height)))
dwg.add(dwg.text(last_col, (x_start + (new_cols - 1) * box_size,
y_start + (i + 1) * box_height)))
if draw_box:
dwg.add(dwg.rect((x_start, y_start + 0),
(new_cols*box_size, rows*box_height),
style="stroke-width:1; "
"stroke:#000000; fill:none"))
if draw_name:
dwg.add(dwg.text(name,
(x_start + box_size * new_cols / 2.0,
y_start + box_height * (rows) + 13),
style="text-anchor: middle;"))
if total_width is not None:
if spacer is None:
x_start += total_width * 1.1
else:
x_start += total_width + spacer
else:
if spacer is None:
x_start += new_cols * box_size + box_size
else:
x_start += new_cols * box_size + spacer
y_diff = new_rows * box_height + 30
if x_start + total_width >= max_width:
x_start = 0
y_start += new_rows*box_height + vspacer
if draw_row_labels:
for i in range(rows):
dwg.add(dwg.text(row_labels[i],
(x_start, y_start + i*box_height+box_height),
style='font-size:{}'.format(box_height),
))
pbar.finish()
dwg.saveas(filename)
