def process_mnist_input(input_list):
"""
Process a list of inputs, where one input is a list of integers corresponding to the pixels of the images, by
dividing each input by 256 to limit the inputs to values between 0 and 1.
:param input_list: a list of lists of integers, where each list is one input.
:return: a list of numpy column arrays with length 784, with floats between 0 and 1.
"""
# Progress Bar
start_time = time.time()
print("Processing Input")
ProgressBar.draw_bar(0, 30, 0)
inputs_completed = 0
processed_inputs = []
for old_input in input_list:
new_input = []
for integer in old_input:
new_input.append([integer / 256.0])
processed_inputs.append(np.array(new_input))
inputs_completed += 1.0
ProgressBar.draw_bar(inputs_completed / len(input_list), 30,
time.time() - start_time)
print("Input processing took " +
ProgressBar.time_to_string(time.time() - start_time) + ".")
return processed_inputs
def process_mnist_output(output_list):
"""
Process a list of outputs, where one output is an integer
corresponding to the group that the input should be classified under.
:param output_list: a list of integers, where each integer is one output.
:return: a list of numpy arrays with length 10, with 0 at all indices except the integer from data_list
"""
start_time = time.time()
print("Processing Output")
ProgressBar.draw_bar(0, 30, 0)
outputs_completed = 0
processed_outputs = []
for old_output in output_list:
new_output = []
for i in range(10):
if i == old_output:
new_output.append([1])
else:
new_output.append([0])
processed_outputs.append(np.array(new_output))
outputs_completed += 1.0
ProgressBar.draw_bar(outputs_completed / len(output_list), 30,
time.time() - start_time)
print("Output processing took " +
ProgressBar.time_to_string(time.time() - start_time) + ".")
return processed_outputs
def augment_input_outputs(io_list):
"""
Takes a zipped list of inputs and their corresponding outputs
:param io_list: list of tuples, each with one input and its corresponding output
:return: a zipped list of augmented inputs and their corresponding outputs
"""
inputs_completed = 0
new_inputs = []
new_outputs = []
for i, o in io_list:
# doing the original input and output
new_inputs.append(i)
new_outputs.append(o)
inputs_completed += 1.0
ProgressBar.draw_bar(
inputs_completed / len(io_list * num_augments), 30,
time.time() - start_time)
for n in range(num_augments - 1):
new_input = augment_input(i, width, height)
new_inputs.append(new_input)
new_outputs.append(o)
inputs_completed += 1.0
ProgressBar.draw_bar(
inputs_completed / len(io_list * num_augments), 30,
time.time() - start_time)
return list(zip(new_inputs, new_outputs))
def classify_images(net, transformer, img_test_names, img_test_labels, settings):
pb = ProgressBar(len(img_test_names))
cm = ConfusionMatrix(li.labels)
pred_hit = np.zeros(len(img_test_names))
class_preds = []
for i, (id_, true_class) in enumerate(zip(img_test_names, img_test_labels)):
pb.print_progress()
if (settings["bulk"]):
settings["test_dir"] = settings["bulk_dir"]
pred_class = classify_from_bulk(net, transformer, id_, true_class, settings)
else:
pred_class = classify_image(net, transformer, id_, settings)[0]
class_preds.append(pred_class)
if (pred_class == true_class):
pred_hit[i] = 1
cm.actualize(true_class, pred_class)
submission_file = print_submission(img_test_names, class_preds, settings)
accuracy = compute_accuracy(pred_hit)
return submission_file, cm.get_confusion_matrix(), accuracy
def CallOne(self):
self.pgpg = ProgressBar(self.master)
self.pgpg.LoadingTweet()
t = threading.Thread(target=self.getJsonTextFichierDuJour,
args=(self.pgpg, ))
self.pgpg.fil.after(1000, t.start())
self.pgpg.fil.mainloop()
self.pgpg.fil.quit()
def __init__(self, master):
self.master = master
self.TabLinkImage = {}
self.Nom_Dossier = "DonneJson"
self.donne = {}
self.LastTabLinkImage = {}
self.ville = {}
self.pgpg = ProgressBar(self.master)
def regrid_file(self):
"""
regrid the file
"""
progress = ProgressBar(n_iter=1 , total_width= 30 , description='regrid_file')
list = ['ncremap','-i',self.filepath,'-d',self.model_grid,'-o',self.local_temp_file_path]
command = list
subprocess.run(command)
self.filepath = self.local_temp_file_path
progress.update()
print("")
def change_unit(self):
"""
Chage the unit of the dataset
"""
progress = ProgressBar(n_iter=1, total_width= 30 , description='Change Unit')
string1 = 'units,'+self.variable+',o,c,'+self.output_unit
list = ['ncatted','-O','-a',string1,'-o',self.local_file_path]
command = list + [self.filepath]
subprocess.run(command)
self.filepath = self.local_file_path
progress.update()
print("")
def augment_mnist_digits_data(num_augments, width, height):
"""
Takes the mnist digits dataset and augments it, multiplying its size by num_augments.
Writes the new dataset to a pickle file. Requires the mnist digits data to already be written.
:param num_augments: the number of new inputs to create from one base input in the mnist dataset.
:param width: the width of the image in the input
:param height: the height of the image in the input
"""
start_time = time.time()
print("Augmenting Digit Training Data")
ProgressBar.draw_bar(0, 30, 0)
def augment_input_outputs(io_list):
"""
Takes a zipped list of inputs and their corresponding outputs
:param io_list: list of tuples, each with one input and its corresponding output
:return: a zipped list of augmented inputs and their corresponding outputs
"""
inputs_completed = 0
new_inputs = []
new_outputs = []
for i, o in io_list:
# doing the original input and output
new_inputs.append(i)
new_outputs.append(o)
inputs_completed += 1.0
ProgressBar.draw_bar(
inputs_completed / len(io_list * num_augments), 30,
time.time() - start_time)
for n in range(num_augments - 1):
new_input = augment_input(i, width, height)
new_inputs.append(new_input)
new_outputs.append(o)
inputs_completed += 1.0
ProgressBar.draw_bar(
inputs_completed / len(io_list * num_augments), 30,
time.time() - start_time)
return list(zip(new_inputs, new_outputs))
old_training, old_testing = read_mnist_data('mnist_digits')
augmented_train_io = augment_input_outputs(old_training)
augmented_test_io = augment_input_outputs(old_testing)
print("Data augmenting took " +
ProgressBar.time_to_string(time.time() - start_time) + ".")
training_title = 'data/' + 'augmented_digits' + '_training' + '.pickle'
testing_title = 'data/' + 'augmented_digits' + '_testing' + '.pickle'
pickle.dump(augmented_train_io, open(training_title, 'wb'))
pickle.dump(augmented_test_io, open(testing_title, 'wb'))
def run_MCMC_onebin(binnum, nwalkers, burnlinks, links, hist_bincnt):
ndim = 3
p0 = np.random.rand(ndim * nwalkers).reshape((nwalkers, ndim))
sampler = emcee.EnsembleSampler(nwalkers, ndim, lambda param: log_likelihood_onebin(binnum-1,param))
progbar.set_totalsteps(float(burnlinks)*nwalkers)
print 'Begin burn in'
pos, prob, state = sampler.run_mcmc(p0,burnlinks)
progbar.update_progress(1)
print 'Burn in completed'
sampler.reset()
progbar.set_totalsteps(float(links)*nwalkers)
sampler.run_mcmc(pos, links)
progbar.update_progress(1)
print('Main chain completed')
print("Mean acceptance fraction: {0:.3f}".format(np.mean(sampler.acceptance_fraction)))
# Determine the 95% confidence level by sorting the marginalized chain and finding the
# eta element that is 95% of the way down the sorted chain
sorted_eta = np.sort(sampler.flatchain[:,0])
conf95_index = np.round(0.95*len(sorted_eta) - 1).astype(np.int)
conf998_index = np.round(0.998*len(sorted_eta) - 1).astype(np.int)
conf95 = sorted_eta[conf95_index]
conf998 = sorted_eta[conf998_index]
print '95% upper limit: ' + str(conf95)
plt.figure()
y, x, o = plt.hist(sampler.flatchain[:,0], hist_bincnt, range=[0,conf998], histtype="step", normed = 1)
yciel = 1.1*y.max()
plt.ylim([0,yciel])
plt.vlines(conf95, 0, yciel ,colors='r')
plt.title('Dark Matter Signal Posterior PDF (bin %d)' % binnum,fontsize = 16)
plt.xlabel(r'Signal Strength [$\eta$]',fontsize = 14)
plt.ylabel(r'Probability Density', fontsize = 14)
plt.text(1.05*conf95,0.75*yciel, '95% conf: {0:.3f}'.format(conf95), bbox=dict(facecolor='red',alpha=0.5))
plt.show()
def train(self):
self.k = 0
pg = ProgressBar()
num_examples = len(self.x)
for i in range(0, num_examples):
if self.y[i] * (self.dot_pdt(self.x[i],self.w) + self.b) <= 0:
self.w = self.mat_add(self.w, self.mat_mul(self.x[i],self.y[i]))
self.b += self.y[i] * self.r2
self.k += 1
pg.update(float(i)/num_examples, 'Iterating...')
pg.update(1, 'Done')
result = (self.w, self.b, self.k)
self.iterations.append(result)
return result
def log_likelihood_all(pos):
eta = pos[0]
xs = pos[1]
xb = pos[2]
if eta < 0.:
return np.NINF
s = N_sig * (1 + E_sig/N_sig)**xs
b = N_bkg * (1 + E_bkg/N_bkg)**xb
lam = eta * s + b
log_Normal_xs = np.log(1./np.sqrt(2*np.pi)) -xs**2/2.
log_Normal_xb = np.log(1./np.sqrt(2*np.pi)) -xb**2/2.
log_Poisson_lam = np.log(poisson.pmf(Data_obs, lam))
log_likelihood = np.log(prior((eta,s,b))) + np.sum(log_Poisson_lam) + log_Normal_xs + log_Normal_xb
progbar.step_progress()
return log_likelihood
def log_likelihood_onebin(binnum, pos):
eta = pos[0]
xs = pos[1]
xb = pos[2]
if eta < 0.:
return np.NINF
s = N_sig[binnum] * (1 + E_sig[binnum]/N_sig[binnum])**xs
b = N_bkg[binnum] * (1 + E_bkg[binnum]/N_bkg[binnum])**xb
lam = eta * s + b
log_Normal_xs = np.log(1./np.sqrt(2*np.pi)) -xs**2/2.
log_Normal_xb = np.log(1./np.sqrt(2*np.pi)) -xb**2/2.
log_Poisson_lam = np.log(poisson.pmf(Data_obs[binnum], lam))
log_likelihood = np.log(prior((eta,s,b))) + log_Poisson_lam + log_Normal_xs + log_Normal_xb
progbar.step_progress()
return log_likelihood
def read_wider_mat(WIDER_ROOT, SET_TYPE, MAT):
# return imgname, WHC, bboxes
mat = h5py.File(MAT, 'r')
filenames = []
WHC = [] #width, height, channel
img_bboxes = []
discard_num = 0
file_list = mat['file_list'][:]
event_list = mat['event_list'][:]
bbx_list = mat['face_bbx_list'][:]
print 'Reading wider mat files...'
pbar = ProgressBar(file_list.size)
for i in range(file_list.size):
pbar+=1
file_list_sub = mat[file_list[0,i]][:]
bbx_list_sub = mat[bbx_list[0, i]][:]
event_value = ''.join(chr(x) for x in mat[event_list[0,i]][:])
for j in range(file_list_sub.size):
root = osp.join(WIDER_ROOT,SET_TYPE,'images',event_value)
filename = osp.join(root , ''.join([chr(x) for x in mat[file_list_sub[0, j]][:]])+'.jpg')
im = io.imread(filename)
bboxes = mat[bbx_list_sub[0, j]][:]
bboxes = bbox_filter(bboxes, filename)
if(bboxes.shape[1] == 0):
# print '[+] Discard:',filename
discard_num += 1
continue
filenames.append(filename)
WHC.append(np.array([im.shape[1], im.shape[0], im.shape[2]]))
img_bboxes.append(bboxes.T)
print '\n[+] Dicard {} bad images'.format(discard_num)
del im
return filenames, WHC, img_bboxes
def tokenize_stories(reviews, tokenized_stories_dir):
"""Maps a whole directory of .story files to a tokenized version using Stanford CoreNLP Tokenizer"""
progress = ProgressBar.ProgressBar(len(reviews),
fmt=ProgressBar.ProgressBar.FULL)
for i, row in reviews.iterrows():
#if i==20:
# break
filename = str(i) + '.tok'
with open(os.path.join(tokenized_stories_dir, filename),
'w',
encoding="utf-8") as temp_file:
text = row["content"]
text = clean_text(text, remove_stopwords=True)
tok = nltk.word_tokenize(text)
tok.append("@highlight")
Summary = row["title"]
Summary = clean_text(Summary, remove_stopwords=False)
tok.extend(nltk.word_tokenize(Summary))
list = tok.copy()
for i in tok:
if (i == '``' or i == "''"):
list.remove(i)
tok_string = "\n".join(str(x) for x in list)
temp_file.write(tok_string)
progress.current += 1
progress()
print("Successfully finished tokenizing to %s .\n" %
(tokenized_stories_dir))
def __createMessageBar(self):
# Create the message bar area for help and status messages.
frame = self.createcomponent('bottomtray', (), None,
Frame,(self._hull,), relief=SUNKEN)
self.__messageBar = self.createcomponent('messagebar',
(), None,
Pmw.MessageBar,
(frame,),
#entry_width = 40,
entry_relief=SUNKEN,
entry_bd=1,
labelpos=None)
self.__messageBar.pack(side=LEFT, expand=YES, fill=X)
self.__progressBar = ProgressBar.ProgressBar(frame,
fillColor='slateblue',
doLabel=1,
width=150)
self.__progressBar.frame.pack(side=LEFT, expand=NO, fill=NONE)
self.updateProgress(0)
frame.pack(side=BOTTOM, expand=NO, fill=X)
self.__balloon.configure(statuscommand = \
self.__messageBar.helpmessage)
def runProgressBarDownload(downloadCmd, quitFilter=True):
sys.path.insert(
0, '%s/Resources/ProgressBar.scptd/Contents/Resources/' % CURRENT_PATH)
import ProgressBar
progressBar = ProgressBar.ProgressBar('Luxinate', '100')
try:
mediaTitle = open(TEMPFILE, 'r').readline()
progressBar.start()
progressBar.update(mediaTitle)
runDownload = subprocess.Popen([downloadCmd],
stdout=subprocess.PIPE,
shell=True)
while runDownload.poll() is None:
newLine = runDownload.stdout.readline()
try:
if newLine.split(' ')[2] == '':
newPercent = newLine.split(' ')[3].replace('%', '')
else:
newPercent = newLine.split(' ')[2].replace('%', '')
infoText = ' '.join(newLine.split(' ')[4:]).replace('\n', '')
except IndexError:
newPercent = '100'
infoText = 'Download Complete'
try:
float(newPercent)
progressBar.increment(mediaTitle, infoText, newPercent)
except ValueError:
pass
progressBar.update('Download Complete')
if quitFilter:
progressBar.quit()
except:
pass
if quitFilter:
progressBar.quit()
def uploadToDrive(self, population, File_Name, x1, y1, x2, y2):
geometry = ee.Geometry.Rectangle([x1, y1, x2, y2])
print(File_Name)
task_config = {
'region': geometry.coordinates().getInfo(),
'scale': 50,
'description': File_Name,
'bucket': 'earth_engine_forest',
}
task = ee.batch.Export.image.toCloudStorage(population, **task_config)
task.start()
task.status()
print(task, task.status())
counter = 0
pb = ProgressBar.ProgressBar()
while task.status()['state'] in ['READY', 'RUNNING']:
# print(task.status())
pb.printProgress(min(counter, 66), 66)
time.sleep(1)
# print(task, task.status())
counter += 1
# print(task.status())
pb.printProgress(100, 100)
print("finish download to bucket")
self.downloadFromDrive(File_Name)
return
def train(self):
self.k = 0
pg = ProgressBar()
i = 0
num_examples = len(self.x)
for x,y in zip(self.x, self.y):
if y * (self.dot_pdt(x,self.w) + self.b) <= 0:
self.mat_add_mul(self.w, x, y)
self.b += y * self.r2
self.k += 1
pg.update(float(i)/num_examples, 'Iterating...')
i += 1
pg.update(1, 'Done')
result = (self.w, self.b, self.k)
self.iterations.append(result)
return result
def simulate(self):
# Initialize the progressbar for console
p = ProgressBar.ProgressBar(0, self.Total_Inserts, 77)
# Loop through all insertions
inserted_descriptors = 0
while inserted_descriptors < self.Total_Inserts:
# Write IO stats regularly as dictated by parameters
if inserted_descriptors % self.Output_Freq == 0:
# Update the progress bar
p.update_progress(inserted_descriptors)
# Print IO stats and tree stats
out_stats = self.IO.get_io_stats()
self.print_stats(inserted_descriptors, out_stats)
self.print_stats(inserted_descriptors,
self.Index.get_index_stats())
# Make sure the stuff is printed to disk
sys.stdout.flush()
# Insert the descriptors, one at a time
self.Index.insert()
inserted_descriptors += 1
# Get the final IO stats and print them to console
p.update_progress(inserted_descriptors)
#flush OS buffer (if any)
self.Index.clear_osbuf()
if self.Write_IOtrace == True:
self.print_ioqueue(self.IO.IODepthDict)
out_stats = self.IO.get_io_stats()
self.print_stats(inserted_descriptors, out_stats)
self.print_stats(inserted_descriptors, self.Index.get_index_stats())
def main(argv):
global debug #debug flag
debug = False
try:
parms = PrintParameters.PrintParameters(argv)
parms.Display_parameters()
w = ExcelControl.ExcelControl()
wb = w.openworkbook(parms.wbInputFileName)
ws = wb.get_sheet_by_name("Sheet1")
pl = PrintLabel.PrintLabel(ws, parms)
pb = ProgressBar.ProgressBar(25, "Progress:")
ToDo = 0
RowCounter = 0
LastRow = ws.max_row + 1
for row in range(2, LastRow):
rc = RowContents.RowContents(ws, row)
if rc.ProcessRow:
ToDo = ToDo + 1
print("reading rows")
for row in range(2, LastRow):
assert row <= LastRow
rc = RowContents.RowContents(ws, row)
if rc.ProcessRow:
# print (str(row-1) + " of " + str(counter))
pl.printlabel(rc, ws)
RowCounter = RowCounter + 1
pb.update_progress(float(RowCounter), float(ToDo), str(rc.Label))
wb.save(parms.wbInputFileName)
except LookupError as e:
print("Lookup Error " + str(e))
quit(-2)
except NameError as e:
print("Error! - Invalid Filename = " + str(e))
quit(-3)
except StandardError as e:
print("Error! - Standard Error = " + str(e))
quit(-3)
except Warning as e:
print("Warning " + str(e))
quit(-1)
return True
def training(self, num_networks):
"""
Train a certain number of randomly initialized networks, writing weight files for each network after
training and testing is completed.
:param num_networks: the number of new, randomly initialized networks to train with the dataset
:return: a list of file names and the index of the file name storing the information for the best network
:rtype: list, int
"""
num_correct_lists = []
file_names = []
path = FileProcessor.get_complete_title(self.name + " Training",
'weight_database', '')
os.mkdir(path)
for i in range(num_networks):
start_time = time.time()
print(f"Training Network {i}")
net = NeuralNet(self.net_size, self.learning_rate)
num_correct_list = []
for trial_num in range(self.num_trials):
print(f"Trial {trial_num}: ", end='')
def update_progress_bar(epoch_index):
ProgressBar.draw_bar(
float(epoch_index) / self.num_epochs, 30,
time.time() - start_time)
print(f"Trial {trial_num}: ", end='')
ProgressBar.draw_bar(0, 30, 0)
net.stochastic_training_input(self.train_inputs_outputs,
self.num_epochs, self.batch_size,
update_progress_bar)
correct_images = self.testing(net)
print(f"{correct_images} Correct Images", " " * 10)
num_correct_list.append(correct_images)
print("Network training took " +
ProgressBar.time_to_string(time.time() - start_time) + ".")
file_name = FileProcessor.write_net_file(net, self.name, path)
file_names.append(file_name)
num_correct_lists.append(num_correct_list)
index = FileProcessor.write_meta_net_file(
self.net_size, self.learning_rate, self.name, self.num_trials,
self.num_epochs, self.batch_size, num_correct_lists)
return file_names, index
def __init__(self, url, start, end):
self.url = url
self.headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
'Cookie':'Hm_lvt_9e49ea2ec9c7da90d2fc4246e717f460=1492345083; Hm_lpvt_9e49ea2ec9c7da90d2fc4246e717f460=1492345095'
}
self.bar = ProgressBar.ProgressBar(end-start+1)
self.bar.grow(0)
self.get(start, end)
def combine(normalbins, tumorbins, tumorbafs, normalbafs, diploidbaf, totalcounts, chromosomes, samples, normal, mode, draws, bafsd, gamma, verbose=False, disable=False):
res = {}
ctumorbafs = {c : sorted(tumorbafs[c], key=(lambda x : x[1])) for c in tumorbafs}
if not disable: progress_bar = pb.ProgressBar(total=len(tumorbins), length=40, verbose=verbose)
for bi in sorted(tumorbins, key=(lambda x : (sp.numericOrder(x[0]), int(x[1]), int(x[2])))):
if bi[0] in ctumorbafs:
# Extract counts
normal_count = normalbins[bi][1]
tumor_counts = {x[0] : x[1] for x in tumorbins[bi]}
# Extract overlapping SNPs: option 2
#tumorover = [x for x in ctumorbafs[bi[0]] if bi[1] <= x[1] <= bi[2]]
#ctumorbafs[bi[0]] = [x for x in ctumorbafs[bi[0]] if x not in tumorover]
tumorover = []
for i, x in enumerate(ctumorbafs[bi[0]]):
if bi[1] <= x[1] <= bi[2]:
tumorover.append(x)
elif x[1] > bi[2]:
ctumorbafs[bi[0]] = ctumorbafs[bi[0]][i:]
break
if normalbafs != None:
normalover = [normalbafs[bi[0], x[1]] for x in tumorover]
else:
normalover = None
# Partition the overlapping SNPs by samples
tpartition = {sample : [x for x in tumorover if x[0] == sample] for sample in samples}
# Check the bin to be covered in each sample and non-zero count in normal
if sum(len(tpartition[key]) != 0 for key in tpartition) == len(samples) and normal_count != 0:
# Compute ratios
ratios = {sample : float(tumor_counts[sample]) / float(normal_count) for sample in samples}
# Compute normalizing factor by total number of reads if provided
if totalcounts is not None:
totalfactor = {sample : float(totalcounts[normal]) / float(totalcounts[sample]) for sample in samples}
ratios = {sample : float(ratios[sample]) * float(totalfactor[sample]) for sample in samples}
# Compute number of SNPs covering each bin and the average coverage
snps = {sample : len(tpartition[sample]) for sample in samples}
cov = {sample : float(sum(x[2]+x[3] for x in tpartition[sample])) / float(len(tpartition[sample])) for sample in samples}
#Compute BAFs
records = computeBAFs(mode=mode, partition=tpartition, normal=normalover, diploidbaf=diploidbaf, samples=samples, chro=bi[0], draws=draws, bafsd=bafsd, gamma=gamma)
parsed = {record[0] : record for record in records}
res[bi] = [(parsed[sample][0], ratios[sample], snps[sample], cov[sample], parsed[sample][1], parsed[sample][2], parsed[sample][3]) for sample in samples]
else:
if verbose and normal_count != 0:
sp.log(msg='The bin ({}, {}) in chromosomes "{}" has been discarded because there are no covering SNPs in each tumor or normal sample\n'.format(bi[1], bi[2], bi[0]), level="WARN")
elif verbose and normal_count == 0:
sp.log(msg='The bin ({}, {}) in chromosomes "{}" has been discarded because normal read count is zero\n'.format(bi[1], bi[2], bi[0]), level="WARN")
if not disable: progress_bar.progress(advance=True, msg="Combine bin ({}, {}) in chromosome {}".format(bi[1], bi[2], bi[0]))
return res
def _balance_histograms(self):
""" Compute the cost of balancing a histogram for each
row in the affinity matrix
"""
if self.verbose:
""" start progress bar """
prog = pb.progressBar(0, self.total_instances, 77)
oldprog = str(prog)
""" end progress bar """
for i in xrange(self.total_instances):
histogram = self._build_histogram(self.affinity_matrix[i, :])
max_cost = -inf
bin_count = 1
for j in xrange(2, len(histogram) + 1):
""" Explicitely create deep copies,
Getting shallow copies, otherwise (Python 2.6)
"""
temp_hist = histogram.copy()[:j]
mu = ceil(mean(temp_hist))
cost = 0
while max(temp_hist) > mu:
max_list = where(max(temp_hist) == temp_hist)[0]
min_list = where(min(temp_hist) == temp_hist)[0]
""" Cost is 1 regardless of distance """
if min_list[-1] > max_list[0]:
cost += 1
else:
cost -= 2
temp_hist[min_list[-1]] += 1
temp_hist[max_list[0]] -= 1
if cost > max_cost:
max_cost = cost
bin_count = j
k = sum(histogram[:bin_count])
self._cost_list[i] = max_cost
self._bin_count_list[i] = bin_count
self._k_list[i] = k
if self.verbose:
""" start progress bar """
prog.updateAmount(i)
if oldprog != str(prog):
print prog, "\r",
sys.stdout.flush()
oldprog = str(prog)
""" end progress bar """
if self.verbose:
print "\n"
def log_likelihood_all_legacy(pos):
eta = pos[0]
xs = pos[1]
xb = pos[2]
if eta < 0.:
return np.NINF
s = N_sig * (1 + E_sig/N_sig)**xs
b = N_bkg * (1 + E_bkg/N_bkg)**xb
lam = eta * s + b
log_Normal_xs = np.log(1./np.sqrt(2*np.pi)) -xs**2/2.
log_Normal_xb = np.log(1./np.sqrt(2*np.pi)) -xb**2/2.
log_factorial = np.zeros_like(Data_obs)
for i in range(0,len(log_factorial)):
for j in range(1,np.round(Data_obs[i]).astype(np.int)):
log_factorial[i] += np.log(j)
log_Poisson_lam = Data_obs*np.log(lam) -lam - log_factorial
log_likelihood = np.log(prior((eta,s,b))) + np.sum(log_Poisson_lam) + log_Normal_xs + log_Normal_xb
progbar.step_progress()
return log_likelihood
def __init__(self, pages=1):
self.n = pages
self.headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"
}
self.bar = ProgressBar.ProgressBar(20 * pages, 40)
if not os.path.exists('img'):
os.mkdir('img')
print('init complete')
def __init__(self, master):
Tkinter.Frame.__init__(self, master)
self.__master = master
self.__about = 0
self.__rebuildSearch = 0
master.title('CDAT Help')
progressBar = ProgressBar.ProgressBar(master, 'CDAT Help Progress')
progressBar.withdraw()
progressBar.resizable(height=0, width=0)
self.__cdatHelp = CDATHelp.CDATHelp(progressBar)
# Create and install the MenuBar.
self.__menuBar = Pmw.MainMenuBar(master)
master.config(menu=self.__menuBar)
self.__menuBar.addmenu('File', '')
self.__menuBar.addmenuitem('File',
'command',
label='Exit',
command=sys.exit)
self.__menuBar.addmenu('Options', '')
self.__menuBar.addmenuitem('Options',
'command',
label='Rebuild search',
command=self.__rebuildSearchDialog)
self.__menuBar.addmenu('Help', '')
self.__menuBar.addmenuitem('Help',
'command',
label='About',
command=self.__aboutDialog)
# Create vertical panes container.
self.__panes = Pmw.PanedWidget(master, orient='horizontal')
self.__panes.add('query', min=0.2)
self.__queryFrame = QueryFrame.QueryFrame(self.__panes.pane('query'),
self.__cdatHelp)
self.__queryFrame.pack(expand=1, fill='both')
self.__panes.add('results')
self.__resultsFrame = ResultsFrame.ResultsFrame(
self.__panes.pane('results'))
self.__resultsFrame.pack(expand=1, fill='both')
self.__panes.pack(fill='both', expand=1, padx=3, pady=3)
self.__queryFrame.setResultsSetter(self.__resultsFrame.set)
self.startupTimer = threading.Timer(0.0, self.onStartupTimer)
self.startupTimer.start()
def tcount(samtools, samples, chromosomes, num_workers, q, verbose=False):
# Define a Lock and a shared value for log printing through ProgressBar
err_lock = Lock()
counter = Value('i', 0)
progress_bar = pb.ProgressBar(total=len(samples) * len(chromosomes),
length=40,
lock=err_lock,
counter=counter,
verbose=verbose)
# Establish communication queues
tasks = JoinableQueue()
results = Queue()
# Enqueue jobs
jobs_count = 0
for bam in samples:
for chro in chromosomes:
tasks.put((bam[0], bam[1], chro))
jobs_count += 1
# Setting up the workers
workers = [
TotalCounter(tasks, results, progress_bar, samtools, q, verbose)
for i in range(min(num_workers, jobs_count))
]
# Add a poison pill for each worker
for i in range(len(workers)):
tasks.put(None)
# Start the workers
for w in workers:
w.start()
# Wait for all of the tasks to finish
tasks.join()
# Get the results
sorted_results = {}
for i in range(jobs_count):
res = results.get()
sorted_results[res[0], res[1]] = res[2]
# Close Queues
tasks.close()
results.close()
# Ensure each worker terminates
for w in workers:
w.terminate()
w.join()
return sorted_results
def chunk(self):
"""
chunk the dataset into small pieces
"""
if not os.path.isdir(self.output_folder):
os.makedirs(self.output_folder)
interval = int(self.num_time/self.pieces[0])
year_interval = int(self.num_year/self.pieces[0])
# command_list=[]
progress = ProgressBar(n_iter=self.pieces[0], total_width= 30 , description='Chunk Process')
for i in range(self.pieces[0]):
current_start = interval*i
current_end = interval*(i+1)-1
current_str = 'time,'+str(current_start)+','+str(current_end)
current_list = ['ncks','-d',current_str]
output_file_path = self.variable+'_'+str(year_interval*i+self.start_year[0])+'_'+str(year_interval*(i+1)+self.start_year[0]-1)+'.nc'
current_command = current_list+ [self.filepath] + [output_file_path]
# print(current_command)
# command_list.append(current_command)
subprocess.run(current_command)
shutil.move(output_file_path,self.output_folder)
progress.update()
print("")
progress.finish()
print("")
def __readFaces(self):
train_data_path = self._settings["trainDataPath"]
self._file_names = os.listdir(train_data_path)
d = [] # contains all the images data
if self._settings["showSteps"]:
print(f"Detected {len(self._file_names)} files, processing")
ProgressBar.initializeProgressBar(len(self._file_names))
for i, filename in enumerate(self._file_names, start=0):
img = imf.read_image(train_data_path + '/' + filename)
# Detect edges if settings say to do so
if self._settings["detectEdges"]:
img = self.__detectEdges(img)
# If image size was not initialized, write the first image size
# !!! Assumption: all images in the folder have the same size !!!
if self._size == None or len(d) == 0:
self._size = len(img.data[0])
d = np.zeros((self._size**2, len(self._file_names)),
dtype=np.int32)
max_shade, data = img
reshaped = data.reshape(-1)
d[:, i] = reshaped
if self._settings["showSteps"]: ProgressBar.increaseProgressBar()
if self._settings["showSteps"]: ProgressBar.completeProgressBar()
return d
def convert(fcn_data_dir, wider_root, wider_mat_dir):
dataset_root, img_dir, cls_dir = mk_dir_tree(fcn_data_dir)
mats = ['wider_face_val.mat','wider_face_train.mat']
set_types = ['WIDER_val','WIDER_train']
for mat, set_type in zip(mats, set_types):
print "[{}]".format(set_type)
wider_mat = osp.join(wider_mat_dir,mat)
filenames,WHC,img_bboxes = read_wider_mat(wider_root, set_type, wider_mat)
# lab_imgs =gen_label_image(filenames,WHC,img_bboxes)
# del WHC, img_bboxes
txt_content = []
filtered_out = []
print "Writting..."
pbar = ProgressBar(len(filenames))
for img_path, whc, bboxes in zip(filenames, WHC, img_bboxes):
short_name = img_path.split('/')[-1]
pbar+=1
if(WH_THRES > 0):
if(whc[0]>WH_THRES or whc[1]>WH_THRES):
filtered_out.append(short_name+"({}x{})".format(whc[0],whc[1]))
continue #discard this image
#cp image
non_ext_name = short_name.split('.')[0]
target_path = osp.join(img_dir, short_name)
shutil.copyfile(img_path, target_path)
if not os.path.exists(target_path):
print '[{}] failed'.format(img_path)
#os.system('cp {} {}'.format(img_path, osp.join(img_dir,short_name)))
#save mat
label = gen_label_image(whc, bboxes) #to read: label = mat['GTcls']
mat_path = osp.join(cls_dir,non_ext_name+'.mat')
savemat(mat_path,{'GTcls':label})
#append to txt_content
txt_content.append(non_ext_name)
#write .txt
if 'train' in set_type:
txtname = osp.join(dataset_root,'train.txt')
if 'val' in set_type:
txtname = osp.join(dataset_root,'val.txt')
with open(txtname, 'w') as f:
f.write('\n'.join(txt_content))
del filenames,WHC,img_bboxes
print 'TotalNumber:',len(txt_content)
if len(filtered_out) != 0:
print 'Filtered out: \n{}'.format('\n'.join(filtered_out))
print '\n'
def __init__(self, parent, message='Working', cancel=None):
TopLevelModalDialog.__init__(self, parent)
self.parent = parent
self.title(message)
progrlbl = Label(self.body, text='Progress:')
progrlbl.pack(fill=BOTH, padx=2, pady=4)
self.bar = ProgressBar.ProgressBar(self.body)
self.bar.frame.pack(fill=Y, padx=2, pady=4)
if cancel != None:
self.cancel = Button(self.body, text='cancel', command=cancel)
self.cancel.pack()
return
def main():
list_path, gt_path, result_path = process_arguments(sys.argv)
gt_ext = 'png'
result_ext = 'mat' # bin, mat
pa_list = []
ma_list = []
m_IU_list = []
fw_IU_list = []
list_images = load_list(list_path)
pb = ProgressBar(len(list_images))
for image_name in list_images:
gt_fullpath = os.path.join(gt_path, image_name) + '.' + gt_ext
label = imread(gt_fullpath)
if result_ext == 'bin':
result_fullpath = os.path.join(result_path,
image_name) + '.' + result_ext
pred = load_binary_segmentation(result_fullpath, dtype='int16')
elif result_ext == 'mat':
result_fullpath = os.path.join(
result_path, image_name) + '_blob_0.' + result_ext
pred = convert_segmentation_mat2numpy(result_fullpath)
pred = pred[0:label.shape[0], 0:label.shape[1]]
pa_list.append(pixel_accuracy(pred, label))
ma_list.append(mean_accuracy(pred, label))
m_IU_list.append(mean_IU(pred, label))
fw_IU_list.append(frequency_weighted_IU(pred, label))
pb.print_progress()
print("pixel_accuracy: " + str(np.mean(pa_list)))
print("mean_accuracy: " + str(np.mean(ma_list)))
print("mean_IU: " + str(np.mean(m_IU_list)))
print("frequency_weighted: " + str(np.mean(fw_IU_list)))
def runProgressBarConvert(convertCmd):
sys.path.insert(
0, '%s/Resources/ProgressBar.scptd/Contents/Resources/' % CURRENT_PATH)
import ProgressBar
progressBar = ProgressBar.ProgressBar('Luxinate', '100')
try:
mediaTitle = open(TEMPFILE, 'r').readline()
progressBar.start()
progressBar.increment(mediaTitle, 'Converting...', '100')
runProcess(convertCmd)
except:
progressBar.quit()
progressBar.quit()
def predict(self, data):
num_incorrect, total, prediction_array = 0, len(data), []
tx, ty = self.matricize(data)
pg = ProgressBar()
num_examples = len(tx)
for i in range(0, num_examples):
if ty[i] * (self.dot_pdt(tx[i], self.w) + self.b) <= 0:
num_incorrect += 1
prediction_array.append(0)
else:
prediction_array.append(1)
pg.update(float(i)/num_examples, 'Predicting...')
pg.update(1, 'Predicted')
return (prediction_array, num_incorrect, total, float(num_incorrect)/total)
# d1 = Example('-1 1:0.5 8:0.6')
# d2 = Example('-1 1:0.4 9:-0.8')
# #
# d3 = Example('-1 1:0.3 9:0.5')
# d4 = Example('1 1:0.6 9:0.7')
# #
# # # a = sparse.lil_matrix((2,1))
# # # b = sparse.lil_matrix((2,1))
# # # c = a * b.T
# # # print c[0,0]
# #
# p = FastPerceptron([d1,d2])
# # print "y"
# # print p.y
# # print "hi"
# # print p.x
# print p.train()
# # p.train()
# # p.train()
# # p.train()
# # print [x for a,b,x in p.iterations]
# #
# print p.predict([d3, d4])
def setIdPlayersPlanet(self,p_id):
if not (p_id in self.__ids_reachable_planets):
self.__ids_reachable_planets.append(p_id )
else:
#sound
SfxPlayer.teleport()
found = False
for pl in self.__l_planets:
if pl.getId()==p_id:
found = True
self.current_coords = (pl.x,pl.y)
self.__id_p_player = p_id
break
if(not found ):
raise Exception('cannot set players planet to '+str(p_id) )
#init bars if the player just landed
if not self.__bars_init:
self.__info_o2 = ProgressBar( RESSOURCE_DUR ,False)
self.__info_food = ProgressBar( RESSOURCE_DUR ,False)
self.__info_water = ProgressBar( RESSOURCE_DUR ,False)
self.__bars_init =True
def main():
list_path, gt_path, result_path = process_arguments(sys.argv)
gt_ext = 'png'
result_ext = 'mat' # bin, mat
pa_list = []
ma_list = []
m_IU_list = []
fw_IU_list = []
list_images = load_list(list_path)
pb = ProgressBar(len(list_images))
for image_name in list_images:
gt_fullpath = os.path.join(gt_path, image_name) + '.' + gt_ext
label = imread(gt_fullpath)
if result_ext == 'bin':
result_fullpath = os.path.join(result_path, image_name) + '.' + result_ext
pred = load_binary_segmentation(result_fullpath, dtype='int16')
elif result_ext == 'mat':
result_fullpath = os.path.join(result_path, image_name) + '_blob_0.' + result_ext
pred = convert_segmentation_mat2numpy(result_fullpath)
pred = pred[0:label.shape[0], 0:label.shape[1]]
pa_list.append(pixel_accuracy(pred, label))
ma_list.append(mean_accuracy(pred, label))
m_IU_list.append(mean_IU(pred, label))
fw_IU_list.append(frequency_weighted_IU(pred, label))
pb.print_progress()
print("pixel_accuracy: " + str(np.mean(pa_list)))
print("mean_accuracy: " + str(np.mean(ma_list)))
print("mean_IU: " + str(np.mean(m_IU_list)))
print("frequency_weighted: " + str(np.mean(fw_IU_list)))
def test_net(net_path, model_path, images, labels, lut, gpu_id):
net = Segmenter(net_path, model_path, gpu_id)
mean_vec = np.array([103.939, 116.779, 123.68], dtype=np.float32)
reshaped_mean_vec = mean_vec.reshape(1, 1, 3);
pa_list = []
ma_list = []
m_IU_list = []
fw_IU_list = []
pb = ProgressBar(len(images))
for img_path, label_path in zip(images, labels):
im, cur_h, cur_w = preprocess_image(img_path, reshaped_mean_vec)
label = imread(label_path)
label = lut[label]
segmentation = net.predict([im])
pred = segmentation[0:cur_h, 0:cur_w]
pa = pixel_accuracy(pred, label)
ma = mean_accuracy(pred, label)
m_IU = mean_IU(pred, label)
fw_IU = frequency_weighted_IU(pred, label)
pa_list.append(pa)
ma_list.append(ma)
m_IU_list.append(m_IU)
fw_IU_list.append(fw_IU)
pb.print_progress()
print("pixel_accuracy: " + str(np.mean(pa_list)))
print("mean_accuracy: " + str(np.mean(ma_list)))
print("mean_IU: " + str(np.mean(m_IU_list)))
print("frequency_weighted: " + str(np.mean(fw_IU_list)))
def __init__(self, parent, ProgressBar=1):
Frame.__init__(self, parent)
# Zweitgeteilt: links der Statustext, rechts der Inikatortext
# und dann evtl. noch der ProgressBar
self.status1 = Label(self, bd=1, relief=SUNKEN, anchor=W)
self.status1.pack(side=LEFT, fill=X, expand=YES, padx=2, pady=1)
if ProgressBar == 1:
import ProgressBar
self.progress = ProgressBar.ProgressBar(self, bd=1)
self.progress.updateProgress(0)
self.progress.frame.pack(side=RIGHT, padx=2, pady=1)
self.status2 = Label(self, bd=1, relief=SUNKEN, anchor=E)
self.status2.pack(side=RIGHT, padx=2, pady=1)
def startup_MultiBody(self):
import numpy as np
import ProgressBar
# Load in data
print("\n-- Reading in '%s' File" % self.dfile)
with open(self.dfile, 'r', errors='replace') as f:
lines_ = f.readlines()
lines = lines_[3:]
self.Ndata = len(lines)
self.ExpData = np.zeros((self.Ndata, 7), dtype=np.float64)
for i in range(self.Ndata):
self.ExpData[i][:] = lines[i].split()
precent = float(i + 1) / float(self.Ndata) * 100.0
ProgressBar.ProgressBar(i + 1, self.Ndata, precent)
def test_MCMC():
read_data("DMm400AVu_cf.txt")
set_prior(uniform)
ndim = 3
nwalkers = 90
burnlinks = 300
links = 100
p0 = np.random.rand(ndim * nwalkers).reshape((nwalkers, ndim))
sampler = emcee.EnsembleSampler(nwalkers, ndim, log_likelihood_all)
progbar.set_totalsteps(float(burnlinks)*nwalkers)
print 'Begin burn in'
pos, prob, state = sampler.run_mcmc(p0,burnlinks)
progbar.update_progress(1)
print 'Burn in completed'
sampler.reset()
progbar.set_totalsteps(float(links)*nwalkers)
sampler.run_mcmc(pos, links)
progbar.update_progress(1)
print('Main chain completed')
print("Mean acceptance fraction: {0:.3f}".format(np.mean(sampler.acceptance_fraction)))
sorted_eta = np.sort(sampler.flatchain[:,0])
conf95_index = np.round(0.95*len(sorted_eta) - 1).astype(np.int)
conf998_index = np.round(0.998*len(sorted_eta) - 1).astype(np.int)
conf95 = sorted_eta[conf95_index]
conf998 = sorted_eta[conf998_index]
print '95% upper limit: ' + str(conf95)
plt.figure()
y, x, o = plt.hist(sampler.flatchain[:,0], 300, range=[0,conf998], histtype="step", normed = 1)
yciel = 1.1*y.max()
plt.ylim([0,yciel])
plt.vlines(conf95, 0, yciel ,colors='r')
plt.title('Dark Matter Signal Posterior PDF (all bins)',fontsize = 16)
plt.xlabel(r'Signal Strength [$\eta$]',fontsize = 14)
plt.ylabel(r'Probability Density', fontsize = 14)
plt.text(1.05*conf95,0.75*yciel, '95% conf: {0:.3f}'.format(conf95), bbox=dict(facecolor='red',alpha=0.5))
plt.show()
#!/usr/bin/python3.4
# coding: utf8
import sys,os,subprocess,re
from ProgressBar import *
rootFolder = sys.argv[1]
listRootFolder = os.listdir(rootFolder) #list of the elements in the root folder
# Step one : sort unsorted images wich are in the root folder
Bar1 = ProgressBar(len(listRootFolder), 30 , "Sort unsorted images (Step 1/2)")
for imgUnsorted in listRootFolder:
Bar1.update()
if re.findall('.png', imgUnsorted): #if this element is an image :
#get the char name of the glyph
glyphName=imgUnsorted[0]
#create dir if it's necessary, move the image in this dir
if glyphName == ".": #to fix "." name
subprocess.call(["mkdir","-p",rootFolder+".point"])
subprocess.call(["mv",rootFolder+imgUnsorted,rootFolder+".point/"])
else :
subprocess.call(["mkdir","-p",rootFolder+glyphName])
subprocess.call(["mv",rootFolder+imgUnsorted,rootFolder+glyphName])
# Step two : rename file if it does not correspond with the name of the folder in which it is.
# prepares the estimation of the process time.
totalFile=0
nbfolder=0
for r,d,f in os.walk(rootFolder):
nbfolder+=1
for i in f:
totalFile+=1
#!/usr/bin/python3.4
# coding: utf8
import sys,os,subprocess,re
from ProgressBar import *
letter = sys.argv[1]
levels = sys.argv[2:]
resize = "300%" #define rescale factor
delta = 10 #define delta of the levels : the contrast factor
char = letter.split("/")[-1][0] #get the name of the glyph
progressBar = ProgressBar(len(levels), 30 , "Niveaux : ")
subprocess.call(["convert",letter,"-resize",resize,char+".png"]) #save a rescale version
#save rescale and modify versions
for level in levels:
print(level)
m = int(level)-delta/2
M = int(level)+delta/2
parameter = str(m)+"%,"+str(M)+"%" #ex : 45%,55% with level=50% and delta=10%
output = char+level+"pc"+str(delta)+"d.png"
subprocess.call(["convert",letter,"-resize",resize,"-level",parameter,output])
progressBar.update()
s_paras = [1, 5, 9, 17, 25, 37, 43, 49]
#s_paras = [49]
s_len = float(len(s_paras))
# Splits the list l into npieces pieces
def parts(l, npieces):
parts = []
size = len(l)/npieces
for i in range(0, npieces-1):
parts.append(l[i*size:(i+1)*size])
parts.append(l[size*(npieces-1):])
return parts
avg_error = []
tsets = parts(training_set, 10)
pg, prog = ProgressBar(), 1.0
for s_para in s_paras:
op = prog/s_len
ip = 1.0
test_error = []
for p in tsets:
train = []
for q in tsets:
if p != q:
train += q
test = p
dt = DecisionTree(train, stopping_parameter=s_para)
test_error.append(dt.prediction_error(test)[1])
pg.update(-(1-ip/10)/s_len+op, "Stopping Para %d" % s_para)
ip += 1
#print test_error
class World:
'''this class models the world where the player is located'''
@classmethod
def __populate_universe(cls, pl_list ):
''' populating the list of nodes+associated radii '''
while( len(pl_list) < NB_PLANETS):
tmp = Planet()
# if this doesnt fit our goals, we roll dices again!
while cls.__is_bad_pick(tmp,pl_list):
tmp = Planet()
pl_list.append(tmp)
@classmethod
def __is_bad_pick(cls, t_planet, pl_list):
if (t_planet.x+t_planet.rad>=DISP_W-1) or (t_planet.x-t_planet.rad<=0):
return True
if (t_planet.y+t_planet.rad>=DISP_H-1) or (t_planet.y-t_planet.rad<=BORDER_UP):
return True
for oth_planet in pl_list:
if t_planet.overlaps( oth_planet):
return True
return False
#
# CONSTRUCTOR ---------------------------------------------
#
def __init__(self):
self.game_restart = False
self.__info_o2 = None
self.__info_food = None
self.__info_water = None
self.__bars_init = False
self.paused = False
self.__lost = False
self.__rewards = dict()
self.__game_exit = False;
self.__bars_init = False
#the player has not landed at the beginning
self.__id_p_player = None
self.resetLinks()
self.__l_planets = list()
World.__populate_universe( self.__l_planets )
def signalQuit(self):
self.__game_exit = True;
def isGameLost(self):
return self.__lost
def getDefeatCause(self):
if(not self.__lost):
raise Exception('bad usage, player did not loose yet')
if( self.__info_o2.isFinished() ):
return "You died of hypoxemia!"
if( self.__info_water.isFinished() ):
return "You died of dehydration!"
if( self.__info_food.isFinished() ):
return "You died of starvation!"
def gameRestart(self):
return self.game_restart
def gameExit(self):
return self.__game_exit
def resetLinks(self):
self.__assoc_id_lk = dict()
self.__ids_reachable_planets = list()
#there are no teleport links in the beginning
self.__links = list()
def produceReward(self):
#generating
res = None
if( self.__id_p_player in self.__rewards ):
res = self.__rewards[ self.__id_p_player]
else:
res = random.choice(REWARDS)
self.__rewards[ self.__id_p_player ] = res
#updating bars
if res==None:
raise Exception('cannot produce reward!')
if( res==REWARD_O2):
self.incrOxygen()
elif( res==REWARD_FOOD):
self.incrFood()
elif( res==REWARD_WATER):
self.incrAmmu()
self.last_reward = res
def getDescLastReward(self):
if( self.last_reward==REWARD_O2):
return "oxygen."
if( self.last_reward==REWARD_FOOD):
return "food."
if( self.last_reward==REWARD_WATER):
return "water."
def getPlanets(self):
return self.__l_planets
def getLinks(self):
return self.__links
def isPaused(self):
return self.paused
def isPlayerHere(self, planet):
return planet.getId()==self.__id_p_player
def getIdPlayersPlanet(self):
return self.__id_p_player
#landing / teleporting
def setIdPlayersPlanet(self,p_id):
if not (p_id in self.__ids_reachable_planets):
self.__ids_reachable_planets.append(p_id )
else:
#sound
SfxPlayer.teleport()
found = False
for pl in self.__l_planets:
if pl.getId()==p_id:
found = True
self.current_coords = (pl.x,pl.y)
self.__id_p_player = p_id
break
if(not found ):
raise Exception('cannot set players planet to '+str(p_id) )
#init bars if the player just landed
if not self.__bars_init:
self.__info_o2 = ProgressBar( RESSOURCE_DUR ,False)
self.__info_food = ProgressBar( RESSOURCE_DUR ,False)
self.__info_water = ProgressBar( RESSOURCE_DUR ,False)
self.__bars_init =True
def pause(self):
self.__info_o2.pause()
self.__info_food.pause()
self.__info_water.pause()
self.paused = True
def unpause(self):
self.__info_o2.unpause()
self.__info_food.unpause()
self.__info_water.unpause()
self.paused = False
def incrOxygen(self):
self.__info_o2.addDuration( INCREMENT_DURATION)
def incrFood(self):
self.__info_food.addDuration( INCREMENT_DURATION)
def incrAmmu(self):
self.__info_water.addDuration( INCREMENT_DURATION)
def update(self):
self.__info_o2.update()
self.__info_food.update()
self.__info_water.update()
# if a progress bar arrived to 0, the player lost
if( self.__info_o2.isFinished() ):
self.__lost = True
if( self.__info_food.isFinished() ):
self.__lost = True
if( self.__info_water.isFinished() ):
self.__lost = True
def getInfo(self):
res = list()
res.append( compl("Oxygen:")+ str(self.__info_o2) )
res.append( compl("Water:")+ str(self.__info_water) )
res.append( compl("Food:")+ str(self.__info_food) )
return res
def addLink(self, lk_dest, id_p ):
''' lk_dest represents a pair of coord for the dest planet'''
tmp_lk = (self.current_coords, lk_dest)
if( id_p in self.__assoc_id_lk.keys() and
tmp_lk in self.__assoc_id_lk[id_p ] ):
return
if(len(self.__links) >= NB_EDGES ):
raise Exception('max nb edges')
#add a new edge
try:
self.__assoc_id_lk[ id_p ]
except Exception:
self.__assoc_id_lk[ id_p ]= list()
self.__assoc_id_lk[ id_p ].append( tmp_lk)
self.__links.append( tmp_lk )
self.__ids_reachable_planets.append( id_p )
#sound
SfxPlayer.buildStuff()
def hasMaxTeleporters(self):
return (len(self.__links) >= NB_EDGES )
def getIdsRP(self):
return self.__ids_reachable_planets
def getCurrentCoord(self):
return self.current_coords
def read_in_info(placement_file):
"""Reads in an ALE placement file, returns a list of Contigs.
Args:
placement_file: An ALE placement file (*.ale)
must be in the following format::
# comments/metadata
# can have multiple lines, all starting with #
# Reference: gi|170079663|ref|NC_010473.1| 350000 24.3
# contig position depth ln(depthLike) ln(placeLike) ln(insertLike) ln(kmerLike)
0 0 1.000000 -60.000000 0.194888 -5.760798 -65.565910
0 1 3.000000 -60.000000 0.466271 -5.608334 -65.142063
0 2 5.000000 -60.000000 0.010585 -5.541655 -65.531071
0 3 12.000000 -60.000000 -0.057731 -5.380759 -65.438491
Specific lines (using the above as an example):
0. Any number of comment lines starting with #
1. The length of the contig is::
length = int(line.split(' ')[3]) == 350000
The name of the contig is::
name = line.split(' ')[2] == gi|170079663|ref|NC_010473.1|
name **cannot** be 'position'
2. The following line is ignored and lists what is in the columns of following lines
3. The data corresponding to the column headers for each position in the contig
4. See 2.
Returns:
A list of Contigs (see class :py:mod:`plotter3.Contig`)
Raises:
IOError: An error occured accessing the placement file.
FormattingError: The placement file was not formatted correctly.
"""
MINIMUM_VALUE = -60.0 # minimum value we allow a position to have
ale_placement_file = open(placement_file, "r")
contigs = []
previous_line_one = ""
previous_line_two = ""
for line in ale_placement_file:
if line[0] == "#":
if previous_line_one == "":
previous_line_one = line
else:
previous_line_two = previous_line_one
previous_line_one = line
else:
if previous_line_two != "":
tName = previous_line_two.split(" ")[2]
tLen = int(previous_line_two.split(" ")[-2])
contigs.append(Contig(tLen, name=tName))
place = 0
print "Reading in contig: " + tName + " len " + str(tLen)
print ""
bar = ProgressBar.progressBar(0, tLen, 42)
previous_line_two = ""
data = line.split(" ")
contigs[-1].depth[place] = numpy.double(data[2])
for i in range(1, 7):
if (
"-nan" == data[i]
or "nan" == data[i]
or "inf" == data[i]
or "-inf" == data[i]
or numpy.double(data[i]) != numpy.double(data[i])
):
data[i] = MINIMUM_VALUE # Predefined threshold
contigs[-1].prob_vecs["d"].prob[place] = numpy.double(data[3])
contigs[-1].prob_vecs["p"].prob[place] = numpy.double(data[4])
contigs[-1].prob_vecs["i"].prob[place] = numpy.double(data[5])
contigs[-1].prob_vecs["k"].prob[place] = numpy.double(data[6])
place += 1
if tLen > 40:
if (place) % (int(tLen) / 40) == 0:
bar(place)
print "\nYou now have a list of contigs, try contig[0].plot()"
return contigs
for f in inputXMLs:
# get page number and match it with it XMLPAGE file parsed in a dictionnary : XMLs[page] = page.xml
pageXML = re.findall('\d+', f)[0]
XMLs[pageXML] = minidom.parse(folderInputXMLs+f)
for i in inputImgs:
# get page number and match it with it .png file parsed by PIL in a dictionnary : Imgs[page] = page.png
pageImg = re.findall('\d+', i)[0]
Imgs[pageImg]= Image.open(folderInputImgs+"/"+i)
for pageNumber, xmlPage in XMLs.items(): # page by page
imgPage = Imgs[pageNumber]
root = xmlPage.documentElement
# xml browsing
nodeGlyphs = root.getElementsByTagName('Glyph')
BarByPage = ProgressBar(len(nodeGlyphs), 30 , 'Extraction page '+pageNumber)
#unicodeChars = []
coordsCorpList = []
for n in nodeGlyphs : # glyph by glyph
area,outputName = extract.extract(n)
area.save(folderOutputPath + outputName)
BarByPage.update()
#coordsCorpList.append(coordCrop)
else:
print("PAGE files and page image don't match")
level = level_test,shaker = shaker, p_0 = p_0_test)
#test
rare_test.adaptive_levels(N = N_test, shake_times = 1, reject_rate = 0.5,
sigma_default = 0.5, descent_step = 0.05,status_tracking=True)
##############################
### simualtion & plotting ####
##############################
list_p = []
list_n_0 = []
list_r = []
list_V = []
list_s_called_times = []
bar = ProgressBar(total = num_simulation)
for i in range(num_simulation):
###ProgressBar###
bar.move()
bar.log()
#################
##fixed_levels
# iter_output = rare_test.fixed_levels\
# (N = N_test, L = L_ideal, shake_times = shake_times,reject_rate = reject_rate, \
# sigma_default = 0.5, descent_step = 0.1,status_tracking=False)
#adaptive_levels
iter_output = rare_test.adaptive_levels\
(N = N_test, shake_times = shake_times,reject_rate = reject_rate, \
sigma_default = 0.5, descent_step = 0.05,status_tracking=False)
def processtrajectories(self,n=None):
# processes next n trajectories
if n == None:
n = self.numtrajectories-self.trajectoryindex
current = self.trajectoryindex
# do some pre-processing for use in approximate inference
# this pre-processing should be independent of the kernel chosen,
# so that this time-intensive step need-not be repeated for
# different choice of regularization
# do sanity check to make sure that n is not too large
if(current+n > self.numtrajectories):
n = self.numtrajectories - current
print "Inputed number of trajectories to process is too high, using ", n, " trajectories instead."
# current starts at 0
trajectories = self.trajectories[:,current:current+n]
positions = np.ndarray.flatten( (trajectories[:-1,:]).transpose())
indices = np.argsort(positions)
times = np.tile(self.times[:-1],n)[indices]
jumps = np.ndarray.flatten((trajectories[1:,:] - trajectories[:-1,:]).transpose())
jumps = jumps[indices]
positions = positions[indices]
binpos = np.floor((positions-self.x0)/self.dy)
muval = self.mu(positions)
spring = self.x0+times*self.V # location of the spring
m = muval - self.K*(positions-spring)
# compute the coefficient matrices
#print("Initializing coefficient matrices")
numobs = len(positions)
currentbin = 0
freq = 0
dist = positions - (self.midpoints[0]+self.dy*binpos)
a = 0.5*power(dist/self.dy,2)-0.5*dist/self.dy
b = 1.0-dist*dist/self.dy/self.dy
c = 0.5*power(dist/self.dy,2)+0.5*dist/self.dy
aa = a*a
ab = a*b
ac = a*c
bb = b*b
bc = b*c
cc = c*c
prog = ProgressBar(self.bins)
#print("Processing the data, this can take a while...")
jumpsum, groups = self.sum_by_group(jumps,binpos)
jumpsqsum, groups = self.sum_by_group(jumps**2,binpos)
msum, groups = self.sum_by_group(m,binpos)
asum, groups = self.sum_by_group(a,binpos)
bsum, groups = self.sum_by_group(b,binpos)
csum, groups = self.sum_by_group(c,binpos)
masum, groups = self.sum_by_group(m*a,binpos)
mbsum, groups = self.sum_by_group(m*b,binpos)
mcsum, groups = self.sum_by_group(m*c,binpos)
aasum, groups = self.sum_by_group(aa,binpos)
bbsum, groups = self.sum_by_group(bb,binpos)
ccsum, groups = self.sum_by_group(cc,binpos)
absum, groups = self.sum_by_group(ab,binpos)
acsum, groups = self.sum_by_group(ac,binpos)
bcsum, groups = self.sum_by_group(bc,binpos)
bincounts, groups = self.sum_by_group(np.ones(binpos.shape[0]),binpos)
ajumpsum, groups = self.sum_by_group(a*jumps,binpos)
bjumpsum, groups = self.sum_by_group(b*jumps,binpos)
cjumpsum, groups = self.sum_by_group(c*jumps,binpos)
jump_sum = np.zeros(self.bins)
jumpsq_sum = np.zeros(self.bins)
m_sum = np.zeros(self.bins)
a_sum = np.zeros(self.bins)
b_sum = np.zeros(self.bins)
c_sum = np.zeros(self.bins)
ma_sum = np.zeros(self.bins)
mb_sum = np.zeros(self.bins)
mc_sum = np.zeros(self.bins)
aa_sum = np.zeros(self.bins)
bb_sum = np.zeros(self.bins)
cc_sum = np.zeros(self.bins)
ab_sum = np.zeros(self.bins)
bc_sum = np.zeros(self.bins)
ac_sum = np.zeros(self.bins)
for k in range(self.bins):
try:
jump_sum[k] = jumpsum[groups==k]
jumpsq_sum[k] = jumpsqsum[groups==k]
m_sum[k] = msum[groups==k]
a_sum[k] = asum[groups==k]
b_sum[k] = bsum[groups==k]
c_sum[k] = csum[groups==k]
ma_sum[k] = masum[groups==k]
mb_sum[k] = mbsum[groups==k]
mc_sum[k] = mcsum[groups==k]
aa_sum[k] = aasum[groups==k]
bb_sum[k] = bbsum[groups==k]
cc_sum[k] = ccsum[groups==k]
ab_sum[k] = absum[groups==k]
ac_sum[k] = acsum[groups==k]
bc_sum[k] = bcsum[groups==k]
self.binfreqs[k] += bincounts[groups==k]
except:
continue
self.fO1 += -jump_sum
self.fODp[:,0] += a_sum*self.dt
self.fODp[:,1] += b_sum*self.dt
self.fODp[:,2] += c_sum*self.dt
self.fOD[:,0] += ma_sum*self.dt
self.fOD[:,1] += mb_sum*self.dt
self.fOD[:,2] += mc_sum*self.dt
self.fOfD[:,0,0] += aa_sum*self.dt
self.fOfD[:,1,1] += bb_sum*self.dt
self.fOfD[:,2,2] += cc_sum*self.dt
self.fOfD[:,0,1] += ab_sum*self.dt
self.fOfD[:,1,0] += ab_sum*self.dt # redundant
self.fOfD[:,0,2] += ac_sum*self.dt
self.fOfD[:,2,0] += ac_sum*self.dt
self.fOfD[:,1,2] += bc_sum*self.dt
self.fOfD[:,2,1] += bc_sum*self.dt
for k in range(self.bins):
prog.animate(k+1)
if self.mode != 'fonly':
self.gODinv[k,0] += -np.sum(jumps[binpos==k]**2*a[binpos==k])*0.5/self.dt
self.gODinv[k,1] += -np.sum(jumps[binpos==k]**2*b[binpos==k])*0.5/self.dt
self.gODinv[k,2] += -np.sum(jumps[binpos==k]**2*c[binpos==k])*0.5/self.dt
self.gODf1[k,0,0] += np.sum(aa[binpos==k]*m[binpos==k])*self.dt*0.5
self.gODf1[k,1,1] += np.sum(bb[binpos==k]*m[binpos==k])*self.dt*0.5
self.gODf1[k,2,2] += np.sum(cc[binpos==k]*m[binpos==k])*self.dt*0.5
gODf1ab = np.sum(ab[binpos==k]*m[binpos==k])*self.dt*0.5
gODf1bc = np.sum(bc[binpos==k]*m[binpos==k])*self.dt*0.5
gODf1ac = np.sum(ac[binpos==k]*m[binpos==k])*self.dt*0.5
self.gODf1[k,0,1] += gODf1ab
self.gODf1[k,1,0] += gODf1ab
self.gODf1[k,0,2] += gODf1ac
self.gODf1[k,2,0] += gODf1ac
self.gODf1[k,1,2] += gODf1bc
self.gODf1[k,2,1] += gODf1bc
# aaa, aab, aac
self.gODff[k,0,0,0] += np.sum(a[binpos==k]**3)*self.dt*0.5
self.gODff[k,1,1,1] += np.sum(b[binpos==k]**3)*self.dt*0.5
self.gODff[k,2,2,2] += np.sum(c[binpos==k]**3)*self.dt*0.5
# aab terms
gODfaab = np.sum(a[binpos==k]**2*b[binpos==k])*self.dt*0.5
self.gODff[k,0,0,1] += gODfaab
self.gODff[k,0,1,0] += gODfaab
self.gODff[k,1,0,0] += gODfaab
# aac terms
gODfaac = np.sum(a[binpos==k]**2*c[binpos==k])*self.dt*0.5
self.gODff[k,0,0,2] += gODfaac
self.gODff[k,0,2,0] += gODfaac
self.gODff[k,2,0,0] += gODfaac
#abb terms
gODfabb = np.sum(a[binpos==k]*b[binpos==k]**2)*self.dt*0.5
self.gODff[k,0,1,1] += gODfaab
self.gODff[k,1,0,1] += gODfaab
self.gODff[k,1,1,0] += gODfaab
#bbc terms
gODfbbc = np.sum(b[binpos==k]**2*c[binpos==k])*self.dt*0.5
self.gODff[k,1,1,2] += gODfbbc
self.gODff[k,1,2,1] += gODfbbc
self.gODff[k,2,1,1] += gODfbbc
#acc terms
gODfacc = np.sum(a[binpos==k]*c[binpos==k]**2)*self.dt*0.5
self.gODff[k,0,2,2] += gODfacc
self.gODff[k,2,0,2] += gODfacc
self.gODff[k,2,2,0] += gODfacc
#bcc terms
gODfbcc = np.sum(b[binpos==k]*c[binpos==k]**2)*self.dt*0.5
self.gODff[k,1,2,2] += gODfbcc
self.gODff[k,2,1,2] += gODfbcc
self.gODff[k,2,2,1] += gODfbcc
# abc terms
gODfabc = np.sum(a[binpos==k]*b[binpos==k]*c[binpos==k])*self.dt*0.5
self.gODff[k,0,1,2] += gODfabc
self.gODff[k,1,2,0] += gODfabc
self.gODff[k,2,0,1] += gODfabc
self.gODff[k,2,1,0] += gODfabc
self.gODff[k,1,0,2] += gODfabc
self.gODff[k,0,2,1] += gODfabc
if self.mode != "fonly":
self.gO11 = self.binfreqs
self.gOD1 = self.fOD*0.5
self.gODf2 = self.fOfD
self.gOgpDp = self.gODf2*0.5
self.gOD2 = self.fOD
self.gODp1 = self.fOD
self.gODpf = self.fOfD
self.gODp2 = self.fODp
self.gO12 = self.fO1
self.trajectoryindex += n
# merge data with prior data if available
try:
newpositions = np.append(self.positions,positions)
indices = np.argsort(newpositions)
self.positions = newpositions[indices]
self.jumps = np.append(self.jumps,jumps)[indices]
self.a = np.append(self.a,a)[indices]
self.b = np.append(self.b,b)[indices]
self.c = np.append(self.c,c)[indices]
self.binpos = np.append(self.binpos,binpos)[indices]
self.m = np.append(self.m,m)[indices]
self.sortedtimes = np.append(self.sortedtimes,times)[indices]
except:
self.positions = positions
self.jumps = jumps
self.a = a
self.b = b
self.c = c
self.binpos = binpos
self.m = m
self.sortedtimes = times
try:
self.jump_sum += jump_sum
self.jumpsq_sum += jumpsq_sum
self.m_sum += m_sum
except:
self.jump_sum = jump_sum
self.jumpsq_sum = jumpsq_sum
self.m_sum = m_sum
return
def generate_diff(self, method="dice", k=None, lower_bound=None, k_fixed=False, alpha=0.3):
""" Given a similarity matrix generate a
new similarity matrix based on the similarity
of the top k closest matches for each pair of
shape instances
"""
if k is None:
if self.verbose:
print "k not set. Finding a good value..."
k = self._get_k()
if self.verbose:
print "Found k:", k
if k_fixed is True:
k_i = k
else:
upper_bound = k
if lower_bound is None:
lower_bound = int(upper_bound * alpha) + 1
idx_top_k = self.affinity_matrix.argsort(axis=1)[:, 0:k]
if self.verbose:
print "Building new similarity matrix..."
""" start progress bar """
prog = pb.progressBar(0, self.total_instances, 77)
oldprog = str(prog)
""" end progress bar """
for i in xrange(self.total_instances):
shape_ranks = idx_top_k[i, :]
for j in xrange(i + 1, self.total_instances):
if k_fixed is not True:
j_idx = where(shape_ranks == j)[0]
if not j_idx:
k_i = upper_bound
else:
k_i = j_idx[0] + 1
if k_i < lower_bound:
k_i = lower_bound
a = set(idx_top_k[i, 0:k_i])
b = set(idx_top_k[j, 0:k_i])
if method is "dice":
distance = 1 - self._dice_set_diff(a, b)
elif method is "jaccard":
distance = 1 - self._jaccard_set_diff(a, b)
else:
raise ValueError(
"Comparison algorithm not found. \
Currently implemented: dice, jaccard"
)
self.processed_matrix[i, j] = distance
if self.verbose:
""" start progress bar """
prog.updateAmount(i)
if oldprog != str(prog):
print prog, "\r",
sys.stdout.flush()
oldprog = str(prog)
""" end progress bar """
if self.verbose:
print "\n"
self.processed_matrix += self.processed_matrix.transpose()
return self.processed_matrix
