def ShowResults(sampleFile="sample.txt",sample_infoFile="sample_info.txt", levelsFile="levels.txt",posteriorFile="posterior_sample.txt",DataFile='Data/dummy_data.dat',weightFile='weights.txt',plotfilebase=None,clearall=True,showfullsample=True): import postprocess import display if clearall: for p in range(0,6): figure(p+1) clf() postprocess.postprocess(sampleFile=sampleFile,sample_infoFile=sample_infoFile, levelsFile=levelsFile,posteriorFile=posteriorFile,weightFile=weightFile) display.Display(posteriorFile=posteriorFile,DataFile=DataFile,plotfilebase=plotfilebase,showfullsample=showfullsample)
def main(): questions = open("qs1.txt", "r") qs = questions.readlines() for line in qs: print(line) firstWord = line.partition(' ')[0] wQuestions = ["who", "what", "when", "where", "why", "is"] query = "" found = False if line.lower() == "what is the meaning of life?": print "42" elif firstWord.lower() in wQuestions: #print "in wQuestions" query = rewriteQuestion(line) plainQuery = query[1:] print(query) if query != "none": query = "\"" + query + "\"" allTweets = returnTweets(query) if firstWord.lower() == "who" or firstWord.lower() == "where" or firstWord.lower() == "what": #print "postprocessWhoWhere" #print "query is" + plainQuery found = postprocessWhoWhereWhat(allTweets, plainQuery) else: found = postprocess(firstWord.lower(), allTweets, query) if not found: #print "hi" findAnswer(line) else: findAnswer(line)
def main(): parser = argparse.ArgumentParser(description='Postprocess word sense induction file for all files in a directory.') parser.add_argument('ddt_dir', help='Path to a directory with csv files with DDTs: "word<TAB>sense-id<TAB>keyword<TAB>cluster" w/o header by default. Here <cluster> is "word:sim<SPACE><SPACE>word:sim<SPACE><SPACE>..."') parser.add_argument('-min_size', help='Minimum cluster size. Default -- 5.', default="5") args = parser.parse_args() print "Input DDT directory (pattern):", args.ddt_dir print "Min size:", args.min_size #postprocess(args.ddt, output_fpath, filtered_fpath, int(args.min_size)) for cluster_fpath in glob.glob(args.ddt_dir): if splitext(cluster_fpath)[-1] == ".csv": print "\n>>>", cluster_fpath postprocess( cluster_fpath, cluster_fpath+"-minsize" + args.min_size + ".csv", cluster_fpath+"-minsize" + args.min_size + "-filtered.csv", args.min_size)
def main(): # Define a timer t = Timer() # Start the timer t.start() # Start by getting all the data from twitter # Note that this should be commented out if there is no API keys # Note max is 100 tweets per senator due to twitter limits createCSV.getData(100) # Preprocess the data and create new CSV files preprocess.preprocess() # Analyze the data and generate new CSV files analyze.analyze() # Postprocess the data and create new CSV files postprocess.postprocess() # Generate the visualizations of the data visualize.visualize() # Stop the timer and display runtime elapsed_time = t.stop()
def collect_images(file_name, input_dir, output_dir, num_views): """Collects images from sim.""" if not os.path.exists(output_dir): os.makedirs(output_dir) f = h5py.File(os.path.join(input_dir, file_name), 'r') pregrasp, postgrasp = postprocess(f['pregrasp'], f['postgrasp']) if pregrasp is None or postgrasp is None: print('No data in %s' % file_name) return num_samples = len(pregrasp[pregrasp.keys()[0]]) out_file = h5py.File(os.path.join(output_dir, file_name), 'w') pregrasp_group = out_file.create_group('pregrasp') postgrasp_group = out_file.create_group('postgrasp') dt = h5py.special_dtype(vlen=unicode) for key in pregrasp.keys(): num_var = pregrasp[key].shape[1] pregrasp_group.create_dataset(key, (num_samples * num_views, num_var)) postgrasp_group.create_dataset(key, (num_samples * num_views, num_var)) # Collecting this from the sim pregrasp_group.create_dataset('grasp_wrt_cam', (num_samples * num_views, 18)) pregrasp_group.create_dataset('image_name', (num_samples * num_views, ), dtype=dt) pregrasp_group.create_dataset('frame_work2cam', (num_samples * num_views, 12)) # Start collection object_name = file_name.split('.')[0] for i in range(num_samples): print('Querying for image set %d / %d ' % (i, num_samples)) low = i * num_views high = (i + 1) * num_views for key in pregrasp.keys(): pregrasp_group[key][low:high] = np.repeat(pregrasp[key][i:i + 1], num_views, axis=0) postgrasp_group[key][low:high] = np.repeat(postgrasp[key][i:i + 1], num_views, axis=0) # Query the simulator for some images; note that we're only collecting # this info for the pregrasp here. grasp_wrt_cam, frame_work2cam, image_names = \ query_minibatch(pregrasp, i, num_views, object_name) pregrasp_group['grasp_wrt_cam'][low:high] = grasp_wrt_cam pregrasp_group['image_name'][low:high] = image_names pregrasp_group['frame_work2cam'][low:high] = frame_work2cam f.close()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--path', type=str, help='path to the image') parser.add_argument('--sl', type=str, help='source language') parser.add_argument('--tl', type=str, default='en', help='target language') parser.add_argument('--region', type=str, default='us-east-2', help='AWS region name') args = parser.parse_args() path = args.path sl = args.sl tl = args.tl region = args.region mask_generator(path=path, region_name=region) inpainter() postprocess() bucket = '' document = '' block_count, text, coord = text_extractor(bucket, document, path, region_name=region) #Getting the translated text i.e. ttext ttext = translator(text, sl, tl, region_name=region) #Opening the inpainted image image = cv2.imread('./results/image.jpg') #Writing the text on inpainted image written_img = writer(image, ttext, coord) #Output Image cv2.imwrite('./Output/Output.jpg', written_img) print("Blocks detected: " + str(block_count))
def diffusive_integrator(sampler, tolerance = 0.01): # sample from sampler samples = [] # coordinates samplesInfo = [] # logX, likelihood, (tieBreaker), livePointID finfo = open('sample_info.txt', 'w') finfo.write("# index, logLikelihood, tieBreaker, ID.\n") #numpy.savetxt(f, self.samplesInfo, fmt='%d %e %f %d') fsample = open('sample.txt', 'w') fsample.write("# Samples file. One sample per line.\n") #numpy.savetxt(f, self.samples) for i in range(1000): # keep sampling sample, sampleInfo = sampler.next() print 'new sample:', sample, sampleInfo # each sample file contains one line per live point / particle # sampleFile :: self.live_pointsu # sampleInfoFile :: self.indices, self.live_pointsL, (tieBreaker), ID? # levelFile :: logX, cutoff, (tieBreaker), accepts, tries, exceeds, visits # adding an extra number because postprocess can't deal with 1d data fsample.write(' '.join(['%e' % s for s in sample]) + " 1\n") fsample.flush() finfo.write("%d %e %f %d\n" % sampleInfo) finfo.flush() samples.append(sample) samplesInfo.append(sampleInfo) levels = numpy.array([[l.logX, l.cutoff, numpy.random.uniform(), l.accepts, l.tries, l.exceeds, l.visits] for l in sampler.levels]) flevels = open('levels.txt', 'w') flevels.write("# logX, logLikelihood, tieBreaker, accepts, tries, exceeds, visits.\n") numpy.savetxt(flevels, levels, fmt='%f %e %f %d %d %d %d') flevels.close() if i % 20 == 19: # check if tolerance achieved already logZ, H, weights, logZerr = postprocess.postprocess( loaded=(levels, numpy.asarray(samplesInfo), numpy.asarray(samples)), save=False, plot=False, verbose=False, numResampleLogX=10) print 'logZ = %.3f +- %.3f' % (logZ, logZerr) if logZerr < tolerance / 3: break return dict(logZ=logZ, logZerr=logZerr, samples=sampler.samples, weights=weights, information=H)
def main(args): '''Example script to train a model on the sample lyrics dataset''' c = Configuration() if args.artist: c.set_artist(args.artist) print("Hyperparameters: ", c) print("Loading data from path: ", c.path) lyrics_dataset = pp.read_lyrics_files(c.path) tokenized = pp.tokenize(lyrics_dataset) x, y, dictionary = pp.preprocess(tokenized, c.window_size) training_data = DataLoader(list(zip(x, y)), batch_size=c.train_batch_size, shuffle=True) model = LyricPredictor(len(dictionary), c.output_size) print("Training model...") model, _, _ = train(model=model, training_data=training_data, num_epochs=c.num_epochs, lr=c.lr, grad_norm=c.grad_max_norm) print("Saving model: ", c.model_path) torch.save(model, c.model_path) print("Saving dictionary: ", c.dictionary_path) torch.save(dictionary, c.dictionary_path) print("Generating lyrics...") seed_lyrics = generate_seed_lyrics(tokenized, c.window_size, args.censored) predicted_lyrics = predict(model, seed_lyrics, dictionary, num_words=args.words, topk=c.predict_topk) predicted_lyrics = postprocess(predicted_lyrics, args.censored) print(predicted_lyrics)
def main(config_path, results_dir): r""" This function runs the whole analysis pipeline """ starttime = time.time() logger.define_logging(logpath=results_dir + '/optimisation_results') # Preproccessing logging.info('Preprocess data') connect_to_oep(config_path=config_path, results_dir=results_dir) prepare_timeseries(config_path=config_path, results_dir=results_dir) preprocess_closed_data(config_path=config_path, results_dir=results_dir) # Run the optimisation model logging.info('Run optimisation model') run_model_dessau(config_path=config_path, results_dir=results_dir) # Postprocessing logging.info('Postprocess data') postprocess(config_path=config_path, results_dir=results_dir) # Plotting logging.info('Create plots') create_plots(config_path=config_path, results_dir=results_dir) # Build a report # cmd = ['pdflatex', '-interaction=nonstopmode', '--output-directory={0}/presentation/build'.format(abs_path), '{0}/presentation/report.tex'.format(results_dir)] # process = subprocess.call(cmd) # , stdout=open(os.devnull, 'wb')) endtime = time.time() logging.info(f'Analysis lastet {endtime-starttime} sec.') return True
def simulation_wrapper_noparams(host, COMSOL_model, paramfile): #CST #Preprocessing - calculate current distribution current_density_file, paramfile, frequency = preprocess(paramfile) if not within_frequency_bounds(frequency, 7.03e09, 100e6): return (-dist_from_clock(frequency, 7.03e09), 0 ) # Negative as want to optimize against this else: #COMSOL simulation remote_interface(host, COMSOL_model, paramfile, current_density_file) #Postprocess - generate g_ens and pi_fidelity file_gens2_number = os.getcwd( ) + '/downloads/exports/g_ens2_number.csv' g_ens, FWHM = postprocess(file_gens2_number) return (g_ens, FWHM)
def detect_card_details(card_img, net, save=False, image_path=".jpg", prepro=False): # Preprocess image if prepro: card_img = preprocess(card_img) # Predict classes from image card_img, outs = detection(card_img, net) # Postprocess to extract card details image_name = os.path.split(image_path)[1] _card_details = postprocess(card_img, outs, image_name, save) return _card_details
def main(args): global NUMBER_FILES_TO_GENERATE if args and len(args) > 0: try: NUMBER_FILES_TO_GENERATE = int(args[0]) except: raise ValueError("The first argument to main.py must be an integer!") util.init() global heads global pcfg global objects global primitives # Call the main rule-parsing/generation module first to get our pcfg # and other data ready to go (heads, pcfg, objects, raw_primitives) = generate_rules.process_all(); # Prepare primitives dictionaries by doing some pre-processing on them # so that future stuff is accessible much quicker: primitives = prepare_primitives(raw_primitives) print "\nStarting generation process. Don't be alarmed if it takes awhile." numGenerated = 0 while True: try: print "Attempting to generate artificial Python file {}/{}".format(numGenerated+1, NUMBER_FILES_TO_GENERATE) tree = makeNode("Module", 0, []) outcode = "generated/code{}.py".format(numGenerated+1) outast = "generated/AST{}.txt".format(numGenerated+1) with open(outcode, "w") as out: Unparser.Unparser(tree, out) with open(outcode, "r") as in_f: text = in_f.read() with open(outcode, 'w') as out: out.write(postprocess.postprocess(text)) with open(outast, "w") as out: out.write(ast.dump(tree)) print "Successfully created {} and {}".format(outcode, outast) numGenerated += 1 if numGenerated >= NUMBER_FILES_TO_GENERATE: break except Exception as e: pass # The Fifth Amendment allows me to do this. Shhh...
def preview(api: sly.Api, task_id, context, state, app_logger): bg_images = update_bg_images(api, state) if len(bg_images) == 0: sly.logger.warn("There are no background images") else: cache_dir = os.path.join(app.data_dir, "cache_images_preview") sly.fs.mkdir(cache_dir) sly.fs.clean_dir(cache_dir) img, ann, res_meta = synthesize(api, task_id, state, meta, images_info, labels, bg_images, cache_dir) res_meta, ann = postprocess(state, ann, res_meta, sly.ProjectMeta()) if state["taskType"] == "inst-seg" and state[ "highlightInstances"] is True: res_meta, ann = highlight_instances(res_meta, ann) src_img_path = os.path.join(cache_dir, "res.png") dst_img_path = os.path.join(f"/flying_object/{task_id}", "res.png") sly.image.write(src_img_path, img) file_info = None if api.file.exists(team_id, dst_img_path): api.file.remove(team_id, dst_img_path) file_info = api.file.upload(team_id, src_img_path, dst_img_path) gallery = dict(empty_gallery) gallery["content"]["projectMeta"] = res_meta.to_json() gallery["content"]["annotations"] = { "preview": { "url": file_info.full_storage_url, "figures": [label.to_json() for label in ann.labels] } } gallery["content"]["layout"] = [["preview"]] fields = [ { "field": "data.gallery", "payload": gallery }, { "field": "state.previewLoading", "payload": False }, ] api.task.set_fields(task_id, fields)
def app_estimate(): if request.headers.getlist("X-Forwarded-For"): ip = request.headers.getlist("X-Forwarded-For")[0] else: ip = request.remote_addr slack.notify(text=f"[ESTIMATE] : \n from {ip}") test_csv = request.files.get('test_csv') if test_csv is None: return abort(404) if test_csv.content_type == 'text/csv' or test_csv.content_type == 'application/vnd.ms-excel': test_df = pd.read_csv(test_csv) else: return abort( 404, { 'code': 'Wrong file', 'message': 'the file is not csv please send me csv' }) test_encoded = preprocess(test_df) test_preds_df = estimate(test_encoded) submit_df = postprocess(test_preds_df, test_df) temp_time = time.time() test_df.to_csv(osp.join(DATA_DIR, f'dip_input_{temp_time}.csv'), index=False, header=True) submit_df.to_csv(osp.join(OUTPUT_DIR, f'dip_estimete_{temp_time}.csv'), index=False, header=True) output_name = f"estimate_{test_csv.filename}" if output_name[-4:] != ".csv": output_name += ".csv" textStream = StringIO() submit_df.to_csv(textStream, index=False, header=True) return Response( textStream.getvalue(), mimetype="text/csv", headers={"Content-disposition": f"attachment; filename={output_name}"})
def quantile_regression(): df = init() raw_input, processing_info = preprocess(df) raw_predictions = train_and_predict(raw_input) postprocess(raw_input, raw_predictions, processing_info)
def put_out(input): # this code sounds weird :-) return sep.join(postprocess.postprocess(input.split(sep), sep))
def generate(api: sly.Api, task_id, context, state, app_logger): bg_images = update_bg_images(api, state) if len(bg_images) == 0: sly.logger.warn("There are no background images") else: cache_dir = os.path.join(app.data_dir, "cache_images_generate") sly.fs.mkdir(cache_dir) sly.fs.clean_dir(cache_dir) if state["destProject"] == "newProject": res_project_name = state["resProjectName"] if res_project_name == "": res_project_name = "synthetic" res_project = api.project.create(workspace_id, res_project_name, change_name_if_conflict=True) elif state["destProject"] == "existingProject": res_project = api.project.get_info_by_id(state["destProjectId"]) res_dataset = api.dataset.get_or_create(res_project.id, state["resDatasetName"]) res_meta = sly.ProjectMeta.from_json( api.project.get_meta(res_project.id)) progress = sly.Progress("Generating images", state["imagesCount"]) refresh_progress_images(api, task_id, progress) for i in range(state["imagesCount"]): img, ann, cur_meta = synthesize(api, task_id, state, meta, images_info, labels, bg_images, cache_dir, preview=False) merged_meta, new_ann = postprocess(state, ann, cur_meta, res_meta) if res_meta != merged_meta: api.project.update_meta(res_project.id, merged_meta.to_json()) res_meta = merged_meta image_info = api.image.upload_np( res_dataset.id, f"{i + res_dataset.items_count}.png", img) api.annotation.upload_ann(image_info.id, new_ann) progress.iter_done_report() if progress.need_report(): refresh_progress_images(api, task_id, progress) res_project = api.project.get_info_by_id(res_project.id) fields = [ { "field": "data.started", "payload": False }, { "field": "data.resProjectId", "payload": res_project.id }, { "field": "data.resProjectName", "payload": res_project.name }, { "field": "data.resProjectPreviewUrl", "payload": api.image.preview_url(res_project.reference_image_url, 100, 100) }, ] api.task.set_fields(task_id, fields)
f = open('OPTIONS_TEMPLATE', 'r') o = f.read() f.close() o = o.replace('XXX', '500') o = o.replace('YYY', '400') f = open('OPTIONS', 'w') f.write(o) f.close() os.system('./main -t 8 -s 0') l = loadtxt('levels.txt') l = l[l[:,1] <= -30., :] f = open('OPTIONS_TEMPLATE', 'r') o = f.read() f.close() o = o.replace('XXX', str(l.shape[0])) o = o.replace('YYY', '3000') f = open('OPTIONS', 'w') f.write(o) f.close() os.system('./main -t 8 -s 0 -l levels.txt') postprocess.postprocess(cut=0.25, plot=False) os.system('cp posterior_sample.txt Results/' + str(i) + '.txt')
def main(): # constants num_channels = cfg.config['num_channels'] classes = cfg.config['classes'] num_classes = len(classes) height = cfg.config['height'] width = cfg.config['width'] anchors = cfg.config['anchors'] num_anchors = len(anchors[0]) confidency = cfg.config['confidency'] tp_iou = cfg.config['TP_IoU'] nms_iou = cfg.config['NMS_IoU'] cuda = cfg.config['CUDA'] target_dir = cfg.config['path']['test'] image_dir = cfg.config['path']['image'] weight_dir = cfg.config['path']['weight_test'] image_paths = pre.load_image_paths(image_dir, target_dir) target_paths = pre.load_dir_paths(target_dir) weight_paths = pre.load_dir_paths(weight_dir) num_images = len(image_paths) # network net = model.YOLOv3(num_channels, num_classes, num_anchors) if cuda: net = net.cuda() # calculate loss or mAP for each weights for weight_path in weight_paths: net.load_state_dict(torch.load(weight_path)) net.eval() loss_giou = 0.0 loss_obj = 0.0 loss_cls = 0.0 loss_blc = 0.0 predictions = [] targets = [] t0 = time.time() for i in range(num_images): # load image and target image = pre.load_image(image_paths[i], height, width, cuda) image = image.unsqueeze(0) target = pre.load_targets(target_paths[i:i + 1], num_classes, height, width, cuda) # predict bbox prediction = [pred.detach() for pred in net(image)] # calculate loss loss = post.calculate_loss(prediction, target, anchors, height, width, cuda).detach() loss_giou += float(loss[0]) loss_obj += float(loss[1]) loss_cls += float(loss[2]) loss_blc += float(loss[3]) # save prediction and target as numpy array prediction = post.postprocess(prediction, anchors, height, width, confidency, nms_iou, cuda) predictions.extend([pred.detach() for pred in prediction]) targets.extend([tagt.detach() for tagt in target]) # normalize loss loss_giou /= num_images loss_obj /= num_images loss_cls /= num_images loss_blc /= num_images # calculate AP AP = post.calculate_AP(predictions, targets, tp_iou, cuda) elapsed_time = time.time() - t0 print(( 'Weight: {}, Elapsed Time: {:.2f}s, ' + # 'GIoU Loss: {:.2f}, ' + # 'Objectness Loss: {:.2f}, ' + # 'Class Loss: {:.2f}, ' + # 'Balance Loss: {:.2f}, ' + 'Loss: {:.2f}, ' + 'AP: ' + ', '.join(['{:.2f}'.format(ap * 100) for ap in AP]) + ', ' 'mAP: {:.2f}'.format(100 * sum(AP) / len(AP))).format( weight_path, elapsed_time, loss_giou, loss_obj, loss_cls, loss_blc, loss_giou + loss_obj + loss_cls + loss_blc))
# Copyright (c) 2009, 2010, 2011, 2012 Brendon J. Brewer. # # This file is part of DNest3. # # DNest3 is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # DNest3 is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with DNest3. If not, see <http://www.gnu.org/licenses/>. # Import postprocess from two directories up # http://stackoverflow.com/a/9806045 import os parentdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) parentdir = os.path.dirname(parentdir) os.sys.path.insert(0, parentdir) import postprocess postprocess.postprocess() import display
import postprocess postprocess.postprocess(cut=0., plot=False, moreSamples=1.) # import display import make_plots import histograms
import postprocess postprocess.postprocess(cut=0.) import display
def convert(self, text): text = '\n'+text+'\n'# add embed in newlines (makes regex replaces work better) # first we search for [[code]] statements as we don't want any replacement to happen inside those code blocks! code_blocks = dict() code_blocks_found = re.findall(re.compile(r'(\[\[code( type="([\S]+)")?\]\]([\s\S ]*?)\[\[/code\]\])',re.MULTILINE), text) for code_block_found in code_blocks_found: tmp_hash = str(uuid.uuid4()) text = text.replace(code_block_found[0],tmp_hash,1) # replace code block with a hash - to fill it in later code_blocks[tmp_hash] = "\n"+string.join([" " + l for l in code_block_found[-1].strip().split("\n") ],"\n")+"\n" for search, replacement in self.static_replacements.items(): text = text.replace(search,replacement,1) # search for any of the simpler replacements in the dictionary regex_replacements for s_reg, r_reg in self.regex_replacements.items(): text = re.sub(re.compile(s_reg,re.MULTILINE),r_reg,text) # TITLES -- replace '+++ X' with '=== X ===' for titles in re.finditer(r"^(\++)([^\n]*)$", text, re.MULTILINE): header = ("=" * len(titles.group(1))) text = text.replace(titles.group(0), header + (titles.group(2) + " ") + header) # LISTS(*) -- replace ' *' with '***' and so on for stars in re.finditer(r"^([ \t]+)\*", text, re.MULTILINE): text = text[:stars.start(1)] + ("*" * len(stars.group(1))) + text[stars.end(1):] # LISTS(#) -- replace ' #' with '###' and so on for hashes in re.finditer(r"^([ \t]+)\*", text, re.MULTILINE): text = text[:hashes.start(1)] + ("#" * len(hashes.group(1))) + text[hashes.end(1):] # INTERNAL LINKS -- replace [[[bink]]] with [[bink]] for inlink in re.finditer(r"\[\[\[([\s\S ]*?)\]\]\]", text): text = text.replace(inlink.group(0), "[["+inlink.group(1)+"]]") # IMAGES for image in re.finditer(r"\[\[image([\s\S ]*?)\]\]", text): text = text.replace(image.group(0), "[[File:" + image.group(1) + "]]") # START TABLE for table in re.finditer(r"\[\[table([\s\S ]*?)\]\]", text): #text = text.replace(table.group(0), "{|" + table.group(1)) text = text.replace(table.group(0), "{|") # START ROW for row in re.finditer(r"\[\[row([\s\S ]*?)\]\]", text): #text = text.replace(row.group(0), "|-" + row.group(1)) text = text.replace(row.group(0), "|-") # START CELL for cell in re.finditer(r"\[\[cell([\s\S ]*?)\]\]", text): #text = text.replace(cell.group(0), "|" + cell.group(1)) text = text.replace(cell.group(0), "|") # ENDS for end in re.finditer(r"\[\[/([\s\S ]*?)\]\]", text): token = end.group(1) if token == "table": text = text.replace(end.group(0), "|}") elif token == "row": # end row tabs are not necessary in mediawiki text = text.replace(end.group(0), "") elif token == "cell": # end cell tabs are not necessary in mediawiki text = text.replace(end.group(0), "") # now we substitute back our code blocks for tmp_hash, code in code_blocks.items(): text = text.replace(tmp_hash, code, 1) # Process color corrections startpos = 0 while -1 != startpos: startpos = text.find("##", startpos) pipepos = text.find("|", startpos + 2) endpos = text.find("##", startpos + 2) if startpos != -1 and pipepos != -1 and endpos != -1 and endpos > pipepos: color = text[startpos + 2 : pipepos].strip() colored = text[pipepos + 1 : endpos].strip() text = text[: startpos] + "<span style=\"color:" + color + "\">" + colored + \ "</span>" + text[endpos + 2 :] startpos = endpos # Process math corrections startpos = 0 while -1 != startpos: startpos = text.find("[[$", startpos) endpos = text.find("$]]", startpos) if startpos != -1 and endpos != -1: mathtext = text[startpos + 3 : endpos].strip() text = text[: startpos] + "<math>" + mathtext + "</math>" + text[endpos + 3 :] startpos = endpos # Process table corrections startpos = 0 while -1 != startpos: startpos = text.find("\n||", startpos) if startpos == -1: break # Find end of table endpos = text.find("\n", startpos + 3) while endpos < len(text) - 3 and "||" == text[endpos + 1: endpos + 3]: endpos = text.find("\n", endpos + 3) # Found bounds of text chunk: reformat table fixup = text[startpos + 1 : endpos].replace("||~", "!!") fixup = fixup.split("\n") fixout = ["", "{| class=\"wikitable\""] for i in xrange(len(fixup)): if fixup[i][0 : 2] == "||" or fixup[i][0 : 2] == "!!": out = fixup[i].strip()[1 : ] fixout.append(out[ : -2 if out[-2 : ] in ["||", "!!"] else 0]) else: print("Failed to parse item %d/%d: '%s'" % (i, len(fixup), fixup[i])) sys.exit(-1) fixout.append("|}" if i == len(fixup) - 1 else "|-") # Construct output table text fullout = "\n".join(fixout) text = text[ : startpos] + fullout + text[endpos : ] startpos = startpos + len(fullout) # Repair multi-newlines text = re.sub(r"\n\n+", "\n\n", text, re.M) # Repair starting newlines text = text.strip() # Optional postprocessing stage text = postprocess.postprocess(text) return text
def main(): parser = argparse_ArgumentParser("Input parameters") parser.add_argument("--input_file_name", default="input_toy.yaml", help="Input parameters file name") parser.add_argument("--out_dir_name", default="/results", help="Output directory name") parser.add_argument("--train_test_files_dir", default="", help="Train test file path") parser.add_argument("--n_pts", default=1, help="number of partitions (computers)") parser.add_argument("--over_t", help="Overlap threshold") parser.add_argument("--model_dir", help="Directory containing model") parser.add_argument("--sample_dir", help="Sample files dir + /res") parser.add_argument("--sample_folders_prefix", help="Input parameters file name /results..") parser.add_argument( "--sample_folders_prefix_final", help="Input file name to use final merged results Use as /results..") parser.add_argument( "--sample_folders_list", nargs='+', help="Input parameters file name /results.. separated by commas") parser.add_argument("--graph_files_dir", default="", help="Graph files' folder path") parser.add_argument("--overlap_method", default=1, help="Overlap method option: qi, default: jaccard") parser.add_argument("--infer_overlap_threshold", default='n', help="y or n") args = parser.parse_args() with open(args.input_file_name, 'r') as f: inputs = yaml_load(f, yaml_Loader) if args.overlap_method: inputs['overlap_method'] = args.overlap_method if args.over_t: inputs['over_t'] = float(args.over_t) if args.sample_dir: inputs['sample_dir'] = args.sample_dir if args.model_dir: inputs['model_dir'] = args.model_dir if args.infer_overlap_threshold: inputs['infer_overlap_threshold'] = args.infer_overlap_threshold # Override output directory name if same as gen if args.out_dir_name or inputs['out_comp_nm'] == "/results/res": if not os_path.exists(inputs['dir_nm'] + args.out_dir_name): os_mkdir(inputs['dir_nm'] + args.out_dir_name) inputs['out_comp_nm'] = args.out_dir_name + "/res" inputs['train_test_files_dir'] = '' if args.train_test_files_dir: if not os_path.exists(inputs['dir_nm'] + args.train_test_files_dir): os_mkdir(inputs['dir_nm'] + args.train_test_files_dir) inputs['train_test_files_dir'] = args.train_test_files_dir inputs['graph_files_dir'] = '' if args.graph_files_dir: if not os_path.exists(inputs['dir_nm'] + args.graph_files_dir): os_mkdir(inputs['dir_nm'] + args.graph_files_dir) inputs['graph_files_dir'] = args.graph_files_dir logging_basicConfig(filename=inputs['dir_nm'] + inputs['out_comp_nm'] + "_logs.yaml", level=logging_INFO) # fin_list_graphs = control(myGraph,inputs,n=50) if "sample_dir" not in inputs: inputs['sample_dir'] = inputs['out_comp_nm'] myGraphName = inputs['dir_nm'] + inputs['graph_files_dir'] + "/res_myGraph" with open(myGraphName, 'rb') as f: myGraph = pickle_load(f) if 'infer_overlap_threshold' in inputs: if inputs['infer_overlap_threshold'] == 'y': pp_flag = 0 if inputs['dir_nm'] == 'yeast': pp_flag = 1 if 'overlap_method' in inputs: if inputs['overlap_method'] == 'qi': inputs['over_t'] = get_overlap_threshold_qi( inputs, pp_flag, myGraph) else: inputs['over_t'] = get_overlap_threshold( inputs, pp_flag, myGraph) with open(inputs['dir_nm'] + inputs['out_comp_nm'] + "_input_pp.yaml", 'w') as outfile: yaml_dump(inputs, outfile, default_flow_style=False) out_comp_nm = inputs['dir_nm'] + inputs['out_comp_nm'] out_comp_nm_sample = inputs['dir_nm'] + inputs['sample_dir'] out_comp_nm_model = inputs['dir_nm'] + inputs['model_dir'] modelfname = out_comp_nm_model + "_model" scalerfname = out_comp_nm_model + "_scaler" with open(scalerfname, 'rb') as f: scaler = pickle_load(f) pred_comp_list = [] sdndap = pred_comp_list.append if args.sample_folders_list: for folder in args.sample_folders_list: allfiles = './' + inputs['dir_nm'] + folder + '/res_pred_comp_list*' for fname in glob(allfiles, recursive=True): with open(fname, 'rb') as f: pred_comp_tmp = pickle_load(f) for snode in pred_comp_tmp: sdndap(snode) elif args.sample_folders_prefix_final: allsubd = './' + inputs[ 'dir_nm'] + args.sample_folders_prefix_final + '*/res_pred.out' for fname in glob(allsubd, recursive=True): with open(fname) as f: complexes_score = [ line.rstrip().split() for line in f.readlines() ] pred_comp_tmp = [(frozenset(comp[:-1]), float(comp[-1])) for comp in complexes_score] for snode in pred_comp_tmp: sdndap(snode) elif args.sample_folders_prefix: allsubd = './' + inputs[ 'dir_nm'] + args.sample_folders_prefix + '*/res_pred_comp_list*' for fname in glob(allsubd, recursive=True): with open(fname, 'rb') as f: pred_comp_tmp = pickle_load(f) for snode in pred_comp_tmp: sdndap(snode) else: for i in range(int(args.n_pts)): with open(out_comp_nm_sample + "_pred_comp_list" + str(i), 'rb') as f: pred_comp_tmp = pickle_load(f) for snode in pred_comp_tmp: sdndap(snode) len_pred_comp_list = 'No. of complexes before pp = ' + str( len(pred_comp_list)) logging_info(len_pred_comp_list) test_complex_path = inputs['dir_nm'] + inputs[ 'train_test_files_dir'] + "/res_test_known_complex_nodes_list" test_prot_list = get_prot_list(test_complex_path) train_complex_path = inputs['dir_nm'] + inputs[ 'train_test_files_dir'] + "/res_train_known_complex_nodes_list" train_prot_list = get_prot_list(train_complex_path) protlistfname = inputs['dir_nm'] + inputs[ 'train_test_files_dir'] + "/res_protlist" with open(protlistfname, 'rb') as f: prot_list = pickle_load(f) start_time_pp = time_time() fin_list_graphs = postprocess(pred_comp_list, modelfname, scaler, inputs, myGraph, prot_list, train_prot_list, test_prot_list) pp_time = time_time() - start_time_pp tot_time = time_time() - start_time # Write to yaml file instead with open(out_comp_nm + '_runtime_performance.out', "a") as fid: print("--- Runtime performance ---", file=fid) print("Post processing complex time (s) = ", pp_time, "[", round(100 * float(pp_time) / tot_time, 2), "%]", file=fid) print("Total time (s) = ", tot_time, file=fid)
# # Copyright (C) 2011 Brendon J. Brewer # This file is part of DNest, the Diffusive Nested Sampler. # # DNest is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # DNest is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with DNest. If not, see <http://www.gnu.org/licenses/>. import postprocess postprocess.postprocess(temperature=10.) import display
def hanso(func, x0=None, grad=None, nvar=None, nstart=None, sampgrad=False, funcrtol=1e-20, gradnormtol=1e-6, verbose=2, fvalquit=-np.inf, cpumax=np.inf, maxit=100, callback=None, **kwargs): """ HANSO: Hybrid Algorithm for Nonsmooth Optimization The algorithm is two-fold. Viz, BFGS phase: BFGS is run from multiple starting points, taken from the columns of x0 parameter, if provided, and otherwise 10 points generated randomly. If the termination test was satisfied at the best point found by BFGS, or if nvar > 100, HANSO terminates; otherwise, it continues to: Gradient sampling phases: 3 gradient sampling phases are run from lowest point found, using sampling radii: 10*evaldist, evaldist, evaldist/10 Termination takes place immediately during any phase if cpumax CPU time is exceeded. References ---------- A.S. Lewis and M.L. Overton, Nonsmooth Optimization via Quasi-Newton Methods, Math Programming, 2012 J.V. Burke, A.S. Lewis and M.L. Overton, A Robust Gradient Sampling Algorithm for Nonsmooth, Nonconvex Optimization SIAM J. Optimization 15 (2005), pp. 751-779 Parameters ---------- func: callable function on 1D arrays of length nvar function being optimized grad: callable function gradient of func fvalquit: float, optional (default -inf) param passed to bfgs1run function gradnormtol: float, optional (default 1e-4) termination tolerance for smallest vector in convex hull of saved gradients verbose: int, optional (default 1) param passed to bfgs1run function cpumax: float, optional (default inf) quit if cpu time in secs exceeds this (applies to total running time) sampgrad: boolean, optional (default False) if set, the gradient-sampling will be used to continue the algorithm in case the BFGS fails **kwargs: param-value dict optional parameters passed to bfgs backend. Possible key/values are: x0: 2D array of shape (nvar, nstart), optional (default None) intial points, one per column nvar: int, optional (default None) number of dimensions in the problem (exclusive x0) nstart: int, optional (default None) number of starting points for BFGS algorithm (exclusive x0) maxit: int, optional (default 100) param passed to bfgs1run function wolfe1: float, optional (default 0) param passed to bfgs1run function wolfe2: float, optional (default .5) param passed to bfgs1run function Returns ------- x: D array of same length nvar = len(x0) final iterate f: list of nstart floats final function values, one per run of bfgs1run d: list of nstart 1D arrays, each of same length as input nvar final smallest vectors in convex hull of saved gradients, one array per run of bfgs1run H: list of nstarts 2D arrays, each of shape (nvar, nvar) final inverse Hessian approximations, one array per run of bfgs1run itrecs: list of nstart int numbers of iterations, one per run of bfgs1run; see bfgs1run for details inforecs: list of int reason for termination; see bfgs1run for details pobj: list of tuples of the form (duration of iteration, final func value) trajectory for best starting point (i.e of the starting point that led to the greatest overall decrease in the cost function. Note that the O(1) time consumed by the gradient-sampling stage is not counted. Optional Outputs (in case output_records is True): Xrecs: list of nstart 2D arrays, each of shape (iter, nvar) iterates where saved gradients were evaluated; one array per run of bfgs1run; see bfgs1run for details Grecs: ist of nstart 2D arrays, each of shape (nvar, nvar) gradients evaluated at these points, one per run of bfgs1run; see bfgs1run for details wrecs: list of nstart 1D arrays, each of length iter weights defining convex combinations d = G*w; one array per run of bfgs1run; see bfgs1run for details fevalrecs: list of nstart 1D arrays, each of length iter records of all function evaluations in the line searches; one array per run of bfgs1run; see bfgs1run for details xrecs: list of nstart 2D arrays, each of length (iter, nvar) record of x iterates Hrecs: list of nstart 2D arrays, each of shape (iter, nvar) record of H (Hessian) iterates; one array per run of bfgs1run; see bfgs1run for details Raises ------ RuntimeError """ def _log(msg, level=0): if verbose > level: print msg # sanitize x0 if x0 is None: assert not nvar is None, ( "No value specified for x0, expecting a value for nvar") assert not nstart is None, ( "No value specified for x0, expecting a value for nstart") x0 = setx0(nvar, nstart) else: assert nvar is None, ( "Value specified for x0, expecting no value for nvar") assert nstart is None, ( "Value specified for x0, expecting no value for nstart") x0 = np.array(x0) if x0.ndim == 1: x0 = x0.reshape((-1, 1)) nvar, nstart = x0.shape cpufinish = time.time() + cpumax # run BFGS step kwargs['output_records'] = 1 x, f, d, H, _, info, X, G, w, pobj = bfgs( func, x0=x0, grad=grad, fvalquit=fvalquit, funcrtol=funcrtol, gradnormtol=gradnormtol, cpumax=cpumax, maxit=maxit, verbose=verbose, callback=callback, **kwargs) # throw away all but the best result assert len(f) == np.array(x).shape[1], np.array(x).shape indx = np.argmin(f) f = f[indx] x = x[..., indx] d = d[..., indx] H = H[indx] # bug if do this when only one start point: H already matrix X = X[indx] G = G[indx] w = w[indx] pobj = pobj[indx] dnorm = linalg.norm(d, 2) # the 2nd argument will not be used since x == X(:,1) after bfgs loc, X, G, w = postprocess(x, np.nan, dnorm, X, G, w, verbose=verbose) if np.isnan(f) or np.isinf(f): _log('hanso: f is infinite or nan at all starting points') return x, f, loc, X, G, w, H, pobj if time.time() > cpufinish: _log('hanso: cpu time limit exceeded') _log('hanso: best point found has f = %g with local optimality ' 'measure: dnorm = %5.1e, evaldist = %5.1e' % ( f, loc['dnorm'], loc['evaldist'])) return x, f, loc, X, G, w, H, pobj if f < fvalquit: _log('hanso: reached target objective') _log('hanso: best point found has f = %g with local optimality' ' measure: dnorm = %5.1e, evaldist = %5.1e' % ( f, loc['dnorm'], loc['evaldist'])) return x, f, loc, X, G, w, H, pobj if dnorm < gradnormtol: _log('hanso: verified optimality within tolerance in bfgs phase') _log('hanso: best point found has f = %g with local optimality ' 'measure: dnorm = %5.1e, evaldist = %5.1e' % ( f, loc['dnorm'], loc['evaldist'])) return x, f, loc, X, G, w, H, pobj if sampgrad: # launch gradient sampling # time0 = time.time() f_BFGS = f # save optimality certificate info in case gradient sampling cannot # improve the one provided by BFGS dnorm_BFGS = dnorm loc_BFGS = loc d_BFGS = d X_BFGS = X G_BFGS = G w_BFGS = w x0 = x.reshape((-1, 1)) # otherwise gradient sampling is too expensivea if maxit > 100: maxit = 100 # # otherwise grad sampling will augment with random starts # x0 = x0[..., :1] # assert 0, x0.shape cpumax = cpufinish - time.time() # time left # run gradsamp proper x, f, g, dnorm, X, G, w = gradsamp(func, x0, grad=grad, maxit=maxit, cpumax=cpumax) if f == f_BFGS: # gradient sampling did not reduce f _log('hanso: gradient sampling did not reduce f below best point' ' found by BFGS\n') # use the better optimality certificate if dnorm > dnorm_BFGS: loc = loc_BFGS d = d_BFGS X = X_BFGS G = G_BFGS w = w_BFGS elif f < f_BFGS: loc, X, G, w = postprocess(x, g, dnorm, X, G, w, verbose=verbose) _log('hanso: gradient sampling reduced f below best point found' ' by BFGS\n') else: raise RuntimeError( 'hanso: f > f_BFGS: this should never happen' ) # this should never happen x = x[0] f = f[0] if callback: callback(x) return x, f, loc, X, G, w, H, pobj else: if callback: callback(x) return x, f, loc, X, G, w, H, pobj
from pylab import * import postprocess from postprocess import logsumexp # Load the levels from the DNest run levels = loadtxt('levels.txt') sample_info = loadtxt('sample_info.txt') # Get logx of samples from the DNest run result = postprocess.postprocess() # Relationship between logl and logx based on the DNest run logx = hstack([levels[:,0], result[2].flatten()]) logl = vstack([levels[:,1:3], sample_info[:,1:3]]) order = argsort(logx)[::-1] logx = logx[order] logl = logl[order, :] # Now treat logx, logl as "levels" and use them as scaffolding to locate # the points from the directional runs levels = empty((logx.size, levels.shape[1])) levels[:, 0] = logx levels[:, 1:3] = logl scalars = loadtxt('scalars.txt') sample = loadtxt('output.txt') sample_info = empty((scalars.shape[0], sample_info.shape[1])) sample_info[:,1:3] = scalars[:, 0:2] # log prior weights result = postprocess.postprocess(loaded=[levels, sample_info, sample],\ temperature=1E300, plot=False)
def main(): img = cv2.imread(sys.argv[1]) gray= cv2.cvtColor(img,cv2.COLOR_BGR2GRAY) p = postprocess()
def hanso(func, x0=None, grad=None, nvar=None, nstart=None, sampgrad=False, funcrtol=1e-20, gradnormtol=1e-6, verbose=2, fvalquit=-np.inf, cpumax=np.inf, maxit=100, **kwargs): """ HANSO: Hybrid Algorithm for Nonsmooth Optimization The algorithm is two-fold. Viz, BFGS phase: BFGS is run from multiple starting points, taken from the columns of x0 parameter, if provided, and otherwise 10 points generated randomly. If the termination test was satisfied at the best point found by BFGS, or if nvar > 100, HANSO terminates; otherwise, it continues to: Gradient sampling phases: 3 gradient sampling phases are run from lowest point found, using sampling radii: 10*evaldist, evaldist, evaldist/10 Termination takes place immediately during any phase if cpumax CPU time is exceeded. References ---------- A.S. Lewis and M.L. Overton, Nonsmooth Optimization via Quasi-Newton Methods, Math Programming, 2012 J.V. Burke, A.S. Lewis and M.L. Overton, A Robust Gradient Sampling Algorithm for Nonsmooth, Nonconvex Optimization SIAM J. Optimization 15 (2005), pp. 751-779 Parameters ---------- func: callable function on 1D arrays of length nvar function being optimized grad: callable function gradient of func fvalquit: float, optional (default -inf) param passed to bfgs1run function gradnormtol: float, optional (default 1e-4) termination tolerance for smallest vector in convex hull of saved gradients verbose: int, optional (default 1) param passed to bfgs1run function cpumax: float, optional (default inf) quit if cpu time in secs exceeds this (applies to total running time) sampgrad: boolean, optional (default False) if set, the gradient-sampling will be used to continue the algorithm in case the BFGS fails **kwargs: param-value dict optional parameters passed to bfgs backend. Possible key/values are: x0: 2D array of shape (nvar, nstart), optional (default None) intial points, one per column nvar: int, optional (default None) number of dimensions in the problem (exclusive x0) nstart: int, optional (default None) number of starting points for BFGS algorithm (exclusive x0) maxit: int, optional (default 100) param passed to bfgs1run function wolfe1: float, optional (default 0) param passed to bfgs1run function wolfe2: float, optional (default .5) param passed to bfgs1run function Returns ------- x: D array of same length nvar = len(x0) final iterate f: list of nstart floats final function values, one per run of bfgs1run d: list of nstart 1D arrays, each of same length as input nvar final smallest vectors in convex hull of saved gradients, one array per run of bfgs1run H: list of nstarts 2D arrays, each of shape (nvar, nvar) final inverse Hessian approximations, one array per run of bfgs1run itrecs: list of nstart int numbers of iterations, one per run of bfgs1run; see bfgs1run for details inforecs: list of int reason for termination; see bfgs1run for details pobj: list of tuples of the form (duration of iteration, final func value) trajectory for best starting point (i.e of the starting point that led to the greatest overall decrease in the cost function. Note that the O(1) time consumed by the gradient-sampling stage is not counted. Optional Outputs (in case output_records is True): Xrecs: list of nstart 2D arrays, each of shape (iter, nvar) iterates where saved gradients were evaluated; one array per run of bfgs1run; see bfgs1run for details Grecs: ist of nstart 2D arrays, each of shape (nvar, nvar) gradients evaluated at these points, one per run of bfgs1run; see bfgs1run for details wrecs: list of nstart 1D arrays, each of length iter weights defining convex combinations d = G*w; one array per run of bfgs1run; see bfgs1run for details fevalrecs: list of nstart 1D arrays, each of length iter records of all function evaluations in the line searches; one array per run of bfgs1run; see bfgs1run for details xrecs: list of nstart 2D arrays, each of length (iter, nvar) record of x iterates Hrecs: list of nstart 2D arrays, each of shape (iter, nvar) record of H (Hessian) iterates; one array per run of bfgs1run; see bfgs1run for details Raises ------ RuntimeError """ def _log(msg, level=0): if verbose > level: print msg # sanitize x0 if x0 is None: assert not nvar is None, ( "No value specified for x0, expecting a value for nvar") assert not nstart is None, ( "No value specified for x0, expecting a value for nstart") x0 = setx0(nvar, nstart) else: assert nvar is None, ( "Value specified for x0, expecting no value for nvar") assert nstart is None, ( "Value specified for x0, expecting no value for nstart") x0 = np.array(x0) if x0.ndim == 1: x0 = x0.reshape((-1, 1)) nvar, nstart = x0.shape cpufinish = time.time() + cpumax # run BFGS step kwargs['output_records'] = 1 x, f, d, H, _, info, X, G, w, pobj = bfgs(func, x0=x0, grad=grad, fvalquit=fvalquit, funcrtol=funcrtol, gradnormtol=gradnormtol, cpumax=cpumax, maxit=maxit, verbose=verbose, **kwargs) # throw away all but the best result assert len(f) == np.array(x).shape[1], np.array(x).shape indx = np.argmin(f) f = f[indx] x = x[..., indx] d = d[..., indx] H = H[indx] # bug if do this when only one start point: H already matrix X = X[indx] G = G[indx] w = w[indx] pobj = pobj[indx] dnorm = linalg.norm(d, 2) # the 2nd argument will not be used since x == X(:,1) after bfgs loc, X, G, w = postprocess(x, np.nan, dnorm, X, G, w, verbose=verbose) if np.isnan(f) or np.isinf(f): _log('hanso: f is infinite or nan at all starting points') return x, f, loc, X, G, w, H, pobj if time.time() > cpufinish: _log('hanso: cpu time limit exceeded') _log('hanso: best point found has f = %g with local optimality ' 'measure: dnorm = %5.1e, evaldist = %5.1e' % (f, loc['dnorm'], loc['evaldist'])) return x, f, loc, X, G, w, H, pobj if f < fvalquit: _log('hanso: reached target objective') _log('hanso: best point found has f = %g with local optimality' ' measure: dnorm = %5.1e, evaldist = %5.1e' % (f, loc['dnorm'], loc['evaldist'])) return x, f, loc, X, G, w, H, pobj if dnorm < gradnormtol: _log('hanso: verified optimality within tolerance in bfgs phase') _log('hanso: best point found has f = %g with local optimality ' 'measure: dnorm = %5.1e, evaldist = %5.1e' % (f, loc['dnorm'], loc['evaldist'])) return x, f, loc, X, G, w, H, pobj if sampgrad: # launch gradient sampling # time0 = time.time() f_BFGS = f # save optimality certificate info in case gradient sampling cannot # improve the one provided by BFGS dnorm_BFGS = dnorm loc_BFGS = loc d_BFGS = d X_BFGS = X G_BFGS = G w_BFGS = w x0 = x.reshape((-1, 1)) # otherwise gradient sampling is too expensivea if maxit > 100: maxit = 100 # # otherwise grad sampling will augment with random starts # x0 = x0[..., :1] # assert 0, x0.shape cpumax = cpufinish - time.time() # time left # run gradsamp proper x, f, g, dnorm, X, G, w = gradsamp(func, x0, grad=grad, maxit=maxit, cpumax=cpumax) if f == f_BFGS: # gradient sampling did not reduce f _log('hanso: gradient sampling did not reduce f below best point' ' found by BFGS\n') # use the better optimality certificate if dnorm > dnorm_BFGS: loc = loc_BFGS d = d_BFGS X = X_BFGS G = G_BFGS w = w_BFGS elif f < f_BFGS: loc, X, G, w = postprocess(x, g, dnorm, X, G, w, verbose=verbose) _log('hanso: gradient sampling reduced f below best point found' ' by BFGS\n') else: raise RuntimeError('hanso: f > f_BFGS: this should never happen' ) # this should never happen x = x[0] f = f[0] # pobj.append((time.time() - time0, f)) return x, f, loc, X, G, w, H, pobj else: return x, f, loc, X, G, w, H, pobj
# Copyright (c) 2009, 2010, 2011, 2012 Brendon J. Brewer. # # This file is part of DNest3. # # DNest3 is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # DNest3 is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with DNest3. If not, see <http://www.gnu.org/licenses/>. import postprocess postprocess.postprocess(cut=0.5) import display
def interactive_plot(): # import data from post-processing script run_data = postprocess() # setup accuracies = [] losses = [] test_accuracies = [] step = 0 acc = 0 loss = 0 handle = 0 ####################################################### # GRAPHING VALIDATION ACCURACY AS A FUNCTION OF EPOCH # ####################################################### fig, ax = plt.subplots() ax.set_title('Validation Accuracy vs Epoch') for run_num in run_data: # print "processing run", run_num step = np.asarray(run_data[run_num]['step']) acc = 100 * np.asarray(run_data[run_num]['validation_accuracy']) loss = np.asarray(run_data[run_num]['loss']) handle, = ax.plot(step, acc, lw=1.0, label=str(run_num)) accuracies.append(handle) test_accuracies.append(run_data[run_num]['test_accuracy']) # shrink x axis by 20% so we can fit our huge legend box = ax.get_position() ax.set_position([box.x0, box.y0, box.width * 0.9, box.height]) # create legend, put to right of plot area leg = ax.legend(fancybox=True, shadow=True, ncol=3, loc='upper right', bbox_to_anchor=(1.2, 1.0)) leg.get_frame().set_alpha(0.4) # I have no idea what this does, but I think it links legend entries to the data lined = dict() for legline, origline in zip(leg.get_lines(), accuracies): legline.set_picker(5) # 5 pts tolerance # origline.set_visible(False) # legline.set_alpha(0.2) lined[legline] = origline # legend click callback def onpick(event): legline = event.artist origline = lined[legline] vis = not origline.get_visible() origline.set_visible(vis) # change the alpha on the line in the legend so we can see what lines have been toggled if vis: legline.set_alpha(1.0) else: legline.set_alpha(0.2) fig.canvas.draw() fig.canvas.mpl_connect('pick_event', onpick) ax.set_xlim([0, 500]) ax.set_ylim([80, 100]) plt.xlabel('Epoch Number') plt.ylabel('Validation Accuracy [%]') plt.show() ######################################## # GRAPHING LOSS AS A FUNCTION OF EPOCH # ######################################## fig, ax = plt.subplots() ax.set_title('Softmax Loss vs Epoch') for run_num in run_data: # print "processing run", run_num step = np.asarray(run_data[run_num]['step']) loss = np.asarray(run_data[run_num]['loss']) handle, = ax.plot(step, loss, lw=1.0, label=str(run_num)) # losses[run_num] = handle losses.append(handle) # shrink x axis by 20% so we can fit our huge legend box = ax.get_position() ax.set_position([box.x0, box.y0, box.width * 0.9, box.height]) # create legend, put to right of plot area leg = ax.legend(fancybox=True, shadow=True, ncol=3, loc='upper right', bbox_to_anchor=(1.2, 1.0)) leg.get_frame().set_alpha(0.4) # I have no idea what this does, but I think it links legend entries to the data lined = dict() for legline, origline in zip(leg.get_lines(), losses): legline.set_picker(5) # 5 pts tolerance # origline.set_visible(False) # legline.set_alpha(0.2) lined[legline] = origline # legend click callback def onpick2(event): legline = event.artist origline = lined[legline] vis = not origline.get_visible() origline.set_visible(vis) # change the alpha on the line in the legend so we can see what lines have been toggled if vis: legline.set_alpha(1.0) else: legline.set_alpha(0.2) fig.canvas.draw() fig.canvas.mpl_connect('pick_event', onpick2) ax.set_xlim([0, 500]) ax.set_ylim([0.0, 0.5]) plt.xlabel('Epoch Number') plt.ylabel('Softmax Loss') plt.show() ####################################################### # GRAPHING FINAL TEST ACCURACY AS A FUNCTION OF EPOCH # ####################################################### plt.plot(test_accuracies) # ax.set_xlim([0, 500]) ax.set_ylim([0.0, 0.5]) plt.xlabel('Epoch Number') plt.ylabel('Softmax Loss') plt.show() plt.show()
# Copyright (c) 2009, 2010, 2011, 2012 Brendon J. Brewer. # # This file is part of DNest3. # # DNest3 is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # DNest3 is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with DNest3. If not, see <http://www.gnu.org/licenses/>. # Import postprocess from two directories up # http://stackoverflow.com/a/9806045 import os parentdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) parentdir = os.path.dirname(parentdir) os.sys.path.insert(0, parentdir) import postprocess postprocess.postprocess(temperature=1) import display
def print_help(): print('''Usage: {0}: [OPTION] scan|text|learn file [file...]\ \nExtract string from image file \nvalid argument may be:\ \n -h, --help: print this help message\ \n -d, --dir: dataset directory\ \n scan: scan for valid text in file'''.format(sys.argv[0])) sys.exit(0) if __name__ == "__main__": optlist, args = getopt.getopt(sys.argv[1:], "hd:", ["help", "dir"]) direc = "dataset" p = ps.postprocess() for o, a in optlist: if o in ("-h", "--help"): print_help() if o in ("-d", "--dir"): direc = a if len(args) >= 2 and args[0] in ("s", "scan"): knn = learnLetter() for filename in args[1:]: scan(knn, filename, p) elif len(args) >= 2 and args[0] in ("t", "text"): print(direc) knn = learnLetter(directory=direc) for filename in args[1:]: scantext(knn, filename, p) elif len(args) >= 1 and args[0] in ("l", "learn"):
def main(): st.title("picTranslate") '''Hii there! We are happy to help you. ''' '''picTranslate is an AI based app that enables you to intuitively translate the language of the text on your images, so that you can understand your picture completely.''' st.subheader('Instructions') '''1. Just select the image you want to change the text of.''' '''2. Select the source language i.e. the current language on the image.''' '''3. Select the target language i.e. the language you want the translated text to be in and Press Start.''' '''That's it!!''' st.header('Upload the required image') uploaded_file = st.file_uploader("") if uploaded_file is not None: image = Image.open(uploaded_file) image.save('Uploadedimg.jpg') st.image(image, caption='Uploaded Image.', use_column_width=True) st.success('Uploaded Successfully!!') sl = st.text_input("Source Language", 'en') tl = st.text_input("Target Language", 'en') if st.button('More Target Languages'): webbrowser.open_new_tab(url) region = st.text_input("AWS Region name", 'us-east-2') url = 'https://docs.aws.amazon.com/translate/latest/dg/what-is.html' if st.button('Start Translation'): path = './Uploadedimg.jpg' st.write("(1/4) Generating Mask...") mask_generator(path, region_name=region) st.write( "(2/4) Inpainting the image (This might take sometime if running on CPU)..." ) inpainter() postprocess() bucket = '' document = '' st.write('(3/4) Detecting text...') block_count, text, coord = text_extractor(bucket, document, path, region_name=region) #Getting the translated text i.e. ttext ttext = translator(text, sl, tl, region_name=region) #Opening the inpainted image image = cv2.imread('./results/image.jpg') #Writing the text on inpainted image st.write("(4/4) Just there...") written_img = writer(image, ttext, coord) #Output Image cv2.imwrite('./Output/Output.jpg', written_img) st.success('Finished!!') image2 = Image.open('./Output/Output.jpg') st.image(image2, caption='Output image', use_column_width=True) st.warning('Very rich backgrounds might create problems!!') st.markdown( "<h1 style='text-align: center; color: black; font-size:16px;'>Developed by Vaibhav Tiwari</h1>", unsafe_allow_html=True)
# Copyright (c) 2009, 2010, 2011, 2012 Brendon J. Brewer. # # This file is part of DNest3. # # DNest3 is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # DNest3 is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with DNest3. If not, see <http://www.gnu.org/licenses/>. # Import postprocess from two directories up # http://stackoverflow.com/a/9806045 import os #parentdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) #parentdir = os.path.dirname(parentdir) #os.sys.path.insert(0, parentdir) import postprocess postprocess.postprocess(plot=False)
# [micro m] SL1 = SL0 * .97 # [micro m] SLt0 = .05 # [s] SLt1 = .35 # [s] SLtau0 = .05 # [s] SLtau1 = .02 # [s] SL_base = lambda t: SL0 + (SL1 - SL0) * (np.clip( 1 - np.exp((SLt0 - t) / SLtau0), 0, None) - np.clip( 1 - np.exp((SLt1 - t) / SLtau1), 0, None)) #%% Input definition inputs = {} inputs['times'] = np.arange(0, Tmax, 1e-4) inputs['Ca'] = Ca_base(inputs['times']) inputs['SL'] = SL_base(inputs['times']) #%% Simulation output = model.solve(inputs) #%% Saving results to csv file pandas.DataFrame(output).to_csv('output_' + model.model_name + '.csv', index=False) #%% Postprocessing postprocess(output, 'output_' + model.model_name + '.pdf')
print e.content except TA_error.Preprocess_Error, e: # preprocess has some problems n_tests_fail += 1 print_error(self.test_name, t_elapsed, "PREPROCESS") print e.content except TA_error.Process_Error, e: # process has some problems n_tests_fail += 1 print_error(self.test_name, t_elapsed, "PROCESS") print e.content finally: try: if test_pass == True: print_pass(self.test_name, t_elapsed) postprocess.postprocess(self.parser) except TA_error.Postprocess_Error, e: # print "postprocess error" print_error(self.test_name, t_elapsed, "POSTPROCESS") print e.content def main(self): self._parse_test_cfg() # parse test cfg file if self.macro == "open": # print "open" self.macro_ele_list = extract_macro(self.parser) macro.new_macro(self.macro_ele_list, self) else: """ no macro, only run once """
def _process(video): """ This route will perform 3 steps: 1. split video into frames directory and audio file 2. add frames into service bus queue 3. this function will poll storage until the number of input images matches the number of processed images 4. download processed frames and stitch video back together """ # get varaibles from environment namespace = os.getenv("SB_NAMESPACE") queue = os.getenv("SB_QUEUE") sb_key_name = os.getenv("SB_SHARED_ACCESS_KEY_NAME") sb_key_value = os.getenv("SB_SHARED_ACCESS_KEY_VALUE") mount_dir = os.getenv("MOUNT_DIR", "data") terminate = os.getenv("TERMINATE") # start time t0 = time.time() # set video name video_name = video.split(".")[0] input_dir = os.path.join(mount_dir, video_name, "input_frames") output_dir = os.path.join(mount_dir, video_name, "output_frames") audio_file = os.path.join(mount_dir, video_name, "audio.aac") # create parent directory in mount_dir if not os.path.exists(os.path.join(mount_dir, video_name)): os.makedirs(os.path.join(mount_dir, video_name)) # setup logger handler_format = get_handler_format() console_handler = logging.StreamHandler(sys.stdout) console_handler.setFormatter(handler_format) log_file = "{}.log".format(video_name) file_handler = RotatingFileHandler(os.path.join(mount_dir, video_name, log_file), maxBytes=20000) file_handler.setFormatter(handler_format) logger = logging.getLogger("root") logger.setLevel(logging.DEBUG) logger.addHandler(console_handler) logger.addHandler(file_handler) logger.propagate = False # process video and upload output frames and audio file to blob logger.debug("Preprocessing video {}".format(video)) preprocess(video=video, mount_dir=mount_dir) t1 = time.time() # service bus client bus_service = ServiceBusService( service_namespace=namespace, shared_access_key_name=sb_key_name, shared_access_key_value=sb_key_value, ) # add all images from frame_dir to the queue logger.debug("Adding images from {} to queue {}".format(input_dir, queue)) image_count = add_images_to_queue( mount_dir=mount_dir, queue=queue, video_name=video_name, bus_service=bus_service, ) t2 = time.time() # terminate if testing if terminate: exit(0) # poll storage for output logger.debug( "Polling for input images {} to equal output images {}".format( input_dir, output_dir)) while True: path, dirs, files = next(os.walk(output_dir)) output_frames_length = len(files) if output_frames_length == image_count: t3 = time.time() # postprocess video logger.debug( "Stitching video together with processed frames dir '{}' and audio file '{}'." .format(output_dir, audio_file)) postprocess(video_name=video_name, mount_dir=mount_dir) t4 = time.time() break else: logger.debug( "Images are still processing. Retrying in 10 seconds...") time.sleep(10) continue t5 = time.time() logger.debug( "Preprocessing video finished.... Time taken in seconds: {:.2f}". format(t1 - t0)) logger.debug( "Adding image to queue finished.. Time taken in seconds: {:.2f}". format(t2 - t1)) logger.debug( "Detecting vehicles.............. Time taken in seconds: {:.2f}". format(t3 - t2)) logger.debug( "Postprocessing video finished... Time taken in seconds: {:.2f}". format(t4 - t3)) logger.debug( "Total process................... Time taken in seconds: {:.2f}". format(t5 - t0))
loadName = loadName[ 0:: 2] # get the elements in the even position. start at the first element (0) and skip 1 each time # print(loadName) # debug # CREATE HEADERS IN RESULTS TABLE column_names = ["specimen", "timePL", "cyclesPL", "timeYP", "cyclesYP"] results = pd.DataFrame(columns=column_names) # CALL POSTPROCESS FUNCTION AND FILL IN RESULTS TABLE for i in range(len(names)): # range creates iterable object index = loadName.index( names[i].split(" ")[0] ) # find the name that is being evaluated and return target load from load files output = pp.postprocess(os.path.join(entry_file_path, names[i] + '.csv'), os.path.join(script_dir, 'temporary.csv'), float(load[index].split(" ")[1]), 0.01, 0.05, 5) #0.01 and 0.05 for all files print('aqui', output) preLoadData = list(map( float, output[0].split(",")))[0:2] # get first and second value yieldPointData = [] for j in output[1].split(","): if j == '': yieldPointData.append('not found') yieldPointData.append('not found') else: yieldPointData.append(float(j)) yieldPointData = yieldPointData[0:2] row = [names[i].split(" ")[0]] # extract name until the first blank space row.extend(preLoadData) # extend te alarga la lista row.extend(yieldPointData)
def diffusive_integrator(sampler, tolerance=0.01, maxEvaluations=0, minEvaluations=0): # sample from sampler samples = [] # coordinates samplesInfo = [] # logX, likelihood, (tieBreaker), livePointID finfo = open('sample_info.txt', 'w') finfo.write("# index, logLikelihood, tieBreaker, ID.\n") #numpy.savetxt(f, self.samplesInfo, fmt='%d %e %f %d') fsample = open('sample.txt', 'w') fsample.write("# Samples file. One sample per line.\n") #numpy.savetxt(f, self.samples) logZs = [] i = 1 nlevelcomplete = None while True: # keep sampling sample, sampleInfo = next(sampler) #print 'new sample:', sample, sampleInfo # each sample file contains one line per live point / particle # sampleFile :: self.live_pointsu # sampleInfoFile :: self.indices, self.live_pointsL, (tieBreaker), ID? # levelFile :: logX, cutoff, (tieBreaker), accepts, tries, exceeds, visits # adding an extra number because postprocess can't deal with 1d data fsample.write(' '.join(['%e' % s for s in sample]) + "\n") fsample.flush() finfo.write("%d %e %f %d\n" % sampleInfo) finfo.flush() samples.append(sample) samplesInfo.append(sampleInfo) levels = numpy.array([[ l.logX, l.cutoff, numpy.random.uniform(), l.accepts, l.tries, l.exceeds, l.visits ] for l in sampler.levels]) flevels = open('levels.txt', 'w') flevels.write( "# logX, logLikelihood, tieBreaker, accepts, tries, exceeds, visits.\n" ) numpy.savetxt(flevels, levels, fmt='%f %e %f %d %d %d %d') flevels.close() if i % 20 == 0: # check if tolerance achieved already logZ, H, weights, logZerr, ESS = postprocess.postprocess( loaded=(levels, numpy.asarray(samplesInfo), numpy.asarray(samples)), save=False, plot=False, verbose=False, numResampleLogX=10) logZs.append(logZ) print( 'logZ = %.3f +- %.3f +- %.3f | %.1f samples | iteration %d[warmup:%s,neval:%d]' % (logZ, logZerr, numpy.std( logZs[-10:]), ESS, i, nlevelcomplete, sampler.neval)) if nlevelcomplete is None and ESS > 60 and sampler.maxLevels <= 0: sampler.maxLevels = len(sampler.levels) if nlevelcomplete is None and sampler.levelsComplete(): nlevelcomplete = i print('# levels sufficient after %d iterations' % nlevelcomplete) logZerr_total = (logZerr**2 + numpy.var(logZs[-10:]))**0.5 converged = nlevelcomplete is not None and logZerr_total < tolerance and numpy.std( logZs[-10:]) * 10 < tolerance if (minEvaluations is not None and sampler.neval > minEvaluations ) and (converged or (maxEvaluations > 0 and sampler.neval >= maxEvaluations)): # do some plotting postprocess.postprocess(loaded=(levels, numpy.asarray(samplesInfo), numpy.asarray(samples)), save=True, plot=True, numResampleLogX=1) break i = i + 1 return dict(logZ=logZ, logZerr=logZerr, samples=samples, points=[(p, numpy.median(w), si[0]) for p, si, w in zip(samples, samplesInfo, weights)], information=H)