def main(): data_path_dir = 'data' if not os.path.exists(data_path_dir): os.makedirs(data_path_dir) data_path = 'data.json' try: os.remove(data_path) except: pass subprocess.call([ 'scrapy', 'runspider', 'scrapy/crawler/spiders/quotes_spider.py', '-o', data_path ]) post_process(data_path) with open(data_path, 'r') as fn: data = json.loads(fn.read()) new_data = [] for item in data: download(item['url']) new_item = { 'content': item['content'], 'title': item['title'], 'thumbnail': item['thumbnail'], 'audio': os.path.join(data_path_dir, glob.glob('*.mp3')[0]), 'type': 'audio' } shutil.move(glob.glob('*.mp3')[0], data_path_dir) new_data.append(new_item) os.remove(data_path) with open(data_path, 'w') as fn: json.dump(new_data, fn)
def publish(item, board, helper, channel, web): item_type = helper.item_type(item) post_process(item, board, helper, web) while True: try: channel.basic_publish(exchange='chan', routing_key="%s.%s.%s" % (chan, item_type, board), body=json.dumps(item, separators=(',', ':'), ensure_ascii=False, sort_keys=True)) if MONITORING: distance = datetime.utcnow() - datetime.utcfromtimestamp( helper.item_mtime(item)) monitoring.log([{ "measurement": chan, "time": str(datetime.utcnow()), "tags": { "board": board }, "fields": { "distance": distance.total_seconds() } }]) break except Exception as e: logger.debug(traceback.format_exc()) logger.error(str(e)) channel = connect()
def predict(input_path, output_path, start_extract, end_extract): tmp_dir = 'tmp_files/' tmp_features = 'tmp.features' tmp_prob = 'tmp.prob' tmp_prediction = 'tmp.prediction' if not os.path.exists(input_path): print >>sys.stderr, "wav file does not exits" return length = utils.get_wav_file_length(input_path) feature_file = tmp_dir+tmp_features prob_file = tmp_dir+tmp_prob predict_file = tmp_dir+tmp_prediction # remove tmo dir if exists if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir) os.mkdir(tmp_dir) print '\n1) Extracting features and classifying ...' extract_features(input_path, feature_file, start_extract, end_extract) run(feature_file, prob_file) print '\n3) Extract Durations ...' post_process(prob_file, predict_file) print '\n4) Writing TextGrid file to %s ...' % output_path create_text_grid(predict_file, output_path, length, float(start_extract))
def predict(input_path, output_path, start_extract, end_extract): tmp_dir = 'tmp_files/' tmp_features = 'tmp.features' tmp_prob = 'tmp.prob' tmp_prediction = 'tmp.prediction' if not os.path.exists(input_path): print >> sys.stderr, "wav file does not exits" return length = utils.get_wav_file_length(input_path) feature_file = tmp_dir + tmp_features prob_file = tmp_dir + tmp_prob predict_file = tmp_dir + tmp_prediction # remove tmo dir if exists if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir) os.mkdir(tmp_dir) print '\n1) Extracting features and classifying ...' extract_features(input_path, feature_file, start_extract, end_extract) run(feature_file, prob_file) print '\n3) Extract Durations ...' post_process(prob_file, predict_file) print '\n4) Writing TextGrid file to %s ...' % output_path create_text_grid(predict_file, output_path, length, float(start_extract))
def main(): #inputs = area, sweep, thickness, thrust inputs = np.array([92.00,22.0 * Units.deg, 0.11, 20300.0]) # Have the optimizer call the wrapper mywrap = lambda inputs:wrap(inputs) opt_prob = pyOpt.Optimization('Embraer',mywrap) opt_prob.addObj('Fuel') opt_prob.addVar('Wing Area','c',lower=70,upper=110,value=inputs[0]) opt_prob.addVar('Wing Sweep','c',lower=15,upper=35,value=inputs[1]) opt_prob.addVar('Wing thickness','c',lower=0.10,upper=0.15,value=inputs[2]) opt_prob.addVar('Engine Thrust','c',lower=18000,upper=25000,value=inputs[3]) opt_prob.addConGroup('g',4,'i') opt = pyOpt.pySLSQP.SLSQP() post_process(vehicle,mission,results) outputs = opt(opt_prob, sens_type='FD',sens_mode='pgc') print outputs return
def main(): vehicle, mission = full_setup() results = the_aircraft_function(vehicle,mission) post_process(vehicle,mission,results) return
def main(): vehicle, mission = full_setup() results = the_aircraft_function(vehicle,mission) post_process(vehicle,mission,results) return
def predict(input_path, output_path, model, csv_filename): if not os.path.exists(input_path): print >> sys.stderr, "%s file does not exits" % input_path return t_model = model.upper() if t_model == 'RNN': model_path = 'results/1_layer_model.net' print '==> using single layer RNN' elif t_model == '2RNN': model_path = 'results/2_layer_model.net' print '==> using 2 stacked layers RNN' elif t_model == 'BIRNN': model_path = 'results/bi_model.net' print '==> using bi-directional RNN' else: model_path = 'results/1_layer_model.net' print '==> unknown model, using default model: single RNN' try: length = utils.get_wav_file_length(input_path) except: print "The input file ", input_path, " is probably not a valid WAV file." exit(-1) feature_file = generate_tmp_filename('features') prob_file = generate_tmp_filename('prob') predict_file = generate_tmp_filename('prediction') dur_file = generate_tmp_filename('dur') print '\n1) Extracting features and classifying ...' abs_path = os.path.abspath(input_path) os.chdir("front_end/") fe.main(abs_path, feature_file) os.chdir("..") print '\n2) Model predictions ...' cmd = 'th classify.lua -x_filename %s -class_path %s -prob_path %s -model_path %s' % ( feature_file, predict_file, prob_file, model_path) os.chdir("back_end/") utils.easy_call(cmd) os.chdir("..") print '\n3) Extracting duration' post_process(os.path.abspath(predict_file), dur_file) print '\n4) Writing TextGrid file to %s ...' % output_path create_text_grid(dur_file, input_path, output_path, length, float(0.0), csv_filename) # remove leftovers os.remove(feature_file) os.remove(prob_file) os.remove(predict_file) os.remove(dur_file)
def publish_worker(queue: Queue, helper): for item, board in queue_iter(queue): try: post_process(item, board, helper) redis_publish( rdb, item=json.dumps(item, separators=(',', ':'), ensure_ascii=False, sort_keys=True), item_project="chan", item_subproject=CHAN, item_type=helper.item_type(item), item_category=board ) except Exception as e: logger.error(str(e) + ": " + traceback.format_exc())
def publish(thing): thing_type = type(thing).__name__.lower() j = serialize(thing) post_process(j) routing_key = "%s.%s" % (thing_type, str(thing.subreddit).lower()) while True: try: r.rpush( "q.reddit." + routing_key, json.dumps(j, separators=(',', ':'), ensure_ascii=False, sort_keys=True)) break except Exception as e: logger.error(str(e)) time.sleep(0.5)
def ocr_box_file(self, box_fname): # Set up the names of output files replace = lambda s: box_fname.replace('.box', s) asis_fname = replace('.ml.txt') nogram_out_fname = replace('.nogram.txt') ngram_out_fname = replace('.gram.txt') log_fname = replace('.{}.log'.format(self.loglevelname)) logging.basicConfig(filename=log_fname, level=self.loglevel, filemode="w") # Read Bantries & get Most likely output bf = BantryFile(box_fname) with open(asis_fname, 'w', encoding='utf-8') as f: f.write(post_process(bf.text)) # Process using ngrams ngrammed_lines, notgrammed_lines = [], [] for linenum in range(bf.num_lines): print("Line ", linenum) line_bantries = bf.get_line_bantires(linenum) gramgraph = GramGraph(line_bantries) gramgraph.process_tree() notgrammed_lines.append(gramgraph.get_best_apriori_str()) ngrammed_lines.append(gramgraph.get_best_str()) nogram_out = post_process("\n".join(notgrammed_lines)) with open(nogram_out_fname, 'w', encoding='utf-8') as out_file: out_file.write(nogram_out) ngram_out = post_process("\n".join(ngrammed_lines)) with open(ngram_out_fname, 'w', encoding='utf-8') as out_file: out_file.write(ngram_out) print("Input : ", box_fname) print("As is output : ", asis_fname) print("Without ngram : ", nogram_out_fname) print("With ngram : ", ngram_out_fname) print("Log : ", log_fname)
def ocr_box_file(self, box_fname): # Set up the names of output files replace = lambda s: box_fname.replace('.box', s) asis_fname = replace('.ml.txt') nogram_out_fname = replace('.nogram.txt') ngram_out_fname = replace('.gram.txt') log_fname = replace('.{}.log'.format(self.loglevelname)) logging.basicConfig(filename=log_fname, level=self.loglevel, filemode="w") # Read Bantries & get Most likely output bf = BantryFile(box_fname) with open(asis_fname, 'w', encoding='utf-8') as f: f.write(post_process(bf.text)) # Process using ngrams ngrammed_lines, notgrammed_lines = [], [] for linenum in range(bf.num_lines): print("Line ", linenum) line_bantries = bf.get_line_bantires(linenum) gramgraph = GramGraph(line_bantries) gramgraph.process_tree() notgrammed_lines.append(gramgraph.get_best_apriori_str()) ngrammed_lines.append(gramgraph.get_best_str()) nogram_out = post_process("\n".join(notgrammed_lines)) with open(nogram_out_fname, 'w', encoding='utf-8') as out_file: out_file.write(nogram_out) ngram_out = post_process("\n".join(ngrammed_lines)) with open(ngram_out_fname, 'w', encoding='utf-8') as out_file: out_file.write(ngram_out) print("Input : ", box_fname) print("As is output : ", asis_fname) print("Without ngram : ", nogram_out_fname) print("With ngram : ", ngram_out_fname) print("Log : ", log_fname)
def predict(wav_path, textgrid_path, start_extract, end_extract): tmp_feature_file = generate_tmp_filename('features') tmp_prob_file = generate_tmp_filename('prob') tmp_predict_file = generate_tmp_filename('prediction') if not os.path.exists(wav_path): print >> sys.stderr, "wav file %s does not exits" % wav_path return length = utils.get_wav_file_length(wav_path) print '\n1) Extracting features and classifying ...' extract_features(wav_path, tmp_feature_file, start_extract, end_extract) run(tmp_feature_file, tmp_prob_file) print '\n3) Extract Durations ...' post_process(tmp_prob_file, tmp_predict_file) print '\n4) Writing TextGrid file to %s ...' % textgrid_path create_text_grid(tmp_predict_file, textgrid_path, length, float(start_extract)) # remove leftovers os.remove(tmp_feature_file) os.remove(tmp_prob_file) os.remove(tmp_predict_file)
def end_to_end(impath, show=False, demo=False): """ Return a Label object from a label in the provided image. """ start = timer() # try: img = Image.open(impath) exif_data = img._getexif() orientation = 1 if exif_data is not None: # orientation tag is 0x0112 o_data = exif_data[274] if o_data is not None: orientation = int(o_data) label_im = contour(impath, demo=demo, orientation=orientation) if label_im is False: label_im = contour(impath, invert=True, demo=demo) if label_im is False: return False end = timer() print('Pre process time: %2f' % (end - start)) basename = os.path.basename(impath) label_impath = impath.replace(basename, 'tmp_' + basename) # label_impath = impath.lower().replace('.jpg', 'label_tmp.jpg') cv2.imwrite(label_impath, label_im) start = timer() # Apply Tesseract to image output = apply_tesseract(label_impath, demo=demo) end = timer() print('OCR time: %2f' % (end - start)) os.remove(label_impath) start = timer() ocr_label = post_process(output, demo=demo) end = timer() print('Post process time: %2f' % (end - start)) if show: draw_image(label_im, 'Transformed label: %s' % impath) return ocr_label
def end_to_end(impath, show=False, demo=False): """ Return a Label object from a label in the provided image. """ start = timer() # try: img = Image.open(impath) exif_data = img._getexif() orientation = 1 if exif_data is not None: # orientation tag is 0x0112 o_data = exif_data[274] if o_data is not None: orientation = int(o_data) label_im = contour(impath, demo=demo, orientation=orientation) if label_im is False: label_im = contour(impath, invert=True, demo=demo) if label_im is False: return False end = timer() print('Pre process time: %2f' % (end-start)) basename = os.path.basename(impath) label_impath = impath.replace(basename, 'tmp_' + basename) # label_impath = impath.lower().replace('.jpg', 'label_tmp.jpg') cv2.imwrite(label_impath, label_im) start = timer() # Apply Tesseract to image output = apply_tesseract(label_impath, demo=demo) end = timer() print('OCR time: %2f' % (end-start)) os.remove(label_impath) start = timer() ocr_label = post_process(output, demo=demo) end = timer() print('Post process time: %2f' % (end-start)) if show: draw_image(label_im, 'Transformed label: %s' % impath) return ocr_label
def predict_from_textgrid(wav_path, textgrid_path, tier, csv_outfile): tmp_feature_file = generate_tmp_filename('features') tmp_prob_file = generate_tmp_filename('prob') tmp_predict_file = generate_tmp_filename('prediction') # defines msc_2_sec = 0.001 if not os.path.exists(wav_path): print >> sys.stderr, "wav file %s does not exits" % wav_path return length = utils.get_wav_file_length(wav_path) # read the whole input text grid textgrid = TextGrid() textgrid.read(textgrid_path) tier_names = textgrid.tierNames() tier_index = tier_names.index(tier) # print all its interval, which has some value in their description (mark) for (i, interval) in enumerate(textgrid[tier_index]): if re.search(r'\S', interval.mark()): # define processing window start_extract = textgrid[tier_index][i].xmin() end_extract = textgrid[tier_index][i].xmax() print '\n1) Extracting features and classifying ...' extract_features(wav_path, tmp_feature_file, start_extract, end_extract) print '\n2) Make predictions ...' run(tmp_feature_file, tmp_prob_file) print '\n3) Extract Durations ...' p = post_process(tmp_prob_file, tmp_predict_file) print '\n4) Writing TextGrid file to %s ...' % textgrid_path onset = -1 offset = -1 prevoiced = -1 # prepare TextGrid vot_tier = IntervalTier(name='vot', xmin=0.0, xmax=float(length)) if p[1] == -1: onset = p[0] offset = p[2] vot_tier.append(Interval(0, float(onset) * msc_2_sec + start_extract, "")) vot_tier.append( Interval(float(onset) * msc_2_sec + start_extract, float(offset) * msc_2_sec + start_extract, "DeepVOT")) vot_tier.append(Interval(float(offset) * msc_2_sec + start_extract, float(length), "")) # write textgrid textgrid.append(vot_tier) textgrid.write(textgrid_path) if p[1] != -1: prevoiced = p[0] onset = p[1] offset = p[2] vot_tier.append(Interval(0, float(prevoiced) * msc_2_sec + start_extract, "")) vot_tier.append(Interval(float(prevoiced) * msc_2_sec + start_extract, float(onset) * msc_2_sec + start_extract, "prevoicing")) vot_tier.append( Interval(float(onset) * msc_2_sec + start_extract, float(offset) * msc_2_sec + start_extract, "DeepVOT")) vot_tier.append(Interval(float(offset) * msc_2_sec + start_extract, float(length), "")) # write textgrid textgrid.append(vot_tier) textgrid.write(textgrid_path) # remove leftovers os.remove(tmp_feature_file) os.remove(tmp_prob_file) os.remove(tmp_predict_file) if csv_outfile: with open(csv_outfile, 'w') as f: f.write("FILE, PREVOICE TIME, START_TIME, END_TIME\n") f.write("%s, %f, %f, %f\n" % (wav_path, (float(prevoiced) * msc_2_sec + start_extract) * 10, (float(onset) * msc_2_sec + start_extract) * 10, (float(offset) * msc_2_sec + start_extract) * 10))
P_plot[:, n] = P[:, 0] #saving initial pressure # time marching for i in range(n, nmax): P_old = sp.csr_matrix(P) #dummy vector as a placeholder for old P IM = (1 - numerical.theta ) * T + B / numerical.dt #implicit part coefficient in Eq. 3.44 EX = ((B / numerical.dt - numerical.theta * T) @ P_old) + Q #explicit part or RHS of Eq. 3.44 P = np.transpose([linalg.spsolve(IM, EX)]) #solving IM*P = EX or Ax=B P_plot[:, i + 1] = P[:, 0] #storing the pressure values inside the domain time[i + 1] = time[i] + numerical.dt #marching time post_process(P_plot, numerical.xc[:, 0], time, nmax) ''' #plotting fig = plt.figure(figsize=(15,7.5) , dpi=100) plot = plt.plot(numerical.xc[:,0],P_plot[:,0],'r-',label=r"$%0.1f$ [days]" % time[0]) plt.plot(numerical.xc[:,0],P_plot[:,10],'r--',label=r"$%0.1f$ [days]" % time[10]) plt.plot(numerical.xc[:,0],P_plot[:,20],'r-.',label=r"$%0.1f$ [days]" % time[20]) plt.plot(numerical.xc[:,0],P_plot[:,30],'b-',label=r"$%0.1f$ [days]" % time[30]) plt.plot(numerical.xc[:,0],P_plot[:,40],'b--',label=r"$%0.1f$ [days]" % time[40]) plt.plot(numerical.xc[:,0],P_plot[:,50],'b-.',label=r"$%0.1f$ [days]" % time[50]) plt.plot(numerical.xc[:,0],P_plot[:,100],'g-',label=r"$%0.1f$ [days]" % time[100]) plt.plot(numerical.xc[:,0],P_plot[:,150],'g--',label=r"$%0.1f$ [days]" % time[150]) plt.plot(numerical.xc[:,0],P_plot[:,200],'g-.',label=r"$%0.1f$ [days]" % time[200]) manager = plt.get_current_fig_manager() manager.window.showMaximized() plt.ylabel(r'$Pressure$ [psi]')
# time marching initialization time = 0 #initializing time n = 0 #time step initialization nmax = int(numerical.tfinal / numerical.dt) #total number of time steps time = np.zeros((nmax + 1)) #initializing time vector P_plot = np.zeros((numerical.N, nmax + 1)) #matrix to save pressure P = (IC.P).copy() #initializing iterable current pressure P_plot[:, n] = P[:, 0] #saving initial pressure B = B.tocsr() Q = Q.tocsr() # time marching for i in range(n, nmax): P_old = csr_matrix(P) #dummy vector as a placeholder for old P IM = (1 - numerical.theta ) * T + B / numerical.dt #implicit part coefficient in Eq. 3.44 EX = ((B / numerical.dt - numerical.theta * T) @ P_old) + Q + G #explicit part or RHS of Eq. 3.44 P = np.transpose([spsolve(IM, EX)]) #solving IM*P = EX or Ax=B P_plot[:, i + 1] = P[:, 0] #storing the pressure values inside the domain time[i + 1] = time[i] + numerical.dt #marching time end = timer.clock() print(end - start) post_process(P_plot, numerical.xc[:, 0], time, 10000)
"--findblocktext", type=str, default="False", help="find_block_text method that is applied to the image") ap.add_argument("-m", "--model", type=str, default="vie_best", help="model tesseract") args = vars(ap.parse_args()) model_lang = args["model"] file_name = args["image"] if args["preprocess"] == "True": img = pre_process.process_image_for_ocr(file_name) else: img = cv2.imread(file_name) Text = pytesseract.image_to_string(img, lang=model_lang) loi_sai = dict() if args["postprocess"] == "True": Text = post_process.post_process(Text) loi_sai = post_process.check_cta(Text) print(Text) if bool(loi_sai): print(u'Các từ sai là: \n') for key, value in loi_sai.items(): print(u'Từ số' + str(key) + ':' + value + ' \n')
def main(data_root, challengeset_fn, out_root, n_factors=1000, use_gpu=False): """""" # setup model names mf_name = MF_MODEL_NAME.format(n_factors) rnn_name = RNN_MODEL_NAME.format(n_factors) mfrnn_pl_name = MFRNN_PLAYLIST_MODEL_NAME.format(n_factors) print('>>> 1. Processing dataset!...') # only process fullset (we don't need subset now) data = (DataPrepper(subset=False).process(data_root, out_root, challengeset_fn)) print('>>> 2. Pre-training MF (WRMF) model!...') pretrain_cf(train_fn=join(out_root, "full", "playlist_track_train.csv"), r=n_factors, model_out_root=out_root, model_name=mf_name) print('>>> 3. Training RNN (Char-Ngram-LSTM) model!...') # > build tmp config rnn_conf = { 'path': { 'embeddings': { 'U': join(out_root, '{}_U.npy'.format(mf_name)), 'V': join(out_root, '{}_V.npy'.format(mf_name)) }, 'data': { 'playlists': join(out_root, 'full', 'playlist_hash.csv'), 'tracks': join(out_root, 'full', 'track_hash.csv'), 'artists': join(out_root, 'full', 'artist_hash.csv'), 'train': join(out_root, 'full', 'playlist_track_train.csv'), 'artist2track': join(out_root, 'full', 'artist_track.csv') }, 'model_out': { 'U': join(out_root, '{}_U.npy'.format(rnn_name)), 'V': join(out_root, '{}_V.npy'.format(rnn_name)), "rnn": join(out_root, "rnn_checkpoint.pth.tar") } }, 'hyper_parameters': { "early_stop": True, "use_gpu": use_gpu, "eval_while_fit": True, "sample_weight": False, "sample_weight_power": 0.75, "sample_threshold": 1e-6, "with_context": False, "num_epochs": 100, "neg_sample": 4, "loss": "all", "learn_rate": 0.005, "batch_size": 1024, "n_embedding": 300, "n_out_embedding": n_factors, "n_hid": 1000, "n_layers": 1, "drop_out": 0, "l2": 1e-4, "alpha": 0, "ngram_n": 3 }, } with tempfile.NamedTemporaryFile(suffix='.json') as tmpf: json.dump(rnn_conf, open(tmpf.name, 'w')) train_rnn(tmpf.name) # 4. post process (merge two playlist factors (mf/rnn)) print('>>> 4. Combining MF-RNN playlist factors!...') post_process(rnn_conf['path']['embeddings']['U'], rnn_conf['path']['model_out']['U'], join(out_root, '{}_U.npy'.format(mfrnn_pl_name)), rnn_conf['path']['data']['train']) # 5. prepare submission file print('>>> 5. Preparing submission file!!...') prep_conf = { 'path': { 'models': { 'als_rnn': { 'name': 'main', 'P': join(out_root, '{}_U.npy'.format(mfrnn_pl_name)), 'Q': join(out_root, rnn_conf['path']['embeddings']['V']), 'importance': 1, 'logistic': False } }, 'data': { 'playlists': join(out_root, "full", 'playlist_hash.csv'), 'tracks': join(out_root, "full", 'track_hash.csv'), 'train': join(out_root, "full", 'playlist_track_train.csv'), 'challenge_set': challengeset_fn }, 'output': './data/wrmf_titlernn_spotif_ai.csv' } } with tempfile.NamedTemporaryFile(suffix='.json') as tmpf: json.dump(prep_conf, open(tmpf.name, 'w')) prepare_submission(tmpf.name) # verify if the submission file is valid print('>>>>>> verifying submission file...') errors = verify_submission(challengeset_fn, prep_conf['path']['output']) assert errors == 0 print('>>>>>>>>> No errors found!...') print print('>>>>>> All process finished!!') print print
def translate(sentence): sentence = pre_process(sentence) decoder = Decoder() sentence = decoder.decode(sentence) sentence = post_process(sentence) return sentence
nogram_out_fname = replace('.nogram.txt') ngram_out_fname = replace('.gram.txt') logging.basicConfig(filename=log_fname, level=loglevel, filemode="w") ############################## Set-up scaler, classifier, ngram etc. Bantry.scaler = ScalerFactory(scaler_fname) Bantry.classifier = Classifier(nnet_fname, labels_fname, logbase=1) ng = Ngram(ngram_fname) Bantry.ngram = ng GramGraph.set_ngram(ng) ############################## Read Bantries & get Most likely output bf = BantryFile(banti_fname) with open(asis_fname, 'w', encoding='utf-8') as f: f.write(post_process(bf.text)) ############################## Process using ngrams ngrammed_lines, notgrammed_lines = [], [] for linenum in range(bf.num_lines): print("Line ", linenum) line_bantries = bf.get_line_bantires(linenum) gramgraph = GramGraph(line_bantries) gramgraph.process_tree() notgrammed_lines.append(gramgraph.get_best_apriori_str()) ngrammed_lines.append(gramgraph.get_best_str()) nogram_out = post_process("\n".join(notgrammed_lines)) with open(nogram_out_fname, 'w', encoding='utf-8') as out_file: out_file.write(nogram_out)
def stitch(paras_file, fasta_list_file, output_dir, REFALIGNER, BLASTN, GLPSOL, optmap, optmap_type, num_threads, threshold_1, threshold_2, threshold_3, threshold_4, threshold_5, threshold_6, threshold_7): #tools DIGEST = "./tools/fa2cmap_multi.pl" CPP_1 = "./tools/Script_mapping_contigs2contigs" CPP_2 = "./tools/Script_mapping_contigs2stitched_contigs" #intermediate files unrelated to each iteration input_merged_file = output_dir + "/input_merged.fasta" input_merged_file_chname = output_dir + "/input_merged_chname.fasta" input_merged_file_chname_long = output_dir + "/input_merged_chname_long.fasta" input_merged_file_chname_nochi = output_dir + "/input_merged_chname_nochi.fasta" sub_dir_prefix = output_dir + "/iteration_" last_stitched_file = output_dir + "/last_stitched.fasta" last_mtp_stitched_file = output_dir + "/last_mtp_stitched.fasta" final_stitched_file = output_dir + "/final_stitched.fasta" output_file = output_dir + "/final_stitched.nochi.fasta" last_refaligner_dir = output_dir + "/last_refaligner" last_refaligner_prefix = "stitched_contigs" last_silicomap = last_refaligner_dir + "/" + last_refaligner_prefix + "_" + optmap_type + ".cmap" last_list_file = last_refaligner_dir + "/" + last_refaligner_prefix + "_" + optmap_type + "_BNG_VS_seq_list.txt" last_key_file = last_refaligner_dir + "/" + last_refaligner_prefix + "_" + optmap_type + "_key.txt" chimeric_start_dir = output_dir + "/chimeric_start" chimeric_end_dir = output_dir + "/chimeric_end" #check existence of output_dir if not os.path.isdir(output_dir): os.makedirs(output_dir) #pre-operations output_paras(fasta_list_file, output_dir, optmap, optmap_type, num_threads, REFALIGNER, BLASTN, GLPSOL, threshold_1, threshold_2, threshold_3, threshold_4, threshold_5, threshold_6, threshold_7) merge_inputs(output_dir, fasta_list_file, input_merged_file) chname_fasta(input_merged_file, input_merged_file_chname) fasta_long_seqs(input_merged_file_chname, input_merged_file_chname_long) #iterations num_iterations = 1 old_adjusted_fasta_file = "" i = 0 while True: #intermediate files and directories for this iteration i += 1 sub_dir = sub_dir_prefix + str(i) refaligner_dir = sub_dir + "/refaligner" contigs_dir = sub_dir + "/grouped_contigs" listfiles_dir = sub_dir + "/grouped_listfiles" best_dir = sub_dir + "/best_hits" stitched_dir = sub_dir + "/stitched_contigs" alms_dir = sub_dir + "/alignments" input_fasta_file = sub_dir + "/input_contigs.fasta" mtp_fasta_file = sub_dir + "/input_contigs_mtp.fasta" mtp_fasta_file_chname = sub_dir + "/input_contigs_mtp_chname.fasta" unaligned_fasta_file = sub_dir + "/input_contigs_unaligned.fasta" lowconf_fasta_file = sub_dir + "/input_contigs_lowconf.fasta" stitched_fasta_file = sub_dir + "/stitched_contigs_total.fasta" stitch_info_file = sub_dir + "/stitch_info.log" adjusted_fasta_file = sub_dir + "/adjusted_contigs_total.fasta" refaligner_prefix = "input_contigs" silicomap = refaligner_dir + "/" + refaligner_prefix + "_" + optmap_type + ".cmap" list_file = refaligner_dir + "/" + refaligner_prefix + "_" + optmap_type + "_BNG_VS_seq_list.txt" key_file = refaligner_dir + "/" + refaligner_prefix + "_" + optmap_type + "_key.txt" group_list_file = sub_dir + "/group_id_list.txt" cancel_list_file = sub_dir + "/cancel_id_list.txt" filenames_list_file = sub_dir + "/new_filenames.txt" confid_file = sub_dir + "/confidences.txt" #pre-processing if not os.path.isdir(sub_dir): os.makedirs(sub_dir) if i == 1: copyfile(input_merged_file_chname_long, input_fasta_file) else: src = sub_dir_prefix + str(i - 1) + "/adjusted_contigs_total.fasta" copyfile(src, input_fasta_file) #core operations Phase1_reduction(REFALIGNER, DIGEST, GLPSOL, sub_dir, refaligner_dir, input_fasta_file, optmap, optmap_type, silicomap, refaligner_prefix, list_file, key_file, mtp_fasta_file, mtp_fasta_file_chname, unaligned_fasta_file, lowconf_fasta_file, num_threads, threshold_6, threshold_7) Phase2_stitch(BLASTN, CPP_1, mtp_fasta_file_chname, list_file, key_file, group_list_file, contigs_dir, listfiles_dir, best_dir, stitched_dir, stitched_fasta_file, stitch_info_file, num_threads, threshold_1, threshold_2) Phase3_check(BLASTN, CPP_2, contigs_dir, listfiles_dir, best_dir, stitched_dir, alms_dir, group_list_file, cancel_list_file, filenames_list_file, stitch_info_file, confid_file, adjusted_fasta_file, num_threads, threshold_3, threshold_4, threshold_5) if num_iterations != 1: if iterations_stop(old_adjusted_fasta_file, adjusted_fasta_file) == True: break old_adjusted_fasta_file = adjusted_fasta_file num_iterations += 1 #post-operations src = sub_dir_prefix + str( num_iterations) + "/adjusted_contigs_total.fasta" copyfile(src, last_stitched_file) post_process(REFALIGNER, DIGEST, GLPSOL, sub_dir, last_refaligner_dir, last_stitched_file, optmap, optmap_type, last_silicomap, last_refaligner_prefix, last_list_file, last_key_file, last_mtp_stitched_file, final_stitched_file, num_threads, threshold_6, threshold_7)
parser.add_argument("--config_deepsort", type=str, default="./configs/deep_sort.yaml") # parser.add_argument("--ignore_display", dest="display", action="store_false", default=True) parser.add_argument("--display", action="store_true", default=False) parser.add_argument("--frame_interval", type=int, default=1) parser.add_argument("--display_width", type=int, default=800) parser.add_argument("--display_height", type=int, default=600) parser.add_argument("--save_path", type=str, default="./output/") parser.add_argument("--cpu", dest="use_cuda", action="store_false", default=True) parser.add_argument("--camera", action="store", dest="cam", type=int, default="-1") return parser.parse_args() if __name__ == "__main__": print('start mot') start_time = time.time() args = parse_args() prepare_data(args.data_path) tracks = os.listdir(r'./dataset/test-c') for track in tracks: track_data_folder = os.path.join(r'./dataset/test-c', track, 'img1') im = cv2.imread(os.path.join(track_data_folder, '00000.jpg')) i_h, i_w,_ = im.shape os.system('cgexec -g memory:myGroup python vis_zhongxing.py --data_path '+track_data_folder+'--track_name'+track) post_process(args.result_path, './output/'+track+'.txt')
if WEIGHTS: yolo.model.load_weights(WEIGHTS) yolov2 = yolo.model yolov2.summary() # Load class names with open(CATEGORIES, mode='r') as txt_file: class_names = [c.strip() for c in txt_file.readlines()] img_shape = K.placeholder(shape=(2, )) # Start prediction boxes, classes, scores = post_process(yolov2, img_shape, n_classes=80, anchors=np.array(ANCHORS), iou_threshold=0.5, score_threshold=0.5, mode=2) orig_img = cv2.cvtColor(cv2.imread(IMG_PATH), cv2.COLOR_BGR2RGB) height, width, _ = orig_img.shape img = yolo.feature_extractor.preprocess( cv2.resize(orig_img, (IMG_INPUT, IMG_INPUT))) img = np.expand_dims(img, 0) pred_bboxes, pred_classes, pred_scores = sess.run( [boxes, classes, scores], feed_dict={ yolov2.input: img, img_shape: [height, width],
crawler_mgr.start_crawlers() while progress.active: progress.print() for url in url_mgr.get(): crawler_mgr.add_to_queue(url) for result, url, url_str, anchor_text in doc_mgr.get_parsed(n=300): if result == CR.SUCCESS: url_mgr.set(url_str, anchor_text, parent_URL=url) elif result == CR.NEED_RETRY: url_mgr.set(url) else: url_mgr.deactive_url(url) logger.save_to_disk() progress.print(force=True) print('\n') # end crawling doc_mgr.stop_doc_parsers() crawler_mgr.stop_crawlers() while doc_mgr.num_running_process or crawler_mgr.num_running_process: time.sleep(.5) logger.save_to_disk(force=True) if __name__ == '__main__': main() post_process(config.STORAGE_FOLDER)
def predict(input_path, output_path, model): tmp_dir = 'tmp/' tmp_features = 'tmp.features' tmp_prob = 'tmp.prob' tmp_prediction = 'tmp.prediction' tmp_duration = 'tmp.dur' if not os.path.exists(input_path): print >> sys.stderr, "wav file does not exits" return t_model = model.upper() if t_model == 'RNN': model_path = 'results/1_layer_model.net' print '==> using single RNN layer' elif t_model == '2RNN': model_path = 'results/2_layer_model.net' print '==> using 2 stacked layers of RNN' elif t_model == 'BIRNN': model_path = 'results/1_bi_model.net' print '==> using single bi-directional RNN layer' elif t_model == '2BIRNN': model_path = 'results/2_bi_model.net' print '==> using two stacked layers of bi-directional RNN' else: model_path = 'results/1_layer_model.net' print '==> unknown model, using default model: single layer of RNN' length = utils.get_wav_file_length(input_path) prob_file = tmp_dir + tmp_prob predict_file = tmp_dir + tmp_prediction dur_file = tmp_dir+tmp_duration # remove tmo dir if exists if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir) os.mkdir(tmp_dir) print '\n1) Extracting features and classifying ...' cmd = 'python predict_single_file.py %s %s ' % ( os.path.abspath(os.path.abspath(input_path)), os.path.abspath(tmp_dir) + '/' + tmp_features) os.chdir("front_end/") utils.easy_call(cmd) os.chdir("..") print '\n2) Model predictions ...' cmd = 'th classify.lua -folder_path %s -x_filename %s -class_path %s -prob_path %s -model_path %s' % ( os.path.abspath(tmp_dir), tmp_features, os.path.abspath(predict_file), os.path.abspath(prob_file), model_path) os.chdir("back_end/") utils.easy_call(cmd) os.chdir("..") print '\n3) Extracting duration' post_process(os.path.abspath(predict_file), dur_file) print '\n4) Writing TextGrid file to %s ...' % output_path create_text_grid(dur_file, output_path, length, float(0.0)) # remove leftovers if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir)
def main(): data_gen() run_experiment() post_process()
def __init__(self, classes, blocks): self.blocks = blocks self.classes = classes post_process(self)