def main():
data_path_dir = 'data'
if not os.path.exists(data_path_dir):
os.makedirs(data_path_dir)
data_path = 'data.json'
try:
os.remove(data_path)
except:
pass
subprocess.call([
'scrapy', 'runspider', 'scrapy/crawler/spiders/quotes_spider.py', '-o',
data_path
])
post_process(data_path)
with open(data_path, 'r') as fn:
data = json.loads(fn.read())
new_data = []
for item in data:
download(item['url'])
new_item = {
'content': item['content'],
'title': item['title'],
'thumbnail': item['thumbnail'],
'audio': os.path.join(data_path_dir,
glob.glob('*.mp3')[0]),
'type': 'audio'
}
shutil.move(glob.glob('*.mp3')[0], data_path_dir)
new_data.append(new_item)
os.remove(data_path)
with open(data_path, 'w') as fn:
json.dump(new_data, fn)
def publish(item, board, helper, channel, web):
item_type = helper.item_type(item)
post_process(item, board, helper, web)
while True:
try:
channel.basic_publish(exchange='chan',
routing_key="%s.%s.%s" %
(chan, item_type, board),
body=json.dumps(item,
separators=(',', ':'),
ensure_ascii=False,
sort_keys=True))
if MONITORING:
distance = datetime.utcnow() - datetime.utcfromtimestamp(
helper.item_mtime(item))
monitoring.log([{
"measurement": chan,
"time": str(datetime.utcnow()),
"tags": {
"board": board
},
"fields": {
"distance": distance.total_seconds()
}
}])
break
except Exception as e:
logger.debug(traceback.format_exc())
logger.error(str(e))
channel = connect()
def predict(input_path, output_path, start_extract, end_extract):
tmp_dir = 'tmp_files/'
tmp_features = 'tmp.features'
tmp_prob = 'tmp.prob'
tmp_prediction = 'tmp.prediction'
if not os.path.exists(input_path):
print >>sys.stderr, "wav file does not exits"
return
length = utils.get_wav_file_length(input_path)
feature_file = tmp_dir+tmp_features
prob_file = tmp_dir+tmp_prob
predict_file = tmp_dir+tmp_prediction
# remove tmo dir if exists
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
os.mkdir(tmp_dir)
print '\n1) Extracting features and classifying ...'
extract_features(input_path, feature_file, start_extract, end_extract)
run(feature_file, prob_file)
print '\n3) Extract Durations ...'
post_process(prob_file, predict_file)
print '\n4) Writing TextGrid file to %s ...' % output_path
create_text_grid(predict_file, output_path, length, float(start_extract))
def predict(input_path, output_path, start_extract, end_extract):
tmp_dir = 'tmp_files/'
tmp_features = 'tmp.features'
tmp_prob = 'tmp.prob'
tmp_prediction = 'tmp.prediction'
if not os.path.exists(input_path):
print >> sys.stderr, "wav file does not exits"
return
length = utils.get_wav_file_length(input_path)
feature_file = tmp_dir + tmp_features
prob_file = tmp_dir + tmp_prob
predict_file = tmp_dir + tmp_prediction
# remove tmo dir if exists
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
os.mkdir(tmp_dir)
print '\n1) Extracting features and classifying ...'
extract_features(input_path, feature_file, start_extract, end_extract)
run(feature_file, prob_file)
print '\n3) Extract Durations ...'
post_process(prob_file, predict_file)
print '\n4) Writing TextGrid file to %s ...' % output_path
create_text_grid(predict_file, output_path, length, float(start_extract))
def main():
#inputs = area, sweep, thickness, thrust
inputs = np.array([92.00,22.0 * Units.deg, 0.11, 20300.0])
# Have the optimizer call the wrapper
mywrap = lambda inputs:wrap(inputs)
opt_prob = pyOpt.Optimization('Embraer',mywrap)
opt_prob.addObj('Fuel')
opt_prob.addVar('Wing Area','c',lower=70,upper=110,value=inputs[0])
opt_prob.addVar('Wing Sweep','c',lower=15,upper=35,value=inputs[1])
opt_prob.addVar('Wing thickness','c',lower=0.10,upper=0.15,value=inputs[2])
opt_prob.addVar('Engine Thrust','c',lower=18000,upper=25000,value=inputs[3])
opt_prob.addConGroup('g',4,'i')
opt = pyOpt.pySLSQP.SLSQP()
post_process(vehicle,mission,results)
outputs = opt(opt_prob, sens_type='FD',sens_mode='pgc')
print outputs
return
def main():
vehicle, mission = full_setup()
results = the_aircraft_function(vehicle,mission)
post_process(vehicle,mission,results)
return
def main():
vehicle, mission = full_setup()
results = the_aircraft_function(vehicle,mission)
post_process(vehicle,mission,results)
return
def predict(input_path, output_path, model, csv_filename):
if not os.path.exists(input_path):
print >> sys.stderr, "%s file does not exits" % input_path
return
t_model = model.upper()
if t_model == 'RNN':
model_path = 'results/1_layer_model.net'
print '==> using single layer RNN'
elif t_model == '2RNN':
model_path = 'results/2_layer_model.net'
print '==> using 2 stacked layers RNN'
elif t_model == 'BIRNN':
model_path = 'results/bi_model.net'
print '==> using bi-directional RNN'
else:
model_path = 'results/1_layer_model.net'
print '==> unknown model, using default model: single RNN'
try:
length = utils.get_wav_file_length(input_path)
except:
print "The input file ", input_path, " is probably not a valid WAV file."
exit(-1)
feature_file = generate_tmp_filename('features')
prob_file = generate_tmp_filename('prob')
predict_file = generate_tmp_filename('prediction')
dur_file = generate_tmp_filename('dur')
print '\n1) Extracting features and classifying ...'
abs_path = os.path.abspath(input_path)
os.chdir("front_end/")
fe.main(abs_path, feature_file)
os.chdir("..")
print '\n2) Model predictions ...'
cmd = 'th classify.lua -x_filename %s -class_path %s -prob_path %s -model_path %s' % (
feature_file, predict_file, prob_file, model_path)
os.chdir("back_end/")
utils.easy_call(cmd)
os.chdir("..")
print '\n3) Extracting duration'
post_process(os.path.abspath(predict_file), dur_file)
print '\n4) Writing TextGrid file to %s ...' % output_path
create_text_grid(dur_file, input_path, output_path, length, float(0.0), csv_filename)
# remove leftovers
os.remove(feature_file)
os.remove(prob_file)
os.remove(predict_file)
os.remove(dur_file)
def publish_worker(queue: Queue, helper):
for item, board in queue_iter(queue):
try:
post_process(item, board, helper)
redis_publish(
rdb,
item=json.dumps(item, separators=(',', ':'), ensure_ascii=False, sort_keys=True),
item_project="chan",
item_subproject=CHAN,
item_type=helper.item_type(item),
item_category=board
)
except Exception as e:
logger.error(str(e) + ": " + traceback.format_exc())
def publish(thing):
thing_type = type(thing).__name__.lower()
j = serialize(thing)
post_process(j)
routing_key = "%s.%s" % (thing_type, str(thing.subreddit).lower())
while True:
try:
r.rpush(
"q.reddit." + routing_key,
json.dumps(j,
separators=(',', ':'),
ensure_ascii=False,
sort_keys=True))
break
except Exception as e:
logger.error(str(e))
time.sleep(0.5)
def ocr_box_file(self, box_fname):
# Set up the names of output files
replace = lambda s: box_fname.replace('.box', s)
asis_fname = replace('.ml.txt')
nogram_out_fname = replace('.nogram.txt')
ngram_out_fname = replace('.gram.txt')
log_fname = replace('.{}.log'.format(self.loglevelname))
logging.basicConfig(filename=log_fname,
level=self.loglevel,
filemode="w")
# Read Bantries & get Most likely output
bf = BantryFile(box_fname)
with open(asis_fname, 'w', encoding='utf-8') as f:
f.write(post_process(bf.text))
# Process using ngrams
ngrammed_lines, notgrammed_lines = [], []
for linenum in range(bf.num_lines):
print("Line ", linenum)
line_bantries = bf.get_line_bantires(linenum)
gramgraph = GramGraph(line_bantries)
gramgraph.process_tree()
notgrammed_lines.append(gramgraph.get_best_apriori_str())
ngrammed_lines.append(gramgraph.get_best_str())
nogram_out = post_process("\n".join(notgrammed_lines))
with open(nogram_out_fname, 'w', encoding='utf-8') as out_file:
out_file.write(nogram_out)
ngram_out = post_process("\n".join(ngrammed_lines))
with open(ngram_out_fname, 'w', encoding='utf-8') as out_file:
out_file.write(ngram_out)
print("Input : ", box_fname)
print("As is output : ", asis_fname)
print("Without ngram : ", nogram_out_fname)
print("With ngram : ", ngram_out_fname)
print("Log : ", log_fname)
def ocr_box_file(self, box_fname):
# Set up the names of output files
replace = lambda s: box_fname.replace('.box', s)
asis_fname = replace('.ml.txt')
nogram_out_fname = replace('.nogram.txt')
ngram_out_fname = replace('.gram.txt')
log_fname = replace('.{}.log'.format(self.loglevelname))
logging.basicConfig(filename=log_fname,
level=self.loglevel,
filemode="w")
# Read Bantries & get Most likely output
bf = BantryFile(box_fname)
with open(asis_fname, 'w', encoding='utf-8') as f:
f.write(post_process(bf.text))
# Process using ngrams
ngrammed_lines, notgrammed_lines = [], []
for linenum in range(bf.num_lines):
print("Line ", linenum)
line_bantries = bf.get_line_bantires(linenum)
gramgraph = GramGraph(line_bantries)
gramgraph.process_tree()
notgrammed_lines.append(gramgraph.get_best_apriori_str())
ngrammed_lines.append(gramgraph.get_best_str())
nogram_out = post_process("\n".join(notgrammed_lines))
with open(nogram_out_fname, 'w', encoding='utf-8') as out_file:
out_file.write(nogram_out)
ngram_out = post_process("\n".join(ngrammed_lines))
with open(ngram_out_fname, 'w', encoding='utf-8') as out_file:
out_file.write(ngram_out)
print("Input : ", box_fname)
print("As is output : ", asis_fname)
print("Without ngram : ", nogram_out_fname)
print("With ngram : ", ngram_out_fname)
print("Log : ", log_fname)
def predict(wav_path, textgrid_path, start_extract, end_extract):
tmp_feature_file = generate_tmp_filename('features')
tmp_prob_file = generate_tmp_filename('prob')
tmp_predict_file = generate_tmp_filename('prediction')
if not os.path.exists(wav_path):
print >> sys.stderr, "wav file %s does not exits" % wav_path
return
length = utils.get_wav_file_length(wav_path)
print '\n1) Extracting features and classifying ...'
extract_features(wav_path, tmp_feature_file, start_extract, end_extract)
run(tmp_feature_file, tmp_prob_file)
print '\n3) Extract Durations ...'
post_process(tmp_prob_file, tmp_predict_file)
print '\n4) Writing TextGrid file to %s ...' % textgrid_path
create_text_grid(tmp_predict_file, textgrid_path, length, float(start_extract))
# remove leftovers
os.remove(tmp_feature_file)
os.remove(tmp_prob_file)
os.remove(tmp_predict_file)
def end_to_end(impath, show=False, demo=False):
"""
Return a Label object from a label in the provided image.
"""
start = timer()
# try:
img = Image.open(impath)
exif_data = img._getexif()
orientation = 1
if exif_data is not None:
# orientation tag is 0x0112
o_data = exif_data[274]
if o_data is not None:
orientation = int(o_data)
label_im = contour(impath, demo=demo, orientation=orientation)
if label_im is False:
label_im = contour(impath, invert=True, demo=demo)
if label_im is False:
return False
end = timer()
print('Pre process time: %2f' % (end - start))
basename = os.path.basename(impath)
label_impath = impath.replace(basename, 'tmp_' + basename)
# label_impath = impath.lower().replace('.jpg', 'label_tmp.jpg')
cv2.imwrite(label_impath, label_im)
start = timer()
# Apply Tesseract to image
output = apply_tesseract(label_impath, demo=demo)
end = timer()
print('OCR time: %2f' % (end - start))
os.remove(label_impath)
start = timer()
ocr_label = post_process(output, demo=demo)
end = timer()
print('Post process time: %2f' % (end - start))
if show:
draw_image(label_im, 'Transformed label: %s' % impath)
return ocr_label
def end_to_end(impath, show=False, demo=False):
"""
Return a Label object from a label in the provided image.
"""
start = timer()
# try:
img = Image.open(impath)
exif_data = img._getexif()
orientation = 1
if exif_data is not None:
# orientation tag is 0x0112
o_data = exif_data[274]
if o_data is not None:
orientation = int(o_data)
label_im = contour(impath, demo=demo, orientation=orientation)
if label_im is False:
label_im = contour(impath, invert=True, demo=demo)
if label_im is False:
return False
end = timer()
print('Pre process time: %2f' % (end-start))
basename = os.path.basename(impath)
label_impath = impath.replace(basename, 'tmp_' + basename)
# label_impath = impath.lower().replace('.jpg', 'label_tmp.jpg')
cv2.imwrite(label_impath, label_im)
start = timer()
# Apply Tesseract to image
output = apply_tesseract(label_impath, demo=demo)
end = timer()
print('OCR time: %2f' % (end-start))
os.remove(label_impath)
start = timer()
ocr_label = post_process(output, demo=demo)
end = timer()
print('Post process time: %2f' % (end-start))
if show:
draw_image(label_im, 'Transformed label: %s' % impath)
return ocr_label
def predict_from_textgrid(wav_path, textgrid_path, tier, csv_outfile):
tmp_feature_file = generate_tmp_filename('features')
tmp_prob_file = generate_tmp_filename('prob')
tmp_predict_file = generate_tmp_filename('prediction')
# defines
msc_2_sec = 0.001
if not os.path.exists(wav_path):
print >> sys.stderr, "wav file %s does not exits" % wav_path
return
length = utils.get_wav_file_length(wav_path)
# read the whole input text grid
textgrid = TextGrid()
textgrid.read(textgrid_path)
tier_names = textgrid.tierNames()
tier_index = tier_names.index(tier)
# print all its interval, which has some value in their description (mark)
for (i, interval) in enumerate(textgrid[tier_index]):
if re.search(r'\S', interval.mark()):
# define processing window
start_extract = textgrid[tier_index][i].xmin()
end_extract = textgrid[tier_index][i].xmax()
print '\n1) Extracting features and classifying ...'
extract_features(wav_path, tmp_feature_file, start_extract, end_extract)
print '\n2) Make predictions ...'
run(tmp_feature_file, tmp_prob_file)
print '\n3) Extract Durations ...'
p = post_process(tmp_prob_file, tmp_predict_file)
print '\n4) Writing TextGrid file to %s ...' % textgrid_path
onset = -1
offset = -1
prevoiced = -1
# prepare TextGrid
vot_tier = IntervalTier(name='vot', xmin=0.0, xmax=float(length))
if p[1] == -1:
onset = p[0]
offset = p[2]
vot_tier.append(Interval(0, float(onset) * msc_2_sec + start_extract, ""))
vot_tier.append(
Interval(float(onset) * msc_2_sec + start_extract, float(offset) * msc_2_sec + start_extract, "DeepVOT"))
vot_tier.append(Interval(float(offset) * msc_2_sec + start_extract, float(length), ""))
# write textgrid
textgrid.append(vot_tier)
textgrid.write(textgrid_path)
if p[1] != -1:
prevoiced = p[0]
onset = p[1]
offset = p[2]
vot_tier.append(Interval(0, float(prevoiced) * msc_2_sec + start_extract, ""))
vot_tier.append(Interval(float(prevoiced) * msc_2_sec + start_extract, float(onset) * msc_2_sec + start_extract,
"prevoicing"))
vot_tier.append(
Interval(float(onset) * msc_2_sec + start_extract, float(offset) * msc_2_sec + start_extract, "DeepVOT"))
vot_tier.append(Interval(float(offset) * msc_2_sec + start_extract, float(length), ""))
# write textgrid
textgrid.append(vot_tier)
textgrid.write(textgrid_path)
# remove leftovers
os.remove(tmp_feature_file)
os.remove(tmp_prob_file)
os.remove(tmp_predict_file)
if csv_outfile:
with open(csv_outfile, 'w') as f:
f.write("FILE, PREVOICE TIME, START_TIME, END_TIME\n")
f.write("%s, %f, %f, %f\n" % (wav_path, (float(prevoiced) * msc_2_sec + start_extract) * 10,
(float(onset) * msc_2_sec + start_extract) * 10,
(float(offset) * msc_2_sec + start_extract) * 10))
P_plot[:, n] = P[:, 0] #saving initial pressure
# time marching
for i in range(n, nmax):
P_old = sp.csr_matrix(P) #dummy vector as a placeholder for old P
IM = (1 - numerical.theta
) * T + B / numerical.dt #implicit part coefficient in Eq. 3.44
EX = ((B / numerical.dt - numerical.theta * T)
@ P_old) + Q #explicit part or RHS of Eq. 3.44
P = np.transpose([linalg.spsolve(IM, EX)]) #solving IM*P = EX or Ax=B
P_plot[:, i + 1] = P[:, 0] #storing the pressure values inside the domain
time[i + 1] = time[i] + numerical.dt #marching time
post_process(P_plot, numerical.xc[:, 0], time, nmax)
'''
#plotting
fig = plt.figure(figsize=(15,7.5) , dpi=100)
plot = plt.plot(numerical.xc[:,0],P_plot[:,0],'r-',label=r"$%0.1f$ [days]" % time[0])
plt.plot(numerical.xc[:,0],P_plot[:,10],'r--',label=r"$%0.1f$ [days]" % time[10])
plt.plot(numerical.xc[:,0],P_plot[:,20],'r-.',label=r"$%0.1f$ [days]" % time[20])
plt.plot(numerical.xc[:,0],P_plot[:,30],'b-',label=r"$%0.1f$ [days]" % time[30])
plt.plot(numerical.xc[:,0],P_plot[:,40],'b--',label=r"$%0.1f$ [days]" % time[40])
plt.plot(numerical.xc[:,0],P_plot[:,50],'b-.',label=r"$%0.1f$ [days]" % time[50])
plt.plot(numerical.xc[:,0],P_plot[:,100],'g-',label=r"$%0.1f$ [days]" % time[100])
plt.plot(numerical.xc[:,0],P_plot[:,150],'g--',label=r"$%0.1f$ [days]" % time[150])
plt.plot(numerical.xc[:,0],P_plot[:,200],'g-.',label=r"$%0.1f$ [days]" % time[200])
manager = plt.get_current_fig_manager()
manager.window.showMaximized()
plt.ylabel(r'$Pressure$ [psi]')
# time marching initialization
time = 0 #initializing time
n = 0 #time step initialization
nmax = int(numerical.tfinal / numerical.dt) #total number of time steps
time = np.zeros((nmax + 1)) #initializing time vector
P_plot = np.zeros((numerical.N, nmax + 1)) #matrix to save pressure
P = (IC.P).copy() #initializing iterable current pressure
P_plot[:, n] = P[:, 0] #saving initial pressure
B = B.tocsr()
Q = Q.tocsr()
# time marching
for i in range(n, nmax):
P_old = csr_matrix(P) #dummy vector as a placeholder for old P
IM = (1 - numerical.theta
) * T + B / numerical.dt #implicit part coefficient in Eq. 3.44
EX = ((B / numerical.dt - numerical.theta * T)
@ P_old) + Q + G #explicit part or RHS of Eq. 3.44
P = np.transpose([spsolve(IM, EX)]) #solving IM*P = EX or Ax=B
P_plot[:, i + 1] = P[:, 0] #storing the pressure values inside the domain
time[i + 1] = time[i] + numerical.dt #marching time
end = timer.clock()
print(end - start)
post_process(P_plot, numerical.xc[:, 0], time, 10000)
"--findblocktext",
type=str,
default="False",
help="find_block_text method that is applied to the image")
ap.add_argument("-m",
"--model",
type=str,
default="vie_best",
help="model tesseract")
args = vars(ap.parse_args())
model_lang = args["model"]
file_name = args["image"]
if args["preprocess"] == "True":
img = pre_process.process_image_for_ocr(file_name)
else:
img = cv2.imread(file_name)
Text = pytesseract.image_to_string(img, lang=model_lang)
loi_sai = dict()
if args["postprocess"] == "True":
Text = post_process.post_process(Text)
loi_sai = post_process.check_cta(Text)
print(Text)
if bool(loi_sai):
print(u'Các từ sai là: \n')
for key, value in loi_sai.items():
print(u'Từ số' + str(key) + ':' + value + ' \n')
def main(data_root, challengeset_fn, out_root, n_factors=1000, use_gpu=False):
""""""
# setup model names
mf_name = MF_MODEL_NAME.format(n_factors)
rnn_name = RNN_MODEL_NAME.format(n_factors)
mfrnn_pl_name = MFRNN_PLAYLIST_MODEL_NAME.format(n_factors)
print('>>> 1. Processing dataset!...')
# only process fullset (we don't need subset now)
data = (DataPrepper(subset=False).process(data_root, out_root,
challengeset_fn))
print('>>> 2. Pre-training MF (WRMF) model!...')
pretrain_cf(train_fn=join(out_root, "full", "playlist_track_train.csv"),
r=n_factors,
model_out_root=out_root,
model_name=mf_name)
print('>>> 3. Training RNN (Char-Ngram-LSTM) model!...')
# > build tmp config
rnn_conf = {
'path': {
'embeddings': {
'U': join(out_root, '{}_U.npy'.format(mf_name)),
'V': join(out_root, '{}_V.npy'.format(mf_name))
},
'data': {
'playlists': join(out_root, 'full', 'playlist_hash.csv'),
'tracks': join(out_root, 'full', 'track_hash.csv'),
'artists': join(out_root, 'full', 'artist_hash.csv'),
'train': join(out_root, 'full', 'playlist_track_train.csv'),
'artist2track': join(out_root, 'full', 'artist_track.csv')
},
'model_out': {
'U': join(out_root, '{}_U.npy'.format(rnn_name)),
'V': join(out_root, '{}_V.npy'.format(rnn_name)),
"rnn": join(out_root, "rnn_checkpoint.pth.tar")
}
},
'hyper_parameters': {
"early_stop": True,
"use_gpu": use_gpu,
"eval_while_fit": True,
"sample_weight": False,
"sample_weight_power": 0.75,
"sample_threshold": 1e-6,
"with_context": False,
"num_epochs": 100,
"neg_sample": 4,
"loss": "all",
"learn_rate": 0.005,
"batch_size": 1024,
"n_embedding": 300,
"n_out_embedding": n_factors,
"n_hid": 1000,
"n_layers": 1,
"drop_out": 0,
"l2": 1e-4,
"alpha": 0,
"ngram_n": 3
},
}
with tempfile.NamedTemporaryFile(suffix='.json') as tmpf:
json.dump(rnn_conf, open(tmpf.name, 'w'))
train_rnn(tmpf.name)
# 4. post process (merge two playlist factors (mf/rnn))
print('>>> 4. Combining MF-RNN playlist factors!...')
post_process(rnn_conf['path']['embeddings']['U'],
rnn_conf['path']['model_out']['U'],
join(out_root, '{}_U.npy'.format(mfrnn_pl_name)),
rnn_conf['path']['data']['train'])
# 5. prepare submission file
print('>>> 5. Preparing submission file!!...')
prep_conf = {
'path': {
'models': {
'als_rnn': {
'name': 'main',
'P': join(out_root, '{}_U.npy'.format(mfrnn_pl_name)),
'Q': join(out_root, rnn_conf['path']['embeddings']['V']),
'importance': 1,
'logistic': False
}
},
'data': {
'playlists': join(out_root, "full", 'playlist_hash.csv'),
'tracks': join(out_root, "full", 'track_hash.csv'),
'train': join(out_root, "full", 'playlist_track_train.csv'),
'challenge_set': challengeset_fn
},
'output': './data/wrmf_titlernn_spotif_ai.csv'
}
}
with tempfile.NamedTemporaryFile(suffix='.json') as tmpf:
json.dump(prep_conf, open(tmpf.name, 'w'))
prepare_submission(tmpf.name)
# verify if the submission file is valid
print('>>>>>> verifying submission file...')
errors = verify_submission(challengeset_fn, prep_conf['path']['output'])
assert errors == 0
print('>>>>>>>>> No errors found!...')
print
print('>>>>>> All process finished!!')
print
print
def translate(sentence): sentence = pre_process(sentence) decoder = Decoder() sentence = decoder.decode(sentence) sentence = post_process(sentence) return sentence
nogram_out_fname = replace('.nogram.txt')
ngram_out_fname = replace('.gram.txt')
logging.basicConfig(filename=log_fname, level=loglevel, filemode="w")
############################## Set-up scaler, classifier, ngram etc.
Bantry.scaler = ScalerFactory(scaler_fname)
Bantry.classifier = Classifier(nnet_fname, labels_fname, logbase=1)
ng = Ngram(ngram_fname)
Bantry.ngram = ng
GramGraph.set_ngram(ng)
############################## Read Bantries & get Most likely output
bf = BantryFile(banti_fname)
with open(asis_fname, 'w', encoding='utf-8') as f:
f.write(post_process(bf.text))
############################## Process using ngrams
ngrammed_lines, notgrammed_lines = [], []
for linenum in range(bf.num_lines):
print("Line ", linenum)
line_bantries = bf.get_line_bantires(linenum)
gramgraph = GramGraph(line_bantries)
gramgraph.process_tree()
notgrammed_lines.append(gramgraph.get_best_apriori_str())
ngrammed_lines.append(gramgraph.get_best_str())
nogram_out = post_process("\n".join(notgrammed_lines))
with open(nogram_out_fname, 'w', encoding='utf-8') as out_file:
out_file.write(nogram_out)
def stitch(paras_file, fasta_list_file, output_dir, REFALIGNER, BLASTN, GLPSOL,
optmap, optmap_type, num_threads, threshold_1, threshold_2,
threshold_3, threshold_4, threshold_5, threshold_6, threshold_7):
#tools
DIGEST = "./tools/fa2cmap_multi.pl"
CPP_1 = "./tools/Script_mapping_contigs2contigs"
CPP_2 = "./tools/Script_mapping_contigs2stitched_contigs"
#intermediate files unrelated to each iteration
input_merged_file = output_dir + "/input_merged.fasta"
input_merged_file_chname = output_dir + "/input_merged_chname.fasta"
input_merged_file_chname_long = output_dir + "/input_merged_chname_long.fasta"
input_merged_file_chname_nochi = output_dir + "/input_merged_chname_nochi.fasta"
sub_dir_prefix = output_dir + "/iteration_"
last_stitched_file = output_dir + "/last_stitched.fasta"
last_mtp_stitched_file = output_dir + "/last_mtp_stitched.fasta"
final_stitched_file = output_dir + "/final_stitched.fasta"
output_file = output_dir + "/final_stitched.nochi.fasta"
last_refaligner_dir = output_dir + "/last_refaligner"
last_refaligner_prefix = "stitched_contigs"
last_silicomap = last_refaligner_dir + "/" + last_refaligner_prefix + "_" + optmap_type + ".cmap"
last_list_file = last_refaligner_dir + "/" + last_refaligner_prefix + "_" + optmap_type + "_BNG_VS_seq_list.txt"
last_key_file = last_refaligner_dir + "/" + last_refaligner_prefix + "_" + optmap_type + "_key.txt"
chimeric_start_dir = output_dir + "/chimeric_start"
chimeric_end_dir = output_dir + "/chimeric_end"
#check existence of output_dir
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
#pre-operations
output_paras(fasta_list_file, output_dir, optmap, optmap_type, num_threads,
REFALIGNER, BLASTN, GLPSOL, threshold_1, threshold_2,
threshold_3, threshold_4, threshold_5, threshold_6,
threshold_7)
merge_inputs(output_dir, fasta_list_file, input_merged_file)
chname_fasta(input_merged_file, input_merged_file_chname)
fasta_long_seqs(input_merged_file_chname, input_merged_file_chname_long)
#iterations
num_iterations = 1
old_adjusted_fasta_file = ""
i = 0
while True:
#intermediate files and directories for this iteration
i += 1
sub_dir = sub_dir_prefix + str(i)
refaligner_dir = sub_dir + "/refaligner"
contigs_dir = sub_dir + "/grouped_contigs"
listfiles_dir = sub_dir + "/grouped_listfiles"
best_dir = sub_dir + "/best_hits"
stitched_dir = sub_dir + "/stitched_contigs"
alms_dir = sub_dir + "/alignments"
input_fasta_file = sub_dir + "/input_contigs.fasta"
mtp_fasta_file = sub_dir + "/input_contigs_mtp.fasta"
mtp_fasta_file_chname = sub_dir + "/input_contigs_mtp_chname.fasta"
unaligned_fasta_file = sub_dir + "/input_contigs_unaligned.fasta"
lowconf_fasta_file = sub_dir + "/input_contigs_lowconf.fasta"
stitched_fasta_file = sub_dir + "/stitched_contigs_total.fasta"
stitch_info_file = sub_dir + "/stitch_info.log"
adjusted_fasta_file = sub_dir + "/adjusted_contigs_total.fasta"
refaligner_prefix = "input_contigs"
silicomap = refaligner_dir + "/" + refaligner_prefix + "_" + optmap_type + ".cmap"
list_file = refaligner_dir + "/" + refaligner_prefix + "_" + optmap_type + "_BNG_VS_seq_list.txt"
key_file = refaligner_dir + "/" + refaligner_prefix + "_" + optmap_type + "_key.txt"
group_list_file = sub_dir + "/group_id_list.txt"
cancel_list_file = sub_dir + "/cancel_id_list.txt"
filenames_list_file = sub_dir + "/new_filenames.txt"
confid_file = sub_dir + "/confidences.txt"
#pre-processing
if not os.path.isdir(sub_dir):
os.makedirs(sub_dir)
if i == 1:
copyfile(input_merged_file_chname_long, input_fasta_file)
else:
src = sub_dir_prefix + str(i - 1) + "/adjusted_contigs_total.fasta"
copyfile(src, input_fasta_file)
#core operations
Phase1_reduction(REFALIGNER, DIGEST, GLPSOL, sub_dir, refaligner_dir,
input_fasta_file, optmap, optmap_type, silicomap,
refaligner_prefix, list_file, key_file,
mtp_fasta_file, mtp_fasta_file_chname,
unaligned_fasta_file, lowconf_fasta_file, num_threads,
threshold_6, threshold_7)
Phase2_stitch(BLASTN, CPP_1, mtp_fasta_file_chname, list_file,
key_file, group_list_file, contigs_dir, listfiles_dir,
best_dir, stitched_dir, stitched_fasta_file,
stitch_info_file, num_threads, threshold_1, threshold_2)
Phase3_check(BLASTN, CPP_2, contigs_dir, listfiles_dir, best_dir,
stitched_dir, alms_dir, group_list_file, cancel_list_file,
filenames_list_file, stitch_info_file, confid_file,
adjusted_fasta_file, num_threads, threshold_3,
threshold_4, threshold_5)
if num_iterations != 1:
if iterations_stop(old_adjusted_fasta_file,
adjusted_fasta_file) == True:
break
old_adjusted_fasta_file = adjusted_fasta_file
num_iterations += 1
#post-operations
src = sub_dir_prefix + str(
num_iterations) + "/adjusted_contigs_total.fasta"
copyfile(src, last_stitched_file)
post_process(REFALIGNER, DIGEST, GLPSOL, sub_dir, last_refaligner_dir,
last_stitched_file, optmap, optmap_type, last_silicomap,
last_refaligner_prefix, last_list_file, last_key_file,
last_mtp_stitched_file, final_stitched_file, num_threads,
threshold_6, threshold_7)
parser.add_argument("--config_deepsort", type=str, default="./configs/deep_sort.yaml")
# parser.add_argument("--ignore_display", dest="display", action="store_false", default=True)
parser.add_argument("--display", action="store_true", default=False)
parser.add_argument("--frame_interval", type=int, default=1)
parser.add_argument("--display_width", type=int, default=800)
parser.add_argument("--display_height", type=int, default=600)
parser.add_argument("--save_path", type=str, default="./output/")
parser.add_argument("--cpu", dest="use_cuda", action="store_false", default=True)
parser.add_argument("--camera", action="store", dest="cam", type=int, default="-1")
return parser.parse_args()
if __name__ == "__main__":
print('start mot')
start_time = time.time()
args = parse_args()
prepare_data(args.data_path)
tracks = os.listdir(r'./dataset/test-c')
for track in tracks:
track_data_folder = os.path.join(r'./dataset/test-c', track, 'img1')
im = cv2.imread(os.path.join(track_data_folder, '00000.jpg'))
i_h, i_w,_ = im.shape
os.system('cgexec -g memory:myGroup python vis_zhongxing.py --data_path '+track_data_folder+'--track_name'+track)
post_process(args.result_path, './output/'+track+'.txt')
if WEIGHTS:
yolo.model.load_weights(WEIGHTS)
yolov2 = yolo.model
yolov2.summary()
# Load class names
with open(CATEGORIES, mode='r') as txt_file:
class_names = [c.strip() for c in txt_file.readlines()]
img_shape = K.placeholder(shape=(2, ))
# Start prediction
boxes, classes, scores = post_process(yolov2,
img_shape,
n_classes=80,
anchors=np.array(ANCHORS),
iou_threshold=0.5,
score_threshold=0.5,
mode=2)
orig_img = cv2.cvtColor(cv2.imread(IMG_PATH), cv2.COLOR_BGR2RGB)
height, width, _ = orig_img.shape
img = yolo.feature_extractor.preprocess(
cv2.resize(orig_img, (IMG_INPUT, IMG_INPUT)))
img = np.expand_dims(img, 0)
pred_bboxes, pred_classes, pred_scores = sess.run(
[boxes, classes, scores],
feed_dict={
yolov2.input: img,
img_shape: [height, width],
crawler_mgr.start_crawlers()
while progress.active:
progress.print()
for url in url_mgr.get():
crawler_mgr.add_to_queue(url)
for result, url, url_str, anchor_text in doc_mgr.get_parsed(n=300):
if result == CR.SUCCESS:
url_mgr.set(url_str, anchor_text, parent_URL=url)
elif result == CR.NEED_RETRY:
url_mgr.set(url)
else:
url_mgr.deactive_url(url)
logger.save_to_disk()
progress.print(force=True)
print('\n')
# end crawling
doc_mgr.stop_doc_parsers()
crawler_mgr.stop_crawlers()
while doc_mgr.num_running_process or crawler_mgr.num_running_process:
time.sleep(.5)
logger.save_to_disk(force=True)
if __name__ == '__main__':
main()
post_process(config.STORAGE_FOLDER)
def predict(input_path, output_path, model):
tmp_dir = 'tmp/'
tmp_features = 'tmp.features'
tmp_prob = 'tmp.prob'
tmp_prediction = 'tmp.prediction'
tmp_duration = 'tmp.dur'
if not os.path.exists(input_path):
print >> sys.stderr, "wav file does not exits"
return
t_model = model.upper()
if t_model == 'RNN':
model_path = 'results/1_layer_model.net'
print '==> using single RNN layer'
elif t_model == '2RNN':
model_path = 'results/2_layer_model.net'
print '==> using 2 stacked layers of RNN'
elif t_model == 'BIRNN':
model_path = 'results/1_bi_model.net'
print '==> using single bi-directional RNN layer'
elif t_model == '2BIRNN':
model_path = 'results/2_bi_model.net'
print '==> using two stacked layers of bi-directional RNN'
else:
model_path = 'results/1_layer_model.net'
print '==> unknown model, using default model: single layer of RNN'
length = utils.get_wav_file_length(input_path)
prob_file = tmp_dir + tmp_prob
predict_file = tmp_dir + tmp_prediction
dur_file = tmp_dir+tmp_duration
# remove tmo dir if exists
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
os.mkdir(tmp_dir)
print '\n1) Extracting features and classifying ...'
cmd = 'python predict_single_file.py %s %s ' % (
os.path.abspath(os.path.abspath(input_path)), os.path.abspath(tmp_dir) + '/' + tmp_features)
os.chdir("front_end/")
utils.easy_call(cmd)
os.chdir("..")
print '\n2) Model predictions ...'
cmd = 'th classify.lua -folder_path %s -x_filename %s -class_path %s -prob_path %s -model_path %s' % (
os.path.abspath(tmp_dir), tmp_features, os.path.abspath(predict_file), os.path.abspath(prob_file), model_path)
os.chdir("back_end/")
utils.easy_call(cmd)
os.chdir("..")
print '\n3) Extracting duration'
post_process(os.path.abspath(predict_file), dur_file)
print '\n4) Writing TextGrid file to %s ...' % output_path
create_text_grid(dur_file, output_path, length, float(0.0))
# remove leftovers
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
def main():
data_gen()
run_experiment()
post_process()
def __init__(self, classes, blocks):
self.blocks = blocks
self.classes = classes
post_process(self)
