def process_scgink(l, request_id):
try:
with open(l, 'r') as f:
scgink_str = ''
line = f.readline()
while line:
scgink_str += line
line = f.readline()
f.close()
except requests.exceptions.RequestException as e: # all exception
return "ERROR"
try:
scgink_data = ScgImage(scgink_str, request_id)
# empty traces due to scgink data
if not scgink_data.traces:
return "ERROR"
img_file_path = outdir + '/' + request_id + '_input.png'
#convert to png format
scgink_data.save_image(img_file_path)
#preprocess image
filename, postfix, processed_img, = img_file_path, '.png', outdir + '/' + request_id + '_preprocessed.png',
crop_blank_default_size, pad_size, buckets, downsample_ratio = [
600, 60
], (8, 8, 8, 8), default_buckets, 2
l = (filename, postfix, processed_img, crop_blank_default_size,
pad_size, buckets, downsample_ratio)
preprocess(l)
except:
return "ERROR"
return "SUCC"
def process_hw_record_table(self):
'''
generate png images from scg content
:return:
'''
cursor = self.db.cursor()
try:
cursor.execute(self.hw_record_sql)
# Fetch all the rows in a list of lists.
results = cursor.fetchall()
total = len(results)
print 'total hw_record records: ', total
count = 0
for row in results:
count += 1
print 'process ' + str(count) + ' of ' + str(total)
id, scg_content = row[0], row[1]
if count == 1537:
print id, scg_content
break
scg = ScgImage(scg_content, id)
scg.save_image(batch.png_dir + str(id) + '.png')
except:
logging.exception("DB reading exception!!!")
raise
class TestScgImage(TestCase):
def setUp(self):
# Load in the workbook
excel = Excel('data/scg/test.xlsx')
scg_id, scg_content, truth, dt1, dt2 = excel.get_scg_record(1)
self.scg = ScgImage(scg_content, scg_id) # x + y
def test_save_image(self):
self.scg.save_image('temp/' + str(self.scg.id) + '.png')
def test_scg_file(self):
scgink = json.load(open('data/x_+_y_square.json'))
print '\n', scgink
print '\n', scgink['scg_ink']
ink = scgink['scg_ink']
scg = ScgImage(ink, 999)
scg.save_image('temp/' + str(999) + '.png')
def test_scg_file_bug(self):
num = str(610)
file_name = 'temp/2018-09-17/' + num + '_scg.txt'
with open(file_name, 'r') as myfile:
data = myfile.read()
print data
scg = ScgImage(data, num)
scg.save_image('temp/' + num + '.png')
def test_scg_file(self):
scgink = json.load(open('data/x_+_y_square.json'))
print '\n', scgink
print '\n', scgink['scg_ink']
ink = scgink['scg_ink']
scg = ScgImage(ink, 999)
scg.save_image('temp/' + str(999) + '.png')
def test_scg_file_bug(self):
num = str(610)
file_name = 'temp/2018-09-17/' + num + '_scg.txt'
with open(file_name, 'r') as myfile:
data = myfile.read()
print data
scg = ScgImage(data, num)
scg.save_image('temp/' + num + '.png')
class TestScgImage(TestCase):
def setUp(self):
# Load in the workbook
excel = Excel('data/scg/test.xlsx')
scg_id, scg_content, truth = excel.get_scg_record(1)
self.scg = ScgImage(scg_content, scg_id) # x + y
def test_save_image(self):
self.scg.save_image('temp/' + str(self.scg.id) + '.png')
def test_scg_file(self):
scgink = json.load(open('data/x_+_y_square.json'))
print '\n', scgink
print '\n', scgink['scg_ink']
ink = scgink['scg_ink']
scg = ScgImage(ink, 999)
scg.save_image('temp/' + str(999) + '.png')
def model_api(input_data):
"""
Args:
input_data: submitted to the API, json string
Returns:
output_data: after some transformation, to be
returned to the API
"""
# process input
global hw_count
global start_0
res = {}
request_id = str(uuid.uuid4())
res['id'] = input_data['id']
scgink = input_data['scg_ink']
try:
scgink_data = ScgImage(scgink, request_id)
except:
res['status'] = 'error'
res['info'] = 'bad scgink data'
return res
# empty traces due to scgink data
if not scgink_data.traces:
res['info'] = 'wrong scgink data'
res['status'] = 'error'
return res
start_t = current_milli_time()
img_file_path = outdir + '/' + request_id + '_input.png'
#convert to png format
scgink_data.save_image(img_file_path)
#preprocess image
filename, postfix, processed_img = img_file_path, '.png', outdir + '/' + request_id + '_preprocessed.png'
crop_blank_default_size, pad_size, buckets, downsample_ratio = [
600, 60
], (8, 8, 8, 8), default_buckets, 2
l = (filename, postfix, processed_img, crop_blank_default_size,
pad_size, buckets, downsample_ratio)
if not preprocess(l):
res['status'] = 'error'
return res
# construct data
os.system('echo ' + request_id + '_preprocessed.png ' +
'>temp/test.txt')
src = 'temp/test.txt'
src_dir = 'temp'
#print "src=", src
#print "src_dir=", src_dir
data = onmt.io.build_dataset(fields,
opt.data_type,
src,
None,
src_dir=src_dir,
sample_rate=opt.sample_rate,
window_size=opt.window_size,
window_stride=opt.window_stride,
window=opt.window,
use_filter_pred=False)
# Sort batch by decreasing lengths of sentence required by pytorch.
# sort=False means "Use dataset's sortkey instead of iterator's".
data_iter = onmt.io.OrderedIterator(dataset=data,
device=opt.gpu,
batch_size=opt.batch_size,
train=False,
sort=False,
sort_within_batch=True,
shuffle=False)
# Inference
builder = onmt.translate.TranslationBuilder(data, translator.fields,
opt.n_best,
opt.replace_unk, opt.tgt)
cnt = 0
for batch in data_iter:
batch_data = translator.translate_batch(batch, data)
translations = builder.from_batch(batch_data)
for trans in translations:
cnt += 1
n_best_preds = [
" ".join(pred) for pred in trans.pred_sents[:opt.n_best]
]
now_t = current_milli_time()
#hw_count = hw_count + 1
#if hw_count %100 == 0 :
# app.logger.debug( "last 100 "+(now_t - start_0 ))
# start_0 = now_t
# app.logger.debug( "time spent "+( now_t -start_t))
# process the output
n_best_latex = []
for pred in n_best_preds:
n_best_latex.append(detokenizer(pred))
n_best_ascii = []
for pred in n_best_latex:
n_best_ascii.append(latex_asciimath(pred))
# return the output for the api
res['status'] = "succuss"
res['info'] = now_t - start_t
res['mathml'] = ''
res['latex'] = n_best_latex[0]
res['asciimath'] = n_best_ascii[0]
res['n_best_latex'] = n_best_latex
res['n_best_ascii'] = n_best_ascii
app.logger.debug(request_id + "\t" + n_best_latex[0] + "\n")
return res
def setUp(self):
# Load in the workbook
excel = Excel('data/scg/test.xlsx')
scg_id, scg_content, truth = excel.get_scg_record(1)
self.scg = ScgImage(scg_content, scg_id) # x + y