def main():
"""
Get a list of formulas.
Parameters
----------
cursor : a database cursor
dataset : string
Either 'all' or a path to a yaml symbol file.
Returns
-------
list :
A list of formulas
"""
cfg = utils.get_database_configuration()
mysql = cfg['mysql_online']
connection = pymysql.connect(host=mysql['host'],
user=mysql['user'],
passwd=mysql['passwd'],
db=mysql['db'],
cursorclass=pymysql.cursors.DictCursor,
charset='utf8')
cursor = connection.cursor()
sql = (
"SELECT `id`, `formula_in_latex` FROM `wm_formula` "
# "WHERE `formula_type` = 'single symbol' "
"WHERE `formula_type` = 'nesting symbol' "
"ORDER BY `formula_in_latex` ASC")
cursor.execute(sql)
symbols = cursor.fetchall()
store_symbols(symbols)
def test_execution():
"""Test if the functions execute at all."""
utils.get_project_root()
utils.get_latest_model(".", "model")
utils.get_latest_working_model(".")
utils.get_latest_successful_run(".")
assert utils.get_readable_time(123) == "123ms"
assert utils.get_readable_time(1000 * 30) == "30s 0ms"
assert utils.get_readable_time(1000 * 60) == "1 minutes 0s 0ms"
assert utils.get_readable_time(1000 * 60 * 60) == "1h, 0 minutes 0s 0ms"
assert utils.get_readable_time(2 * 1000 * 60 *
60) == "2h, 0 minutes 0s 0ms"
assert utils.get_readable_time(25 * 1000 * 60 * 60 +
3) == "25h, 0 minutes 0s 3ms"
utils.print_status(3, 1, 123)
utils.get_nntoolkit()
utils.get_database_config_file()
utils.get_database_configuration()
assert utils.sizeof_fmt(1) == "1.0 bytes"
assert utils.sizeof_fmt(1111) == "1.1 KB"
def main():
cfg = utils.get_database_configuration()
mysql = cfg['mysql_online']
logging.info("Start dumping structure and constraints...")
dir_s = "/home/moose/GitHub/write-math"
tables = dump_structure(mysql,
prefix='wm_',
filename_strucutre=(("%s/database/structure/"
"write-math.sql") % dir_s),
filename_constraints=(("%s/database/structure/"
"foreign-keys.sql") %
dir_s))
logging.info(tables)
def execution_test():
"""Test if the functions execute at all."""
utils.get_project_root()
utils.get_latest_model(".", "model")
utils.get_latest_working_model(".")
utils.get_latest_successful_run(".")
nose.tools.assert_equal(utils.get_readable_time(123), "123ms")
nose.tools.assert_equal(utils.get_readable_time(1000*30),
"30s 0ms")
nose.tools.assert_equal(utils.get_readable_time(1000*60),
"1 minutes 0s 0ms")
nose.tools.assert_equal(utils.get_readable_time(1000*60*60),
"1h, 0 minutes 0s 0ms")
nose.tools.assert_equal(utils.get_readable_time(2*1000*60*60),
"2h, 0 minutes 0s 0ms")
nose.tools.assert_equal(utils.get_readable_time(25*1000*60*60+3),
"25h, 0 minutes 0s 3ms")
utils.print_status(3, 1, 123)
utils.get_nntoolkit()
utils.get_database_config_file()
utils.get_database_configuration()
nose.tools.assert_equal(utils.sizeof_fmt(1), "1.0 bytes")
nose.tools.assert_equal(utils.sizeof_fmt(1111), "1.1 KB")
def main(destination=os.path.join(utils.get_project_root(), "raw-datasets"),
dataset='all',
renderings=False):
"""Main part of the backup script."""
time_prefix = time.strftime("%Y-%m-%d-%H-%M")
filename = ("%s-handwriting_datasets-%s-raw.pickle" %
(time_prefix, dataset.replace('/', '-')))
destination_path = os.path.join(destination, filename)
logging.info("Data will be written to '%s'", destination_path)
cfg = utils.get_database_configuration()
mysql = cfg['mysql_online']
connection = pymysql.connect(host=mysql['host'],
user=mysql['user'],
passwd=mysql['passwd'],
db=mysql['db'],
cursorclass=pymysql.cursors.DictCursor)
cursor = connection.cursor()
formulas = get_formulas(cursor, dataset)
logging.info('Received %i formulas.', len(formulas))
handwriting_datasets = []
formula_id2latex = {}
# Go through each formula and download every raw_data instance
for formula in formulas:
formula_id2latex[formula['id']] = formula['formula_in_latex']
sql = ((
"SELECT `wm_raw_draw_data`.`id`, `data`, `is_in_testset`, "
"`wild_point_count`, `missing_line`, `user_id`, "
"`display_name` "
"FROM `wm_raw_draw_data` "
"JOIN `wm_users` ON "
"(`wm_users`.`id` = `wm_raw_draw_data`.`user_id`) "
"WHERE `accepted_formula_id` = %s "
# "AND `display_name` LIKE 'MfrDB::%%'"
) % str(formula['id']))
cursor.execute(sql)
raw_datasets = cursor.fetchall()
logging.info("%s (%i)", formula['formula_in_latex'], len(raw_datasets))
for raw_data in raw_datasets:
try:
handwriting = HandwrittenData(
raw_data['data'],
formula['id'],
raw_data['id'],
formula['formula_in_latex'],
raw_data['wild_point_count'],
raw_data['missing_line'],
raw_data['user_id'],
user_name=raw_data['display_name'])
handwriting_datasets.append({
'handwriting':
handwriting,
'id':
raw_data['id'],
'formula_id':
formula['id'],
'formula_in_latex':
formula['formula_in_latex'],
'is_in_testset':
raw_data['is_in_testset']
})
except Exception as e:
logging.info("Raw data id: %s", raw_data['id'])
logging.info(e)
pickle.dump(
{
'handwriting_datasets': handwriting_datasets,
'formula_id2latex': formula_id2latex
}, open(destination_path, "wb"), 2)
if renderings:
logging.info("Start downloading SVG renderings...")
svgfolder = tempfile.mkdtemp()
sql = """SELECT t1.formula_id, t1.svg from wm_renderings t1
LEFT JOIN wm_renderings t2 ON t1.formula_id = t2.formula_id
AND t1.creation_time < t2.creation_time
WHERE t2.id is null"""
cursor.execute(sql)
formulas = cursor.fetchall()
logging.info("Create svg...")
for formula in formulas:
filename = os.path.join(svgfolder,
"%s.svg" % str(formula['formula_id']))
with open(filename, 'wb') as temp_file:
temp_file.write(formula['svg'])
logging.info("Tar at %s", os.path.abspath("renderings.tar"))
tar = tarfile.open("renderings.tar.bz2", "w:bz2")
for fn in os.listdir(svgfolder):
filename = os.path.join(svgfolder, fn)
if os.path.isfile(filename):
print(filename)
tar.add(filename, arcname=os.path.basename(filename))
tar.close()
def main(dataset='all'):
"""
Parameters
----------
dataset : string
Either 'all' or a path to a yaml symbol file.
"""
cfg = utils.get_database_configuration()
mysql = cfg['mysql_online']
connection = pymysql.connect(host=mysql['host'],
user=mysql['user'],
passwd=mysql['passwd'],
db=mysql['db'],
cursorclass=pymysql.cursors.DictCursor)
cursor = connection.cursor()
# TODO: no formulas, only single-symbol ones.
formulas = get_formulas(cursor, dataset)
prob = {}
# Go through each formula and download every raw_data instance
for formula in formulas:
stroke_counts = []
recordings = []
sql = (("SELECT `wm_raw_draw_data`.`id`, `data`, `is_in_testset`, "
"`wild_point_count`, `missing_line`, `user_id`, "
"`display_name` "
"FROM `wm_raw_draw_data` "
"JOIN `wm_users` ON "
"(`wm_users`.`id` = `wm_raw_draw_data`.`user_id`) "
"WHERE `accepted_formula_id` = %s "
"AND wild_point_count=0 "
"AND has_correction=0 "
# "AND `display_name` LIKE 'MfrDB::%%'"
) %
str(formula['id']))
cursor.execute(sql)
raw_datasets = cursor.fetchall()
logging.info("%s (%i)", formula['formula_in_latex'], len(raw_datasets))
for raw_data in raw_datasets:
try:
handwriting = HandwrittenData(raw_data['data'],
formula['id'],
raw_data['id'],
formula['formula_in_latex'],
raw_data['wild_point_count'],
raw_data['missing_line'],
raw_data['user_id'])
stroke_counts.append(len(handwriting.get_pointlist()))
recordings.append(handwriting)
except Exception as e:
logging.info("Raw data id: %s", raw_data['id'])
logging.info(e)
if len(stroke_counts) > 0:
logging.info("\t[%i - %i]", min(stroke_counts), max(stroke_counts))
median = numpy.median(stroke_counts)
logging.info("\tMedian: %0.2f\tMean: %0.2f\tstd: %0.2f",
median,
numpy.mean(stroke_counts),
numpy.std(stroke_counts))
# Make prob
s = sorted(Counter(stroke_counts).items(),
key=lambda n: n[1],
reverse=True)
key = formula['formula_in_latex']
prob[key] = {}
for stroke_nr, count in s:
prob[key][stroke_nr] = count
# Outliers
modes = get_modes(stroke_counts)
logging.info("\tModes: %s", modes)
exceptions = []
for rec in recordings:
if len(rec.get_pointlist()) not in modes:
url = (("http://www.martin-thoma.de/"
"write-math/view/?raw_data_id=%i - "
"%i strokes") % (rec.raw_data_id,
len(rec.get_pointlist())))
dist = get_dist(len(rec.get_pointlist()), modes)
exceptions.append((url, len(rec.get_pointlist()), dist))
print_exceptions(exceptions, max_print=10)
else:
logging.debug("No recordings for symbol "
"'http://www.martin-thoma.de/"
"write-math/symbol/?id=%s'.",
formula['id'])
write_prob(prob, "prob_stroke_count_by_symbol.yml")
for el in add_new:
print("\thttp://write-math.com/view/?raw_data_id=%i" % el)
for rid in add_new:
sql = ("UPDATE `wm_raw_draw_data` SET `is_in_testset`=1 "
"WHERE `id` = %i LIMIT 1") % rid
cursor.execute(sql)
connection.commit()
def get_parser():
"""Return the parser object for this script."""
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
parser = ArgumentParser(description=__doc__,
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument("-s", "--symbol",
dest="symbol_filename",
type=lambda x: utils.is_valid_file(parser, x),
required=True,
help="symbol yml file",
metavar="FILE")
return parser
if __name__ == '__main__':
args = get_parser().parse_args()
cfg = utils.get_database_configuration()
if 'mysql_online' in cfg:
main(cfg['mysql_online'], args.symbol_filename)
if 'mysql_local' in cfg:
main(cfg['mysql_local'], args.symbol_filename)
def main(cfg, raw_data_start_id):
cfg = utils.get_database_configuration()
mysql = cfg['mysql_online']
connection = pymysql.connect(host=mysql['host'],
user=mysql['user'],
passwd=mysql['passwd'],
db=mysql['db'],
cursorclass=pymysql.cursors.DictCursor)
cursor = connection.cursor()
# Get formulas
print("Get formulas")
sql = ("SELECT `id`, `formula_in_latex` FROM `wm_formula` WHERE `id` > %s")
cursor.execute(sql, (raw_data_start_id, ))
formulas = cursor.fetchall()
formulaid2latex = {}
for el in formulas:
formulaid2latex[el['id']] = el['formula_in_latex']
preprocessing_queue = [preprocessing.ScaleAndShift(),
# preprocessing.Douglas_peucker(EPSILON=0.2),
# preprocessing.Space_evenly(number=100,
# kind='cubic')
]
checked_formulas = 0
checked_raw_data_instances = 0
for formula_id in formulaid2latex.keys():
if formula_id == 1:
# This formula id is for trash. No need to look at it.
continue
# Get data
print("Get data for formula_id %i (%s)" % (formula_id,
formulaid2latex[formula_id])
)
sql = ("SELECT `id`, `data`, `accepted_formula_id`, "
"`wild_point_count`, `missing_line`, `has_hook`, "
"`has_too_long_line`, `is_image`, `administrator_edit`, "
"`other_problem`, `has_interrupted_line` "
"FROM `wm_raw_draw_data` "
"WHERE `accepted_formula_id` = %i "
"ORDER BY `administrator_edit` DESC, "
"`creation_date` ASC;") % formula_id
cursor.execute(sql)
raw_datasets = cursor.fetchall()
print("Raw datasets: %i" % len(raw_datasets))
checked_raw_data_instances += len(raw_datasets)
checked_formulas += 1
if len(raw_datasets) < 100:
continue
for i, data in enumerate(raw_datasets):
if data['data'] == "[]":
continue
B = HandwrittenDataM(data['data'],
data['accepted_formula_id'],
data['wild_point_count'],
data['missing_line'],
data['has_hook'],
data['has_too_long_line'],
data['is_image'],
data['other_problem'],
data['has_interrupted_line'],
data['id'],
formulaid2latex[formula_id])
B.preprocessing(preprocessing_queue)
Bs = deepcopy(B)
Bs.preprocessing([preprocessing.DotReduction(0.01)])
if B != Bs:
before_pointcount = sum([len(line)
for line in B.get_pointlist()])
after_pointcount = sum([len(line)
for line in Bs.get_pointlist()])
print("Reduced %i lines to %i lines." %
(len(B.get_pointlist()), len(Bs.get_pointlist())))
print("Reduced %i points to %i points." %
(before_pointcount, after_pointcount))
if before_pointcount - after_pointcount > 2:
B.show()
Bs.show()
print("[Status] Checked formulas: %i of %i" % (checked_formulas,
len(formulaid2latex)))
print("[Status] Checked raw_data_instances: %i" %
checked_raw_data_instances)
print("done")
def main(dataset='all'):
"""
Parameters
----------
dataset : string
Either 'all' or a path to a yaml symbol file.
"""
cfg = utils.get_database_configuration()
mysql = cfg['mysql_online']
connection = pymysql.connect(host=mysql['host'],
user=mysql['user'],
passwd=mysql['passwd'],
db=mysql['db'],
cursorclass=pymysql.cursors.DictCursor)
cursor = connection.cursor()
# TODO: no formulas, only single-symbol ones.
formulas = get_formulas(cursor, dataset)
prob = {}
# Go through each formula and download every raw_data instance
for formula in formulas:
stroke_counts = []
recordings = []
sql = ((
"SELECT `wm_raw_draw_data`.`id`, `data`, `is_in_testset`, "
"`wild_point_count`, `missing_line`, `user_id`, "
"`display_name` "
"FROM `wm_raw_draw_data` "
"JOIN `wm_users` ON "
"(`wm_users`.`id` = `wm_raw_draw_data`.`user_id`) "
"WHERE `accepted_formula_id` = %s "
"AND wild_point_count=0 "
"AND has_correction=0 "
# "AND `display_name` LIKE 'MfrDB::%%'"
) % str(formula['id']))
cursor.execute(sql)
raw_datasets = cursor.fetchall()
logging.info("%s (%i)", formula['formula_in_latex'], len(raw_datasets))
for raw_data in raw_datasets:
try:
handwriting = HandwrittenData(raw_data['data'], formula['id'],
raw_data['id'],
formula['formula_in_latex'],
raw_data['wild_point_count'],
raw_data['missing_line'],
raw_data['user_id'])
stroke_counts.append(len(handwriting.get_pointlist()))
recordings.append(handwriting)
except Exception as e:
logging.info("Raw data id: %s", raw_data['id'])
logging.info(e)
if len(stroke_counts) > 0:
logging.info("\t[%i - %i]", min(stroke_counts), max(stroke_counts))
median = numpy.median(stroke_counts)
logging.info("\tMedian: %0.2f\tMean: %0.2f\tstd: %0.2f", median,
numpy.mean(stroke_counts), numpy.std(stroke_counts))
# Make prob
s = sorted(Counter(stroke_counts).items(),
key=lambda n: n[1],
reverse=True)
key = formula['formula_in_latex']
prob[key] = {}
for stroke_nr, count in s:
prob[key][stroke_nr] = count
# Outliers
modes = get_modes(stroke_counts)
logging.info("\tModes: %s", modes)
exceptions = []
for rec in recordings:
if len(rec.get_pointlist()) not in modes:
url = (("http://www.martin-thoma.de/"
"write-math/view/?raw_data_id=%i - "
"%i strokes") %
(rec.raw_data_id, len(rec.get_pointlist())))
dist = get_dist(len(rec.get_pointlist()), modes)
exceptions.append((url, len(rec.get_pointlist()), dist))
print_exceptions(exceptions, max_print=10)
else:
logging.debug(
"No recordings for symbol "
"'http://www.martin-thoma.de/"
"write-math/symbol/?id=%s'.", formula['id'])
write_prob(prob, "prob_stroke_count_by_symbol.yml")
def main(cfg, raw_data_start_id):
cfg = utils.get_database_configuration()
mysql = cfg['mysql_online']
connection = pymysql.connect(host=mysql['host'],
user=mysql['user'],
passwd=mysql['passwd'],
db=mysql['db'],
cursorclass=pymysql.cursors.DictCursor)
cursor = connection.cursor()
# Get formulas
print("Get formulas")
sql = ("SELECT `id`, `formula_in_latex` FROM `wm_formula` WHERE `id` > %s")
cursor.execute(sql, (raw_data_start_id, ))
formulas = cursor.fetchall()
formulaid2latex = {}
for el in formulas:
formulaid2latex[el['id']] = el['formula_in_latex']
preprocessing_queue = [
preprocessing.ScaleAndShift(),
# preprocessing.Douglas_peucker(EPSILON=0.2),
# preprocessing.Space_evenly(number=100,
# kind='cubic')
]
checked_formulas = 0
checked_raw_data_instances = 0
for formula_id in formulaid2latex.keys():
if formula_id == 1:
# This formula id is for trash. No need to look at it.
continue
# Get data
print("Get data for formula_id %i (%s)" %
(formula_id, formulaid2latex[formula_id]))
sql = ("SELECT `id`, `data`, `accepted_formula_id`, "
"`wild_point_count`, `missing_line`, `has_hook`, "
"`has_too_long_line`, `is_image`, `administrator_edit`, "
"`other_problem`, `has_interrupted_line` "
"FROM `wm_raw_draw_data` "
"WHERE `accepted_formula_id` = %i "
"ORDER BY `administrator_edit` DESC, "
"`creation_date` ASC;") % formula_id
cursor.execute(sql)
raw_datasets = cursor.fetchall()
print("Raw datasets: %i" % len(raw_datasets))
checked_raw_data_instances += len(raw_datasets)
checked_formulas += 1
if len(raw_datasets) < 100:
continue
for i, data in enumerate(raw_datasets):
if data['data'] == "[]":
continue
B = HandwrittenDataM(data['data'], data['accepted_formula_id'],
data['wild_point_count'],
data['missing_line'], data['has_hook'],
data['has_too_long_line'], data['is_image'],
data['other_problem'],
data['has_interrupted_line'], data['id'],
formulaid2latex[formula_id])
B.preprocessing(preprocessing_queue)
Bs = deepcopy(B)
Bs.preprocessing([preprocessing.DotReduction(0.01)])
if B != Bs:
before_pointcount = sum(
[len(line) for line in B.get_pointlist()])
after_pointcount = sum(
[len(line) for line in Bs.get_pointlist()])
print("Reduced %i lines to %i lines." %
(len(B.get_pointlist()), len(Bs.get_pointlist())))
print("Reduced %i points to %i points." %
(before_pointcount, after_pointcount))
if before_pointcount - after_pointcount > 2:
B.show()
Bs.show()
print("[Status] Checked formulas: %i of %i" %
(checked_formulas, len(formulaid2latex)))
print("[Status] Checked raw_data_instances: %i" %
checked_raw_data_instances)
print("done")
def main():
cfg = utils.get_database_configuration()
mysql = cfg['mysql_online']
find_wrong_count(mysql)
def main():
cfg = utils.get_database_configuration()
mysql = cfg['mysql_online']
find_wrong_count(mysql)
def main(destination=os.path.join(utils.get_project_root(),
"raw-datasets"),
dataset='all',
renderings=False):
"""Main part of the backup script."""
time_prefix = time.strftime("%Y-%m-%d-%H-%M")
filename = ("%s-handwriting_datasets-%s-raw.pickle" %
(time_prefix, dataset.replace('/', '-')))
destination_path = os.path.join(destination, filename)
logging.info("Data will be written to '%s'", destination_path)
cfg = utils.get_database_configuration()
mysql = cfg['mysql_online']
connection = pymysql.connect(host=mysql['host'],
user=mysql['user'],
passwd=mysql['passwd'],
db=mysql['db'],
cursorclass=pymysql.cursors.DictCursor)
cursor = connection.cursor()
formulas = get_formulas(cursor, dataset)
logging.info('Received %i formulas.', len(formulas))
handwriting_datasets = []
formula_id2latex = {}
# Go through each formula and download every raw_data instance
for formula in formulas:
formula_id2latex[formula['id']] = formula['formula_in_latex']
sql = (("SELECT `wm_raw_draw_data`.`id`, `data`, `is_in_testset`, "
"`wild_point_count`, `missing_line`, `user_id`, "
"`display_name` "
"FROM `wm_raw_draw_data` "
"JOIN `wm_users` ON "
"(`wm_users`.`id` = `wm_raw_draw_data`.`user_id`) "
"WHERE `accepted_formula_id` = %s "
# "AND `display_name` LIKE 'MfrDB::%%'"
) %
str(formula['id']))
cursor.execute(sql)
raw_datasets = cursor.fetchall()
logging.info("%s (%i)", formula['formula_in_latex'], len(raw_datasets))
for raw_data in raw_datasets:
try:
handwriting = HandwrittenData(raw_data['data'],
formula['id'],
raw_data['id'],
formula['formula_in_latex'],
raw_data['wild_point_count'],
raw_data['missing_line'],
raw_data['user_id'],
user_name=raw_data['display_name'])
handwriting_datasets.append({'handwriting': handwriting,
'id': raw_data['id'],
'formula_id': formula['id'],
'formula_in_latex':
formula['formula_in_latex'],
'is_in_testset':
raw_data['is_in_testset']})
except Exception as e:
logging.info("Raw data id: %s", raw_data['id'])
logging.info(e)
pickle.dump({'handwriting_datasets': handwriting_datasets,
'formula_id2latex': formula_id2latex},
open(destination_path, "wb"),
2)
if renderings:
logging.info("Start downloading SVG renderings...")
svgfolder = tempfile.mkdtemp()
sql = """SELECT t1.formula_id, t1.svg from wm_renderings t1
LEFT JOIN wm_renderings t2 ON t1.formula_id = t2.formula_id
AND t1.creation_time < t2.creation_time
WHERE t2.id is null"""
cursor.execute(sql)
formulas = cursor.fetchall()
logging.info("Create svg...")
for formula in formulas:
filename = os.path.join(svgfolder,
"%s.svg" % str(formula['formula_id']))
with open(filename, 'wb') as temp_file:
temp_file.write(formula['svg'])
logging.info("Tar at %s", os.path.abspath("renderings.tar"))
tar = tarfile.open("renderings.tar.bz2", "w:bz2")
for fn in os.listdir(svgfolder):
filename = os.path.join(svgfolder, fn)
if os.path.isfile(filename):
print(filename)
tar.add(filename, arcname=os.path.basename(filename))
tar.close()