def ScaleAndShift_test_simple_3():
preprocessing_queue = [preprocessing.ScaleAndShift()]
s = '[[{"x":0, "y":10, "time": 0}]]'
a = HandwrittenData(s)
a.preprocessing(preprocessing_queue)
s = a.get_pointlist()
expectation = [[{"x": 0, "y": 0, "time": 0}]]
assert s == expectation, "Got: %s; expected %s" % (s, expectation)
def test_ScaleAndShift_test_simple_4():
preprocessing_queue = [preprocessing.ScaleAndShift()]
s = '[[{"x":0, "y":0, "time": 10}]]'
a = HandwrittenData(s)
a.preprocessing(preprocessing_queue)
s = a.get_pointlist()
expectation = [[{"x": 0, "y": 0, "time": 0}]]
assert s == expectation, f"Got: {s}; expected {expectation}"
def main(dataset='all'):
"""
Parameters
----------
dataset : string
Either 'all' or a path to a yaml symbol file.
"""
cfg = utils.get_database_configuration()
mysql = cfg['mysql_online']
connection = pymysql.connect(host=mysql['host'],
user=mysql['user'],
passwd=mysql['passwd'],
db=mysql['db'],
cursorclass=pymysql.cursors.DictCursor)
cursor = connection.cursor()
# TODO: no formulas, only single-symbol ones.
formulas = get_formulas(cursor, dataset)
prob = {}
# Go through each formula and download every raw_data instance
for formula in formulas:
stroke_counts = []
recordings = []
sql = (("SELECT `wm_raw_draw_data`.`id`, `data`, `is_in_testset`, "
"`wild_point_count`, `missing_line`, `user_id`, "
"`display_name` "
"FROM `wm_raw_draw_data` "
"JOIN `wm_users` ON "
"(`wm_users`.`id` = `wm_raw_draw_data`.`user_id`) "
"WHERE `accepted_formula_id` = %s "
"AND wild_point_count=0 "
"AND has_correction=0 "
# "AND `display_name` LIKE 'MfrDB::%%'"
) %
str(formula['id']))
cursor.execute(sql)
raw_datasets = cursor.fetchall()
logging.info("%s (%i)", formula['formula_in_latex'], len(raw_datasets))
for raw_data in raw_datasets:
try:
handwriting = HandwrittenData(raw_data['data'],
formula['id'],
raw_data['id'],
formula['formula_in_latex'],
raw_data['wild_point_count'],
raw_data['missing_line'],
raw_data['user_id'])
stroke_counts.append(len(handwriting.get_pointlist()))
recordings.append(handwriting)
except Exception as e:
logging.info("Raw data id: %s", raw_data['id'])
logging.info(e)
if len(stroke_counts) > 0:
logging.info("\t[%i - %i]", min(stroke_counts), max(stroke_counts))
median = numpy.median(stroke_counts)
logging.info("\tMedian: %0.2f\tMean: %0.2f\tstd: %0.2f",
median,
numpy.mean(stroke_counts),
numpy.std(stroke_counts))
# Make prob
s = sorted(Counter(stroke_counts).items(),
key=lambda n: n[1],
reverse=True)
key = formula['formula_in_latex']
prob[key] = {}
for stroke_nr, count in s:
prob[key][stroke_nr] = count
# Outliers
modes = get_modes(stroke_counts)
logging.info("\tModes: %s", modes)
exceptions = []
for rec in recordings:
if len(rec.get_pointlist()) not in modes:
url = (("http://www.martin-thoma.de/"
"write-math/view/?raw_data_id=%i - "
"%i strokes") % (rec.raw_data_id,
len(rec.get_pointlist())))
dist = get_dist(len(rec.get_pointlist()), modes)
exceptions.append((url, len(rec.get_pointlist()), dist))
print_exceptions(exceptions, max_print=10)
else:
logging.debug("No recordings for symbol "
"'http://www.martin-thoma.de/"
"write-math/symbol/?id=%s'.",
formula['id'])
write_prob(prob, "prob_stroke_count_by_symbol.yml")
def main(dataset='all'):
"""
Parameters
----------
dataset : string
Either 'all' or a path to a yaml symbol file.
"""
cfg = utils.get_database_configuration()
mysql = cfg['mysql_online']
connection = pymysql.connect(host=mysql['host'],
user=mysql['user'],
passwd=mysql['passwd'],
db=mysql['db'],
cursorclass=pymysql.cursors.DictCursor)
cursor = connection.cursor()
# TODO: no formulas, only single-symbol ones.
formulas = get_formulas(cursor, dataset)
prob = {}
# Go through each formula and download every raw_data instance
for formula in formulas:
stroke_counts = []
recordings = []
sql = ((
"SELECT `wm_raw_draw_data`.`id`, `data`, `is_in_testset`, "
"`wild_point_count`, `missing_line`, `user_id`, "
"`display_name` "
"FROM `wm_raw_draw_data` "
"JOIN `wm_users` ON "
"(`wm_users`.`id` = `wm_raw_draw_data`.`user_id`) "
"WHERE `accepted_formula_id` = %s "
"AND wild_point_count=0 "
"AND has_correction=0 "
# "AND `display_name` LIKE 'MfrDB::%%'"
) % str(formula['id']))
cursor.execute(sql)
raw_datasets = cursor.fetchall()
logging.info("%s (%i)", formula['formula_in_latex'], len(raw_datasets))
for raw_data in raw_datasets:
try:
handwriting = HandwrittenData(raw_data['data'], formula['id'],
raw_data['id'],
formula['formula_in_latex'],
raw_data['wild_point_count'],
raw_data['missing_line'],
raw_data['user_id'])
stroke_counts.append(len(handwriting.get_pointlist()))
recordings.append(handwriting)
except Exception as e:
logging.info("Raw data id: %s", raw_data['id'])
logging.info(e)
if len(stroke_counts) > 0:
logging.info("\t[%i - %i]", min(stroke_counts), max(stroke_counts))
median = numpy.median(stroke_counts)
logging.info("\tMedian: %0.2f\tMean: %0.2f\tstd: %0.2f", median,
numpy.mean(stroke_counts), numpy.std(stroke_counts))
# Make prob
s = sorted(Counter(stroke_counts).items(),
key=lambda n: n[1],
reverse=True)
key = formula['formula_in_latex']
prob[key] = {}
for stroke_nr, count in s:
prob[key][stroke_nr] = count
# Outliers
modes = get_modes(stroke_counts)
logging.info("\tModes: %s", modes)
exceptions = []
for rec in recordings:
if len(rec.get_pointlist()) not in modes:
url = (("http://www.martin-thoma.de/"
"write-math/view/?raw_data_id=%i - "
"%i strokes") %
(rec.raw_data_id, len(rec.get_pointlist())))
dist = get_dist(len(rec.get_pointlist()), modes)
exceptions.append((url, len(rec.get_pointlist()), dist))
print_exceptions(exceptions, max_print=10)
else:
logging.debug(
"No recordings for symbol "
"'http://www.martin-thoma.de/"
"write-math/symbol/?id=%s'.", formula['id'])
write_prob(prob, "prob_stroke_count_by_symbol.yml")