def parse_directory(self):
self.parser = ScienceLogParser(database=self)
self._delete_indexes() # Takes too long while parsing.
filenames = [os.path.join(self.log_dir, filename) for filename in \
sorted(os.listdir(unicode(self.log_dir))) if \
filename.endswith(".bz2")]
filenames_count = len(filenames)
for counter, filename in enumerate(filenames):
sys.stdout.flush()
if self.con.execute(\
"select log_name from parsed_logs where parsed_logs.log_name=?",
(os.path.basename(filename), )).fetchone() is not None:
print "(%d/%d) %1.1f%% %s already parsed" % \
(counter + 1, filenames_count,
(counter + 1.) / filenames_count * 100, \
os.path.basename(filename))
continue
print "(%d/%d) %1.1f%% %s" % (counter + 1, filenames_count,
(counter + 1.) / filenames_count * 100, \
os.path.basename(filename))
try:
self.parser.parse(filename)
except KeyboardInterrupt:
print "Interrupted!"
self.con.commit()
exit()
except:
print "Can't open file, ignoring."
self.con.execute("insert into parsed_logs(log_name) values(?)",
(os.path.basename(filename), ))
self.con.commit()
self._create_indexes()
def test_logs_imported_1(self):
self.database().update_card_after_log_import = (lambda x, y, z: 0)
self.database().before_1x_log_import()
filename = os.path.join(os.getcwd(), "tests", "files", "imported_1.txt")
ScienceLogParser(self.database()).parse(filename)
assert self.database().con.execute(\
"select count() from log where event_type=?",
(EventTypes.ADDED_CARD, )).fetchone()[0] == 1
assert self.database().con.execute(\
"select count() from log where event_type=?",
(EventTypes.REPETITION, )).fetchone()[0] == 3
assert self.database().con.execute(\
"select acq_reps from log where event_type=? and object_id='f5d9bbe7'",
(EventTypes.REPETITION, )).fetchone()[0] == 1
assert self.database().con.execute(\
"select ret_reps from log where event_type=? and object_id='f5d9bbe7'",
(EventTypes.REPETITION, )).fetchone()[0] == 0
assert self.database().con.execute(\
"select acq_reps_since_lapse from log where event_type=? and object_id='f5d9bbe7'",
(EventTypes.REPETITION, )).fetchone()[0] == 1
assert self.database().con.execute(\
"""select acq_reps from log where event_type=? and object_id='f5d9bbe7'
order by _id desc limit 1""",
(EventTypes.REPETITION, )).fetchone()[0] == 1
assert self.database().con.execute(\
"""select ret_reps from log where event_type=? and object_id='f5d9bbe7'
order by _id desc limit 1""",
(EventTypes.REPETITION, )).fetchone()[0] == 2
assert self.database().con.execute(\
"""select acq_reps_since_lapse from log where event_type=? and object_id='f5d9bbe7'
order by _id desc limit 1""",
(EventTypes.REPETITION, )).fetchone()[0] == 1
def test_restored_1(self):
self.database().update_card_after_log_import = (lambda x, y, z: 0)
self.database().before_1x_log_import()
filename = os.path.join(os.getcwd(), "tests", "files", "restored_1.txt")
ScienceLogParser(self.database()).parse(filename)
assert self.database().con.execute(\
"select count() from log where event_type=?",
(EventTypes.ADDED_CARD, )).fetchone()[0] == 1
assert self.database().con.execute(\
"select count() from log where event_type=?",
(EventTypes.REPETITION, )).fetchone()[0] == 1
sql_res = self.database().con.execute(\
"select * from log where event_type=?",
(EventTypes.REPETITION, )).fetchone()
assert sql_res[4] == 1
assert sql_res[5] == 2.36
assert sql_res[6] == 23
assert sql_res[7] == 8
assert sql_res[8] == 2
assert sql_res[9] == 0
assert sql_res[10] == 0
assert sql_res[11] == 89 * 24 * 60 * 60
assert sql_res[12] == 0 # No last rep data.
assert sql_res[14] - sql_res[2] == 0
assert sql_res[13] == 5
def test_logs_new_5(self):
self.database().update_card_after_log_import = (lambda x, y, z: 0)
self.database().before_1x_log_import()
filename = os.path.join(os.getcwd(), "tests", "files", "new_5.txt")
ScienceLogParser(self.database()).parse(filename)
assert self.database().con.execute(\
"select count() from log where event_type=?",
(EventTypes.ADDED_CARD, )).fetchone()[0] == 1
assert self.database().con.execute(\
"select count() from log where event_type=?",
(EventTypes.REPETITION, )).fetchone()[0] == 2
assert self.database().con.execute(\
"select acq_reps from log where event_type=? and object_id='9c8ce28e-1a4b-4148-8287-b8a7790d86d0.1.1'",
(EventTypes.REPETITION, )).fetchone()[0] == 1
assert self.database().con.execute(\
"select ret_reps from log where event_type=? and object_id='9c8ce28e-1a4b-4148-8287-b8a7790d86d0.1.1'",
(EventTypes.REPETITION, )).fetchone()[0] == 0
assert self.database().con.execute(\
"select acq_reps_since_lapse from log where event_type=? and object_id='9c8ce28e-1a4b-4148-8287-b8a7790d86d0.1.1'",
(EventTypes.REPETITION, )).fetchone()[0] == 1
assert self.database().con.execute(\
"""select acq_reps from log where event_type=? and object_id='9c8ce28e-1a4b-4148-8287-b8a7790d86d0.1.1'
order by _id desc limit 1""",
(EventTypes.REPETITION, )).fetchone()[0] == 2
assert self.database().con.execute(\
"""select ret_reps from log where event_type=? and object_id='9c8ce28e-1a4b-4148-8287-b8a7790d86d0.1.1'
order by _id desc limit 1""",
(EventTypes.REPETITION, )).fetchone()[0] == 0
assert self.database().con.execute(\
"""select acq_reps_since_lapse from log where event_type=? and object_id='9c8ce28e-1a4b-4148-8287-b8a7790d86d0.1.1'
order by _id desc limit 1""",
(EventTypes.REPETITION, )).fetchone()[0] == 2
assert self.database().con.execute(\
"""select object_id from log where event_type=?""",
(EventTypes.STARTED_SCHEDULER, )).fetchone()[0] == "SM2 Mnemosyne"
def test_restored_2(self):
self.database().update_card_after_log_import = (lambda x, y, z: 0)
self.database().before_1x_log_import()
filename = os.path.join(os.getcwd(), "tests", "files", "restored_2.txt")
ScienceLogParser(self.database()).parse(filename)
assert self.database().con.execute(\
"select count() from log where event_type=?",
(EventTypes.ADDED_CARD, )).fetchone()[0] == 1
def test_past_schedule(self):
self.database().update_card_after_log_import = (lambda x, y, z: 0)
self.database().before_1x_log_import()
filename = os.path.join(os.getcwd(), "tests", "files", "schedule_1.txt")
ScienceLogParser(self.database()).parse(filename)
days_elapsed = datetime.date.today() - datetime.date(2009, 8, 15)
assert self.scheduler().card_count_scheduled_n_days_from_now(\
-days_elapsed.days) == 124
assert self.scheduler().card_count_scheduled_n_days_from_now(-1) == 0
def test_logs_act_interval(self):
self.database().update_card_after_log_import = (lambda x, y, z: 0)
self.database().before_1x_log_import()
filename = os.path.join(os.getcwd(), "tests", "files", "actinterval_1.txt")
ScienceLogParser(self.database()).parse(filename)
assert self.database().con.execute(\
"""select actual_interval from log where event_type=? and object_id='f1300e5a'
order by _id desc limit 1""",
(EventTypes.REPETITION, )).fetchone()[0] == 5
def test_logs_corrupt_2(self): # Wrong data, isolated deletion event.
self.database().update_card_after_log_import = (lambda x, y, z: 0)
self.database().before_1x_log_import()
filename = os.path.join(os.getcwd(), "tests", "files", "corrupt_2.txt")
ScienceLogParser(self.database()).parse(filename)
assert self.database().con.execute(\
"select count() from log where event_type=?",
(EventTypes.ADDED_CARD, )).fetchone()[0] == 0
assert self.database().con.execute(\
"select count() from log where object_id=?",
("4b59b830", )).fetchone()[0] == 0
def test_logs_new_1(self):
self.database().update_card_after_log_import = (lambda x, y, z: 0)
self.database().before_1x_log_import()
filename = os.path.join(os.getcwd(), "tests", "files", "new_1.txt")
ScienceLogParser(self.database()).parse(filename)
assert self.database().con.execute(\
"select count() from log where event_type=?",
(EventTypes.ADDED_CARD, )).fetchone()[0] == 1
assert self.database().con.execute(\
"select count() from log where event_type=?",
(EventTypes.REPETITION, )).fetchone()[0] == 10
assert self.database().con.execute(\
"select acq_reps from log where event_type=? and object_id='9525224f'",
(EventTypes.REPETITION, )).fetchone()[0] == 1
assert self.database().con.execute(\
"select acq_reps_since_lapse from log where event_type=? and object_id='9525224f'",
(EventTypes.REPETITION, )).fetchone()[0] == 1
assert self.database().con.execute(\
"""select scheduled_interval from log where event_type=? and object_id='9525224f'
order by _id desc limit 1""",
(EventTypes.REPETITION, )).fetchone()[0] == (6)*60*60*24
assert self.database().con.execute(\
"""select actual_interval from log where event_type=? and object_id='9525224f'
order by _id desc limit 1""",
(EventTypes.REPETITION, )).fetchone()[0] == 0 # This is an artificial log.
timestamp = self.database().con.execute(\
"""select timestamp from log where event_type=? and object_id='9525224f'
order by _id desc limit 1""",
(EventTypes.REPETITION, )).fetchone()[0]
next_rep = self.database().con.execute(\
"""select next_rep from log where event_type=? and object_id='9525224f'
order by _id desc limit 1""",
(EventTypes.REPETITION, )).fetchone()[0]
assert next_rep - timestamp == (14-3)*60*60*24
assert self.database().con.execute(\
"select count() from log").fetchone()[0] == 25
assert self.database().con.execute(\
"select acq_reps from log where event_type=? order by _id desc limit 1",
(EventTypes.LOADED_DATABASE, )).fetchone()[0] == 0
assert self.database().con.execute(\
"select ret_reps from log where event_type=? order by _id desc limit 1",
(EventTypes.LOADED_DATABASE, )).fetchone()[0] == 7
assert self.database().con.execute(\
"select lapses from log where event_type=? order by _id desc limit 1",
(EventTypes.LOADED_DATABASE, )).fetchone()[0] == 336
assert self.database().con.execute(\
"select acq_reps from log where event_type=? order by _id desc limit 1",
(EventTypes.SAVED_DATABASE, )).fetchone()[0] == 0
assert self.database().con.execute(\
"select ret_reps from log where event_type=? order by _id desc limit 1",
(EventTypes.SAVED_DATABASE, )).fetchone()[0] == 12
assert self.database().con.execute(\
"select lapses from log where event_type=? order by _id desc limit 1",
(EventTypes.SAVED_DATABASE, )).fetchone()[0] == 341
def test_score(self):
self.database().update_card_after_log_import = (lambda x, y, z: 0)
self.database().before_1x_log_import()
filename = os.path.join(os.getcwd(), "tests", "files", "score_1.txt")
ScienceLogParser(self.database()).parse(filename)
days_elapsed = datetime.date.today() - datetime.date(2009, 8, 17)
assert self.database().retention_score_n_days_ago(days_elapsed.days) \
== 5/7.*100
assert self.database().retention_score_n_days_ago(0) == 0
from mnemosyne.libmnemosyne.statistics_pages.retention_score import RetentionScore
page = RetentionScore(self.mnemosyne.component_manager)
for i in range(1, 6):
page.prepare_statistics(i)
def test_logs_new_2(self):
self.database().update_card_after_log_import = (lambda x, y, z: 0)
self.database().before_1x_log_import()
filename = os.path.join(os.getcwd(), "tests", "files", "new_2.txt")
ScienceLogParser(self.database()).parse(filename)
assert self.database().con.execute(\
"select count() from log where event_type=?",
(EventTypes.ADDED_CARD, )).fetchone()[0] == 1
assert self.database().con.execute(\
"select count() from log where event_type=?",
(EventTypes.REPETITION, )).fetchone()[0] == 1
assert self.database().con.execute(\
"select acq_reps from log where event_type=? and object_id='8da62cfb'",
(EventTypes.REPETITION, )).fetchone()[0] == 1
assert self.database().con.execute(\
"select acq_reps_since_lapse from log where event_type=? and object_id='8da62cfb'",
(EventTypes.REPETITION, )).fetchone()[0] == 1
def import_logs(self, filename):
w = self.main_widget()
db = self.database()
w.set_progress_text(_("Importing history..."))
log_dir = os.path.join(os.path.dirname(filename), "history")
if not os.path.exists(log_dir):
w.close_progress()
w.show_information(_("No history found to import."))
return
# The events that we import from the science logs obviously should not
# be reexported to these logs (this is true for both the archived logs
# and log.txt). So, before the import, we flush the SQL logs to the
# science logs, and after the import we edit the partership index to
# skip these entries.
db.dump_to_science_log()
# Manage database indexes.
db.before_1x_log_import()
filenames = [os.path.join(log_dir, logname) for logname in \
sorted(os.listdir(log_dir)) if logname.endswith(".bz2")]
# log.txt can also contain data we need to import, especially on the
# initial upgrade from 1.x. 'ids_to_parse' will make sure we only pick
# up the relevant events. (If we do the importing after having used
# 2.x for a while, there could be duplicate load events, etc, but these
# don't matter.)
filenames.append(os.path.join(os.path.dirname(filename), "log.txt"))
w.set_progress_range(len(filenames))
ignored_files = []
parser = ScienceLogParser(self.database(),
ids_to_parse=self.items_by_id,
machine_id=self.config().machine_id())
for filename in filenames:
try:
parser.parse(filename)
except:
ignored_files.append(filename)
w.increase_progress(1)
if ignored_files:
w.show_information(_("Ignoring unparsable files:<br/>") +\
'<br/>'.join(ignored_files))
# Manage database indexes.
db.after_1x_log_import()
db.skip_science_log()
def test_logs_new_6(self):
self.database().update_card_after_log_import = (lambda x, y, z: 0)
self.database().before_1x_log_import()
filename = os.path.join(os.getcwd(), "tests", "files", "new_6.txt")
ScienceLogParser(self.database()).parse(filename)
assert self.database().con.execute(\
"select count() from log where event_type=?",
(EventTypes.ADDED_CARD, )).fetchone()[0] == 1
assert self.database().con.execute(\
"select count() from log where event_type=?",
(EventTypes.REPETITION, )).fetchone()[0] == 2
sql_res = self.database().con.execute(\
"select * from log where event_type=? and object_id='4c53e29a-f9e9-498b-8beb-d3a494f61bca.1.1'",
(EventTypes.REPETITION, )).fetchone()
assert sql_res[4] == 5
assert sql_res[5] == 2.5
assert sql_res[6] == 1
assert sql_res[7] == 0
assert sql_res[8] == 0
assert sql_res[9] == 1
assert sql_res[10] == 0
assert sql_res[11] == 0
assert sql_res[12] == 0
assert sql_res[14] - sql_res[2] == 345600
assert sql_res[13] == 0
sql_res = self.database().con.execute(\
"""select * from log where event_type=? and object_id='4c53e29a-f9e9-498b-8beb-d3a494f61bca.1.1'
order by _id desc limit 1""",
(EventTypes.REPETITION, )).fetchone()
assert sql_res[4] == 2
assert sql_res[5] == 2.5
assert sql_res[6] == 1
assert sql_res[7] == 1
assert sql_res[8] == 0
assert sql_res[9] == 1
assert sql_res[10] == 1
assert sql_res[11] == 302986
assert sql_res[12] == 10
assert sql_res[14] - sql_res[2] == 475774
assert sql_res[13] == 1
def __init__(self, log_dir):
self.log_dir = log_dir
self.parser = ScienceLogParser(database=self)
self.parsed_logs = []
self.log = []
