def test_finish(session):
modification_time = datetime.datetime.utcnow()
job_id = ops.create_new_job([], session)
job = session.query(model.Job).filter_by(id=job_id).first()
job.modification_time = datetime.datetime.utcnow() - datetime.timedelta(
hours=2)
session.commit()
ops.finish_job(job_id, 'result', session)
result = session.query(model.Job).first()
assert result.finished == 1
assert len(result.training_results) == 1
assert result.training_results[0].result == b'result'
assert abs(result.modification_time - modification_time) \
< datetime.timedelta(seconds=10)
def test_purge(session):
job_id_1 = ops.create_new_job([], session)
ops.finish_job(job_id_1, 'result', session)
job_id_2 = ops.create_new_job([], session)
ops.finish_job(job_id_2, 'result', session)
job_2 = session.query(model.Job).filter_by(id=job_id_2).first()
job_2.modification_time = datetime.datetime.utcnow() - datetime.timedelta(
hours=2)
session.commit()
job_id_3 = ops.create_new_job([], session)
job_3 = session.query(model.Job).filter_by(id=job_id_3).first()
job_3.modification_time = datetime.datetime.utcnow() - datetime.timedelta(
hours=2)
session.commit()
job_id_4 = ops.create_new_job([], session)
ops.finish_job(job_id_4, 'result', session)
job_4 = session.query(model.Job).filter_by(id=job_id_4).first()
job_4.modification_time = datetime.datetime.utcnow() + datetime.timedelta(
hours=2)
session.commit()
ops.purge_old_jobs(session)
jobs = session.query(model.Job).order_by(model.Job.modification_time).all()
assert len(jobs) == 3
assert jobs[0].id == job_id_3
assert jobs[1].id == job_id_1
assert jobs[2].id == job_id_4
def test_get_next_job(session):
job_id_1 = ops.create_new_job([], session)
ops.finish_job(job_id_1, 'result', session)
job_id_2 = ops.create_new_job([], session)
ops.finish_job(job_id_2, 'result', session)
job_2 = session.query(model.Job).filter_by(id=job_id_2).first()
job_2.creation_time = datetime.datetime.utcnow() - datetime.timedelta(
hours=2)
session.commit()
job_id_3 = ops.create_new_job([], session)
job_2 = session.query(model.Job).filter_by(id=job_id_3).first()
job_2.creation_time = datetime.datetime.utcnow() - datetime.timedelta(
hours=2)
session.commit()
job_id_4 = ops.create_new_job([], session)
ops.finish_job(job_id_4, 'result', session)
job_4 = session.query(model.Job).filter_by(id=job_id_4).first()
job_4.creation_time = datetime.datetime.utcnow() + datetime.timedelta(
hours=2)
session.commit()
job = ops.get_next_job(session)
assert job.id == job_id_3
def trainer():
"""Thread that performs the actual model training."""
session = Session()
log = logger.new()
while True:
log.debug('trainer::job::wait_for_next')
while True:
next_job = _database_operations.get_next_job(session)
if next_job is None:
time.sleep(1)
continue
else:
break
job_log = log.bind(job_id=next_job.id)
job = next_job
if not job.training_jobs:
job_log.warn('trainer::job::no_training_job')
continue
job_log.info('trainer::job::job_start')
for i, tj in enumerate(job.training_jobs):
subjob_log = job_log.bind(subjob_type='training', subjob=i)
subjob_log.info('trainer::job::subjob_start')
training_data = str(tj.training_job, 'utf8')
training_request = extract_training_request(
json.loads(training_data))
callback = StatusCallback(job.id, session)
try:
result = execute_training_request(training_request, callback)
_database_operations.finish_job(job.id, result, session)
except:
subjob_log.error('trainer::job::error', exc_info=True)
_database_operations.update_status(job.id, -1, session)
_database_operations.finish_job(job.id, traceback.format_exc(),
session)
subjob_log.info('trainer::job::subjob_finished')
job_log.info('trainer::job::job_finished')
def test_get_results(session):
job_id = ops.create_new_job([], session)
ops.finish_job(job_id, 'result', session)
result = ops.get_results(job_id, session)
assert result == 'result'