def enqueue_validation_sets_for_model(train_model_id):
existing_results = db.get_all_validation_results(
train_model_id=train_model_id)
processed_val_model_ids = set([
r['validation_model_id'] for r in existing_results
if r['image_subset'] == 'train'
])
n = 0
for validation_model in db.get_models(details=False, status=db.model_status_dataset)\
+ db.get_models(details=False, status=db.model_status_trained):
if validation_model['_id'] not in processed_val_model_ids:
db.queue_validation(train_model_id=train_model_id,
validation_model_id=validation_model['_id'])
n += 1
return n
def admin_page():
num_images = db.get_sample_count()
num_human_annotations = db.get_human_annotation_count()
datasets = db.get_datasets()
enqueued = db.get_unprocessed_samples()
models = db.get_models(details=True)
model_id_to_name = {m['_id']: m['name'] for m in models}
secondary_items = db.get_queued_samples()
enqueued2 = []
for item in secondary_items:
model_name = model_id_to_name.get(item['model_id'], '???')
if 'sample_id' in item:
target_name = db.get_sample_by_id(item['sample_id'])['filename']
elif 'validation_model_id' in item:
target_name = model_id_to_name.get(item['validation_model_id'],
'?!?')
else:
target_name = '!!!'
enqueued2.append((model_name, target_name, str(item['_id'])))
enqueued2 = sorted(enqueued2, key=lambda item: item[0])
status = [(status_name, db.get_status(status_id))
for status_name, status_id in status_ids]
return render_template('admin.html',
num_images=num_images,
num_human_annotations=num_human_annotations,
datasets=datasets,
enqueued=enqueued,
status=status,
error=pop_last_error(),
models=models,
enqueued2=enqueued2)
def enqueue_all_validation_sets():
n = 0
m = 0
for model in db.get_models(details=False, status=db.model_status_trained):
nm = enqueue_validation_sets_for_model(model['_id'])
if nm:
n += nm
m += 1
set_notice('Enqueued %d sets from %d models.' % (n, m))
return redirect('/admin')
def enqueue_all_images():
n = 0
m = 0
for model in db.get_models(details=False, status=db.model_status_trained):
if not model['primary']:
nm = enqueue_images_for_model(model['_id'])
if nm:
n += nm
m += 1
set_notice('Enqueued %d images from %d models.' % (n, m))
return redirect('/admin')
def export_model_comparison():
models = db.get_models(status=db.model_status_trained)
all_datasets = [
export_model_comparison_ds(get_all_samples(dataset['_id']), not i,
models)
for i, dataset in enumerate(db.get_datasets())
]
results = chain(*all_datasets)
dataset_export_name, _ext = os.path.splitext(
secure_filename('export_model_comparison.csv'))
return Response(results,
mimetype="text/plain",
headers={
"Content-Disposition":
"attachment;filename=%s" % dataset_export_name
})
def run_daemon():
set_status('Daemon startup...')
try:
while True:
scheduled_models = list(db.get_models(details=False, status=db.model_status_scheduled))
if len(scheduled_models):
model = scheduled_models[0]
set_status('Train model %s (%s)...' % (model['name'], str(model['_id'])))
cmdline = list(sys.argv) + ['--model-id', str(model['_id'])]
exec_path, exec_base_path = get_exec_path(model['_id'])
log_filename = os.path.join(exec_path, 'train.log')
cmdline = subprocess.list2cmdline(cmdline) + ' >' + log_filename + ' 2>&1'
print 'Exec:', cmdline
rval = subprocess.call(cmdline, shell=True)
if rval:
print 'ERRORED!'
print open(log_filename, 'rt').read()
set_status('Waiting for scheduled models...')
time.sleep(1)
finally:
set_status('offline')
def worker_process(secondary=False):
# Infinite worker process
try:
set_status('Startup...', secondary=secondary)
model = None
# First find network to load
if secondary:
set_status('Waiting for model/images...', secondary=secondary)
while model is None:
models = db.get_models(details=False, status=db.model_status_trained)
for cmodel in models:
if len(list(db.get_queued_samples(model_id=cmodel['_id']))):
model = cmodel
break
if model is None:
time.sleep(1)
else:
model = db.get_primary_model()
set_status('Loading model %s...' % model['name'], secondary=secondary)
net = load_model_by_record(model)
# Then find samples to process
while True:
if secondary:
process_secondary_models(net, model)
# Secondary model: Always quit after processing is done; going back to model search.
break
else:
process_primary_model(net, model)
# Did the primary model change?
if db.get_primary_model()['_id'] != model['_id']:
break
set_status('Waiting for images...', secondary=secondary)
time.sleep(1)
finally:
set_status('offline', secondary=secondary)
# Trigger restart.
return True
def admin_models():
models = db.get_models(details=True)
return render_template('admin_models.html',
models=models,
error=pop_last_error())
