def enqueue_validation_sets_for_model(train_model_id): existing_results = db.get_all_validation_results( train_model_id=train_model_id) processed_val_model_ids = set([ r['validation_model_id'] for r in existing_results if r['image_subset'] == 'train' ]) n = 0 for validation_model in db.get_models(details=False, status=db.model_status_dataset)\ + db.get_models(details=False, status=db.model_status_trained): if validation_model['_id'] not in processed_val_model_ids: db.queue_validation(train_model_id=train_model_id, validation_model_id=validation_model['_id']) n += 1 return n
def admin_page(): num_images = db.get_sample_count() num_human_annotations = db.get_human_annotation_count() datasets = db.get_datasets() enqueued = db.get_unprocessed_samples() models = db.get_models(details=True) model_id_to_name = {m['_id']: m['name'] for m in models} secondary_items = db.get_queued_samples() enqueued2 = [] for item in secondary_items: model_name = model_id_to_name.get(item['model_id'], '???') if 'sample_id' in item: target_name = db.get_sample_by_id(item['sample_id'])['filename'] elif 'validation_model_id' in item: target_name = model_id_to_name.get(item['validation_model_id'], '?!?') else: target_name = '!!!' enqueued2.append((model_name, target_name, str(item['_id']))) enqueued2 = sorted(enqueued2, key=lambda item: item[0]) status = [(status_name, db.get_status(status_id)) for status_name, status_id in status_ids] return render_template('admin.html', num_images=num_images, num_human_annotations=num_human_annotations, datasets=datasets, enqueued=enqueued, status=status, error=pop_last_error(), models=models, enqueued2=enqueued2)
def enqueue_all_validation_sets(): n = 0 m = 0 for model in db.get_models(details=False, status=db.model_status_trained): nm = enqueue_validation_sets_for_model(model['_id']) if nm: n += nm m += 1 set_notice('Enqueued %d sets from %d models.' % (n, m)) return redirect('/admin')
def enqueue_all_images(): n = 0 m = 0 for model in db.get_models(details=False, status=db.model_status_trained): if not model['primary']: nm = enqueue_images_for_model(model['_id']) if nm: n += nm m += 1 set_notice('Enqueued %d images from %d models.' % (n, m)) return redirect('/admin')
def export_model_comparison(): models = db.get_models(status=db.model_status_trained) all_datasets = [ export_model_comparison_ds(get_all_samples(dataset['_id']), not i, models) for i, dataset in enumerate(db.get_datasets()) ] results = chain(*all_datasets) dataset_export_name, _ext = os.path.splitext( secure_filename('export_model_comparison.csv')) return Response(results, mimetype="text/plain", headers={ "Content-Disposition": "attachment;filename=%s" % dataset_export_name })
def run_daemon(): set_status('Daemon startup...') try: while True: scheduled_models = list(db.get_models(details=False, status=db.model_status_scheduled)) if len(scheduled_models): model = scheduled_models[0] set_status('Train model %s (%s)...' % (model['name'], str(model['_id']))) cmdline = list(sys.argv) + ['--model-id', str(model['_id'])] exec_path, exec_base_path = get_exec_path(model['_id']) log_filename = os.path.join(exec_path, 'train.log') cmdline = subprocess.list2cmdline(cmdline) + ' >' + log_filename + ' 2>&1' print 'Exec:', cmdline rval = subprocess.call(cmdline, shell=True) if rval: print 'ERRORED!' print open(log_filename, 'rt').read() set_status('Waiting for scheduled models...') time.sleep(1) finally: set_status('offline')
def worker_process(secondary=False): # Infinite worker process try: set_status('Startup...', secondary=secondary) model = None # First find network to load if secondary: set_status('Waiting for model/images...', secondary=secondary) while model is None: models = db.get_models(details=False, status=db.model_status_trained) for cmodel in models: if len(list(db.get_queued_samples(model_id=cmodel['_id']))): model = cmodel break if model is None: time.sleep(1) else: model = db.get_primary_model() set_status('Loading model %s...' % model['name'], secondary=secondary) net = load_model_by_record(model) # Then find samples to process while True: if secondary: process_secondary_models(net, model) # Secondary model: Always quit after processing is done; going back to model search. break else: process_primary_model(net, model) # Did the primary model change? if db.get_primary_model()['_id'] != model['_id']: break set_status('Waiting for images...', secondary=secondary) time.sleep(1) finally: set_status('offline', secondary=secondary) # Trigger restart. return True
def admin_models(): models = db.get_models(details=True) return render_template('admin_models.html', models=models, error=pop_last_error())