def examples_page():
# Find the example dataset
example_dataset = db.get_example_dataset()
if example_dataset is None:
set_error('No example dataset defined.')
return redirect('/')
return redirect('/dataset/%s' % example_dataset['_id'])
def delete_dataset(dataset_id_str):
dataset_id = ObjectId(dataset_id_str)
dataset_info = db.get_dataset_by_id(dataset_id)
if dataset_info is None:
return render_template("404.html")
if db.is_readonly_dataset(dataset_info):
set_error('Dataset is protected.')
return redirect('/dataset/' + str(dataset_info['_id']))
db.delete_dataset(dataset_id)
set_notice('Dataset "%s" deleted.' % dataset_info['name'])
return redirect('/')
def add_dataset():
# Add by name. Forward to newly created dataset
dataset_name = request.form['dataset_name'].strip()
if dataset_name == '':
set_error('Invalid dataset name.')
return redirect('/')
dataset_info = db.get_dataset_by_name(dataset_name)
if dataset_info is not None:
set_error('Duplicate dataset name.')
return redirect('/')
dataset_info = db.add_dataset(dataset_name, user_id=get_current_user_id())
return redirect('/dataset/' + str(dataset_info['_id']) + '?new=true')
def add_uploaded_image(dataset_id, full_fn, filename):
# Get some image info
try:
im = Image.open(full_fn)
except:
print 'Invalid image file: %s' % full_fn
set_error('Could not load image. Invalid / Upload error?')
return None
# Add DB entry (after file save to worker can pick it up immediately)
entry = db.add_sample(name=filename, filename=os.path.basename(full_fn), size=im.size, dataset_id=dataset_id)
# Return added entry
return entry
def save_annotations(sid):
sample_id = ObjectId(sid)
sample_entry = db.get_sample_by_id(sample_id)
# Caution: Load dataset id from DB (not from form), for security of the readonly check!
# dataset_id = ObjectId(request.form['dataset_id'].strip())
dataset_id = sample_entry['dataset_id']
readonly = db.is_readonly_dataset_id(dataset_id)
if readonly:
set_error('Dataset is protected.')
return redirect('/dataset/' + str(dataset_id))
is_differential = request.args.get('differential')
if is_differential:
base_annotations = db.get_machine_annotations(sample_id)
redirect_params = '?differential=1'
else:
base_annotations = None
redirect_params = ''
# Save annotations for sample
annotations = json.loads(request.form['annotations'].strip())
margin = int(request.form['margin'].strip())
print 'Saving annotations.', sid, margin, annotations
db.set_human_annotation(sample_id,
get_current_user_id(),
annotations,
margin,
base_annotations=base_annotations)
# Forward either to info page or to annotation of next un-annotated entry in DB if found
annotate_next = ("save_and_continue" in request.form)
if annotate_next:
next_sample_id = db.get_next_sample_id(dataset_id,
sample_id,
annotated=False)
if next_sample_id is not None:
return redirect('/annotate/' + str(next_sample_id) +
redirect_params)
else:
set_error('No more samples to annotate.')
return redirect('/dataset/' + str(dataset_id))
# Nothing
return redirect('/info/' + sid)
def delete_entry(str_id):
# For return path
id = ObjectId(str_id)
sample = db.get_sample_by_id(id)
if sample is None:
set_error('Item to delete not found.')
return redirect('/')
dataset_id = sample['dataset_id']
readonly = db.is_readonly_dataset_id(dataset_id)
sample_index, sample_count, prev_sample_id, next_sample_id = db.get_sample_index(dataset_id, id)
if readonly:
set_error('Dataset is protected.')
elif db.delete_sample(id):
set_notice('Item deleted.')
else:
set_error('Could not delete item.')
# Redirect: To next sample in set if it exists. Otherwise to the dataset page.
if next_sample_id is not None and next_sample_id != id:
# Return to same kind of page (info/annotation/diff_annotation)
is_annotation = request.args.get('annotate')
is_differential = request.args.get('differential')
if is_annotation:
suffix = '?differential=1' if is_differential else ''
return redirect('/annotate/' + str(next_sample_id) + suffix)
else:
return redirect('/info/' + str(next_sample_id))
else:
return redirect('/dataset/' + str(dataset_id))
def dataset_info(dataset_id_str):
print 'request.method', request.method
if dataset_id_str == 'new' and request.method == 'POST':
dataset_id = None
dataset_info = None
new_dataset_zoom = request.form['size']
else:
dataset_id = ObjectId(dataset_id_str)
dataset_info = db.get_dataset_by_id(dataset_id)
new_dataset_zoom = None
if dataset_info is None:
return render_template("404.html")
if request.method == 'POST':
# File upload
if dataset_info is not None:
if db.is_readonly_dataset(dataset_info):
set_error('Dataset is protected.')
return redirect('/dataset/' + dataset_id_str)
return upload_file(dataset_id, image_zoom=new_dataset_zoom)
enqueued = db.get_unprocessed_samples(dataset_id=dataset_id)
finished = db.get_processed_samples(dataset_id=dataset_id)
for i, sample in enumerate(finished):
sample['machine_distance'] = 1.0 / max(
[0.001, sqrt(float(sample['machine_position_count']))])
sample['index'] = i
errored = db.get_error_samples(dataset_id=dataset_id)
# Get request data
return render_template("dataset.html",
dataset_name=dataset_info['name'],
dataset_id=dataset_id_str,
enqueued=enqueued,
finished=finished,
errored=errored,
status=db.get_status('worker'),
readonly=db.is_readonly_dataset(dataset_info),
error=pop_last_error(),
dataset_user=dataset_info.get('user'),
image_zoom=dataset_info.get('image_zoom',
'default'))
def admin_retrain():
data = request.form
model_name = data['train_model_name']
# Race condition here; but it's just for the admin page anyway
if re.match('^[\w-]+$', model_name) is None:
set_error(
'Invalid model name. Must be non-empty, only alphanumeric characters, dashes and underscores.'
)
return redirect('/admin')
if db.get_model_by_name(model_name):
set_error('Model exists. Please pick a different name.')
return redirect('/admin')
# Validate tag
tag_name = data['train_label']
dsets = list(db.get_datasets_by_tag(tag_name))
if not len(dsets):
set_error('Train tag does not match any datasets.')
return redirect('/admin')
is_primary = (data.get('train_primary') == 'on')
dataset_only = (data.get('dataset_only') == 'on')
train_sample_limit_s = data['train_sample_limit']
if len(train_sample_limit_s):
try:
train_sample_limit = int(train_sample_limit_s)
if train_sample_limit <= 0:
raise ValueError()
except ValueError as e:
set_error('Invalid sample limit. Either leave empty for no limit, '
'or supply a valid number greater than zero.')
return redirect('/admin')
else:
train_sample_limit = None
# Add scheduled model training to DB. Will be picked up by worker.
rec = db.add_model(model_name=model_name,
margin=96,
sample_limit=train_sample_limit,
train_tag=tag_name,
scheduled_primary=is_primary,
status=db.model_status_scheduled,
dataset_only=dataset_only)
set_notice('Model training scheduled.')
return redirect('/model/' + str(rec['_id']))
def do_claim_dataset(dataset_id_str, ignore_errors):
dataset_id = ObjectId(dataset_id_str)
if current_user is None:
if ignore_errors:
return
set_error('Not logged in.')
return redirect('/dataset/' + dataset_id_str)
dataset = db.get_dataset_by_id(dataset_id)
if dataset is None:
if ignore_errors:
return
set_error('Dataset not found.')
return redirect('/user_datasets')
if dataset.get('user') is not None:
if ignore_errors:
return
set_error('Dataset already owned by %s.' %
dataset['user'].get('email'))
return redirect('/dataset/' + dataset_id_str)
db.set_dataset_user(dataset_id, current_user.id)
return redirect('/user_datasets')
def dataset_info(dataset_id_str):
print 'request.method', request.method
new_dataset_threshold_prob = None
new_allow_reuse = False
if dataset_id_str == 'new':
print 'Creating new dataset'
if request.method != 'POST':
return redirect('/')
dataset_id = None
dataset_info = None
new_dataset_zoom = request.form['size']
print 'Threshold prob:'
print request.form['threshold']
try:
v = float(request.form['threshold'])
new_dataset_threshold_prob = min(max(v, 0.5), 1.0)
print 'Specified thresh prob:', new_dataset_threshold_prob
except ValueError:
print 'Invalid threshold. Ignored.'
try:
new_allow_reuse = bool(request.form.get('reuseCheck'))
print 'Specified allow reuse:', request.form.get(
'reuseCheck'), new_allow_reuse
except ValueError:
print 'Invalid reuse setting. Ignored.'
else:
dataset_id = ObjectId(dataset_id_str)
db.access_dataset(dataset_id)
dataset_info = db.get_dataset_by_id(dataset_id)
new_dataset_zoom = None
if dataset_info is None:
return render_template("404.html")
if request.method == 'POST':
# File upload
if dataset_info is not None:
if db.is_readonly_dataset(dataset_info):
set_error('Dataset is protected.')
return redirect('/dataset/' + dataset_id_str)
return upload_file(dataset_id,
image_zoom=new_dataset_zoom,
threshold_prob=new_dataset_threshold_prob,
allow_reuse=new_allow_reuse)
enqueued = db.get_unprocessed_samples(dataset_id=dataset_id)
finished = db.get_processed_samples(dataset_id=dataset_id)
for i, sample in enumerate(finished):
sample['machine_distance'] = 1.0 / max(
[0.001, sqrt(float(sample['machine_position_count']))])
sample['index'] = i
errored = db.get_error_samples(dataset_id=dataset_id)
threshold_prob = round(dataset_info.get('threshold_prob')
or fc8_to_prob(default_prob_threshold),
ndigits=3)
# Get request data
return render_template("dataset.html",
dataset_name=dataset_info['name'],
dataset_id=dataset_id_str,
enqueued=enqueued,
finished=finished,
errored=errored,
status=db.get_status('worker'),
readonly=db.is_readonly_dataset(dataset_info),
error=pop_last_error(),
dataset_user=dataset_info.get('user'),
image_zoom=dataset_info.get('image_zoom',
'default'),
threshold_prob=threshold_prob)