def image_classification_model_create():
"""
Create a new ImageClassificationModelJob
Returns JSON when requested: {job_id,name,status} or {errors:[]}
"""
form = ImageClassificationModelForm()
form.dataset.choices = get_datasets()
form.standard_networks.choices = get_standard_networks()
form.standard_networks.default = get_default_standard_network()
form.previous_networks.choices = get_previous_networks()
prev_network_snapshots = get_previous_network_snapshots()
if not form.validate_on_submit():
if request_wants_json():
return flask.jsonify({"errors": form.errors}), 400
else:
return (
flask.render_template(
"models/images/classification/new.html",
form=form,
previous_network_snapshots=prev_network_snapshots,
multi_gpu=config_value("caffe_root")["multi_gpu"],
),
400,
)
datasetJob = scheduler.get_job(form.dataset.data)
if not datasetJob:
raise werkzeug.exceptions.BadRequest('Unknown dataset job_id "%s"' % form.dataset.data)
job = None
try:
job = ImageClassificationModelJob(name=form.model_name.data, dataset_id=datasetJob.id())
network = caffe_pb2.NetParameter()
pretrained_model = None
if form.method.data == "standard":
found = False
networks_dir = os.path.join(os.path.dirname(digits.__file__), "standard-networks")
for filename in os.listdir(networks_dir):
path = os.path.join(networks_dir, filename)
if os.path.isfile(path):
match = re.match(r"%s.prototxt" % form.standard_networks.data, filename)
if match:
with open(path) as infile:
text_format.Merge(infile.read(), network)
found = True
break
if not found:
raise werkzeug.exceptions.BadRequest('Unknown standard model "%s"' % form.standard_networks.data)
elif form.method.data == "previous":
old_job = scheduler.get_job(form.previous_networks.data)
if not old_job:
raise werkzeug.exceptions.BadRequest("Job not found: %s" % form.previous_networks.data)
network.CopyFrom(old_job.train_task().network)
# Rename the final layer
# XXX making some assumptions about network architecture here
ip_layers = [l for l in network.layer if l.type == "InnerProduct"]
if len(ip_layers) > 0:
ip_layers[-1].name = "%s_retrain" % ip_layers[-1].name
for choice in form.previous_networks.choices:
if choice[0] == form.previous_networks.data:
epoch = float(flask.request.form["%s-snapshot" % form.previous_networks.data])
if epoch != 0:
for filename, e in old_job.train_task().snapshots:
if e == epoch:
pretrained_model = filename
break
if pretrained_model is None:
raise werkzeug.exceptions.BadRequest(
"For the job %s, selected pretrained_model for epoch %d is invalid!"
% (form.previous_networks.data, epoch)
)
if not (os.path.exists(pretrained_model)):
raise werkzeug.exceptions.BadRequest(
"Pretrained_model for the selected epoch doesn't exists. May be deleted by another user/process. Please restart the server to load the correct pretrained_model details"
)
break
elif form.method.data == "custom":
text_format.Merge(form.custom_network.data, network)
pretrained_model = form.custom_network_snapshot.data.strip()
else:
raise werkzeug.exceptions.BadRequest('Unrecognized method: "%s"' % form.method.data)
policy = {"policy": form.lr_policy.data}
if form.lr_policy.data == "fixed":
pass
elif form.lr_policy.data == "step":
policy["stepsize"] = form.lr_step_size.data
policy["gamma"] = form.lr_step_gamma.data
elif form.lr_policy.data == "multistep":
policy["stepvalue"] = form.lr_multistep_values.data
policy["gamma"] = form.lr_multistep_gamma.data
elif form.lr_policy.data == "exp":
policy["gamma"] = form.lr_exp_gamma.data
elif form.lr_policy.data == "inv":
policy["gamma"] = form.lr_inv_gamma.data
policy["power"] = form.lr_inv_power.data
elif form.lr_policy.data == "poly":
policy["power"] = form.lr_poly_power.data
elif form.lr_policy.data == "sigmoid":
policy["stepsize"] = form.lr_sigmoid_step.data
policy["gamma"] = form.lr_sigmoid_gamma.data
else:
raise werkzeug.exceptions.BadRequest("Invalid learning rate policy")
if config_value("caffe_root")["multi_gpu"]:
if form.select_gpu_count.data:
gpu_count = form.select_gpu_count.data
selected_gpus = None
else:
selected_gpus = [str(gpu) for gpu in form.select_gpus.data]
gpu_count = None
else:
if form.select_gpu.data == "next":
gpu_count = 1
selected_gpus = None
else:
selected_gpus = [str(form.select_gpu.data)]
gpu_count = None
job.tasks.append(
tasks.CaffeTrainTask(
job_dir=job.dir(),
dataset=datasetJob,
train_epochs=form.train_epochs.data,
snapshot_interval=form.snapshot_interval.data,
learning_rate=form.learning_rate.data,
lr_policy=policy,
gpu_count=gpu_count,
selected_gpus=selected_gpus,
batch_size=form.batch_size.data,
val_interval=form.val_interval.data,
pretrained_model=pretrained_model,
crop_size=form.crop_size.data,
use_mean=bool(form.use_mean.data),
network=network,
random_seed=form.random_seed.data,
solver_type=form.solver_type.data,
)
)
scheduler.add_job(job)
if request_wants_json():
return flask.jsonify(job.json_dict())
else:
return flask.redirect(flask.url_for("models_show", job_id=job.id()))
except:
if job:
scheduler.delete_job(job)
raise
def image_classification_model_create():
"""
Create a new ImageClassificationModelJob
"""
form = ImageClassificationModelForm()
form.dataset.choices = get_datasets()
form.standard_networks.choices = get_standard_networks()
form.standard_networks.default = get_default_standard_network()
form.previous_networks.choices = get_previous_networks()
prev_network_snapshots = get_previous_network_snapshots()
if not form.validate_on_submit():
return render_template('models/images/classification/new.html',
form = form,
previous_network_snapshots=prev_network_snapshots,
multi_gpu = MULTI_GPU,
), 400
datasetJob = scheduler.get_job(form.dataset.data)
if not datasetJob:
return 'Unknown dataset job_id "%s"' % form.dataset.data, 500
job = None
try:
job = ImageClassificationModelJob(
name = form.model_name.data,
dataset_id = datasetJob.id(),
)
network = caffe_pb2.NetParameter()
pretrained_model = None
if form.method.data == 'standard':
found = False
networks_dir = os.path.join(os.path.dirname(digits.__file__), 'standard-networks')
for filename in os.listdir(networks_dir):
path = os.path.join(networks_dir, filename)
if os.path.isfile(path):
match = re.match(r'%s.prototxt' % form.standard_networks.data, filename)
if match:
with open(path) as infile:
text_format.Merge(infile.read(), network)
found = True
break
if not found:
raise Exception('Unknown standard model "%s"' % form.standard_networks.data)
elif form.method.data == 'previous':
old_job = scheduler.get_job(form.previous_networks.data)
if not old_job:
raise Exception('Job not found: %s' % form.previous_networks.data)
network.CopyFrom(old_job.train_task().network)
# Rename the final layer
# XXX making some assumptions about network architecture here
ip_layers = [l for l in network.layer if l.type == 'InnerProduct']
if len(ip_layers) > 0:
ip_layers[-1].name = '%s_retrain' % ip_layers[-1].name
for i, choice in enumerate(form.previous_networks.choices):
if choice[0] == form.previous_networks.data:
epoch = float(request.form['%s-snapshot' % form.previous_networks.data])
if epoch != 0:
for filename, e in old_job.train_task().snapshots:
if e == epoch:
pretrained_model = filename
break
if pretrained_model is None:
raise Exception("For the job %s, selected pretrained_model for epoch %d is invalid!" % (form.previous_networks.data, epoch))
if not (os.path.exists(pretrained_model)):
raise Exception("Pretrained_model for the selected epoch doesn't exists. May be deleted by another user/process. Please restart the server to load the correct pretrained_model details")
break
elif form.method.data == 'custom':
text_format.Merge(form.custom_network.data, network)
pretrained_model = form.custom_network_snapshot.data.strip()
else:
raise Exception('Unrecognized method: "%s"' % form.method.data)
policy = {'policy': form.lr_policy.data}
if form.lr_policy.data == 'fixed':
pass
elif form.lr_policy.data == 'step':
policy['stepsize'] = form.lr_step_size.data
policy['gamma'] = form.lr_step_gamma.data
elif form.lr_policy.data == 'multistep':
policy['stepvalue'] = form.lr_multistep_values.data
policy['gamma'] = form.lr_multistep_gamma.data
elif form.lr_policy.data == 'exp':
policy['gamma'] = form.lr_exp_gamma.data
elif form.lr_policy.data == 'inv':
policy['gamma'] = form.lr_inv_gamma.data
policy['power'] = form.lr_inv_power.data
elif form.lr_policy.data == 'poly':
policy['power'] = form.lr_poly_power.data
elif form.lr_policy.data == 'sigmoid':
policy['stepsize'] = form.lr_sigmoid_step.data
policy['gamma'] = form.lr_sigmoid_gamma.data
else:
return 'Invalid policy', 404
if MULTI_GPU:
if form.select_gpu_count.data:
gpu_count = form.select_gpu_count.data
selected_gpus = None
else:
selected_gpus = [str(gpu) for gpu in form.select_gpus.data]
gpu_count = None
else:
if form.select_gpu.data == 'next':
gpu_count = 1
selected_gpus = None
else:
selected_gpus = [str(form.select_gpu.data)]
gpu_count = None
job.tasks.append(
tasks.CaffeTrainTask(
job_dir = job.dir(),
dataset = datasetJob,
train_epochs = form.train_epochs.data,
snapshot_interval = form.snapshot_interval.data,
learning_rate = form.learning_rate.data,
lr_policy = policy,
gpu_count = gpu_count,
selected_gpus = selected_gpus,
batch_size = form.batch_size.data,
val_interval = form.val_interval.data,
pretrained_model= pretrained_model,
crop_size = form.crop_size.data,
use_mean = form.use_mean.data,
network = network,
random_seed = form.random_seed.data,
solver_type = form.solver_type.data,
)
)
scheduler.add_job(job)
return redirect(url_for('models_show', job_id=job.id()))
except:
if job:
scheduler.delete_job(job)
raise
def image_classification_model_create():
"""
Create a new ImageClassificationModelJob
Returns JSON when requested: {job_id,name,status} or {errors:[]}
"""
form = ImageClassificationModelForm()
form.dataset.choices = get_datasets()
form.standard_networks.choices = get_standard_networks()
form.standard_networks.default = get_default_standard_network()
form.previous_networks.choices = get_previous_networks()
prev_network_snapshots = get_previous_network_snapshots()
## Is there a request to clone a job with ?clone=<job_id>
fill_form_if_cloned(form)
if not form.validate_on_submit():
if request_wants_json():
return flask.jsonify({'errors': form.errors}), 400
else:
return flask.render_template('models/images/classification/new.html',
form = form,
frameworks = frameworks.get_frameworks(),
previous_network_snapshots = prev_network_snapshots,
previous_networks_fullinfo = get_previous_networks_fulldetails(),
multi_gpu = config_value('caffe_root')['multi_gpu'],
), 400
datasetJob = scheduler.get_job(form.dataset.data)
if not datasetJob:
raise werkzeug.exceptions.BadRequest(
'Unknown dataset job_id "%s"' % form.dataset.data)
job = None
try:
job = ImageClassificationModelJob(
name = form.model_name.data,
dataset_id = datasetJob.id(),
)
# get handle to framework object
fw = frameworks.get_framework_by_id(form.framework.data)
pretrained_model = None
if form.method.data == 'standard':
found = False
# can we find it in standard networks?
network_desc = fw.get_standard_network_desc(form.standard_networks.data)
if network_desc:
found = True
network = fw.get_network_from_desc(network_desc)
if not found:
raise werkzeug.exceptions.BadRequest(
'Unknown standard model "%s"' % form.standard_networks.data)
elif form.method.data == 'previous':
old_job = scheduler.get_job(form.previous_networks.data)
if not old_job:
raise werkzeug.exceptions.BadRequest(
'Job not found: %s' % form.previous_networks.data)
network = fw.get_network_from_previous(old_job.train_task().network)
for choice in form.previous_networks.choices:
if choice[0] == form.previous_networks.data:
epoch = float(flask.request.form['%s-snapshot' % form.previous_networks.data])
if epoch == 0:
pass
elif epoch == -1:
pretrained_model = old_job.train_task().pretrained_model
else:
for filename, e in old_job.train_task().snapshots:
if e == epoch:
pretrained_model = filename
break
if pretrained_model is None:
raise werkzeug.exceptions.BadRequest(
"For the job %s, selected pretrained_model for epoch %d is invalid!"
% (form.previous_networks.data, epoch))
if not (os.path.exists(pretrained_model)):
raise werkzeug.exceptions.BadRequest(
"Pretrained_model for the selected epoch doesn't exists. May be deleted by another user/process. Please restart the server to load the correct pretrained_model details")
break
elif form.method.data == 'custom':
network = fw.get_network_from_desc(form.custom_network.data)
pretrained_model = form.custom_network_snapshot.data.strip()
else:
raise werkzeug.exceptions.BadRequest(
'Unrecognized method: "%s"' % form.method.data)
policy = {'policy': form.lr_policy.data}
if form.lr_policy.data == 'fixed':
pass
elif form.lr_policy.data == 'step':
policy['stepsize'] = form.lr_step_size.data
policy['gamma'] = form.lr_step_gamma.data
elif form.lr_policy.data == 'multistep':
policy['stepvalue'] = form.lr_multistep_values.data
policy['gamma'] = form.lr_multistep_gamma.data
elif form.lr_policy.data == 'exp':
policy['gamma'] = form.lr_exp_gamma.data
elif form.lr_policy.data == 'inv':
policy['gamma'] = form.lr_inv_gamma.data
policy['power'] = form.lr_inv_power.data
elif form.lr_policy.data == 'poly':
policy['power'] = form.lr_poly_power.data
elif form.lr_policy.data == 'sigmoid':
policy['stepsize'] = form.lr_sigmoid_step.data
policy['gamma'] = form.lr_sigmoid_gamma.data
else:
raise werkzeug.exceptions.BadRequest(
'Invalid learning rate policy')
if config_value('caffe_root')['multi_gpu']:
if form.select_gpus.data:
selected_gpus = [str(gpu) for gpu in form.select_gpus.data]
gpu_count = None
elif form.select_gpu_count.data:
gpu_count = form.select_gpu_count.data
selected_gpus = None
else:
gpu_count = 1
selected_gpus = None
else:
if form.select_gpu.data == 'next':
gpu_count = 1
selected_gpus = None
else:
selected_gpus = [str(form.select_gpu.data)]
gpu_count = None
# Python Layer File may be on the server or copied from the client.
fs.copy_python_layer_file(
bool(form.python_layer_from_client.data),
job.dir(),
(flask.request.files[form.python_layer_client_file.name]
if form.python_layer_client_file.name in flask.request.files
else ''), form.python_layer_server_file.data)
job.tasks.append(fw.create_train_task(
job_dir = job.dir(),
dataset = datasetJob,
train_epochs = form.train_epochs.data,
snapshot_interval = form.snapshot_interval.data,
learning_rate = form.learning_rate.data,
lr_policy = policy,
gpu_count = gpu_count,
selected_gpus = selected_gpus,
batch_size = form.batch_size.data,
val_interval = form.val_interval.data,
pretrained_model= pretrained_model,
crop_size = form.crop_size.data,
use_mean = form.use_mean.data,
network = network,
random_seed = form.random_seed.data,
solver_type = form.solver_type.data,
shuffle = form.shuffle.data,
)
)
## Save form data with the job so we can easily clone it later.
save_form_to_job(job, form)
scheduler.add_job(job)
if request_wants_json():
return flask.jsonify(job.json_dict())
else:
return flask.redirect(flask.url_for('models_show', job_id=job.id()))
except:
if job:
scheduler.delete_job(job)
raise
def image_classification_model_create():
"""
Create a new ImageClassificationModelJob
Returns JSON when requested: {job_id,name,status} or {errors:[]}
"""
form = ImageClassificationModelForm()
form.dataset.choices = get_datasets()
form.standard_networks.choices = get_standard_networks()
form.standard_networks.default = get_default_standard_network()
form.previous_networks.choices = get_previous_networks()
prev_network_snapshots = get_previous_network_snapshots()
if not form.validate_on_submit():
if request_wants_json():
return flask.jsonify({"errors": form.errors}), 400
else:
return (
flask.render_template(
"models/images/classification/new.html",
form=form,
frameworks=frameworks.get_frameworks(),
previous_network_snapshots=prev_network_snapshots,
previous_networks_fullinfo=get_previous_networks_fulldetails(),
multi_gpu=config_value("caffe_root")["multi_gpu"],
),
400,
)
datasetJob = scheduler.get_job(form.dataset.data)
if not datasetJob:
raise werkzeug.exceptions.BadRequest('Unknown dataset job_id "%s"' % form.dataset.data)
job = None
try:
job = ImageClassificationModelJob(name=form.model_name.data, dataset_id=datasetJob.id())
# get handle to framework object
fw = frameworks.get_framework_by_id(form.framework.data)
pretrained_model = None
if form.method.data == "standard":
found = False
# can we find it in standard networks?
network_desc = fw.get_standard_network_desc(form.standard_networks.data)
if network_desc:
found = True
network = fw.get_network_from_desc(network_desc)
if not found:
raise werkzeug.exceptions.BadRequest('Unknown standard model "%s"' % form.standard_networks.data)
elif form.method.data == "previous":
old_job = scheduler.get_job(form.previous_networks.data)
if not old_job:
raise werkzeug.exceptions.BadRequest("Job not found: %s" % form.previous_networks.data)
network = fw.get_network_from_previous(old_job.train_task().network)
for choice in form.previous_networks.choices:
if choice[0] == form.previous_networks.data:
epoch = float(flask.request.form["%s-snapshot" % form.previous_networks.data])
if epoch == 0:
pass
elif epoch == -1:
pretrained_model = old_job.train_task().pretrained_model
else:
for filename, e in old_job.train_task().snapshots:
if e == epoch:
pretrained_model = filename
break
if pretrained_model is None:
raise werkzeug.exceptions.BadRequest(
"For the job %s, selected pretrained_model for epoch %d is invalid!"
% (form.previous_networks.data, epoch)
)
if not (os.path.exists(pretrained_model)):
raise werkzeug.exceptions.BadRequest(
"Pretrained_model for the selected epoch doesn't exists. May be deleted by another user/process. Please restart the server to load the correct pretrained_model details"
)
break
elif form.method.data == "custom":
network = fw.get_network_from_desc(form.custom_network.data)
pretrained_model = form.custom_network_snapshot.data.strip()
else:
raise werkzeug.exceptions.BadRequest('Unrecognized method: "%s"' % form.method.data)
policy = {"policy": form.lr_policy.data}
if form.lr_policy.data == "fixed":
pass
elif form.lr_policy.data == "step":
policy["stepsize"] = form.lr_step_size.data
policy["gamma"] = form.lr_step_gamma.data
elif form.lr_policy.data == "multistep":
policy["stepvalue"] = form.lr_multistep_values.data
policy["gamma"] = form.lr_multistep_gamma.data
elif form.lr_policy.data == "exp":
policy["gamma"] = form.lr_exp_gamma.data
elif form.lr_policy.data == "inv":
policy["gamma"] = form.lr_inv_gamma.data
policy["power"] = form.lr_inv_power.data
elif form.lr_policy.data == "poly":
policy["power"] = form.lr_poly_power.data
elif form.lr_policy.data == "sigmoid":
policy["stepsize"] = form.lr_sigmoid_step.data
policy["gamma"] = form.lr_sigmoid_gamma.data
else:
raise werkzeug.exceptions.BadRequest("Invalid learning rate policy")
if config_value("caffe_root")["multi_gpu"]:
if form.select_gpus.data:
selected_gpus = [str(gpu) for gpu in form.select_gpus.data]
gpu_count = None
elif form.select_gpu_count.data:
gpu_count = form.select_gpu_count.data
selected_gpus = None
else:
gpu_count = 1
selected_gpus = None
else:
if form.select_gpu.data == "next":
gpu_count = 1
selected_gpus = None
else:
selected_gpus = [str(form.select_gpu.data)]
gpu_count = None
job.tasks.append(
fw.create_train_task(
job_dir=job.dir(),
dataset=datasetJob,
train_epochs=form.train_epochs.data,
snapshot_interval=form.snapshot_interval.data,
learning_rate=form.learning_rate.data,
lr_policy=policy,
gpu_count=gpu_count,
selected_gpus=selected_gpus,
batch_size=form.batch_size.data,
val_interval=form.val_interval.data,
pretrained_model=pretrained_model,
crop_size=form.crop_size.data,
use_mean=bool(form.use_mean.data),
network=network,
random_seed=form.random_seed.data,
solver_type=form.solver_type.data,
shuffle=form.shuffle.data,
)
)
scheduler.add_job(job)
if request_wants_json():
return flask.jsonify(job.json_dict())
else:
return flask.redirect(flask.url_for("models_show", job_id=job.id()))
except:
if job:
scheduler.delete_job(job)
raise
def image_classification_model_create(dataset_id):
"""
Create a new ImageClassificationModelJob
Returns JSON when requested: {job_id,name,status} or {errors:[]}
"""
form = ImageClassificationModelForm()
dataset_name, form.category_names.choices = get_category_names(dataset_id)
form.standard_networks.choices = get_standard_networks()
form.standard_networks.default = get_default_standard_network()
form.previous_networks.choices = get_previous_networks()
prev_network_snapshots = get_previous_network_snapshots()
## Is there a request to clone a job with ?clone=<job_id>
fill_form_if_cloned(form)
if not form.validate_on_submit():
if request_wants_json():
return flask.jsonify({'errors': form.errors}), 400
else:
return flask.render_template(
'models/images/classification/new.html',
form=form,
dataset_name=dataset_name,
dataset_id=dataset_id,
frameworks=frameworks.get_frameworks(),
previous_network_snapshots=prev_network_snapshots,
previous_networks_fullinfo=get_previous_networks_fulldetails(),
multi_gpu=config_value('caffe_root')['multi_gpu'],
), 400
job = None
try:
job = ImageClassificationModelJob(
name=form.model_name.data,
dataset_id=dataset_id,
category_label=form.category_names.choices[
form.category_names.data][1],
)
# get handle to framework object
fw = frameworks.get_framework_by_id(form.framework.data)
pretrained_model = None
if form.method.data == 'standard':
found = False
# can we find it in standard networks?
network_desc = fw.get_standard_network_desc(
form.standard_networks.data)
if network_desc:
found = True
network = fw.get_network_from_desc(network_desc)
if not found:
raise werkzeug.exceptions.BadRequest(
'Unknown standard model "%s"' %
form.standard_networks.data)
elif form.method.data == 'previous':
old_job = scheduler.get_job(form.previous_networks.data)
if not old_job:
raise werkzeug.exceptions.BadRequest(
'Job not found: %s' % form.previous_networks.data)
network = fw.get_network_from_previous(
old_job.train_task().network)
for choice in form.previous_networks.choices:
if choice[0] == form.previous_networks.data:
epoch = float(
flask.request.form['%s-snapshot' %
form.previous_networks.data])
if epoch == 0:
pass
elif epoch == -1:
pretrained_model = old_job.train_task(
).pretrained_model
else:
for filename, e in old_job.train_task().snapshots:
if e == epoch:
pretrained_model = filename
break
if pretrained_model is None:
raise werkzeug.exceptions.BadRequest(
"For the job %s, selected pretrained_model for epoch %d is invalid!"
% (form.previous_networks.data, epoch))
if not (os.path.exists(pretrained_model)):
raise werkzeug.exceptions.BadRequest(
"Pretrained_model for the selected epoch doesn't exists. May be deleted by another user/process. Please restart the server to load the correct pretrained_model details"
)
break
elif form.method.data == 'custom':
network = fw.get_network_from_desc(form.custom_network.data)
pretrained_model = form.custom_network_snapshot.data.strip()
else:
raise werkzeug.exceptions.BadRequest('Unrecognized method: "%s"' %
form.method.data)
policy = {'policy': form.lr_policy.data}
if form.lr_policy.data == 'fixed':
pass
elif form.lr_policy.data == 'step':
policy['stepsize'] = form.lr_step_size.data
policy['gamma'] = form.lr_step_gamma.data
elif form.lr_policy.data == 'multistep':
policy['stepvalue'] = form.lr_multistep_values.data
policy['gamma'] = form.lr_multistep_gamma.data
elif form.lr_policy.data == 'exp':
policy['gamma'] = form.lr_exp_gamma.data
elif form.lr_policy.data == 'inv':
policy['gamma'] = form.lr_inv_gamma.data
policy['power'] = form.lr_inv_power.data
elif form.lr_policy.data == 'poly':
policy['power'] = form.lr_poly_power.data
elif form.lr_policy.data == 'sigmoid':
policy['stepsize'] = form.lr_sigmoid_step.data
policy['gamma'] = form.lr_sigmoid_gamma.data
else:
raise werkzeug.exceptions.BadRequest(
'Invalid learning rate policy')
if config_value('caffe_root')['multi_gpu']:
if form.select_gpus.data:
selected_gpus = [str(gpu) for gpu in form.select_gpus.data]
gpu_count = None
elif form.select_gpu_count.data:
gpu_count = form.select_gpu_count.data
selected_gpus = None
else:
gpu_count = 1
selected_gpus = None
else:
if form.select_gpu.data == 'next':
gpu_count = 1
selected_gpus = None
else:
selected_gpus = [str(form.select_gpu.data)]
gpu_count = None
# Python Layer File may be on the server or copied from the client.
fs.copy_python_layer_file(
bool(form.python_layer_from_client.data), job.dir(),
(flask.request.files[form.python_layer_client_file.name]
if form.python_layer_client_file.name in flask.request.files else
''), form.python_layer_server_file.data)
job.tasks.append(
fw.create_train_task(
job_dir=job.dir(),
dataset=job.dataset,
train_epochs=form.train_epochs.data,
snapshot_interval=form.snapshot_interval.data,
learning_rate=form.learning_rate.data,
lr_policy=policy,
gpu_count=gpu_count,
selected_gpus=selected_gpus,
batch_size=form.batch_size.data,
val_interval=form.val_interval.data,
pretrained_model=pretrained_model,
crop_size=form.crop_size.data,
use_mean=form.use_mean.data,
network=network,
random_seed=form.random_seed.data,
solver_type=form.solver_type.data,
shuffle=form.shuffle.data,
category_index=form.category_names.data,
))
## Save form data with the job so we can easily clone it later.
save_form_to_job(job, form)
scheduler.add_job(job)
if request_wants_json():
return flask.jsonify(job.json_dict())
else:
return flask.redirect(flask.url_for('models_show',
job_id=job.id()))
except:
if job:
scheduler.delete_job(job)
raise
def image_classification_model_create():
"""
Create a new ImageClassificationModelJob
Returns JSON when requested: {job_id,name,status} or {errors:[]}
"""
form = ImageClassificationModelForm()
form.dataset.choices = get_datasets()
form.standard_networks.choices = get_standard_networks()
form.standard_networks.default = get_default_standard_network()
form.previous_networks.choices = get_previous_networks()
prev_network_snapshots = get_previous_network_snapshots()
if not form.validate_on_submit():
if request_wants_json():
return flask.jsonify({'errors': form.errors}), 400
else:
return flask.render_template(
'models/images/classification/new.html',
form=form,
previous_network_snapshots=prev_network_snapshots,
multi_gpu=config_value('caffe_root')['multi_gpu'],
), 400
datasetJob = scheduler.get_job(form.dataset.data)
if not datasetJob:
raise werkzeug.exceptions.BadRequest('Unknown dataset job_id "%s"' %
form.dataset.data)
job = None
try:
job = ImageClassificationModelJob(
name=form.model_name.data,
dataset_id=datasetJob.id(),
)
network = caffe_pb2.NetParameter()
pretrained_model = None
if form.method.data == 'standard':
found = False
networks_dir = os.path.join(os.path.dirname(digits.__file__),
'standard-networks')
for filename in os.listdir(networks_dir):
path = os.path.join(networks_dir, filename)
if os.path.isfile(path):
match = re.match(
r'%s.prototxt' % form.standard_networks.data, filename)
if match:
with open(path) as infile:
text_format.Merge(infile.read(), network)
found = True
break
if not found:
raise werkzeug.exceptions.BadRequest(
'Unknown standard model "%s"' %
form.standard_networks.data)
elif form.method.data == 'previous':
old_job = scheduler.get_job(form.previous_networks.data)
if not old_job:
raise werkzeug.exceptions.BadRequest(
'Job not found: %s' % form.previous_networks.data)
network.CopyFrom(old_job.train_task().network)
# Rename the final layer
# XXX making some assumptions about network architecture here
ip_layers = [l for l in network.layer if l.type == 'InnerProduct']
if len(ip_layers) > 0:
ip_layers[-1].name = '%s_retrain' % ip_layers[-1].name
for choice in form.previous_networks.choices:
if choice[0] == form.previous_networks.data:
epoch = float(
flask.request.form['%s-snapshot' %
form.previous_networks.data])
if epoch != 0:
for filename, e in old_job.train_task().snapshots:
if e == epoch:
pretrained_model = filename
break
if pretrained_model is None:
raise werkzeug.exceptions.BadRequest(
"For the job %s, selected pretrained_model for epoch %d is invalid!"
% (form.previous_networks.data, epoch))
if not (os.path.exists(pretrained_model)):
raise werkzeug.exceptions.BadRequest(
"Pretrained_model for the selected epoch doesn't exists. May be deleted by another user/process. Please restart the server to load the correct pretrained_model details"
)
break
elif form.method.data == 'custom':
text_format.Merge(form.custom_network.data, network)
pretrained_model = form.custom_network_snapshot.data.strip()
else:
raise werkzeug.exceptions.BadRequest('Unrecognized method: "%s"' %
form.method.data)
policy = {'policy': form.lr_policy.data}
if form.lr_policy.data == 'fixed':
pass
elif form.lr_policy.data == 'step':
policy['stepsize'] = form.lr_step_size.data
policy['gamma'] = form.lr_step_gamma.data
elif form.lr_policy.data == 'multistep':
policy['stepvalue'] = form.lr_multistep_values.data
policy['gamma'] = form.lr_multistep_gamma.data
elif form.lr_policy.data == 'exp':
policy['gamma'] = form.lr_exp_gamma.data
elif form.lr_policy.data == 'inv':
policy['gamma'] = form.lr_inv_gamma.data
policy['power'] = form.lr_inv_power.data
elif form.lr_policy.data == 'poly':
policy['power'] = form.lr_poly_power.data
elif form.lr_policy.data == 'sigmoid':
policy['stepsize'] = form.lr_sigmoid_step.data
policy['gamma'] = form.lr_sigmoid_gamma.data
else:
raise werkzeug.exceptions.BadRequest(
'Invalid learning rate policy')
if config_value('caffe_root')['multi_gpu']:
if form.select_gpus.data:
selected_gpus = [str(gpu) for gpu in form.select_gpus.data]
gpu_count = None
elif form.select_gpu_count.data:
gpu_count = form.select_gpu_count.data
selected_gpus = None
else:
gpu_count = 1
selected_gpus = None
else:
if form.select_gpu.data == 'next':
gpu_count = 1
selected_gpus = None
else:
selected_gpus = [str(form.select_gpu.data)]
gpu_count = None
job.tasks.append(
tasks.CaffeTrainTask(
job_dir=job.dir(),
dataset=datasetJob,
train_epochs=form.train_epochs.data,
snapshot_interval=form.snapshot_interval.data,
learning_rate=form.learning_rate.data,
lr_policy=policy,
gpu_count=gpu_count,
selected_gpus=selected_gpus,
batch_size=form.batch_size.data,
val_interval=form.val_interval.data,
pretrained_model=pretrained_model,
crop_size=form.crop_size.data,
use_mean=bool(form.use_mean.data),
network=network,
random_seed=form.random_seed.data,
solver_type=form.solver_type.data,
))
scheduler.add_job(job)
if request_wants_json():
return flask.jsonify(job.json_dict())
else:
return flask.redirect(flask.url_for('models_show',
job_id=job.id()))
except:
if job:
scheduler.delete_job(job)
raise
def image_classification_model_create():
form = ImageClassificationModelForm()
form.dataset.choices = get_datasets()
form.standard_networks.choices = get_standard_networks()
form.standard_networks.default = get_default_standard_network()
form.previous_networks.choices = get_previous_networks()
prev_network_snapshots = get_previous_network_snapshots()
if not form.validate_on_submit():
return render_template(
'models/images/classification/new.html',
form=form,
previous_network_snapshots=prev_network_snapshots), 400
datasetJob = scheduler.get_job(form.dataset.data)
if not datasetJob:
return 'Unknown dataset job_id "%s"' % form.dataset.data, 500
job = None
try:
job = ImageClassificationModelJob(
name=form.model_name.data,
dataset_id=datasetJob.id(),
)
network = caffe_pb2.NetParameter()
pretrained_model = None
if form.method.data == 'standard':
found = False
networks_dir = os.path.join(os.path.dirname(digits.__file__),
'standard-networks')
for filename in os.listdir(networks_dir):
path = os.path.join(networks_dir, filename)
if os.path.isfile(path):
match = re.match(
r'%s.prototxt' % form.standard_networks.data, filename)
if match:
with open(path) as infile:
text_format.Merge(infile.read(), network)
found = True
break
if not found:
raise Exception('Unknown standard model "%s"' %
form.standard_networks.data)
elif form.method.data == 'previous':
old_job = scheduler.get_job(form.previous_networks.data)
if not old_job:
raise Exception('Job not found: %s' %
form.previous_networks.data)
network.CopyFrom(old_job.train_task().network)
for i, choice in enumerate(form.previous_networks.choices):
if choice[0] == form.previous_networks.data:
epoch = int(request.form['%s-snapshot' %
form.previous_networks.data])
if epoch != 0:
for filename, e in old_job.train_task().snapshots:
if e == epoch:
pretrained_model = filename
break
if pretrained_model is None:
raise Exception(
"For the job %s, selected pretrained_model for epoch %d is invalid!"
% (form.previous_networks.data, epoch))
if not (os.path.exists(pretrained_model)):
raise Exception(
"Pretrained_model for the selected epoch doesn't exists. May be deleted by another user/process. Please restart the server to load the correct pretrained_model details"
)
break
elif form.method.data == 'custom':
text_format.Merge(form.custom_network.data, network)
pretrained_model = form.custom_network_snapshot.data.strip()
else:
raise Exception('Unrecognized method: "%s"' % form.method.data)
policy = {'policy': form.lr_policy.data}
if form.lr_policy.data == 'fixed':
pass
elif form.lr_policy.data == 'step':
policy['stepsize'] = form.lr_step_size.data
policy['gamma'] = form.lr_step_gamma.data
elif form.lr_policy.data == 'multistep':
policy['stepvalue'] = form.lr_multistep_values.data
policy['gamma'] = form.lr_multistep_gamma.data
elif form.lr_policy.data == 'exp':
policy['gamma'] = form.lr_exp_gamma.data
elif form.lr_policy.data == 'inv':
policy['gamma'] = form.lr_inv_gamma.data
policy['power'] = form.lr_inv_power.data
elif form.lr_policy.data == 'poly':
policy['power'] = form.lr_poly_power.data
elif form.lr_policy.data == 'sigmoid':
policy['stepsize'] = form.lr_sigmoid_step.data
policy['gamma'] = form.lr_sigmoid_gamma.data
else:
return 'Invalid policy', 404
job.tasks.append(
tasks.CaffeTrainTask(
job_dir=job.dir(),
dataset=datasetJob,
train_epochs=form.train_epochs.data,
learning_rate=form.learning_rate.data,
lr_policy=policy,
batch_size=form.batch_size.data,
val_interval=form.val_interval.data,
pretrained_model=pretrained_model,
crop_size=form.crop_size.data,
use_mean=form.use_mean.data,
network=network,
))
scheduler.add_job(job)
return redirect(url_for('models_show', job_id=job.id()))
except:
if job:
scheduler.delete_job(job)
raise