class InferenceForm(Form):
"""
A form used to perform inference on a gradient regression model
"""
gradient_x = utils.forms.FloatField(
'Gradient (x)',
validators=[
validate_required_iff(test_image_count=None),
validators.NumberRange(min=-0.5, max=0.5),
],
tooltip="Specify a number between -0.5 and 0.5")
gradient_y = utils.forms.FloatField(
'Gradient (y)',
validators=[
validate_required_iff(test_image_count=None),
validators.NumberRange(min=-0.5, max=0.5),
],
tooltip="Specify a number between -0.5 and 0.5")
test_image_count = utils.forms.IntegerField(
'Test Image count',
validators=[
validators.Optional(),
validators.NumberRange(min=0),
],
tooltip="Number of images to create in test set")
class GenericImageDatasetForm(ImageDatasetForm):
"""
Defines the form used to create a new GenericImageDatasetJob
"""
# Use a SelectField instead of a HiddenField so that the default value
# is used when nothing is provided (through the REST API)
method = wtforms.SelectField(
u'Dataset type',
choices=[
('prebuilt', 'Prebuilt'),
],
default='prebuilt',
)
def validate_lmdb_path(form, field):
if not field.data:
pass
else:
# make sure the filesystem path exists
if not os.path.exists(field.data) or not os.path.isdir(field.data):
raise validators.ValidationError('Folder does not exist')
def validate_file_path(form, field):
if not field.data:
pass
else:
# make sure the filesystem path exists
if not os.path.exists(field.data) or not os.path.isfile(
field.data):
raise validators.ValidationError('File does not exist')
### Method - prebuilt
prebuilt_train_images = wtforms.StringField(
'Training Images',
validators=[
validate_required_iff(method='prebuilt'),
validate_lmdb_path,
])
prebuilt_train_labels = wtforms.StringField('Training Labels',
validators=[
validate_lmdb_path,
])
prebuilt_val_images = wtforms.StringField('Validation Images',
validators=[
validate_lmdb_path,
])
prebuilt_val_labels = wtforms.StringField('Validation Labels',
validators=[
validate_lmdb_path,
])
prebuilt_mean_file = wtforms.StringField('Mean Image',
validators=[
validate_file_path,
])
class ImageClassificationDatasetForm(ImageDatasetForm):
"""
Defines the form used to create a new ImageClassificationDatasetJob
"""
backend = wtforms.SelectField('DB backend',
choices = [
('lmdb', 'LMDB'),
('hdf5', 'HDF5'),
],
default='lmdb',
)
def validate_backend(form, field):
if field.data == 'lmdb':
form.compression.data = 'none'
elif field.data == 'hdf5':
form.encoding.data = 'none'
compression = utils.forms.SelectField('DB compression',
choices = [
('none', 'None'),
('gzip', 'GZIP'),
],
default='none',
tooltip='Compressing the dataset may significantly decrease the size of your database files, but it may increase read and write times.',
)
# Use a SelectField instead of a HiddenField so that the default value
# is used when nothing is provided (through the REST API)
method = wtforms.SelectField(u'Dataset type',
choices = [
('folder', 'Folder'),
('textfile', 'Textfiles'),
],
default='folder',
)
def validate_folder_path(form, field):
if not field.data:
pass
elif utils.is_url(field.data):
# make sure the URL exists
try:
r = requests.get(field.data,
allow_redirects=False,
timeout=utils.HTTP_TIMEOUT)
if r.status_code not in [requests.codes.ok, requests.codes.moved, requests.codes.found]:
raise validators.ValidationError('URL not found')
except Exception as e:
raise validators.ValidationError('Caught %s while checking URL: %s' % (type(e).__name__, e))
else:
return True
else:
# make sure the filesystem path exists
if not os.path.exists(field.data) or not os.path.isdir(field.data):
raise validators.ValidationError('Folder does not exist')
else:
return True
### Method - folder
folder_train = utils.forms.StringField(u'Training Images',
validators=[
validate_required_iff(method='folder'),
validate_folder_path,
],
tooltip = "Indicate a folder which holds subfolders full of images. Each subfolder should be named according to the desired label for the images that it holds. Can also be a URL for an apache/nginx auto-indexed folder."
)
folder_pct_val = utils.forms.IntegerField(u'% for validation',
default=25,
validators=[
validate_required_iff(method='folder'),
validators.NumberRange(min=0, max=100)
],
tooltip = "You can choose to set apart a certain percentage of images from the training images for the validation set."
)
folder_pct_test = utils.forms.IntegerField(u'% for testing',
default=0,
validators=[
validate_required_iff(method='folder'),
validators.NumberRange(min=0, max=100)
],
tooltip = "You can choose to set apart a certain percentage of images from the training images for the test set."
)
folder_train_min_per_class = utils.forms.IntegerField(u'Minimum samples per class',
default=2,
validators=[
validators.Optional(),
validators.NumberRange(min=1),
],
tooltip = "You can choose to specify a minimum number of samples per class. If a class has fewer samples than the specified amount it will be ignored. Leave blank to ignore this feature."
)
folder_train_max_per_class = utils.forms.IntegerField(u'Maximum samples per class',
validators=[
validators.Optional(),
validators.NumberRange(min=1),
validate_greater_than('folder_train_min_per_class'),
],
tooltip = "You can choose to specify a maximum number of samples per class. If a class has more samples than the specified amount extra samples will be ignored. Leave blank to ignore this feature."
)
has_val_folder = wtforms.BooleanField('Separate validation images folder',
default = False,
validators=[
validate_required_iff(method='folder')
]
)
folder_val = wtforms.StringField(u'Validation Images',
validators=[
validate_required_iff(
method='folder',
has_val_folder=True),
]
)
folder_val_min_per_class = utils.forms.IntegerField(u'Minimum samples per class',
default=2,
validators=[
validators.Optional(),
validators.NumberRange(min=1),
],
tooltip = "You can choose to specify a minimum number of samples per class. If a class has fewer samples than the specified amount it will be ignored. Leave blank to ignore this feature."
)
folder_val_max_per_class = utils.forms.IntegerField(u'Maximum samples per class',
validators=[
validators.Optional(),
validators.NumberRange(min=1),
validate_greater_than('folder_val_min_per_class'),
],
tooltip = "You can choose to specify a maximum number of samples per class. If a class has more samples than the specified amount extra samples will be ignored. Leave blank to ignore this feature."
)
has_test_folder = wtforms.BooleanField('Separate test images folder',
default = False,
validators=[
validate_required_iff(method='folder')
]
)
folder_test = wtforms.StringField(u'Test Images',
validators=[
validate_required_iff(
method='folder',
has_test_folder=True),
validate_folder_path,
]
)
folder_test_min_per_class = utils.forms.IntegerField(u'Minimum samples per class',
default=2,
validators=[
validators.Optional(),
validators.NumberRange(min=1)
],
tooltip = "You can choose to specify a minimum number of samples per class. If a class has fewer samples than the specified amount it will be ignored. Leave blank to ignore this feature."
)
folder_test_max_per_class = utils.forms.IntegerField(u'Maximum samples per class',
validators=[
validators.Optional(),
validators.NumberRange(min=1),
validate_greater_than('folder_test_min_per_class'),
],
tooltip = "You can choose to specify a maximum number of samples per class. If a class has more samples than the specified amount extra samples will be ignored. Leave blank to ignore this feature."
)
### Method - textfile
textfile_use_local_files = wtforms.BooleanField(u'Use local files',
default=False)
textfile_train_images = utils.forms.FileField(u'Training images',
validators=[
validate_required_iff(method='textfile',
textfile_use_local_files=False)
]
)
textfile_local_train_images = wtforms.StringField(u'Training images',
validators=[
validate_required_iff(method='textfile',
textfile_use_local_files=True)
]
)
textfile_train_folder = wtforms.StringField(u'Training images folder')
def validate_textfile_train_folder(form, field):
if form.method.data != 'textfile':
field.errors[:] = []
raise validators.StopValidation()
if not field.data.strip():
# allow null
return True
if not os.path.exists(field.data) or not os.path.isdir(field.data):
raise validators.ValidationError('folder does not exist')
return True
textfile_use_val = wtforms.BooleanField(u'Validation set',
default=True,
validators=[
validate_required_iff(method='textfile')
]
)
textfile_val_images = utils.forms.FileField(u'Validation images',
validators=[
validate_required_iff(
method='textfile',
textfile_use_val=True,
textfile_use_local_files=False)
]
)
textfile_local_val_images = wtforms.StringField(u'Validation images',
validators=[
validate_required_iff(
method='textfile',
textfile_use_val=True,
textfile_use_local_files=True)
]
)
textfile_val_folder = wtforms.StringField(u'Validation images folder')
def validate_textfile_val_folder(form, field):
if form.method.data != 'textfile' or not form.textfile_use_val.data:
field.errors[:] = []
raise validators.StopValidation()
if not field.data.strip():
# allow null
return True
if not os.path.exists(field.data) or not os.path.isdir(field.data):
raise validators.ValidationError('folder does not exist')
return True
textfile_use_test = wtforms.BooleanField(u'Test set',
default=False,
validators=[
validate_required_iff(method='textfile')
]
)
textfile_test_images = utils.forms.FileField(u'Test images',
validators=[
validate_required_iff(
method='textfile',
textfile_use_test=True,
textfile_use_local_files=False)
]
)
textfile_local_test_images = wtforms.StringField(u'Test images',
validators=[
validate_required_iff(
method='textfile',
textfile_use_test=True,
textfile_use_local_files=True)
]
)
textfile_test_folder = wtforms.StringField(u'Test images folder')
def validate_textfile_test_folder(form, field):
if form.method.data != 'textfile' or not form.textfile_use_test.data:
field.errors[:] = []
raise validators.StopValidation()
if not field.data.strip():
# allow null
return True
if not os.path.exists(field.data) or not os.path.isdir(field.data):
raise validators.ValidationError('folder does not exist')
return True
# Can't use a BooleanField here because HTML doesn't submit anything
# for an unchecked checkbox. Since we want to use a REST API and have
# this default to True when nothing is supplied, we have to use a
# SelectField
textfile_shuffle = utils.forms.SelectField('Shuffle lines',
choices = [
(1, 'Yes'),
(0, 'No'),
],
coerce=int,
default=1,
tooltip = "Shuffle the list[s] of images before creating the database."
)
textfile_labels_file = utils.forms.FileField(u'Labels',
validators=[
validate_required_iff(method='textfile',
textfile_use_local_files=False)
],
tooltip = "The 'i'th line of the file should give the string label associated with the '(i-1)'th numberic label. (E.g. the string label for the numeric label 0 is supposed to be on line 1.)"
)
textfile_local_labels_file = utils.forms.StringField(u'Labels',
validators=[
validate_required_iff(method='textfile',
textfile_use_local_files=True)
],
tooltip = "The 'i'th line of the file should give the string label associated with the '(i-1)'th numberic label. (E.g. the string label for the numeric label 0 is supposed to be on line 1.)"
)
class ModelForm(Form):
### Methods
def selection_exists_in_choices(form, field):
found = False
for choice in field.choices:
if choice[0] == field.data:
found = True
if not found:
raise validators.ValidationError(
"Selected job doesn't exist. Maybe it was deleted by another user."
)
def validate_NetParameter(form, field):
fw = frameworks.get_framework_by_id(form['framework'].data)
try:
# below function raises a BadNetworkException in case of validation error
fw.validate_network(field.data)
except frameworks.errors.BadNetworkError as e:
raise validators.ValidationError('Bad network: %s' % e.message)
def validate_file_exists(form, field):
from_client = bool(form.python_layer_from_client.data)
filename = ''
if not from_client and field.type == 'StringField':
filename = field.data
if filename == '': return
if not os.path.isfile(filename):
raise validators.ValidationError(
'Server side file, %s, does not exist.' % filename)
def validate_py_ext(form, field):
from_client = bool(form.python_layer_from_client.data)
filename = ''
if from_client and field.type == 'FileField':
filename = flask.request.files[field.name].filename
elif not from_client and field.type == 'StringField':
filename = field.data
if filename == '': return
(root, ext) = os.path.splitext(filename)
if ext != '.py' and ext != '.pyc':
raise validators.ValidationError(
'Python file, %s, needs .py or .pyc extension.' % filename)
### Fields
# The options for this get set in the view (since they are dynamic)
dataset = utils.forms.SelectField(
'Select Dataset',
choices=[],
tooltip="Choose the dataset to use for this model.")
python_layer_from_client = utils.forms.BooleanField(
u'Use client-side file', default=False)
python_layer_client_file = utils.forms.FileField(
u'Client-side file',
validators=[validate_py_ext],
tooltip=
"Choose a Python file on the client containing layer definitions.")
python_layer_server_file = utils.forms.StringField(
u'Server-side file',
validators=[validate_file_exists, validate_py_ext],
tooltip=
"Choose a Python file on the server containing layer definitions.")
train_epochs = utils.forms.IntegerField(
'Training epochs',
validators=[validators.NumberRange(min=1)],
default=30,
tooltip="How many passes through the training data?")
snapshot_interval = utils.forms.FloatField(
'Snapshot interval (in epochs)',
default=1,
validators=[
validators.NumberRange(min=0),
],
tooltip="How many epochs of training between taking a snapshot?")
val_interval = utils.forms.FloatField(
'Validation interval (in epochs)',
default=1,
validators=[validators.NumberRange(min=0)],
tooltip=
"How many epochs of training between running through one pass of the validation data?"
)
random_seed = utils.forms.IntegerField(
'Random seed',
validators=[
validators.NumberRange(min=0),
validators.Optional(),
],
tooltip=
"If you provide a random seed, then back-to-back runs with the same model and dataset should give identical results."
)
batch_size = utils.forms.MultiIntegerField(
'Batch size',
validators=[
utils.forms.MultiNumberRange(min=1),
utils.forms.MultiOptional(),
],
tooltip=
"How many images to process at once. If blank, values are used from the network definition."
)
batch_accumulation = utils.forms.IntegerField(
'Batch Accumulation',
validators=[
validators.NumberRange(min=1),
validators.Optional(),
],
tooltip=
"Accumulate gradients over multiple batches (useful when you need a bigger batch size for training but it doesn't fit in memory)."
)
### Solver types
solver_type = utils.forms.SelectField(
'Solver type',
choices=[
('SGD', 'Stochastic gradient descent (SGD)'),
('NESTEROV', "Nesterov's accelerated gradient (NAG)"),
('ADAGRAD', 'Adaptive gradient (AdaGrad)'),
('RMSPROP', 'RMSprop'),
('ADADELTA', 'AdaDelta'),
('ADAM', 'Adam'),
],
default='SGD',
tooltip="What type of solver will be used?",
)
def validate_solver_type(form, field):
fw = frameworks.get_framework_by_id(form.framework)
if fw is not None:
if not fw.supports_solver_type(field.data):
raise validators.ValidationError(
'Solver type not supported by this framework')
### Learning rate
learning_rate = utils.forms.MultiFloatField(
'Base Learning Rate',
default=0.01,
validators=[
utils.forms.MultiNumberRange(min=0),
],
tooltip=
"Affects how quickly the network learns. If you are getting NaN for your loss, you probably need to lower this value."
)
lr_policy = wtforms.SelectField('Policy',
choices=[
('fixed', 'Fixed'),
('step', 'Step Down'),
('multistep',
'Step Down (arbitrary steps)'),
('exp', 'Exponential Decay'),
('inv', 'Inverse Decay'),
('poly', 'Polynomial Decay'),
('sigmoid', 'Sigmoid Decay'),
],
default='step')
lr_step_size = wtforms.FloatField('Step Size', default=33)
lr_step_gamma = wtforms.FloatField('Gamma', default=0.1)
lr_multistep_values = wtforms.StringField('Step Values', default="50,85")
def validate_lr_multistep_values(form, field):
if form.lr_policy.data == 'multistep':
for value in field.data.split(','):
try:
float(value)
except ValueError:
raise validators.ValidationError('invalid value')
lr_multistep_gamma = wtforms.FloatField('Gamma', default=0.5)
lr_exp_gamma = wtforms.FloatField('Gamma', default=0.95)
lr_inv_gamma = wtforms.FloatField('Gamma', default=0.1)
lr_inv_power = wtforms.FloatField('Power', default=0.5)
lr_poly_power = wtforms.FloatField('Power', default=3)
lr_sigmoid_step = wtforms.FloatField('Step', default=50)
lr_sigmoid_gamma = wtforms.FloatField('Gamma', default=0.1)
### Network
# Use a SelectField instead of a HiddenField so that the default value
# is used when nothing is provided (through the REST API)
method = wtforms.SelectField(
u'Network type',
choices=[
('standard', 'Standard network'),
('previous', 'Previous network'),
('pretrained', 'Pretrained network'),
('custom', 'Custom network'),
],
default='standard',
)
## framework - hidden field, set by Javascript to the selected framework ID
framework = wtforms.HiddenField(
'framework',
validators=[
validators.AnyOf(
[fw.get_id() for fw in frameworks.get_frameworks()],
message='The framework you choose is not currently supported.')
],
default=frameworks.get_frameworks()[0].get_id())
# The options for this get set in the view (since they are dependent on the data type)
standard_networks = wtforms.RadioField(
'Standard Networks',
validators=[
validate_required_iff(method='standard'),
],
)
previous_networks = wtforms.RadioField(
'Previous Networks',
choices=[],
validators=[
validate_required_iff(method='previous'),
selection_exists_in_choices,
],
)
pretrained_networks = wtforms.RadioField(
'Pretrained Networks',
choices=[],
validators=[
validate_required_iff(method='pretrained'),
selection_exists_in_choices,
],
)
custom_network = utils.forms.TextAreaField(
'Custom Network',
validators=[
validate_required_iff(method='custom'),
validate_NetParameter,
],
)
custom_network_snapshot = utils.forms.TextField(
'Pretrained model(s)',
tooltip=
"Paths to pretrained model files, separated by '%s'. Only edit this field if you understand how fine-tuning works in caffe or torch."
% os.path.pathsep)
def validate_custom_network_snapshot(form, field):
if form.method.data == 'custom':
for filename in field.data.strip().split(os.path.pathsep):
if filename and not os.path.exists(filename):
raise validators.ValidationError(
'File "%s" does not exist' % filename)
# Select one of several GPUs
select_gpu = wtforms.RadioField(
'Select which GPU you would like to use',
choices=[('next', 'Next available')] + [(
index,
'#%s - %s (%s memory)' %
(index, get_device(index).name,
sizeof_fmt(
get_nvml_info(index)['memory']['total']
if get_nvml_info(index) and 'memory' in get_nvml_info(index)
else get_device(index).totalGlobalMem)),
) for index in config_value('gpu_list').split(',') if index],
default='next',
)
# Select N of several GPUs
select_gpus = utils.forms.SelectMultipleField(
'Select which GPU[s] you would like to use',
choices=[(
index,
'#%s - %s (%s memory)' %
(index, get_device(index).name,
sizeof_fmt(
get_nvml_info(index)['memory']['total']
if get_nvml_info(index) and 'memory' in get_nvml_info(index)
else get_device(index).totalGlobalMem)),
) for index in config_value('gpu_list').split(',') if index],
tooltip=
"The job won't start until all of the chosen GPUs are available.")
# XXX For testing
# The Flask test framework can't handle SelectMultipleFields correctly
select_gpus_list = wtforms.StringField(
'Select which GPU[s] you would like to use (comma separated)')
def validate_select_gpus(form, field):
if form.select_gpus_list.data:
field.data = form.select_gpus_list.data.split(',')
# Use next available N GPUs
select_gpu_count = wtforms.IntegerField(
'Use this many GPUs (next available)',
validators=[
validators.NumberRange(min=1,
max=len(
config_value('gpu_list').split(',')))
],
default=1,
)
def validate_select_gpu_count(form, field):
if field.data is None:
if form.select_gpus.data:
# Make this field optional
field.errors[:] = []
raise validators.StopValidation()
model_name = utils.forms.StringField(
'Model Name',
validators=[validators.DataRequired()],
tooltip=
"An identifier, later used to refer to this model in the Application.")
# allows shuffling data during training (for frameworks that support this, as indicated by
# their Framework.can_shuffle_data() method)
shuffle = utils.forms.BooleanField(
'Shuffle Train Data',
default=True,
tooltip='For every epoch, shuffle the data before training.')
class ModelForm(Form):
### Methods
def selection_exists_in_choices(form, field):
found = False
for choice in field.choices:
if choice[0] == field.data:
found = True
if not found:
raise validators.ValidationError(
"Selected job doesn't exist. Maybe it was deleted by another user."
)
def validate_NetParameter(form, field):
pb = caffe_pb2.NetParameter()
try:
text_format.Merge(field.data, pb)
except text_format.ParseError as e:
raise validators.ValidationError('Not a valid NetParameter: %s' %
e)
### Fields
# The options for this get set in the view (since they are dynamic)
dataset = wtforms.SelectField('Select Dataset', choices=[])
train_epochs = wtforms.IntegerField(
'Training epochs',
validators=[validators.NumberRange(min=1)],
default=30,
)
snapshot_interval = wtforms.FloatField(
'Snapshot interval (in epochs)',
default=1,
validators=[
validators.NumberRange(min=0),
],
)
val_interval = wtforms.FloatField(
'Validation interval (in epochs)',
default=1,
validators=[validators.NumberRange(min=0)],
)
random_seed = wtforms.IntegerField(
'Random seed',
validators=[
validators.NumberRange(min=0),
validators.Optional(),
],
)
batch_size = wtforms.IntegerField(
'Batch size',
validators=[
validators.NumberRange(min=1),
validators.Optional(),
],
)
### Solver types
solver_type = wtforms.SelectField(
'Solver type',
choices=[
('SGD', 'Stochastic gradient descent (SGD)'),
('ADAGRAD', 'Adaptive gradient (AdaGrad)'),
('NESTEROV', "Nesterov's accelerated gradient (NAG)"),
],
default='SGD')
### Learning rate
learning_rate = wtforms.FloatField('Base Learning Rate',
default=0.01,
validators=[
validators.NumberRange(min=0),
])
lr_policy = wtforms.SelectField('Policy',
choices=[
('fixed', 'Fixed'),
('step', 'Step Down'),
('multistep',
'Step Down (arbitrary steps)'),
('exp', 'Exponential Decay'),
('inv', 'Inverse Decay'),
('poly', 'Polynomial Decay'),
('sigmoid', 'Sigmoid Decay'),
],
default='step')
lr_step_size = wtforms.FloatField('Step Size', default=33)
lr_step_gamma = wtforms.FloatField('Gamma', default=0.1)
lr_multistep_values = wtforms.StringField('Step Values', default="50,85")
def validate_lr_multistep_values(form, field):
if form.lr_policy.data == 'multistep':
for value in field.data.split(','):
try:
float(value)
except ValueError:
raise validators.ValidationError('invalid value')
lr_multistep_gamma = wtforms.FloatField('Gamma', default=0.5)
lr_exp_gamma = wtforms.FloatField('Gamma', default=0.95)
lr_inv_gamma = wtforms.FloatField('Gamma', default=0.1)
lr_inv_power = wtforms.FloatField('Power', default=0.5)
lr_poly_power = wtforms.FloatField('Power', default=3)
lr_sigmoid_step = wtforms.FloatField('Step', default=50)
lr_sigmoid_gamma = wtforms.FloatField('Gamma', default=0.1)
### Network
# Use a SelectField instead of a HiddenField so that the default value
# is used when nothing is provided (through the REST API)
method = wtforms.SelectField(
u'Network type',
choices=[
('standard', 'Standard network'),
('previous', 'Previous network'),
('custom', 'Custom network'),
],
default='standard',
)
# The options for this get set in the view (since they are dependent on the data type)
standard_networks = wtforms.RadioField(
'Standard Networks',
validators=[
validate_required_iff(method='standard'),
],
)
previous_networks = wtforms.RadioField(
'Previous Networks',
choices=[],
validators=[
validate_required_iff(method='previous'),
selection_exists_in_choices,
],
)
custom_network = wtforms.TextAreaField(
'Custom Network',
validators=[
validate_required_iff(method='custom'),
validate_NetParameter,
])
custom_network_snapshot = wtforms.TextField('Pretrained model')
def validate_custom_network_snapshot(form, field):
if form.method.data == 'custom':
snapshot = field.data.strip()
if snapshot:
if not os.path.exists(snapshot):
raise validators.ValidationError('File does not exist')
# Select one of several GPUs
select_gpu = wtforms.RadioField(
'Select which GPU you would like to use',
choices=[('next', 'Next available')] + [(
index,
'#%s - %s%s' % (
index,
get_device(index).name,
' (%s memory)' %
sizeof_fmt(get_nvml_info(index)['memory']['total'])
if get_nvml_info(index) and 'memory' in get_nvml_info(index)
else '',
),
) for index in config_value('gpu_list').split(',') if index],
default='next',
)
# Select N of several GPUs
select_gpus = wtforms.SelectMultipleField(
'Select which GPU[s] you would like to use',
choices=[(
index,
'#%s - %s%s' % (
index,
get_device(index).name,
' (%s memory)' %
sizeof_fmt(get_nvml_info(index)['memory']['total'])
if get_nvml_info(index) and 'memory' in get_nvml_info(index)
else '',
),
) for index in config_value('gpu_list').split(',') if index])
# XXX For testing
# The Flask test framework can't handle SelectMultipleFields correctly
select_gpus_list = wtforms.StringField(
'Select which GPU[s] you would like to use (comma separated)')
def validate_select_gpus(form, field):
if form.select_gpus_list.data:
field.data = form.select_gpus_list.data.split(',')
# Use next available N GPUs
select_gpu_count = wtforms.IntegerField(
'Use this many GPUs (next available)',
validators=[
validators.NumberRange(min=1,
max=len(
config_value('gpu_list').split(',')))
],
default=1,
)
def validate_select_gpu_count(form, field):
if field.data is None:
if form.select_gpus.data:
# Make this field optional
field.errors[:] = []
raise validators.StopValidation()
model_name = wtforms.StringField('Model Name',
validators=[validators.DataRequired()])
class GenericImageDatasetForm(ImageDatasetForm):
"""
Defines the form used to create a new GenericImageDatasetJob
"""
# Use a SelectField instead of a HiddenField so that the default value
# is used when nothing is provided (through the REST API)
method = wtforms.SelectField(
u'Dataset type',
choices=[
('prebuilt', 'Prebuilt'),
],
default='prebuilt',
)
def validate_lmdb_path(form, field):
if not field.data:
pass
else:
# make sure the filesystem path exists
if not os.path.exists(field.data) or not os.path.isdir(field.data):
raise validators.ValidationError('Folder does not exist')
def validate_file_path(form, field):
if not field.data:
pass
else:
# make sure the filesystem path exists
if not os.path.exists(field.data) or not os.path.isfile(
field.data):
raise validators.ValidationError('File does not exist')
### Method - prebuilt
prebuilt_train_images = wtforms.StringField(
'Training Images',
validators=[
validate_required_iff(method='prebuilt'),
validate_lmdb_path,
])
prebuilt_train_labels = wtforms.StringField('Training Labels',
validators=[
validate_lmdb_path,
])
prebuilt_val_images = wtforms.StringField('Validation Images',
validators=[
validate_lmdb_path,
])
prebuilt_val_labels = wtforms.StringField('Validation Labels',
validators=[
validate_lmdb_path,
])
# Can't use a BooleanField here because HTML doesn't submit anything
# for an unchecked checkbox. Since we want to use a REST API and have
# this default to True when nothing is supplied, we have to use a
# SelectField
force_same_shape = utils.forms.SelectField(
'Enforce same shape',
choices=[
(1, 'Yes'),
(0, 'No'),
],
coerce=int,
default=1,
tooltip=
'Check that each entry in the database has the same shape (can be time-consuming)'
)
prebuilt_mean_file = utils.forms.StringField(
'Mean Image',
validators=[
validate_file_path,
],
tooltip="Path to a .binaryproto file on the server")
class DatasetForm(Form):
"""
A form used to create an image processing dataset
"""
def validate_folder_path(form, field):
if not field.data:
pass
else:
# make sure the filesystem path exists
if not os.path.exists(field.data) or not os.path.isdir(field.data):
raise validators.ValidationError(
'Folder does not exist or is not reachable')
else:
return True
feature_folder = utils.forms.StringField(
'Feature image folder',
validators=[
validators.DataRequired(),
validate_folder_path,
],
tooltip="Indicate a folder full of images.")
label_folder = utils.forms.StringField(
'Label image folder',
validators=[
validators.DataRequired(),
validate_folder_path,
],
tooltip="Indicate a folder full of images. For each image in the feature"
" image folder there must be one corresponding image in the label"
" image folder. The label image must have the same filename except"
" for the extension, which may differ.")
folder_pct_val = utils.forms.IntegerField(
'% for validation',
default=10,
validators=[validators.NumberRange(min=0, max=100)],
tooltip="You can choose to set apart a certain percentage of images "
"from the training images for the validation set.")
has_val_folder = utils.forms.BooleanField(
'Separate validation images',
default=False,
)
validation_feature_folder = utils.forms.StringField(
'Validation feature image folder',
validators=[
validate_required_iff(has_val_folder=True),
validate_folder_path,
],
tooltip="Indicate a folder full of images.")
validation_label_folder = utils.forms.StringField(
'Validation label image folder',
validators=[
validate_required_iff(has_val_folder=True),
validate_folder_path,
],
tooltip="Indicate a folder full of images. For each image in the feature"
" image folder there must be one corresponding image in the label"
" image folder. The label image must have the same filename except"
" for the extension, which may differ.")
channel_conversion = utils.forms.SelectField(
'Channel conversion',
choices=[
('RGB', 'RGB'),
('L', 'Grayscale'),
('none', 'None'),
],
default='none',
tooltip="Perform selected channel conversion.")
class ImageClassificationDatasetForm(ImageDatasetForm):
"""
Defines the form used to create a new ImageClassificationDatasetJob
"""
# Use a SelectField instead of a HiddenField so that the default value
# is used when nothing is provided (through the REST API)
method = wtforms.SelectField(
u'Dataset type',
choices=[
('folder', 'Folder'),
('textfile', 'Textfiles'),
],
default='folder',
)
def validate_folder_path(form, field):
if not field.data:
pass
elif utils.is_url(field.data):
# make sure the URL exists
try:
r = requests.get(field.data,
allow_redirects=False,
timeout=utils.HTTP_TIMEOUT)
if r.status_code not in [
requests.codes.ok, requests.codes.moved,
requests.codes.found
]:
raise validators.ValidationError('URL not found')
except Exception as e:
raise validators.ValidationError(
'Caught %s while checking URL: %s' % (type(e).__name__, e))
else:
return True
else:
# make sure the filesystem path exists
if not os.path.exists(field.data) or not os.path.isdir(field.data):
raise validators.ValidationError('Folder does not exist')
else:
return True
### Method - folder
folder_train = wtforms.StringField(
u'Training Images',
validators=[
validate_required_iff(method='folder'),
validate_folder_path,
])
folder_pct_val = wtforms.IntegerField(
u'% for validation',
default=25,
validators=[
validate_required_iff(method='folder'),
validators.NumberRange(min=0, max=100)
])
folder_pct_test = wtforms.IntegerField(
u'% for testing',
default=0,
validators=[
validate_required_iff(method='folder'),
validators.NumberRange(min=0, max=100)
])
has_val_folder = wtforms.BooleanField(
'Separate validation images folder',
default=False,
validators=[validate_required_iff(method='folder')])
folder_val = wtforms.StringField(u'Validation Images',
validators=[
validate_required_iff(
method='folder',
has_val_folder=True),
validate_folder_path,
])
has_test_folder = wtforms.BooleanField(
'Separate test images folder',
default=False,
validators=[validate_required_iff(method='folder')])
folder_test = wtforms.StringField(u'Test Images',
validators=[
validate_required_iff(
method='folder',
has_test_folder=True),
validate_folder_path,
])
### Method - textfile
textfile_train_images = wtforms.FileField(
u'Training images',
validators=[validate_required_iff(method='textfile')])
textfile_train_folder = wtforms.StringField(u'Training images folder')
def validate_textfile_train_folder(form, field):
if form.method.data != 'textfile':
field.errors[:] = []
raise validators.StopValidation()
if not field.data.strip():
# allow null
return True
if not os.path.exists(field.data) or not os.path.isdir(field.data):
raise validators.ValidationError('folder does not exist')
return True
textfile_use_val = wtforms.BooleanField(
u'Validation set',
default=True,
validators=[validate_required_iff(method='textfile')])
textfile_val_images = wtforms.FileField(u'Validation images',
validators=[
validate_required_iff(
method='textfile',
textfile_use_val=True)
])
textfile_val_folder = wtforms.StringField(u'Validation images folder')
def validate_textfile_val_folder(form, field):
if form.method.data != 'textfile' or not form.textfile_use_val.data:
field.errors[:] = []
raise validators.StopValidation()
if not field.data.strip():
# allow null
return True
if not os.path.exists(field.data) or not os.path.isdir(field.data):
raise validators.ValidationError('folder does not exist')
return True
textfile_use_test = wtforms.BooleanField(
u'Test set',
default=False,
validators=[validate_required_iff(method='textfile')])
textfile_test_images = wtforms.FileField(u'Test images',
validators=[
validate_required_iff(
method='textfile',
textfile_use_test=True)
])
textfile_test_folder = wtforms.StringField(u'Test images folder')
def validate_textfile_test_folder(form, field):
if form.method.data != 'textfile' or not form.textfile_use_test.data:
field.errors[:] = []
raise validators.StopValidation()
if not field.data.strip():
# allow null
return True
if not os.path.exists(field.data) or not os.path.isdir(field.data):
raise validators.ValidationError('folder does not exist')
return True
# Can't use a BooleanField here because HTML doesn't submit anything
# for an unchecked checkbox. Since we want to use a REST API and have
# this default to True when nothing is supplied, we have to use a
# SelectField
textfile_shuffle = wtforms.SelectField(
'Shuffle lines',
choices=[
(1, 'Yes'),
(0, 'No'),
],
coerce=int,
default=1,
)
textfile_labels_file = wtforms.FileField(
u'Labels', validators=[validate_required_iff(method='textfile')])
class DatasetForm(Form):
"""
A form used to create an image processing dataset
"""
def validate_folder_path(form, field):
if not field.data:
pass
else:
# make sure the filesystem path exists
if not os.path.exists(field.data) or not os.path.isdir(field.data):
raise validators.ValidationError(
'Folder does not exist or is not reachable')
else:
return True
def validate_file_path(form, field):
if not field.data:
pass
else:
# make sure the filesystem path exists
if not os.path.exists(field.data) and not os.path.isdir(
field.data):
raise validators.ValidationError(
'File does not exist or is not reachable')
else:
return True
feature_folder = utils.forms.StringField(
'Feature image folder',
validators=[
validators.DataRequired(),
validate_folder_path,
],
tooltip="Indicate a folder full of images.")
label_folder = utils.forms.StringField(
'Label image folder',
validators=[
validators.DataRequired(),
validate_folder_path,
],
tooltip="Indicate a folder full of images. For each image in the feature"
" image folder there must be one corresponding image in the label"
" image folder. The label image must have the same filename except"
" for the extension, which may differ. Label images are expected"
" to be single-channel images (paletted or grayscale), or RGB"
" images, in which case the color/class mappings need to be"
" specified through a separate text file.")
folder_pct_val = utils.forms.IntegerField(
'% for validation',
default=10,
validators=[validators.NumberRange(min=0, max=100)],
tooltip="You can choose to set apart a certain percentage of images "
"from the training images for the validation set.")
has_val_folder = utils.forms.BooleanField(
'Separate validation images',
default=False,
)
validation_feature_folder = utils.forms.StringField(
'Validation feature image folder',
validators=[
validate_required_iff(has_val_folder=True),
validate_folder_path,
],
tooltip="Indicate a folder full of images.")
validation_label_folder = utils.forms.StringField(
'Validation label image folder',
validators=[
validate_required_iff(has_val_folder=True),
validate_folder_path,
],
tooltip="Indicate a folder full of images. For each image in the feature"
" image folder there must be one corresponding image in the label"
" image folder. The label image must have the same filename except"
" for the extension, which may differ. Label images are expected"
" to be single-channel images (paletted or grayscale), or RGB"
" images, in which case the color/class mappings need to be"
" specified through a separate text file.")
channel_conversion = utils.forms.SelectField(
'Channel conversion',
choices=[
('RGB', 'RGB'),
('L', 'Grayscale'),
('none', 'None'),
],
default='none',
tooltip="Perform selected channel conversion on feature images. Label"
" images are single channel and not affected by this parameter.")
class_labels_file = utils.forms.StringField(
'Class labels (optional)',
validators=[
validate_file_path,
],
tooltip="The 'i'th line of the file should give the string label "
"associated with the '(i-1)'th numeric label. (E.g. the "
"string label for the numeric label 0 is supposed to be "
"on line 1.)")
colormap_method = utils.forms.SelectField(
'Color map specification',
choices=[
('label', 'From label image'),
('textfile', 'From text file'),
],
default='label',
tooltip="Specify how to map class IDs to colors. Select 'From label "
"image' to use palette or grayscale from label images. For "
"RGB image labels, select 'From text file' and provide "
"color map in separate text file.")
colormap_text_file = utils.forms.StringField(
'Color map file',
validators=[
validate_required_iff(colormap_method="textfile"),
validate_file_path,
],
tooltip="Specify color/class mappings through a text file. "
"Each line in the file should contain three space-separated "
"integer values, one for each of the Red, Green, Blue "
"channels. The 'i'th line of the file should give the color "
"associated with the '(i-1)'th class. (E.g. the "
"color for class #0 is supposed to be on line 1.)")