class InferenceForm(Form): """ A form used to perform inference on a gradient regression model """ gradient_x = utils.forms.FloatField( 'Gradient (x)', validators=[ validate_required_iff(test_image_count=None), validators.NumberRange(min=-0.5, max=0.5), ], tooltip="Specify a number between -0.5 and 0.5") gradient_y = utils.forms.FloatField( 'Gradient (y)', validators=[ validate_required_iff(test_image_count=None), validators.NumberRange(min=-0.5, max=0.5), ], tooltip="Specify a number between -0.5 and 0.5") test_image_count = utils.forms.IntegerField( 'Test Image count', validators=[ validators.Optional(), validators.NumberRange(min=0), ], tooltip="Number of images to create in test set")
class GenericImageDatasetForm(ImageDatasetForm): """ Defines the form used to create a new GenericImageDatasetJob """ # Use a SelectField instead of a HiddenField so that the default value # is used when nothing is provided (through the REST API) method = wtforms.SelectField( u'Dataset type', choices=[ ('prebuilt', 'Prebuilt'), ], default='prebuilt', ) def validate_lmdb_path(form, field): if not field.data: pass else: # make sure the filesystem path exists if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError('Folder does not exist') def validate_file_path(form, field): if not field.data: pass else: # make sure the filesystem path exists if not os.path.exists(field.data) or not os.path.isfile( field.data): raise validators.ValidationError('File does not exist') ### Method - prebuilt prebuilt_train_images = wtforms.StringField( 'Training Images', validators=[ validate_required_iff(method='prebuilt'), validate_lmdb_path, ]) prebuilt_train_labels = wtforms.StringField('Training Labels', validators=[ validate_lmdb_path, ]) prebuilt_val_images = wtforms.StringField('Validation Images', validators=[ validate_lmdb_path, ]) prebuilt_val_labels = wtforms.StringField('Validation Labels', validators=[ validate_lmdb_path, ]) prebuilt_mean_file = wtforms.StringField('Mean Image', validators=[ validate_file_path, ])
class ImageClassificationDatasetForm(ImageDatasetForm): """ Defines the form used to create a new ImageClassificationDatasetJob """ backend = wtforms.SelectField('DB backend', choices = [ ('lmdb', 'LMDB'), ('hdf5', 'HDF5'), ], default='lmdb', ) def validate_backend(form, field): if field.data == 'lmdb': form.compression.data = 'none' elif field.data == 'hdf5': form.encoding.data = 'none' compression = utils.forms.SelectField('DB compression', choices = [ ('none', 'None'), ('gzip', 'GZIP'), ], default='none', tooltip='Compressing the dataset may significantly decrease the size of your database files, but it may increase read and write times.', ) # Use a SelectField instead of a HiddenField so that the default value # is used when nothing is provided (through the REST API) method = wtforms.SelectField(u'Dataset type', choices = [ ('folder', 'Folder'), ('textfile', 'Textfiles'), ], default='folder', ) def validate_folder_path(form, field): if not field.data: pass elif utils.is_url(field.data): # make sure the URL exists try: r = requests.get(field.data, allow_redirects=False, timeout=utils.HTTP_TIMEOUT) if r.status_code not in [requests.codes.ok, requests.codes.moved, requests.codes.found]: raise validators.ValidationError('URL not found') except Exception as e: raise validators.ValidationError('Caught %s while checking URL: %s' % (type(e).__name__, e)) else: return True else: # make sure the filesystem path exists if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError('Folder does not exist') else: return True ### Method - folder folder_train = utils.forms.StringField(u'Training Images', validators=[ validate_required_iff(method='folder'), validate_folder_path, ], tooltip = "Indicate a folder which holds subfolders full of images. Each subfolder should be named according to the desired label for the images that it holds. Can also be a URL for an apache/nginx auto-indexed folder." ) folder_pct_val = utils.forms.IntegerField(u'% for validation', default=25, validators=[ validate_required_iff(method='folder'), validators.NumberRange(min=0, max=100) ], tooltip = "You can choose to set apart a certain percentage of images from the training images for the validation set." ) folder_pct_test = utils.forms.IntegerField(u'% for testing', default=0, validators=[ validate_required_iff(method='folder'), validators.NumberRange(min=0, max=100) ], tooltip = "You can choose to set apart a certain percentage of images from the training images for the test set." ) folder_train_min_per_class = utils.forms.IntegerField(u'Minimum samples per class', default=2, validators=[ validators.Optional(), validators.NumberRange(min=1), ], tooltip = "You can choose to specify a minimum number of samples per class. If a class has fewer samples than the specified amount it will be ignored. Leave blank to ignore this feature." ) folder_train_max_per_class = utils.forms.IntegerField(u'Maximum samples per class', validators=[ validators.Optional(), validators.NumberRange(min=1), validate_greater_than('folder_train_min_per_class'), ], tooltip = "You can choose to specify a maximum number of samples per class. If a class has more samples than the specified amount extra samples will be ignored. Leave blank to ignore this feature." ) has_val_folder = wtforms.BooleanField('Separate validation images folder', default = False, validators=[ validate_required_iff(method='folder') ] ) folder_val = wtforms.StringField(u'Validation Images', validators=[ validate_required_iff( method='folder', has_val_folder=True), ] ) folder_val_min_per_class = utils.forms.IntegerField(u'Minimum samples per class', default=2, validators=[ validators.Optional(), validators.NumberRange(min=1), ], tooltip = "You can choose to specify a minimum number of samples per class. If a class has fewer samples than the specified amount it will be ignored. Leave blank to ignore this feature." ) folder_val_max_per_class = utils.forms.IntegerField(u'Maximum samples per class', validators=[ validators.Optional(), validators.NumberRange(min=1), validate_greater_than('folder_val_min_per_class'), ], tooltip = "You can choose to specify a maximum number of samples per class. If a class has more samples than the specified amount extra samples will be ignored. Leave blank to ignore this feature." ) has_test_folder = wtforms.BooleanField('Separate test images folder', default = False, validators=[ validate_required_iff(method='folder') ] ) folder_test = wtforms.StringField(u'Test Images', validators=[ validate_required_iff( method='folder', has_test_folder=True), validate_folder_path, ] ) folder_test_min_per_class = utils.forms.IntegerField(u'Minimum samples per class', default=2, validators=[ validators.Optional(), validators.NumberRange(min=1) ], tooltip = "You can choose to specify a minimum number of samples per class. If a class has fewer samples than the specified amount it will be ignored. Leave blank to ignore this feature." ) folder_test_max_per_class = utils.forms.IntegerField(u'Maximum samples per class', validators=[ validators.Optional(), validators.NumberRange(min=1), validate_greater_than('folder_test_min_per_class'), ], tooltip = "You can choose to specify a maximum number of samples per class. If a class has more samples than the specified amount extra samples will be ignored. Leave blank to ignore this feature." ) ### Method - textfile textfile_use_local_files = wtforms.BooleanField(u'Use local files', default=False) textfile_train_images = utils.forms.FileField(u'Training images', validators=[ validate_required_iff(method='textfile', textfile_use_local_files=False) ] ) textfile_local_train_images = wtforms.StringField(u'Training images', validators=[ validate_required_iff(method='textfile', textfile_use_local_files=True) ] ) textfile_train_folder = wtforms.StringField(u'Training images folder') def validate_textfile_train_folder(form, field): if form.method.data != 'textfile': field.errors[:] = [] raise validators.StopValidation() if not field.data.strip(): # allow null return True if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError('folder does not exist') return True textfile_use_val = wtforms.BooleanField(u'Validation set', default=True, validators=[ validate_required_iff(method='textfile') ] ) textfile_val_images = utils.forms.FileField(u'Validation images', validators=[ validate_required_iff( method='textfile', textfile_use_val=True, textfile_use_local_files=False) ] ) textfile_local_val_images = wtforms.StringField(u'Validation images', validators=[ validate_required_iff( method='textfile', textfile_use_val=True, textfile_use_local_files=True) ] ) textfile_val_folder = wtforms.StringField(u'Validation images folder') def validate_textfile_val_folder(form, field): if form.method.data != 'textfile' or not form.textfile_use_val.data: field.errors[:] = [] raise validators.StopValidation() if not field.data.strip(): # allow null return True if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError('folder does not exist') return True textfile_use_test = wtforms.BooleanField(u'Test set', default=False, validators=[ validate_required_iff(method='textfile') ] ) textfile_test_images = utils.forms.FileField(u'Test images', validators=[ validate_required_iff( method='textfile', textfile_use_test=True, textfile_use_local_files=False) ] ) textfile_local_test_images = wtforms.StringField(u'Test images', validators=[ validate_required_iff( method='textfile', textfile_use_test=True, textfile_use_local_files=True) ] ) textfile_test_folder = wtforms.StringField(u'Test images folder') def validate_textfile_test_folder(form, field): if form.method.data != 'textfile' or not form.textfile_use_test.data: field.errors[:] = [] raise validators.StopValidation() if not field.data.strip(): # allow null return True if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError('folder does not exist') return True # Can't use a BooleanField here because HTML doesn't submit anything # for an unchecked checkbox. Since we want to use a REST API and have # this default to True when nothing is supplied, we have to use a # SelectField textfile_shuffle = utils.forms.SelectField('Shuffle lines', choices = [ (1, 'Yes'), (0, 'No'), ], coerce=int, default=1, tooltip = "Shuffle the list[s] of images before creating the database." ) textfile_labels_file = utils.forms.FileField(u'Labels', validators=[ validate_required_iff(method='textfile', textfile_use_local_files=False) ], tooltip = "The 'i'th line of the file should give the string label associated with the '(i-1)'th numberic label. (E.g. the string label for the numeric label 0 is supposed to be on line 1.)" ) textfile_local_labels_file = utils.forms.StringField(u'Labels', validators=[ validate_required_iff(method='textfile', textfile_use_local_files=True) ], tooltip = "The 'i'th line of the file should give the string label associated with the '(i-1)'th numberic label. (E.g. the string label for the numeric label 0 is supposed to be on line 1.)" )
class ModelForm(Form): ### Methods def selection_exists_in_choices(form, field): found = False for choice in field.choices: if choice[0] == field.data: found = True if not found: raise validators.ValidationError( "Selected job doesn't exist. Maybe it was deleted by another user." ) def validate_NetParameter(form, field): fw = frameworks.get_framework_by_id(form['framework'].data) try: # below function raises a BadNetworkException in case of validation error fw.validate_network(field.data) except frameworks.errors.BadNetworkError as e: raise validators.ValidationError('Bad network: %s' % e.message) def validate_file_exists(form, field): from_client = bool(form.python_layer_from_client.data) filename = '' if not from_client and field.type == 'StringField': filename = field.data if filename == '': return if not os.path.isfile(filename): raise validators.ValidationError( 'Server side file, %s, does not exist.' % filename) def validate_py_ext(form, field): from_client = bool(form.python_layer_from_client.data) filename = '' if from_client and field.type == 'FileField': filename = flask.request.files[field.name].filename elif not from_client and field.type == 'StringField': filename = field.data if filename == '': return (root, ext) = os.path.splitext(filename) if ext != '.py' and ext != '.pyc': raise validators.ValidationError( 'Python file, %s, needs .py or .pyc extension.' % filename) ### Fields # The options for this get set in the view (since they are dynamic) dataset = utils.forms.SelectField( 'Select Dataset', choices=[], tooltip="Choose the dataset to use for this model.") python_layer_from_client = utils.forms.BooleanField( u'Use client-side file', default=False) python_layer_client_file = utils.forms.FileField( u'Client-side file', validators=[validate_py_ext], tooltip= "Choose a Python file on the client containing layer definitions.") python_layer_server_file = utils.forms.StringField( u'Server-side file', validators=[validate_file_exists, validate_py_ext], tooltip= "Choose a Python file on the server containing layer definitions.") train_epochs = utils.forms.IntegerField( 'Training epochs', validators=[validators.NumberRange(min=1)], default=30, tooltip="How many passes through the training data?") snapshot_interval = utils.forms.FloatField( 'Snapshot interval (in epochs)', default=1, validators=[ validators.NumberRange(min=0), ], tooltip="How many epochs of training between taking a snapshot?") val_interval = utils.forms.FloatField( 'Validation interval (in epochs)', default=1, validators=[validators.NumberRange(min=0)], tooltip= "How many epochs of training between running through one pass of the validation data?" ) random_seed = utils.forms.IntegerField( 'Random seed', validators=[ validators.NumberRange(min=0), validators.Optional(), ], tooltip= "If you provide a random seed, then back-to-back runs with the same model and dataset should give identical results." ) batch_size = utils.forms.MultiIntegerField( 'Batch size', validators=[ utils.forms.MultiNumberRange(min=1), utils.forms.MultiOptional(), ], tooltip= "How many images to process at once. If blank, values are used from the network definition." ) batch_accumulation = utils.forms.IntegerField( 'Batch Accumulation', validators=[ validators.NumberRange(min=1), validators.Optional(), ], tooltip= "Accumulate gradients over multiple batches (useful when you need a bigger batch size for training but it doesn't fit in memory)." ) ### Solver types solver_type = utils.forms.SelectField( 'Solver type', choices=[ ('SGD', 'Stochastic gradient descent (SGD)'), ('NESTEROV', "Nesterov's accelerated gradient (NAG)"), ('ADAGRAD', 'Adaptive gradient (AdaGrad)'), ('RMSPROP', 'RMSprop'), ('ADADELTA', 'AdaDelta'), ('ADAM', 'Adam'), ], default='SGD', tooltip="What type of solver will be used?", ) def validate_solver_type(form, field): fw = frameworks.get_framework_by_id(form.framework) if fw is not None: if not fw.supports_solver_type(field.data): raise validators.ValidationError( 'Solver type not supported by this framework') ### Learning rate learning_rate = utils.forms.MultiFloatField( 'Base Learning Rate', default=0.01, validators=[ utils.forms.MultiNumberRange(min=0), ], tooltip= "Affects how quickly the network learns. If you are getting NaN for your loss, you probably need to lower this value." ) lr_policy = wtforms.SelectField('Policy', choices=[ ('fixed', 'Fixed'), ('step', 'Step Down'), ('multistep', 'Step Down (arbitrary steps)'), ('exp', 'Exponential Decay'), ('inv', 'Inverse Decay'), ('poly', 'Polynomial Decay'), ('sigmoid', 'Sigmoid Decay'), ], default='step') lr_step_size = wtforms.FloatField('Step Size', default=33) lr_step_gamma = wtforms.FloatField('Gamma', default=0.1) lr_multistep_values = wtforms.StringField('Step Values', default="50,85") def validate_lr_multistep_values(form, field): if form.lr_policy.data == 'multistep': for value in field.data.split(','): try: float(value) except ValueError: raise validators.ValidationError('invalid value') lr_multistep_gamma = wtforms.FloatField('Gamma', default=0.5) lr_exp_gamma = wtforms.FloatField('Gamma', default=0.95) lr_inv_gamma = wtforms.FloatField('Gamma', default=0.1) lr_inv_power = wtforms.FloatField('Power', default=0.5) lr_poly_power = wtforms.FloatField('Power', default=3) lr_sigmoid_step = wtforms.FloatField('Step', default=50) lr_sigmoid_gamma = wtforms.FloatField('Gamma', default=0.1) ### Network # Use a SelectField instead of a HiddenField so that the default value # is used when nothing is provided (through the REST API) method = wtforms.SelectField( u'Network type', choices=[ ('standard', 'Standard network'), ('previous', 'Previous network'), ('pretrained', 'Pretrained network'), ('custom', 'Custom network'), ], default='standard', ) ## framework - hidden field, set by Javascript to the selected framework ID framework = wtforms.HiddenField( 'framework', validators=[ validators.AnyOf( [fw.get_id() for fw in frameworks.get_frameworks()], message='The framework you choose is not currently supported.') ], default=frameworks.get_frameworks()[0].get_id()) # The options for this get set in the view (since they are dependent on the data type) standard_networks = wtforms.RadioField( 'Standard Networks', validators=[ validate_required_iff(method='standard'), ], ) previous_networks = wtforms.RadioField( 'Previous Networks', choices=[], validators=[ validate_required_iff(method='previous'), selection_exists_in_choices, ], ) pretrained_networks = wtforms.RadioField( 'Pretrained Networks', choices=[], validators=[ validate_required_iff(method='pretrained'), selection_exists_in_choices, ], ) custom_network = utils.forms.TextAreaField( 'Custom Network', validators=[ validate_required_iff(method='custom'), validate_NetParameter, ], ) custom_network_snapshot = utils.forms.TextField( 'Pretrained model(s)', tooltip= "Paths to pretrained model files, separated by '%s'. Only edit this field if you understand how fine-tuning works in caffe or torch." % os.path.pathsep) def validate_custom_network_snapshot(form, field): if form.method.data == 'custom': for filename in field.data.strip().split(os.path.pathsep): if filename and not os.path.exists(filename): raise validators.ValidationError( 'File "%s" does not exist' % filename) # Select one of several GPUs select_gpu = wtforms.RadioField( 'Select which GPU you would like to use', choices=[('next', 'Next available')] + [( index, '#%s - %s (%s memory)' % (index, get_device(index).name, sizeof_fmt( get_nvml_info(index)['memory']['total'] if get_nvml_info(index) and 'memory' in get_nvml_info(index) else get_device(index).totalGlobalMem)), ) for index in config_value('gpu_list').split(',') if index], default='next', ) # Select N of several GPUs select_gpus = utils.forms.SelectMultipleField( 'Select which GPU[s] you would like to use', choices=[( index, '#%s - %s (%s memory)' % (index, get_device(index).name, sizeof_fmt( get_nvml_info(index)['memory']['total'] if get_nvml_info(index) and 'memory' in get_nvml_info(index) else get_device(index).totalGlobalMem)), ) for index in config_value('gpu_list').split(',') if index], tooltip= "The job won't start until all of the chosen GPUs are available.") # XXX For testing # The Flask test framework can't handle SelectMultipleFields correctly select_gpus_list = wtforms.StringField( 'Select which GPU[s] you would like to use (comma separated)') def validate_select_gpus(form, field): if form.select_gpus_list.data: field.data = form.select_gpus_list.data.split(',') # Use next available N GPUs select_gpu_count = wtforms.IntegerField( 'Use this many GPUs (next available)', validators=[ validators.NumberRange(min=1, max=len( config_value('gpu_list').split(','))) ], default=1, ) def validate_select_gpu_count(form, field): if field.data is None: if form.select_gpus.data: # Make this field optional field.errors[:] = [] raise validators.StopValidation() model_name = utils.forms.StringField( 'Model Name', validators=[validators.DataRequired()], tooltip= "An identifier, later used to refer to this model in the Application.") # allows shuffling data during training (for frameworks that support this, as indicated by # their Framework.can_shuffle_data() method) shuffle = utils.forms.BooleanField( 'Shuffle Train Data', default=True, tooltip='For every epoch, shuffle the data before training.')
class ModelForm(Form): ### Methods def selection_exists_in_choices(form, field): found = False for choice in field.choices: if choice[0] == field.data: found = True if not found: raise validators.ValidationError( "Selected job doesn't exist. Maybe it was deleted by another user." ) def validate_NetParameter(form, field): pb = caffe_pb2.NetParameter() try: text_format.Merge(field.data, pb) except text_format.ParseError as e: raise validators.ValidationError('Not a valid NetParameter: %s' % e) ### Fields # The options for this get set in the view (since they are dynamic) dataset = wtforms.SelectField('Select Dataset', choices=[]) train_epochs = wtforms.IntegerField( 'Training epochs', validators=[validators.NumberRange(min=1)], default=30, ) snapshot_interval = wtforms.FloatField( 'Snapshot interval (in epochs)', default=1, validators=[ validators.NumberRange(min=0), ], ) val_interval = wtforms.FloatField( 'Validation interval (in epochs)', default=1, validators=[validators.NumberRange(min=0)], ) random_seed = wtforms.IntegerField( 'Random seed', validators=[ validators.NumberRange(min=0), validators.Optional(), ], ) batch_size = wtforms.IntegerField( 'Batch size', validators=[ validators.NumberRange(min=1), validators.Optional(), ], ) ### Solver types solver_type = wtforms.SelectField( 'Solver type', choices=[ ('SGD', 'Stochastic gradient descent (SGD)'), ('ADAGRAD', 'Adaptive gradient (AdaGrad)'), ('NESTEROV', "Nesterov's accelerated gradient (NAG)"), ], default='SGD') ### Learning rate learning_rate = wtforms.FloatField('Base Learning Rate', default=0.01, validators=[ validators.NumberRange(min=0), ]) lr_policy = wtforms.SelectField('Policy', choices=[ ('fixed', 'Fixed'), ('step', 'Step Down'), ('multistep', 'Step Down (arbitrary steps)'), ('exp', 'Exponential Decay'), ('inv', 'Inverse Decay'), ('poly', 'Polynomial Decay'), ('sigmoid', 'Sigmoid Decay'), ], default='step') lr_step_size = wtforms.FloatField('Step Size', default=33) lr_step_gamma = wtforms.FloatField('Gamma', default=0.1) lr_multistep_values = wtforms.StringField('Step Values', default="50,85") def validate_lr_multistep_values(form, field): if form.lr_policy.data == 'multistep': for value in field.data.split(','): try: float(value) except ValueError: raise validators.ValidationError('invalid value') lr_multistep_gamma = wtforms.FloatField('Gamma', default=0.5) lr_exp_gamma = wtforms.FloatField('Gamma', default=0.95) lr_inv_gamma = wtforms.FloatField('Gamma', default=0.1) lr_inv_power = wtforms.FloatField('Power', default=0.5) lr_poly_power = wtforms.FloatField('Power', default=3) lr_sigmoid_step = wtforms.FloatField('Step', default=50) lr_sigmoid_gamma = wtforms.FloatField('Gamma', default=0.1) ### Network # Use a SelectField instead of a HiddenField so that the default value # is used when nothing is provided (through the REST API) method = wtforms.SelectField( u'Network type', choices=[ ('standard', 'Standard network'), ('previous', 'Previous network'), ('custom', 'Custom network'), ], default='standard', ) # The options for this get set in the view (since they are dependent on the data type) standard_networks = wtforms.RadioField( 'Standard Networks', validators=[ validate_required_iff(method='standard'), ], ) previous_networks = wtforms.RadioField( 'Previous Networks', choices=[], validators=[ validate_required_iff(method='previous'), selection_exists_in_choices, ], ) custom_network = wtforms.TextAreaField( 'Custom Network', validators=[ validate_required_iff(method='custom'), validate_NetParameter, ]) custom_network_snapshot = wtforms.TextField('Pretrained model') def validate_custom_network_snapshot(form, field): if form.method.data == 'custom': snapshot = field.data.strip() if snapshot: if not os.path.exists(snapshot): raise validators.ValidationError('File does not exist') # Select one of several GPUs select_gpu = wtforms.RadioField( 'Select which GPU you would like to use', choices=[('next', 'Next available')] + [( index, '#%s - %s%s' % ( index, get_device(index).name, ' (%s memory)' % sizeof_fmt(get_nvml_info(index)['memory']['total']) if get_nvml_info(index) and 'memory' in get_nvml_info(index) else '', ), ) for index in config_value('gpu_list').split(',') if index], default='next', ) # Select N of several GPUs select_gpus = wtforms.SelectMultipleField( 'Select which GPU[s] you would like to use', choices=[( index, '#%s - %s%s' % ( index, get_device(index).name, ' (%s memory)' % sizeof_fmt(get_nvml_info(index)['memory']['total']) if get_nvml_info(index) and 'memory' in get_nvml_info(index) else '', ), ) for index in config_value('gpu_list').split(',') if index]) # XXX For testing # The Flask test framework can't handle SelectMultipleFields correctly select_gpus_list = wtforms.StringField( 'Select which GPU[s] you would like to use (comma separated)') def validate_select_gpus(form, field): if form.select_gpus_list.data: field.data = form.select_gpus_list.data.split(',') # Use next available N GPUs select_gpu_count = wtforms.IntegerField( 'Use this many GPUs (next available)', validators=[ validators.NumberRange(min=1, max=len( config_value('gpu_list').split(','))) ], default=1, ) def validate_select_gpu_count(form, field): if field.data is None: if form.select_gpus.data: # Make this field optional field.errors[:] = [] raise validators.StopValidation() model_name = wtforms.StringField('Model Name', validators=[validators.DataRequired()])
class GenericImageDatasetForm(ImageDatasetForm): """ Defines the form used to create a new GenericImageDatasetJob """ # Use a SelectField instead of a HiddenField so that the default value # is used when nothing is provided (through the REST API) method = wtforms.SelectField( u'Dataset type', choices=[ ('prebuilt', 'Prebuilt'), ], default='prebuilt', ) def validate_lmdb_path(form, field): if not field.data: pass else: # make sure the filesystem path exists if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError('Folder does not exist') def validate_file_path(form, field): if not field.data: pass else: # make sure the filesystem path exists if not os.path.exists(field.data) or not os.path.isfile( field.data): raise validators.ValidationError('File does not exist') ### Method - prebuilt prebuilt_train_images = wtforms.StringField( 'Training Images', validators=[ validate_required_iff(method='prebuilt'), validate_lmdb_path, ]) prebuilt_train_labels = wtforms.StringField('Training Labels', validators=[ validate_lmdb_path, ]) prebuilt_val_images = wtforms.StringField('Validation Images', validators=[ validate_lmdb_path, ]) prebuilt_val_labels = wtforms.StringField('Validation Labels', validators=[ validate_lmdb_path, ]) # Can't use a BooleanField here because HTML doesn't submit anything # for an unchecked checkbox. Since we want to use a REST API and have # this default to True when nothing is supplied, we have to use a # SelectField force_same_shape = utils.forms.SelectField( 'Enforce same shape', choices=[ (1, 'Yes'), (0, 'No'), ], coerce=int, default=1, tooltip= 'Check that each entry in the database has the same shape (can be time-consuming)' ) prebuilt_mean_file = utils.forms.StringField( 'Mean Image', validators=[ validate_file_path, ], tooltip="Path to a .binaryproto file on the server")
class DatasetForm(Form): """ A form used to create an image processing dataset """ def validate_folder_path(form, field): if not field.data: pass else: # make sure the filesystem path exists if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError( 'Folder does not exist or is not reachable') else: return True feature_folder = utils.forms.StringField( 'Feature image folder', validators=[ validators.DataRequired(), validate_folder_path, ], tooltip="Indicate a folder full of images.") label_folder = utils.forms.StringField( 'Label image folder', validators=[ validators.DataRequired(), validate_folder_path, ], tooltip="Indicate a folder full of images. For each image in the feature" " image folder there must be one corresponding image in the label" " image folder. The label image must have the same filename except" " for the extension, which may differ.") folder_pct_val = utils.forms.IntegerField( '% for validation', default=10, validators=[validators.NumberRange(min=0, max=100)], tooltip="You can choose to set apart a certain percentage of images " "from the training images for the validation set.") has_val_folder = utils.forms.BooleanField( 'Separate validation images', default=False, ) validation_feature_folder = utils.forms.StringField( 'Validation feature image folder', validators=[ validate_required_iff(has_val_folder=True), validate_folder_path, ], tooltip="Indicate a folder full of images.") validation_label_folder = utils.forms.StringField( 'Validation label image folder', validators=[ validate_required_iff(has_val_folder=True), validate_folder_path, ], tooltip="Indicate a folder full of images. For each image in the feature" " image folder there must be one corresponding image in the label" " image folder. The label image must have the same filename except" " for the extension, which may differ.") channel_conversion = utils.forms.SelectField( 'Channel conversion', choices=[ ('RGB', 'RGB'), ('L', 'Grayscale'), ('none', 'None'), ], default='none', tooltip="Perform selected channel conversion.")
class ImageClassificationDatasetForm(ImageDatasetForm): """ Defines the form used to create a new ImageClassificationDatasetJob """ # Use a SelectField instead of a HiddenField so that the default value # is used when nothing is provided (through the REST API) method = wtforms.SelectField( u'Dataset type', choices=[ ('folder', 'Folder'), ('textfile', 'Textfiles'), ], default='folder', ) def validate_folder_path(form, field): if not field.data: pass elif utils.is_url(field.data): # make sure the URL exists try: r = requests.get(field.data, allow_redirects=False, timeout=utils.HTTP_TIMEOUT) if r.status_code not in [ requests.codes.ok, requests.codes.moved, requests.codes.found ]: raise validators.ValidationError('URL not found') except Exception as e: raise validators.ValidationError( 'Caught %s while checking URL: %s' % (type(e).__name__, e)) else: return True else: # make sure the filesystem path exists if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError('Folder does not exist') else: return True ### Method - folder folder_train = wtforms.StringField( u'Training Images', validators=[ validate_required_iff(method='folder'), validate_folder_path, ]) folder_pct_val = wtforms.IntegerField( u'% for validation', default=25, validators=[ validate_required_iff(method='folder'), validators.NumberRange(min=0, max=100) ]) folder_pct_test = wtforms.IntegerField( u'% for testing', default=0, validators=[ validate_required_iff(method='folder'), validators.NumberRange(min=0, max=100) ]) has_val_folder = wtforms.BooleanField( 'Separate validation images folder', default=False, validators=[validate_required_iff(method='folder')]) folder_val = wtforms.StringField(u'Validation Images', validators=[ validate_required_iff( method='folder', has_val_folder=True), validate_folder_path, ]) has_test_folder = wtforms.BooleanField( 'Separate test images folder', default=False, validators=[validate_required_iff(method='folder')]) folder_test = wtforms.StringField(u'Test Images', validators=[ validate_required_iff( method='folder', has_test_folder=True), validate_folder_path, ]) ### Method - textfile textfile_train_images = wtforms.FileField( u'Training images', validators=[validate_required_iff(method='textfile')]) textfile_train_folder = wtforms.StringField(u'Training images folder') def validate_textfile_train_folder(form, field): if form.method.data != 'textfile': field.errors[:] = [] raise validators.StopValidation() if not field.data.strip(): # allow null return True if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError('folder does not exist') return True textfile_use_val = wtforms.BooleanField( u'Validation set', default=True, validators=[validate_required_iff(method='textfile')]) textfile_val_images = wtforms.FileField(u'Validation images', validators=[ validate_required_iff( method='textfile', textfile_use_val=True) ]) textfile_val_folder = wtforms.StringField(u'Validation images folder') def validate_textfile_val_folder(form, field): if form.method.data != 'textfile' or not form.textfile_use_val.data: field.errors[:] = [] raise validators.StopValidation() if not field.data.strip(): # allow null return True if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError('folder does not exist') return True textfile_use_test = wtforms.BooleanField( u'Test set', default=False, validators=[validate_required_iff(method='textfile')]) textfile_test_images = wtforms.FileField(u'Test images', validators=[ validate_required_iff( method='textfile', textfile_use_test=True) ]) textfile_test_folder = wtforms.StringField(u'Test images folder') def validate_textfile_test_folder(form, field): if form.method.data != 'textfile' or not form.textfile_use_test.data: field.errors[:] = [] raise validators.StopValidation() if not field.data.strip(): # allow null return True if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError('folder does not exist') return True # Can't use a BooleanField here because HTML doesn't submit anything # for an unchecked checkbox. Since we want to use a REST API and have # this default to True when nothing is supplied, we have to use a # SelectField textfile_shuffle = wtforms.SelectField( 'Shuffle lines', choices=[ (1, 'Yes'), (0, 'No'), ], coerce=int, default=1, ) textfile_labels_file = wtforms.FileField( u'Labels', validators=[validate_required_iff(method='textfile')])
class DatasetForm(Form): """ A form used to create an image processing dataset """ def validate_folder_path(form, field): if not field.data: pass else: # make sure the filesystem path exists if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError( 'Folder does not exist or is not reachable') else: return True def validate_file_path(form, field): if not field.data: pass else: # make sure the filesystem path exists if not os.path.exists(field.data) and not os.path.isdir( field.data): raise validators.ValidationError( 'File does not exist or is not reachable') else: return True feature_folder = utils.forms.StringField( 'Feature image folder', validators=[ validators.DataRequired(), validate_folder_path, ], tooltip="Indicate a folder full of images.") label_folder = utils.forms.StringField( 'Label image folder', validators=[ validators.DataRequired(), validate_folder_path, ], tooltip="Indicate a folder full of images. For each image in the feature" " image folder there must be one corresponding image in the label" " image folder. The label image must have the same filename except" " for the extension, which may differ. Label images are expected" " to be single-channel images (paletted or grayscale), or RGB" " images, in which case the color/class mappings need to be" " specified through a separate text file.") folder_pct_val = utils.forms.IntegerField( '% for validation', default=10, validators=[validators.NumberRange(min=0, max=100)], tooltip="You can choose to set apart a certain percentage of images " "from the training images for the validation set.") has_val_folder = utils.forms.BooleanField( 'Separate validation images', default=False, ) validation_feature_folder = utils.forms.StringField( 'Validation feature image folder', validators=[ validate_required_iff(has_val_folder=True), validate_folder_path, ], tooltip="Indicate a folder full of images.") validation_label_folder = utils.forms.StringField( 'Validation label image folder', validators=[ validate_required_iff(has_val_folder=True), validate_folder_path, ], tooltip="Indicate a folder full of images. For each image in the feature" " image folder there must be one corresponding image in the label" " image folder. The label image must have the same filename except" " for the extension, which may differ. Label images are expected" " to be single-channel images (paletted or grayscale), or RGB" " images, in which case the color/class mappings need to be" " specified through a separate text file.") channel_conversion = utils.forms.SelectField( 'Channel conversion', choices=[ ('RGB', 'RGB'), ('L', 'Grayscale'), ('none', 'None'), ], default='none', tooltip="Perform selected channel conversion on feature images. Label" " images are single channel and not affected by this parameter.") class_labels_file = utils.forms.StringField( 'Class labels (optional)', validators=[ validate_file_path, ], tooltip="The 'i'th line of the file should give the string label " "associated with the '(i-1)'th numeric label. (E.g. the " "string label for the numeric label 0 is supposed to be " "on line 1.)") colormap_method = utils.forms.SelectField( 'Color map specification', choices=[ ('label', 'From label image'), ('textfile', 'From text file'), ], default='label', tooltip="Specify how to map class IDs to colors. Select 'From label " "image' to use palette or grayscale from label images. For " "RGB image labels, select 'From text file' and provide " "color map in separate text file.") colormap_text_file = utils.forms.StringField( 'Color map file', validators=[ validate_required_iff(colormap_method="textfile"), validate_file_path, ], tooltip="Specify color/class mappings through a text file. " "Each line in the file should contain three space-separated " "integer values, one for each of the Red, Green, Blue " "channels. The 'i'th line of the file should give the color " "associated with the '(i-1)'th class. (E.g. the " "color for class #0 is supposed to be on line 1.)")