class ImageClassificationDatasetForm(ImageDatasetForm): """ Defines the form used to create a new ImageClassificationDatasetJob """ backend = wtforms.SelectField('DB backend', choices = [ ('lmdb', 'LMDB'), ('hdf5', 'HDF5'), ], default='lmdb', ) def validate_backend(form, field): if field.data == 'lmdb': form.compression.data = 'none' elif field.data == 'hdf5': form.encoding.data = 'none' compression = utils.forms.SelectField('DB compression', choices = [ ('none', 'None'), ('gzip', 'GZIP'), ], default='none', tooltip='Compressing the dataset may significantly decrease the size of your database files, but it may increase read and write times.', ) # Use a SelectField instead of a HiddenField so that the default value # is used when nothing is provided (through the REST API) method = wtforms.SelectField(u'Dataset type', choices = [ ('folder', 'Folder'), ('textfile', 'Textfiles'), ], default='folder', ) def validate_folder_path(form, field): if not field.data: pass elif utils.is_url(field.data): # make sure the URL exists try: r = requests.get(field.data, allow_redirects=False, timeout=utils.HTTP_TIMEOUT) if r.status_code not in [requests.codes.ok, requests.codes.moved, requests.codes.found]: raise validators.ValidationError('URL not found') except Exception as e: raise validators.ValidationError('Caught %s while checking URL: %s' % (type(e).__name__, e)) else: return True else: # make sure the filesystem path exists if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError('Folder does not exist') else: return True ### Method - folder folder_train = utils.forms.StringField(u'Training Images', validators=[ validate_required_iff(method='folder'), validate_folder_path, ], tooltip = "Indicate a folder which holds subfolders full of images. Each subfolder should be named according to the desired label for the images that it holds. Can also be a URL for an apache/nginx auto-indexed folder." ) folder_pct_val = utils.forms.IntegerField(u'% for validation', default=25, validators=[ validate_required_iff(method='folder'), validators.NumberRange(min=0, max=100) ], tooltip = "You can choose to set apart a certain percentage of images from the training images for the validation set." ) folder_pct_test = utils.forms.IntegerField(u'% for testing', default=0, validators=[ validate_required_iff(method='folder'), validators.NumberRange(min=0, max=100) ], tooltip = "You can choose to set apart a certain percentage of images from the training images for the test set." ) folder_train_min_per_class = utils.forms.IntegerField(u'Minimum samples per class', default=2, validators=[ validators.Optional(), validators.NumberRange(min=1), ], tooltip = "You can choose to specify a minimum number of samples per class. If a class has fewer samples than the specified amount it will be ignored. Leave blank to ignore this feature." ) folder_train_max_per_class = utils.forms.IntegerField(u'Maximum samples per class', validators=[ validators.Optional(), validators.NumberRange(min=1), validate_greater_than('folder_train_min_per_class'), ], tooltip = "You can choose to specify a maximum number of samples per class. If a class has more samples than the specified amount extra samples will be ignored. Leave blank to ignore this feature." ) has_val_folder = wtforms.BooleanField('Separate validation images folder', default = False, validators=[ validate_required_iff(method='folder') ] ) folder_val = wtforms.StringField(u'Validation Images', validators=[ validate_required_iff( method='folder', has_val_folder=True), ] ) folder_val_min_per_class = utils.forms.IntegerField(u'Minimum samples per class', default=2, validators=[ validators.Optional(), validators.NumberRange(min=1), ], tooltip = "You can choose to specify a minimum number of samples per class. If a class has fewer samples than the specified amount it will be ignored. Leave blank to ignore this feature." ) folder_val_max_per_class = utils.forms.IntegerField(u'Maximum samples per class', validators=[ validators.Optional(), validators.NumberRange(min=1), validate_greater_than('folder_val_min_per_class'), ], tooltip = "You can choose to specify a maximum number of samples per class. If a class has more samples than the specified amount extra samples will be ignored. Leave blank to ignore this feature." ) has_test_folder = wtforms.BooleanField('Separate test images folder', default = False, validators=[ validate_required_iff(method='folder') ] ) folder_test = wtforms.StringField(u'Test Images', validators=[ validate_required_iff( method='folder', has_test_folder=True), validate_folder_path, ] ) folder_test_min_per_class = utils.forms.IntegerField(u'Minimum samples per class', default=2, validators=[ validators.Optional(), validators.NumberRange(min=1) ], tooltip = "You can choose to specify a minimum number of samples per class. If a class has fewer samples than the specified amount it will be ignored. Leave blank to ignore this feature." ) folder_test_max_per_class = utils.forms.IntegerField(u'Maximum samples per class', validators=[ validators.Optional(), validators.NumberRange(min=1), validate_greater_than('folder_test_min_per_class'), ], tooltip = "You can choose to specify a maximum number of samples per class. If a class has more samples than the specified amount extra samples will be ignored. Leave blank to ignore this feature." ) ### Method - textfile textfile_use_local_files = wtforms.BooleanField(u'Use local files', default=False) textfile_train_images = utils.forms.FileField(u'Training images', validators=[ validate_required_iff(method='textfile', textfile_use_local_files=False) ] ) textfile_local_train_images = wtforms.StringField(u'Training images', validators=[ validate_required_iff(method='textfile', textfile_use_local_files=True) ] ) textfile_train_folder = wtforms.StringField(u'Training images folder') def validate_textfile_train_folder(form, field): if form.method.data != 'textfile': field.errors[:] = [] raise validators.StopValidation() if not field.data.strip(): # allow null return True if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError('folder does not exist') return True textfile_use_val = wtforms.BooleanField(u'Validation set', default=True, validators=[ validate_required_iff(method='textfile') ] ) textfile_val_images = utils.forms.FileField(u'Validation images', validators=[ validate_required_iff( method='textfile', textfile_use_val=True, textfile_use_local_files=False) ] ) textfile_local_val_images = wtforms.StringField(u'Validation images', validators=[ validate_required_iff( method='textfile', textfile_use_val=True, textfile_use_local_files=True) ] ) textfile_val_folder = wtforms.StringField(u'Validation images folder') def validate_textfile_val_folder(form, field): if form.method.data != 'textfile' or not form.textfile_use_val.data: field.errors[:] = [] raise validators.StopValidation() if not field.data.strip(): # allow null return True if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError('folder does not exist') return True textfile_use_test = wtforms.BooleanField(u'Test set', default=False, validators=[ validate_required_iff(method='textfile') ] ) textfile_test_images = utils.forms.FileField(u'Test images', validators=[ validate_required_iff( method='textfile', textfile_use_test=True, textfile_use_local_files=False) ] ) textfile_local_test_images = wtforms.StringField(u'Test images', validators=[ validate_required_iff( method='textfile', textfile_use_test=True, textfile_use_local_files=True) ] ) textfile_test_folder = wtforms.StringField(u'Test images folder') def validate_textfile_test_folder(form, field): if form.method.data != 'textfile' or not form.textfile_use_test.data: field.errors[:] = [] raise validators.StopValidation() if not field.data.strip(): # allow null return True if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError('folder does not exist') return True # Can't use a BooleanField here because HTML doesn't submit anything # for an unchecked checkbox. Since we want to use a REST API and have # this default to True when nothing is supplied, we have to use a # SelectField textfile_shuffle = utils.forms.SelectField('Shuffle lines', choices = [ (1, 'Yes'), (0, 'No'), ], coerce=int, default=1, tooltip = "Shuffle the list[s] of images before creating the database." ) textfile_labels_file = utils.forms.FileField(u'Labels', validators=[ validate_required_iff(method='textfile', textfile_use_local_files=False) ], tooltip = "The 'i'th line of the file should give the string label associated with the '(i-1)'th numberic label. (E.g. the string label for the numeric label 0 is supposed to be on line 1.)" ) textfile_local_labels_file = utils.forms.StringField(u'Labels', validators=[ validate_required_iff(method='textfile', textfile_use_local_files=True) ], tooltip = "The 'i'th line of the file should give the string label associated with the '(i-1)'th numberic label. (E.g. the string label for the numeric label 0 is supposed to be on line 1.)" )
class ImageClassificationDatasetForm(ImageDatasetForm): """ Defines the form used to create a new ImageClassificationDatasetJob """ # Use a SelectField instead of a HiddenField so that the default value # is used when nothing is provided (through the REST API) method = wtforms.SelectField( u'Dataset type', choices=[ ('folder', 'Folder'), ('textfile', 'Textfiles'), ], default='folder', ) def validate_folder_path(form, field): if not field.data: pass elif utils.is_url(field.data): # make sure the URL exists try: r = requests.get(field.data, allow_redirects=False, timeout=utils.HTTP_TIMEOUT) if r.status_code not in [ requests.codes.ok, requests.codes.moved, requests.codes.found ]: raise validators.ValidationError('URL not found') except Exception as e: raise validators.ValidationError( 'Caught %s while checking URL: %s' % (type(e).__name__, e)) else: return True else: # make sure the filesystem path exists if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError('Folder does not exist') else: return True ### Method - folder folder_train = wtforms.StringField( u'Training Images', validators=[ validate_required_iff(method='folder'), validate_folder_path, ]) folder_pct_val = wtforms.IntegerField( u'% for validation', default=25, validators=[ validate_required_iff(method='folder'), validators.NumberRange(min=0, max=100) ]) folder_pct_test = wtforms.IntegerField( u'% for testing', default=0, validators=[ validate_required_iff(method='folder'), validators.NumberRange(min=0, max=100) ]) folder_train_min_per_class = wtforms.IntegerField( u'Minimum samples per class', default=2, validators=[ validators.Optional(), validators.NumberRange(min=1), ]) folder_train_max_per_class = wtforms.IntegerField( u'Maximum samples per class', validators=[ validators.Optional(), validators.NumberRange(min=1), validate_greater_than('folder_train_min_per_class'), ]) has_val_folder = wtforms.BooleanField( 'Separate validation images folder', default=False, validators=[validate_required_iff(method='folder')]) folder_val = wtforms.StringField(u'Validation Images', validators=[ validate_required_iff( method='folder', has_val_folder=True), ]) folder_val_min_per_class = wtforms.IntegerField( u'Minimum samples per class', default=2, validators=[ validators.Optional(), validators.NumberRange(min=1), ]) folder_val_max_per_class = wtforms.IntegerField( u'Maximum samples per class', validators=[ validators.Optional(), validators.NumberRange(min=1), validate_greater_than('folder_val_min_per_class'), ]) has_test_folder = wtforms.BooleanField( 'Separate test images folder', default=False, validators=[validate_required_iff(method='folder')]) folder_test = wtforms.StringField(u'Test Images', validators=[ validate_required_iff( method='folder', has_test_folder=True), validate_folder_path, ]) folder_test_min_per_class = wtforms.IntegerField( u'Minimum samples per class', default=2, validators=[validators.Optional(), validators.NumberRange(min=1)]) folder_test_max_per_class = wtforms.IntegerField( u'Maximum samples per class', validators=[ validators.Optional(), validators.NumberRange(min=1), validate_greater_than('folder_test_min_per_class'), ]) ### Method - textfile textfile_use_local_files = wtforms.BooleanField(u'Use local files', default=False) textfile_train_images = wtforms.FileField( u'Training images', validators=[ validate_required_iff(method='textfile', textfile_use_local_files=False) ]) textfile_local_train_images = wtforms.StringField( u'Training images', validators=[ validate_required_iff(method='textfile', textfile_use_local_files=True) ]) textfile_train_folder = wtforms.StringField(u'Training images folder') def validate_textfile_train_folder(form, field): if form.method.data != 'textfile': field.errors[:] = [] raise validators.StopValidation() if not field.data.strip(): # allow null return True if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError('folder does not exist') return True textfile_use_val = wtforms.BooleanField( u'Validation set', default=True, validators=[validate_required_iff(method='textfile')]) textfile_val_images = wtforms.FileField( u'Validation images', validators=[ validate_required_iff(method='textfile', textfile_use_val=True, textfile_use_local_files=False) ]) textfile_local_val_images = wtforms.StringField( u'Validation images', validators=[ validate_required_iff(method='textfile', textfile_use_val=True, textfile_use_local_files=True) ]) textfile_val_folder = wtforms.StringField(u'Validation images folder') def validate_textfile_val_folder(form, field): if form.method.data != 'textfile' or not form.textfile_use_val.data: field.errors[:] = [] raise validators.StopValidation() if not field.data.strip(): # allow null return True if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError('folder does not exist') return True textfile_use_test = wtforms.BooleanField( u'Test set', default=False, validators=[validate_required_iff(method='textfile')]) textfile_test_images = wtforms.FileField( u'Test images', validators=[ validate_required_iff(method='textfile', textfile_use_test=True, textfile_use_local_files=False) ]) textfile_local_test_images = wtforms.StringField( u'Test images', validators=[ validate_required_iff(method='textfile', textfile_use_test=True, textfile_use_local_files=True) ]) textfile_test_folder = wtforms.StringField(u'Test images folder') def validate_textfile_test_folder(form, field): if form.method.data != 'textfile' or not form.textfile_use_test.data: field.errors[:] = [] raise validators.StopValidation() if not field.data.strip(): # allow null return True if not os.path.exists(field.data) or not os.path.isdir(field.data): raise validators.ValidationError('folder does not exist') return True # Can't use a BooleanField here because HTML doesn't submit anything # for an unchecked checkbox. Since we want to use a REST API and have # this default to True when nothing is supplied, we have to use a # SelectField textfile_shuffle = wtforms.SelectField( 'Shuffle lines', choices=[ (1, 'Yes'), (0, 'No'), ], coerce=int, default=1, ) textfile_labels_file = wtforms.FileField( u'Labels', validators=[ validate_required_iff(method='textfile', textfile_use_local_files=False) ]) textfile_local_labels_file = wtforms.StringField( u'Labels', validators=[ validate_required_iff(method='textfile', textfile_use_local_files=True) ])