Пример #1
0
class InferenceForm(Form):
    """
    A form used to perform inference on a gradient regression model
    """

    gradient_x = utils.forms.FloatField(
        'Gradient (x)',
        validators=[
            validate_required_iff(test_image_count=None),
            validators.NumberRange(min=-0.5, max=0.5),
        ],
        tooltip="Specify a number between -0.5 and 0.5")

    gradient_y = utils.forms.FloatField(
        'Gradient (y)',
        validators=[
            validate_required_iff(test_image_count=None),
            validators.NumberRange(min=-0.5, max=0.5),
        ],
        tooltip="Specify a number between -0.5 and 0.5")

    test_image_count = utils.forms.IntegerField(
        'Test Image count',
        validators=[
            validators.Optional(),
            validators.NumberRange(min=0),
        ],
        tooltip="Number of images to create in test set")
Пример #2
0
class GenericImageDatasetForm(ImageDatasetForm):
    """
    Defines the form used to create a new GenericImageDatasetJob
    """

    # Use a SelectField instead of a HiddenField so that the default value
    # is used when nothing is provided (through the REST API)
    method = wtforms.SelectField(
        u'Dataset type',
        choices=[
            ('prebuilt', 'Prebuilt'),
        ],
        default='prebuilt',
    )

    def validate_lmdb_path(form, field):
        if not field.data:
            pass
        else:
            # make sure the filesystem path exists
            if not os.path.exists(field.data) or not os.path.isdir(field.data):
                raise validators.ValidationError('Folder does not exist')

    def validate_file_path(form, field):
        if not field.data:
            pass
        else:
            # make sure the filesystem path exists
            if not os.path.exists(field.data) or not os.path.isfile(
                    field.data):
                raise validators.ValidationError('File does not exist')

    ### Method - prebuilt

    prebuilt_train_images = wtforms.StringField(
        'Training Images',
        validators=[
            validate_required_iff(method='prebuilt'),
            validate_lmdb_path,
        ])
    prebuilt_train_labels = wtforms.StringField('Training Labels',
                                                validators=[
                                                    validate_lmdb_path,
                                                ])
    prebuilt_val_images = wtforms.StringField('Validation Images',
                                              validators=[
                                                  validate_lmdb_path,
                                              ])
    prebuilt_val_labels = wtforms.StringField('Validation Labels',
                                              validators=[
                                                  validate_lmdb_path,
                                              ])

    prebuilt_mean_file = wtforms.StringField('Mean Image',
                                             validators=[
                                                 validate_file_path,
                                             ])
Пример #3
0
class ImageClassificationDatasetForm(ImageDatasetForm):
    """
    Defines the form used to create a new ImageClassificationDatasetJob
    """

    backend = wtforms.SelectField('DB backend',
            choices = [
                ('lmdb', 'LMDB'),
                ('hdf5', 'HDF5'),
                ],
            default='lmdb',
            )

    def validate_backend(form, field):
        if field.data == 'lmdb':
            form.compression.data = 'none'
        elif field.data == 'hdf5':
            form.encoding.data = 'none'

    compression = utils.forms.SelectField('DB compression',
            choices = [
                ('none', 'None'),
                ('gzip', 'GZIP'),
                ],
            default='none',
            tooltip='Compressing the dataset may significantly decrease the size of your database files, but it may increase read and write times.',
            )

    # Use a SelectField instead of a HiddenField so that the default value
    # is used when nothing is provided (through the REST API)
    method = wtforms.SelectField(u'Dataset type',
            choices = [
                ('folder', 'Folder'),
                ('textfile', 'Textfiles'),
                ],
            default='folder',
            )

    def validate_folder_path(form, field):
        if not field.data:
            pass
        elif utils.is_url(field.data):
            # make sure the URL exists
            try:
                r = requests.get(field.data,
                        allow_redirects=False,
                        timeout=utils.HTTP_TIMEOUT)
                if r.status_code not in [requests.codes.ok, requests.codes.moved, requests.codes.found]:
                    raise validators.ValidationError('URL not found')
            except Exception as e:
                raise validators.ValidationError('Caught %s while checking URL: %s' % (type(e).__name__, e))
            else:
                return True
        else:
            # make sure the filesystem path exists
            if not os.path.exists(field.data) or not os.path.isdir(field.data):
                raise validators.ValidationError('Folder does not exist')
            else:
                return True

    ### Method - folder

    folder_train = utils.forms.StringField(u'Training Images',
            validators=[
                validate_required_iff(method='folder'),
                validate_folder_path,
                ],
            tooltip = "Indicate a folder which holds subfolders full of images. Each subfolder should be named according to the desired label for the images that it holds. Can also be a URL for an apache/nginx auto-indexed folder."
            )

    folder_pct_val = utils.forms.IntegerField(u'% for validation',
            default=25,
            validators=[
                validate_required_iff(method='folder'),
                validators.NumberRange(min=0, max=100)
                ],
            tooltip = "You can choose to set apart a certain percentage of images from the training images for the validation set."
            )

    folder_pct_test = utils.forms.IntegerField(u'% for testing',
            default=0,
            validators=[
                validate_required_iff(method='folder'),
                validators.NumberRange(min=0, max=100)
                ],
            tooltip = "You can choose to set apart a certain percentage of images from the training images for the test set."
            )

    folder_train_min_per_class = utils.forms.IntegerField(u'Minimum samples per class',
            default=2,
            validators=[
                validators.Optional(),
                validators.NumberRange(min=1),
                ],
            tooltip = "You can choose to specify a minimum number of samples per class. If a class has fewer samples than the specified amount it will be ignored. Leave blank to ignore this feature."
            )

    folder_train_max_per_class = utils.forms.IntegerField(u'Maximum samples per class',
            validators=[
                validators.Optional(),
                validators.NumberRange(min=1),
                validate_greater_than('folder_train_min_per_class'),
                ],
            tooltip = "You can choose to specify a maximum number of samples per class. If a class has more samples than the specified amount extra samples will be ignored. Leave blank to ignore this feature."
            )

    has_val_folder = wtforms.BooleanField('Separate validation images folder',
            default = False,
            validators=[
                validate_required_iff(method='folder')
                ]
            )

    folder_val = wtforms.StringField(u'Validation Images',
            validators=[
                validate_required_iff(
                    method='folder',
                    has_val_folder=True),
                ]
            )

    folder_val_min_per_class = utils.forms.IntegerField(u'Minimum samples per class',
            default=2,
            validators=[
                validators.Optional(),
                validators.NumberRange(min=1),
                ],
            tooltip = "You can choose to specify a minimum number of samples per class. If a class has fewer samples than the specified amount it will be ignored. Leave blank to ignore this feature."
            )

    folder_val_max_per_class = utils.forms.IntegerField(u'Maximum samples per class',
            validators=[
                validators.Optional(),
                validators.NumberRange(min=1),
                validate_greater_than('folder_val_min_per_class'),
                ],
            tooltip = "You can choose to specify a maximum number of samples per class. If a class has more samples than the specified amount extra samples will be ignored. Leave blank to ignore this feature."
            )

    has_test_folder = wtforms.BooleanField('Separate test images folder',
            default = False,
            validators=[
                validate_required_iff(method='folder')
                ]
            )

    folder_test = wtforms.StringField(u'Test Images',
            validators=[
                validate_required_iff(
                    method='folder',
                    has_test_folder=True),
                validate_folder_path,
                ]
            )

    folder_test_min_per_class = utils.forms.IntegerField(u'Minimum samples per class',
            default=2,
            validators=[
                validators.Optional(),
                validators.NumberRange(min=1)
                ],
            tooltip = "You can choose to specify a minimum number of samples per class. If a class has fewer samples than the specified amount it will be ignored. Leave blank to ignore this feature."
            )

    folder_test_max_per_class = utils.forms.IntegerField(u'Maximum samples per class',
            validators=[
                validators.Optional(),
                validators.NumberRange(min=1),
                validate_greater_than('folder_test_min_per_class'),
                ],
            tooltip = "You can choose to specify a maximum number of samples per class. If a class has more samples than the specified amount extra samples will be ignored. Leave blank to ignore this feature."
            )

    ### Method - textfile

    textfile_use_local_files = wtforms.BooleanField(u'Use local files',
        default=False)

    textfile_train_images = utils.forms.FileField(u'Training images',
            validators=[
                validate_required_iff(method='textfile',
                                      textfile_use_local_files=False)
                ]
            )
    textfile_local_train_images = wtforms.StringField(u'Training images',
            validators=[
                validate_required_iff(method='textfile',
                                      textfile_use_local_files=True)
                ]
            )
    textfile_train_folder = wtforms.StringField(u'Training images folder')

    def validate_textfile_train_folder(form, field):
        if form.method.data != 'textfile':
            field.errors[:] = []
            raise validators.StopValidation()
        if not field.data.strip():
            # allow null
            return True
        if not os.path.exists(field.data) or not os.path.isdir(field.data):
            raise validators.ValidationError('folder does not exist')
        return True

    textfile_use_val = wtforms.BooleanField(u'Validation set',
            default=True,
            validators=[
                validate_required_iff(method='textfile')
                ]
            )
    textfile_val_images = utils.forms.FileField(u'Validation images',
            validators=[
                validate_required_iff(
                    method='textfile',
                    textfile_use_val=True,
                    textfile_use_local_files=False)
                ]
            )
    textfile_local_val_images = wtforms.StringField(u'Validation images',
            validators=[
                validate_required_iff(
                    method='textfile',
                    textfile_use_val=True,
                    textfile_use_local_files=True)
                ]
            )
    textfile_val_folder = wtforms.StringField(u'Validation images folder')

    def validate_textfile_val_folder(form, field):
        if form.method.data != 'textfile' or not form.textfile_use_val.data:
            field.errors[:] = []
            raise validators.StopValidation()
        if not field.data.strip():
            # allow null
            return True
        if not os.path.exists(field.data) or not os.path.isdir(field.data):
            raise validators.ValidationError('folder does not exist')
        return True

    textfile_use_test = wtforms.BooleanField(u'Test set',
            default=False,
            validators=[
                validate_required_iff(method='textfile')
                ]
            )
    textfile_test_images = utils.forms.FileField(u'Test images',
            validators=[
                validate_required_iff(
                    method='textfile',
                    textfile_use_test=True,
                    textfile_use_local_files=False)
                ]
            )
    textfile_local_test_images = wtforms.StringField(u'Test images',
            validators=[
                validate_required_iff(
                    method='textfile',
                    textfile_use_test=True,
                    textfile_use_local_files=True)
                ]
            )
    textfile_test_folder = wtforms.StringField(u'Test images folder')

    def validate_textfile_test_folder(form, field):
        if form.method.data != 'textfile' or not form.textfile_use_test.data:
            field.errors[:] = []
            raise validators.StopValidation()
        if not field.data.strip():
            # allow null
            return True
        if not os.path.exists(field.data) or not os.path.isdir(field.data):
            raise validators.ValidationError('folder does not exist')
        return True

    # Can't use a BooleanField here because HTML doesn't submit anything
    # for an unchecked checkbox. Since we want to use a REST API and have
    # this default to True when nothing is supplied, we have to use a
    # SelectField
    textfile_shuffle = utils.forms.SelectField('Shuffle lines',
            choices = [
                (1, 'Yes'),
                (0, 'No'),
                ],
            coerce=int,
            default=1,
            tooltip = "Shuffle the list[s] of images before creating the database."
            )

    textfile_labels_file = utils.forms.FileField(u'Labels',
            validators=[
                validate_required_iff(method='textfile',
                                      textfile_use_local_files=False)
                ],
            tooltip = "The 'i'th line of the file should give the string label associated with the '(i-1)'th numberic label. (E.g. the string label for the numeric label 0 is supposed to be on line 1.)"
            )

    textfile_local_labels_file = utils.forms.StringField(u'Labels',
            validators=[
                validate_required_iff(method='textfile',
                                      textfile_use_local_files=True)
                ],
            tooltip = "The 'i'th line of the file should give the string label associated with the '(i-1)'th numberic label. (E.g. the string label for the numeric label 0 is supposed to be on line 1.)"
            )
Пример #4
0
class ModelForm(Form):

    ### Methods

    def selection_exists_in_choices(form, field):
        found = False
        for choice in field.choices:
            if choice[0] == field.data:
                found = True
        if not found:
            raise validators.ValidationError(
                "Selected job doesn't exist. Maybe it was deleted by another user."
            )

    def validate_NetParameter(form, field):
        fw = frameworks.get_framework_by_id(form['framework'].data)
        try:
            # below function raises a BadNetworkException in case of validation error
            fw.validate_network(field.data)
        except frameworks.errors.BadNetworkError as e:
            raise validators.ValidationError('Bad network: %s' % e.message)

    def validate_file_exists(form, field):
        from_client = bool(form.python_layer_from_client.data)

        filename = ''
        if not from_client and field.type == 'StringField':
            filename = field.data

        if filename == '': return

        if not os.path.isfile(filename):
            raise validators.ValidationError(
                'Server side file, %s, does not exist.' % filename)

    def validate_py_ext(form, field):
        from_client = bool(form.python_layer_from_client.data)

        filename = ''
        if from_client and field.type == 'FileField':
            filename = flask.request.files[field.name].filename
        elif not from_client and field.type == 'StringField':
            filename = field.data

        if filename == '': return

        (root, ext) = os.path.splitext(filename)
        if ext != '.py' and ext != '.pyc':
            raise validators.ValidationError(
                'Python file, %s, needs .py or .pyc extension.' % filename)

    ### Fields

    # The options for this get set in the view (since they are dynamic)
    dataset = utils.forms.SelectField(
        'Select Dataset',
        choices=[],
        tooltip="Choose the dataset to use for this model.")

    python_layer_from_client = utils.forms.BooleanField(
        u'Use client-side file', default=False)

    python_layer_client_file = utils.forms.FileField(
        u'Client-side file',
        validators=[validate_py_ext],
        tooltip=
        "Choose a Python file on the client containing layer definitions.")
    python_layer_server_file = utils.forms.StringField(
        u'Server-side file',
        validators=[validate_file_exists, validate_py_ext],
        tooltip=
        "Choose a Python file on the server containing layer definitions.")

    train_epochs = utils.forms.IntegerField(
        'Training epochs',
        validators=[validators.NumberRange(min=1)],
        default=30,
        tooltip="How many passes through the training data?")

    snapshot_interval = utils.forms.FloatField(
        'Snapshot interval (in epochs)',
        default=1,
        validators=[
            validators.NumberRange(min=0),
        ],
        tooltip="How many epochs of training between taking a snapshot?")

    val_interval = utils.forms.FloatField(
        'Validation interval (in epochs)',
        default=1,
        validators=[validators.NumberRange(min=0)],
        tooltip=
        "How many epochs of training between running through one pass of the validation data?"
    )

    random_seed = utils.forms.IntegerField(
        'Random seed',
        validators=[
            validators.NumberRange(min=0),
            validators.Optional(),
        ],
        tooltip=
        "If you provide a random seed, then back-to-back runs with the same model and dataset should give identical results."
    )

    batch_size = utils.forms.MultiIntegerField(
        'Batch size',
        validators=[
            utils.forms.MultiNumberRange(min=1),
            utils.forms.MultiOptional(),
        ],
        tooltip=
        "How many images to process at once. If blank, values are used from the network definition."
    )

    batch_accumulation = utils.forms.IntegerField(
        'Batch Accumulation',
        validators=[
            validators.NumberRange(min=1),
            validators.Optional(),
        ],
        tooltip=
        "Accumulate gradients over multiple batches (useful when you need a bigger batch size for training but it doesn't fit in memory)."
    )

    ### Solver types

    solver_type = utils.forms.SelectField(
        'Solver type',
        choices=[
            ('SGD', 'Stochastic gradient descent (SGD)'),
            ('NESTEROV', "Nesterov's accelerated gradient (NAG)"),
            ('ADAGRAD', 'Adaptive gradient (AdaGrad)'),
            ('RMSPROP', 'RMSprop'),
            ('ADADELTA', 'AdaDelta'),
            ('ADAM', 'Adam'),
        ],
        default='SGD',
        tooltip="What type of solver will be used?",
    )

    def validate_solver_type(form, field):
        fw = frameworks.get_framework_by_id(form.framework)
        if fw is not None:
            if not fw.supports_solver_type(field.data):
                raise validators.ValidationError(
                    'Solver type not supported by this framework')

    ### Learning rate

    learning_rate = utils.forms.MultiFloatField(
        'Base Learning Rate',
        default=0.01,
        validators=[
            utils.forms.MultiNumberRange(min=0),
        ],
        tooltip=
        "Affects how quickly the network learns. If you are getting NaN for your loss, you probably need to lower this value."
    )

    lr_policy = wtforms.SelectField('Policy',
                                    choices=[
                                        ('fixed', 'Fixed'),
                                        ('step', 'Step Down'),
                                        ('multistep',
                                         'Step Down (arbitrary steps)'),
                                        ('exp', 'Exponential Decay'),
                                        ('inv', 'Inverse Decay'),
                                        ('poly', 'Polynomial Decay'),
                                        ('sigmoid', 'Sigmoid Decay'),
                                    ],
                                    default='step')

    lr_step_size = wtforms.FloatField('Step Size', default=33)
    lr_step_gamma = wtforms.FloatField('Gamma', default=0.1)
    lr_multistep_values = wtforms.StringField('Step Values', default="50,85")

    def validate_lr_multistep_values(form, field):
        if form.lr_policy.data == 'multistep':
            for value in field.data.split(','):
                try:
                    float(value)
                except ValueError:
                    raise validators.ValidationError('invalid value')

    lr_multistep_gamma = wtforms.FloatField('Gamma', default=0.5)
    lr_exp_gamma = wtforms.FloatField('Gamma', default=0.95)
    lr_inv_gamma = wtforms.FloatField('Gamma', default=0.1)
    lr_inv_power = wtforms.FloatField('Power', default=0.5)
    lr_poly_power = wtforms.FloatField('Power', default=3)
    lr_sigmoid_step = wtforms.FloatField('Step', default=50)
    lr_sigmoid_gamma = wtforms.FloatField('Gamma', default=0.1)

    ### Network

    # Use a SelectField instead of a HiddenField so that the default value
    # is used when nothing is provided (through the REST API)
    method = wtforms.SelectField(
        u'Network type',
        choices=[
            ('standard', 'Standard network'),
            ('previous', 'Previous network'),
            ('pretrained', 'Pretrained network'),
            ('custom', 'Custom network'),
        ],
        default='standard',
    )

    ## framework - hidden field, set by Javascript to the selected framework ID
    framework = wtforms.HiddenField(
        'framework',
        validators=[
            validators.AnyOf(
                [fw.get_id() for fw in frameworks.get_frameworks()],
                message='The framework you choose is not currently supported.')
        ],
        default=frameworks.get_frameworks()[0].get_id())

    # The options for this get set in the view (since they are dependent on the data type)
    standard_networks = wtforms.RadioField(
        'Standard Networks',
        validators=[
            validate_required_iff(method='standard'),
        ],
    )

    previous_networks = wtforms.RadioField(
        'Previous Networks',
        choices=[],
        validators=[
            validate_required_iff(method='previous'),
            selection_exists_in_choices,
        ],
    )

    pretrained_networks = wtforms.RadioField(
        'Pretrained Networks',
        choices=[],
        validators=[
            validate_required_iff(method='pretrained'),
            selection_exists_in_choices,
        ],
    )

    custom_network = utils.forms.TextAreaField(
        'Custom Network',
        validators=[
            validate_required_iff(method='custom'),
            validate_NetParameter,
        ],
    )

    custom_network_snapshot = utils.forms.TextField(
        'Pretrained model(s)',
        tooltip=
        "Paths to pretrained model files, separated by '%s'. Only edit this field if you understand how fine-tuning works in caffe or torch."
        % os.path.pathsep)

    def validate_custom_network_snapshot(form, field):
        if form.method.data == 'custom':
            for filename in field.data.strip().split(os.path.pathsep):
                if filename and not os.path.exists(filename):
                    raise validators.ValidationError(
                        'File "%s" does not exist' % filename)

    # Select one of several GPUs
    select_gpu = wtforms.RadioField(
        'Select which GPU you would like to use',
        choices=[('next', 'Next available')] + [(
            index,
            '#%s - %s (%s memory)' %
            (index, get_device(index).name,
             sizeof_fmt(
                 get_nvml_info(index)['memory']['total']
                 if get_nvml_info(index) and 'memory' in get_nvml_info(index)
                 else get_device(index).totalGlobalMem)),
        ) for index in config_value('gpu_list').split(',') if index],
        default='next',
    )

    # Select N of several GPUs
    select_gpus = utils.forms.SelectMultipleField(
        'Select which GPU[s] you would like to use',
        choices=[(
            index,
            '#%s - %s (%s memory)' %
            (index, get_device(index).name,
             sizeof_fmt(
                 get_nvml_info(index)['memory']['total']
                 if get_nvml_info(index) and 'memory' in get_nvml_info(index)
                 else get_device(index).totalGlobalMem)),
        ) for index in config_value('gpu_list').split(',') if index],
        tooltip=
        "The job won't start until all of the chosen GPUs are available.")

    # XXX For testing
    # The Flask test framework can't handle SelectMultipleFields correctly
    select_gpus_list = wtforms.StringField(
        'Select which GPU[s] you would like to use (comma separated)')

    def validate_select_gpus(form, field):
        if form.select_gpus_list.data:
            field.data = form.select_gpus_list.data.split(',')

    # Use next available N GPUs
    select_gpu_count = wtforms.IntegerField(
        'Use this many GPUs (next available)',
        validators=[
            validators.NumberRange(min=1,
                                   max=len(
                                       config_value('gpu_list').split(',')))
        ],
        default=1,
    )

    def validate_select_gpu_count(form, field):
        if field.data is None:
            if form.select_gpus.data:
                # Make this field optional
                field.errors[:] = []
                raise validators.StopValidation()

    model_name = utils.forms.StringField(
        'Model Name',
        validators=[validators.DataRequired()],
        tooltip=
        "An identifier, later used to refer to this model in the Application.")

    # allows shuffling data during training (for frameworks that support this, as indicated by
    # their Framework.can_shuffle_data() method)
    shuffle = utils.forms.BooleanField(
        'Shuffle Train Data',
        default=True,
        tooltip='For every epoch, shuffle the data before training.')
Пример #5
0
class ModelForm(Form):

    ### Methods

    def selection_exists_in_choices(form, field):
        found = False
        for choice in field.choices:
            if choice[0] == field.data:
                found = True
        if not found:
            raise validators.ValidationError(
                "Selected job doesn't exist. Maybe it was deleted by another user."
            )

    def validate_NetParameter(form, field):
        pb = caffe_pb2.NetParameter()
        try:
            text_format.Merge(field.data, pb)
        except text_format.ParseError as e:
            raise validators.ValidationError('Not a valid NetParameter: %s' %
                                             e)

    ### Fields

    # The options for this get set in the view (since they are dynamic)
    dataset = wtforms.SelectField('Select Dataset', choices=[])

    train_epochs = wtforms.IntegerField(
        'Training epochs',
        validators=[validators.NumberRange(min=1)],
        default=30,
    )

    snapshot_interval = wtforms.FloatField(
        'Snapshot interval (in epochs)',
        default=1,
        validators=[
            validators.NumberRange(min=0),
        ],
    )

    val_interval = wtforms.FloatField(
        'Validation interval (in epochs)',
        default=1,
        validators=[validators.NumberRange(min=0)],
    )

    random_seed = wtforms.IntegerField(
        'Random seed',
        validators=[
            validators.NumberRange(min=0),
            validators.Optional(),
        ],
    )

    batch_size = wtforms.IntegerField(
        'Batch size',
        validators=[
            validators.NumberRange(min=1),
            validators.Optional(),
        ],
    )

    ### Solver types

    solver_type = wtforms.SelectField(
        'Solver type',
        choices=[
            ('SGD', 'Stochastic gradient descent (SGD)'),
            ('ADAGRAD', 'Adaptive gradient (AdaGrad)'),
            ('NESTEROV', "Nesterov's accelerated gradient (NAG)"),
        ],
        default='SGD')

    ### Learning rate

    learning_rate = wtforms.FloatField('Base Learning Rate',
                                       default=0.01,
                                       validators=[
                                           validators.NumberRange(min=0),
                                       ])

    lr_policy = wtforms.SelectField('Policy',
                                    choices=[
                                        ('fixed', 'Fixed'),
                                        ('step', 'Step Down'),
                                        ('multistep',
                                         'Step Down (arbitrary steps)'),
                                        ('exp', 'Exponential Decay'),
                                        ('inv', 'Inverse Decay'),
                                        ('poly', 'Polynomial Decay'),
                                        ('sigmoid', 'Sigmoid Decay'),
                                    ],
                                    default='step')

    lr_step_size = wtforms.FloatField('Step Size', default=33)
    lr_step_gamma = wtforms.FloatField('Gamma', default=0.1)
    lr_multistep_values = wtforms.StringField('Step Values', default="50,85")

    def validate_lr_multistep_values(form, field):
        if form.lr_policy.data == 'multistep':
            for value in field.data.split(','):
                try:
                    float(value)
                except ValueError:
                    raise validators.ValidationError('invalid value')

    lr_multistep_gamma = wtforms.FloatField('Gamma', default=0.5)
    lr_exp_gamma = wtforms.FloatField('Gamma', default=0.95)
    lr_inv_gamma = wtforms.FloatField('Gamma', default=0.1)
    lr_inv_power = wtforms.FloatField('Power', default=0.5)
    lr_poly_power = wtforms.FloatField('Power', default=3)
    lr_sigmoid_step = wtforms.FloatField('Step', default=50)
    lr_sigmoid_gamma = wtforms.FloatField('Gamma', default=0.1)

    ### Network

    # Use a SelectField instead of a HiddenField so that the default value
    # is used when nothing is provided (through the REST API)
    method = wtforms.SelectField(
        u'Network type',
        choices=[
            ('standard', 'Standard network'),
            ('previous', 'Previous network'),
            ('custom', 'Custom network'),
        ],
        default='standard',
    )

    # The options for this get set in the view (since they are dependent on the data type)
    standard_networks = wtforms.RadioField(
        'Standard Networks',
        validators=[
            validate_required_iff(method='standard'),
        ],
    )

    previous_networks = wtforms.RadioField(
        'Previous Networks',
        choices=[],
        validators=[
            validate_required_iff(method='previous'),
            selection_exists_in_choices,
        ],
    )

    custom_network = wtforms.TextAreaField(
        'Custom Network',
        validators=[
            validate_required_iff(method='custom'),
            validate_NetParameter,
        ])

    custom_network_snapshot = wtforms.TextField('Pretrained model')

    def validate_custom_network_snapshot(form, field):
        if form.method.data == 'custom':
            snapshot = field.data.strip()
            if snapshot:
                if not os.path.exists(snapshot):
                    raise validators.ValidationError('File does not exist')

    # Select one of several GPUs
    select_gpu = wtforms.RadioField(
        'Select which GPU you would like to use',
        choices=[('next', 'Next available')] + [(
            index,
            '#%s - %s%s' % (
                index,
                get_device(index).name,
                ' (%s memory)' %
                sizeof_fmt(get_nvml_info(index)['memory']['total'])
                if get_nvml_info(index) and 'memory' in get_nvml_info(index)
                else '',
            ),
        ) for index in config_value('gpu_list').split(',') if index],
        default='next',
    )

    # Select N of several GPUs
    select_gpus = wtforms.SelectMultipleField(
        'Select which GPU[s] you would like to use',
        choices=[(
            index,
            '#%s - %s%s' % (
                index,
                get_device(index).name,
                ' (%s memory)' %
                sizeof_fmt(get_nvml_info(index)['memory']['total'])
                if get_nvml_info(index) and 'memory' in get_nvml_info(index)
                else '',
            ),
        ) for index in config_value('gpu_list').split(',') if index])

    # XXX For testing
    # The Flask test framework can't handle SelectMultipleFields correctly
    select_gpus_list = wtforms.StringField(
        'Select which GPU[s] you would like to use (comma separated)')

    def validate_select_gpus(form, field):
        if form.select_gpus_list.data:
            field.data = form.select_gpus_list.data.split(',')

    # Use next available N GPUs
    select_gpu_count = wtforms.IntegerField(
        'Use this many GPUs (next available)',
        validators=[
            validators.NumberRange(min=1,
                                   max=len(
                                       config_value('gpu_list').split(',')))
        ],
        default=1,
    )

    def validate_select_gpu_count(form, field):
        if field.data is None:
            if form.select_gpus.data:
                # Make this field optional
                field.errors[:] = []
                raise validators.StopValidation()

    model_name = wtforms.StringField('Model Name',
                                     validators=[validators.DataRequired()])
Пример #6
0
class GenericImageDatasetForm(ImageDatasetForm):
    """
    Defines the form used to create a new GenericImageDatasetJob
    """

    # Use a SelectField instead of a HiddenField so that the default value
    # is used when nothing is provided (through the REST API)
    method = wtforms.SelectField(
        u'Dataset type',
        choices=[
            ('prebuilt', 'Prebuilt'),
        ],
        default='prebuilt',
    )

    def validate_lmdb_path(form, field):
        if not field.data:
            pass
        else:
            # make sure the filesystem path exists
            if not os.path.exists(field.data) or not os.path.isdir(field.data):
                raise validators.ValidationError('Folder does not exist')

    def validate_file_path(form, field):
        if not field.data:
            pass
        else:
            # make sure the filesystem path exists
            if not os.path.exists(field.data) or not os.path.isfile(
                    field.data):
                raise validators.ValidationError('File does not exist')

    ### Method - prebuilt

    prebuilt_train_images = wtforms.StringField(
        'Training Images',
        validators=[
            validate_required_iff(method='prebuilt'),
            validate_lmdb_path,
        ])
    prebuilt_train_labels = wtforms.StringField('Training Labels',
                                                validators=[
                                                    validate_lmdb_path,
                                                ])
    prebuilt_val_images = wtforms.StringField('Validation Images',
                                              validators=[
                                                  validate_lmdb_path,
                                              ])
    prebuilt_val_labels = wtforms.StringField('Validation Labels',
                                              validators=[
                                                  validate_lmdb_path,
                                              ])

    # Can't use a BooleanField here because HTML doesn't submit anything
    # for an unchecked checkbox. Since we want to use a REST API and have
    # this default to True when nothing is supplied, we have to use a
    # SelectField
    force_same_shape = utils.forms.SelectField(
        'Enforce same shape',
        choices=[
            (1, 'Yes'),
            (0, 'No'),
        ],
        coerce=int,
        default=1,
        tooltip=
        'Check that each entry in the database has the same shape (can be time-consuming)'
    )

    prebuilt_mean_file = utils.forms.StringField(
        'Mean Image',
        validators=[
            validate_file_path,
        ],
        tooltip="Path to a .binaryproto file on the server")
Пример #7
0
class DatasetForm(Form):
    """
    A form used to create an image processing dataset
    """
    def validate_folder_path(form, field):
        if not field.data:
            pass
        else:
            # make sure the filesystem path exists
            if not os.path.exists(field.data) or not os.path.isdir(field.data):
                raise validators.ValidationError(
                    'Folder does not exist or is not reachable')
            else:
                return True

    feature_folder = utils.forms.StringField(
        'Feature image folder',
        validators=[
            validators.DataRequired(),
            validate_folder_path,
        ],
        tooltip="Indicate a folder full of images.")

    label_folder = utils.forms.StringField(
        'Label image folder',
        validators=[
            validators.DataRequired(),
            validate_folder_path,
        ],
        tooltip="Indicate a folder full of images. For each image in the feature"
        " image folder there must be one corresponding image in the label"
        " image folder. The label image must have the same filename except"
        " for the extension, which may differ.")

    folder_pct_val = utils.forms.IntegerField(
        '% for validation',
        default=10,
        validators=[validators.NumberRange(min=0, max=100)],
        tooltip="You can choose to set apart a certain percentage of images "
        "from the training images for the validation set.")

    has_val_folder = utils.forms.BooleanField(
        'Separate validation images',
        default=False,
    )

    validation_feature_folder = utils.forms.StringField(
        'Validation feature image folder',
        validators=[
            validate_required_iff(has_val_folder=True),
            validate_folder_path,
        ],
        tooltip="Indicate a folder full of images.")

    validation_label_folder = utils.forms.StringField(
        'Validation label image folder',
        validators=[
            validate_required_iff(has_val_folder=True),
            validate_folder_path,
        ],
        tooltip="Indicate a folder full of images. For each image in the feature"
        " image folder there must be one corresponding image in the label"
        " image folder. The label image must have the same filename except"
        " for the extension, which may differ.")

    channel_conversion = utils.forms.SelectField(
        'Channel conversion',
        choices=[
            ('RGB', 'RGB'),
            ('L', 'Grayscale'),
            ('none', 'None'),
        ],
        default='none',
        tooltip="Perform selected channel conversion.")
Пример #8
0
class ImageClassificationDatasetForm(ImageDatasetForm):
    """
    Defines the form used to create a new ImageClassificationDatasetJob
    """

    # Use a SelectField instead of a HiddenField so that the default value
    # is used when nothing is provided (through the REST API)
    method = wtforms.SelectField(
        u'Dataset type',
        choices=[
            ('folder', 'Folder'),
            ('textfile', 'Textfiles'),
        ],
        default='folder',
    )

    def validate_folder_path(form, field):
        if not field.data:
            pass
        elif utils.is_url(field.data):
            # make sure the URL exists
            try:
                r = requests.get(field.data,
                                 allow_redirects=False,
                                 timeout=utils.HTTP_TIMEOUT)
                if r.status_code not in [
                        requests.codes.ok, requests.codes.moved,
                        requests.codes.found
                ]:
                    raise validators.ValidationError('URL not found')
            except Exception as e:
                raise validators.ValidationError(
                    'Caught %s while checking URL: %s' % (type(e).__name__, e))
            else:
                return True
        else:
            # make sure the filesystem path exists
            if not os.path.exists(field.data) or not os.path.isdir(field.data):
                raise validators.ValidationError('Folder does not exist')
            else:
                return True

    ### Method - folder

    folder_train = wtforms.StringField(
        u'Training Images',
        validators=[
            validate_required_iff(method='folder'),
            validate_folder_path,
        ])

    folder_pct_val = wtforms.IntegerField(
        u'% for validation',
        default=25,
        validators=[
            validate_required_iff(method='folder'),
            validators.NumberRange(min=0, max=100)
        ])

    folder_pct_test = wtforms.IntegerField(
        u'% for testing',
        default=0,
        validators=[
            validate_required_iff(method='folder'),
            validators.NumberRange(min=0, max=100)
        ])

    has_val_folder = wtforms.BooleanField(
        'Separate validation images folder',
        default=False,
        validators=[validate_required_iff(method='folder')])

    folder_val = wtforms.StringField(u'Validation Images',
                                     validators=[
                                         validate_required_iff(
                                             method='folder',
                                             has_val_folder=True),
                                         validate_folder_path,
                                     ])

    has_test_folder = wtforms.BooleanField(
        'Separate test images folder',
        default=False,
        validators=[validate_required_iff(method='folder')])

    folder_test = wtforms.StringField(u'Test Images',
                                      validators=[
                                          validate_required_iff(
                                              method='folder',
                                              has_test_folder=True),
                                          validate_folder_path,
                                      ])

    ### Method - textfile

    textfile_train_images = wtforms.FileField(
        u'Training images',
        validators=[validate_required_iff(method='textfile')])
    textfile_train_folder = wtforms.StringField(u'Training images folder')

    def validate_textfile_train_folder(form, field):
        if form.method.data != 'textfile':
            field.errors[:] = []
            raise validators.StopValidation()
        if not field.data.strip():
            # allow null
            return True
        if not os.path.exists(field.data) or not os.path.isdir(field.data):
            raise validators.ValidationError('folder does not exist')
        return True

    textfile_use_val = wtforms.BooleanField(
        u'Validation set',
        default=True,
        validators=[validate_required_iff(method='textfile')])
    textfile_val_images = wtforms.FileField(u'Validation images',
                                            validators=[
                                                validate_required_iff(
                                                    method='textfile',
                                                    textfile_use_val=True)
                                            ])
    textfile_val_folder = wtforms.StringField(u'Validation images folder')

    def validate_textfile_val_folder(form, field):
        if form.method.data != 'textfile' or not form.textfile_use_val.data:
            field.errors[:] = []
            raise validators.StopValidation()
        if not field.data.strip():
            # allow null
            return True
        if not os.path.exists(field.data) or not os.path.isdir(field.data):
            raise validators.ValidationError('folder does not exist')
        return True

    textfile_use_test = wtforms.BooleanField(
        u'Test set',
        default=False,
        validators=[validate_required_iff(method='textfile')])
    textfile_test_images = wtforms.FileField(u'Test images',
                                             validators=[
                                                 validate_required_iff(
                                                     method='textfile',
                                                     textfile_use_test=True)
                                             ])
    textfile_test_folder = wtforms.StringField(u'Test images folder')

    def validate_textfile_test_folder(form, field):
        if form.method.data != 'textfile' or not form.textfile_use_test.data:
            field.errors[:] = []
            raise validators.StopValidation()
        if not field.data.strip():
            # allow null
            return True
        if not os.path.exists(field.data) or not os.path.isdir(field.data):
            raise validators.ValidationError('folder does not exist')
        return True

    # Can't use a BooleanField here because HTML doesn't submit anything
    # for an unchecked checkbox. Since we want to use a REST API and have
    # this default to True when nothing is supplied, we have to use a
    # SelectField
    textfile_shuffle = wtforms.SelectField(
        'Shuffle lines',
        choices=[
            (1, 'Yes'),
            (0, 'No'),
        ],
        coerce=int,
        default=1,
    )

    textfile_labels_file = wtforms.FileField(
        u'Labels', validators=[validate_required_iff(method='textfile')])
Пример #9
0
class DatasetForm(Form):
    """
    A form used to create an image processing dataset
    """
    def validate_folder_path(form, field):
        if not field.data:
            pass
        else:
            # make sure the filesystem path exists
            if not os.path.exists(field.data) or not os.path.isdir(field.data):
                raise validators.ValidationError(
                    'Folder does not exist or is not reachable')
            else:
                return True

    def validate_file_path(form, field):
        if not field.data:
            pass
        else:
            # make sure the filesystem path exists
            if not os.path.exists(field.data) and not os.path.isdir(
                    field.data):
                raise validators.ValidationError(
                    'File does not exist or is not reachable')
            else:
                return True

    feature_folder = utils.forms.StringField(
        'Feature image folder',
        validators=[
            validators.DataRequired(),
            validate_folder_path,
        ],
        tooltip="Indicate a folder full of images.")

    label_folder = utils.forms.StringField(
        'Label image folder',
        validators=[
            validators.DataRequired(),
            validate_folder_path,
        ],
        tooltip="Indicate a folder full of images. For each image in the feature"
        " image folder there must be one corresponding image in the label"
        " image folder. The label image must have the same filename except"
        " for the extension, which may differ. Label images are expected"
        " to be single-channel images (paletted or grayscale), or RGB"
        " images, in which case the color/class mappings need to be"
        " specified through a separate text file.")

    folder_pct_val = utils.forms.IntegerField(
        '% for validation',
        default=10,
        validators=[validators.NumberRange(min=0, max=100)],
        tooltip="You can choose to set apart a certain percentage of images "
        "from the training images for the validation set.")

    has_val_folder = utils.forms.BooleanField(
        'Separate validation images',
        default=False,
    )

    validation_feature_folder = utils.forms.StringField(
        'Validation feature image folder',
        validators=[
            validate_required_iff(has_val_folder=True),
            validate_folder_path,
        ],
        tooltip="Indicate a folder full of images.")

    validation_label_folder = utils.forms.StringField(
        'Validation label image folder',
        validators=[
            validate_required_iff(has_val_folder=True),
            validate_folder_path,
        ],
        tooltip="Indicate a folder full of images. For each image in the feature"
        " image folder there must be one corresponding image in the label"
        " image folder. The label image must have the same filename except"
        " for the extension, which may differ. Label images are expected"
        " to be single-channel images (paletted or grayscale), or RGB"
        " images, in which case the color/class mappings need to be"
        " specified through a separate text file.")

    channel_conversion = utils.forms.SelectField(
        'Channel conversion',
        choices=[
            ('RGB', 'RGB'),
            ('L', 'Grayscale'),
            ('none', 'None'),
        ],
        default='none',
        tooltip="Perform selected channel conversion on feature images. Label"
        " images are single channel and not affected by this parameter.")

    class_labels_file = utils.forms.StringField(
        'Class labels (optional)',
        validators=[
            validate_file_path,
        ],
        tooltip="The 'i'th line of the file should give the string label "
        "associated with the '(i-1)'th numeric label. (E.g. the "
        "string label for the numeric label 0 is supposed to be "
        "on line 1.)")

    colormap_method = utils.forms.SelectField(
        'Color map specification',
        choices=[
            ('label', 'From label image'),
            ('textfile', 'From text file'),
        ],
        default='label',
        tooltip="Specify how to map class IDs to colors. Select 'From label "
        "image' to use palette or grayscale from label images. For "
        "RGB image labels, select 'From text file' and provide "
        "color map in separate text file.")

    colormap_text_file = utils.forms.StringField(
        'Color map file',
        validators=[
            validate_required_iff(colormap_method="textfile"),
            validate_file_path,
        ],
        tooltip="Specify color/class mappings through a text file. "
        "Each line in the file should contain three space-separated "
        "integer values, one for each of the Red, Green, Blue "
        "channels. The 'i'th line of the file should give the color "
        "associated with the '(i-1)'th class. (E.g. the "
        "color for class #0 is supposed to be on line 1.)")