def __init__(self, *args, **kwargs): handler_id = kwargs.pop('import_handler_id') if handler_id is None: raise ValidationError('Import Handler is required') self.import_handler = XmlImportHandler.query.get(handler_id) if self.import_handler is None: raise ValidationError('Invalid import handler id specified') super(PredictModelForm, self).__init__(*args, **kwargs)
def clean_import_params(self, value, field): if not isinstance(value, dict): raise ValidationError('Should be a dict') for param in self.importhandler.import_params: if param not in value: raise ValidationError( '{0!s} parameter is required'.format(param)) return value
def clean_entity(self, value, field): if value: if not (value.datasource and value.datasource.type == 'pig'): raise ValidationError('Only "pig" entity is allowed') if value and not self.is_edit: query = XmlSqoop.query.filter_by(entity=value) if query.count() >= self.MAX_ITEMS_BY_ENTITY: raise ValidationError( 'There can be no more than {0} elements'.format( self.MAX_ITEMS_BY_ENTITY)) return value
def clean_parameters(self, grid_params, field): params = {} config = CLASSIFIERS[self.model.classifier['type']] config_params = config['parameters'] for pconfig in config_params: name = pconfig['name'] if name in grid_params: value = grid_params[name] if not value: continue value = value.split(',') type_ = pconfig.get('type', 'string') if type_ == 'integer': value = [int(item) for item in value] elif type_ == 'float': value = [float(item) for item in value] elif type_ == 'boolean': value = [item == 'true' for item in value] choices = pconfig.get('choices') if choices: for item in value: if item not in choices: raise ValidationError( 'Invalid {0}: should be one of {1}'.format( name, ','.join(choices))) params[name] = value return params
def clean_name(self, value, field): if not ((self.NO_REQUIRED_FOR_EDIT and self.obj.id) or value): raise ValidationError('name is required field') import_handler_id = self.obj.import_handler_id if \ self.obj.id else self.data['import_handler_id'] query = XmlDataSource.query.filter_by( name=value, import_handler_id=import_handler_id) if self.obj.id: query = query.filter(XmlDataSource.id != self.obj.id) count = query.count() if count: raise ValidationError('Data Source with name "%s" already \ exist. Please choose another one.' % value) return value
def clean_data(self, value, field): try: s = ScriptManager() # this will raise exception in case of incorrect script s.add_python(value) except ImportHandlerException as ex: raise ValidationError(ex.message, ex) return value
def clean_name(self, value, field): query = PredefinedDataSource.query.filter_by(name=value) if self.obj.id: query = query.filter(PredefinedDataSource.id != self.obj.id) count = query.count() if count: raise ValidationError("DataSource with name \"%s\" already exist. " "Please choose another one." % value) return value
def clean_data(self, value, field): if value is None: return value = value.encode('utf-8') try: ExtractionPlan(value, is_file=False) return value except Exception as exc: raise ValidationError(exc.message, exc)
def clean_features(self, value, field): if value: from cloudml.trainer.trainer import Trainer from cloudml.trainer.config import FeatureModel, SchemaException try: # TODO: add support of json dict to FeatureModel feature_model = FeatureModel(json.dumps(value), is_file=False) self.cleaned_data['trainer'] = Trainer(feature_model) except SchemaException, exc: raise ValidationError( 'Features JSON file is invalid: %s' % exc, exc)
def clean_trainer(self, value, field): if value: try: # TODO: find a better way? from cloudml.trainer.store import load_trainer value = value.encode('utf-8').replace('\r', '') trainer_obj = load_trainer(value) self.cleaned_data['status'] = Model.STATUS_TRAINED return trainer_obj except Exception as exc: raise ValidationError( 'Pickled trainer model is invalid: {0!s}'.format(exc), exc)
def save(self, *args, **kwargs): try: script_type = self.cleaned_data.get('type', None) data_file = self.cleaned_data.get('data_file', None) data_url = self.cleaned_data.get('data_url', None) data = self.cleaned_data.get('data', None) if script_type == XmlScript.TYPE_PYTHON_FILE: if data_file: key = XmlScript.to_s3( data_file, self.cleaned_data.get('import_handler_id')) self.cleaned_data['data'] = key elif data_url: self.cleaned_data['data'] = data_url else: raise ValidationError("File upload or URL required " "for type '{0}'".format(script_type)) elif script_type == XmlScript.TYPE_PYTHON_CODE: if not data: raise ValidationError("Code is required for type " "'{0}'".format(script_type)) # type is not passed else: if data_file: key = XmlScript.to_s3( data_file, self.cleaned_data.get('import_handler_id')) self.cleaned_data['data'] = key self.cleaned_data['type'] = XmlScript.TYPE_PYTHON_FILE elif data_url: self.cleaned_data['data'] = data_url self.cleaned_data['type'] = XmlScript.TYPE_PYTHON_FILE else: self.cleaned_data['type'] = XmlScript.TYPE_PYTHON_CODE script = super(XmlScriptForm, self).save() except Exception as e: raise ValidationError(e.message, e) return script
def _save_importhandler(self, fieldname, name, handler_type='xml'): """ Adds new import handler to the system, if it was specified in file field. Use it in the model. """ def determine_name(name, action): if action == 'train': name = "%s import handler" % name else: name = "%s test import handler" % name while True: count = XmlImportHandler.query.filter_by(name=name).count() if not count: return name name += '_' return name data = self.cleaned_data.pop(fieldname, None) if data is not None: if handler_type == 'xml': from api.import_handlers.models import XmlImportHandler cls = XmlImportHandler handler = cls() action = 'test' if fieldname.startswith('test') else 'train' handler.name = determine_name(name, action) handler.import_params = self.cleaned_data.pop('%s_import_params' % action) handler._set_user() try: handler.data = data except Exception, exc: self.add_error('fields', str(exc), exc) raise ValidationError(self.error_messages, exc, errors=self.errors) self.cleaned_data['%s_import_handler' % action] = handler db.session.add(handler) db.session.commit() return handler
def save(self): try: import_handler = XmlImportHandler(name=self.cleaned_data['name'], import_params=[]) import_handler._set_user() db.session.add(import_handler) try: import_handler.data = self.cleaned_data.get('data') except Exception, exc: self.add_error('fields', str(exc), exc) raise ValidationError(self.error_messages, exc, errors=self.errors) except Exception as e: db.session.rollback() raise DBException(e.message, e) else: db.session.commit() return import_handler
def clean_name(self, value, field): count = XmlImportHandler.query.filter_by(name=value).count() if count: raise ValidationError('Import Handler with name "%s" already \ exist. Please choose another one.' % value) return value
def clean_datasource(self, value, field): if value: if value.type != 'db': raise ValidationError('Only "db" datasources are allowed') return value
def clean_type(self, value, field): if value and value not in self.TYPES: raise ValidationError('invalid type') return value
def clean_transformed_field(self, value, field): if value and self.data.get('datasource'): raise ValidationError(self.DATASOURCE_MESSAGE) return value