def validate_data(self):
predefined_selected = self.cleaned_data.get('predefined_selected',
False)
feature_id = self.cleaned_data.get('feature_id', False)
model_id = self.cleaned_data.get('model_id', False)
# It would be predefined obj, when model, feature not specified
# and this form will not used as inner form of another one
self.cleaned_data['is_predefined'] = is_predefined = \
not (feature_id or model_id or self.inner_name)
if predefined_selected and is_predefined:
raise ValidationError(
'item could be predefined or copied from predefined')
if is_predefined:
name = self.cleaned_data.get('name', None)
if not name:
raise ValidationError('name is required for predefined item')
items = self.DOC.query.filter_by(name=name)
if self.obj.id:
items = items.filter(
sqlalchemy.not_(self.DOC.id == self.obj.id))
count = items.count()
if count:
self.add_error(
'name', "Predefined %s with same name already exist. "
"Please choose another one." % self.OBJECT_NAME)
if predefined_selected:
obj = self.cleaned_data.get(self.OBJECT_NAME, None)
if obj:
self._fill_predefined_values(obj)
def clean_classifier(self, value):
if not value:
raise ValidationError('classifier is required')
classifier = PredefinedClassifier.query.get(value)
if not classifier:
raise ValidationError('classifier not found')
return classifier
def clean(self, value):
value = super(ModelField, self).clean(value)
if value is None:
return None
if not isint(value):
raise ValidationError('Invalid {0} id: {1}'.format(
self.Model.__name__, value))
self.model = self.Model.query.get(value)
if self.model is None:
raise ValidationError('{0} not found'.format(self.Model.__name__))
return self.model if self.return_model else value
def clean_type(self, value, field):
if value and value not in NamedFeatureType.TYPES_LIST:
# Try to find type in named types
found = NamedFeatureType.query.filter_by(name=value).count()
if not found:
raise ValidationError('invalid type')
return value
def convert_string_list_none(val, config):
if isinstance(val, dict):
type_ = val.get('type')
value = val.get('value')
if type_ == 'empty':
return None
else:
if config.get('required', None) and value is None:
raise ValidationError('{0} parameter is required'.format(
config.name))
if type_ == 'string':
return str(value) or ''
elif type_ == 'list':
if not isinstance(value, (list, tuple)):
return [
value,
]
return value
else:
if val is None or isinstance(val, basestring) or \
isinstance(val, (list, tuple)):
return val
raise ValidationType("Invalid subtype of the {0} parameter".format(
config.name))
def clean(self, value):
if value is not None:
if value:
from cloudml.trainer.config import FeatureModel, \
SchemaException
try:
value = json.loads(value)
feature_model = FeatureModel(json.dumps(value),
is_file=False)
except SchemaException, exc:
raise ValidationError(
'Features JSON file is invalid: %s' % exc, exc)
except ValueError, e:
raise ValidationError(
'JSON file is corrupted. Can not load it: %s' % value,
e)
def clean_model_id(self, value, field):
if self.model is None:
raise ValidationError('Model not found')
self.cleaned_data['model_name'] = self.model.name
self.cleaned_data['model_id'] = self.model_id
return None
def clean(self, value):
value = super(ChoiceField, self).clean(value)
if value and value not in self._choices:
raise ValidationError('Should be one of %s' %
', '.join(self._choices))
return value
def convert_float_or_int(val, config):
if '.' not in str(val) and isint(val):
return int(val)
elif isfloat(val):
return float(val)
else:
raise ValidationError('Invalid value {0} for type {1}'.format(
val, config.type))
def clean(self, value):
value = super(MultipleModelField, self).clean(value)
if not value:
return None
values = value.split(',')
for item in values:
if not isint(item):
raise ValidationError('Invalid {0} id: {1}'.format(
self.Model.__name__, item))
self.models = self.Model.query.filter(self.Model.id.in_(values)).all()
if not self.models:
raise ValidationError('{0} not found'.format(self.Model.__name__))
return self.models if self.return_model else value
def clean_name(self, value, field):
query = Server.query.filter_by(name=value)
if self.obj.id:
query = query.filter(Server.id != self.obj.id)
count = query.count()
if count:
raise ValidationError('name should be unique')
return value
def clean_parameters(self, params, field):
params = params or {}
if self.model is None:
return params
if not isinstance(params, dict):
raise ValidationError('Invalid parameters format')
return params
def clean_target_variable(self, value, field):
if value:
self.target_feature = Feature.query.filter_by(
name=value,
feature_set_id=self.id).one()
if self.target_feature is None:
raise ValidationError('Feature not found')
return value
def _validate_param(self, data, name):
value = data.get(name, None)
if value is None:
raise ValidationError('Parameter {} is required'.format(name))
param_type = FEATURE_PARAMS_TYPES[name]['type']
if param_type == 'int':
if not isint(value):
raise ValidationError('{} - int is required'.format(value))
elif param_type == 'str':
pass # do nothing
elif param_type == 'text':
if isinstance(value, basestring):
try:
data[name] = json.loads(value)
except ValueError as e:
raise ValidationError('invalid json: {}'.format(value), e)
elif param_type == 'dict':
if not isinstance(value, dict):
raise ValidationError(
'{} should be a dictionary'.format(name))
if not value.keys():
raise ValidationError(
'Map {} should contain at least one value'.format(name))
for key, val in value.items():
if not len(str(val)):
raise ValidationError(
'Value {0} in {1} can\'t be empty'.format(key, name))
elif param_type == 'list':
if not isinstance(value, basestring):
raise ValidationError(
'{} should be a list'.format(name))
def clean(self, value):
value = super(JsonField, self).clean(value)
if value:
try:
return json.loads(value)
except ValueError as e:
raise ValidationError(
'JSON file is corrupted. Can not load it: %s' % value, e)
def _get_weights_download_action(self, **kwargs):
model = self._get_details_query(None, **kwargs)
if model is None:
raise NotFound(self.MESSAGE404 % kwargs)
if model.status != model.STATUS_TRAINED:
raise ValidationError('Model should be trained')
if not model.weights_synchronized:
raise ValidationError('Model weights should be synchronized')
trainer = model.get_trainer(force_load=True)
result = {}
for segment in model.segments:
result[segment.name] = trainer.get_weights(segment.name)
content = json.dumps(result)
resp = Response(content)
resp.headers['Content-Type'] = 'application/json'
resp.headers['Content-Disposition'] = \
'attachment; filename=%s-weights.json' % model.name
return resp
def clean_params(self, value, field):
value_type = self.data.get('type')
if value_type is None and not self.NO_REQUIRED_FOR_EDIT:
raise ValidationError('invalid type')
if value_type not in FEATURE_TYPE_FACTORIES:
return
required_params = FEATURE_TYPE_FACTORIES[value_type].required_params
optional_params = FEATURE_TYPE_FACTORIES[value_type].optional_params
if required_params and value is None:
raise ValidationError('Parameters are required for type {0}, '
'but was not specified'.format(value_type))
data = {}
for name in required_params:
self._validate_param(value, name)
data[name] = self._clean_param(value, name)
for name in optional_params:
v = value.get(name, None)
if v not in ['', None]:
self._validate_param(value, name)
data[name] = self._clean_param(value, name)
return data
def clean(self, value):
if value is None:
return
value = value.encode('utf-8')
from cloudml.importhandler.importhandler import ScriptManager
try:
s = ScriptManager()
s.add_python(value)
except Exception as exc:
raise ValidationError(exc.message, exc)
return value
def clean(self, value):
value = super(UniqueNameField, self).clean(value)
query = self._model_cls.query.filter_by(name=value)
if self.obj.id:
query = query.filter(self._model_cls.id != self.obj.id)
count = query.count()
if count:
raise ValidationError("{0} with name \"{1}\" already exist. "
"Please choose another one.".format(
self.verbose_name, value))
return value
def convert_parameters(config, params):
"""
Tries to convert the parameter value to corresponding type.
Parameters
----------
C : dict
parameter's config with info about type, wheither it's required, etc.
params : dict
parameters filled by user
Note: Use `ParametersConvertorMixin` for converting parameters in forms.
"""
missed_params = []
for param_config in config:
if param_config.get('required', None) and \
param_config['name'] not in params:
missed_params.append(param_config['name'])
if missed_params:
raise ValidationError('These params are required: {}'.format(
', '.join(missed_params)))
for param_config in config:
name = param_config.get('name')
if name in params:
type_ = param_config.get('type')
convertor = TYPE_CONVERTORS[type_]
if params[name] is None:
params.pop(name)
continue
try:
params[name] = convertor(params[name], param_config)
except ValueError, exc:
raise ValidationError(
'Invalid parameter %s value: %s' % (name, exc), exc)
choices = param_config.get('choices')
if choices and type_ == 'string':
check_choices(name, params[name], choices)
def save(self, commit=True, save=True):
if not self.is_valid():
raise ValidationError(self.errors)
for name, val in self.cleaned_data.iteritems():
try:
setattr(self.obj, name, val)
except AttributeError:
pass
self.obj.updated_on = str(datetime.now())
if save:
self.obj.save(commit=commit)
return self.obj
def clean(self, value):
if value is None:
return
value = value.encode('utf-8')
from cloudml.importhandler.importhandler import ExtractionPlan
from api.amazon_utils import amazon_config
try:
plan = ExtractionPlan(value, is_file=False)
plan.amazon_settings = amazon_config()
self.import_params = plan.inputs.keys()
self.import_handler_type = 'xml'
except Exception as exc:
raise ValidationError(exc.message, exc)
return value
def add_error(self, name, msg, exc=None):
"""
Update the content of `self._errors`.
"""
if self.inner_name:
field_name = '%s-%s' % (self.inner_name, name)
else:
field_name = name
tb = None
if exc is not None:
if hasattr(exc, 'traceback'):
tb = exc.traceback
else:
e = ValidationError(msg, exc)
tb = e.traceback
self.errors.append({'name': field_name, 'error': msg, 'traceback': tb})
def clean(self, value):
value = super(DocumentField, self).clean(value)
if value is not None and value != u'undefined':
params = {'name' if self.by_name else 'id': value}
params.update(self.filter_params)
try:
obj = self.Model.query.filter_by(**params)[0]
except:
raise ValidationError('Document not found')
if self.return_doc:
return obj
else:
self.doc = obj
return value
def clean_is_target_variable(self, value, field):
if self.obj.is_target_variable and not value:
feature_set_id = self.cleaned_data.get('feature_set_id')
query = Feature.query.filter_by(
feature_set_id=feature_set_id)
if self.obj.id:
query = query.filter(Feature.id != self.obj.id)
features = query.all()
t_variable = None
for feature in features:
if feature.is_target_variable:
t_variable = feature.name
if not t_variable:
raise ValidationError('Target variable is not set for model '
'feature set. Edit another feature in '
'order to change target variable')
return value
def clean(self, value):
if value is None:
return
value = value.encode('utf-8')
from cloudml.importhandler.importhandler import Script, ScriptManager
from api.import_handlers.models import XmlScript
from api.amazon_utils import amazon_config
try:
xml_scr = XmlScript(data=value, type=XmlScript.TYPE_PYTHON_FILE)
s = Script(xml_scr.to_xml())
s.amazon_settings = amazon_config()
manager = ScriptManager()
manager.add_python(s.get_script_str())
except Exception as exc:
raise ValidationError(exc.message, exc)
return value
def _set_list_query_opts(self, cursor, params):
status = params.pop('status', None)
if status is not None:
if status == 'Active':
cursor = cursor.filter(
Cluster.status.in_(Cluster.ACTIVE_STATUSES))
elif status in Cluster.STATUSES:
cursor = cursor.filter_by(status=status)
else:
raise ValidationError('Invalid status: {0!s}'.format(status))
jobflow_id = params.pop('jobflow_id', None)
if jobflow_id:
cursor = cursor.filter(Cluster.jobflow_id.ilike(
'%{0}%'.format(jobflow_id)))
return cursor
def _get_sample_data_action(self, **kwargs):
ds = self._get_details_query({}, **kwargs)
if ds is None:
raise NotFound('DataSet not found')
if not os.path.exists(ds.filename):
raise NotFound('DataSet file cannot be found')
_, ext = os.path.splitext(ds.filename)
open_fn = gzip.open if ext == '.gz' else open if ext == '' else None
if not open_fn:
raise ValidationError('DataSet has unknown file type')
lines = []
params = self._parse_parameters([('size', int)])
sample_size = params.get('size') or 10
with open_fn(ds.filename, 'rb') as f:
line = f.readline()
while line and len(lines) < sample_size:
lines.append(json.loads(line))
line = f.readline()
return self._render(lines)
def only_one_required(cleaned_data, names, raise_exc=True):
filled_names = []
for name in names:
if cleaned_data.get(name, None):
filled_names.append(name)
count = len(filled_names)
err = None
if count == 0:
err = 'One of %s is required' % ', '.join(names)
elif count > 1:
err = 'Only one parameter from %s is required. \
%s are filled.' % (', '.join(names), ', '.join(filled_names))
if err:
if raise_exc:
raise ValidationError(err)
else:
return err
return ''
def check_choices(name, val, choices):
if choices:
if val not in choices:
raise ValidationError('Invalid {0}: should be one of {1}'.format(
name, ','.join(choices)))
