def __init__(self, *args, **kwargs):
handler_id = kwargs.pop('import_handler_id')
if handler_id is None:
raise ValidationError('Import Handler is required')
self.import_handler = XmlImportHandler.query.get(handler_id)
if self.import_handler is None:
raise ValidationError('Invalid import handler id specified')
super(PredictModelForm, self).__init__(*args, **kwargs)
def clean_import_params(self, value, field):
if not isinstance(value, dict):
raise ValidationError('Should be a dict')
for param in self.importhandler.import_params:
if param not in value:
raise ValidationError(
'{0!s} parameter is required'.format(param))
return value
def clean_entity(self, value, field):
if value:
if not (value.datasource and value.datasource.type == 'pig'):
raise ValidationError('Only "pig" entity is allowed')
if value and not self.is_edit:
query = XmlSqoop.query.filter_by(entity=value)
if query.count() >= self.MAX_ITEMS_BY_ENTITY:
raise ValidationError(
'There can be no more than {0} elements'.format(
self.MAX_ITEMS_BY_ENTITY))
return value
def clean_parameters(self, grid_params, field):
params = {}
config = CLASSIFIERS[self.model.classifier['type']]
config_params = config['parameters']
for pconfig in config_params:
name = pconfig['name']
if name in grid_params:
value = grid_params[name]
if not value:
continue
value = value.split(',')
type_ = pconfig.get('type', 'string')
if type_ == 'integer':
value = [int(item) for item in value]
elif type_ == 'float':
value = [float(item) for item in value]
elif type_ == 'boolean':
value = [item == 'true' for item in value]
choices = pconfig.get('choices')
if choices:
for item in value:
if item not in choices:
raise ValidationError(
'Invalid {0}: should be one of {1}'.format(
name, ','.join(choices)))
params[name] = value
return params
def clean_name(self, value, field):
if not ((self.NO_REQUIRED_FOR_EDIT and self.obj.id) or value):
raise ValidationError('name is required field')
import_handler_id = self.obj.import_handler_id if \
self.obj.id else self.data['import_handler_id']
query = XmlDataSource.query.filter_by(
name=value, import_handler_id=import_handler_id)
if self.obj.id:
query = query.filter(XmlDataSource.id != self.obj.id)
count = query.count()
if count:
raise ValidationError('Data Source with name "%s" already \
exist. Please choose another one.' % value)
return value
def clean_data(self, value, field):
try:
s = ScriptManager()
# this will raise exception in case of incorrect script
s.add_python(value)
except ImportHandlerException as ex:
raise ValidationError(ex.message, ex)
return value
def clean_name(self, value, field):
query = PredefinedDataSource.query.filter_by(name=value)
if self.obj.id:
query = query.filter(PredefinedDataSource.id != self.obj.id)
count = query.count()
if count:
raise ValidationError("DataSource with name \"%s\" already exist. "
"Please choose another one." % value)
return value
def clean_data(self, value, field):
if value is None:
return
value = value.encode('utf-8')
try:
ExtractionPlan(value, is_file=False)
return value
except Exception as exc:
raise ValidationError(exc.message, exc)
def clean_features(self, value, field):
if value:
from cloudml.trainer.trainer import Trainer
from cloudml.trainer.config import FeatureModel, SchemaException
try:
# TODO: add support of json dict to FeatureModel
feature_model = FeatureModel(json.dumps(value), is_file=False)
self.cleaned_data['trainer'] = Trainer(feature_model)
except SchemaException, exc:
raise ValidationError(
'Features JSON file is invalid: %s' % exc, exc)
def clean_trainer(self, value, field):
if value:
try:
# TODO: find a better way?
from cloudml.trainer.store import load_trainer
value = value.encode('utf-8').replace('\r', '')
trainer_obj = load_trainer(value)
self.cleaned_data['status'] = Model.STATUS_TRAINED
return trainer_obj
except Exception as exc:
raise ValidationError(
'Pickled trainer model is invalid: {0!s}'.format(exc), exc)
def save(self, *args, **kwargs):
try:
script_type = self.cleaned_data.get('type', None)
data_file = self.cleaned_data.get('data_file', None)
data_url = self.cleaned_data.get('data_url', None)
data = self.cleaned_data.get('data', None)
if script_type == XmlScript.TYPE_PYTHON_FILE:
if data_file:
key = XmlScript.to_s3(
data_file, self.cleaned_data.get('import_handler_id'))
self.cleaned_data['data'] = key
elif data_url:
self.cleaned_data['data'] = data_url
else:
raise ValidationError("File upload or URL required "
"for type '{0}'".format(script_type))
elif script_type == XmlScript.TYPE_PYTHON_CODE:
if not data:
raise ValidationError("Code is required for type "
"'{0}'".format(script_type))
# type is not passed
else:
if data_file:
key = XmlScript.to_s3(
data_file, self.cleaned_data.get('import_handler_id'))
self.cleaned_data['data'] = key
self.cleaned_data['type'] = XmlScript.TYPE_PYTHON_FILE
elif data_url:
self.cleaned_data['data'] = data_url
self.cleaned_data['type'] = XmlScript.TYPE_PYTHON_FILE
else:
self.cleaned_data['type'] = XmlScript.TYPE_PYTHON_CODE
script = super(XmlScriptForm, self).save()
except Exception as e:
raise ValidationError(e.message, e)
return script
def _save_importhandler(self, fieldname, name, handler_type='xml'):
"""
Adds new import handler to the system,
if it was specified in file field.
Use it in the model.
"""
def determine_name(name, action):
if action == 'train':
name = "%s import handler" % name
else:
name = "%s test import handler" % name
while True:
count = XmlImportHandler.query.filter_by(name=name).count()
if not count:
return name
name += '_'
return name
data = self.cleaned_data.pop(fieldname, None)
if data is not None:
if handler_type == 'xml':
from api.import_handlers.models import XmlImportHandler
cls = XmlImportHandler
handler = cls()
action = 'test' if fieldname.startswith('test') else 'train'
handler.name = determine_name(name, action)
handler.import_params = self.cleaned_data.pop('%s_import_params' %
action)
handler._set_user()
try:
handler.data = data
except Exception, exc:
self.add_error('fields', str(exc), exc)
raise ValidationError(self.error_messages,
exc,
errors=self.errors)
self.cleaned_data['%s_import_handler' % action] = handler
db.session.add(handler)
db.session.commit()
return handler
def save(self):
try:
import_handler = XmlImportHandler(name=self.cleaned_data['name'],
import_params=[])
import_handler._set_user()
db.session.add(import_handler)
try:
import_handler.data = self.cleaned_data.get('data')
except Exception, exc:
self.add_error('fields', str(exc), exc)
raise ValidationError(self.error_messages,
exc,
errors=self.errors)
except Exception as e:
db.session.rollback()
raise DBException(e.message, e)
else:
db.session.commit()
return import_handler
def clean_name(self, value, field):
count = XmlImportHandler.query.filter_by(name=value).count()
if count:
raise ValidationError('Import Handler with name "%s" already \
exist. Please choose another one.' % value)
return value
def clean_datasource(self, value, field):
if value:
if value.type != 'db':
raise ValidationError('Only "db" datasources are allowed')
return value
def clean_type(self, value, field):
if value and value not in self.TYPES:
raise ValidationError('invalid type')
return value
def clean_transformed_field(self, value, field):
if value and self.data.get('datasource'):
raise ValidationError(self.DATASOURCE_MESSAGE)
return value