def _addMetabolizeSchema(self, schema):
scenario = colander.SchemaNode(colander.Mapping())
transformation_types = colander.OneOf([
'phase1', 'phase1_selected', 'phase2', 'phase2_selected',
'glycosidase', 'mass_filter', 'gut'
])
scenario.add(
colander.SchemaNode(colander.String(),
name='type',
validator=transformation_types))
# TODO add validator for steps, but what are valid options for steps?
scenario.add(colander.SchemaNode(colander.String(), name='steps'))
schema.add(
colander.SchemaNode(colander.Sequence(),
scenario,
validator=colander.Length(1),
name='scenario'))
class GpNextPointsStatus(base_schemas.StrictMappingSchema):
"""A status schema for the various subclasses of :class:`moe.views.gp_next_points_pretty_view.GpNextPointsPrettyView`; e.g., :class:`moe.views.rest.gp_next_points_epi.GpNextPointsEpi`.
**Output fields**
:ivar expected_improvement: (*float64 >= 0.0*) EI evaluated at ``points_to_sample`` (:class:`moe.views.schemas.base_schemas.ListOfExpectedImprovements`)
:ivar optimizer_success: (*dict*) Whether or not the optimizer converged to an optimal set of ``points_to_sample``
"""
expected_improvement = colander.SchemaNode(
colander.Float(),
validator=colander.Range(min=0.0),
)
optimizer_success = colander.SchemaNode(
colander.Mapping(unknown='preserve'),
default={'found_update': False},
)
class User(Node, BaseUser):
email = colander.SchemaNode(colander.String(),
validator=colander.Email(),
default='',
missing='')
phone = colander.SchemaNode(colander.String(),
default='',
missing='')
address = colander.SchemaNode(colander.String(),
default='',
missing='')
commentaries = colander.SchemaNode(colander.String(),
default='',
missing='')
memberof = ObjectIdList(missing=[], default=[])
policies = colander.SchemaNode(colander.Mapping(unknown='preserve'),
default={},
missing={})
computers = ObjectIdList(missing=[], default=[])
def get_sequence_model_admin(model, title=u"", excludes=(), **kw):
"""
Return a schema for configuring sequence of models
model
The SQLAlchemy model to configure
"""
node_schema = SQLAlchemySchemaNode(
model,
widget=deform.widget.MappingWidget(template=TEMPLATES_URL +
"clean_mapping.pt", ),
excludes=excludes,
)
node_schema.name = 'data'
default_widget_options = dict(
orderable=True,
template=TEMPLATES_URL + "clean_sequence.pt",
min_len=1,
)
widget_options = kw.get('widget_options', {})
for key, value in widget_options.items():
default_widget_options[key] = value
schema = colander.SchemaNode(colander.Mapping())
schema.add(
colander.SchemaNode(
colander.Sequence(),
node_schema,
widget=deform.widget.SequenceWidget(**default_widget_options),
title=title,
name='datas'))
def dictify(models):
return {'datas': [node_schema.dictify(model) for model in models]}
def objectify(datas):
return [node_schema.objectify(data) for data in datas]
schema.dictify = dictify
schema.objectify = objectify
return schema
def metabolize_one(self, params, has_ms_data=False):
"""Configure job query to metabolize one structure.
``params`` is a MultiDict from which the following keys are used:
* molid, molecule identifier to metabolize
* scenario, transformation scenario
If ``has_ms_data`` is True then
:meth:`~magmaweb.job.JobQuery.annotate` will be called.
"""
schema = colander.SchemaNode(colander.Mapping())
schema.add(colander.SchemaNode(colander.Integer(),
validator=colander.Range(min=0),
name='molid'))
self._addMetabolizeSchema(schema)
if has_ms_data:
self._addAnnotateSchema(schema)
if hasattr(params, 'mixed'):
# unflatten multidict
params = params.mixed()
self._deserialize_scenario(params)
params = schema.deserialize(params)
self._write_scenario_file(params)
self.prestaged.append('scenario.csv')
script = "echo '{molid}' | {{magma}} metabolize -g -j -"
script += " --scenario scenario.csv"
script += " --call_back_url '{call_back_url}' {{db}}"
script_substitutions = {
'molid': self.escape(params['molid']),
'call_back_url': self.status_callback_url,
}
self.script += script.format(**script_substitutions)
if (has_ms_data):
self.script += " |"
self.annotate(params, True)
else:
self.script += "\n"
return self
def __init__(self, request, hide_complex=False, process=None, use_async=False, unknown='ignore', user=None, **kw):
""" Initialise the given mapped schema according to options provided.
Arguments/Keywords
process:
An ``WPS`` process description that you want a ``Colander`` schema
to be generated for.
unknown
Represents the `unknown` argument passed to
:class:`colander.Mapping`.
From Colander:
``unknown`` controls the behavior of this type when an unknown
key is encountered in the cstruct passed to the deserialize
method of this instance.
Default: 'ignore'
\*\*kw
Represents *all* other options able to be passed to a
:class:`colander.SchemaNode`. Keywords passed will influence the
resulting mapped schema accordingly (for instance, passing
``title='My Model'`` means the returned schema will have its
``title`` attribute set accordingly.
See http://docs.pylonsproject.org/projects/colander/en/latest/basics.html for more information.
"""
kwargs = kw.copy()
# The default type of this SchemaNode is Mapping.
colander.SchemaNode.__init__(self, colander.Mapping(unknown), **kwargs)
self.request = request
self.hide_complex = hide_complex
self.process = process
self.unknown = unknown
self.user = user
self.kwargs = kwargs or {}
if use_async:
self.add_async_check()
self.add_nodes(process)
class ConfigurationSchema(colander.Schema):
"""Configuration for Raven, the official Sentry client."""
RAVEN_CONFIG = colander.SchemaNode(
colander.Mapping(unknown='raise'),
missing=colander.required,
default=colander.null,
children=[
colander.SchemaNode(
colander.String(),
name='dsn',
missing=colander.required,
default=colander.null,
)
]
)
SENTRY_CLIENT = colander.SchemaNode(
colander.String(),
missing=colander.drop,
default='raven.contrib.django.raven_compat.DjangoClient',
)
class Project(col.MappingSchema):
name = col.SchemaNode(ProjectNameType())
summary = col.SchemaNode(col.Str(), missing='')
description = col.SchemaNode(col.Str(), missing='')
admin = col.SchemaNode(User())
private = col.SchemaNode(col.Bool(), missing=False)
labels = Labels(missing=[])
external_homepage = col.SchemaNode(col.Str(), missing='')
video_url = col.SchemaNode(col.Str(), missing='')
trove_root_databases = TroveDatabases(missing=None)
trove_developmentstatuses = TroveStatuses(validator=col.Length(max=6), missing=None)
trove_audiences = TroveAudiences(validator=col.Length(max=6), missing=None)
trove_licenses = TroveLicenses(validator=col.Length(max=6), missing=None)
trove_oses = TroveOSes(missing=None)
trove_languages = TroveLanguages(validator=col.Length(max=6), missing=None)
trove_topics = TroveTopics(validator=col.Length(max=3), missing=None)
trove_natlanguages = TroveTranslations(missing=None)
trove_environments = TroveUIs(missing=None)
tool_data = col.SchemaNode(col.Mapping(unknown='preserve'), missing={})
icon = col.SchemaNode(col.Str(), missing=None)
def _get_schema_for_cstruct(self, request, params: dict):
schema = super()._get_schema_for_cstruct(request, params)
if params.get('serialization_form', False) == 'content':
elements = schema['elements']
typ_copy = deepcopy(elements.children[0].typ)
typ_copy.serialization_form = 'content'
elements.children[0].typ = typ_copy
if params.get('show_count', False):
child = colander.SchemaNode(colander.Integer(),
default=0,
missing=colander.drop,
name='count')
schema.add(child)
if params.get('show_frequency', False):
child = colander.SchemaNode(colander.Mapping(unknown='preserve'),
default={},
missing=colander.drop,
name='aggregateby')
schema.add(child)
return schema
def _addAddStructuresSchema(self, has_ms_data, must_metabolize):
def textarea_or_file(node, value):
"""Validator that either textarea or file upload is filled"""
if not (not value['structures'] is colander.null
or not value['structures_file'] is colander.null):
error = 'Either structures or structures_file must be set'
exception = colander.Invalid(node)
exception.add(colander.Invalid(structuresSchema, error))
exception.add(colander.Invalid(structures_fileSchema, error))
raise exception
schema = colander.SchemaNode(colander.Mapping(),
validator=textarea_or_file)
formats = ['smiles', 'sdf']
schema.add(
colander.SchemaNode(colander.String(),
validator=colander.OneOf(formats),
name='structure_format'))
filled = colander.Length(min=1)
structuresSchema = colander.SchemaNode(colander.String(),
validator=filled,
missing=colander.null,
name='structures')
schema.add(structuresSchema)
structures_fileSchema = colander.SchemaNode(self.File(),
missing=colander.null,
name='structures_file')
schema.add(structures_fileSchema)
schema.add(
colander.SchemaNode(colander.Boolean(),
default=False,
missing=False,
name='metabolize'))
if has_ms_data:
self._addAnnotateSchema(schema)
if must_metabolize:
self._addMetabolizeSchema(schema)
return schema
class Plugin(StrictAboutExtraKeysColanderMappingSchema):
name = colander.SchemaNode(
colander.String(),
title="Plugin name",
description=
"Descriptive name for this plugin (to identify it in the web console)",
missing='',
default='',
validator=colander.Regex(
VALID_IDENTIFIER,
"Plugin name must match " + VALID_IDENTIFIER.pattern))
plugin = colander.SchemaNode(
colander.String(),
title="Symbolic plugin name",
description=
"Symbolic name of this plugin, as defined in the main paasmaker options file",
validator=colander.Regex(
VALID_PLUGIN_NAME,
"Plugin name must match " + VALID_PLUGIN_NAME.pattern))
parameters = colander.SchemaNode(colander.Mapping(unknown='preserve'),
missing={},
default={})
def validate(data, relation):
""" Due to some fairly weird interdependencies between the different elements
of the model, validation of relations has to happen in three steps. """
validator = RelationBaseValidator()
sane = validator.deserialize(data)
project = sane.get('project')
schema_validator = colander.SchemaNode(colander.Mapping())
schema_validator.add(colander.SchemaNode(SchemaRef(project),
name='schema'))
schema_validator.add(colander.SchemaNode(EntityRef(project=project),
name='source'))
schema_validator.add(colander.SchemaNode(EntityRef(project=project),
name='target'))
sane.update(schema_validator.deserialize(data))
sane['properties'] = properties_logic.validate('relation', relation,
project, sane.get('schema'),
data.get('properties', []))
return sane
class ReportDetailBaseSchema(colander.MappingSchema):
"""
Validates format of report - ie. request parameters and stats for a request in report group
"""
username = colander.SchemaNode(
colander.String(),
preparer=[shortener_factory(255), lambda x: x or ''],
missing='')
request_id = colander.SchemaNode(colander.String(),
preparer=shortener_factory(40),
missing='')
url = colander.SchemaNode(colander.String(),
preparer=shortener_factory(1024),
missing='')
ip = colander.SchemaNode(colander.String(),
preparer=shortener_factory(39),
missing=None)
start_time = colander.SchemaNode(NonTZDate(),
validator=optional_limited_date,
missing=deferred_utcnow)
end_time = colander.SchemaNode(NonTZDate(),
validator=optional_limited_date,
missing=None)
user_agent = colander.SchemaNode(
colander.String(),
preparer=[shortener_factory(512), lambda x: x or ''],
missing='')
message = colander.SchemaNode(colander.String(),
preparer=shortener_factory(2048),
missing='')
group_string = colander.SchemaNode(colander.String(),
preparer=shortener_factory(512),
missing=None)
request_stats = RequestStatsSchema(missing=None)
request = colander.SchemaNode(colander.Mapping(unknown='preserve'),
missing={})
traceback = FrameInfoListSchema(missing=None)
slow_calls = SlowCallListSchema(missing=[])
extra = ExtraSchemaList()
def dataclass_field_to_colander_schemanode(
prop: dataclasses.Field,
oid_prefix='deformField') -> colander.SchemaNode:
t = dataclass_get_type(prop)
if t['type'] == date:
params = colander_params(prop, oid_prefix, typ=colander.Date())
return colander.SchemaNode(**params)
if t['type'] == datetime:
params = colander_params(prop, oid_prefix, typ=colander.DateTime())
return colander.SchemaNode(**params)
if t['type'] == str:
params = colander_params(prop, oid_prefix, typ=colander.String())
return colander.SchemaNode(**params)
if t['type'] == int:
params = colander_params(prop, oid_prefix, typ=colander.Integer())
return colander.SchemaNode(**params)
if t['type'] == float:
params = colander_params(prop, oid_prefix, typ=colander.Float())
return colander.SchemaNode(**params)
if t['type'] == bool:
params = colander_params(prop, oid_prefix, typ=colander.Boolean())
return colander.SchemaNode(**params)
if dataclass_check_type(prop, ISchema):
subtype = dataclass_to_colander(
t['type'], colander_schema_type=colander.MappingSchema)
return subtype()
if t['type'] == dict:
params = colander_params(prop, oid_prefix, typ=colander.Mapping())
return colander.SchemaNode(**params)
if t['type'] == list:
params = colander_params(prop, oid_prefix, typ=colander.List())
return colander.SchemaNode(**params)
raise KeyError(prop)
def validate(data):
""" Due to some fairly weird interdependencies between the different elements
of the model, validation of entities has to happen in three steps. """
# a bit hacky
data['schemata'] = data.get('schemata', []) + ['base']
validator = EntityBaseValidator()
sane = validator.deserialize(data)
schemata_validator = colander.SchemaNode(colander.Mapping())
schemata_node = colander.SchemaNode(SchemaRef(sane.get('project')))
schemata_validator.add(
colander.SchemaNode(colander.Sequence(),
schemata_node,
name='schemata'))
sane.update(schemata_validator.deserialize(data))
sane['schemata'] = list(set(sane['schemata']))
sane['properties'] = validate_properties(data.get('properties', []),
sane.get('schemata'),
name='properties')
return sane
class Computer(Node):
type = colander.SchemaNode(colander.String(),
default='computer',
validator=colander.OneOf(['computer']))
memberof = ObjectIdList(missing=[], default=[])
family = colander.SchemaNode(colander.String(),
default='desktop',
validator=colander.OneOf(
COMPUTER_FAMILY.keys()))
registry = colander.SchemaNode(colander.String(), default='', missing='')
serial = colander.SchemaNode(colander.String(), default='', missing='')
commentaries = colander.SchemaNode(colander.String(),
default='',
missing='')
policies = colander.SchemaNode(colander.Mapping(unknown='preserve'),
default={},
missing={})
node_chef_id = colander.SchemaNode(colander.String(),
default='',
missing='')
error_last_saved = colander.SchemaNode(RealBoolean(), default=False)
error_last_chef_client = colander.SchemaNode(RealBoolean(), default=False)
gcc_link = colander.SchemaNode(RealBoolean(), default=True)
sudoers = StringList(missing=[], default=[])
def validate_payloads(charm, linter):
"""Validate paylaod configuration in charm metadata.
:param charm: dict of charm metadata parsed from metadata.yaml
:param linter: :class:`CharmLinter` object to which info/warning/error
messages will be written
"""
if 'payloads' not in charm:
return
if (not isinstance(charm['payloads'], dict) or not charm['payloads']):
linter.err('payloads: must be a dictionary of payload definitions')
return
schema = colander.SchemaNode(colander.Mapping())
for payload_def in charm['payloads']:
schema.add(PayloadItem(name=payload_def))
try:
schema.deserialize(charm['payloads'])
except colander.Invalid as e:
for k, v in e.asdict().items():
linter.err('payloads.{}: {}'.format(k, v))
def schema_type(self):
if self.get_option("preserve_unknown") is True:
unknown = "preserve"
else:
unknown = "ignore"
return colander.Mapping(unknown=unknown)
def schema_type():
return colander.Mapping(unknown="raise")
def schema_type():
return colander.Mapping(unknown="ignore")
def schema_type():
return colander.Mapping(unknown="preserve")
def schema_type(self):
if self.known_perms:
return colander.Mapping(unknown="raise")
else:
return colander.Mapping(unknown="preserve")
def schema_type(self, **kw):
return colander.Mapping(unknown='raise')
class VersionResponseSchema(colander.MappingSchema):
body = colander.SchemaNode(colander.Mapping(unknown='preserve'))
def to_schema(cls, seen=None):
""" Returns a colander schema representation of the object.
Recursively evaluates to_schema(...) on its relationships.
"""
if not seen:
seen = []
properties = list(class_mapper(cls).iterate_properties)
relationships = [
p.key for p in properties if type(p) is RelationshipProperty
]
attrs = {}
for prop in properties:
if not prop in relationships:
attrs[prop.key] = prop
schema = colander.SchemaNode(colander.Mapping())
for attr, column_property in attrs.iteritems():
node = None
column = column_property.columns[0]
missing_val = colander.null if column.nullable else colander.required
column_type = column.type
string_types = (sqlalchemy.Unicode, sqlalchemy.UnicodeText,
sqlalchemy.String)
int_types = (sqlalchemy.Integer, sqlalchemy.SmallInteger,
sqlalchemy.BigInteger)
if isinstance(column_type, string_types):
validator = None
if column.type.length:
validator = colander.Length(max=column.type.length)
node = colander.SchemaNode(colander.String())
if validator:
node.validator = validator
elif isinstance(column_type, int_types):
node = colander.SchemaNode(colander.Integer())
elif isinstance(column_type, sqlalchemy.Boolean):
node = colander.SchemaNode(colander.Boolean())
node.name = attr
node.missing = missing_val
schema.add(node)
# d = dict([(attr, getattr(obj, attr)) for attr in attrs])
#TODO: Support relationships
# for attr in relationships:
# d[attr] = expand(obj, getattr(obj, attr), seen)
#
return schema
class LbHeartbeatResponseSchema(colander.MappingSchema):
body = colander.SchemaNode(colander.Mapping())
class HeartbeatResponseSchema(colander.MappingSchema):
body = colander.SchemaNode(colander.Mapping(unknown='preserve'))
import colander
import deform
from pyramid.httpexceptions import HTTPFound
from .residence import ResidenceCRUD
from ..models import Residence, WeightFactor, WeightMapping, SmootherstepMapping
import json
class SmoothstepMapping(colander.MappingSchema):
lower=colander.SchemaNode(colander.Float())
upper=colander.SchemaNode(colander.Float())
class FieldWeightSchema(colander.MappingSchema):
mapping=SmoothstepMapping()
weight=colander.SchemaNode(colander.Float())
weight_schema=colander.SchemaNode(colander.Mapping())
for field in Residence.score_fields:
fieldmapping=colander.SchemaNode(
colander.Mapping(),
name=field,
label=Residence.score_field_labels.get(field,field)
)
MappingType=Residence.score_mapping_types.get(
field, SmootherstepMapping)
if issubclass(MappingType,SmootherstepMapping):
fieldmapping.add(
SmoothstepMapping(name='mapping')
)
fieldmapping.add(colander.SchemaNode(colander.Float(),name='weight'))
weight_schema.add(fieldmapping)
class ContributeResponseSchema(colander.MappingSchema):
body = colander.SchemaNode(colander.Mapping(unknown="preserve"))
def dataclass_field_to_colander_schemanode(
prop: dataclasses.Field,
schema,
request,
oid_prefix="deformField",
mode=None,
default_tzinfo=pytz.UTC,
metadata=None,
) -> colander.SchemaNode:
t = dataclass_get_type(prop)
field_factory = t["metadata"].get("colander.field_factory", None)
if field_factory:
params = colander_params(
prop,
oid_prefix,
typ=field_factory(request),
schema=schema,
request=request,
mode=mode,
)
return SchemaNode(**params)
if t["type"] == date:
params = colander_params(prop,
oid_prefix,
typ=Date(),
schema=schema,
request=request,
mode=mode)
return SchemaNode(**params)
if t["type"] == datetime:
params = colander_params(
prop,
oid_prefix,
typ=DateTime(default_tzinfo=default_tzinfo),
schema=schema,
request=request,
mode=mode,
)
return SchemaNode(**params)
if t["type"] == str:
params = colander_params(prop,
oid_prefix,
typ=Str(),
schema=schema,
request=request,
mode=mode)
return SchemaNode(**params)
if t["type"] == int:
params = colander_params(prop,
oid_prefix,
typ=Int(),
schema=schema,
request=request,
mode=mode)
return SchemaNode(**params)
if t["type"] == float:
params = colander_params(prop,
oid_prefix,
typ=Float(),
schema=schema,
request=request,
mode=mode)
return SchemaNode(**params)
if t["type"] == bool:
params = colander_params(prop,
oid_prefix,
typ=Boolean(),
schema=schema,
request=request,
mode=mode)
return SchemaNode(**params)
if is_dataclass_field(prop):
subtype = dc2colanderESjson(
prop,
colander_schema_type=colander.MappingSchema,
request=request,
mode=mode,
field_metadata=metadata,
default_tzinfo=default_tzinfo,
)
return subtype()
if t["type"] == dict:
params = colander_params(
prop,
oid_prefix,
typ=colander.Mapping(unknown="preserve"),
schema=schema,
request=request,
mode=mode,
)
return SchemaNode(**params)
return orig_dc2colander_node(
prop=prop,
schema=schema,
request=request,
oid_prefix=oid_prefix,
mode=mode,
default_tzinfo=default_tzinfo,
metadata=metadata,
)
