class AdvertSchema(Schema):
id = fields.Integer(dump_only=True)
title = fields.String()
images_set = fields.Relationship(schema=ImageSchema,
many=True,
dump_only=True,
dump_to='images')
description = fields.String()
author = fields.Relationship(
schema=UserSchema,
type_='user',
related_url='/api/v1/users/{author_id}',
related_url_kwargs={'author_id': '<author.id>'},
include_resource_linkage=False)
# timestamp = TimeStampConvertedField()
price = fields.Float()
timestamp = fields.LocalDateTime('%Y%m%d%H%M%S', dump_only=True)
key = fields.Method('ExampleKeyGenerated', dump_only=True)
class Meta:
type_ = 'advert'
additional = ('id', )
def ExampleKeyGenerated(self, obj):
return ''.join(('@', str(obj.item)))
@post_load
def make_advert(self, data):
return Adverts(**data)
class ActivatedApplicationSchema(ma.Schema):
id = fields.String(dump_only=True)
application = fields.Relationship(
related_url='/applications/{application_id}',
related_url_kwargs={'application_id': '<id>'},
many=True,
schema=ApplicationSchema,
include_resource_linkage=True,
type_='applications',
dump_only=True
)
owner = fields.Relationship(
related_url='/users/{user_id}',
related_url_kwargs={'user_id': '<id>'},
many=False,
schema=UserSchema,
include_resource_linkage=True,
type_='users',
dump_only=True
)
activate_date = fields.DateTime(dump_only=True)
class Meta:
type_ = 'activated-applications'
strict = True
inflect = common.dasherize
class FavoriteSchema(BaseSchema):
"""Schema for models.Favorite"""
class Meta:
type_ = 'favorite'
self_url = '/favorites/{uuid}'
self_url_many = '/favorites/'
self_url_kwargs = {'uuid': '<id>'}
json_module = simplejson
id = fields.Str(dump_only=True, attribute='uuid')
item_uuid = fields.Str(required=True, validate=NOT_BLANK)
item = fields.Relationship(
include_resource_linkage=True,
type_='item',
schema='ItemSchema',
id_field='uuid',
)
user = fields.Relationship(
include_resource_linkage=True,
type_='user',
schema='UserSchema',
id_field='uuid',
)
class StockSchema(Schema):
id = fields.String(dump_only=True)
name = fields.String(required=True,
validator=ma.validate.Length(min=3, max=20))
description = fields.String()
tags = fields.List(fields.String())
status = fields.String(requred=True, default='deactive')
owner = fields.Relationship(related_url='/users/{user_id}',
related_url_kwargs={'user_id': '<id>'},
many=False,
schema=common.UserSchema,
include_resource_linkage=True,
type_='users',
dump_only=True)
building = fields.Relationship(
related_url='/buildings/{building_id}',
related_url_kwargs={'building_id': '<id>'},
many=False,
schema=common.BuildingSchema,
include_resource_linkage=True,
type_='buildings',
)
created_date = fields.DateTime(dump_only=True)
updated_date = fields.DateTime(dump_only=True)
class Meta:
type_ = 'stocks'
strict = True
inflect = common.dasherize
class AnnotationSchema(BaseAnnotationSchema):
required = fields.Boolean(default=False)
ignore = fields.Boolean(default=False)
ignore_beneath = fields.Boolean(default=False)
variant = fields.Integer(default=False)
slice = fields.List(fields.Integer())
pre_text = fields.Str()
post_text = fields.Str()
selection_mode = fields.Str()
field = fields.Relationship(
related_url='/api/projects/{project_id}/schemas/{schema_id}/fields/'
'{field_id}',
related_url_kwargs={
'project_id': '<project_id>',
'schema_id': '<schema_id>',
'field_id': '<field.id>'
},
type_='fields',
include_resource_linkage=True)
extractors = fields.Relationship(
related_url='/api/projects/{project_id}/extractors',
related_url_kwargs={'project_id': '<project_id>'},
many=True,
include_resource_linkage=True,
type_='extractors')
class Meta:
type_ = 'annotations'
class ProcessSchema(EntitySchema):
spatial_scope = fields.Str(attribute='SpatialScope')
temporal_scope = fields.Str(attribute='TemporalScope')
Classification = fields.List(fields.Str())
reference_entity = fields.Relationship(
self_url='/{ref_link}/reference',
self_url_kwargs={'ref_link': '<link>'},
related_url='/{ref_link}',
related_url_kwargs={'ref_link': '<link>'},
many=True,
include_resource_linkage=True,
type_='reference_exchange',
id_field='link',
schema=ReferenceExchangeSchema)
exchanges = fields.Relationship(
self_url='/{origin}/{id}/exchanges',
self_url_kwargs={
'id': '<external_ref>',
'origin': '<origin>'
},
)
class Meta:
type_ = 'process'
self_url_many = '/processes/'
class PostSchema(Schema):
id = fields.Int()
post_title = fields.Str(attribute='title', dump_to='title')
author = fields.Relationship('http://test.test/posts/{id}/author/',
related_url_kwargs={'id': '<id>'},
schema=AuthorSchema,
many=False,
type_='people')
post_comments = fields.Relationship(
'http://test.test/posts/{id}/comments/',
related_url_kwargs={'id': '<id>'},
attribute='comments',
load_from='post-comments',
dump_to='post-comments',
schema='CommentSchema',
many=True,
type_='comments')
post_keywords = fields.Relationship(
'http://test.test/posts/{id}/keywords/',
related_url_kwargs={'id': '<id>'},
attribute='keywords',
dump_to='post-keywords',
schema='KeywordSchema',
many=True,
type_='keywords')
class Meta:
type_ = 'posts'
class CommentSchema(BaseSchema):
id = fields.String()
content = fields.String()
created_on = fields.DateTime()
poster = fields.String(load_only=True)
class Meta:
type_ = 'comments'
self_url = '/api/comments/{id}'
self_url_kwargs = {'id': '<id>'}
self_url_many = '/api/comments'
user = fields.Relationship(
type_='users',
self_url = '/api/comments/{id}/relationships/user',
self_url_kwargs = {'id': '<id>'},
related_url='/api/comments/{id}/user',
related_url_kwargs={'id': '<id>'},
include_resource_linkage=True,
schema='UserSchema'
)
post = fields.Relationship(
type_='posts',
self_url = '/api/comments/{id}/relationships/post',
self_url_kwargs = {'id': '<id>'},
related_url='/api/comments/{id}/post',
related_url_kwargs={'id': '<id>'},
include_resource_linkage=True,
schema='PostSchema'
)
class WorksheetSchema(Schema):
id = fields.String(validate=validate_uuid, attribute='uuid')
uuid = fields.String(attribute='uuid') # for backwards compatibility
name = fields.String(validate=validate_name)
owner = fields.Relationship(include_resource_linkage=True,
type_='users',
attribute='owner_id')
title = fields.String()
frozen = fields.DateTime(allow_none=True)
is_anonymous = fields.Bool()
tags = fields.List(fields.String())
group_permissions = fields.Relationship(include_resource_linkage=True,
type_='worksheet-permissions',
id_field='id',
many=True)
items = fields.Relationship(include_resource_linkage=True,
type_='worksheet-items',
id_field='id',
many=True)
last_item_id = fields.Integer(dump_only=True)
# Bundle permission of the authenticated user for convenience, read-only
permission = fields.Integer()
permission_spec = PermissionSpec(attribute='permission')
class Meta:
type_ = 'worksheets'
class WorksheetPermissionSchema(Schema):
id = fields.Integer(as_string=True, dump_only=True)
worksheet = fields.Relationship(
include_resource_linkage=True,
attribute='object_uuid',
type_='worksheets',
load_only=True,
required=True,
)
group = fields.Relationship(include_resource_linkage=True,
attribute='group_uuid',
type_='groups',
required=True)
group_name = fields.String(dump_only=True) # for convenience
permission = fields.Integer(validate=lambda p: 0 <= p <= 2)
permission_spec = PermissionSpec(attribute='permission') # for convenience
@validates_schema
def check_permission_exists(self, data):
if 'permission' not in data:
raise ValidationError(
"One of either permission or permission_spec must be provided."
)
class Meta:
type_ = 'worksheet-permissions'
class BundleSchema(Schema):
id = fields.String(validate=validate_uuid, attribute='uuid')
uuid = fields.String(attribute='uuid') # for backwards compatibility
bundle_type = fields.String(
validate=validate.OneOf({bsc.BUNDLE_TYPE
for bsc in BUNDLE_SUBCLASSES}))
command = fields.String(allow_none=True)
data_hash = fields.String()
state = fields.String()
owner = fields.Relationship(include_resource_linkage=True,
type_='users',
attribute='owner_id')
is_anonymous = fields.Bool()
metadata = fields.Dict()
dependencies = fields.Nested(BundleDependencySchema, many=True)
children = fields.Relationship(include_resource_linkage=True,
type_='bundles',
id_field='uuid',
many=True)
group_permissions = fields.Relationship(include_resource_linkage=True,
type_='bundle-permissions',
id_field='id',
many=True)
host_worksheets = fields.Relationship(include_resource_linkage=True,
type_='worksheets',
id_field='uuid',
many=True)
args = fields.String()
# Bundle permission of the authenticated user for convenience, read-only
permission = fields.Integer()
permission_spec = PermissionSpec(attribute='permission')
class Meta:
type_ = 'bundles'
class ProjectSchema(SlydSchema):
id = fields.Str(load_from='name')
name = fields.Str()
spiders = fields.Relationship(
related_url='/api/projects/{project_id}/spiders',
related_url_kwargs={'project_id': '<id>'},
type_='spiders',
include_resource_linkage=True,
many=True)
schemas = fields.Relationship(
related_url='/api/projects/{project_id}/schemas',
related_url_kwargs={'project_id': '<id>'},
type_='schemas',
include_resource_linkage=True,
many=True)
extractors = fields.Relationship(
related_url='/api/projects/{project_id}/extractors',
related_url_kwargs={'project_id': '<id>'},
type_='extractors',
include_resource_linkage=True,
many=True)
project = fields.Relationship(self_url='/api/projects/{project_id}',
self_url_kwargs={'project_id': '<id>'},
type_='projects')
class Meta:
type_ = 'projects'
class SpiderSchema(SlydSchema):
id = fields.Str(dump_only=True, load_from='name')
name = fields.Str()
start_urls = fields.List(fields.Str(), default=[])
links_to_follow = fields.Str(default='patterns')
follow_patterns = fields.List(fields.Str(), default=[])
exclude_patterns = fields.List(fields.Str(), default=[])
js_enabled = fields.Boolean(default=False)
js_enable_patterns = fields.List(fields.Str(), default=[])
js_disable_patterns = fields.List(fields.Str(), default=[])
respect_nofollow = fields.Boolean(default=True)
allowed_domains = fields.List(fields.Str(), default=[])
login_url = fields.Str()
login_user = fields.Str()
login_password = fields.Str()
perform_login = fields.Boolean(default=False)
template_names = fields.List(fields.Str(), default=[])
samples = fields.Relationship(
related_url='/api/projects/{project_id}/spider/{spider_id}/samples',
related_url_kwargs={
'project_id': '<project_id>',
'spider_id': '<spider_id>'
},
many=True,
include_resource_linkage=True,
type_='samples')
project = fields.Relationship(
related_url='/api/projects/{project_id}',
related_url_kwargs={'project_id': '<project_id>'},
type_='projects',
include_resource_linkage=True)
@pre_dump
def _dump_login_data(self, item):
init_requests = item.pop('init_requests', None)
if init_requests:
login_request = init_requests[0]
item['login_url'] = login_request['loginurl']
item['login_user'] = login_request['username']
item['login_password'] = login_request['password']
return item
@post_load
def _load_login_data(self, item):
fields = ('login_url', 'login_user', 'login_password')
if all(field in item and item[field] for field in fields):
item['init_requests'] = [{
'type': 'login',
'loginurl': item.pop('login_url'),
'username': item.pop('login_user'),
'password': item.pop('login_password')
}]
for field in fields:
item.pop(field, None)
return item
class Meta:
type_ = 'spiders'
class Category(Schema):
id = fields.Integer()
name = fields.String()
category = fields.Relationship(schema='Category')
categories = fields.Relationship(schema='Category')
class Meta:
inflect = dasherize
type_ = 'categories'
class ItemSchema(SlydSchema):
"""Instance of a schema. Meta item built from sample."""
id = fields.Str()
sample = fields.Relationship(
related_url='/api/projects/{project_id}/spider/{spider_id}/samples/'
'{sample_id}',
related_url_kwargs={
'project_id': '<project_id>',
'spider_id': '<spider_id>',
'sample_id': '<sample_id>'
},
include_resource_linkage=True,
type_='samples')
schema = fields.Relationship(
related_url='/api/projects/{project_id}/schemas/{schema_id}',
related_url_kwargs={
'project_id': '<project_id>',
'schema_id': '<schema_id>'
},
type_='schemas',
include_resource_linkage=True)
annotations = fields.Relationship(
related_url='/api/projects/{project_id}/spider/{spider_id}/samples/'
'{sample_id}/items/{item_id}/annotations',
related_url_kwargs={
'project_id': '<project_id>',
'spider_id': '<spider_id>',
'sample_id': '<sample_id>',
'item_id': '<id>'
},
many=True,
include_resource_linkage=True,
type_='annotations')
item_annotation = fields.Relationship(
related_url='/api/projects/{project_id}/spider/{spider_id}/samples/'
'{sample_id}/items/{item_id}/item_annotation',
related_url_kwargs={
'project_id': '<project_id>',
'spider_id': '<spider_id>',
'sample_id': '<sample_id>',
'item_id': '<id>'
},
include_resource_linkage=True,
type_='item_annotations')
parent = fields.Relationship(type_='items', include_resource_linkage=True)
@pre_dump
def _dump_parent_id(self, item):
parent_id = item.get('container_id') or ''
if parent_id:
item['parent'] = {'id': parent_id}
if parent_id and item.get('parent_id') is None:
item['parent_id'] = parent_id
class Meta:
type_ = 'items'
class SampleSchema(SlydSchema):
id = fields.Str(dump_only=True)
name = fields.Str()
url = fields.Str(required=True)
page_id = fields.Str()
page_type = fields.Str(default='item')
scrapes = fields.Str()
extractors = fields.Dict(default={})
project = fields.Relationship(
related_url='/api/projects/{project_id}',
related_url_kwargs={'project_id': '<project_id>'},
type_='projects', include_resource_linkage=True
)
spider = fields.Relationship(
related_url='/api/projects/{project_id}/spiders/{spider_id}',
related_url_kwargs={'project_id': '<project_id>',
'spider_id': '<spider_id>'},
type_='spiders', include_resource_linkage=True
)
original_body = fields.Relationship(
related_url='/api/projects/{project_id}/spider/{spider_id}/samples/'
'{sample_id}/original_body',
related_url_kwargs={'project_id': '<project_id>',
'spider_id': '<spider_id>',
'sample_id': '<id>'},
type_='html', include_resource_linkage=False
)
rendered_body = fields.Relationship(
related_url='/api/projects/{project_id}/spider/{spider_id}/samples/'
'{sample_id}/rendered_body',
related_url_kwargs={'project_id': '<project_id>',
'spider_id': '<spider_id>',
'sample_id': '<id>'},
type_='html', include_resource_linkage=False
)
items = fields.Relationship(
related_url='/api/projects/{project_id}/spider/{spider_id}/samples/'
'{sample_id}/items',
related_url_kwargs={'project_id': '<project_id>',
'spider_id': '<spider_id>',
'sample_id': '<id>'},
type_='items', many=True, include_resource_linkage=True
)
def dump(self, obj, many=None, update_fields=True, **kwargs):
many = self.many if many is None else bool(many)
if many:
for o in obj:
o.setdefault('items', [])
else:
obj.setdefault('items', [])
return super(SampleSchema, self).dump(obj, many, update_fields,
**kwargs)
class Meta:
type_ = 'samples'
class ArticleSchema(Schema):
id = fields.Integer()
body = fields.String()
author = fields.Relationship(
dump_only=False, include_resource_linkage=True, many=False, type_='people')
comments = fields.Relationship(
dump_only=False, include_resource_linkage=True, many=True, type_='comments')
class Meta:
type_ = 'articles'
class ExampleSchema(marshmallow_jsonapi.Schema):
id = fields.UUID(required=True)
first_body = fields.Str()
second_body = fields.Str()
is_active = fields.Boolean(attribute='active')
related = fields.Relationship(attribute='related_id')
other_related = fields.Relationship(id_field='id')
class Meta:
type_ = 'example'
class GroupSchema(Schema):
id = fields.String(validate=validate_uuid, attribute='uuid')
name = fields.String(required=True, validate=validate_name)
user_defined = fields.Bool(dump_only=True)
owner = fields.Relationship(include_resource_linkage=True, type_='users', attribute='owner_id')
admins = fields.Relationship(include_resource_linkage=True, type_='users', many=True)
members = fields.Relationship(include_resource_linkage=True, type_='users', many=True)
class Meta:
type_ = 'groups'
class WorksheetItemSchema(Schema):
id = fields.Integer(as_string=True, dump_only=True)
worksheet = fields.Relationship(include_data=True, attribute='worksheet_uuid', type_='worksheets', required=True)
subworksheet = fields.Relationship(include_data=True, type_='worksheets', attribute='subworksheet_uuid', allow_none=True)
bundle = fields.Relationship(include_data=True, type_='bundles', attribute='bundle_uuid', allow_none=True)
value = fields.String()
type = fields.String(validate=validate.OneOf(set(WORKSHEET_ITEM_TYPES)), required=True)
sort_key = fields.Integer(allow_none=True)
class Meta:
type_ = "worksheet-items"
class Student(Schema):
id = fields.Integer()
school = fields.Relationship(schema='School',
include_resource_linkage=True,
type_='schools')
person = fields.Relationship(schema='Person',
include_resource_linkage=True,
type_='people')
class Meta:
inflect = dasherize
type_ = 'students'
class UserSchema(BaseSchema):
id = fields.String()
username = fields.String()
created_on = fields.DateTime()
user_id = fields.String()
active = fields.Boolean(load_from='is_active', dump_to='is_active')
class Meta:
type_ = 'users'
strict = False
self_url = '/api/users/{id}'
self_url_kwargs = {'id': '<id>'}
self_url_many = '/api/users'
comments = fields.Relationship(
type_='comments',
self_url='/api/users/{id}/relationships/comments',
self_url_kwargs={'id': '<id>'},
related_url='/api/users/{id}/comments',
related_url_kwargs={'id': '<id>'},
many=True,
include_resource_linkage=True,
schema='CommentSchema')
posts = fields.Relationship(type_='posts',
self_url='/api/users/{id}/relationships/posts',
self_url_kwargs={'id': '<id>'},
related_url='/api/users/{id}/posts',
related_url_kwargs={'id': '<id>'},
many=True,
include_resource_linkage=True,
schema='PostSchema')
likes = fields.Relationship(dump_only=True,
type_='likes',
self_url='/api/users/{id}/relationships/likes',
self_url_kwargs={'id': '<id>'},
related_url='/api/users/{id}/likes',
related_url_kwargs={'id': '<id>'},
many=True,
include_resource_linkage=True,
schema='PostSchema')
likes = fields.Relationship(load_only=True,
type_='posts',
self_url='/api/users/{id}/relationships/likes',
self_url_kwargs={'id': '<id>'},
related_url='/api/users/{id}/likes',
related_url_kwargs={'id': '<id>'},
many=True,
include_resource_linkage=True,
schema='PostSchema')
class BaseAnnotationSchema(SlydSchema):
id = fields.Str()
attribute = fields.Str(required=True)
accept_selectors = fields.List(fields.Str(), default=[])
reject_selectors = fields.List(fields.Str(), default=[])
tagid = fields.Integer(required=True)
text_content = fields.Str()
selector = fields.Str()
sample = fields.Relationship(
related_url='/api/projects/{project_id}/spiders/{spider_id}/samples/'
'{sample_id}',
related_url_kwargs={
'project_id': '<project_id>',
'spider_id': '<spider_id>',
'sample_id': '<sample_id>'
},
type_='samples',
include_resource_linkage=True)
parent = fields.Relationship(related_url_kwargs={
'project_id': '<project_id>',
'spider_id': '<spider_id>',
'sample_id': '<sample_id>',
'item_id': '<parent_id>'
},
type_='items',
include_resource_linkage=True)
@property
def parent_id(self):
return self.context.get('container_id', self.item_id)
@pre_dump
def _dump_parent_id(self, item):
parent_id = None
if 'parent' in item:
parent_id = item['parent']['id']
if not parent_id:
parent_id = item.get('container_id', self.parent_id) or ''
if (item['id'].split('#')[0] == parent_id
or parent_id.split('#')[0] == item['id']):
item.pop('parent', None)
item.pop('parent_id', None)
return
if parent_id:
item['parent'] = {'id': parent_id}
if parent_id and item.get('parent_id') is None:
item['parent_id'] = parent_id
class SequenceSchema(Schema):
id = fields.String(dump_only=True)
start_time = fields.String(required=True)
start_range = fields.String(required=True)
end_time = fields.String(required=True)
end_range = fields.String(required=True)
pin = fields.Relationship(
related_url='/api/pins/{pin_id}',
related_url_kwargs={'pin_id': '<pin>'},
# Include resource linkage
many=False, include_data=True,
type_='pins'
)
@post_load
def make_sequence(self, data):
return Sequence(**data)
def handle_error(self, exc, data):
raise ValidationError('An error occurred with input: {0} \n {1}'.format(data, exc.messages))
def __str__(self):
if self.pin is None:
return "<Sequence: Start " + self.start_time + " End " +\
self.end_time + " Pin none>"
else:
return "<Sequence: Start " + self.start_time + " End " +\
self.end_time + " Pin " + str(self.pin) + ">"
class Meta:
type_ = 'sequences'
strict = True
class RefSchema(Schema):
id = fields.Str()
data = fields.Str()
children = fields.Relationship(schema='self', many=True)
class Meta:
type_ = 'refs'
class RefSchema(Schema):
id = fields.Int()
data = fields.Str()
parent = fields.Relationship(schema='self', many=False)
class Meta:
type_ = 'refs'
class PictureSchema(BaseSchema):
"""
Schema for describing :any:`Picture` resources.
Attributes:
id (uuid): Picture's ID
extension (str): picture file's extension
filename (str): full name for the picture
item (``Relationship``): any:`Item` related to the picture
resource.
"""
class Meta:
type_ = 'picture'
self_url = '/pictures/{id}'
self_url_kwargs = {'id': '<id>'}
json_module = simplejson
id = fields.Str(dump_only=True, attribute='uuid')
# TODO: Make extensions validation rule oneOf
extension = fields.Str(required=True, validate=NOT_BLANK)
filename = fields.Str(dump_only=True)
item = fields.Relationship(
include_resource_linkage=True,
type_='item',
schema='ItemSchema',
id_field='uuid',
required=True,
)
class PinSchema(Schema):
id = fields.Str(dump_only=True)
number = fields.Integer(required=True)
name = fields.String(attribute='name')
state = fields.Integer()
sequences = fields.Relationship(
related_url='/api/pins/{pin_id}/sequences',
related_url_kwargs={'pin_id': '<id>'},
# Include resource linkage
many=True,
include_data=True,
type_='sequences',
schema='SequenceSchema'
)
@post_load
def make_pin(self, data):
return Pin(**data)
def handle_error(self, exc, data):
raise ValidationError('An error occurred with input: {0} \n {1}'.format(data, exc.messages))
class Meta:
type_ = 'pins'
strict = True
class ArticleSchema(Schema):
id = fields.Str(dump_only=True)
title = fields.Str()
author = fields.Relationship(
related_url='/articles/{article_id}/author',
related_url_kwargs={'article_id': '<id>'},
include_resource_linkage=True,
type_='people',
# define a schema for rendering included data
schema='PersonSchema')
# comments = fields.Relationship(
# related_url='/posts/{post_id}/comments',
# related_url_kwargs={'post_id': '<id>'},
# many=True, include_resource_linkage=True,
# type_='comments',
# # define a schema for rendering included data
# schema='CommentSchema'
# )
class Meta:
type_ = 'articles'
self_url = '/articles/{id}'
self_url_kwargs = {'id': '<id>'}
self_url_many = '/articles/'
strict = True
class FlowTermination(Schema):
id = fields.Function(lambda x: '(%s, %s, %s)' % (
x.flow.external_ref, x.termination.direction, x.termination))
flow = fields.Relationship(related_url='/{flow_link}',
related_url_kwargs={'flow_link': '<flow.link>'})
direction = fields.Str(attribute='termination.direction')
termination = fields.Str()
parent = fields.Relationship(
related_url='/{parent_link}',
related_url_kwargs={'parent_link': '<parent.link>'})
class Meta:
type_ = 'termination'
strict = True
