class CollectionSchema(BaseSchema):
EXPAND = [
('creator', Role, 'creator', RoleReferenceSchema, False),
]
label = String(validate=Length(min=2, max=500), required=True)
foreign_id = String(missing=None)
kind = String(dump_only=True)
casefile = Boolean(missing=None)
summary = String(allow_none=True)
publisher = String(allow_none=True)
publisher_url = Url(allow_none=True)
data_url = Url(allow_none=True)
info_url = Url(allow_none=True)
countries = List(Country())
languages = List(Language())
secret = Boolean(dump_only=True)
category = Category(missing=Collection.DEFAULT)
creator_id = String(allow_none=True)
creator = Nested(RoleReferenceSchema(), dump_only=True)
team = List(Nested(RoleReferenceSchema()), dump_only=True)
count = Integer(dump_only=True)
schemata = Dict(dump_only=True, default={})
@pre_load()
def flatten_collection(self, data):
flatten_id(data, 'creator_id', 'creator')
@pre_dump()
def visibility(self, data):
if not is_mapping(data):
return
roles = data.get('roles', [])
public = Role.public_roles()
data['secret'] = len(public.intersection(roles)) == 0
@post_dump
def transient(self, data):
pk = str(data.get('id'))
data['links'] = {
'self': url_for('collections_api.view', id=pk),
'ui': collection_url(pk)
}
data['writeable'] = request.authz.can_write(pk)
return data
class UASZoneSchema(BaseSchema):
identifier = String(required=True)
country = String(required=True)
name = String()
type = String(required=True)
restriction = String(required=True)
restriction_conditions = List(String(), data_key='restrictionConditions')
region = Integer()
reason = List(String(), validate=validate.Length(max=9))
other_reason_info = String(data_key='otherReasonInfo')
regulation_exemption = String(data_key='regulationExemption')
u_space_class = String(data_key='uSpaceClass')
message = String()
zone_authority = Nested(AuthoritySchema,
data_key='zoneAuthority',
required=True)
applicability = Nested(TimePeriodSchema)
geometry = Nested(AirspaceVolumeSchema, required=True, many=True)
extended_properties = Dict(data_key='extendedProperties')
@post_load
def make_mongo_object(self, data, **kwargs):
"""
A document schema will be eventually loaded in a mongoengine object for possible storing in
DB
:param data:
:param kwargs:
:return:
"""
return UASZone(**data)
@post_dump
def handle_mongoengine_dict_field(self, data, **kwargs):
"""
Apparently the dict field is not serialized properly to a dict object so it has to be done
manually.
:param data:
:param kwargs:
:return:
"""
data['extendedProperties'] = dict(data['extendedProperties'])
return data
class DocumentSchema(Schema, DatedSchema):
id = String(dump_only=True)
collection_id = Integer(dump_only=True, required=True)
schema = SchemaName(dump_only=True)
schemata = List(SchemaName(), dump_only=True)
status = String(dump_only=True)
type = String(dump_only=True)
foreign_id = String(dump_only=True)
content_hash = String(dump_only=True)
parent = Dict(dump_only=True) # TODO: make writeable?
uploader_id = Integer(dump_only=True)
error_message = String(dump_only=True)
# title = String(validate=Length(min=2, max=5000), missing=None)
title = String(missing=None)
summary = String(missing=None)
countries = List(Country(), missing=[])
languages = List(Language(), missing=[])
keywords = List(String(validate=Length(min=1, max=5000)), missing=[])
dates = List(PartialDate(), dump_only=True)
file_name = String(dump_only=True)
file_size = Integer(dump_only=True)
author = String(dump_only=True)
mime_type = String(dump_only=True)
extension = String(dump_only=True)
encoding = String(dump_only=True)
source_url = String(dump_only=True)
pdf_version = String(dump_only=True)
columns = List(String(), dump_only=True)
children = Boolean(dump_to='$children',
attribute='$children',
dump_only=True)
@post_dump
def transient(self, data):
data['$uri'] = url_for('documents_api.view',
document_id=data.get('id'))
data['$ui'] = document_url(data.get('id'))
collection_id = data.get('collection_id')
data['$writeable'] = request.authz.can_write(collection_id)
return data
class TransactionLogScheme(BaseModelScheme):
transaction_log_uuid = Str(validate=[validate.Length(max=36)])
transaction_time = DateTime()
license_lrn_uuid = Str(validate=[validate.Length(max=36)])
license_switch_uuid = Str(validate=[validate.Length(max=36)])
type = Int()
amount_total = Float()
amount_lrn = Float()
amount_switch = Float()
transaction_id = Str(validate=[validate.Length(max=255)])
transaction_type = Str(validate=[validate.Length(max=255)])
from_ip = Str(validate=[validate.Length(max=36)])
transaction_src = Dict()
status = Int()
result = Str()
payment_uuid = Str(validate=[validate.Length(max=36)])
license_lrn_plan_name = Str()
license_switch_plan_name = Str()
user_uuid = Str()
user_name = Str()
class Meta:
model = model.TransactionLog
fields = (
'transaction_time',
'license_lrn_uuid',
'license_switch_uuid',
'type',
'amount_total',
'amount_lrn',
'amount_switch',
'transaction_id',
'transaction_type',
'from_ip',
'transaction_src',
'status',
'result',
'payment_uuid',
)
class CollectionSchema(BaseSchema):
EXPAND = [
('creator', Role, 'creator', RoleReferenceSchema, False),
]
label = String(validate=Length(min=2, max=500), required=True)
foreign_id = String(missing=None)
kind = String(dump_only=True)
casefile = Boolean(missing=None)
summary = String(allow_none=True)
publisher = String(allow_none=True)
publisher_url = Url(allow_none=True)
data_url = Url(allow_none=True)
info_url = Url(allow_none=True)
countries = List(Country())
languages = List(Language())
secret = Boolean(dump_only=True)
category = Category(missing=Collection.DEFAULT)
creator_id = String(allow_none=True)
creator = Nested(RoleReferenceSchema(), dump_only=True)
team = List(Nested(RoleReferenceSchema()), dump_only=True)
count = Integer(dump_only=True)
schemata = Dict(dump_only=True)
@pre_load
def flatten_collection(self, data):
flatten_id(data, 'creator_id', 'creator')
@post_dump
def hypermedia(self, data):
pk = str(data.get('id'))
data['links'] = {
'self': url_for('collections_api.view', id=pk),
'xref': url_for('xref_api.index', id=pk),
'xref_csv': url_for('xref_api.csv_export', id=pk, _authorize=True),
'ui': collection_url(pk)
}
data['writeable'] = request.authz.can(pk, request.authz.WRITE)
return data
class FileSchema(Schema):
"""Service schema for files."""
key = SanitizedUnicode(dump_only=True)
created = TZDateTime(timezone=timezone.utc, format='iso', dump_only=True)
updated = TZDateTime(timezone=timezone.utc, format='iso', dump_only=True)
status = GenMethod('dump_status')
metadata = Dict(dump_only=True)
checksum = Str(dump_only=True, attribute='file.checksum')
storage_class = Str(dump_only=True, attribute='file.storage_class')
mimetype = Str(dump_only=True, attribute='file.mimetype')
size = Number(attribute='file.size')
version_id = UUID(attribute='file.version_id')
file_id = UUID(attribute='file.file_id')
bucket_id = UUID(attribute='file.bucket_id')
links = Links()
def dump_status(self, obj):
"""Dump file status."""
return 'completed' if obj.file else 'pending'
class SchemaWithDict(Schema):
dict_field = Dict(values=Nested(PetSchema))
class OverlaysEndpoint(BaseEndpoint):
"""
This endpoint is responsible for handing all requests regarding the status of overlays.
"""
def __init__(self):
super(OverlaysEndpoint, self).__init__()
self.statistics_supported = None
def setup_routes(self):
self.app.add_routes([
web.get('', self.get_overlays),
web.get('/statistics', self.get_statistics),
web.post('/statistics', self.enable_statistics)
])
def initialize(self, session):
super(OverlaysEndpoint, self).initialize(session)
self.statistics_supported = isinstance(session.endpoint, StatisticsEndpoint) \
or isinstance(getattr(session.endpoint, 'endpoint', None), StatisticsEndpoint)
@docs(tags=["Overlays"],
summary="Return information about all currently loaded overlays.",
responses={
200: {
"schema":
schema(OverlayResponse={"overlays": [OverlaySchema]})
}
})
async def get_overlays(self, _):
overlay_stats = []
for overlay in self.session.overlays:
peers = overlay.get_peers()
statistics = self.session.endpoint.get_aggregate_statistics(overlay.get_prefix()) \
if isinstance(self.session.endpoint, StatisticsEndpoint) else {}
overlay_stats.append({
"master_peer":
hexlify(overlay.master_peer.public_key.key_to_bin()).decode(
'utf-8'),
"my_peer":
hexlify(
overlay.my_peer.public_key.key_to_bin()).decode('utf-8'),
"global_time":
overlay.global_time,
"peers": [{
'ip':
peer.address[0],
'port':
peer.address[1],
'public_key':
hexlify(peer.public_key.key_to_bin()).decode('utf-8')
} for peer in peers],
"overlay_name":
overlay.__class__.__name__,
"statistics":
statistics
})
return Response({"overlays": overlay_stats})
@docs(tags=["Overlays"],
summary="Return statistics for all currently loaded overlays.",
responses={
200: {
"schema":
schema(
StatisticsResponse={
"statistics":
List(
Dict(keys=String,
values=Nested(OverlayStatisticsSchema))),
}),
"examples": {
'Success': {
"statistics": [{
"DiscoveryCommunity": {
'num_up': 0,
'num_down': 0,
'bytes_up': 0,
'bytes_down': 0,
'diff_time': 0
}
}]
}
}
}
})
async def get_statistics(self, _):
overlay_stats = []
for overlay in self.session.overlays:
statistics = self.session.endpoint.get_statistics(
overlay.get_prefix()) if self.statistics_supported else {}
overlay_stats.append({
overlay.__class__.__name__:
self.statistics_by_name(statistics, overlay)
})
return Response({"statistics": overlay_stats})
def statistics_by_name(self, statistics, overlay):
named_statistics = {}
for message_id, network_stats in statistics.items():
if overlay.decode_map.get(chr(message_id)):
mapped_name = str(message_id) + ":" + overlay.decode_map[chr(
message_id)].__name__
else:
mapped_name = str(message_id) + ":unknown"
mapped_value = network_stats.to_dict()
named_statistics[mapped_name] = mapped_value
return named_statistics
@docs(tags=["Overlays"],
summary="Enable/disable statistics for a given overlay.",
responses={
200: {
"schema": DefaultResponseSchema,
"examples": {
'Success': {
"success": True
}
}
},
HTTP_PRECONDITION_FAILED: {
"schema": DefaultResponseSchema,
"examples": {
'Statistics disabled': {
"success": False,
"error": "StatisticsEndpoint is not enabled."
}
}
},
HTTP_BAD_REQUEST: {
"schema": DefaultResponseSchema,
"examples": {
'Missing parameter': {
"success": False,
"error": "Parameter 'enable' is required."
}
}
}
})
@json_schema(
schema(
EnableStatisticsRequest={
'enable*': (Boolean,
'Whether to enable or disable the statistics'),
'all': (Boolean, 'Whether update applies to all overlays'),
'overlay_name': (String, 'Class name of the overlay'),
}))
async def enable_statistics(self, request):
if not self.statistics_supported:
return Response(
{
"success": False,
"error": "StatisticsEndpoint is not enabled."
},
status=HTTP_PRECONDITION_FAILED)
all_overlays = False
overlay_name = None
args = await request.json()
if 'enable' not in args or not args['enable']:
return Response(
{
"success": False,
"error": "Parameter 'enable' is required"
},
status=HTTP_BAD_REQUEST)
enable = args['enable'].lower() == 'true'
if 'all' in args and args['all']:
all_overlays = args['all'].lower() == 'true'
elif 'overlay_name' in args and args['overlay_name']:
overlay_name = args['overlay_name']
else:
return Response(
{
"success": False,
"error": "Parameter 'all' or 'overlay_name' is required"
},
status=HTTP_PRECONDITION_FAILED)
self.enable_overlay_statistics(enable=enable,
class_name=overlay_name,
all_overlays=all_overlays)
return Response({"success": True})
def enable_overlay_statistics(self,
enable=False,
class_name=None,
all_overlays=False):
if all_overlays:
for overlay in self.session.overlays:
self.session.endpoint.enable_community_statistics(
overlay.get_prefix(), enable)
elif class_name:
for overlay in self.session.overlays:
if overlay.__class__.__name__ == class_name:
self.session.endpoint.enable_community_statistics(
overlay.get_prefix(), enable)
class ShallowCombinedSchema(BaseSchema):
collection_id = String()
# Joint entity/document attributes
collection = Nested(CollectionSchema())
schema = SchemaName()
schemata = List(SchemaName())
names = List(String())
addresses = List(String())
phones = List(String())
emails = List(String())
identifiers = List(String())
countries = List(Country())
dates = List(PartialDate())
bulk = Boolean()
# Entity attributes
foreign_id = String()
name = String()
entities = List(String())
properties = Dict()
# Document attributes
status = String()
content_hash = String()
uploader_id = String()
uploader = Nested(RoleReferenceSchema())
error_message = String()
title = String()
summary = String()
languages = List(Language())
keywords = List(String())
date = PartialDate()
authored_at = PartialDate()
modified_at = PartialDate()
published_at = PartialDate()
retrieved_at = PartialDate()
file_name = String()
file_size = Integer()
author = String()
generator = String()
mime_type = String()
extension = String()
encoding = String()
source_url = String()
pdf_version = String()
columns = List(String())
headers = Dict()
children = Integer()
# TODO: is this a separate endpoint?
text = String()
html = String()
def document_links(self, data, pk, schemata):
links = {
'self': url_for('documents_api.view', document_id=pk),
'tags': url_for('entities_api.tags', id=pk),
'ui': document_url(pk)
}
if data.get('content_hash'):
links['file'] = url_for('documents_api.file',
document_id=pk,
_authorize=True)
if schemata.intersection([Document.SCHEMA_PDF]):
links['pdf'] = url_for('documents_api.pdf',
document_id=pk,
_authorize=True)
if schemata.intersection([Document.SCHEMA_PDF, Document.SCHEMA_TABLE]):
links['records'] = url_for('documents_api.records', document_id=pk)
if schemata.intersection([Document.SCHEMA_FOLDER]):
query = (('filter:parent.id', pk),)
links['children'] = url_for('documents_api.index', _query=query)
return links
def entity_links(self, data, pk, schemata):
return {
'self': url_for('entities_api.view', id=pk),
# 'similar': url_for('entities_api.similar', id=pk),
# 'documents': url_for('entities_api.documents', id=pk),
'references': url_for('entities_api.references', id=pk),
'tags': url_for('entities_api.tags', id=pk),
'ui': entity_url(pk)
}
@post_dump()
def hypermedia(self, data):
pk = str(data.get('id'))
collection = data.get('collection', {})
collection_id = collection.get('id')
collection_id = collection_id or data.get('collection_id')
schemata = set(data.get('schemata', []))
if Document.SCHEMA in schemata:
data['links'] = self.document_links(data, pk, schemata)
else:
data['links'] = self.entity_links(data, pk, schemata)
if data.get('bulk'):
data['writeable'] = False
else:
data['writeable'] = request.authz.can_write(collection_id)
return data
class ChannelsEndpoint(ChannelsEndpointBase):
def setup_routes(self):
self.app.add_routes(
[
web.get('', self.get_channels),
web.get(r'/{channel_pk:\w*}/{channel_id:\w*}', self.get_channel_contents),
web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/copy', self.copy_channel),
web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/channels', self.create_channel),
web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/collections', self.create_collection),
web.put(r'/{channel_pk:\w*}/{channel_id:\w*}/torrents', self.add_torrent_to_channel),
web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/commit', self.post_commit),
web.get(r'/{channel_pk:\w*}/{channel_id:\w*}/commit', self.is_channel_dirty),
]
)
def get_channel_from_request(self, request):
channel_pk = (
self.session.mds.my_key.pub().key_to_bin()[10:]
if request.match_info['channel_pk'] == 'mychannel'
else unhexlify(request.match_info['channel_pk'])
)
channel_id = int(request.match_info['channel_id'])
return channel_pk, channel_id
@docs(
tags=['Metadata'],
summary='Get a list of all channels known to the system.',
responses={
200: {
'schema': schema(
GetChannelsResponse={
'results': [ChannelSchema],
'first': Integer(),
'last': Integer(),
'sort_by': String(),
'sort_desc': Integer(),
'total': Integer(),
}
)
}
},
)
# TODO: DRY it with SpecificChannel endpoint?
async def get_channels(self, request):
sanitized = self.sanitize_parameters(request.query)
sanitized['subscribed'] = None if 'subscribed' not in request.query else bool(int(request.query['subscribed']))
include_total = request.query.get('include_total', '')
sanitized.update({"origin_id": 0})
with db_session:
channels = self.session.mds.ChannelMetadata.get_entries(**sanitized)
total = self.session.mds.ChannelMetadata.get_total_count(**sanitized) if include_total else None
channels_list = [channel.to_simple_dict() for channel in channels]
response_dict = {
"results": channels_list,
"first": sanitized["first"],
"last": sanitized["last"],
"sort_by": sanitized["sort_by"],
"sort_desc": int(sanitized["sort_desc"]),
}
if total is not None:
response_dict.update({"total": total})
return RESTResponse(response_dict)
@docs(
tags=['Metadata'],
summary='Get a list of the channel\'s contents (torrents/channels/etc.).',
responses={
200: {
'schema': schema(
GetChannelContentsResponse={
'results': [Dict()],
'first': Integer(),
'last': Integer(),
'sort_by': String(),
'sort_desc': Integer(),
'total': Integer(),
}
)
}
},
)
async def get_channel_contents(self, request):
sanitized = self.sanitize_parameters(request.query)
include_total = request.query.get('include_total', '')
channel_pk, channel_id = self.get_channel_from_request(request)
sanitized.update({"channel_pk": channel_pk, "origin_id": channel_id})
with db_session:
contents = self.session.mds.MetadataNode.get_entries(**sanitized)
contents_list = [c.to_simple_dict() for c in contents]
total = self.session.mds.MetadataNode.get_total_count(**sanitized) if include_total else None
response_dict = {
"results": contents_list,
"first": sanitized['first'],
"last": sanitized['last'],
"sort_by": sanitized['sort_by'],
"sort_desc": int(sanitized['sort_desc']),
}
if total is not None:
response_dict.update({"total": total})
return RESTResponse(response_dict)
@docs(
tags=['Metadata'],
summary='Create a copy of an entry/entries from another channel.',
parameters=[
{
'in': 'body',
'name': 'entries',
'description': 'List of entries to copy',
'example': [{'public_key': '1234567890', 'id': 123}],
'required': True,
}
],
responses={
200: {'description': 'Returns a list of copied content'},
HTTP_NOT_FOUND: {'schema': HandledErrorSchema, 'example': {"error": "Target channel not found"}},
HTTP_BAD_REQUEST: {'schema': HandledErrorSchema, 'example': {"error": "Source entry not found"}},
},
)
async def copy_channel(self, request):
with db_session:
channel_pk, channel_id = self.get_channel_from_request(request)
personal_root = channel_id == 0 and channel_pk == self.session.mds.my_key.pub().key_to_bin()[10:]
# TODO: better error handling
target_collection = self.session.mds.CollectionNode.get(
public_key=database_blob(channel_pk), id_=channel_id
)
try:
request_parsed = await request.json()
except (ContentTypeError, ValueError):
return RESTResponse({"error": "Bad JSON"}, status=HTTP_BAD_REQUEST)
if not target_collection and not personal_root:
return RESTResponse({"error": "Target channel not found"}, status=HTTP_NOT_FOUND)
results_list = []
for entry in request_parsed:
public_key, id_ = database_blob(unhexlify(entry["public_key"])), entry["id"]
source = self.session.mds.ChannelNode.get(public_key=public_key, id_=id_)
if not source:
return RESTResponse({"error": "Source entry not found"}, status=HTTP_BAD_REQUEST)
# We must upgrade Collections to Channels when moving them to root channel, and, vice-versa,
# downgrade Channels to Collections when moving them into existing channels
if isinstance(source, self.session.mds.CollectionNode):
src_dict = source.to_dict()
if channel_id == 0:
rslt = self.session.mds.ChannelMetadata.create_channel(title=source.title)
else:
dst_dict = {'origin_id': channel_id, "status": NEW}
for k in self.session.mds.CollectionNode.nonpersonal_attributes:
dst_dict[k] = src_dict[k]
dst_dict.pop("metadata_type")
rslt = self.session.mds.CollectionNode(**dst_dict)
for child in source.actual_contents:
child.make_copy(rslt.id_)
else:
rslt = source.make_copy(channel_id)
results_list.append(rslt.to_simple_dict())
return RESTResponse(results_list)
@docs(
tags=['Metadata'],
summary='Create a new channel entry in the given channel.',
responses={
200: {
'description': 'Returns the newly created channel',
'schema': schema(CreateChannelResponse={'results': [Dict()]}),
}
},
)
async def create_channel(self, request):
with db_session:
_, channel_id = self.get_channel_from_request(request)
request_parsed = await request.json()
channel_name = request_parsed.get("name", "New channel")
md = self.session.mds.ChannelMetadata.create_channel(channel_name, origin_id=channel_id)
return RESTResponse({"results": [md.to_simple_dict()]})
@docs(
tags=['Metadata'],
summary='Create a new collection entry in the given channel.',
responses={
200: {
'description': 'Returns the newly created collection',
'schema': schema(CreateCollectionResponse={'results': [Dict()]}),
}
},
)
async def create_collection(self, request):
with db_session:
_, channel_id = self.get_channel_from_request(request)
request_parsed = await request.json()
collection_name = request_parsed.get("name", "New collection")
md = self.session.mds.CollectionNode(origin_id=channel_id, title=collection_name, status=NEW)
return RESTResponse({"results": [md.to_simple_dict()]})
@docs(
tags=['Metadata'],
summary='Add a torrent file to your own channel.',
responses={
200: {
'schema': schema(
AddTorrentToChannelResponse={'added': (Integer, 'Number of torrent that were added to the channel')}
)
},
HTTP_NOT_FOUND: {'schema': HandledErrorSchema, 'example': {"error": "Unknown channel"}},
HTTP_BAD_REQUEST: {'schema': HandledErrorSchema, 'example': {"error": "unknown uri type"}},
},
)
@json_schema(
schema(
AddTorrentToChannelRequest={
'torrent': (String, 'Base64-encoded torrent file'),
'uri': (String, 'Add a torrent from a magnet link or URL'),
'torrents_dir': (String, 'Add all .torrent files from a chosen directory'),
'recursive': (Boolean, 'Toggle recursive scanning of the chosen directory for .torrent files'),
'description': (String, 'Description for the torrent'),
'filesize': (Integer, "Filesize of the torrent file, this parameter is used for "
"skipping metadata check when uri is a magnet link"),
}
)
)
async def add_torrent_to_channel(self, request):
channel_pk, channel_id = self.get_channel_from_request(request)
with db_session:
channel = self.session.mds.CollectionNode.get(public_key=database_blob(channel_pk), id_=channel_id)
if not channel:
return RESTResponse({"error": "Unknown channel"}, status=HTTP_NOT_FOUND)
parameters = await request.json()
extra_info = {}
if parameters.get('description', None):
extra_info = {'description': parameters['description']}
# First, check whether we did upload a magnet link or URL
if parameters.get('uri', None):
uri = parameters['uri']
if uri.startswith("http:") or uri.startswith("https:"):
async with ClientSession() as session:
response = await session.get(uri)
data = await response.read()
tdef = TorrentDef.load_from_memory(data)
elif uri.startswith("magnet:"):
_, xt, _ = parse_magnetlink(uri)
if (
xt
and is_infohash(codecs.encode(xt, 'hex'))
and (channel.torrent_exists(xt) or channel.copy_torrent_from_infohash(xt))
):
return RESTResponse({"added": 1})
filesize = parameters.get("filesize")
if filesize and not (isinstance(filesize, int) or int is None):
return RESTResponse({"error": "filesize must be an integer"},
status=HTTP_BAD_REQUEST,)
if filesize:
dn, xt, _ = parse_magnetlink(uri)
tdef = TorrentDefNoMetainfo(xt, dn, uri, filesize)
else:
meta_info = await self.session.dlmgr.get_metainfo(xt, timeout=30, url=uri)
if not meta_info:
raise RuntimeError("Metainfo timeout")
tdef = TorrentDef.load_from_dict(meta_info)
else:
return RESTResponse({"error": "unknown uri type"}, status=HTTP_BAD_REQUEST)
added = 0
if tdef:
channel.add_torrent_to_channel(tdef, extra_info)
added = 1
return RESTResponse({"added": added})
torrents_dir = None
if parameters.get('torrents_dir', None):
torrents_dir = parameters['torrents_dir']
if not path_util.isabs(torrents_dir):
return RESTResponse({"error": "the torrents_dir should point to a directory"}, status=HTTP_BAD_REQUEST)
recursive = False
if parameters.get('recursive'):
recursive = parameters['recursive']
if not torrents_dir:
return RESTResponse(
{"error": "the torrents_dir parameter should be provided when the recursive parameter is set"},
status=HTTP_BAD_REQUEST,
)
if torrents_dir:
torrents_list, errors_list = channel.add_torrents_from_dir(torrents_dir, recursive)
return RESTResponse({"added": len(torrents_list), "errors": errors_list})
if not parameters.get('torrent', None):
return RESTResponse({"error": "torrent parameter missing"}, status=HTTP_BAD_REQUEST)
# Try to parse the torrent data
# Any errors will be handled by the error_middleware
torrent = base64.b64decode(parameters['torrent'])
torrent_def = TorrentDef.load_from_memory(torrent)
channel.add_torrent_to_channel(torrent_def, extra_info)
return RESTResponse({"added": 1})
@docs(
tags=['Metadata'],
summary='Commit a channel.',
responses={200: {'schema': schema(CommitResponse={'success': Boolean()})}},
)
async def post_commit(self, request):
channel_pk, channel_id = self.get_channel_from_request(request)
with db_session:
if channel_id == 0:
for t in self.session.mds.CollectionNode.commit_all_channels():
self.session.gigachannel_manager.updated_my_channel(TorrentDef.load_from_dict(t))
else:
coll = self.session.mds.CollectionNode.get(public_key=database_blob(channel_pk), id_=channel_id)
if not coll:
return RESTResponse({"success": False}, status=HTTP_NOT_FOUND)
torrent_dict = coll.commit_channel_torrent()
if torrent_dict:
self.session.gigachannel_manager.updated_my_channel(TorrentDef.load_from_dict(torrent_dict))
return RESTResponse({"success": True})
@docs(
tags=['Metadata'],
summary='Check if a channel has uncommitted changes.',
responses={200: {'schema': schema(IsChannelDirtyResponse={'dirty': Boolean()})}},
)
async def is_channel_dirty(self, request):
channel_pk, _ = self.get_channel_from_request(request)
with db_session:
dirty = self.session.mds.MetadataNode.exists(
lambda g: g.public_key == database_blob(channel_pk) and g.status in DIRTY_STATUSES
)
return RESTResponse({"dirty": dirty})
class SchemaWithDict(Schema):
dict_field = Dict()
class AllRequiredTripletOfSideImport2Schema(Schema):
a = TestsAbsThreeSideImport2Field(dump_to="a", load_from="a", allow_none=False)
b = List(
TestsBasicTestLiteralTopField(), dump_to="b", load_from="b", allow_none=False
)
c = Dict(dump_to="c", load_from="c", allow_none=False)
class CommandSchema(Schema):
name = String()
data = Dict()
class DatabaseRequestSchema(Schema):
currencies = Dict(key=Str(), values=Float())
class CollectionIndexSchema(CollectionSchema):
count = Integer(dump_only=True)
schemata = Dict(dump_only=True, default={})
class Device(Thing):
__doc__ = m.Device.__doc__
id = Integer(description=m.Device.id.comment, dump_only=True)
hid = SanitizedStr(lower=True, description=m.Device.hid.comment)
tags = NestedOn('Tag',
many=True,
collection_class=OrderedSet,
description='A set of tags that identify the device.')
model = SanitizedStr(lower=True,
validate=Length(max=STR_BIG_SIZE),
description=m.Device.model.comment)
manufacturer = SanitizedStr(lower=True,
validate=Length(max=STR_SIZE),
description=m.Device.manufacturer.comment)
serial_number = SanitizedStr(lower=True,
validate=Length(max=STR_BIG_SIZE),
data_key='serialNumber')
brand = SanitizedStr(validate=Length(max=STR_BIG_SIZE),
description=m.Device.brand.comment)
generation = Integer(validate=Range(1, 100),
description=m.Device.generation.comment)
version = SanitizedStr(description=m.Device.version)
weight = Float(validate=Range(0.1, 5),
unit=UnitCodes.kgm,
description=m.Device.weight.comment)
width = Float(validate=Range(0.1, 5),
unit=UnitCodes.m,
description=m.Device.width.comment)
height = Float(validate=Range(0.1, 5),
unit=UnitCodes.m,
description=m.Device.height.comment)
depth = Float(validate=Range(0.1, 5),
unit=UnitCodes.m,
description=m.Device.depth.comment)
# TODO TimeOut 2. Comment actions and lots if there are time out.
actions = NestedOn('Action',
many=True,
dump_only=True,
description=m.Device.actions.__doc__)
# TODO TimeOut 2. Comment actions_one and lots if there are time out.
actions_one = NestedOn('Action',
many=True,
load_only=True,
collection_class=OrderedSet)
problems = NestedOn('Action',
many=True,
dump_only=True,
description=m.Device.problems.__doc__)
url = URL(dump_only=True, description=m.Device.url.__doc__)
# TODO TimeOut 2. Comment actions and lots if there are time out.
lots = NestedOn(
'Lot',
many=True,
dump_only=True,
description='The lots where this device is directly under.')
rate = NestedOn('Rate', dump_only=True, description=m.Device.rate.__doc__)
price = NestedOn('Price',
dump_only=True,
description=m.Device.price.__doc__)
tradings = Dict(dump_only=True, description='')
physical = EnumField(states.Physical,
dump_only=True,
description=m.Device.physical.__doc__)
traking = EnumField(states.Traking,
dump_only=True,
description=m.Device.physical.__doc__)
usage = EnumField(states.Usage,
dump_only=True,
description=m.Device.physical.__doc__)
revoke = UUID(dump_only=True)
physical_possessor = NestedOn('Agent',
dump_only=True,
data_key='physicalPossessor')
production_date = DateTime('iso',
description=m.Device.updated.comment,
data_key='productionDate')
working = NestedOn('Action',
many=True,
dump_only=True,
description=m.Device.working.__doc__)
variant = SanitizedStr(description=m.Device.variant.comment)
sku = SanitizedStr(description=m.Device.sku.comment)
image = URL(description=m.Device.image.comment)
allocated = Boolean(description=m.Device.allocated.comment)
devicehub_id = SanitizedStr(data_key='devicehubID',
description=m.Device.devicehub_id.comment)
@pre_load
def from_actions_to_actions_one(self, data: dict):
"""
Not an elegant way of allowing submitting actions to a device
(in the context of Snapshots) without creating an ``actions``
field at the model (which is not possible).
:param data:
:return:
"""
# Note that it is secure to allow uploading actions_one
# as the only time an user can send a device object is
# in snapshots.
data['actions_one'] = data.pop('actions', [])
return data
@post_load
def validate_snapshot_actions(self, data):
"""Validates that only snapshot-related actions can be uploaded."""
from ereuse_devicehub.resources.action.models import EraseBasic, Test, Rate, Install, \
Benchmark
for action in data['actions_one']:
if not isinstance(action,
(Install, EraseBasic, Rate, Test, Benchmark)):
raise ValidationError('You cannot upload {}'.format(action),
field_names=['actions'])
class FeatureExtractionOutput(Schema):
cell_features = Nested(CellFeatures)
sweep_features = Dict(keys=Str(), values=Nested(SweepFeatures))
cell_record = Nested(CellRecord)
sweep_records = List(Nested(SweepRecord))
cell_state = Nested(CellState)
class Meta:
unknown = EXCLUDE
fields = Dict(required=True)
tags = Nested(ImportantTags, required=True)
timestamp = Float(required=True, validate=UnixEpoch())
class DataProvider(AfterglowSchema):
"""
Base class for data provider plugins
Plugin modules are placed in the :mod:`resources.data_provider_plugins`
subpackage and must subclass from :class:`DataProvider`, e.g.
class MyDataProvider(DataProvider):
name = 'my_provider'
search_fields = {...}
def get_asset(self, path):
...
def get_asset_data(self, path):
...
def get_child_assets(self, path):
...
def find_assets(self, path=None, **kwargs):
...
Methods:
get_asset(): return asset at the given path; must be implemented by any
data provider
get_asset_data(): return data for a non-collection asset at the given
path; must be implemented by any data provider
get_child_assets(): return child assets of a collection asset at the
given path; must be implemented by any browsable data provider
find_assets(): return assets matching the given parameters; must be
implemented by any searchable data provider
create_asset(): create a new non-collection asset from data file at the
given path, or an empty collection asset at the given path; must be
implemented by a read-write provider if it supports adding new
assets
update_asset(): update an existing non-collection asset at the given
path with a data file; must be implemented by a read-write provider
if it supports modifying existing assets
delete_asset(): delete an asset at the given path; must be implemented
by a read-write provider if it supports deleting assets
Attributes:
id: unique integer ID of the data provider; assigned automatically on
initialization
name: unique data provider name; can be used by the clients in requests
like GET /data-providers/[id]/assets in place of the integer
data provider ID
auth_methods: list of data provider-specific authentication methods;
if None, defaults to DEFAULT_DATA_PROVIDER_AUTH -> DATA_FILE_AUTH
-> all auth methods available
icon: optional data provider icon name
display_name: data provider plugin visible in the Afterglow UI
description: a longer description of the data provider
columns: list of dictionary
{name: string, field_name: string, sortable: boolean}
sort_by: string - name of column to use for initial sort
sort_asc: boolean - initial sort order should be ascending
browseable: True if the data provider supports browsing (i.e. getting
child assets of a collection asset at the given path);
automatically set depending on whether the provider implements
get_child_assets()
searchable: True if the data provider supports searching (i.e. querying
using the custom search keywords defined by `search_fields`);
automatically set depending on whether the provider implements
find_assets()
search_fields: dictionary
{field_name: {"label": label, "type": type, ...}, ...}
containing names and descriptions of search fields used on the
client side to create search forms
readonly: True if the data provider assets cannot be modified (created,
updated, or deleted); automatically set depending on whether the
provider implements create_asset(), update_asset(), or
delete_asset()
allow_upload: if readonly=False, allow uploading user images
to the data provider
quota: data provider storage quota, in bytes, if applicable
usage: current usage of the data provider storage, in bytes, if
applicable
"""
__polymorphic_on__ = 'name'
id: int = Integer(default=None)
name: str = String(default=None)
auth_methods: Optional[TList[str]] = List(String(), default=None)
display_name: str = String(default=None)
icon: str = String(default=None)
description: str = String(default=None)
columns: TList[TDict[str, Any]] = List(Dict(), default=[])
sort_by: str = String(default=None)
sort_asc: bool = Boolean(default=True)
browseable: bool = Boolean(default=False)
searchable: bool = Boolean(default=False)
search_fields: TDict[str, TDict[str, Any]] = Dict(default={})
readonly: bool = Boolean(default=True)
allow_upload: bool = Boolean(default=False)
quota: int = Integer(default=None)
usage: int = Integer(default=None)
def __init__(self, **kwargs):
"""
Create a DataProvider instance
:param kwargs: data provider initialization parameters
"""
super(DataProvider, self).__init__(_set_defaults=True, **kwargs)
# Automatically set browseable, searchable, and readonly flags
# depending on what methods are reimplemented by provider; method attr
# of a class is an unbound method instance in Python 2 and a function
# in Python 3
if 'browseable' not in kwargs:
self.browseable = is_overridden(DataProvider, self,
'get_child_assets')
if 'searchable' not in kwargs:
self.searchable = is_overridden(DataProvider, self, 'find_assets')
if 'readonly' not in kwargs:
self.readonly = \
not is_overridden(DataProvider, self, 'create_asset') and \
not is_overridden(DataProvider, self, 'update_asset') and \
not is_overridden(DataProvider, self, 'delete_asset')
if self.auth_methods is None:
# Use default data provider authentication
self.auth_methods = app.config.get('DEFAULT_DATA_PROVIDER_AUTH')
if self.auth_methods is None:
# Inherit auth methods from data files
self.auth_methods = app.config.get('DATA_FILE_AUTH')
if isinstance(self.auth_methods, str):
self.auth_methods = self.auth_methods.split(',')
def get_asset(self, path: str) -> DataProviderAsset:
"""
Return an asset at the given path
:param path: asset path
:return: asset object
"""
raise errors.MethodNotImplementedError(
class_name=self.__class__.__name__, method_name='get_assets')
def get_child_assets(self, path: str, sort_by: Optional[str] = None,
page_size: Optional[int] = None,
page: Optional[Union[int, str]] = None) \
-> Tuple[TList[DataProviderAsset], Optional[PaginationInfo]]:
"""
Return child assets of a collection asset at the given path
:param path: asset path; must identify a collection asset
:param sort_by: optional sorting key (e.g. column name); reverse
sorting is indicated by prepending a hyphen to the key; data
provider may assume a certain default sorting mode and must return
it in the pagination info
:param page_size: optional number of assets per page; None means don't
use pagination (used only internally, never via the API);
if not None, data provider may enforce a hard limit on the page
size
:param page: page-based pagination: optional 0-based page number (data
provider returns at most `page_size` assets sorted by the sorting
key at offset = `page`*`page_size`);
keyset-based pagination: ">value" = return at most `page_size`
assets with the value of `sort_by` key greater than the given
value, "<value": return at most `page_size` assets with the value
of the `sort_by` key smaller than the given value;
for any pagination type, two special values "first" and "last"
are used to return first and last page, respectively
:return: list of :class:`DataProviderAsset` objects for child assets
and pagination info or None if pagination is not supported
"""
raise errors.MethodNotImplementedError(
class_name=self.__class__.__name__, method_name='get_child_assets')
def find_assets(self, path: Optional[str] = None,
sort_by: Optional[str] = None,
page_size: Optional[int] = None,
page: Optional[Union[int, str]] = None,
**kwargs) \
-> Tuple[TList[DataProviderAsset], Optional[PaginationInfo]]:
"""
Return a list of assets matching the given parameters
:param path: optional path to the collection asset to search in;
by default (and for providers that do not have collection assets),
search in the data provider root
:param sort_by: optional sorting key; see :meth:`get_child_assets`
:param page_size: optional number of assets per page
:param page: optional 0-based page number, ">value", "<value", "first",
or "last"
:param kwargs: provider-specific keyword=value pairs defining the
asset(s), like name, image type or dimensions
:return: list of :class:`DataProviderAsset` objects for assets matching
the search query parameters and pagination info or None
if pagination is not supported
"""
raise errors.MethodNotImplementedError(
class_name=self.__class__.__name__, method_name='find_assets')
def get_asset_data(self, path: str) -> bytes:
"""
Return data for a non-collection asset at the given path
:param path: asset path; must identify a non-collection asset
:return: asset data
"""
raise errors.MethodNotImplementedError(
class_name=self.__class__.__name__, method_name='get_asset_data')
def create_asset(self, path: str, data: Optional[bytes] = None, **kwargs) \
-> DataProviderAsset:
"""
Create an asset at the given path
:param path: path at which to create the asset
:param data: FITS image data; if omitted, create a collection
asset
:param kwargs: optional extra provider specific parameters
:return: new data provider asset object
"""
raise errors.MethodNotImplementedError(
class_name=self.__class__.__name__, method_name='create_asset')
def rename_asset(self, path: str, name: str, **kwargs) \
-> DataProviderAsset:
"""
Rename asset at the given path
:param path: path at which to create the asset
:param name: new asset name
:param kwargs: optional extra provider specific parameters
:return: updated data provider asset object
"""
raise errors.MethodNotImplementedError(
class_name=self.__class__.__name__, method_name='rename_asset')
def update_asset(self, path: str, data: Optional[bytes], **kwargs) \
-> DataProviderAsset:
"""
Update an asset at the given path
:param path: path of the asset to update
:param data: asset data; create collection asset if None
:param kwargs: optional extra provider-specific parameters
:return: updated data provider asset object
"""
raise errors.MethodNotImplementedError(
class_name=self.__class__.__name__, method_name='update_asset')
def delete_asset(self, path: str, **kwargs) -> None:
"""
Delete an asset at the given path; recursively delete non-collection
assets
:param path: path of the asset to delete
:param kwargs: optional extra provider-specific parameters
"""
raise errors.MethodNotImplementedError(
class_name=self.__class__.__name__, method_name='delete_asset')
def check_quota(self: DataProvider, path: Optional[str], data: bytes) \
-> None:
"""
Check that the new asset data will not exceed the data provider's quota
:param path: asset path; must be set if updating existing asset
:param data: asset data being saved
"""
quota = self.quota
if quota:
usage = self.usage or 0
size = len(data) if data is not None else 0
if path is not None:
usage -= self.get_asset(path).metadata.get('size', 0)
if usage + size > quota:
raise QuotaExceededError(quota=quota, usage=usage, size=size)
def check_auth(self) -> None:
"""
Check that the user is authenticated with any of the auth methods
required for the data provider; raises NotAuthenticatedError if not
"""
if not app.config.get('USER_AUTH'):
# User auth disabled, always succeed
return
auth_methods = self.auth_methods
if not auth_methods:
# No specific auth methods requested
return
# Check that any of the auth methods requested is present
# in any of the user's identities
for required_method in auth_methods:
from .. import auth
if required_method == 'http':
# HTTP auth requires username and password being set
if auth.current_user is not None and \
auth.current_user.username and \
auth.current_user.password:
return
continue
# For non-HTTP methods, check identities
try:
for identity in auth.current_user.identities:
if identity.auth_method == required_method:
return
except AttributeError:
pass
raise NotAuthenticatedError(
error_msg='Data provider "{}" requires authentication with either '
'of the methods: {}'.format(self.id, ', '.join(auth_methods)))
def recursive_copy(self, provider: DataProvider,
src_path: str, dst_path: str,
move: bool = False, update: Optional[bool] = None,
force: bool = False, limit: int = 0, _depth: int = 0,
**kwargs) \
-> DataProviderAsset:
"""
Copy the whole asset from another data provider or a different path
within the same data provider
:param provider: source data provider; can be the same as the current
provider
:param src_path: asset path within the source data provider
:param dst_path: destination asset path within the current data
provider
:param move: delete source asset after successful copy
:param update: update existing asset at `dst_path` vs create
a new asset; None (default) means auto
:param force: overwrite existing top-level collection asset if updating
:param limit: recursion limit for the copy
:param _depth: current recursion depth; keep as is
:param kwargs: optional provider-specific keyword arguments to
:meth:`create_asset`, :meth:`update_asset`, and
:meth:`delete_asset`
:return: new data provider asset
"""
if update is None:
# Create or update top-level asset?
try:
self.get_asset(dst_path)
except AssetNotFoundError:
update = False
else:
update = True
src_asset = provider.get_asset(src_path)
if src_asset.collection:
# Copying the whole collection asset tree; first, create/update
# empty collection asset at dst_path
if not provider.browseable:
raise NonBrowseableDataProviderError(id=provider.id)
if update:
res = self.update_asset(dst_path, None, force=force, **kwargs)
else:
res = self.create_asset(dst_path, None, **kwargs)
if not limit or _depth < limit - 1:
for child_asset in provider.get_child_assets(src_path)[0]:
# For each child asset of a collection asset, recursively
# copy its data; calculate the destination path by
# appending the source asset name; always create
# destination asset since no asset exists there yet
self.recursive_copy(provider,
child_asset.path,
dst_path + '/' + child_asset.name,
move=move,
update=False,
limit=limit,
_depth=_depth + 1,
**kwargs)
else:
# Copying a non-collection asset
src_data = provider.get_asset_data(src_path)
self.check_quota(dst_path if update else None, src_data)
if update:
# Updating top-level destination asset
res = self.update_asset(dst_path,
src_data,
force=force,
**kwargs)
else:
# Creating non-collection asset
res = self.create_asset(dst_path, src_data, **kwargs)
if move:
# Delete the source asset after successful copy
self.delete_asset(src_path, **kwargs)
return res
class OAuthServerPluginBase(AuthnPluginBase):
"""
Class for OAuth plugins
"""
# Fields visible on the client side
authorize_url = String(default=None)
request_token_params = Dict(default=None)
client_id = String(default=None)
# Internal fields related to access token exchange
client_secret = None
access_token_url = None
access_token_method = None
access_token_headers = None
access_token_params = None
def __init__(self,
id: Optional[str] = None,
description: Optional[str] = None,
icon: Optional[str] = None,
register_users: Optional[bool] = None,
authorize_url: Optional[str] = None,
request_token_params: Optional[dict] = None,
client_id: str = None,
client_secret: str = None,
access_token_url: str = None,
access_token_method: str = 'POST',
access_token_headers: Optional[dict] = None,
access_token_params: Optional[dict] = None):
"""
Initialize OAuth plugin
:param id: plugin ID
:param description: plugin description
:param icon: plugin icon ID used by the client UI
:param register_users: automatically register authenticated users
if missing from the local user database; overrides
REGISTER_AUTHENTICATED_USERS
:param authorize_url: URL for authorization (needed by client)
:param request_token_params: additional parameters for auth code
exchange, like scope
:param client_id: client ID
:param client_secret: client secret
:param access_token_url: URL for token exchange
:param access_token_method: HTTP method for access token URL;
default: "POST"
:param access_token_headers: additional headers for token exchange
:param access_token_params: additional parameters for token exchange
"""
super().__init__(id=id,
description=description,
icon=icon,
register_users=register_users)
self.type = 'oauth_server'
self.authorize_url = authorize_url
if request_token_params:
self.request_token_params = request_token_params
else:
self.request_token_params = {}
if not client_id:
raise ValueError('Missing OAuth client ID')
self.client_id = client_id
if not client_secret:
raise ValueError('Missing OAuth client secret')
self.client_secret = str(client_secret)
if not access_token_url:
raise ValueError('Missing OAuth access token URL')
self.access_token_url = str(access_token_url)
if not access_token_method:
raise ValueError('Missing OAuth access token method')
access_token_method = str(access_token_method).upper()
if access_token_method not in ('GET', 'POST'):
raise ValueError('Invalid OAuth access token method "{}"'.format(
access_token_method))
self.access_token_method = access_token_method
if access_token_headers:
try:
access_token_headers = dict(access_token_headers)
except (TypeError, ValueError):
raise ValueError(
'Invalid OAuth access token headers "{}"'.format(
access_token_headers))
self.access_token_headers = access_token_headers
if access_token_params:
try:
access_token_params = dict(access_token_params)
except (TypeError, ValueError):
raise ValueError(
'Invalid OAuth access token parameters "{}"'.format(
access_token_params))
self.access_token_params = access_token_params
def construct_authorize_url(self, state: Optional[dict] = None) -> str:
"""
Generic authorization url formatter; implemented by OAuth plugin base
that creates the OAuth server's authorization URL from state parameters
:param state: additional application state to be added to OAuth state
query parameter
:return: authorization URL
"""
if state is None:
state = {}
state_json = json.dumps(state)
qs = urlencode(
dict(state=state_json,
redirect_uri=url_for('oauth2_authorized',
_external=True,
plugin_id=self.id),
client_id=self.client_id,
**self.request_token_params))
return '{}?{}'.format(self.authorize_url, qs)
def get_token(self, code: str, redirect_uri: str) -> OAuthToken:
"""
Generic token getter; implemented by OAuth plugin base that retrieves
the token using an authorization code
:param code: authorization code
:param base_url: root URL
:return: OAuthToken containing access, refresh, and expiration
"""
args = {
'grant_type': 'authorization_code',
'code': code,
'client_id': self.client_id,
'client_secret': self.client_secret,
'redirect_uri': redirect_uri,
}
if self.access_token_params:
args.update(self.access_token_params)
if self.access_token_method == 'POST':
data = args
args = None
else:
data = None
try:
resp = requests.request(
self.access_token_method,
self.access_token_url,
params=args,
data=data,
headers=self.access_token_headers,
verify=False if app.config.get('DEBUG') else None)
if resp.status_code not in (200, 201):
raise Exception(
'OAuth server returned HTTP status {}, message: {}'.format(
resp.status_code, resp.text))
data = resp.json()
# Get token expiration time
expires = data.get('expires_in')
if expires is not None:
expires = datetime.utcnow() + timedelta(seconds=expires)
return OAuthToken(access=data.get('access_token'),
refresh=data.get('refresh_token'),
expiration=expires)
except Exception as e:
raise NotAuthenticatedError(error_msg=str(e))
def get_user(self, token: OAuthToken) -> dict:
"""
Provider-specific user getter; implemented by OAuth plugin that
retrieves the user using the provider API and token
:param token: provider API access, refresh, expiration token info
:return: user profile
"""
raise errors.MethodNotImplementedError(
class_name=self.__class__.__name__, method_name='get_user')
class RunSpiderRequest(ma.Schema):
spider = Str()
project = Str()
params = Dict()
class QuestionLoaderSchema(Schema):
id = Integer()
order = Integer()
question = String()
options = Dict()
type = String()
subQuestions = Nested("self", many=True)
@validates_schema
def validate_schema(self, data):
if data.get("id") is not None:
if data.get("question") is None:
raise ValidationError(
"you must provide the text for the question to create the question",
"question")
if data.get("type") is None:
raise ValidationError(
"you must provide the type of the question", "type")
if (data.get("type") is not None
and (data["type"] == "matrix" or data["type"] == "ranking")
and data.get("subQuestions") is None):
raise ValidationError(
"you must provide at least one sub question for matrix and ranking type questions",
"subQuestions")
@post_load
def load_question(self, data):
session = get_db()
if data.get("id") is not None:
question = session.query(QuestionModel).filter_by(
id=data["id"]).one_or_none()
if question is None:
raise ValidationError(
f"The id {data[id]} for the question you have provided does not exist",
"id")
else:
question = QuestionModel()
if data.get("question") is not None:
question.question = data["question"]
if data.get("type") is not None:
type = session.query(QuestionTypeModel).filter_by(
type=data['type']).one_or_none()
if type is None:
raise ValidationError("The type " + data.get("type") +
"is not supported")
else:
question.type = type
if data.get("options") is not None:
question.options = data["options"]
if data.get("subQuestions") is not None:
if question.type.type != "matrix" and question.type.type != "ranking":
raise ValidationError(
"Only matrix and ranking type questions may have sub questions"
)
for sub in question.subQuestions:
if sub not in data["subQuestions"]:
sub.status = "deactive"
for sub in data['subQuestions']:
if sub['object'] not in question.subQuestions:
question.subQuestions.append(sub['object'])
elif sub.status == "deactive":
sub.status = "active"
for sub in data['subQuestions']:
if sub.get('order') is not None:
question.set_item_order(sub['order'], sub['object'])
elif (question.type.type == "matrix"
or question.type.type == "ranking"):
for sub in question.subQuestions:
if sub.status == "active":
sub.status == "deactive"
question.status = "deactive"
# handle the case of matrix and ranking questions
# the sub questions of these types need the parent questionKey as a prefix
if (question.questionKey is None and question.type.type == "matrix"
and question.type.type == "ranking"):
for sub in question.subQuestions:
if sub.questionKey is None:
sub.set_prefix(question.questionKey)
deserialized_return = {"object": question}
if data.get("order") is not None:
deserialized_return['order'] = data['order']
return deserialized_return
class EntityUpdateSchema(Schema):
schema = SchemaName(required=True)
properties = Dict()
class ChannelsEndpoint(ChannelsEndpointBase):
def setup_routes(self):
self.app.add_routes([
web.get('', self.get_channels),
web.get(r'/{channel_pk:\w*}/{channel_id:\w*}',
self.get_channel_contents),
web.get(r'/{channel_pk:\w*}/{channel_id:\w*}/description',
self.get_channel_description),
web.put(r'/{channel_pk:\w*}/{channel_id:\w*}/description',
self.put_channel_description),
web.get(r'/{channel_pk:\w*}/{channel_id:\w*}/thumbnail',
self.get_channel_thumbnail),
web.put(r'/{channel_pk:\w*}/{channel_id:\w*}/thumbnail',
self.put_channel_thumbnail),
web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/copy',
self.copy_channel),
web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/channels',
self.create_channel),
web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/collections',
self.create_collection),
web.put(r'/{channel_pk:\w*}/{channel_id:\w*}/torrents',
self.add_torrent_to_channel),
web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/commit',
self.post_commit),
web.get(r'/{channel_pk:\w*}/{channel_id:\w*}/commit',
self.is_channel_dirty),
web.get('/popular_torrents', self.get_popular_torrents_channel),
])
def add_download_progress_to_metadata_list(self, contents_list):
for torrent in contents_list:
if torrent['type'] == REGULAR_TORRENT:
dl = self.session.dlmgr.get_download(
unhexlify(torrent['infohash']))
if dl is not None and dl.tdef.infohash not in self.session.dlmgr.metainfo_requests:
torrent['progress'] = dl.get_state().get_progress()
def get_channel_from_request(self, request):
channel_pk = (self.session.mds.my_key.pub().key_to_bin()[10:]
if request.match_info['channel_pk'] == 'mychannel' else
unhexlify(request.match_info['channel_pk']))
channel_id = int(request.match_info['channel_id'])
return channel_pk, channel_id
@docs(
tags=['Metadata'],
summary='Get a list of all channels known to the system.',
responses={
200: {
'schema':
schema(
GetChannelsResponse={
'results': [ChannelSchema],
'first': Integer(),
'last': Integer(),
'sort_by': String(),
'sort_desc': Integer(),
'total': Integer(),
})
}
},
)
async def get_channels(self, request):
sanitized = self.sanitize_parameters(request.query)
sanitized[
'subscribed'] = None if 'subscribed' not in request.query else bool(
int(request.query['subscribed']))
include_total = request.query.get('include_total', '')
sanitized.update({"origin_id": 0})
sanitized['metadata_type'] = CHANNEL_TORRENT
with db_session:
channels = self.session.mds.get_entries(**sanitized)
total = self.session.mds.get_total_count(
**sanitized) if include_total else None
channels_list = []
for channel in channels:
channel_dict = channel.to_simple_dict()
# Add progress info for those channels that are still being processed
if channel.subscribed:
if channel_dict["state"] == CHANNEL_STATE.UPDATING.value:
try:
progress = self.session.mds.compute_channel_update_progress(
channel)
channel_dict["progress"] = progress
except (ZeroDivisionError, FileNotFoundError) as e:
self._logger.error(
"Error %s when calculating channel update progress. Channel data: %s-%i %i/%i",
e,
hexlify(channel.public_key),
channel.id_,
channel.start_timestamp,
channel.local_version,
)
elif channel_dict[
"state"] == CHANNEL_STATE.METAINFO_LOOKUP.value:
if not self.session.dlmgr.metainfo_requests.get(
bytes(channel.infohash)
) and self.session.dlmgr.download_exists(
bytes(channel.infohash)):
channel_dict[
"state"] = CHANNEL_STATE.DOWNLOADING.value
channels_list.append(channel_dict)
response_dict = {
"results": channels_list,
"first": sanitized["first"],
"last": sanitized["last"],
"sort_by": sanitized["sort_by"],
"sort_desc": int(sanitized["sort_desc"]),
}
if total is not None:
response_dict.update({"total": total})
return RESTResponse(response_dict)
@docs(
tags=['Metadata'],
summary=
'Get a list of the channel\'s contents (torrents/channels/etc.).',
responses={
200: {
'schema':
schema(
GetChannelContentsResponse={
'results': [Dict()],
'first': Integer(),
'last': Integer(),
'sort_by': String(),
'sort_desc': Integer(),
'total': Integer(),
})
}
},
)
async def get_channel_contents(self, request):
sanitized = self.sanitize_parameters(request.query)
include_total = request.query.get('include_total', '')
channel_pk, channel_id = self.get_channel_from_request(request)
sanitized.update({"channel_pk": channel_pk, "origin_id": channel_id})
remote = sanitized.pop("remote", None)
total = None
remote_failed = False
if remote:
try:
contents_list = await self.session.gigachannel_community.remote_select_channel_contents(
**sanitized)
except (RequestTimeoutException, NoChannelSourcesException,
CancelledError):
remote_failed = True
if not remote or remote_failed:
with db_session:
contents = self.session.mds.get_entries(**sanitized)
contents_list = [c.to_simple_dict() for c in contents]
total = self.session.mds.get_total_count(
**sanitized) if include_total else None
self.add_download_progress_to_metadata_list(contents_list)
response_dict = {
"results": contents_list,
"first": sanitized['first'],
"last": sanitized['last'],
"sort_by": sanitized['sort_by'],
"sort_desc": int(sanitized['sort_desc']),
}
if total is not None:
response_dict.update({"total": total})
return RESTResponse(response_dict)
async def get_channel_description(self, request):
channel_pk, channel_id = self.get_channel_from_request(request)
with db_session:
channel_description = self.session.mds.ChannelDescription.select(
lambda g: g.public_key == channel_pk and g.origin_id ==
channel_id).first()
response_dict = loads(channel_description.json_text) if (
channel_description is not None) else {}
return RESTResponse(response_dict)
async def put_channel_description(self, request):
channel_pk, channel_id = self.get_channel_from_request(request)
request_parsed = await request.json()
updated_json_text = dumps(
{"description_text": request_parsed["description_text"]})
with db_session:
channel_description = self.session.mds.ChannelDescription.select(
lambda g: g.public_key == channel_pk and g.origin_id ==
channel_id).first()
if channel_description is not None:
channel_description.update_properties(
{"json_text": updated_json_text})
else:
channel_description = self.session.mds.ChannelDescription(
public_key=channel_pk,
origin_id=channel_id,
json_text=updated_json_text,
status=NEW)
return RESTResponse(loads(channel_description.json_text))
async def get_channel_thumbnail(self, request):
channel_pk, channel_id = self.get_channel_from_request(request)
with db_session:
obj = self.session.mds.ChannelThumbnail.select(
lambda g: g.public_key == channel_pk and g.origin_id ==
channel_id).first()
return web.Response(
body=obj.binary_data,
content_type=obj.data_type) if obj else web.Response(status=400)
async def put_channel_thumbnail(self, request):
content_type = request.headers["Content-Type"]
post_body = await request.read()
channel_pk, channel_id = self.get_channel_from_request(request)
obj_properties = {"binary_data": post_body, "data_type": content_type}
with db_session:
obj = self.session.mds.ChannelThumbnail.select(
lambda g: g.public_key == channel_pk and g.origin_id ==
channel_id, ).first()
if obj is not None:
obj.update_properties(obj_properties)
else:
self.session.mds.ChannelThumbnail(public_key=channel_pk,
origin_id=channel_id,
status=NEW,
**obj_properties)
return web.Response(status=201)
@docs(
tags=['Metadata'],
summary='Create a copy of an entry/entries from another channel.',
parameters=[{
'in': 'body',
'name': 'entries',
'description': 'List of entries to copy',
'example': [{
'public_key': '1234567890',
'id': 123
}],
'required': True,
}],
responses={
200: {
'description': 'Returns a list of copied content'
},
HTTP_NOT_FOUND: {
'schema': HandledErrorSchema,
'example': {
"error": "Target channel not found"
}
},
HTTP_BAD_REQUEST: {
'schema': HandledErrorSchema,
'example': {
"error": "Source entry not found"
}
},
},
)
async def copy_channel(self, request):
with db_session:
channel_pk, channel_id = self.get_channel_from_request(request)
personal_root = channel_id == 0 and channel_pk == self.session.mds.my_key.pub(
).key_to_bin()[10:]
# TODO: better error handling
target_collection = self.session.mds.CollectionNode.get(
public_key=database_blob(channel_pk), id_=channel_id)
try:
request_parsed = await request.json()
except (ContentTypeError, ValueError):
return RESTResponse({"error": "Bad JSON"},
status=HTTP_BAD_REQUEST)
if not target_collection and not personal_root:
return RESTResponse({"error": "Target channel not found"},
status=HTTP_NOT_FOUND)
results_list = []
for entry in request_parsed:
public_key, id_ = database_blob(unhexlify(
entry["public_key"])), entry["id"]
source = self.session.mds.ChannelNode.get(
public_key=public_key, id_=id_)
if not source:
return RESTResponse({"error": "Source entry not found"},
status=HTTP_BAD_REQUEST)
# We must upgrade Collections to Channels when moving them to root channel, and, vice-versa,
# downgrade Channels to Collections when moving them into existing channels
if isinstance(source, self.session.mds.CollectionNode):
src_dict = source.to_dict()
if channel_id == 0:
rslt = self.session.mds.ChannelMetadata.create_channel(
title=source.title)
else:
dst_dict = {'origin_id': channel_id, "status": NEW}
for k in self.session.mds.CollectionNode.nonpersonal_attributes:
dst_dict[k] = src_dict[k]
dst_dict.pop("metadata_type")
rslt = self.session.mds.CollectionNode(**dst_dict)
for child in source.actual_contents:
child.make_copy(rslt.id_)
else:
rslt = source.make_copy(channel_id)
results_list.append(rslt.to_simple_dict())
return RESTResponse(results_list)
@docs(
tags=['Metadata'],
summary='Create a new channel entry in the given channel.',
responses={
200: {
'description': 'Returns the newly created channel',
'schema': schema(CreateChannelResponse={'results': [Dict()]}),
}
},
)
async def create_channel(self, request):
with db_session:
_, channel_id = self.get_channel_from_request(request)
request_parsed = await request.json()
channel_name = request_parsed.get("name", "New channel")
md = self.session.mds.ChannelMetadata.create_channel(
channel_name, origin_id=channel_id)
return RESTResponse({"results": [md.to_simple_dict()]})
@docs(
tags=['Metadata'],
summary='Create a new collection entry in the given channel.',
responses={
200: {
'description': 'Returns the newly created collection',
'schema':
schema(CreateCollectionResponse={'results': [Dict()]}),
}
},
)
async def create_collection(self, request):
with db_session:
_, channel_id = self.get_channel_from_request(request)
request_parsed = await request.json()
collection_name = request_parsed.get("name", "New collection")
md = self.session.mds.CollectionNode(origin_id=channel_id,
title=collection_name,
status=NEW)
return RESTResponse({"results": [md.to_simple_dict()]})
@docs(
tags=['Metadata'],
summary='Add a torrent file to your own channel.',
responses={
200: {
'schema':
schema(
AddTorrentToChannelResponse={
'added': (
Integer,
'Number of torrent that were added to the channel')
})
},
HTTP_NOT_FOUND: {
'schema': HandledErrorSchema,
'example': {
"error": "Unknown channel"
}
},
HTTP_BAD_REQUEST: {
'schema': HandledErrorSchema,
'example': {
"error": "unknown uri type"
}
},
},
)
@json_schema(
schema(
AddTorrentToChannelRequest={
'torrent': (String, 'Base64-encoded torrent file'),
'uri': (String, 'Add a torrent from a magnet link or URL'),
'torrents_dir': (
String, 'Add all .torrent files from a chosen directory'),
'recursive':
(Boolean,
'Toggle recursive scanning of the chosen directory for .torrent files'
),
'description': (String, 'Description for the torrent'),
}))
async def add_torrent_to_channel(self, request):
channel_pk, channel_id = self.get_channel_from_request(request)
with db_session:
channel = self.session.mds.CollectionNode.get(
public_key=database_blob(channel_pk), id_=channel_id)
if not channel:
return RESTResponse({"error": "Unknown channel"},
status=HTTP_NOT_FOUND)
parameters = await request.json()
extra_info = {}
if parameters.get('description', None):
extra_info = {'description': parameters['description']}
# First, check whether we did upload a magnet link or URL
if parameters.get('uri', None):
uri = parameters['uri']
if uri.startswith("http:") or uri.startswith("https:"):
async with ClientSession() as session:
response = await session.get(uri)
data = await response.read()
tdef = TorrentDef.load_from_memory(data)
elif uri.startswith("magnet:"):
_, xt, _ = parse_magnetlink(uri)
if (xt and is_infohash(codecs.encode(xt, 'hex')) and
(self.session.mds.torrent_exists_in_personal_channel(xt)
or channel.copy_torrent_from_infohash(xt))):
return RESTResponse({"added": 1})
meta_info = await self.session.dlmgr.get_metainfo(xt,
timeout=30,
url=uri)
if not meta_info:
raise RuntimeError("Metainfo timeout")
tdef = TorrentDef.load_from_dict(meta_info)
else:
return RESTResponse({"error": "unknown uri type"},
status=HTTP_BAD_REQUEST)
added = 0
if tdef:
channel.add_torrent_to_channel(tdef, extra_info)
added = 1
return RESTResponse({"added": added})
torrents_dir = None
if parameters.get('torrents_dir', None):
torrents_dir = parameters['torrents_dir']
if not path_util.isabs(torrents_dir):
return RESTResponse(
{"error": "the torrents_dir should point to a directory"},
status=HTTP_BAD_REQUEST)
recursive = False
if parameters.get('recursive'):
recursive = parameters['recursive']
if not torrents_dir:
return RESTResponse(
{
"error":
"the torrents_dir parameter should be provided when the recursive parameter is set"
},
status=HTTP_BAD_REQUEST,
)
if torrents_dir:
torrents_list, errors_list = channel.add_torrents_from_dir(
torrents_dir, recursive)
return RESTResponse({
"added": len(torrents_list),
"errors": errors_list
})
if not parameters.get('torrent', None):
return RESTResponse({"error": "torrent parameter missing"},
status=HTTP_BAD_REQUEST)
# Try to parse the torrent data
# Any errors will be handled by the error_middleware
torrent = base64.b64decode(parameters['torrent'])
torrent_def = TorrentDef.load_from_memory(torrent)
channel.add_torrent_to_channel(torrent_def, extra_info)
return RESTResponse({"added": 1})
@docs(
tags=['Metadata'],
summary='Commit a channel.',
responses={
200: {
'schema': schema(CommitResponse={'success': Boolean()})
}
},
)
async def post_commit(self, request):
channel_pk, channel_id = self.get_channel_from_request(request)
with db_session:
if channel_id == 0:
for t in self.session.mds.CollectionNode.commit_all_channels():
self.session.gigachannel_manager.updated_my_channel(
TorrentDef.load_from_dict(t))
else:
coll = self.session.mds.CollectionNode.get(
public_key=database_blob(channel_pk), id_=channel_id)
if not coll:
return RESTResponse({"success": False},
status=HTTP_NOT_FOUND)
torrent_dict = coll.commit_channel_torrent()
if torrent_dict:
self.session.gigachannel_manager.updated_my_channel(
TorrentDef.load_from_dict(torrent_dict))
return RESTResponse({"success": True})
@docs(
tags=['Metadata'],
summary='Check if a channel has uncommitted changes.',
responses={
200: {
'schema': schema(IsChannelDirtyResponse={'dirty': Boolean()})
}
},
)
async def is_channel_dirty(self, request):
channel_pk, _ = self.get_channel_from_request(request)
with db_session:
dirty = self.session.mds.MetadataNode.exists(
lambda g: g.public_key == database_blob(
channel_pk) and g.status in DIRTY_STATUSES)
return RESTResponse({"dirty": dirty})
@docs(
tags=['Metadata'],
summary=
'Get the list of most popular torrents. Functions as a pseudo-channel.',
responses={
200: {
'schema':
schema(GetChannelContentsResponse={
'results': [Dict()],
'first': Integer(),
'last': Integer(),
})
}
},
)
async def get_popular_torrents_channel(self, request):
sanitized = self.sanitize_parameters(request.query)
sanitized["metadata_type"] = REGULAR_TORRENT
sanitized["popular"] = True
with db_session:
contents = self.session.mds.get_entries(**sanitized)
contents_list = [c.to_simple_dict() for c in contents]
self.add_download_progress_to_metadata_list(contents_list)
response_dict = {
"results": contents_list,
"first": sanitized['first'],
"last": sanitized['last'],
}
return RESTResponse(response_dict)
class SchemaWithDict(Schema):
dict_field = Dict(values=String())
class EntityUpdateSchema(Schema):
name = String(allow_none=True)
schema = SchemaName(required=True)
properties = Dict()
class Catalog(AfterglowSchema):
"""
Base class for catalog plugins
Plugin modules are placed in the :mod:`resources.catalog_plugins`
subpackage and must directly or indirectly subclass from :class:`Catalog`,
e.g.
class MyCatalog(Catalog):
name = 'my_catalog'
num_sources = 1000000
mags = {'B': ('Bmag', 'eBmag'), 'V': ('Vmag', 'eVmag'),
'R': ('Rmag', 'eRmag'), 'I': ('Imag', 'eImag')}
filter_lookup = {'Open': '(3*B + 5*R)/8', '*': 'R'}
# '*' stands for "use this for any unknown filter"
def query_objects(self, names): # optional
...
def query_rect(self, ra_hours, dec_degs, width_arcmins, height_arcmins,
constraints=None):
...
def query_circ(self, ra_hours, dec_degs, radius_arcmins,
constraints=None):
...
Methods:
query_objects: return a list of catalog objects with the specified
names
query_box: return catalog objects within the specified rectangular
region
query_circ: return catalog objects within the specified circular
region
"""
__polymorphic_on__ = 'name'
name: str = String(default=None)
display_name: str = String(default=None)
num_sources: int = Integer()
mags: TDict[str, TList[str]] = Dict(
keys=String, values=List(String()), default={})
filter_lookup: TDict[str, str] = Dict(keys=String, values=String)
def __init__(self, **kwargs):
"""
Create a Catalog instance
:param kwargs: catalog-specific initialization parameters
"""
# Override catalog option defaults with CATALOG_OPTIONS config var
# for the current catalog
kwargs = dict(kwargs)
kwargs.update(app.config.get('CATALOG_OPTIONS', {}).get(self.name, {}))
super().__init__(**kwargs)
if self.display_name is None:
self.display_name = self.name
def query_objects(self, names: TList[str]) -> TList[CatalogSource]:
"""
Return a list of catalog objects with the specified names
:param names: object names
:return: list of catalog objects with the specified names
"""
raise errors.MethodNotImplementedError(
class_name=self.__class__.__name__, method_name='query_objects')
def query_box(self, ra_hours: float, dec_degs: float, width_arcmins: float,
height_arcmins: Optional[float] = None,
constraints: Optional[TDict[str, str]] = None,
limit: Optional[int] = None) \
-> TList[CatalogSource]:
"""
Return catalog objects within the specified rectangular region
:param ra_hours: right ascension of region center in hours
:param dec_degs: declination of region center in degrees
:param width_arcmins: width of region in arcminutes
:param height_arcmins: optional height of region in arcminutes;
defaults to `width_arcmins`
:param constraints: optional constraints on the column values
:param limit: optional limit on the number of objects to return
:return: list of catalog objects within the specified rectangular
region
"""
raise errors.MethodNotImplementedError(
class_name=self.__class__.__name__, method_name='query_rect')
def query_circ(self, ra_hours: float, dec_degs: float,
radius_arcmins: float,
constraints: Optional[TDict[str, str]] = None,
limit: Optional[int] = None) \
-> TList[CatalogSource]:
"""
Return catalog objects within the specified circular region
:param ra_hours: right ascension of region center in hours
:param dec_degs: declination of region center in degrees
:param radius_arcmins: region radius in arcminutes
:param constraints: optional constraints on the column values
:param limit: optional limit on the number of objects to return
:return: list of catalog objects
"""
raise errors.MethodNotImplementedError(
class_name=self.__class__.__name__, method_name='query_circ')
class ErrorSchema(Schema):
code = Str(required=True)
message = Str(required=True)
fields = Dict()
