class CollectionSchema(BaseSchema): EXPAND = [ ('creator', Role, 'creator', RoleReferenceSchema, False), ] label = String(validate=Length(min=2, max=500), required=True) foreign_id = String(missing=None) kind = String(dump_only=True) casefile = Boolean(missing=None) summary = String(allow_none=True) publisher = String(allow_none=True) publisher_url = Url(allow_none=True) data_url = Url(allow_none=True) info_url = Url(allow_none=True) countries = List(Country()) languages = List(Language()) secret = Boolean(dump_only=True) category = Category(missing=Collection.DEFAULT) creator_id = String(allow_none=True) creator = Nested(RoleReferenceSchema(), dump_only=True) team = List(Nested(RoleReferenceSchema()), dump_only=True) count = Integer(dump_only=True) schemata = Dict(dump_only=True, default={}) @pre_load() def flatten_collection(self, data): flatten_id(data, 'creator_id', 'creator') @pre_dump() def visibility(self, data): if not is_mapping(data): return roles = data.get('roles', []) public = Role.public_roles() data['secret'] = len(public.intersection(roles)) == 0 @post_dump def transient(self, data): pk = str(data.get('id')) data['links'] = { 'self': url_for('collections_api.view', id=pk), 'ui': collection_url(pk) } data['writeable'] = request.authz.can_write(pk) return data
class UASZoneSchema(BaseSchema): identifier = String(required=True) country = String(required=True) name = String() type = String(required=True) restriction = String(required=True) restriction_conditions = List(String(), data_key='restrictionConditions') region = Integer() reason = List(String(), validate=validate.Length(max=9)) other_reason_info = String(data_key='otherReasonInfo') regulation_exemption = String(data_key='regulationExemption') u_space_class = String(data_key='uSpaceClass') message = String() zone_authority = Nested(AuthoritySchema, data_key='zoneAuthority', required=True) applicability = Nested(TimePeriodSchema) geometry = Nested(AirspaceVolumeSchema, required=True, many=True) extended_properties = Dict(data_key='extendedProperties') @post_load def make_mongo_object(self, data, **kwargs): """ A document schema will be eventually loaded in a mongoengine object for possible storing in DB :param data: :param kwargs: :return: """ return UASZone(**data) @post_dump def handle_mongoengine_dict_field(self, data, **kwargs): """ Apparently the dict field is not serialized properly to a dict object so it has to be done manually. :param data: :param kwargs: :return: """ data['extendedProperties'] = dict(data['extendedProperties']) return data
class DocumentSchema(Schema, DatedSchema): id = String(dump_only=True) collection_id = Integer(dump_only=True, required=True) schema = SchemaName(dump_only=True) schemata = List(SchemaName(), dump_only=True) status = String(dump_only=True) type = String(dump_only=True) foreign_id = String(dump_only=True) content_hash = String(dump_only=True) parent = Dict(dump_only=True) # TODO: make writeable? uploader_id = Integer(dump_only=True) error_message = String(dump_only=True) # title = String(validate=Length(min=2, max=5000), missing=None) title = String(missing=None) summary = String(missing=None) countries = List(Country(), missing=[]) languages = List(Language(), missing=[]) keywords = List(String(validate=Length(min=1, max=5000)), missing=[]) dates = List(PartialDate(), dump_only=True) file_name = String(dump_only=True) file_size = Integer(dump_only=True) author = String(dump_only=True) mime_type = String(dump_only=True) extension = String(dump_only=True) encoding = String(dump_only=True) source_url = String(dump_only=True) pdf_version = String(dump_only=True) columns = List(String(), dump_only=True) children = Boolean(dump_to='$children', attribute='$children', dump_only=True) @post_dump def transient(self, data): data['$uri'] = url_for('documents_api.view', document_id=data.get('id')) data['$ui'] = document_url(data.get('id')) collection_id = data.get('collection_id') data['$writeable'] = request.authz.can_write(collection_id) return data
class TransactionLogScheme(BaseModelScheme): transaction_log_uuid = Str(validate=[validate.Length(max=36)]) transaction_time = DateTime() license_lrn_uuid = Str(validate=[validate.Length(max=36)]) license_switch_uuid = Str(validate=[validate.Length(max=36)]) type = Int() amount_total = Float() amount_lrn = Float() amount_switch = Float() transaction_id = Str(validate=[validate.Length(max=255)]) transaction_type = Str(validate=[validate.Length(max=255)]) from_ip = Str(validate=[validate.Length(max=36)]) transaction_src = Dict() status = Int() result = Str() payment_uuid = Str(validate=[validate.Length(max=36)]) license_lrn_plan_name = Str() license_switch_plan_name = Str() user_uuid = Str() user_name = Str() class Meta: model = model.TransactionLog fields = ( 'transaction_time', 'license_lrn_uuid', 'license_switch_uuid', 'type', 'amount_total', 'amount_lrn', 'amount_switch', 'transaction_id', 'transaction_type', 'from_ip', 'transaction_src', 'status', 'result', 'payment_uuid', )
class CollectionSchema(BaseSchema): EXPAND = [ ('creator', Role, 'creator', RoleReferenceSchema, False), ] label = String(validate=Length(min=2, max=500), required=True) foreign_id = String(missing=None) kind = String(dump_only=True) casefile = Boolean(missing=None) summary = String(allow_none=True) publisher = String(allow_none=True) publisher_url = Url(allow_none=True) data_url = Url(allow_none=True) info_url = Url(allow_none=True) countries = List(Country()) languages = List(Language()) secret = Boolean(dump_only=True) category = Category(missing=Collection.DEFAULT) creator_id = String(allow_none=True) creator = Nested(RoleReferenceSchema(), dump_only=True) team = List(Nested(RoleReferenceSchema()), dump_only=True) count = Integer(dump_only=True) schemata = Dict(dump_only=True) @pre_load def flatten_collection(self, data): flatten_id(data, 'creator_id', 'creator') @post_dump def hypermedia(self, data): pk = str(data.get('id')) data['links'] = { 'self': url_for('collections_api.view', id=pk), 'xref': url_for('xref_api.index', id=pk), 'xref_csv': url_for('xref_api.csv_export', id=pk, _authorize=True), 'ui': collection_url(pk) } data['writeable'] = request.authz.can(pk, request.authz.WRITE) return data
class FileSchema(Schema): """Service schema for files.""" key = SanitizedUnicode(dump_only=True) created = TZDateTime(timezone=timezone.utc, format='iso', dump_only=True) updated = TZDateTime(timezone=timezone.utc, format='iso', dump_only=True) status = GenMethod('dump_status') metadata = Dict(dump_only=True) checksum = Str(dump_only=True, attribute='file.checksum') storage_class = Str(dump_only=True, attribute='file.storage_class') mimetype = Str(dump_only=True, attribute='file.mimetype') size = Number(attribute='file.size') version_id = UUID(attribute='file.version_id') file_id = UUID(attribute='file.file_id') bucket_id = UUID(attribute='file.bucket_id') links = Links() def dump_status(self, obj): """Dump file status.""" return 'completed' if obj.file else 'pending'
class SchemaWithDict(Schema): dict_field = Dict(values=Nested(PetSchema))
class OverlaysEndpoint(BaseEndpoint): """ This endpoint is responsible for handing all requests regarding the status of overlays. """ def __init__(self): super(OverlaysEndpoint, self).__init__() self.statistics_supported = None def setup_routes(self): self.app.add_routes([ web.get('', self.get_overlays), web.get('/statistics', self.get_statistics), web.post('/statistics', self.enable_statistics) ]) def initialize(self, session): super(OverlaysEndpoint, self).initialize(session) self.statistics_supported = isinstance(session.endpoint, StatisticsEndpoint) \ or isinstance(getattr(session.endpoint, 'endpoint', None), StatisticsEndpoint) @docs(tags=["Overlays"], summary="Return information about all currently loaded overlays.", responses={ 200: { "schema": schema(OverlayResponse={"overlays": [OverlaySchema]}) } }) async def get_overlays(self, _): overlay_stats = [] for overlay in self.session.overlays: peers = overlay.get_peers() statistics = self.session.endpoint.get_aggregate_statistics(overlay.get_prefix()) \ if isinstance(self.session.endpoint, StatisticsEndpoint) else {} overlay_stats.append({ "master_peer": hexlify(overlay.master_peer.public_key.key_to_bin()).decode( 'utf-8'), "my_peer": hexlify( overlay.my_peer.public_key.key_to_bin()).decode('utf-8'), "global_time": overlay.global_time, "peers": [{ 'ip': peer.address[0], 'port': peer.address[1], 'public_key': hexlify(peer.public_key.key_to_bin()).decode('utf-8') } for peer in peers], "overlay_name": overlay.__class__.__name__, "statistics": statistics }) return Response({"overlays": overlay_stats}) @docs(tags=["Overlays"], summary="Return statistics for all currently loaded overlays.", responses={ 200: { "schema": schema( StatisticsResponse={ "statistics": List( Dict(keys=String, values=Nested(OverlayStatisticsSchema))), }), "examples": { 'Success': { "statistics": [{ "DiscoveryCommunity": { 'num_up': 0, 'num_down': 0, 'bytes_up': 0, 'bytes_down': 0, 'diff_time': 0 } }] } } } }) async def get_statistics(self, _): overlay_stats = [] for overlay in self.session.overlays: statistics = self.session.endpoint.get_statistics( overlay.get_prefix()) if self.statistics_supported else {} overlay_stats.append({ overlay.__class__.__name__: self.statistics_by_name(statistics, overlay) }) return Response({"statistics": overlay_stats}) def statistics_by_name(self, statistics, overlay): named_statistics = {} for message_id, network_stats in statistics.items(): if overlay.decode_map.get(chr(message_id)): mapped_name = str(message_id) + ":" + overlay.decode_map[chr( message_id)].__name__ else: mapped_name = str(message_id) + ":unknown" mapped_value = network_stats.to_dict() named_statistics[mapped_name] = mapped_value return named_statistics @docs(tags=["Overlays"], summary="Enable/disable statistics for a given overlay.", responses={ 200: { "schema": DefaultResponseSchema, "examples": { 'Success': { "success": True } } }, HTTP_PRECONDITION_FAILED: { "schema": DefaultResponseSchema, "examples": { 'Statistics disabled': { "success": False, "error": "StatisticsEndpoint is not enabled." } } }, HTTP_BAD_REQUEST: { "schema": DefaultResponseSchema, "examples": { 'Missing parameter': { "success": False, "error": "Parameter 'enable' is required." } } } }) @json_schema( schema( EnableStatisticsRequest={ 'enable*': (Boolean, 'Whether to enable or disable the statistics'), 'all': (Boolean, 'Whether update applies to all overlays'), 'overlay_name': (String, 'Class name of the overlay'), })) async def enable_statistics(self, request): if not self.statistics_supported: return Response( { "success": False, "error": "StatisticsEndpoint is not enabled." }, status=HTTP_PRECONDITION_FAILED) all_overlays = False overlay_name = None args = await request.json() if 'enable' not in args or not args['enable']: return Response( { "success": False, "error": "Parameter 'enable' is required" }, status=HTTP_BAD_REQUEST) enable = args['enable'].lower() == 'true' if 'all' in args and args['all']: all_overlays = args['all'].lower() == 'true' elif 'overlay_name' in args and args['overlay_name']: overlay_name = args['overlay_name'] else: return Response( { "success": False, "error": "Parameter 'all' or 'overlay_name' is required" }, status=HTTP_PRECONDITION_FAILED) self.enable_overlay_statistics(enable=enable, class_name=overlay_name, all_overlays=all_overlays) return Response({"success": True}) def enable_overlay_statistics(self, enable=False, class_name=None, all_overlays=False): if all_overlays: for overlay in self.session.overlays: self.session.endpoint.enable_community_statistics( overlay.get_prefix(), enable) elif class_name: for overlay in self.session.overlays: if overlay.__class__.__name__ == class_name: self.session.endpoint.enable_community_statistics( overlay.get_prefix(), enable)
class ShallowCombinedSchema(BaseSchema): collection_id = String() # Joint entity/document attributes collection = Nested(CollectionSchema()) schema = SchemaName() schemata = List(SchemaName()) names = List(String()) addresses = List(String()) phones = List(String()) emails = List(String()) identifiers = List(String()) countries = List(Country()) dates = List(PartialDate()) bulk = Boolean() # Entity attributes foreign_id = String() name = String() entities = List(String()) properties = Dict() # Document attributes status = String() content_hash = String() uploader_id = String() uploader = Nested(RoleReferenceSchema()) error_message = String() title = String() summary = String() languages = List(Language()) keywords = List(String()) date = PartialDate() authored_at = PartialDate() modified_at = PartialDate() published_at = PartialDate() retrieved_at = PartialDate() file_name = String() file_size = Integer() author = String() generator = String() mime_type = String() extension = String() encoding = String() source_url = String() pdf_version = String() columns = List(String()) headers = Dict() children = Integer() # TODO: is this a separate endpoint? text = String() html = String() def document_links(self, data, pk, schemata): links = { 'self': url_for('documents_api.view', document_id=pk), 'tags': url_for('entities_api.tags', id=pk), 'ui': document_url(pk) } if data.get('content_hash'): links['file'] = url_for('documents_api.file', document_id=pk, _authorize=True) if schemata.intersection([Document.SCHEMA_PDF]): links['pdf'] = url_for('documents_api.pdf', document_id=pk, _authorize=True) if schemata.intersection([Document.SCHEMA_PDF, Document.SCHEMA_TABLE]): links['records'] = url_for('documents_api.records', document_id=pk) if schemata.intersection([Document.SCHEMA_FOLDER]): query = (('filter:parent.id', pk),) links['children'] = url_for('documents_api.index', _query=query) return links def entity_links(self, data, pk, schemata): return { 'self': url_for('entities_api.view', id=pk), # 'similar': url_for('entities_api.similar', id=pk), # 'documents': url_for('entities_api.documents', id=pk), 'references': url_for('entities_api.references', id=pk), 'tags': url_for('entities_api.tags', id=pk), 'ui': entity_url(pk) } @post_dump() def hypermedia(self, data): pk = str(data.get('id')) collection = data.get('collection', {}) collection_id = collection.get('id') collection_id = collection_id or data.get('collection_id') schemata = set(data.get('schemata', [])) if Document.SCHEMA in schemata: data['links'] = self.document_links(data, pk, schemata) else: data['links'] = self.entity_links(data, pk, schemata) if data.get('bulk'): data['writeable'] = False else: data['writeable'] = request.authz.can_write(collection_id) return data
class ChannelsEndpoint(ChannelsEndpointBase): def setup_routes(self): self.app.add_routes( [ web.get('', self.get_channels), web.get(r'/{channel_pk:\w*}/{channel_id:\w*}', self.get_channel_contents), web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/copy', self.copy_channel), web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/channels', self.create_channel), web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/collections', self.create_collection), web.put(r'/{channel_pk:\w*}/{channel_id:\w*}/torrents', self.add_torrent_to_channel), web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/commit', self.post_commit), web.get(r'/{channel_pk:\w*}/{channel_id:\w*}/commit', self.is_channel_dirty), ] ) def get_channel_from_request(self, request): channel_pk = ( self.session.mds.my_key.pub().key_to_bin()[10:] if request.match_info['channel_pk'] == 'mychannel' else unhexlify(request.match_info['channel_pk']) ) channel_id = int(request.match_info['channel_id']) return channel_pk, channel_id @docs( tags=['Metadata'], summary='Get a list of all channels known to the system.', responses={ 200: { 'schema': schema( GetChannelsResponse={ 'results': [ChannelSchema], 'first': Integer(), 'last': Integer(), 'sort_by': String(), 'sort_desc': Integer(), 'total': Integer(), } ) } }, ) # TODO: DRY it with SpecificChannel endpoint? async def get_channels(self, request): sanitized = self.sanitize_parameters(request.query) sanitized['subscribed'] = None if 'subscribed' not in request.query else bool(int(request.query['subscribed'])) include_total = request.query.get('include_total', '') sanitized.update({"origin_id": 0}) with db_session: channels = self.session.mds.ChannelMetadata.get_entries(**sanitized) total = self.session.mds.ChannelMetadata.get_total_count(**sanitized) if include_total else None channels_list = [channel.to_simple_dict() for channel in channels] response_dict = { "results": channels_list, "first": sanitized["first"], "last": sanitized["last"], "sort_by": sanitized["sort_by"], "sort_desc": int(sanitized["sort_desc"]), } if total is not None: response_dict.update({"total": total}) return RESTResponse(response_dict) @docs( tags=['Metadata'], summary='Get a list of the channel\'s contents (torrents/channels/etc.).', responses={ 200: { 'schema': schema( GetChannelContentsResponse={ 'results': [Dict()], 'first': Integer(), 'last': Integer(), 'sort_by': String(), 'sort_desc': Integer(), 'total': Integer(), } ) } }, ) async def get_channel_contents(self, request): sanitized = self.sanitize_parameters(request.query) include_total = request.query.get('include_total', '') channel_pk, channel_id = self.get_channel_from_request(request) sanitized.update({"channel_pk": channel_pk, "origin_id": channel_id}) with db_session: contents = self.session.mds.MetadataNode.get_entries(**sanitized) contents_list = [c.to_simple_dict() for c in contents] total = self.session.mds.MetadataNode.get_total_count(**sanitized) if include_total else None response_dict = { "results": contents_list, "first": sanitized['first'], "last": sanitized['last'], "sort_by": sanitized['sort_by'], "sort_desc": int(sanitized['sort_desc']), } if total is not None: response_dict.update({"total": total}) return RESTResponse(response_dict) @docs( tags=['Metadata'], summary='Create a copy of an entry/entries from another channel.', parameters=[ { 'in': 'body', 'name': 'entries', 'description': 'List of entries to copy', 'example': [{'public_key': '1234567890', 'id': 123}], 'required': True, } ], responses={ 200: {'description': 'Returns a list of copied content'}, HTTP_NOT_FOUND: {'schema': HandledErrorSchema, 'example': {"error": "Target channel not found"}}, HTTP_BAD_REQUEST: {'schema': HandledErrorSchema, 'example': {"error": "Source entry not found"}}, }, ) async def copy_channel(self, request): with db_session: channel_pk, channel_id = self.get_channel_from_request(request) personal_root = channel_id == 0 and channel_pk == self.session.mds.my_key.pub().key_to_bin()[10:] # TODO: better error handling target_collection = self.session.mds.CollectionNode.get( public_key=database_blob(channel_pk), id_=channel_id ) try: request_parsed = await request.json() except (ContentTypeError, ValueError): return RESTResponse({"error": "Bad JSON"}, status=HTTP_BAD_REQUEST) if not target_collection and not personal_root: return RESTResponse({"error": "Target channel not found"}, status=HTTP_NOT_FOUND) results_list = [] for entry in request_parsed: public_key, id_ = database_blob(unhexlify(entry["public_key"])), entry["id"] source = self.session.mds.ChannelNode.get(public_key=public_key, id_=id_) if not source: return RESTResponse({"error": "Source entry not found"}, status=HTTP_BAD_REQUEST) # We must upgrade Collections to Channels when moving them to root channel, and, vice-versa, # downgrade Channels to Collections when moving them into existing channels if isinstance(source, self.session.mds.CollectionNode): src_dict = source.to_dict() if channel_id == 0: rslt = self.session.mds.ChannelMetadata.create_channel(title=source.title) else: dst_dict = {'origin_id': channel_id, "status": NEW} for k in self.session.mds.CollectionNode.nonpersonal_attributes: dst_dict[k] = src_dict[k] dst_dict.pop("metadata_type") rslt = self.session.mds.CollectionNode(**dst_dict) for child in source.actual_contents: child.make_copy(rslt.id_) else: rslt = source.make_copy(channel_id) results_list.append(rslt.to_simple_dict()) return RESTResponse(results_list) @docs( tags=['Metadata'], summary='Create a new channel entry in the given channel.', responses={ 200: { 'description': 'Returns the newly created channel', 'schema': schema(CreateChannelResponse={'results': [Dict()]}), } }, ) async def create_channel(self, request): with db_session: _, channel_id = self.get_channel_from_request(request) request_parsed = await request.json() channel_name = request_parsed.get("name", "New channel") md = self.session.mds.ChannelMetadata.create_channel(channel_name, origin_id=channel_id) return RESTResponse({"results": [md.to_simple_dict()]}) @docs( tags=['Metadata'], summary='Create a new collection entry in the given channel.', responses={ 200: { 'description': 'Returns the newly created collection', 'schema': schema(CreateCollectionResponse={'results': [Dict()]}), } }, ) async def create_collection(self, request): with db_session: _, channel_id = self.get_channel_from_request(request) request_parsed = await request.json() collection_name = request_parsed.get("name", "New collection") md = self.session.mds.CollectionNode(origin_id=channel_id, title=collection_name, status=NEW) return RESTResponse({"results": [md.to_simple_dict()]}) @docs( tags=['Metadata'], summary='Add a torrent file to your own channel.', responses={ 200: { 'schema': schema( AddTorrentToChannelResponse={'added': (Integer, 'Number of torrent that were added to the channel')} ) }, HTTP_NOT_FOUND: {'schema': HandledErrorSchema, 'example': {"error": "Unknown channel"}}, HTTP_BAD_REQUEST: {'schema': HandledErrorSchema, 'example': {"error": "unknown uri type"}}, }, ) @json_schema( schema( AddTorrentToChannelRequest={ 'torrent': (String, 'Base64-encoded torrent file'), 'uri': (String, 'Add a torrent from a magnet link or URL'), 'torrents_dir': (String, 'Add all .torrent files from a chosen directory'), 'recursive': (Boolean, 'Toggle recursive scanning of the chosen directory for .torrent files'), 'description': (String, 'Description for the torrent'), 'filesize': (Integer, "Filesize of the torrent file, this parameter is used for " "skipping metadata check when uri is a magnet link"), } ) ) async def add_torrent_to_channel(self, request): channel_pk, channel_id = self.get_channel_from_request(request) with db_session: channel = self.session.mds.CollectionNode.get(public_key=database_blob(channel_pk), id_=channel_id) if not channel: return RESTResponse({"error": "Unknown channel"}, status=HTTP_NOT_FOUND) parameters = await request.json() extra_info = {} if parameters.get('description', None): extra_info = {'description': parameters['description']} # First, check whether we did upload a magnet link or URL if parameters.get('uri', None): uri = parameters['uri'] if uri.startswith("http:") or uri.startswith("https:"): async with ClientSession() as session: response = await session.get(uri) data = await response.read() tdef = TorrentDef.load_from_memory(data) elif uri.startswith("magnet:"): _, xt, _ = parse_magnetlink(uri) if ( xt and is_infohash(codecs.encode(xt, 'hex')) and (channel.torrent_exists(xt) or channel.copy_torrent_from_infohash(xt)) ): return RESTResponse({"added": 1}) filesize = parameters.get("filesize") if filesize and not (isinstance(filesize, int) or int is None): return RESTResponse({"error": "filesize must be an integer"}, status=HTTP_BAD_REQUEST,) if filesize: dn, xt, _ = parse_magnetlink(uri) tdef = TorrentDefNoMetainfo(xt, dn, uri, filesize) else: meta_info = await self.session.dlmgr.get_metainfo(xt, timeout=30, url=uri) if not meta_info: raise RuntimeError("Metainfo timeout") tdef = TorrentDef.load_from_dict(meta_info) else: return RESTResponse({"error": "unknown uri type"}, status=HTTP_BAD_REQUEST) added = 0 if tdef: channel.add_torrent_to_channel(tdef, extra_info) added = 1 return RESTResponse({"added": added}) torrents_dir = None if parameters.get('torrents_dir', None): torrents_dir = parameters['torrents_dir'] if not path_util.isabs(torrents_dir): return RESTResponse({"error": "the torrents_dir should point to a directory"}, status=HTTP_BAD_REQUEST) recursive = False if parameters.get('recursive'): recursive = parameters['recursive'] if not torrents_dir: return RESTResponse( {"error": "the torrents_dir parameter should be provided when the recursive parameter is set"}, status=HTTP_BAD_REQUEST, ) if torrents_dir: torrents_list, errors_list = channel.add_torrents_from_dir(torrents_dir, recursive) return RESTResponse({"added": len(torrents_list), "errors": errors_list}) if not parameters.get('torrent', None): return RESTResponse({"error": "torrent parameter missing"}, status=HTTP_BAD_REQUEST) # Try to parse the torrent data # Any errors will be handled by the error_middleware torrent = base64.b64decode(parameters['torrent']) torrent_def = TorrentDef.load_from_memory(torrent) channel.add_torrent_to_channel(torrent_def, extra_info) return RESTResponse({"added": 1}) @docs( tags=['Metadata'], summary='Commit a channel.', responses={200: {'schema': schema(CommitResponse={'success': Boolean()})}}, ) async def post_commit(self, request): channel_pk, channel_id = self.get_channel_from_request(request) with db_session: if channel_id == 0: for t in self.session.mds.CollectionNode.commit_all_channels(): self.session.gigachannel_manager.updated_my_channel(TorrentDef.load_from_dict(t)) else: coll = self.session.mds.CollectionNode.get(public_key=database_blob(channel_pk), id_=channel_id) if not coll: return RESTResponse({"success": False}, status=HTTP_NOT_FOUND) torrent_dict = coll.commit_channel_torrent() if torrent_dict: self.session.gigachannel_manager.updated_my_channel(TorrentDef.load_from_dict(torrent_dict)) return RESTResponse({"success": True}) @docs( tags=['Metadata'], summary='Check if a channel has uncommitted changes.', responses={200: {'schema': schema(IsChannelDirtyResponse={'dirty': Boolean()})}}, ) async def is_channel_dirty(self, request): channel_pk, _ = self.get_channel_from_request(request) with db_session: dirty = self.session.mds.MetadataNode.exists( lambda g: g.public_key == database_blob(channel_pk) and g.status in DIRTY_STATUSES ) return RESTResponse({"dirty": dirty})
class SchemaWithDict(Schema): dict_field = Dict()
class AllRequiredTripletOfSideImport2Schema(Schema): a = TestsAbsThreeSideImport2Field(dump_to="a", load_from="a", allow_none=False) b = List( TestsBasicTestLiteralTopField(), dump_to="b", load_from="b", allow_none=False ) c = Dict(dump_to="c", load_from="c", allow_none=False)
class CommandSchema(Schema): name = String() data = Dict()
class DatabaseRequestSchema(Schema): currencies = Dict(key=Str(), values=Float())
class CollectionIndexSchema(CollectionSchema): count = Integer(dump_only=True) schemata = Dict(dump_only=True, default={})
class Device(Thing): __doc__ = m.Device.__doc__ id = Integer(description=m.Device.id.comment, dump_only=True) hid = SanitizedStr(lower=True, description=m.Device.hid.comment) tags = NestedOn('Tag', many=True, collection_class=OrderedSet, description='A set of tags that identify the device.') model = SanitizedStr(lower=True, validate=Length(max=STR_BIG_SIZE), description=m.Device.model.comment) manufacturer = SanitizedStr(lower=True, validate=Length(max=STR_SIZE), description=m.Device.manufacturer.comment) serial_number = SanitizedStr(lower=True, validate=Length(max=STR_BIG_SIZE), data_key='serialNumber') brand = SanitizedStr(validate=Length(max=STR_BIG_SIZE), description=m.Device.brand.comment) generation = Integer(validate=Range(1, 100), description=m.Device.generation.comment) version = SanitizedStr(description=m.Device.version) weight = Float(validate=Range(0.1, 5), unit=UnitCodes.kgm, description=m.Device.weight.comment) width = Float(validate=Range(0.1, 5), unit=UnitCodes.m, description=m.Device.width.comment) height = Float(validate=Range(0.1, 5), unit=UnitCodes.m, description=m.Device.height.comment) depth = Float(validate=Range(0.1, 5), unit=UnitCodes.m, description=m.Device.depth.comment) # TODO TimeOut 2. Comment actions and lots if there are time out. actions = NestedOn('Action', many=True, dump_only=True, description=m.Device.actions.__doc__) # TODO TimeOut 2. Comment actions_one and lots if there are time out. actions_one = NestedOn('Action', many=True, load_only=True, collection_class=OrderedSet) problems = NestedOn('Action', many=True, dump_only=True, description=m.Device.problems.__doc__) url = URL(dump_only=True, description=m.Device.url.__doc__) # TODO TimeOut 2. Comment actions and lots if there are time out. lots = NestedOn( 'Lot', many=True, dump_only=True, description='The lots where this device is directly under.') rate = NestedOn('Rate', dump_only=True, description=m.Device.rate.__doc__) price = NestedOn('Price', dump_only=True, description=m.Device.price.__doc__) tradings = Dict(dump_only=True, description='') physical = EnumField(states.Physical, dump_only=True, description=m.Device.physical.__doc__) traking = EnumField(states.Traking, dump_only=True, description=m.Device.physical.__doc__) usage = EnumField(states.Usage, dump_only=True, description=m.Device.physical.__doc__) revoke = UUID(dump_only=True) physical_possessor = NestedOn('Agent', dump_only=True, data_key='physicalPossessor') production_date = DateTime('iso', description=m.Device.updated.comment, data_key='productionDate') working = NestedOn('Action', many=True, dump_only=True, description=m.Device.working.__doc__) variant = SanitizedStr(description=m.Device.variant.comment) sku = SanitizedStr(description=m.Device.sku.comment) image = URL(description=m.Device.image.comment) allocated = Boolean(description=m.Device.allocated.comment) devicehub_id = SanitizedStr(data_key='devicehubID', description=m.Device.devicehub_id.comment) @pre_load def from_actions_to_actions_one(self, data: dict): """ Not an elegant way of allowing submitting actions to a device (in the context of Snapshots) without creating an ``actions`` field at the model (which is not possible). :param data: :return: """ # Note that it is secure to allow uploading actions_one # as the only time an user can send a device object is # in snapshots. data['actions_one'] = data.pop('actions', []) return data @post_load def validate_snapshot_actions(self, data): """Validates that only snapshot-related actions can be uploaded.""" from ereuse_devicehub.resources.action.models import EraseBasic, Test, Rate, Install, \ Benchmark for action in data['actions_one']: if not isinstance(action, (Install, EraseBasic, Rate, Test, Benchmark)): raise ValidationError('You cannot upload {}'.format(action), field_names=['actions'])
class FeatureExtractionOutput(Schema): cell_features = Nested(CellFeatures) sweep_features = Dict(keys=Str(), values=Nested(SweepFeatures)) cell_record = Nested(CellRecord) sweep_records = List(Nested(SweepRecord)) cell_state = Nested(CellState)
class Meta: unknown = EXCLUDE fields = Dict(required=True) tags = Nested(ImportantTags, required=True) timestamp = Float(required=True, validate=UnixEpoch())
class DataProvider(AfterglowSchema): """ Base class for data provider plugins Plugin modules are placed in the :mod:`resources.data_provider_plugins` subpackage and must subclass from :class:`DataProvider`, e.g. class MyDataProvider(DataProvider): name = 'my_provider' search_fields = {...} def get_asset(self, path): ... def get_asset_data(self, path): ... def get_child_assets(self, path): ... def find_assets(self, path=None, **kwargs): ... Methods: get_asset(): return asset at the given path; must be implemented by any data provider get_asset_data(): return data for a non-collection asset at the given path; must be implemented by any data provider get_child_assets(): return child assets of a collection asset at the given path; must be implemented by any browsable data provider find_assets(): return assets matching the given parameters; must be implemented by any searchable data provider create_asset(): create a new non-collection asset from data file at the given path, or an empty collection asset at the given path; must be implemented by a read-write provider if it supports adding new assets update_asset(): update an existing non-collection asset at the given path with a data file; must be implemented by a read-write provider if it supports modifying existing assets delete_asset(): delete an asset at the given path; must be implemented by a read-write provider if it supports deleting assets Attributes: id: unique integer ID of the data provider; assigned automatically on initialization name: unique data provider name; can be used by the clients in requests like GET /data-providers/[id]/assets in place of the integer data provider ID auth_methods: list of data provider-specific authentication methods; if None, defaults to DEFAULT_DATA_PROVIDER_AUTH -> DATA_FILE_AUTH -> all auth methods available icon: optional data provider icon name display_name: data provider plugin visible in the Afterglow UI description: a longer description of the data provider columns: list of dictionary {name: string, field_name: string, sortable: boolean} sort_by: string - name of column to use for initial sort sort_asc: boolean - initial sort order should be ascending browseable: True if the data provider supports browsing (i.e. getting child assets of a collection asset at the given path); automatically set depending on whether the provider implements get_child_assets() searchable: True if the data provider supports searching (i.e. querying using the custom search keywords defined by `search_fields`); automatically set depending on whether the provider implements find_assets() search_fields: dictionary {field_name: {"label": label, "type": type, ...}, ...} containing names and descriptions of search fields used on the client side to create search forms readonly: True if the data provider assets cannot be modified (created, updated, or deleted); automatically set depending on whether the provider implements create_asset(), update_asset(), or delete_asset() allow_upload: if readonly=False, allow uploading user images to the data provider quota: data provider storage quota, in bytes, if applicable usage: current usage of the data provider storage, in bytes, if applicable """ __polymorphic_on__ = 'name' id: int = Integer(default=None) name: str = String(default=None) auth_methods: Optional[TList[str]] = List(String(), default=None) display_name: str = String(default=None) icon: str = String(default=None) description: str = String(default=None) columns: TList[TDict[str, Any]] = List(Dict(), default=[]) sort_by: str = String(default=None) sort_asc: bool = Boolean(default=True) browseable: bool = Boolean(default=False) searchable: bool = Boolean(default=False) search_fields: TDict[str, TDict[str, Any]] = Dict(default={}) readonly: bool = Boolean(default=True) allow_upload: bool = Boolean(default=False) quota: int = Integer(default=None) usage: int = Integer(default=None) def __init__(self, **kwargs): """ Create a DataProvider instance :param kwargs: data provider initialization parameters """ super(DataProvider, self).__init__(_set_defaults=True, **kwargs) # Automatically set browseable, searchable, and readonly flags # depending on what methods are reimplemented by provider; method attr # of a class is an unbound method instance in Python 2 and a function # in Python 3 if 'browseable' not in kwargs: self.browseable = is_overridden(DataProvider, self, 'get_child_assets') if 'searchable' not in kwargs: self.searchable = is_overridden(DataProvider, self, 'find_assets') if 'readonly' not in kwargs: self.readonly = \ not is_overridden(DataProvider, self, 'create_asset') and \ not is_overridden(DataProvider, self, 'update_asset') and \ not is_overridden(DataProvider, self, 'delete_asset') if self.auth_methods is None: # Use default data provider authentication self.auth_methods = app.config.get('DEFAULT_DATA_PROVIDER_AUTH') if self.auth_methods is None: # Inherit auth methods from data files self.auth_methods = app.config.get('DATA_FILE_AUTH') if isinstance(self.auth_methods, str): self.auth_methods = self.auth_methods.split(',') def get_asset(self, path: str) -> DataProviderAsset: """ Return an asset at the given path :param path: asset path :return: asset object """ raise errors.MethodNotImplementedError( class_name=self.__class__.__name__, method_name='get_assets') def get_child_assets(self, path: str, sort_by: Optional[str] = None, page_size: Optional[int] = None, page: Optional[Union[int, str]] = None) \ -> Tuple[TList[DataProviderAsset], Optional[PaginationInfo]]: """ Return child assets of a collection asset at the given path :param path: asset path; must identify a collection asset :param sort_by: optional sorting key (e.g. column name); reverse sorting is indicated by prepending a hyphen to the key; data provider may assume a certain default sorting mode and must return it in the pagination info :param page_size: optional number of assets per page; None means don't use pagination (used only internally, never via the API); if not None, data provider may enforce a hard limit on the page size :param page: page-based pagination: optional 0-based page number (data provider returns at most `page_size` assets sorted by the sorting key at offset = `page`*`page_size`); keyset-based pagination: ">value" = return at most `page_size` assets with the value of `sort_by` key greater than the given value, "<value": return at most `page_size` assets with the value of the `sort_by` key smaller than the given value; for any pagination type, two special values "first" and "last" are used to return first and last page, respectively :return: list of :class:`DataProviderAsset` objects for child assets and pagination info or None if pagination is not supported """ raise errors.MethodNotImplementedError( class_name=self.__class__.__name__, method_name='get_child_assets') def find_assets(self, path: Optional[str] = None, sort_by: Optional[str] = None, page_size: Optional[int] = None, page: Optional[Union[int, str]] = None, **kwargs) \ -> Tuple[TList[DataProviderAsset], Optional[PaginationInfo]]: """ Return a list of assets matching the given parameters :param path: optional path to the collection asset to search in; by default (and for providers that do not have collection assets), search in the data provider root :param sort_by: optional sorting key; see :meth:`get_child_assets` :param page_size: optional number of assets per page :param page: optional 0-based page number, ">value", "<value", "first", or "last" :param kwargs: provider-specific keyword=value pairs defining the asset(s), like name, image type or dimensions :return: list of :class:`DataProviderAsset` objects for assets matching the search query parameters and pagination info or None if pagination is not supported """ raise errors.MethodNotImplementedError( class_name=self.__class__.__name__, method_name='find_assets') def get_asset_data(self, path: str) -> bytes: """ Return data for a non-collection asset at the given path :param path: asset path; must identify a non-collection asset :return: asset data """ raise errors.MethodNotImplementedError( class_name=self.__class__.__name__, method_name='get_asset_data') def create_asset(self, path: str, data: Optional[bytes] = None, **kwargs) \ -> DataProviderAsset: """ Create an asset at the given path :param path: path at which to create the asset :param data: FITS image data; if omitted, create a collection asset :param kwargs: optional extra provider specific parameters :return: new data provider asset object """ raise errors.MethodNotImplementedError( class_name=self.__class__.__name__, method_name='create_asset') def rename_asset(self, path: str, name: str, **kwargs) \ -> DataProviderAsset: """ Rename asset at the given path :param path: path at which to create the asset :param name: new asset name :param kwargs: optional extra provider specific parameters :return: updated data provider asset object """ raise errors.MethodNotImplementedError( class_name=self.__class__.__name__, method_name='rename_asset') def update_asset(self, path: str, data: Optional[bytes], **kwargs) \ -> DataProviderAsset: """ Update an asset at the given path :param path: path of the asset to update :param data: asset data; create collection asset if None :param kwargs: optional extra provider-specific parameters :return: updated data provider asset object """ raise errors.MethodNotImplementedError( class_name=self.__class__.__name__, method_name='update_asset') def delete_asset(self, path: str, **kwargs) -> None: """ Delete an asset at the given path; recursively delete non-collection assets :param path: path of the asset to delete :param kwargs: optional extra provider-specific parameters """ raise errors.MethodNotImplementedError( class_name=self.__class__.__name__, method_name='delete_asset') def check_quota(self: DataProvider, path: Optional[str], data: bytes) \ -> None: """ Check that the new asset data will not exceed the data provider's quota :param path: asset path; must be set if updating existing asset :param data: asset data being saved """ quota = self.quota if quota: usage = self.usage or 0 size = len(data) if data is not None else 0 if path is not None: usage -= self.get_asset(path).metadata.get('size', 0) if usage + size > quota: raise QuotaExceededError(quota=quota, usage=usage, size=size) def check_auth(self) -> None: """ Check that the user is authenticated with any of the auth methods required for the data provider; raises NotAuthenticatedError if not """ if not app.config.get('USER_AUTH'): # User auth disabled, always succeed return auth_methods = self.auth_methods if not auth_methods: # No specific auth methods requested return # Check that any of the auth methods requested is present # in any of the user's identities for required_method in auth_methods: from .. import auth if required_method == 'http': # HTTP auth requires username and password being set if auth.current_user is not None and \ auth.current_user.username and \ auth.current_user.password: return continue # For non-HTTP methods, check identities try: for identity in auth.current_user.identities: if identity.auth_method == required_method: return except AttributeError: pass raise NotAuthenticatedError( error_msg='Data provider "{}" requires authentication with either ' 'of the methods: {}'.format(self.id, ', '.join(auth_methods))) def recursive_copy(self, provider: DataProvider, src_path: str, dst_path: str, move: bool = False, update: Optional[bool] = None, force: bool = False, limit: int = 0, _depth: int = 0, **kwargs) \ -> DataProviderAsset: """ Copy the whole asset from another data provider or a different path within the same data provider :param provider: source data provider; can be the same as the current provider :param src_path: asset path within the source data provider :param dst_path: destination asset path within the current data provider :param move: delete source asset after successful copy :param update: update existing asset at `dst_path` vs create a new asset; None (default) means auto :param force: overwrite existing top-level collection asset if updating :param limit: recursion limit for the copy :param _depth: current recursion depth; keep as is :param kwargs: optional provider-specific keyword arguments to :meth:`create_asset`, :meth:`update_asset`, and :meth:`delete_asset` :return: new data provider asset """ if update is None: # Create or update top-level asset? try: self.get_asset(dst_path) except AssetNotFoundError: update = False else: update = True src_asset = provider.get_asset(src_path) if src_asset.collection: # Copying the whole collection asset tree; first, create/update # empty collection asset at dst_path if not provider.browseable: raise NonBrowseableDataProviderError(id=provider.id) if update: res = self.update_asset(dst_path, None, force=force, **kwargs) else: res = self.create_asset(dst_path, None, **kwargs) if not limit or _depth < limit - 1: for child_asset in provider.get_child_assets(src_path)[0]: # For each child asset of a collection asset, recursively # copy its data; calculate the destination path by # appending the source asset name; always create # destination asset since no asset exists there yet self.recursive_copy(provider, child_asset.path, dst_path + '/' + child_asset.name, move=move, update=False, limit=limit, _depth=_depth + 1, **kwargs) else: # Copying a non-collection asset src_data = provider.get_asset_data(src_path) self.check_quota(dst_path if update else None, src_data) if update: # Updating top-level destination asset res = self.update_asset(dst_path, src_data, force=force, **kwargs) else: # Creating non-collection asset res = self.create_asset(dst_path, src_data, **kwargs) if move: # Delete the source asset after successful copy self.delete_asset(src_path, **kwargs) return res
class OAuthServerPluginBase(AuthnPluginBase): """ Class for OAuth plugins """ # Fields visible on the client side authorize_url = String(default=None) request_token_params = Dict(default=None) client_id = String(default=None) # Internal fields related to access token exchange client_secret = None access_token_url = None access_token_method = None access_token_headers = None access_token_params = None def __init__(self, id: Optional[str] = None, description: Optional[str] = None, icon: Optional[str] = None, register_users: Optional[bool] = None, authorize_url: Optional[str] = None, request_token_params: Optional[dict] = None, client_id: str = None, client_secret: str = None, access_token_url: str = None, access_token_method: str = 'POST', access_token_headers: Optional[dict] = None, access_token_params: Optional[dict] = None): """ Initialize OAuth plugin :param id: plugin ID :param description: plugin description :param icon: plugin icon ID used by the client UI :param register_users: automatically register authenticated users if missing from the local user database; overrides REGISTER_AUTHENTICATED_USERS :param authorize_url: URL for authorization (needed by client) :param request_token_params: additional parameters for auth code exchange, like scope :param client_id: client ID :param client_secret: client secret :param access_token_url: URL for token exchange :param access_token_method: HTTP method for access token URL; default: "POST" :param access_token_headers: additional headers for token exchange :param access_token_params: additional parameters for token exchange """ super().__init__(id=id, description=description, icon=icon, register_users=register_users) self.type = 'oauth_server' self.authorize_url = authorize_url if request_token_params: self.request_token_params = request_token_params else: self.request_token_params = {} if not client_id: raise ValueError('Missing OAuth client ID') self.client_id = client_id if not client_secret: raise ValueError('Missing OAuth client secret') self.client_secret = str(client_secret) if not access_token_url: raise ValueError('Missing OAuth access token URL') self.access_token_url = str(access_token_url) if not access_token_method: raise ValueError('Missing OAuth access token method') access_token_method = str(access_token_method).upper() if access_token_method not in ('GET', 'POST'): raise ValueError('Invalid OAuth access token method "{}"'.format( access_token_method)) self.access_token_method = access_token_method if access_token_headers: try: access_token_headers = dict(access_token_headers) except (TypeError, ValueError): raise ValueError( 'Invalid OAuth access token headers "{}"'.format( access_token_headers)) self.access_token_headers = access_token_headers if access_token_params: try: access_token_params = dict(access_token_params) except (TypeError, ValueError): raise ValueError( 'Invalid OAuth access token parameters "{}"'.format( access_token_params)) self.access_token_params = access_token_params def construct_authorize_url(self, state: Optional[dict] = None) -> str: """ Generic authorization url formatter; implemented by OAuth plugin base that creates the OAuth server's authorization URL from state parameters :param state: additional application state to be added to OAuth state query parameter :return: authorization URL """ if state is None: state = {} state_json = json.dumps(state) qs = urlencode( dict(state=state_json, redirect_uri=url_for('oauth2_authorized', _external=True, plugin_id=self.id), client_id=self.client_id, **self.request_token_params)) return '{}?{}'.format(self.authorize_url, qs) def get_token(self, code: str, redirect_uri: str) -> OAuthToken: """ Generic token getter; implemented by OAuth plugin base that retrieves the token using an authorization code :param code: authorization code :param base_url: root URL :return: OAuthToken containing access, refresh, and expiration """ args = { 'grant_type': 'authorization_code', 'code': code, 'client_id': self.client_id, 'client_secret': self.client_secret, 'redirect_uri': redirect_uri, } if self.access_token_params: args.update(self.access_token_params) if self.access_token_method == 'POST': data = args args = None else: data = None try: resp = requests.request( self.access_token_method, self.access_token_url, params=args, data=data, headers=self.access_token_headers, verify=False if app.config.get('DEBUG') else None) if resp.status_code not in (200, 201): raise Exception( 'OAuth server returned HTTP status {}, message: {}'.format( resp.status_code, resp.text)) data = resp.json() # Get token expiration time expires = data.get('expires_in') if expires is not None: expires = datetime.utcnow() + timedelta(seconds=expires) return OAuthToken(access=data.get('access_token'), refresh=data.get('refresh_token'), expiration=expires) except Exception as e: raise NotAuthenticatedError(error_msg=str(e)) def get_user(self, token: OAuthToken) -> dict: """ Provider-specific user getter; implemented by OAuth plugin that retrieves the user using the provider API and token :param token: provider API access, refresh, expiration token info :return: user profile """ raise errors.MethodNotImplementedError( class_name=self.__class__.__name__, method_name='get_user')
class RunSpiderRequest(ma.Schema): spider = Str() project = Str() params = Dict()
class QuestionLoaderSchema(Schema): id = Integer() order = Integer() question = String() options = Dict() type = String() subQuestions = Nested("self", many=True) @validates_schema def validate_schema(self, data): if data.get("id") is not None: if data.get("question") is None: raise ValidationError( "you must provide the text for the question to create the question", "question") if data.get("type") is None: raise ValidationError( "you must provide the type of the question", "type") if (data.get("type") is not None and (data["type"] == "matrix" or data["type"] == "ranking") and data.get("subQuestions") is None): raise ValidationError( "you must provide at least one sub question for matrix and ranking type questions", "subQuestions") @post_load def load_question(self, data): session = get_db() if data.get("id") is not None: question = session.query(QuestionModel).filter_by( id=data["id"]).one_or_none() if question is None: raise ValidationError( f"The id {data[id]} for the question you have provided does not exist", "id") else: question = QuestionModel() if data.get("question") is not None: question.question = data["question"] if data.get("type") is not None: type = session.query(QuestionTypeModel).filter_by( type=data['type']).one_or_none() if type is None: raise ValidationError("The type " + data.get("type") + "is not supported") else: question.type = type if data.get("options") is not None: question.options = data["options"] if data.get("subQuestions") is not None: if question.type.type != "matrix" and question.type.type != "ranking": raise ValidationError( "Only matrix and ranking type questions may have sub questions" ) for sub in question.subQuestions: if sub not in data["subQuestions"]: sub.status = "deactive" for sub in data['subQuestions']: if sub['object'] not in question.subQuestions: question.subQuestions.append(sub['object']) elif sub.status == "deactive": sub.status = "active" for sub in data['subQuestions']: if sub.get('order') is not None: question.set_item_order(sub['order'], sub['object']) elif (question.type.type == "matrix" or question.type.type == "ranking"): for sub in question.subQuestions: if sub.status == "active": sub.status == "deactive" question.status = "deactive" # handle the case of matrix and ranking questions # the sub questions of these types need the parent questionKey as a prefix if (question.questionKey is None and question.type.type == "matrix" and question.type.type == "ranking"): for sub in question.subQuestions: if sub.questionKey is None: sub.set_prefix(question.questionKey) deserialized_return = {"object": question} if data.get("order") is not None: deserialized_return['order'] = data['order'] return deserialized_return
class EntityUpdateSchema(Schema): schema = SchemaName(required=True) properties = Dict()
class ChannelsEndpoint(ChannelsEndpointBase): def setup_routes(self): self.app.add_routes([ web.get('', self.get_channels), web.get(r'/{channel_pk:\w*}/{channel_id:\w*}', self.get_channel_contents), web.get(r'/{channel_pk:\w*}/{channel_id:\w*}/description', self.get_channel_description), web.put(r'/{channel_pk:\w*}/{channel_id:\w*}/description', self.put_channel_description), web.get(r'/{channel_pk:\w*}/{channel_id:\w*}/thumbnail', self.get_channel_thumbnail), web.put(r'/{channel_pk:\w*}/{channel_id:\w*}/thumbnail', self.put_channel_thumbnail), web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/copy', self.copy_channel), web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/channels', self.create_channel), web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/collections', self.create_collection), web.put(r'/{channel_pk:\w*}/{channel_id:\w*}/torrents', self.add_torrent_to_channel), web.post(r'/{channel_pk:\w*}/{channel_id:\w*}/commit', self.post_commit), web.get(r'/{channel_pk:\w*}/{channel_id:\w*}/commit', self.is_channel_dirty), web.get('/popular_torrents', self.get_popular_torrents_channel), ]) def add_download_progress_to_metadata_list(self, contents_list): for torrent in contents_list: if torrent['type'] == REGULAR_TORRENT: dl = self.session.dlmgr.get_download( unhexlify(torrent['infohash'])) if dl is not None and dl.tdef.infohash not in self.session.dlmgr.metainfo_requests: torrent['progress'] = dl.get_state().get_progress() def get_channel_from_request(self, request): channel_pk = (self.session.mds.my_key.pub().key_to_bin()[10:] if request.match_info['channel_pk'] == 'mychannel' else unhexlify(request.match_info['channel_pk'])) channel_id = int(request.match_info['channel_id']) return channel_pk, channel_id @docs( tags=['Metadata'], summary='Get a list of all channels known to the system.', responses={ 200: { 'schema': schema( GetChannelsResponse={ 'results': [ChannelSchema], 'first': Integer(), 'last': Integer(), 'sort_by': String(), 'sort_desc': Integer(), 'total': Integer(), }) } }, ) async def get_channels(self, request): sanitized = self.sanitize_parameters(request.query) sanitized[ 'subscribed'] = None if 'subscribed' not in request.query else bool( int(request.query['subscribed'])) include_total = request.query.get('include_total', '') sanitized.update({"origin_id": 0}) sanitized['metadata_type'] = CHANNEL_TORRENT with db_session: channels = self.session.mds.get_entries(**sanitized) total = self.session.mds.get_total_count( **sanitized) if include_total else None channels_list = [] for channel in channels: channel_dict = channel.to_simple_dict() # Add progress info for those channels that are still being processed if channel.subscribed: if channel_dict["state"] == CHANNEL_STATE.UPDATING.value: try: progress = self.session.mds.compute_channel_update_progress( channel) channel_dict["progress"] = progress except (ZeroDivisionError, FileNotFoundError) as e: self._logger.error( "Error %s when calculating channel update progress. Channel data: %s-%i %i/%i", e, hexlify(channel.public_key), channel.id_, channel.start_timestamp, channel.local_version, ) elif channel_dict[ "state"] == CHANNEL_STATE.METAINFO_LOOKUP.value: if not self.session.dlmgr.metainfo_requests.get( bytes(channel.infohash) ) and self.session.dlmgr.download_exists( bytes(channel.infohash)): channel_dict[ "state"] = CHANNEL_STATE.DOWNLOADING.value channels_list.append(channel_dict) response_dict = { "results": channels_list, "first": sanitized["first"], "last": sanitized["last"], "sort_by": sanitized["sort_by"], "sort_desc": int(sanitized["sort_desc"]), } if total is not None: response_dict.update({"total": total}) return RESTResponse(response_dict) @docs( tags=['Metadata'], summary= 'Get a list of the channel\'s contents (torrents/channels/etc.).', responses={ 200: { 'schema': schema( GetChannelContentsResponse={ 'results': [Dict()], 'first': Integer(), 'last': Integer(), 'sort_by': String(), 'sort_desc': Integer(), 'total': Integer(), }) } }, ) async def get_channel_contents(self, request): sanitized = self.sanitize_parameters(request.query) include_total = request.query.get('include_total', '') channel_pk, channel_id = self.get_channel_from_request(request) sanitized.update({"channel_pk": channel_pk, "origin_id": channel_id}) remote = sanitized.pop("remote", None) total = None remote_failed = False if remote: try: contents_list = await self.session.gigachannel_community.remote_select_channel_contents( **sanitized) except (RequestTimeoutException, NoChannelSourcesException, CancelledError): remote_failed = True if not remote or remote_failed: with db_session: contents = self.session.mds.get_entries(**sanitized) contents_list = [c.to_simple_dict() for c in contents] total = self.session.mds.get_total_count( **sanitized) if include_total else None self.add_download_progress_to_metadata_list(contents_list) response_dict = { "results": contents_list, "first": sanitized['first'], "last": sanitized['last'], "sort_by": sanitized['sort_by'], "sort_desc": int(sanitized['sort_desc']), } if total is not None: response_dict.update({"total": total}) return RESTResponse(response_dict) async def get_channel_description(self, request): channel_pk, channel_id = self.get_channel_from_request(request) with db_session: channel_description = self.session.mds.ChannelDescription.select( lambda g: g.public_key == channel_pk and g.origin_id == channel_id).first() response_dict = loads(channel_description.json_text) if ( channel_description is not None) else {} return RESTResponse(response_dict) async def put_channel_description(self, request): channel_pk, channel_id = self.get_channel_from_request(request) request_parsed = await request.json() updated_json_text = dumps( {"description_text": request_parsed["description_text"]}) with db_session: channel_description = self.session.mds.ChannelDescription.select( lambda g: g.public_key == channel_pk and g.origin_id == channel_id).first() if channel_description is not None: channel_description.update_properties( {"json_text": updated_json_text}) else: channel_description = self.session.mds.ChannelDescription( public_key=channel_pk, origin_id=channel_id, json_text=updated_json_text, status=NEW) return RESTResponse(loads(channel_description.json_text)) async def get_channel_thumbnail(self, request): channel_pk, channel_id = self.get_channel_from_request(request) with db_session: obj = self.session.mds.ChannelThumbnail.select( lambda g: g.public_key == channel_pk and g.origin_id == channel_id).first() return web.Response( body=obj.binary_data, content_type=obj.data_type) if obj else web.Response(status=400) async def put_channel_thumbnail(self, request): content_type = request.headers["Content-Type"] post_body = await request.read() channel_pk, channel_id = self.get_channel_from_request(request) obj_properties = {"binary_data": post_body, "data_type": content_type} with db_session: obj = self.session.mds.ChannelThumbnail.select( lambda g: g.public_key == channel_pk and g.origin_id == channel_id, ).first() if obj is not None: obj.update_properties(obj_properties) else: self.session.mds.ChannelThumbnail(public_key=channel_pk, origin_id=channel_id, status=NEW, **obj_properties) return web.Response(status=201) @docs( tags=['Metadata'], summary='Create a copy of an entry/entries from another channel.', parameters=[{ 'in': 'body', 'name': 'entries', 'description': 'List of entries to copy', 'example': [{ 'public_key': '1234567890', 'id': 123 }], 'required': True, }], responses={ 200: { 'description': 'Returns a list of copied content' }, HTTP_NOT_FOUND: { 'schema': HandledErrorSchema, 'example': { "error": "Target channel not found" } }, HTTP_BAD_REQUEST: { 'schema': HandledErrorSchema, 'example': { "error": "Source entry not found" } }, }, ) async def copy_channel(self, request): with db_session: channel_pk, channel_id = self.get_channel_from_request(request) personal_root = channel_id == 0 and channel_pk == self.session.mds.my_key.pub( ).key_to_bin()[10:] # TODO: better error handling target_collection = self.session.mds.CollectionNode.get( public_key=database_blob(channel_pk), id_=channel_id) try: request_parsed = await request.json() except (ContentTypeError, ValueError): return RESTResponse({"error": "Bad JSON"}, status=HTTP_BAD_REQUEST) if not target_collection and not personal_root: return RESTResponse({"error": "Target channel not found"}, status=HTTP_NOT_FOUND) results_list = [] for entry in request_parsed: public_key, id_ = database_blob(unhexlify( entry["public_key"])), entry["id"] source = self.session.mds.ChannelNode.get( public_key=public_key, id_=id_) if not source: return RESTResponse({"error": "Source entry not found"}, status=HTTP_BAD_REQUEST) # We must upgrade Collections to Channels when moving them to root channel, and, vice-versa, # downgrade Channels to Collections when moving them into existing channels if isinstance(source, self.session.mds.CollectionNode): src_dict = source.to_dict() if channel_id == 0: rslt = self.session.mds.ChannelMetadata.create_channel( title=source.title) else: dst_dict = {'origin_id': channel_id, "status": NEW} for k in self.session.mds.CollectionNode.nonpersonal_attributes: dst_dict[k] = src_dict[k] dst_dict.pop("metadata_type") rslt = self.session.mds.CollectionNode(**dst_dict) for child in source.actual_contents: child.make_copy(rslt.id_) else: rslt = source.make_copy(channel_id) results_list.append(rslt.to_simple_dict()) return RESTResponse(results_list) @docs( tags=['Metadata'], summary='Create a new channel entry in the given channel.', responses={ 200: { 'description': 'Returns the newly created channel', 'schema': schema(CreateChannelResponse={'results': [Dict()]}), } }, ) async def create_channel(self, request): with db_session: _, channel_id = self.get_channel_from_request(request) request_parsed = await request.json() channel_name = request_parsed.get("name", "New channel") md = self.session.mds.ChannelMetadata.create_channel( channel_name, origin_id=channel_id) return RESTResponse({"results": [md.to_simple_dict()]}) @docs( tags=['Metadata'], summary='Create a new collection entry in the given channel.', responses={ 200: { 'description': 'Returns the newly created collection', 'schema': schema(CreateCollectionResponse={'results': [Dict()]}), } }, ) async def create_collection(self, request): with db_session: _, channel_id = self.get_channel_from_request(request) request_parsed = await request.json() collection_name = request_parsed.get("name", "New collection") md = self.session.mds.CollectionNode(origin_id=channel_id, title=collection_name, status=NEW) return RESTResponse({"results": [md.to_simple_dict()]}) @docs( tags=['Metadata'], summary='Add a torrent file to your own channel.', responses={ 200: { 'schema': schema( AddTorrentToChannelResponse={ 'added': ( Integer, 'Number of torrent that were added to the channel') }) }, HTTP_NOT_FOUND: { 'schema': HandledErrorSchema, 'example': { "error": "Unknown channel" } }, HTTP_BAD_REQUEST: { 'schema': HandledErrorSchema, 'example': { "error": "unknown uri type" } }, }, ) @json_schema( schema( AddTorrentToChannelRequest={ 'torrent': (String, 'Base64-encoded torrent file'), 'uri': (String, 'Add a torrent from a magnet link or URL'), 'torrents_dir': ( String, 'Add all .torrent files from a chosen directory'), 'recursive': (Boolean, 'Toggle recursive scanning of the chosen directory for .torrent files' ), 'description': (String, 'Description for the torrent'), })) async def add_torrent_to_channel(self, request): channel_pk, channel_id = self.get_channel_from_request(request) with db_session: channel = self.session.mds.CollectionNode.get( public_key=database_blob(channel_pk), id_=channel_id) if not channel: return RESTResponse({"error": "Unknown channel"}, status=HTTP_NOT_FOUND) parameters = await request.json() extra_info = {} if parameters.get('description', None): extra_info = {'description': parameters['description']} # First, check whether we did upload a magnet link or URL if parameters.get('uri', None): uri = parameters['uri'] if uri.startswith("http:") or uri.startswith("https:"): async with ClientSession() as session: response = await session.get(uri) data = await response.read() tdef = TorrentDef.load_from_memory(data) elif uri.startswith("magnet:"): _, xt, _ = parse_magnetlink(uri) if (xt and is_infohash(codecs.encode(xt, 'hex')) and (self.session.mds.torrent_exists_in_personal_channel(xt) or channel.copy_torrent_from_infohash(xt))): return RESTResponse({"added": 1}) meta_info = await self.session.dlmgr.get_metainfo(xt, timeout=30, url=uri) if not meta_info: raise RuntimeError("Metainfo timeout") tdef = TorrentDef.load_from_dict(meta_info) else: return RESTResponse({"error": "unknown uri type"}, status=HTTP_BAD_REQUEST) added = 0 if tdef: channel.add_torrent_to_channel(tdef, extra_info) added = 1 return RESTResponse({"added": added}) torrents_dir = None if parameters.get('torrents_dir', None): torrents_dir = parameters['torrents_dir'] if not path_util.isabs(torrents_dir): return RESTResponse( {"error": "the torrents_dir should point to a directory"}, status=HTTP_BAD_REQUEST) recursive = False if parameters.get('recursive'): recursive = parameters['recursive'] if not torrents_dir: return RESTResponse( { "error": "the torrents_dir parameter should be provided when the recursive parameter is set" }, status=HTTP_BAD_REQUEST, ) if torrents_dir: torrents_list, errors_list = channel.add_torrents_from_dir( torrents_dir, recursive) return RESTResponse({ "added": len(torrents_list), "errors": errors_list }) if not parameters.get('torrent', None): return RESTResponse({"error": "torrent parameter missing"}, status=HTTP_BAD_REQUEST) # Try to parse the torrent data # Any errors will be handled by the error_middleware torrent = base64.b64decode(parameters['torrent']) torrent_def = TorrentDef.load_from_memory(torrent) channel.add_torrent_to_channel(torrent_def, extra_info) return RESTResponse({"added": 1}) @docs( tags=['Metadata'], summary='Commit a channel.', responses={ 200: { 'schema': schema(CommitResponse={'success': Boolean()}) } }, ) async def post_commit(self, request): channel_pk, channel_id = self.get_channel_from_request(request) with db_session: if channel_id == 0: for t in self.session.mds.CollectionNode.commit_all_channels(): self.session.gigachannel_manager.updated_my_channel( TorrentDef.load_from_dict(t)) else: coll = self.session.mds.CollectionNode.get( public_key=database_blob(channel_pk), id_=channel_id) if not coll: return RESTResponse({"success": False}, status=HTTP_NOT_FOUND) torrent_dict = coll.commit_channel_torrent() if torrent_dict: self.session.gigachannel_manager.updated_my_channel( TorrentDef.load_from_dict(torrent_dict)) return RESTResponse({"success": True}) @docs( tags=['Metadata'], summary='Check if a channel has uncommitted changes.', responses={ 200: { 'schema': schema(IsChannelDirtyResponse={'dirty': Boolean()}) } }, ) async def is_channel_dirty(self, request): channel_pk, _ = self.get_channel_from_request(request) with db_session: dirty = self.session.mds.MetadataNode.exists( lambda g: g.public_key == database_blob( channel_pk) and g.status in DIRTY_STATUSES) return RESTResponse({"dirty": dirty}) @docs( tags=['Metadata'], summary= 'Get the list of most popular torrents. Functions as a pseudo-channel.', responses={ 200: { 'schema': schema(GetChannelContentsResponse={ 'results': [Dict()], 'first': Integer(), 'last': Integer(), }) } }, ) async def get_popular_torrents_channel(self, request): sanitized = self.sanitize_parameters(request.query) sanitized["metadata_type"] = REGULAR_TORRENT sanitized["popular"] = True with db_session: contents = self.session.mds.get_entries(**sanitized) contents_list = [c.to_simple_dict() for c in contents] self.add_download_progress_to_metadata_list(contents_list) response_dict = { "results": contents_list, "first": sanitized['first'], "last": sanitized['last'], } return RESTResponse(response_dict)
class SchemaWithDict(Schema): dict_field = Dict(values=String())
class EntityUpdateSchema(Schema): name = String(allow_none=True) schema = SchemaName(required=True) properties = Dict()
class Catalog(AfterglowSchema): """ Base class for catalog plugins Plugin modules are placed in the :mod:`resources.catalog_plugins` subpackage and must directly or indirectly subclass from :class:`Catalog`, e.g. class MyCatalog(Catalog): name = 'my_catalog' num_sources = 1000000 mags = {'B': ('Bmag', 'eBmag'), 'V': ('Vmag', 'eVmag'), 'R': ('Rmag', 'eRmag'), 'I': ('Imag', 'eImag')} filter_lookup = {'Open': '(3*B + 5*R)/8', '*': 'R'} # '*' stands for "use this for any unknown filter" def query_objects(self, names): # optional ... def query_rect(self, ra_hours, dec_degs, width_arcmins, height_arcmins, constraints=None): ... def query_circ(self, ra_hours, dec_degs, radius_arcmins, constraints=None): ... Methods: query_objects: return a list of catalog objects with the specified names query_box: return catalog objects within the specified rectangular region query_circ: return catalog objects within the specified circular region """ __polymorphic_on__ = 'name' name: str = String(default=None) display_name: str = String(default=None) num_sources: int = Integer() mags: TDict[str, TList[str]] = Dict( keys=String, values=List(String()), default={}) filter_lookup: TDict[str, str] = Dict(keys=String, values=String) def __init__(self, **kwargs): """ Create a Catalog instance :param kwargs: catalog-specific initialization parameters """ # Override catalog option defaults with CATALOG_OPTIONS config var # for the current catalog kwargs = dict(kwargs) kwargs.update(app.config.get('CATALOG_OPTIONS', {}).get(self.name, {})) super().__init__(**kwargs) if self.display_name is None: self.display_name = self.name def query_objects(self, names: TList[str]) -> TList[CatalogSource]: """ Return a list of catalog objects with the specified names :param names: object names :return: list of catalog objects with the specified names """ raise errors.MethodNotImplementedError( class_name=self.__class__.__name__, method_name='query_objects') def query_box(self, ra_hours: float, dec_degs: float, width_arcmins: float, height_arcmins: Optional[float] = None, constraints: Optional[TDict[str, str]] = None, limit: Optional[int] = None) \ -> TList[CatalogSource]: """ Return catalog objects within the specified rectangular region :param ra_hours: right ascension of region center in hours :param dec_degs: declination of region center in degrees :param width_arcmins: width of region in arcminutes :param height_arcmins: optional height of region in arcminutes; defaults to `width_arcmins` :param constraints: optional constraints on the column values :param limit: optional limit on the number of objects to return :return: list of catalog objects within the specified rectangular region """ raise errors.MethodNotImplementedError( class_name=self.__class__.__name__, method_name='query_rect') def query_circ(self, ra_hours: float, dec_degs: float, radius_arcmins: float, constraints: Optional[TDict[str, str]] = None, limit: Optional[int] = None) \ -> TList[CatalogSource]: """ Return catalog objects within the specified circular region :param ra_hours: right ascension of region center in hours :param dec_degs: declination of region center in degrees :param radius_arcmins: region radius in arcminutes :param constraints: optional constraints on the column values :param limit: optional limit on the number of objects to return :return: list of catalog objects """ raise errors.MethodNotImplementedError( class_name=self.__class__.__name__, method_name='query_circ')
class ErrorSchema(Schema): code = Str(required=True) message = Str(required=True) fields = Dict()