def retrieve(fid, actor=None, serializer_class=None, queryset=None): if queryset: try: # queryset can also be a Model class existing = queryset.filter(fid=fid).first() except AttributeError: existing = queryset.objects.filter(fid=fid).first() if existing: return existing auth = (None if not actor else signing.get_auth(actor.private_key, actor.private_key_id)) response = session.get_session().get( fid, auth=auth, timeout=5, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, headers={ "Accept": "application/activity+json", "Content-Type": "application/activity+json", }, ) response.raise_for_status() data = response.json() if not serializer_class: return data serializer = serializer_class(data=data) serializer.is_valid(raise_exception=True) return serializer.save()
def deliver_to_remote(delivery): if not preferences.get("federation__enabled"): # federation is disabled, we only deliver to local recipients return actor = delivery.activity.actor logger.info("Preparing activity delivery to %s", delivery.inbox_url) auth = signing.get_auth(actor.private_key, actor.private_key_id) try: response = session.get_session().post( auth=auth, json=delivery.activity.payload, url=delivery.inbox_url, headers={"Content-Type": "application/activity+json"}, ) logger.debug("Remote answered with %s", response.status_code) response.raise_for_status() except Exception: delivery.last_attempt_date = timezone.now() delivery.attempts = F("attempts") + 1 delivery.save(update_fields=["last_attempt_date", "attempts"]) raise else: delivery.last_attempt_date = timezone.now() delivery.attempts = F("attempts") + 1 delivery.is_delivered = True delivery.save( update_fields=["last_attempt_date", "attempts", "is_delivered"])
def download_audio_from_remote(self, actor): from funkwhale_api.federation import signing if actor: auth = signing.get_auth(actor.private_key, actor.private_key_id) else: auth = None remote_response = session.get_session().get( self.source, auth=auth, stream=True, timeout=20, headers={"Content-Type": "application/octet-stream"}, ) with remote_response as r: remote_response.raise_for_status() extension = utils.get_ext_from_type(self.mimetype) title_parts = [] title_parts.append(self.track.title) if self.track.album: title_parts.append(self.track.album.title) title_parts.append(self.track.artist.name) title = " - ".join(title_parts) filename = "{}.{}".format(title, extension) tmp_file = tempfile.TemporaryFile() for chunk in r.iter_content(chunk_size=512): tmp_file.write(chunk) self.audio_file.save(filename, tmp_file, save=False) self.save(update_fields=["audio_file"])
def download_audio_from_remote(self, user): from funkwhale_api.common import session from funkwhale_api.federation import signing if user.is_authenticated and user.actor: auth = signing.get_auth(user.actor.private_key, user.actor.private_key_id) else: auth = None remote_response = session.get_session().get( self.source, auth=auth, stream=True, timeout=20, headers={"Content-Type": "application/octet-stream"}, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, ) with remote_response as r: remote_response.raise_for_status() extension = utils.get_ext_from_type(self.mimetype) title = " - ".join([ self.track.title, self.track.album.title, self.track.artist.name ]) filename = "{}.{}".format(title, extension) tmp_file = tempfile.TemporaryFile() for chunk in r.iter_content(chunk_size=512): tmp_file.write(chunk) self.audio_file.save(filename, tmp_file, save=False) self.save(update_fields=["audio_file"])
def get_actor_data(actor_url): response = session.get_session().get( actor_url, headers={"Accept": "application/activity+json"}, ) response.raise_for_status() try: return response.json() except Exception: raise ValueError("Invalid actor payload: {}".format(response.text))
def get_resource(resource_string): resource_type, resource = clean_resource(resource_string) username, hostname = clean_acct(resource, ensure_local=False) url = "https://{}/.well-known/webfinger?resource={}".format( hostname, resource_string) response = session.get_session().get( url, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, timeout=5) response.raise_for_status() serializer = serializers.ActorWebfingerSerializer(data=response.json()) serializer.is_valid(raise_exception=True) return serializer.validated_data
def get_actor_data(actor_url): response = session.get_session().get( actor_url, timeout=5, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, headers={"Accept": "application/activity+json"}, ) response.raise_for_status() try: return response.json() except Exception: raise ValueError("Invalid actor payload: {}".format(response.text))
def handle(self, *args, **options): logger = logging.getLogger("funkwhale.mrf") logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(stream=sys.stderr)) input = options["input"] if not input: registry = getattr(mrf, options["type"]) self.stdout.write( "No input given, listing registered policies for '{}' MRF:". format(options["type"])) for name in registry.keys(): self.stdout.write("- {}".format(name)) return raw_content = None content = None if input == "-": raw_content = sys.stdin.read() elif is_uuid(input): self.stderr.write("UUID provided, retrieving payload from db") content = models.Activity.objects.get(uuid=input).payload elif is_url(input): response = session.get_session().get( input, headers={"Accept": "application/activity+json"}, ) response.raise_for_status() content = response.json() else: with open(input, "rb") as f: raw_content = f.read() content = json.loads(raw_content) if content is None else content policies = options["policies"] or [] registry = getattr(mrf, options["type"]) for policy in policies: if policy not in registry: raise CommandError("Unknown policy '{}' for MRF '{}'".format( policy, options["type"])) payload, updated = registry.apply(content, policies=policies) if not payload: self.stderr.write("Payload was discarded by MRF") elif updated: self.stderr.write("Payload was modified by MRF") self.stderr.write("Initial payload:\n") self.stdout.write(json.dumps(content, indent=2, sort_keys=True)) self.stderr.write("Modified payload:\n") self.stdout.write(json.dumps(payload, indent=2, sort_keys=True)) else: self.stderr.write("Payload left untouched by MRF")
def get_library_page(library, page_url, actor): auth = signing.get_auth(actor.private_key, actor.private_key_id) response = session.get_session().get( page_url, auth=auth, timeout=5, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, headers={"Content-Type": "application/activity+json"}, ) serializer = serializers.CollectionPageSerializer( data=response.json(), context={"library": library, "item_serializer": serializers.UploadSerializer}, ) serializer.is_valid(raise_exception=True) return serializer.validated_data
def get_library_page(library, page_url, actor): auth = signing.get_auth(actor.private_key, actor.private_key_id) response = session.get_session().get( page_url, auth=auth, headers={"Accept": "application/activity+json"}, ) serializer = serializers.CollectionPageSerializer( data=response.json(), context={ "library": library, "item_serializer": serializers.UploadSerializer }, ) serializer.is_valid(raise_exception=True) return serializer.validated_data
def fetch_nodeinfo(domain_name): s = session.get_session() wellknown_url = "https://{}/.well-known/nodeinfo".format(domain_name) response = s.get(url=wellknown_url) response.raise_for_status() serializer = serializers.NodeInfoSerializer(data=response.json()) serializer.is_valid(raise_exception=True) nodeinfo_url = None for link in serializer.validated_data["links"]: if link["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0": nodeinfo_url = link["href"] break response = s.get(url=nodeinfo_url) response.raise_for_status() return response.json()
def retrieve_ap_object(fid, actor, serializer_class=None, queryset=None, apply_instance_policies=True): # we have a duplicate check here because it's less expensive to do those checks # twice than to trigger a HTTP request payload, updated = mrf.inbox.apply({"id": fid}) if not payload: raise exceptions.BlockedActorOrDomain() if queryset: try: # queryset can also be a Model class existing = queryset.filter(fid=fid).first() except AttributeError: existing = queryset.objects.filter(fid=fid).first() if existing: return existing auth = (None if not actor else signing.get_auth(actor.private_key, actor.private_key_id)) response = session.get_session().get( fid, auth=auth, headers={ "Accept": "application/activity+json", "Content-Type": "application/activity+json", }, ) response.raise_for_status() data = response.json() # we match against mrf here again, because new data may yield different # results data, updated = mrf.inbox.apply(data) if not data: raise exceptions.BlockedActorOrDomain() if not serializer_class: return data serializer = serializer_class(data=data, context={"fetch_actor": actor}) serializer.is_valid(raise_exception=True) try: return serializer.save() except NotImplementedError: return serializer.validated_data
def download_audio(self): from . import actors auth = actors.SYSTEM_ACTORS["library"].get_request_auth() remote_response = session.get_session().get( self.audio_url, auth=auth, stream=True, timeout=20, headers={"Accept": "application/activity+json"}, ) with remote_response as r: remote_response.raise_for_status() extension = music_utils.get_ext_from_type(self.audio_mimetype) title = " - ".join( [self.title, self.album_title, self.artist_name]) filename = "{}.{}".format(title, extension) tmp_file = tempfile.TemporaryFile() for chunk in r.iter_content(chunk_size=512): tmp_file.write(chunk) self.audio_file.save(filename, tmp_file)
def retrieve_feed(url): try: logger.info("Fetching RSS feed at %s", url) response = session.get_session().get(url) response.raise_for_status() except requests.exceptions.HTTPError as e: if e.response: raise FeedFetchException( "Error while fetching feed: HTTP {}".format(e.response.status_code) ) raise FeedFetchException("Error while fetching feed: unknown error") except requests.exceptions.Timeout: raise FeedFetchException("Error while fetching feed: timeout") except requests.exceptions.ConnectionError: raise FeedFetchException("Error while fetching feed: connection error") except requests.RequestException as e: raise FeedFetchException("Error while fetching feed: {}".format(e)) except Exception as e: raise FeedFetchException("Error while fetching feed: {}".format(e)) return response
def get_image(self, data=None): if data: extensions = { "image/jpeg": "jpg", "image/png": "png", "image/gif": "gif" } extension = extensions.get(data["mimetype"], "jpg") if data.get("content"): # we have to cover itself f = ContentFile(data["content"]) elif data.get("url"): # we can fetch from a url try: response = session.get_session().get( data.get("url"), timeout=3, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, ) response.raise_for_status() except Exception as e: logger.warn("Cannot download cover at url %s: %s", data.get("url"), e) return else: f = ContentFile(response.content) self.cover.save("{}.{}".format(self.uuid, extension), f, save=False) self.save(update_fields=["cover"]) return self.cover.file if self.mbid: image_data = musicbrainz.api.images.get_front(str(self.mbid)) f = ContentFile(image_data) self.cover.save("{0}.jpg".format(self.mbid), f, save=False) self.save(update_fields=["cover"]) return self.cover.file
def get_library_data(library_url, actor): auth = signing.get_auth(actor.private_key, actor.private_key_id) try: response = session.get_session().get( library_url, auth=auth, headers={"Accept": "application/activity+json"}, ) except requests.ConnectionError: return {"errors": ["This library is not reachable"]} scode = response.status_code if scode == 401: return {"errors": ["This library requires authentication"]} elif scode == 403: return {"errors": ["Permission denied while scanning library"]} elif scode >= 400: return { "errors": ["Error {} while fetching the library".format(scode)] } serializer = serializers.LibrarySerializer(data=response.json()) if not serializer.is_valid(): return {"errors": ["Invalid ActivityPub response from remote library"]} return serializer.validated_data
def get_session(): from funkwhale_api.common import session return session.get_session()
def fetch(fetch_obj): def error(code, **kwargs): fetch_obj.status = "errored" fetch_obj.fetch_date = timezone.now() fetch_obj.detail = {"error_code": code} fetch_obj.detail.update(kwargs) fetch_obj.save(update_fields=["fetch_date", "status", "detail"]) url = fetch_obj.url mrf_check_url = url if not mrf_check_url.startswith("webfinger://"): payload, updated = mrf.inbox.apply({"id": mrf_check_url}) if not payload: return error("blocked", message="Blocked by MRF") actor = fetch_obj.actor if settings.FEDERATION_AUTHENTIFY_FETCHES: auth = signing.get_auth(actor.private_key, actor.private_key_id) else: auth = None auth = None try: if url.startswith("webfinger://"): # we first grab the correpsonding webfinger representation # to get the ActivityPub actor ID webfinger_data = webfinger.get_resource( "acct:" + url.replace("webfinger://", "")) url = webfinger.get_ap_url(webfinger_data["links"]) if not url: return error("webfinger", message="Invalid or missing webfinger data") payload, updated = mrf.inbox.apply({"id": url}) if not payload: return error("blocked", message="Blocked by MRF") response = session.get_session().get( auth=auth, url=url, headers={"Accept": "application/activity+json"}, ) logger.debug("Remote answered with %s", response.status_code) response.raise_for_status() except requests.exceptions.HTTPError as e: return error( "http", status_code=e.response.status_code if e.response else None) except requests.exceptions.Timeout: return error("timeout") except requests.exceptions.ConnectionError as e: return error("connection", message=str(e)) except requests.RequestException as e: return error("request", message=str(e)) except Exception as e: return error("unhandled", message=str(e)) try: payload = response.json() except json.decoder.JSONDecodeError: # we attempt to extract a <link rel=alternate> that points # to an activity pub resource, if possible, and retry with this URL alternate_url = utils.find_alternate(response.text) if alternate_url: fetch_obj.url = alternate_url fetch_obj.save(update_fields=["url"]) return fetch(fetch_id=fetch_obj.pk) return error("invalid_json") payload, updated = mrf.inbox.apply(payload) if not payload: return error("blocked", message="Blocked by MRF") try: doc = jsonld.expand(payload) except ValueError: return error("invalid_jsonld") try: type = doc.get("@type", [])[0] except IndexError: return error("missing_jsonld_type") try: serializer_classes = fetch_obj.serializers[type] model = serializer_classes[0].Meta.model except (KeyError, AttributeError): fetch_obj.status = "skipped" fetch_obj.fetch_date = timezone.now() fetch_obj.detail = {"reason": "unhandled_type", "type": type} return fetch_obj.save(update_fields=["fetch_date", "status", "detail"]) try: id = doc.get("@id") except IndexError: existing = None else: existing = model.objects.filter(fid=id).first() serializer = None for serializer_class in serializer_classes: serializer = serializer_class(existing, data=payload) if not serializer.is_valid(): continue else: break if serializer.errors: return error("validation", validation_errors=serializer.errors) try: obj = serializer.save() except Exception as e: error("save", message=str(e)) raise # special case for channels # when obj is an actor, we check if the actor has a channel associated with it # if it is the case, we consider the fetch obj to be a channel instead # and also trigger a fetch on the channel outbox if isinstance(obj, models.Actor) and obj.get_channel(): obj = obj.get_channel() if obj.actor.outbox_url: try: # first page fetch is synchronous, so that at least some data is available # in the UI after subscription result = fetch_collection( obj.actor.outbox_url, channel_id=obj.pk, max_pages=1, ) except Exception: logger.exception("Error while fetching actor outbox: %s", obj.actor.outbox.url) else: if result.get("next_page"): # additional pages are fetched in the background result = fetch_collection.delay( result["next_page"], channel_id=obj.pk, max_pages=settings.FEDERATION_COLLECTION_MAX_PAGES - 1, is_page=True, ) fetch_obj.object = obj fetch_obj.status = "finished" fetch_obj.fetch_date = timezone.now() return fetch_obj.save(update_fields=[ "fetch_date", "status", "object_id", "object_content_type" ])
def test_get_session(): expected = session.get_user_agent() assert session.get_session().headers["User-Agent"] == expected