def dispatch_extra(self, request, *args, **kwargs): _ = gettext context = self.get_context_data() # fallback to intentions if no request_intentions # remove "domain", "sl" context["intentions"] = set( self.object.extra.get("request_intentions", self.object.extra.get("intentions", []))) # sl or domain should NOT be allowed for token updates # sl is only for clients, domain only initial and should be removed context["intentions"].difference_update({"domain", "sl"}) context["action"] = "update" context["uc"] = self.object.usercomponent rreferrer = request.POST.get("referrer", None) if rreferrer: context["referrer"] = merge_get_url(rreferrer) if not get_settings_func( "SPIDER_URL_VALIDATOR", "spkcspider.apps.spider.functions.validate_url_default")( context["referrer"], self): context["action"] = "referrer_invalid" # for donotact if self.object.referrer and self.object.referrer.url == rreferrer: rreferrer = None else: rreferrer = self.object.extra.get("request_referrer", None) if rreferrer: context["referrer"] = merge_get_url(rreferrer) if not get_settings_func( "SPIDER_URL_VALIDATOR", "spkcspider.apps.spider.functions.validate_url_default" )(context["referrer"], self): return HttpResponse(status=400, content=_('Insecure url: %(url)s') % {"url": context["referrer"]}) elif self.object.referrer: context["referrer"] = self.object.referrer.url else: context["referrer"] = "" context["ids"] = self.object.usercomponent.contents.values_list( "id", flat=True) context["ids"] = set(context["ids"]) # if requested referrer is available DO delete invalid and DO care ret = self.handle_referrer_request(context, self.object, dontact=not rreferrer, no_oldtoken=True) if isinstance(ret, HttpResponseRedirect): if context.get("post_success", False): messages.success(request, _("Intention update successful")) else: messages.error(request, _("Intention update failed")) return HttpResponseRedirect( self.get_redirect_url(context["sanitized_GET"])) return ret
def check(self, url=None): if not url or url.startswith(self.url): merged_url, headers = self.merge_and_headers( self.url, raw="embed", search="\x1etype=PostBox\x1e" ) response = self.session.get( merge_get_url( merged_url, raw="embed", search="\x1etype=PostBox\x1e" ), headers=headers, ) response.raise_for_status() graph = Graph() graph.parse(data=response.content, format="turtle") for page in get_pages(graph): with self.session.get( merge_get_url(merged_url, page=page), headers=headers ) as response: response.raise_for_status() graph.parse(data=response.content, format="turtle") else: response = self.session.get( merge_get_url(url, raw="embed", search="\x1etype=PostBox\x1e") ) response.raise_for_status() graph = Graph() graph.parse(data=response.content, format="turtle") for page in get_pages(graph): response = self.session.get( merge_get_url( self.url, raw="embed", search="\x1etype=PostBox\x1e", page=page, ) ) response.raise_for_status() graph.parse(data=response.content, format="turtle") if not url or url.startswith(self.url): result = self.simple_check( graph, url=url, checker=self.attestation_checker, auto_add=True ) key_hashes = set(map(lambda x: x[0], result["key_list"])) if self.hash_key_public not in key_hashes: self.state = AttestationResult.error raise CheckError("Key is not part of chain") self.state = result["result"] self.client_list = result["key_list"] return result else: return self.simple_check( graph, url=url, checker=self.attestation_checker, auto_add=True )
def retrieve_missing(cls, graph_or_url, filters=None, x_token=None, timeout=60, session=None): if not session: session = requests.Session() if isinstance(graph_or_url, "str"): assert filters is None, "No filters specified and url given" graph = cls.retrieve_filtered_graph(graph_or_url, filters, x_token=x_token, timeout=timeout, session=session) else: graph = graph_or_url retrieved_url, missing_pages = get_pages(graph) for page in missing_pages: with session.get(merge_get_url(retrieved_url, page=page), headers={"X-TOKEN": x_token or ""}, timeout=timeout) as response: response.raise_for_status() graph.parse(data=response.content, format="turtle") return graph, retrieved_url
def action_view(context): token = getattr(context["request"], "auth_token", None) if token: token = token.token url2 = merge_get_url(context["hostpart"] + context["request"].path, token=token) return Literal(url2, datatype=XSD.anyURI)
def handle_referrer(self): _ = gettext if (self.request.user != self.usercomponent.user and not self.request.auth_token): if self.request.user.is_authenticated: return self.handle_no_permission() return HttpResponseRedirect(redirect_to="{}?{}={}".format( self.get_login_url(), REDIRECT_FIELD_NAME, quote_plus( merge_get_url(self.request.build_absolute_uri(), token=None)))) context = self.get_context_data() context["action"] = "create" context["intentions"] = set(self.request.GET.getlist("intention")) if "referrer" in self.request.POST: context["referrer"] = merge_get_url(self.request.POST["referrer"]) if not get_settings_func( "SPIDER_URL_VALIDATOR", "spkcspider.apps.spider.functions.validate_url_default")( context["referrer"], self): context["action"] = "referrer_invalid" else: context["referrer"] = merge_get_url(self.request.GET["referrer"]) if not get_settings_func( "SPIDER_URL_VALIDATOR", "spkcspider.apps.spider.functions.validate_url_default")( context["referrer"], self): return HttpResponse(status=400, content=_('Insecure url: %(url)s') % {"url": context["referrer"]}) context["payload"] = self.request.GET.get("payload", None) token = self.request.auth_token if not token: token = AuthToken(usercomponent=self.usercomponent, extra={ "strength": 10, "taint": False }) if "domain" in context["intentions"]: return self.handle_domain_auth(context, token) else: context["ids"] = set(self.object_list.values_list("id", flat=True)) context["search"] = set(self.request.POST.getlist("search")) return self.handle_referrer_request(context, token)
def retrieve_missing(self, graph, url=None, timeout=60): retrieved_url, missing_pages = get_pages(graph) if not url: url = retrieved_url if self.url and url.startswith(self.url): merged_url, headers = self.merge_and_headers(url) else: merged_url = url headers = {} for page in missing_pages: with self.session.get(merge_get_url(merged_url, page=page), headers=headers, timeout=timeout) as response: response.raise_for_status() graph.parse(data=response.content, format="turtle") return retrieved_url
def retrieve_filtered_graph(self, url=None, timeout=60, out=None): if not url or url == self.url: merged_url, headers = self.merge_and_headers( self.url, raw="embed", search="\x1etype=PostBox\x1e") else: merged_url = merge_get_url(url, raw="embed", search="\x1etype=PostBox\x1e") headers = {} response = self.session.get(merged_url, headers=headers, timeout=timeout) response.raise_for_status() if out is None: graph = Graph() else: graph = out graph.parse(data=response.content, format="turtle") return graph
def clean(self): ret = super().clean() if not ret.get("url", None) and not ret.get("dvfile", None): raise forms.ValidationError(_('Require either url or dvfile'), code="missing_parameter") if ret.get("url", None): self.cleaned_data["url"] = merge_get_url(self.cleaned_data["url"], raw="embed") url = self.cleaned_data["url"] if not get_settings_func( "SPIDER_URL_VALIDATOR", "spkcspider.apps.spider.functions.validate_url_default")( url): self.add_error( "url", forms.ValidationError(_('invalid url: %(url)s'), params={"url": url}, code="invalid_url")) return ret return ret
def _send_dest(self, aes_key, fetch_url, dest): dest_url = merge_get_url(dest, raw="embed", search="\x1etype=PostBox\x1e") response_dest = self.session.get(dest_url, timeout=60) try: response_dest.raise_for_status() g_dest = Graph() g_dest.parse(data=response_dest.content, format="turtle") self.retrieve_missing(g_dest, dest_url) except Exception as exc: raise DestException("postbox retrieval failed") from exc dest_postboxes = get_postboxes(g_dest) if len(dest_postboxes) != 1: raise DestException("No postbox found/more than one found") dest_postbox_url, dest_options = next(iter(dest_postboxes.items())) webref_url = replace_action(dest_postbox_url, "push_webref/") attestation = dest_options["attestation"] bdomain = dest_postbox_url.split("?", 1)[0] result_dest, errors, dest_keys = self.attestation_checker.check( bdomain, map(lambda x: (x["key"], x["signature"]), dest_options["signatures"].values()), attestation=attestation, algo=dest_options["hash_algorithm"], auto_add=True) if result_dest == AttestationResult.domain_unknown: logger.info("add domain: %s", bdomain) self.attestation_checker.attestation.add( bdomain, dest_keys, algo=dest_options["hash_algorithm"], embed=True) elif result_dest == AttestationResult.error: if len(dest_keys) == 0: raise DestException("No keys found") raise DestSecurityException("dest contains invalid keys", errors) dest_key_list = {} for k in dest_keys: enc = k[1].encrypt( aes_key, padding.OAEP( mgf=padding.MGF1(algorithm=dest_options["hash_algorithm"]), algorithm=dest_options["hash_algorithm"], label=None)) # encrypt decryption key dest_key_list["%s=%s" % (dest_options["hash_algorithm"].name, k[0].hex())] = base64.b64encode(enc).decode("ascii") try: response_dest = self.session.post(webref_url, data={ "url": fetch_url, "key_list": json.dumps(dest_key_list) }, timeout=60) response_dest.raise_for_status() except Exception as exc: raise DestException("post webref failed") from exc
def merge_and_headers(self, _url, **kwargs): if self.use_get_token: return merge_get_url(_url, token=self.token, **kwargs), {} return (merge_get_url(_url, **kwargs), {"X-TOKEN": self.token or ""})
def action_view(argv, priv_key, pem_public, own_url, session, g_message): if argv.message_id is not None: assert isinstance(argv.message_id, int) result = list(g_message.query( """ SELECT DISTINCT ?base ?hash_algorithm ?type WHERE { ?base a <https://spkcspider.net/static/schemes/spkcgraph#spkc:Content> ; spkc:type ?type ; spkc:properties ?propalg , ?propid . ?propid spkc:name ?idname ; spkc:value ?idvalue . ?propalg spkc:name ?algname ; spkc:value ?hash_algorithm . } """, # noqa E501 initNs={"spkc": spkcgraph}, initBindings={ "idvalue": Literal(argv.message_id), "algname": Literal( "hash_algorithm", datatype=XSD.string ), "idname": Literal( "id", datatype=XSD.string ), } )) if not result or result[0].type.toPython() not in { "WebReference", "MessageContent" }: parser.exit(0, "message not found\n") pub_key_hasher = getattr( hashes, result[0].hash_algorithm.upper() )() digest = hashes.Hash(pub_key_hasher, backend=default_backend()) digest.update(pem_public) pub_key_hashalg = "%s=%s" % ( pub_key_hasher.name, digest.finalize().hex() ) retrieve_url = merge_get_url( replace_action( result[0].base, "message/" ) ) data = { "max_size": argv.max } if argv.action == "view": data["keyhash"] = pub_key_hashalg response = session.post( retrieve_url, stream=True, headers={ "X-TOKEN": argv.token }, data=data ) if not response.ok: logger.info("Message retrieval failed: %s", response.text) parser.exit(0, "message could not be fetched\n") key_list = json.loads(response.headers["X-KEYLIST"]) key = key_list.get(pub_key_hashalg, None) if not key: parser.exit(0, "message not for me\n") decrypted_key = priv_key.decrypt( base64.b64decode(key), padding.OAEP( mgf=padding.MGF1(algorithm=argv.src_hash_algo), algorithm=argv.src_hash_algo, label=None ) ) headblock = b"" fdecryptor = None eparser = emailparser.BytesFeedParser(policy=policy.default) headers = None for chunk in response.iter_content(chunk_size=256): blob = None if not fdecryptor: headblock = b"%b%b" % (headblock, chunk) if b"\0" in headblock: nonce, headblock = headblock.split(b"\0", 1) nonce = base64.b64decode(nonce) fdecryptor = Cipher( algorithms.AES(decrypted_key), modes.GCM(nonce), backend=default_backend() ).decryptor() blob = fdecryptor.update(headblock[:-16]) headblock = headblock[-16:] else: continue else: blob = fdecryptor.update( b"%b%b" % (headblock, chunk[:-16]) ) headblock = chunk[-16:] if not headers: if b"\n\n" not in blob: eparser.feed(blob) continue headersrest, blob = blob.split(b"\n\n", 1) eparser.feed(headersrest) headers = eparser.close() # check what to do t = headers.get("SPKC-Type", MessageType.email) if t == MessageType.email: argv.file.write(headers.as_bytes( unixfrom=True, policy=policy.SMTP )) argv.file.write(blob) argv.file.write(fdecryptor.finalize_with_tag(headblock)) else: queried_webrefs = {} queried_messages = {} for i in g_message.query( """ SELECT DISTINCT ?base ?idvalue ?namevalue ?type WHERE { ?base a <https://spkcspider.net/static/schemes/spkcgraph#spkc:Content> ; spkc:type ?type ; spkc:properties ?propname , ?propid . ?propid spkc:name ?idname ; spkc:value ?idvalue . ?propname spkc:name ?namename ; spkc:value ?namevalue . } """, # noqa E501 initNs={"spkc": spkcgraph}, initBindings={ "idname": Literal( "id", datatype=XSD.string ), "namename": Literal( "name", datatype=XSD.string ), } ): if i.type.toPython() == "WebReference": queried = queried_webrefs elif i.type.toPython() == "MessageContent": queried = queried_messages else: continue queried.setdefault(str(i.base), {}) queried[str(i.base)]["id"] = i.idvalue queried[str(i.base)]["name"] = i.namevalue print("Received Messages:") for i in sorted(queried_webrefs.values(), key=lambda x: x["id"]): print(i["id"], i["name"]) print("Own Messages:") for i in sorted(queried_messages.values(), key=lambda x: x["id"]): print(i["id"], i["name"])
def handle_referrer_request(self, context, token, keep=False, dontact=False, no_oldtoken=False): """ no_oldtoken: don't use old token for calculating: old_ids, old_search (performance and manual old_* possible) dontact: dont act if probing results fails """ _ = gettext context.setdefault("action", "create") context.setdefault("model", self.model) context.setdefault("payload", None) context.setdefault("ids", set()) context.setdefault("filter", set()) context.setdefault("old_ids", set()) context.setdefault("old_search", set()) context["is_serverless"] = "sl" in context["intentions"] if keep is True: context["ids"].update(token.extra.get("ids", [])) context["search"].update(token.extra.get("filter", [])) action = self.request.POST.get("action", None) if context["action"].endswith("invalid"): action = context["action"] if action == "confirm": newtoken = None # if persist try to find old token if "persist" in context["intentions"]: oldtoken = AuthToken.objects.filter( Q(persist__gte=0, usercomponent=token.usercomponent), referrer__url=context["referrer"]).first() if oldtoken: newtoken = token token = oldtoken # either reuse persistent token with auth token tokenstring # or just reuse auth token if newtoken: # steal token value token.token = newtoken.token # set to zero as prot_strength can elevate perms token.extra["taint"] = False token.extra["prot_strength"] = 0 token.extra["intentions"] = list(context["intentions"]) token.extra.pop("request_referrer", None) token.extra.pop("request_intentions", None) token.extra.pop("request_search", None) if not self.clean_refer_intentions(context, token): return HttpResponseRedirect(redirect_to=merge_get_url( context["referrer"], error="intentions_incorrect")) token.extra["search"] = list(context["search"]) if "live" in context["intentions"]: token.extra.pop("ids", None) else: token.extra["ids"] = list(context["ids"]) token.referrer = ReferrerObject.objects.get_or_create( url=context["referrer"])[0] # after cleanup, save try: with transaction.atomic(): # must be done here, elsewise other token can (unlikely) # take token as it is free for a short time, better be safe if newtoken: newtoken.delete() token.save() except TokenCreationError: logger.exception("Token creation failed") return HttpResponseServerError( _("Token creation failed, try again")) if context["is_serverless"]: context["post_success"] = True ret = self.refer_with_get(context, token) else: context["post_success"] = False ret = self.refer_with_post(context, token) if dontact: return ret if not context["post_success"]: if newtoken: logger.warning("Updating persisting token failed") else: token.delete() return ret elif action == "cancel": if not dontact: token.delete() return HttpResponseRedirect( redirect_to=merge_get_url(context["referrer"], status="canceled", payload=context["payload"])) else: if (no_oldtoken and "persist" in context["intentions"]): oldtoken = AuthToken.objects.filter( Q(persist__gte=0, usercomponent=token.usercomponent), referrer__url=context["referrer"], ).first() if oldtoken: context["old_ids"].update(oldtoken.extra.get("ids", [])) context["old_search"].update( oldtoken.extra.get("filter", [])) if not self.clean_refer_intentions(context, token): return HttpResponse(status=400, content=_('Error: intentions incorrect')) context["object_list"] = context["model"].objects.filter( id__in=context["ids"]) # remove other media context["media"] = Media( css={ 'all': [ 'node_modules/choices.js/public/assets/styles/choices%s.css' % _extra # noqa:E501 ] }, js=[ 'node_modules/choices.js/public/assets/scripts/choices%s.js' % _extra, # noqa: E501 ]) return self.response_class( request=self.request, template=self.get_referrer_template_names(), context=context, using=self.template_engine, content_type=self.content_type)
def simple_check(url_or_graph, session=None, checker=None, auto_add=False, token=None): if isinstance(url_or_graph, Graph): graph = url_or_graph retrieve_url, pages = get_pages(graph) url = retrieve_url.split("?", 1)[0] else: url = url_or_graph retrieve_url = merge_get_url(url_or_graph, raw="embed", search="\x1etype=PostBox\x1e") if not session: session = requests.Session() response = session.get(retrieve_url, headers={"X-TOKEN": token or ""}) response.raise_for_status() graph = Graph() graph.parse(data=response.content, format="turtle") pages = get_pages(graph)[1] for page in pages: response = session.get(merge_get_url(retrieve_url, page=page), headers={"X-TOKEN": token or ""}) response.raise_for_status() graph.parse(data=response.content, format="turtle") postboxes = get_postboxes(graph) if len(postboxes) != 1: raise CheckError("No postbox found/more than one found") postbox, options = next(iter(postboxes.items())) if not isinstance(options.get("hash_algorithm"), hashes.HashAlgorithm): raise CheckError("Hash algorithm not found") if not isinstance(options.get("attestation"), bytes): raise CheckError("Attestation not found/wrong type") attestation, errors, key_list = AttestationChecker.check_signatures( map(lambda x: (x["key"], x["signature"]), options["signatures"].values()), attestation=options["attestation"], algo=options["hash_algorithm"]) if errors: raise CheckError("Missmatch attestation with signatures", errored=errors, key_list=key_list, attestation=attestation) if not checker: return { "result": AttestationResult.success, "errors": [], "key_list": key_list, **options } url = url.split("?", 1)[0] ret = checker.check(url, key_list, algo=options["hash_algorithm"], auto_add=auto_add, embed=True, attestation=attestation) if ret[0] == AttestationResult.error: raise CheckError("Checker validation failed", errored=ret[1], key_list=ret[2], attestation=ret[0]) return { "result": ret[0], "errors": ret[1], "key_list": key_list, **options }
def test_token(self, minstrength=0, force_token=False, taint=False): expire = timezone.now()-self.usercomponent.token_duration no_token = not force_token and self.usercomponent.required_passes == 0 ptype = ProtectionType.access_control if minstrength >= 4: no_token = False ptype = ProtectionType.authentication # delete old token, so no confusion happen self.remove_old_tokens(expire) # only valid tokens here tokenstring = self.request.GET.get("token", None) token = None if tokenstring: # find by tokenstring token = self.usercomponent.authtokens.filter( token=tokenstring ).first() elif self.request.session.session_key: # use session_key token = self.usercomponent.authtokens.filter( session_key=self.request.session.session_key ).first() elif not no_token: # generate session key if it not exist and token is required self.request.session.cycle_key() if token and token.extra.get("prot_strength", 0) >= minstrength: self.request.token_expires = \ token.created+self.usercomponent.token_duration # case will never enter # if not token.session_key and "token" not in self.request.GET: # return self.replace_token() if ( token.extra["prot_strength"] >= 4 and not token.extra.get("taint", False) ): self.request.is_special_user = True self.request.is_owner = True self.request.auth_token = token # after, for having access to token self.usercomponent.auth( request=self.request, scope=self.scope, ptype=ptype, side_effect=True ) return True # if result is impossible and token invalid try to login if minstrength >= 4 and not self.usercomponent.can_auth: # auth won't work for logged in users if self.request.user.is_authenticated: return False # remove token and redirect # login_url can be also on a different host => merge_get_url target = "{}?{}={}".format( self.get_login_url(), REDIRECT_FIELD_NAME, quote_plus( merge_get_url( self.request.build_absolute_uri(), token=None ) ) ) return HttpResponseRedirect(redirect_to=target) protection_codes = None if "protection" in self.request.GET: protection_codes = self.request.GET.getlist("protection") # execute protections for side effects even no_token self.request.protections = self.usercomponent.auth( request=self.request, scope=self.scope, protection_codes=protection_codes, ptype=ptype ) if ( type(self.request.protections) is int and # because: False==0 self.request.protections >= minstrength ): # generate only tokens if required if no_token: return True token = self.create_token( extra={ "strength": self.usercomponent.strength, "prot_strength": self.request.protections, "taint": taint } ) if ( token.extra["prot_strength"] >= 4 and not token.extra.get("taint", False) ): self.request.is_special_user = True self.request.is_owner = True self.request.token_expires = \ token.created+self.usercomponent.token_duration self.request.auth_token = token if "token" in self.request.GET: return self.replace_token() return True return False
def refer_with_get(self, context, token): return HttpResponseRedirect(redirect_to=merge_get_url( context["referrer"], token=token.token, payload=context["payload"]))
def render_to_response(self, context): if context["scope"] != "export" and "raw" not in self.request.GET: return super().render_to_response(context) session_dict = { "request": self.request, "context": context, "scope": context["scope"], "uc": self.usercomponent, "hostpart": context["hostpart"], "sourceref": URIRef("%s%s" % (context["hostpart"], self.request.path)) } g = Graph() g.namespace_manager.bind("spkc", spkcgraph, replace=True) embed = False if (context["scope"] == "export" or self.request.GET.get("raw", "") == "embed"): embed = True if context["object_list"]: p = paginate_stream( context["object_list"], getattr(settings, "SPIDER_SERIALIZED_PER_PAGE", settings.SPIDER_OBJECTS_PER_PAGE), settings.SPIDER_MAX_EMBED_DEPTH) else: # no content, pagination works here only this way p = paginate_stream( UserComponent.objects.filter(pk=self.usercomponent.pk), 1, 1) context["object_list"] page = 1 try: page = int(self.request.GET.get("page", "1")) except Exception: pass if hasattr(self.request, "token_expires"): session_dict["expires"] = self.request.token_expires.strftime( "%a, %d %b %Y %H:%M:%S %z") if page <= 1: # "expires" (string) different from "token_expires" (datetime) if session_dict.get("expires"): add_property(g, "token_expires", ob=session_dict["request"], ref=session_dict["sourceref"]) assert not context.get("machine_variants") or self.request.is_owner for machinec in context.get("machine_variants", []): g.add((session_dict["sourceref"], spkcgraph["create:name"], Literal(machinec, datatype=XSD.string))) g.add((session_dict["sourceref"], spkcgraph["scope"], Literal(context["scope"], datatype=XSD.string))) g.add((session_dict["sourceref"], spkcgraph["type"], Literal("Component", datatype=XSD.string))) g.add((session_dict["sourceref"], spkcgraph["strength"], Literal(self.usercomponent.strength, datatype=XSD.integer))) if context["referrer"]: g.add((session_dict["sourceref"], spkcgraph["referrer"], Literal(context["referrer"], datatype=XSD.anyURI))) token = getattr(session_dict["request"], "auth_token", None) if token: token = token.token g.add((session_dict["sourceref"], spkcgraph["action:view"], Literal(merge_get_url(str(session_dict["sourceref"]), token=token), datatype=XSD.anyURI))) if context["token_strength"]: add_property(g, "token_strength", ref=session_dict["sourceref"], literal=context["token_strength"], datatype=XSD.integer) add_property(g, "intentions", ref=session_dict["sourceref"], literal=context["intentions"], datatype=XSD.string, iterate=True) serialize_stream(g, p, session_dict, page=page, embed=embed) ret = HttpResponse(g.serialize(format="turtle"), content_type="text/turtle;charset=utf-8") if session_dict.get("expires", None): ret['X-Token-Expires'] = session_dict["expires"] # allow cors requests for raw ret["Access-Control-Allow-Origin"] = "*" return ret
def action_send(argv, priv_key, pub_key_hash, src_keys, session, g_src): postboxes = get_postboxes(g_src) if len(postboxes) != 1: parser.exit(1, "Source cannot create messages, logged in?") component_uriref, src_options = next(postboxes.items()) dest_url = merge_get_url( argv.dest, raw="embed", search="\x1etype=PostBox\x1e" ) response_dest = session.get(dest_url) if not response_dest.ok: logger.info("Dest returned error: %s", response_dest.text) parser.exit(1, "retrieval failed, invalid url?\n") g_dest = Graph() g_dest.parse(data=response_dest.content, format="turtle") for page in get_pages(g_dest)[1]: with session.get( merge_get_url(dest_url, page=page) ) as response: response.raise_for_status() g_dest.parse(data=response.content, format="turtle") dest_postboxes = get_postboxes(g_dest) if len(dest_postboxes) != 1: parser.exit(1, "Dest has no/too many postboxes") dest_postbox_url, dest_options = next(dest_postboxes.items()) webref_url = replace_action(dest_postbox_url, "push_webref/") dest_hash_algo = dest_options["hash_algorithm"] attestation = dest_options["attestation"] bdomain = dest_postbox_url.split("?", 1)[0] result_dest, _, dest_keys = argv.attestation.check( bdomain, map( lambda x: (x["key"], x["signature"]), dest_options["signatures"].values() ), attestation=attestation, algo=dest_hash_algo ) if result_dest == AttestationResult.domain_unknown: logger.info("add domain: %s", bdomain) argv.attestation.add( bdomain, dest_keys, algo=dest_hash_algo ) elif result_dest == AttestationResult.error: logger.critical("Dest base url contains invalid keys.") parser.exit(1, "dest contains invalid keys\n") # 256 bit aes_key = os.urandom(32) nonce = os.urandom(13) fencryptor = Cipher( algorithms.AES(aes_key), modes.GCM(nonce), backend=default_backend() ).encryptor() src_key_list = {} dest_key_list = {} if argv.stealth: pass elif src_options["shared"]: for k in src_keys: enc = k[1].encrypt( aes_key, padding.OAEP( mgf=padding.MGF1(algorithm=argv.src_hash_algo), algorithm=argv.src_hash_algo, label=None ) ) # encrypt decryption key src_key_list[ "%s=%s" % (argv.src_hash_algo.name, k[0].hex()) ] = base64.b64encode(enc).decode("ascii") else: enc = priv_key.public_key().encrypt( aes_key, padding.OAEP( mgf=padding.MGF1(algorithm=argv.src_hash_algo), algorithm=argv.src_hash_algo, label=None ) ) # encrypt decryption key src_key_list[ "%s=%s" % (argv.src_hash_algo.name, pub_key_hash) ] = base64.b64encode(enc).decode("ascii") for k in dest_keys: enc = k[1].encrypt( aes_key, padding.OAEP( mgf=padding.MGF1(algorithm=dest_hash_algo), algorithm=dest_hash_algo, label=None ) ) # encrypt decryption key dest_key_list[ "%s=%s" % (dest_hash_algo.name, k[0].hex()) ] = base64.b64encode(enc).decode("ascii") headers = b"SPKC-Type: %b\n" % MessageType.file # remove raw as we parse html message_create_url = merge_get_url( replace_action(str(component_uriref), "add/MessageContent/"), raw=None ) response = session.get( message_create_url, headers={ "X-TOKEN": argv.token } ) if not response.ok: logger.error("retrieval csrftoken failed: %s", response.text) parser.exit(1, "retrieval csrftoken failed: %s" % response.text) g = Graph() g.parse(data=response.content, format="html") csrftoken = list(g.objects(predicate=spkcgraph["csrftoken"]))[0] # create message object response = session.post( message_create_url, data={ "own_hash": pub_key_hash, "key_list": json.dumps(src_key_list), "amount_tokens": 1 }, headers={ "X-CSRFToken": csrftoken, "X-TOKEN": argv.token # only for src }, files={ "encrypted_content": EncryptedFile( fencryptor, argv.file, nonce, headers ) } ) if not response.ok or message_create_url == response.url: logger.error("Message creation failed: %s", response.text) parser.exit(1, "Message creation failed: %s" % response.text) g = Graph() g.parse(data=response.content, format="html") fetch_url = list(map(lambda x: x.value, g.query( """ SELECT ?value WHERE { ?property spkc:name ?name ; spkc:value ?value . } """, initNs={"spkc": spkcgraph}, initBindings={ "name": Literal( "fetch_url", datatype=XSD.string ) } ))) tokens = list(map(lambda x: x.value, g.query( """ SELECT ?value WHERE { ?property spkc:name ?name ; spkc:value ?value . } """, initNs={"spkc": spkcgraph}, initBindings={ "name": Literal( "tokens", datatype=XSD.string ) } ))) if not fetch_url or not tokens: logger.error("Message creation failed: %s", response.text) parser.exit(1, "Message creation failed") # extract url response_dest = session.post( webref_url, data={ "url": merge_get_url(fetch_url[0], token=str(tokens[0])), "key_list": json.dumps(dest_key_list) } ) if not response_dest.ok: logger.error("Sending message failed: %s", response_dest.text) parser.exit(1, "Sending message failed")
def main(argv): argv = parser.parse_args(argv) argv.attestation = AttestationChecker(argv.attestation) if not os.path.exists(argv.key): parser.exit(1, "key does not exist\n") match = static_token_matcher.match(argv.url) if not match: parser.exit(1, "invalid url scheme\n") if argv.verbose >= 2: logging.basicConfig(level=logging.DEBUG) elif argv.verbose >= 1: logging.basicConfig(level=logging.INFO) else: logging.basicConfig(level=logging.WARNING) access = match.groupdict()["access"] if ( argv.action == "check" and access not in {"view", "list"} ): parser.exit(1, "url doesn't match action\n") if ( argv.action in {"view", "peek"} and access not in {"view", "list"} ): parser.exit(1, "url doesn't match action\n") if argv.action == "send": match2 = static_token_matcher.match(argv.dest) if not match2: parser.exit(1, "invalid url scheme\n") access2 = match2.groupdict()["access"] if ( access not in {"list", "view", "push_webref"} or access2 not in {"list", "view", "push_webref"} ): parser.exit(1, "url doesn't match action\n") argv.access_type = access with open(argv.key, "rb") as f: priv_key = load_priv_key(f.read())[0] if not priv_key: parser.exit(1, "invalid key: %s\n" % argv.key) pem_public = priv_key.public_key().public_bytes( encoding=serialization.Encoding.PEM, format=serialization.PublicFormat.SubjectPublicKeyInfo ).strip() with requests.Session() as s: if access == "list": own_url = merge_get_url( argv.url, raw="embed", search="\x1etype=PostBox\x1e" ) else: own_url = merge_get_url(argv.url, raw="embed") response = s.get(own_url, headers={ "X-TOKEN": argv.token }) if not response.ok: logger.info("Url returned error: %s\n", response.text) parser.exit(1, "retrieval failed, invalid url?\n") g = Graph() g.parse(data=response.content, format="turtle") for page in get_pages(g)[1]: with s.get( merge_get_url(own_url, page=page), headers={ "X-TOKEN": argv.token } ) as response: response.raise_for_status() g.parse(data=response.content, format="turtle") src = {} for i in g.query( """ SELECT ?postbox ?postbox_value ?key_name ?key_value WHERE { ?postbox spkc:properties ?property . ?property spkc:name ?postbox_name ; spkc:value ?postbox_value . ?postbox_value spkc:properties ?key_base_prop . ?key_base_prop spkc:name ?key_name ; spkc:value ?key_value . } """, initNs={"spkc": spkcgraph}, initBindings={ "postbox_name": Literal( "signatures", datatype=XSD.string ) } ): argv.postbox_base = i.postbox src.setdefault(str(i.postbox_value), {}) src[str(i.postbox_value)][str(i.key_name)] = i.key_value # algorithm for hashing argv.src_hash_algo = getattr( hashes, next(iter(src.values()))["hash_algorithm"].upper() )() digest = hashes.Hash(argv.src_hash_algo, backend=default_backend()) digest.update(pem_public) pub_key_hash = digest.finalize().hex() argv.attestation.add( own_url.split("?", 1)[0], [pub_key_hash], argv.src_hash_algo ) src_keys = None if argv.action != "check": result_own, errored, src_keys = argv.attestation.check( own_url.split("?", 1)[0], map( lambda x: (x["key"], x["signature"]), src.values() ), algo=argv.src_hash_algo, auto_add=True ) if result_own == AttestationResult.domain_unknown: logger.critical( "home url unknown, should not happen" ) parser.exit(1, "invalid home url\n") elif result_own != AttestationResult.success: logger.critical( "Home base url contains invalid keys, hacked?" ) parser.exit(1, "invalid keys\n") # check own domain if argv.action == "send": return action_send( argv, priv_key, pub_key_hash, src_keys, s, g ) elif argv.action in {"view", "peek"}: return action_view(argv, priv_key, pem_public, own_url, s, g) elif argv.action == "check": ret = action_check(argv, priv_key, pub_key_hash, s, g) if ret is not True: parser.exit(2, "check failed: %s\n" % ret) print("check successful")
def send(self, inp, receivers, headers=b"\n", mode=SendMethod.shared, aes_key=None): if not self.ok: raise NotReady() if isinstance(receivers, str): receivers = [receivers] if isinstance(inp, (bytes, memoryview)): inp = io.BytesIO(bytes(inp)) elif isinstance(inp, str): inp = io.BytesIO(inp.encode("utf8")) # 256 bit if not aes_key: aes_key = os.urandom(32) assert len(aes_key) == 32 nonce = os.urandom(13) fencryptor = Cipher(algorithms.AES(aes_key), modes.GCM(nonce), backend=default_backend()).encryptor() src_key_list = {} if mode == SendMethod.stealth: pass elif mode == SendMethod.private: enc = self.priv_key.public_key().encrypt( aes_key, padding.OAEP(mgf=padding.MGF1(algorithm=self.hash_algo), algorithm=self.hash_algo, label=None)) # encrypt decryption key src_key_list["%s=%s" % (self.hash_algo.name, self.pub_key_hash )] = base64.b64encode(enc).decode("ascii") elif mode == SendMethod.shared: for k in self.client_list: enc = k[1].encrypt( aes_key, padding.OAEP(mgf=padding.MGF1(algorithm=self.hash_algo), algorithm=self.hash_algo, label=None)) # encrypt decryption key src_key_list["%s=%s" % (self.hash_algo.name, k[0].hex() )] = base64.b64encode(enc).decode("ascii") else: raise NotImplementedError() # remove raw as we parse html message_create_url, src_headers = self.merge_and_headers( replace_action(self.component_url, "add/MessageContent/"), raw=None) response = self.session.get(message_create_url, headers=src_headers) try: response.raise_for_status() except Exception as exc: raise SrcException("retrieval csrftoken failed", response.text) from exc g = Graph() g.parse(data=response.content, format="html") csrftoken = list(g.objects(predicate=spkcgraph["csrftoken"]))[0] # create message object response = self.session.post( message_create_url, data={ "own_hash": self.hash_key_public, "key_list": json.dumps(src_key_list), "amount_tokens": len(receivers) }, headers={ "X-CSRFToken": csrftoken, **src_headers # only for src }, files={ "encrypted_content": EncryptedFile(fencryptor, inp, nonce, headers) }) try: response.raise_for_status() except Exception as exc: raise SrcException("Message creation failed", response.text) from exc if message_create_url == response.url: raise SrcException("Message creation failed", response.text) g = Graph() g.parse(data=response.content, format="html") fetch_url = list( map( lambda x: x.value, g.query(""" SELECT ?value WHERE { ?property spkc:name ?name ; spkc:value ?value . } """, initNs={"spkc": spkcgraph}, initBindings={ "name": Literal("fetch_url", datatype=XSD.string) }))) tokens = list( map( lambda x: x.value, g.query(""" SELECT ?value WHERE { ?property spkc:name ?name ; spkc:value ?value . } """, initNs={"spkc": spkcgraph}, initBindings={ "name": Literal("tokens", datatype=XSD.string) }))) if not fetch_url or not tokens: raise SrcException("Message creation failed", response.text) fetch_url = fetch_url[0].toPython() exceptions = [] final_fetch_urls = [] for receiver, token in zip(receivers, tokens): furl = merge_get_url(fetch_url, token=token.toPython()) try: self._send_dest(aes_key, furl, receiver) final_fetch_urls.append(furl) except Exception as exc: exceptions.append(exc) # for autoremoval simulate access self.session.get(furl) return exceptions, final_fetch_urls, aes_key
def access_message(self, **kwargs): cached_content = self.associated.attachedfiles.filter( name="cache").first() if not cached_content: cached_content = AttachedFile(content=self.associated, unique=True, name="cache") max_size = kwargs["request"].POST.get("max_size") or math.inf if isinstance(max_size, str): max_size = int(max_size) max_configured_size = \ self.associated.attached_to_content.content.free_data.get( "max_receive_size", math.inf ) if max_configured_size is None: max_configured_size = math.inf max_size = min(self.associated.user_info.get_free_space("remote"), max_configured_size, max_size) params, inline_domain = get_requests_params(self.quota_data["url"]) fp = None if inline_domain: try: resp = Client().post( self.quota_data["url"], { "max_size": (max_size if max_size != math.inf else "") }, follow=True, secure=True, Connection="close", Referer=merge_get_url( "%s%s" % (kwargs["hostpart"], kwargs["request"].path)), SERVER_NAME=inline_domain) if resp.status_code != 200: logging.info("file retrieval failed: \"%s\" failed", self.url) return HttpResponse("other error", status=502) c_length = resp.get("content-length", math.inf) if isinstance(c_length, str): c_length = int(c_length) if max_size < c_length: return HttpResponse("Too big/not specified", status=413) written_size = 0 fp = NamedTemporaryFile(suffix='.upload', dir=settings.FILE_UPLOAD_TEMP_DIR) for chunk in resp: written_size += fp.write(chunk) self.update_used_space(written_size, "remote") # saves object cached_content.file.save("", File(fp)) except ValidationError as exc: del fp logging.info("Quota exceeded", exc_info=exc) return HttpResponse("Quota", status=413) else: try: with requests.post( self.quota_data["url"], data={ "max_size": (max_size if max_size != math.inf else "") }, headers={ "Referer": merge_get_url("%s%s" % (kwargs["hostpart"], kwargs["request"].path)), "Connection": "close" }, stream=True, **params) as resp: resp.raise_for_status() c_length = resp.get("content-length", math.inf) if isinstance(c_length, str): c_length = int(c_length) if max_size < c_length: return HttpResponse("Too big/not specified", status=413) fp = NamedTemporaryFile( suffix='.upload', dir=settings.FILE_UPLOAD_TEMP_DIR) written_size = 0 for chunk in resp.iter_content(fp.DEFAULT_CHUNK_SIZE): written_size += fp.write(chunk) self.update_used_space(written_size, "remote") # saves object cached_content.file.save("", File(fp)) except requests.exceptions.SSLError as exc: logger.info( "referrer: \"%s\" has a broken ssl configuration", self.url, exc_info=exc) return HttpResponse("ssl error", status=502) except ValidationError as exc: del fp logging.info("Quota exceeded", exc_info=exc) return HttpResponse("Quota", status=413) except Exception as exc: del fp logging.info("file retrieval failed: \"%s\" failed", self.url, exc_info=exc) return HttpResponse("other error", status=502) ret = CbFileResponse(cached_content.file.open("rb")) q = models.Q() for i in kwargs["request"].POST.getlist("keyhash"): q |= models.Q(target__info__contains="\x1epubkeyhash=%s" % i) ret.refcopies = self.associated.smarttags.filter(q) # ret["X-TYPE"] = kwargs["rtype"].name ret["X-KEYLIST"] = json.dumps(self.quota_data["key_list"]) ret["X-KEYHASH-ALGO"] = self.free_data["hash_algorithm"] return ret
def receive(self, message_id, outfp=None, access_method=AccessMethod.view, extra_key_hashes=None, max_size=None): if not self.ok: raise NotReady() merged_url, headers = self.merge_and_headers(self.url, raw="embed") response = self.session.get(merge_get_url(self.url, raw="embed"), headers=headers) response.raise_for_status() graph = Graph() graph.parse(data=response.content, format="turtle") self.retrieve_missing(graph, merged_url) result = list( graph.query( """ SELECT DISTINCT ?base ?hash_algorithm ?type WHERE { ?base a <https://spkcspider.net/static/schemes/spkcgraph#spkc:Content> ; spkc:type ?type ; spkc:properties ?propalg , ?propid . ?propid spkc:name ?idname ; spkc:value ?idvalue . ?propalg spkc:name ?algname ; spkc:value ?hash_algorithm . } """, # noqa E501 initNs={"spkc": spkcgraph}, initBindings={ "idvalue": Literal(message_id), "algname": Literal("hash_algorithm", datatype=XSD.string), "idname": Literal("id", datatype=XSD.string), })) if not result or result[0].type.toPython() not in { "WebReference", "MessageContent" }: raise SrcException("No Message") # every object has it's own copy of the hash algorithm, used hash_algo = getattr(hashes, result[0].hash_algorithm.upper())() if not outfp: outfp = tempfile.TempFile() if hash_algo == self.hash_algo: pub_key_hashalg = "%s=%s" % (hash_algo.name, self.hash_key_public.hex()) else: digest = hashes.Hash(hash_algo, backend=default_backend()) digest.update(self.pem_key_public) pub_key_hashalg = "%s=%s" % (hash_algo.name, digest.finalize().hex()) key_hashes = list() if extra_key_hashes: extra_key_hashes = set(extra_key_hashes) extra_key_hashes.discard(pub_key_hashalg) for key in self.client_list: if key[0] in extra_key_hashes: if hash_algo == self.hash_algo: key_hashalg = "%s=%s" % (hash_algo.name, key[0]) else: digest = hashes.Hash(hash_algo, backend=default_backend()) digest.update(key[1]) key_hashalg = "%s=%s" % (hash_algo.name, digest.finalize().hex()) key_hashes.append(key_hashalg) if access_method == AccessMethod.view: key_hashes.append(pub_key_hashalg) retrieve_url, headers = self.merge_and_headers( replace_action( result[0].base, "bypass/" if (access_method == AccessMethod.bypass) else "message/")) data = {} if access_method != AccessMethod.bypass: data.update({"max_size": max_size or "", "keyhash": key_hashes}) response = self.session.post(retrieve_url, stream=True, headers=headers, data=data) try: response.raise_for_status() except Exception as exc: raise DestException("Message retrieval failed", response.text) from exc key_list = json.loads(response.headers["X-KEYLIST"]) key = key_list.get(pub_key_hashalg, None) if not key: raise WrongRecipient("message not for me") decrypted_key = self.priv_key.decrypt( base64.b64decode(key), padding.OAEP(mgf=padding.MGF1(algorithm=hash_algo), algorithm=hash_algo, label=None)) headblock = b"" fdecryptor = None eparser = emailparser.BytesFeedParser(policy=policy.default) headers = None for chunk in response.iter_content(chunk_size=256): blob = None if not fdecryptor: headblock = b"%b%b" % (headblock, chunk) if b"\0" in headblock: nonce, headblock = headblock.split(b"\0", 1) nonce = base64.b64decode(nonce) fdecryptor = Cipher(algorithms.AES(decrypted_key), modes.GCM(nonce), backend=default_backend()).decryptor() blob = fdecryptor.update(headblock[:-16]) headblock = headblock[-16:] else: continue else: blob = fdecryptor.update(b"%b%b" % (headblock, chunk[:-16])) headblock = chunk[-16:] if not headers: if b"\n\n" not in blob: eparser.feed(blob) continue headersrest, blob = blob.split(b"\n\n", 1) eparser.feed(headersrest) headers = eparser.close() # check what to do t = headers.get("SPKC-Type", MessageType.email) if t == MessageType.email: outfp.write( headers.as_bytes(unixfrom=True, policy=policy.SMTP)) outfp.write(blob) outfp.write(fdecryptor.finalize_with_tag(headblock)) return outfp, headers, decrypted_key
def refer_with_post(self, context, token): # application/x-www-form-urlencoded is best here, # for beeing compatible to most webservers # client side rdf is no problem # NOTE: csrf must be disabled or use csrf token from GET, # here is no way to know the token value h = hashlib.sha256(context["referrer"].encode("utf8")).hexdigest() def h_fun(*a): return h # rate limit on errors if ratelimit.get_ratelimit(request=self.request, group="refer_with_post.refer_with_post", key=h_fun, rate=settings.SPIDER_DOMAIN_ERROR_RATE, inc=False)["request_limit"] > 0: return HttpResponseRedirect( redirect_to=merge_get_url(context["referrer"], status="post_failed", error="error_rate_limit")) d = { "token": token.token, "hash_algorithm": settings.SPIDER_HASH_ALGORITHM.name, "action": context["action"], } if context["payload"] is not None: d["payload"] = context["payload"] params, inline_domain = get_requests_params(context["referrer"]) if inline_domain: response = Client().post( context["referrer"], data=d, Connection="close", Referer=merge_get_url( "%s%s" % (context["hostpart"], self.request.path) # sending full url not required anymore, payload ), SERVER_NAME=inline_domain) if response.status_code != 200: return HttpResponseRedirect(redirect_to=merge_get_url( context["referrer"], status="post_failed", error="other")) else: try: with requests.post( context["referrer"], data=d, headers={ "Referer": merge_get_url( "%s%s" % (context["hostpart"], self.request.path) # sending full url not required anymore, payload ), "Connection": "close" }, **params) as resp: resp.raise_for_status() except requests.exceptions.SSLError as exc: logger.info("referrer: \"%s\" has a broken ssl configuration", context["referrer"], exc_info=exc) return HttpResponseRedirect(redirect_to=merge_get_url( context["referrer"], status="post_failed", error="ssl")) except (Exception, requests.exceptions.HTTPError) as exc: apply_error_limit = False if isinstance(exc, (requests.exceptions.ConnectionError, requests.exceptions.Timeout)): apply_error_limit = True elif (isinstance(exc, requests.exceptions.HTTPError) and exc.response.status_code >= 500): apply_error_limit = True if apply_error_limit: ratelimit.get_ratelimit( request=self.request, group="refer_with_post", key=h_fun, rate=settings.SPIDER_DOMAIN_ERROR_RATE, inc=True) logger.info("post failed: \"%s\" failed", context["referrer"], exc_info=exc) return HttpResponseRedirect(redirect_to=merge_get_url( context["referrer"], status="post_failed", error="other")) context["post_success"] = True h = get_hashob() h.update(token.token.encode("utf-8", "ignore")) return HttpResponseRedirect(redirect_to=merge_get_url( context["referrer"], status="success", hash=h.finalize().hex()))
def sign(self, confirm=False): url, headers = self.merge_and_headers(self.url) try: check_result = self.simple_check( url, token=headers.get("X-TOKEN"), session=self.session, ) attestation = check_result["attestation"] errored = set(map(lambda x: x[0], check_result["errors"])) key_list = check_result["key_list"] key_hashes = set(map(lambda x: x[0], key_list)) except CheckError as exc: if not exc.key_list or not exc.errored or not exc.attestation: raise exc attestation = exc.attestation errored = set(map(lambda x: x[0], exc.errored)) key_list = exc.key_list key_hashes = set(map(lambda x: x[0], key_list)) if self.hash_key_public not in key_hashes: raise CheckError("Key is not part of chain") if not confirm: return (self.hash_key_public in errored, key_list) # change to update url postbox_update = merge_get_url(replace_action(self.url, "update/"), raw="embed", search="\x1etype=PostBox\x1e") # retrieve csrftoken response = self.session.get(postbox_update, headers=headers) graph = Graph() graph.parse(data=response.content, format="html") csrftoken = list( graph.objects(predicate=spkcgraph["csrftoken"]))[0].toPython() fields = dict( map( lambda x: (x[0].toPython(), x[1].toPython()), graph.query( """ SELECT DISTINCT ?fieldname ?value WHERE { ?base spkc:fieldname ?fieldname ; spkc:value ?value . } """, initNs={"spkc": spkcgraph}, ))) fields["signatures"] = [] own_signature = None for key in self.client_list: if key[0] != self.hash_key_public: signature = key[2] else: # currently only one priv key is supported signature = self.priv_key.sign( attestation, padding.PSS(mgf=padding.MGF1(self.hash_algo), salt_length=padding.PSS.MAX_LENGTH), self.hash_algo) if signature: fields["signatures"].append({ "hash": f"{self.hash_algo.name}={key[0].hex()}", "signature": "{}={}".format(self.hash_algo.name, base64.b64encode(signature).decode("ascii")) }) if key[0] == self.hash_key_public: own_signature = fields["signatures"][-1]["signature"] fields["signatures"] = json.dumps(fields["signatures"]) # update response = self.session.post(postbox_update, data=fields, headers={ "X-CSRFToken": csrftoken, **headers }) try: response.raise_for_status() graph = Graph() graph.parse(data=response.content, format="html") vals = set(extract_property(graph, "signature").values()) breakpoint() if own_signature not in vals: raise SrcException("could not update signature", own_signature) except SrcException as exc: raise exc except Exception as exc: raise SrcException("could not update signature", own_signature) from exc return (self.hash_key_public in errored, key_list)
def validate(ob, hostpart, task=None, info_filters=None): dvfile = None source = None if not info_filters: info_filters = [] info_filters = set(info_filters) g = Graph() g.namespace_manager.bind("spkc", spkcgraph, replace=True) with requests.session() as session: view_url = None if isinstance(ob, tuple): current_size = ob[1] with open(ob[0], "rb") as f: retrieve_object(f, [current_size], graph=g, session=session) if ob[2]: try: os.unlink(ob[0]) except FileNotFoundError: pass else: current_size = 0 view_url = ob retrieve_object(ob, [current_size], graph=g, session=session) tmp = list(g.query( """ SELECT ?base ?scope ?pages ?view WHERE { ?base spkc:scope ?scope ; spkc:pages.num_pages ?pages ; spkc:pages.current_page ?current_page ; spkc:action:view ?view . } """, initNs={"spkc": spkcgraph}, initBindings={ "current_page": Literal(1, datatype=XSD.positiveInteger) } )) if len(tmp) != 1: raise exceptions.ValidationError( _('Invalid graph'), code="invalid_graph" ) tmp = tmp[0] start = tmp.base pages = tmp.pages.toPython() # scope = tmp.scope.toPython() if not view_url: view_url = tmp.view.toPython() # normalize view_url splitted = view_url.split("?", 1) if len(splitted) != 2: splitted = [splitted[0], ""] # update/create source object source = VerifySourceObject.objects.update_or_create( url=splitted[0], defaults={"get_params": splitted[1]} )[0] # create internal info filter (only those contents are checked) # differs from info field _filter_info = "" for filt in info_filters: fchar = filt[0] if fchar == "\x1e": _filter_info = \ f"{_filter_info}FILTER CONTAINS(?info, {filt})\n" elif fchar == "!" and not filt[1] == "!": _filter_info = \ f"{_filter_info}FILTER NOT CONTAINS(?info, {filt[1:]})\n" else: raise ValueError("Invalid filter") if task: task.update_state( state='RETRIEVING', meta={ 'page': 1, 'num_pages': pages } ) # retrieve further pages for page in range(2, pages+1): url = merge_get_url( view_url, raw="embed", page=str(page) ) retrieve_object(url, [current_size], graph=g, session=session) if task: task.update_state( state='RETRIEVING', meta={ 'page': page, 'num_pages': pages } ) # check and clean graph data_type = get_settings_func( "VERIFIER_CLEAN_GRAPH", "spkcspider.apps.verifier.functions.clean_graph" )(g, start, source, hostpart) if not data_type: raise exceptions.ValidationError( _('Invalid graph (Verification failed)'), code="graph_failed" ) g.remove((None, spkcgraph["csrftoken"], None)) hashable_nodes = g.query( f""" SELECT DISTINCT ?base ?info ?type ?name ?value WHERE {{ ?base spkc:type ?type ; spkc:properties ?pinfo, ?pval . ?pinfo spkc:name "info"^^xsd:string ; spkc:value ?info . ?pval spkc:hashable "true"^^xsd:boolean ; spkc:name ?name ; spkc:value ?value . OPTIONAL {{ ?value spkc:properties ?prop2 ; spkc:name "info"^^xsd:string ; spkc:value ?val_info . }} {_filter_info} }} """, initNs={"spkc": spkcgraph} ) if task: task.update_state( state='HASHING', meta={ 'hashable_nodes_checked': 0 } ) # make sure triples are linked to start # (user can provide arbitary data) g.remove((start, spkcgraph["hashed"], None)) # MAYBE: think about logic for incoperating hashes g.remove((start, spkcgraph["hash"], None)) nodes = {} resources_with_hash = {} for count, val in enumerate(hashable_nodes, start=1): if isinstance(val.value, URIRef): assert(val.val_info) h = get_hashob() # should always hash with xsd.string h.update(XSD.string.encode("utf8")) # don't strip id, as some contents seperate only by id h.update(str(val.val_info).encode("utf8")) _hash = h.finalize() elif val.value.datatype == spkcgraph["hashableURI"]: _hash = resources_with_hash.get(val.value.value) if not _hash: url = merge_get_url(val.value.value, raw="embed") if not get_settings_func( "SPIDER_URL_VALIDATOR", "spkcspider.apps.spider.functions.validate_url_default" )(url): raise exceptions.ValidationError( _('invalid url: %(url)s'), params={"url": url}, code="invalid_url" ) _hash = retrieve_object( url, [current_size], session=session ) # do not use add as it could be corrupted by user # (user can provide arbitary data) _uri = URIRef(val.value.value) g.set(( _uri, spkcgraph["hash"], Literal(_hash.hex()) )) else: h = get_hashob() if val.value.datatype == XSD.base64Binary: h.update(val.value.datatype.encode("utf8")) h.update(val.value.toPython()) elif val.value.datatype: h.update(val.value.datatype.encode("utf8")) h.update(val.value.encode("utf8")) else: h.update(XSD.string.encode("utf8")) h.update(val.value.encode("utf8")) _hash = h.finalize() h = get_hashob() h.update(val.name.toPython().encode("utf8")) h.update(_hash) base = str(val.base) nodes.setdefault(base, ([], val.type)) nodes[base][0].append(h.finalize()) if task: task.update_state( state='HASHING', meta={ 'hashable_nodes_checked': count } ) if task: task.update_state( state='HASHING', meta={ 'hashable_nodes_checked': "all" } ) # first sort hashes per node and create hash over sorted hashes # de-duplicate super-hashes (means: nodes are identical) hashes = set() for val, _type in nodes.values(): h = get_hashob() for _hob in sorted(val): h.update(_hob) h.update(_type.encode("utf8")) hashes.add(h.finalize()) # then create hash over sorted de-duplicated node hashes h = get_hashob() for i in sorted(hashes): h.update(i) # do not use add as it could be corrupted by user # (user can provide arbitary data) digest = h.finalize().hex() g.set(( start, spkcgraph["hash"], Literal(digest) )) with tempfile.NamedTemporaryFile(delete=True) as dvfile: # save in temporary file g.serialize( dvfile, format="turtle" ) result, created = DataVerificationTag.objects.get_or_create( defaults={ "dvfile": File(dvfile), "source": source, "data_type": data_type }, hash=digest ) update_fields = set() # and source, cannot remove source without replacement if not created and source and source != result.source: result.source = source update_fields.add("source") if data_type != result.data_type: result.data_type = data_type update_fields.add("data_type") result.save(update_fields=update_fields) verify_tag(result, task=task, ffrom="validate") if task: task.update_state( state='SUCCESS' ) return result
def render_to_response(self, context): if self.scope != "export" and "raw" not in self.request.GET: return super().render_to_response(context) embed = (self.scope == "export" or self.request.GET.get("raw", "") == "embed") session_dict = { "request": self.request, "context": context, "scope": self.scope, "expires": None, "hostpart": context["hostpart"], "uc_namespace": spkcgraph["components"], "sourceref": URIRef(context["hostpart"] + self.request.path) } g = Graph() g.namespace_manager.bind("spkc", spkcgraph, replace=True) if embed: # embed empty components per_page = getattr(settings, "SPIDER_SERIALIZED_PER_PAGE", settings.SPIDER_OBJECTS_PER_PAGE) // 2 p = [ paginate_stream(self.get_queryset_contents(), per_page, settings.SPIDER_MAX_EMBED_DEPTH), paginate_stream( context["object_list"], # empty components per_page, settings.SPIDER_MAX_EMBED_DEPTH) ] else: p = [ paginate_stream( context["object_list"], getattr(settings, "SPIDER_SERIALIZED_PER_PAGE", settings.SPIDER_OBJECTS_PER_PAGE), settings.SPIDER_MAX_EMBED_DEPTH) ] page = 1 try: page = int(self.request.GET.get("page", "1")) except Exception: pass if page <= 1: g.add((session_dict["sourceref"], spkcgraph["scope"], Literal(context["scope"], datatype=XSD.string))) g.add((session_dict["sourceref"], spkcgraph["strength"], Literal(self.source_strength))) token = getattr(session_dict["request"], "auth_token", None) if token: token = token.token url2 = merge_get_url(str(session_dict["sourceref"]), token=token) g.add((session_dict["sourceref"], spkcgraph["action:view"], Literal(url2, datatype=XSD.anyURI))) serialize_stream(g, p, session_dict, page=page, embed=embed, restrict_embed=(self.source_strength == 10), restrict_inclusion=(self.source_strength == 10)) ret = HttpResponse(g.serialize(format="turtle"), content_type="text/turtle;charset=utf-8") ret["Access-Control-Allow-Origin"] = "*" return ret
def render_serialize(self, **kwargs): from ..models import AssignedContent # ** creates copy of dict, so it is safe to overwrite kwargs here session_dict = { "request": kwargs["request"], "context": kwargs, "scope": kwargs["scope"], "hostpart": kwargs["hostpart"], "ac_namespace": spkcgraph["contents"], "sourceref": URIRef(urljoin( kwargs["hostpart"], kwargs["request"].path )) } g = Graph() g.namespace_manager.bind("spkc", spkcgraph, replace=True) p = paginate_stream( AssignedContent.objects.filter(id=self.associated_id), getattr( settings, "SPIDER_SERIALIZED_PER_PAGE", settings.SPIDER_OBJECTS_PER_PAGE ), settings.SPIDER_MAX_EMBED_DEPTH ) page = 1 try: page = int(session_dict["request"].GET.get("page", "1")) except Exception: pass serialize_stream( g, p, session_dict, page=page, embed=True ) if hasattr(kwargs["request"], "token_expires"): session_dict["expires"] = kwargs["request"].token_expires.strftime( "%a, %d %b %Y %H:%M:%S %z" ) if page <= 1: source = kwargs.get("source", self) # "expires" (string) different from "token_expires" (datetime) if session_dict.get("expires"): add_property( g, "token_expires", ob=session_dict["request"], ref=session_dict["sourceref"] ) if kwargs.get("machine_variants"): assert kwargs["request"].is_owner ucref = URIRef(urljoin( kwargs["hostpart"], source.associated.usercomponent.get_absolute_url() )) for machinec in kwargs["machine_variants"]: g.add(( ucref, spkcgraph["create:name"], Literal(machinec, datatype=XSD.string) )) g.add(( session_dict["sourceref"], spkcgraph["scope"], Literal(kwargs["scope"], datatype=XSD.string) )) uc = kwargs.get("source", self.associated.usercomponent) g.add(( session_dict["sourceref"], spkcgraph["strength"], Literal(uc.strength, datatype=XSD.integer) )) token = getattr(session_dict["request"], "auth_token", None) if token: token = token.token url2 = merge_get_url(str(session_dict["sourceref"]), token=token) g.add( ( session_dict["sourceref"], spkcgraph["action:view"], URIRef(url2) ) ) if kwargs["referrer"]: g.add(( session_dict["sourceref"], spkcgraph["referrer"], Literal(kwargs["referrer"], datatype=XSD.anyURI) )) if kwargs["token_strength"]: add_property( g, "token_strength", ref=session_dict["sourceref"], literal=kwargs["token_strength"] ) add_property( g, "intentions", ref=session_dict["sourceref"], literal=kwargs["intentions"], datatype=XSD.string, iterate=True ) ret = HttpResponse( g.serialize(format="turtle"), content_type="text/turtle;charset=utf-8" ) if session_dict.get("expires", None): ret['X-Token-Expires'] = session_dict["expires"] # allow cors requests for raw ret["Access-Control-Allow-Origin"] = "*" return ret