def dispatch_extra(self, request, *args, **kwargs):
_ = gettext
context = self.get_context_data()
# fallback to intentions if no request_intentions
# remove "domain", "sl"
context["intentions"] = set(
self.object.extra.get("request_intentions",
self.object.extra.get("intentions", [])))
# sl or domain should NOT be allowed for token updates
# sl is only for clients, domain only initial and should be removed
context["intentions"].difference_update({"domain", "sl"})
context["action"] = "update"
context["uc"] = self.object.usercomponent
rreferrer = request.POST.get("referrer", None)
if rreferrer:
context["referrer"] = merge_get_url(rreferrer)
if not get_settings_func(
"SPIDER_URL_VALIDATOR",
"spkcspider.apps.spider.functions.validate_url_default")(
context["referrer"], self):
context["action"] = "referrer_invalid"
# for donotact
if self.object.referrer and self.object.referrer.url == rreferrer:
rreferrer = None
else:
rreferrer = self.object.extra.get("request_referrer", None)
if rreferrer:
context["referrer"] = merge_get_url(rreferrer)
if not get_settings_func(
"SPIDER_URL_VALIDATOR",
"spkcspider.apps.spider.functions.validate_url_default"
)(context["referrer"], self):
return HttpResponse(status=400,
content=_('Insecure url: %(url)s') %
{"url": context["referrer"]})
elif self.object.referrer:
context["referrer"] = self.object.referrer.url
else:
context["referrer"] = ""
context["ids"] = self.object.usercomponent.contents.values_list(
"id", flat=True)
context["ids"] = set(context["ids"])
# if requested referrer is available DO delete invalid and DO care
ret = self.handle_referrer_request(context,
self.object,
dontact=not rreferrer,
no_oldtoken=True)
if isinstance(ret, HttpResponseRedirect):
if context.get("post_success", False):
messages.success(request, _("Intention update successful"))
else:
messages.error(request, _("Intention update failed"))
return HttpResponseRedirect(
self.get_redirect_url(context["sanitized_GET"]))
return ret
def check(self, url=None):
if not url or url.startswith(self.url):
merged_url, headers = self.merge_and_headers(
self.url, raw="embed", search="\x1etype=PostBox\x1e"
)
response = self.session.get(
merge_get_url(
merged_url, raw="embed", search="\x1etype=PostBox\x1e"
),
headers=headers,
)
response.raise_for_status()
graph = Graph()
graph.parse(data=response.content, format="turtle")
for page in get_pages(graph):
with self.session.get(
merge_get_url(merged_url, page=page), headers=headers
) as response:
response.raise_for_status()
graph.parse(data=response.content, format="turtle")
else:
response = self.session.get(
merge_get_url(url, raw="embed", search="\x1etype=PostBox\x1e")
)
response.raise_for_status()
graph = Graph()
graph.parse(data=response.content, format="turtle")
for page in get_pages(graph):
response = self.session.get(
merge_get_url(
self.url,
raw="embed",
search="\x1etype=PostBox\x1e",
page=page,
)
)
response.raise_for_status()
graph.parse(data=response.content, format="turtle")
if not url or url.startswith(self.url):
result = self.simple_check(
graph, url=url, checker=self.attestation_checker, auto_add=True
)
key_hashes = set(map(lambda x: x[0], result["key_list"]))
if self.hash_key_public not in key_hashes:
self.state = AttestationResult.error
raise CheckError("Key is not part of chain")
self.state = result["result"]
self.client_list = result["key_list"]
return result
else:
return self.simple_check(
graph, url=url, checker=self.attestation_checker, auto_add=True
)
def retrieve_missing(cls,
graph_or_url,
filters=None,
x_token=None,
timeout=60,
session=None):
if not session:
session = requests.Session()
if isinstance(graph_or_url, "str"):
assert filters is None, "No filters specified and url given"
graph = cls.retrieve_filtered_graph(graph_or_url,
filters,
x_token=x_token,
timeout=timeout,
session=session)
else:
graph = graph_or_url
retrieved_url, missing_pages = get_pages(graph)
for page in missing_pages:
with session.get(merge_get_url(retrieved_url, page=page),
headers={"X-TOKEN": x_token or ""},
timeout=timeout) as response:
response.raise_for_status()
graph.parse(data=response.content, format="turtle")
return graph, retrieved_url
def action_view(context):
token = getattr(context["request"], "auth_token", None)
if token:
token = token.token
url2 = merge_get_url(context["hostpart"] + context["request"].path,
token=token)
return Literal(url2, datatype=XSD.anyURI)
def handle_referrer(self):
_ = gettext
if (self.request.user != self.usercomponent.user
and not self.request.auth_token):
if self.request.user.is_authenticated:
return self.handle_no_permission()
return HttpResponseRedirect(redirect_to="{}?{}={}".format(
self.get_login_url(), REDIRECT_FIELD_NAME,
quote_plus(
merge_get_url(self.request.build_absolute_uri(),
token=None))))
context = self.get_context_data()
context["action"] = "create"
context["intentions"] = set(self.request.GET.getlist("intention"))
if "referrer" in self.request.POST:
context["referrer"] = merge_get_url(self.request.POST["referrer"])
if not get_settings_func(
"SPIDER_URL_VALIDATOR",
"spkcspider.apps.spider.functions.validate_url_default")(
context["referrer"], self):
context["action"] = "referrer_invalid"
else:
context["referrer"] = merge_get_url(self.request.GET["referrer"])
if not get_settings_func(
"SPIDER_URL_VALIDATOR",
"spkcspider.apps.spider.functions.validate_url_default")(
context["referrer"], self):
return HttpResponse(status=400,
content=_('Insecure url: %(url)s') %
{"url": context["referrer"]})
context["payload"] = self.request.GET.get("payload", None)
token = self.request.auth_token
if not token:
token = AuthToken(usercomponent=self.usercomponent,
extra={
"strength": 10,
"taint": False
})
if "domain" in context["intentions"]:
return self.handle_domain_auth(context, token)
else:
context["ids"] = set(self.object_list.values_list("id", flat=True))
context["search"] = set(self.request.POST.getlist("search"))
return self.handle_referrer_request(context, token)
def retrieve_missing(self, graph, url=None, timeout=60):
retrieved_url, missing_pages = get_pages(graph)
if not url:
url = retrieved_url
if self.url and url.startswith(self.url):
merged_url, headers = self.merge_and_headers(url)
else:
merged_url = url
headers = {}
for page in missing_pages:
with self.session.get(merge_get_url(merged_url, page=page),
headers=headers,
timeout=timeout) as response:
response.raise_for_status()
graph.parse(data=response.content, format="turtle")
return retrieved_url
def retrieve_filtered_graph(self, url=None, timeout=60, out=None):
if not url or url == self.url:
merged_url, headers = self.merge_and_headers(
self.url, raw="embed", search="\x1etype=PostBox\x1e")
else:
merged_url = merge_get_url(url,
raw="embed",
search="\x1etype=PostBox\x1e")
headers = {}
response = self.session.get(merged_url,
headers=headers,
timeout=timeout)
response.raise_for_status()
if out is None:
graph = Graph()
else:
graph = out
graph.parse(data=response.content, format="turtle")
return graph
def clean(self):
ret = super().clean()
if not ret.get("url", None) and not ret.get("dvfile", None):
raise forms.ValidationError(_('Require either url or dvfile'),
code="missing_parameter")
if ret.get("url", None):
self.cleaned_data["url"] = merge_get_url(self.cleaned_data["url"],
raw="embed")
url = self.cleaned_data["url"]
if not get_settings_func(
"SPIDER_URL_VALIDATOR",
"spkcspider.apps.spider.functions.validate_url_default")(
url):
self.add_error(
"url",
forms.ValidationError(_('invalid url: %(url)s'),
params={"url": url},
code="invalid_url"))
return ret
return ret
def _send_dest(self, aes_key, fetch_url, dest):
dest_url = merge_get_url(dest,
raw="embed",
search="\x1etype=PostBox\x1e")
response_dest = self.session.get(dest_url, timeout=60)
try:
response_dest.raise_for_status()
g_dest = Graph()
g_dest.parse(data=response_dest.content, format="turtle")
self.retrieve_missing(g_dest, dest_url)
except Exception as exc:
raise DestException("postbox retrieval failed") from exc
dest_postboxes = get_postboxes(g_dest)
if len(dest_postboxes) != 1:
raise DestException("No postbox found/more than one found")
dest_postbox_url, dest_options = next(iter(dest_postboxes.items()))
webref_url = replace_action(dest_postbox_url, "push_webref/")
attestation = dest_options["attestation"]
bdomain = dest_postbox_url.split("?", 1)[0]
result_dest, errors, dest_keys = self.attestation_checker.check(
bdomain,
map(lambda x: (x["key"], x["signature"]),
dest_options["signatures"].values()),
attestation=attestation,
algo=dest_options["hash_algorithm"],
auto_add=True)
if result_dest == AttestationResult.domain_unknown:
logger.info("add domain: %s", bdomain)
self.attestation_checker.attestation.add(
bdomain,
dest_keys,
algo=dest_options["hash_algorithm"],
embed=True)
elif result_dest == AttestationResult.error:
if len(dest_keys) == 0:
raise DestException("No keys found")
raise DestSecurityException("dest contains invalid keys", errors)
dest_key_list = {}
for k in dest_keys:
enc = k[1].encrypt(
aes_key,
padding.OAEP(
mgf=padding.MGF1(algorithm=dest_options["hash_algorithm"]),
algorithm=dest_options["hash_algorithm"],
label=None))
# encrypt decryption key
dest_key_list["%s=%s" %
(dest_options["hash_algorithm"].name,
k[0].hex())] = base64.b64encode(enc).decode("ascii")
try:
response_dest = self.session.post(webref_url,
data={
"url":
fetch_url,
"key_list":
json.dumps(dest_key_list)
},
timeout=60)
response_dest.raise_for_status()
except Exception as exc:
raise DestException("post webref failed") from exc
def merge_and_headers(self, _url, **kwargs):
if self.use_get_token:
return merge_get_url(_url, token=self.token, **kwargs), {}
return (merge_get_url(_url, **kwargs), {"X-TOKEN": self.token or ""})
def action_view(argv, priv_key, pem_public, own_url, session, g_message):
if argv.message_id is not None:
assert isinstance(argv.message_id, int)
result = list(g_message.query(
"""
SELECT DISTINCT ?base ?hash_algorithm ?type
WHERE {
?base a <https://spkcspider.net/static/schemes/spkcgraph#spkc:Content> ;
spkc:type ?type ;
spkc:properties ?propalg , ?propid .
?propid spkc:name ?idname ;
spkc:value ?idvalue .
?propalg spkc:name ?algname ;
spkc:value ?hash_algorithm .
}
""", # noqa E501
initNs={"spkc": spkcgraph},
initBindings={
"idvalue": Literal(argv.message_id),
"algname": Literal(
"hash_algorithm", datatype=XSD.string
),
"idname": Literal(
"id", datatype=XSD.string
),
}
))
if not result or result[0].type.toPython() not in {
"WebReference", "MessageContent"
}:
parser.exit(0, "message not found\n")
pub_key_hasher = getattr(
hashes, result[0].hash_algorithm.upper()
)()
digest = hashes.Hash(pub_key_hasher, backend=default_backend())
digest.update(pem_public)
pub_key_hashalg = "%s=%s" % (
pub_key_hasher.name,
digest.finalize().hex()
)
retrieve_url = merge_get_url(
replace_action(
result[0].base, "message/"
)
)
data = {
"max_size": argv.max
}
if argv.action == "view":
data["keyhash"] = pub_key_hashalg
response = session.post(
retrieve_url, stream=True, headers={
"X-TOKEN": argv.token
}, data=data
)
if not response.ok:
logger.info("Message retrieval failed: %s", response.text)
parser.exit(0, "message could not be fetched\n")
key_list = json.loads(response.headers["X-KEYLIST"])
key = key_list.get(pub_key_hashalg, None)
if not key:
parser.exit(0, "message not for me\n")
decrypted_key = priv_key.decrypt(
base64.b64decode(key),
padding.OAEP(
mgf=padding.MGF1(algorithm=argv.src_hash_algo),
algorithm=argv.src_hash_algo,
label=None
)
)
headblock = b""
fdecryptor = None
eparser = emailparser.BytesFeedParser(policy=policy.default)
headers = None
for chunk in response.iter_content(chunk_size=256):
blob = None
if not fdecryptor:
headblock = b"%b%b" % (headblock, chunk)
if b"\0" in headblock:
nonce, headblock = headblock.split(b"\0", 1)
nonce = base64.b64decode(nonce)
fdecryptor = Cipher(
algorithms.AES(decrypted_key),
modes.GCM(nonce),
backend=default_backend()
).decryptor()
blob = fdecryptor.update(headblock[:-16])
headblock = headblock[-16:]
else:
continue
else:
blob = fdecryptor.update(
b"%b%b" % (headblock, chunk[:-16])
)
headblock = chunk[-16:]
if not headers:
if b"\n\n" not in blob:
eparser.feed(blob)
continue
headersrest, blob = blob.split(b"\n\n", 1)
eparser.feed(headersrest)
headers = eparser.close()
# check what to do
t = headers.get("SPKC-Type", MessageType.email)
if t == MessageType.email:
argv.file.write(headers.as_bytes(
unixfrom=True,
policy=policy.SMTP
))
argv.file.write(blob)
argv.file.write(fdecryptor.finalize_with_tag(headblock))
else:
queried_webrefs = {}
queried_messages = {}
for i in g_message.query(
"""
SELECT DISTINCT ?base ?idvalue ?namevalue ?type
WHERE {
?base a <https://spkcspider.net/static/schemes/spkcgraph#spkc:Content> ;
spkc:type ?type ;
spkc:properties ?propname , ?propid .
?propid spkc:name ?idname ;
spkc:value ?idvalue .
?propname spkc:name ?namename ;
spkc:value ?namevalue .
}
""", # noqa E501
initNs={"spkc": spkcgraph},
initBindings={
"idname": Literal(
"id", datatype=XSD.string
),
"namename": Literal(
"name", datatype=XSD.string
),
}
):
if i.type.toPython() == "WebReference":
queried = queried_webrefs
elif i.type.toPython() == "MessageContent":
queried = queried_messages
else:
continue
queried.setdefault(str(i.base), {})
queried[str(i.base)]["id"] = i.idvalue
queried[str(i.base)]["name"] = i.namevalue
print("Received Messages:")
for i in sorted(queried_webrefs.values(), key=lambda x: x["id"]):
print(i["id"], i["name"])
print("Own Messages:")
for i in sorted(queried_messages.values(), key=lambda x: x["id"]):
print(i["id"], i["name"])
def handle_referrer_request(self,
context,
token,
keep=False,
dontact=False,
no_oldtoken=False):
"""
no_oldtoken: don't use old token for calculating:
old_ids, old_search
(performance and manual old_* possible)
dontact: dont act if probing results fails
"""
_ = gettext
context.setdefault("action", "create")
context.setdefault("model", self.model)
context.setdefault("payload", None)
context.setdefault("ids", set())
context.setdefault("filter", set())
context.setdefault("old_ids", set())
context.setdefault("old_search", set())
context["is_serverless"] = "sl" in context["intentions"]
if keep is True:
context["ids"].update(token.extra.get("ids", []))
context["search"].update(token.extra.get("filter", []))
action = self.request.POST.get("action", None)
if context["action"].endswith("invalid"):
action = context["action"]
if action == "confirm":
newtoken = None
# if persist try to find old token
if "persist" in context["intentions"]:
oldtoken = AuthToken.objects.filter(
Q(persist__gte=0, usercomponent=token.usercomponent),
referrer__url=context["referrer"]).first()
if oldtoken:
newtoken = token
token = oldtoken
# either reuse persistent token with auth token tokenstring
# or just reuse auth token
if newtoken:
# steal token value
token.token = newtoken.token
# set to zero as prot_strength can elevate perms
token.extra["taint"] = False
token.extra["prot_strength"] = 0
token.extra["intentions"] = list(context["intentions"])
token.extra.pop("request_referrer", None)
token.extra.pop("request_intentions", None)
token.extra.pop("request_search", None)
if not self.clean_refer_intentions(context, token):
return HttpResponseRedirect(redirect_to=merge_get_url(
context["referrer"], error="intentions_incorrect"))
token.extra["search"] = list(context["search"])
if "live" in context["intentions"]:
token.extra.pop("ids", None)
else:
token.extra["ids"] = list(context["ids"])
token.referrer = ReferrerObject.objects.get_or_create(
url=context["referrer"])[0]
# after cleanup, save
try:
with transaction.atomic():
# must be done here, elsewise other token can (unlikely)
# take token as it is free for a short time, better be safe
if newtoken:
newtoken.delete()
token.save()
except TokenCreationError:
logger.exception("Token creation failed")
return HttpResponseServerError(
_("Token creation failed, try again"))
if context["is_serverless"]:
context["post_success"] = True
ret = self.refer_with_get(context, token)
else:
context["post_success"] = False
ret = self.refer_with_post(context, token)
if dontact:
return ret
if not context["post_success"]:
if newtoken:
logger.warning("Updating persisting token failed")
else:
token.delete()
return ret
elif action == "cancel":
if not dontact:
token.delete()
return HttpResponseRedirect(
redirect_to=merge_get_url(context["referrer"],
status="canceled",
payload=context["payload"]))
else:
if (no_oldtoken and "persist" in context["intentions"]):
oldtoken = AuthToken.objects.filter(
Q(persist__gte=0, usercomponent=token.usercomponent),
referrer__url=context["referrer"],
).first()
if oldtoken:
context["old_ids"].update(oldtoken.extra.get("ids", []))
context["old_search"].update(
oldtoken.extra.get("filter", []))
if not self.clean_refer_intentions(context, token):
return HttpResponse(status=400,
content=_('Error: intentions incorrect'))
context["object_list"] = context["model"].objects.filter(
id__in=context["ids"])
# remove other media
context["media"] = Media(
css={
'all': [
'node_modules/choices.js/public/assets/styles/choices%s.css'
% _extra # noqa:E501
]
},
js=[
'node_modules/choices.js/public/assets/scripts/choices%s.js'
% _extra, # noqa: E501
])
return self.response_class(
request=self.request,
template=self.get_referrer_template_names(),
context=context,
using=self.template_engine,
content_type=self.content_type)
def simple_check(url_or_graph,
session=None,
checker=None,
auto_add=False,
token=None):
if isinstance(url_or_graph, Graph):
graph = url_or_graph
retrieve_url, pages = get_pages(graph)
url = retrieve_url.split("?", 1)[0]
else:
url = url_or_graph
retrieve_url = merge_get_url(url_or_graph,
raw="embed",
search="\x1etype=PostBox\x1e")
if not session:
session = requests.Session()
response = session.get(retrieve_url,
headers={"X-TOKEN": token or ""})
response.raise_for_status()
graph = Graph()
graph.parse(data=response.content, format="turtle")
pages = get_pages(graph)[1]
for page in pages:
response = session.get(merge_get_url(retrieve_url, page=page),
headers={"X-TOKEN": token or ""})
response.raise_for_status()
graph.parse(data=response.content, format="turtle")
postboxes = get_postboxes(graph)
if len(postboxes) != 1:
raise CheckError("No postbox found/more than one found")
postbox, options = next(iter(postboxes.items()))
if not isinstance(options.get("hash_algorithm"), hashes.HashAlgorithm):
raise CheckError("Hash algorithm not found")
if not isinstance(options.get("attestation"), bytes):
raise CheckError("Attestation not found/wrong type")
attestation, errors, key_list = AttestationChecker.check_signatures(
map(lambda x: (x["key"], x["signature"]),
options["signatures"].values()),
attestation=options["attestation"],
algo=options["hash_algorithm"])
if errors:
raise CheckError("Missmatch attestation with signatures",
errored=errors,
key_list=key_list,
attestation=attestation)
if not checker:
return {
"result": AttestationResult.success,
"errors": [],
"key_list": key_list,
**options
}
url = url.split("?", 1)[0]
ret = checker.check(url,
key_list,
algo=options["hash_algorithm"],
auto_add=auto_add,
embed=True,
attestation=attestation)
if ret[0] == AttestationResult.error:
raise CheckError("Checker validation failed",
errored=ret[1],
key_list=ret[2],
attestation=ret[0])
return {
"result": ret[0],
"errors": ret[1],
"key_list": key_list,
**options
}
def test_token(self, minstrength=0, force_token=False, taint=False):
expire = timezone.now()-self.usercomponent.token_duration
no_token = not force_token and self.usercomponent.required_passes == 0
ptype = ProtectionType.access_control
if minstrength >= 4:
no_token = False
ptype = ProtectionType.authentication
# delete old token, so no confusion happen
self.remove_old_tokens(expire)
# only valid tokens here
tokenstring = self.request.GET.get("token", None)
token = None
if tokenstring:
# find by tokenstring
token = self.usercomponent.authtokens.filter(
token=tokenstring
).first()
elif self.request.session.session_key:
# use session_key
token = self.usercomponent.authtokens.filter(
session_key=self.request.session.session_key
).first()
elif not no_token:
# generate session key if it not exist and token is required
self.request.session.cycle_key()
if token and token.extra.get("prot_strength", 0) >= minstrength:
self.request.token_expires = \
token.created+self.usercomponent.token_duration
# case will never enter
# if not token.session_key and "token" not in self.request.GET:
# return self.replace_token()
if (
token.extra["prot_strength"] >= 4 and
not token.extra.get("taint", False)
):
self.request.is_special_user = True
self.request.is_owner = True
self.request.auth_token = token
# after, for having access to token
self.usercomponent.auth(
request=self.request, scope=self.scope,
ptype=ptype, side_effect=True
)
return True
# if result is impossible and token invalid try to login
if minstrength >= 4 and not self.usercomponent.can_auth:
# auth won't work for logged in users
if self.request.user.is_authenticated:
return False
# remove token and redirect
# login_url can be also on a different host => merge_get_url
target = "{}?{}={}".format(
self.get_login_url(),
REDIRECT_FIELD_NAME,
quote_plus(
merge_get_url(
self.request.build_absolute_uri(),
token=None
)
)
)
return HttpResponseRedirect(redirect_to=target)
protection_codes = None
if "protection" in self.request.GET:
protection_codes = self.request.GET.getlist("protection")
# execute protections for side effects even no_token
self.request.protections = self.usercomponent.auth(
request=self.request, scope=self.scope,
protection_codes=protection_codes, ptype=ptype
)
if (
type(self.request.protections) is int and # because: False==0
self.request.protections >= minstrength
):
# generate only tokens if required
if no_token:
return True
token = self.create_token(
extra={
"strength": self.usercomponent.strength,
"prot_strength": self.request.protections,
"taint": taint
}
)
if (
token.extra["prot_strength"] >= 4 and
not token.extra.get("taint", False)
):
self.request.is_special_user = True
self.request.is_owner = True
self.request.token_expires = \
token.created+self.usercomponent.token_duration
self.request.auth_token = token
if "token" in self.request.GET:
return self.replace_token()
return True
return False
def refer_with_get(self, context, token):
return HttpResponseRedirect(redirect_to=merge_get_url(
context["referrer"], token=token.token,
payload=context["payload"]))
def render_to_response(self, context):
if context["scope"] != "export" and "raw" not in self.request.GET:
return super().render_to_response(context)
session_dict = {
"request": self.request,
"context": context,
"scope": context["scope"],
"uc": self.usercomponent,
"hostpart": context["hostpart"],
"sourceref":
URIRef("%s%s" % (context["hostpart"], self.request.path))
}
g = Graph()
g.namespace_manager.bind("spkc", spkcgraph, replace=True)
embed = False
if (context["scope"] == "export"
or self.request.GET.get("raw", "") == "embed"):
embed = True
if context["object_list"]:
p = paginate_stream(
context["object_list"],
getattr(settings, "SPIDER_SERIALIZED_PER_PAGE",
settings.SPIDER_OBJECTS_PER_PAGE),
settings.SPIDER_MAX_EMBED_DEPTH)
else:
# no content, pagination works here only this way
p = paginate_stream(
UserComponent.objects.filter(pk=self.usercomponent.pk), 1, 1)
context["object_list"]
page = 1
try:
page = int(self.request.GET.get("page", "1"))
except Exception:
pass
if hasattr(self.request, "token_expires"):
session_dict["expires"] = self.request.token_expires.strftime(
"%a, %d %b %Y %H:%M:%S %z")
if page <= 1:
# "expires" (string) different from "token_expires" (datetime)
if session_dict.get("expires"):
add_property(g,
"token_expires",
ob=session_dict["request"],
ref=session_dict["sourceref"])
assert not context.get("machine_variants") or self.request.is_owner
for machinec in context.get("machine_variants", []):
g.add((session_dict["sourceref"], spkcgraph["create:name"],
Literal(machinec, datatype=XSD.string)))
g.add((session_dict["sourceref"], spkcgraph["scope"],
Literal(context["scope"], datatype=XSD.string)))
g.add((session_dict["sourceref"], spkcgraph["type"],
Literal("Component", datatype=XSD.string)))
g.add((session_dict["sourceref"], spkcgraph["strength"],
Literal(self.usercomponent.strength, datatype=XSD.integer)))
if context["referrer"]:
g.add((session_dict["sourceref"], spkcgraph["referrer"],
Literal(context["referrer"], datatype=XSD.anyURI)))
token = getattr(session_dict["request"], "auth_token", None)
if token:
token = token.token
g.add((session_dict["sourceref"], spkcgraph["action:view"],
Literal(merge_get_url(str(session_dict["sourceref"]),
token=token),
datatype=XSD.anyURI)))
if context["token_strength"]:
add_property(g,
"token_strength",
ref=session_dict["sourceref"],
literal=context["token_strength"],
datatype=XSD.integer)
add_property(g,
"intentions",
ref=session_dict["sourceref"],
literal=context["intentions"],
datatype=XSD.string,
iterate=True)
serialize_stream(g, p, session_dict, page=page, embed=embed)
ret = HttpResponse(g.serialize(format="turtle"),
content_type="text/turtle;charset=utf-8")
if session_dict.get("expires", None):
ret['X-Token-Expires'] = session_dict["expires"]
# allow cors requests for raw
ret["Access-Control-Allow-Origin"] = "*"
return ret
def action_send(argv, priv_key, pub_key_hash, src_keys, session, g_src):
postboxes = get_postboxes(g_src)
if len(postboxes) != 1:
parser.exit(1, "Source cannot create messages, logged in?")
component_uriref, src_options = next(postboxes.items())
dest_url = merge_get_url(
argv.dest, raw="embed", search="\x1etype=PostBox\x1e"
)
response_dest = session.get(dest_url)
if not response_dest.ok:
logger.info("Dest returned error: %s", response_dest.text)
parser.exit(1, "retrieval failed, invalid url?\n")
g_dest = Graph()
g_dest.parse(data=response_dest.content, format="turtle")
for page in get_pages(g_dest)[1]:
with session.get(
merge_get_url(dest_url, page=page)
) as response:
response.raise_for_status()
g_dest.parse(data=response.content, format="turtle")
dest_postboxes = get_postboxes(g_dest)
if len(dest_postboxes) != 1:
parser.exit(1, "Dest has no/too many postboxes")
dest_postbox_url, dest_options = next(dest_postboxes.items())
webref_url = replace_action(dest_postbox_url, "push_webref/")
dest_hash_algo = dest_options["hash_algorithm"]
attestation = dest_options["attestation"]
bdomain = dest_postbox_url.split("?", 1)[0]
result_dest, _, dest_keys = argv.attestation.check(
bdomain,
map(
lambda x: (x["key"], x["signature"]),
dest_options["signatures"].values()
), attestation=attestation,
algo=dest_hash_algo
)
if result_dest == AttestationResult.domain_unknown:
logger.info("add domain: %s", bdomain)
argv.attestation.add(
bdomain,
dest_keys,
algo=dest_hash_algo
)
elif result_dest == AttestationResult.error:
logger.critical("Dest base url contains invalid keys.")
parser.exit(1, "dest contains invalid keys\n")
# 256 bit
aes_key = os.urandom(32)
nonce = os.urandom(13)
fencryptor = Cipher(
algorithms.AES(aes_key),
modes.GCM(nonce),
backend=default_backend()
).encryptor()
src_key_list = {}
dest_key_list = {}
if argv.stealth:
pass
elif src_options["shared"]:
for k in src_keys:
enc = k[1].encrypt(
aes_key,
padding.OAEP(
mgf=padding.MGF1(algorithm=argv.src_hash_algo),
algorithm=argv.src_hash_algo, label=None
)
)
# encrypt decryption key
src_key_list[
"%s=%s" % (argv.src_hash_algo.name, k[0].hex())
] = base64.b64encode(enc).decode("ascii")
else:
enc = priv_key.public_key().encrypt(
aes_key,
padding.OAEP(
mgf=padding.MGF1(algorithm=argv.src_hash_algo),
algorithm=argv.src_hash_algo, label=None
)
)
# encrypt decryption key
src_key_list[
"%s=%s" % (argv.src_hash_algo.name, pub_key_hash)
] = base64.b64encode(enc).decode("ascii")
for k in dest_keys:
enc = k[1].encrypt(
aes_key,
padding.OAEP(
mgf=padding.MGF1(algorithm=dest_hash_algo),
algorithm=dest_hash_algo, label=None
)
)
# encrypt decryption key
dest_key_list[
"%s=%s" % (dest_hash_algo.name, k[0].hex())
] = base64.b64encode(enc).decode("ascii")
headers = b"SPKC-Type: %b\n" % MessageType.file
# remove raw as we parse html
message_create_url = merge_get_url(
replace_action(str(component_uriref), "add/MessageContent/"), raw=None
)
response = session.get(
message_create_url, headers={
"X-TOKEN": argv.token
}
)
if not response.ok:
logger.error("retrieval csrftoken failed: %s", response.text)
parser.exit(1, "retrieval csrftoken failed: %s" % response.text)
g = Graph()
g.parse(data=response.content, format="html")
csrftoken = list(g.objects(predicate=spkcgraph["csrftoken"]))[0]
# create message object
response = session.post(
message_create_url, data={
"own_hash": pub_key_hash,
"key_list": json.dumps(src_key_list),
"amount_tokens": 1
}, headers={
"X-CSRFToken": csrftoken,
"X-TOKEN": argv.token # only for src
},
files={
"encrypted_content": EncryptedFile(
fencryptor, argv.file, nonce, headers
)
}
)
if not response.ok or message_create_url == response.url:
logger.error("Message creation failed: %s", response.text)
parser.exit(1, "Message creation failed: %s" % response.text)
g = Graph()
g.parse(data=response.content, format="html")
fetch_url = list(map(lambda x: x.value, g.query(
"""
SELECT ?value
WHERE {
?property spkc:name ?name ;
spkc:value ?value .
}
""",
initNs={"spkc": spkcgraph},
initBindings={
"name": Literal(
"fetch_url", datatype=XSD.string
)
}
)))
tokens = list(map(lambda x: x.value, g.query(
"""
SELECT ?value
WHERE {
?property spkc:name ?name ;
spkc:value ?value .
}
""",
initNs={"spkc": spkcgraph},
initBindings={
"name": Literal(
"tokens", datatype=XSD.string
)
}
)))
if not fetch_url or not tokens:
logger.error("Message creation failed: %s", response.text)
parser.exit(1, "Message creation failed")
# extract url
response_dest = session.post(
webref_url, data={
"url": merge_get_url(fetch_url[0], token=str(tokens[0])),
"key_list": json.dumps(dest_key_list)
}
)
if not response_dest.ok:
logger.error("Sending message failed: %s", response_dest.text)
parser.exit(1, "Sending message failed")
def main(argv):
argv = parser.parse_args(argv)
argv.attestation = AttestationChecker(argv.attestation)
if not os.path.exists(argv.key):
parser.exit(1, "key does not exist\n")
match = static_token_matcher.match(argv.url)
if not match:
parser.exit(1, "invalid url scheme\n")
if argv.verbose >= 2:
logging.basicConfig(level=logging.DEBUG)
elif argv.verbose >= 1:
logging.basicConfig(level=logging.INFO)
else:
logging.basicConfig(level=logging.WARNING)
access = match.groupdict()["access"]
if (
argv.action == "check" and
access not in {"view", "list"}
):
parser.exit(1, "url doesn't match action\n")
if (
argv.action in {"view", "peek"} and
access not in {"view", "list"}
):
parser.exit(1, "url doesn't match action\n")
if argv.action == "send":
match2 = static_token_matcher.match(argv.dest)
if not match2:
parser.exit(1, "invalid url scheme\n")
access2 = match2.groupdict()["access"]
if (
access not in {"list", "view", "push_webref"} or
access2 not in {"list", "view", "push_webref"}
):
parser.exit(1, "url doesn't match action\n")
argv.access_type = access
with open(argv.key, "rb") as f:
priv_key = load_priv_key(f.read())[0]
if not priv_key:
parser.exit(1, "invalid key: %s\n" % argv.key)
pem_public = priv_key.public_key().public_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PublicFormat.SubjectPublicKeyInfo
).strip()
with requests.Session() as s:
if access == "list":
own_url = merge_get_url(
argv.url, raw="embed", search="\x1etype=PostBox\x1e"
)
else:
own_url = merge_get_url(argv.url, raw="embed")
response = s.get(own_url, headers={
"X-TOKEN": argv.token
})
if not response.ok:
logger.info("Url returned error: %s\n", response.text)
parser.exit(1, "retrieval failed, invalid url?\n")
g = Graph()
g.parse(data=response.content, format="turtle")
for page in get_pages(g)[1]:
with s.get(
merge_get_url(own_url, page=page), headers={
"X-TOKEN": argv.token
}
) as response:
response.raise_for_status()
g.parse(data=response.content, format="turtle")
src = {}
for i in g.query(
"""
SELECT
?postbox ?postbox_value ?key_name ?key_value
WHERE {
?postbox spkc:properties ?property .
?property spkc:name ?postbox_name ;
spkc:value ?postbox_value .
?postbox_value spkc:properties ?key_base_prop .
?key_base_prop spkc:name ?key_name ;
spkc:value ?key_value .
}
""",
initNs={"spkc": spkcgraph},
initBindings={
"postbox_name": Literal(
"signatures", datatype=XSD.string
)
}
):
argv.postbox_base = i.postbox
src.setdefault(str(i.postbox_value), {})
src[str(i.postbox_value)][str(i.key_name)] = i.key_value
# algorithm for hashing
argv.src_hash_algo = getattr(
hashes, next(iter(src.values()))["hash_algorithm"].upper()
)()
digest = hashes.Hash(argv.src_hash_algo, backend=default_backend())
digest.update(pem_public)
pub_key_hash = digest.finalize().hex()
argv.attestation.add(
own_url.split("?", 1)[0], [pub_key_hash], argv.src_hash_algo
)
src_keys = None
if argv.action != "check":
result_own, errored, src_keys = argv.attestation.check(
own_url.split("?", 1)[0],
map(
lambda x: (x["key"], x["signature"]),
src.values()
),
algo=argv.src_hash_algo, auto_add=True
)
if result_own == AttestationResult.domain_unknown:
logger.critical(
"home url unknown, should not happen"
)
parser.exit(1, "invalid home url\n")
elif result_own != AttestationResult.success:
logger.critical(
"Home base url contains invalid keys, hacked?"
)
parser.exit(1, "invalid keys\n")
# check own domain
if argv.action == "send":
return action_send(
argv, priv_key, pub_key_hash, src_keys, s, g
)
elif argv.action in {"view", "peek"}:
return action_view(argv, priv_key, pem_public, own_url, s, g)
elif argv.action == "check":
ret = action_check(argv, priv_key, pub_key_hash, s, g)
if ret is not True:
parser.exit(2, "check failed: %s\n" % ret)
print("check successful")
def send(self,
inp,
receivers,
headers=b"\n",
mode=SendMethod.shared,
aes_key=None):
if not self.ok:
raise NotReady()
if isinstance(receivers, str):
receivers = [receivers]
if isinstance(inp, (bytes, memoryview)):
inp = io.BytesIO(bytes(inp))
elif isinstance(inp, str):
inp = io.BytesIO(inp.encode("utf8"))
# 256 bit
if not aes_key:
aes_key = os.urandom(32)
assert len(aes_key) == 32
nonce = os.urandom(13)
fencryptor = Cipher(algorithms.AES(aes_key),
modes.GCM(nonce),
backend=default_backend()).encryptor()
src_key_list = {}
if mode == SendMethod.stealth:
pass
elif mode == SendMethod.private:
enc = self.priv_key.public_key().encrypt(
aes_key,
padding.OAEP(mgf=padding.MGF1(algorithm=self.hash_algo),
algorithm=self.hash_algo,
label=None))
# encrypt decryption key
src_key_list["%s=%s" % (self.hash_algo.name, self.pub_key_hash
)] = base64.b64encode(enc).decode("ascii")
elif mode == SendMethod.shared:
for k in self.client_list:
enc = k[1].encrypt(
aes_key,
padding.OAEP(mgf=padding.MGF1(algorithm=self.hash_algo),
algorithm=self.hash_algo,
label=None))
# encrypt decryption key
src_key_list["%s=%s" %
(self.hash_algo.name, k[0].hex()
)] = base64.b64encode(enc).decode("ascii")
else:
raise NotImplementedError()
# remove raw as we parse html
message_create_url, src_headers = self.merge_and_headers(
replace_action(self.component_url, "add/MessageContent/"),
raw=None)
response = self.session.get(message_create_url, headers=src_headers)
try:
response.raise_for_status()
except Exception as exc:
raise SrcException("retrieval csrftoken failed",
response.text) from exc
g = Graph()
g.parse(data=response.content, format="html")
csrftoken = list(g.objects(predicate=spkcgraph["csrftoken"]))[0]
# create message object
response = self.session.post(
message_create_url,
data={
"own_hash": self.hash_key_public,
"key_list": json.dumps(src_key_list),
"amount_tokens": len(receivers)
},
headers={
"X-CSRFToken": csrftoken,
**src_headers # only for src
},
files={
"encrypted_content": EncryptedFile(fencryptor, inp, nonce,
headers)
})
try:
response.raise_for_status()
except Exception as exc:
raise SrcException("Message creation failed",
response.text) from exc
if message_create_url == response.url:
raise SrcException("Message creation failed", response.text)
g = Graph()
g.parse(data=response.content, format="html")
fetch_url = list(
map(
lambda x: x.value,
g.query("""
SELECT ?value
WHERE {
?property spkc:name ?name ;
spkc:value ?value .
}
""",
initNs={"spkc": spkcgraph},
initBindings={
"name": Literal("fetch_url", datatype=XSD.string)
})))
tokens = list(
map(
lambda x: x.value,
g.query("""
SELECT ?value
WHERE {
?property spkc:name ?name ;
spkc:value ?value .
}
""",
initNs={"spkc": spkcgraph},
initBindings={
"name": Literal("tokens", datatype=XSD.string)
})))
if not fetch_url or not tokens:
raise SrcException("Message creation failed", response.text)
fetch_url = fetch_url[0].toPython()
exceptions = []
final_fetch_urls = []
for receiver, token in zip(receivers, tokens):
furl = merge_get_url(fetch_url, token=token.toPython())
try:
self._send_dest(aes_key, furl, receiver)
final_fetch_urls.append(furl)
except Exception as exc:
exceptions.append(exc)
# for autoremoval simulate access
self.session.get(furl)
return exceptions, final_fetch_urls, aes_key
def access_message(self, **kwargs):
cached_content = self.associated.attachedfiles.filter(
name="cache").first()
if not cached_content:
cached_content = AttachedFile(content=self.associated,
unique=True,
name="cache")
max_size = kwargs["request"].POST.get("max_size") or math.inf
if isinstance(max_size, str):
max_size = int(max_size)
max_configured_size = \
self.associated.attached_to_content.content.free_data.get(
"max_receive_size", math.inf
)
if max_configured_size is None:
max_configured_size = math.inf
max_size = min(self.associated.user_info.get_free_space("remote"),
max_configured_size, max_size)
params, inline_domain = get_requests_params(self.quota_data["url"])
fp = None
if inline_domain:
try:
resp = Client().post(
self.quota_data["url"], {
"max_size":
(max_size if max_size != math.inf else "")
},
follow=True,
secure=True,
Connection="close",
Referer=merge_get_url(
"%s%s" %
(kwargs["hostpart"], kwargs["request"].path)),
SERVER_NAME=inline_domain)
if resp.status_code != 200:
logging.info("file retrieval failed: \"%s\" failed",
self.url)
return HttpResponse("other error", status=502)
c_length = resp.get("content-length", math.inf)
if isinstance(c_length, str):
c_length = int(c_length)
if max_size < c_length:
return HttpResponse("Too big/not specified",
status=413)
written_size = 0
fp = NamedTemporaryFile(suffix='.upload',
dir=settings.FILE_UPLOAD_TEMP_DIR)
for chunk in resp:
written_size += fp.write(chunk)
self.update_used_space(written_size, "remote")
# saves object
cached_content.file.save("", File(fp))
except ValidationError as exc:
del fp
logging.info("Quota exceeded", exc_info=exc)
return HttpResponse("Quota", status=413)
else:
try:
with requests.post(
self.quota_data["url"],
data={
"max_size":
(max_size if max_size != math.inf else "")
},
headers={
"Referer":
merge_get_url("%s%s" %
(kwargs["hostpart"],
kwargs["request"].path)),
"Connection":
"close"
},
stream=True,
**params) as resp:
resp.raise_for_status()
c_length = resp.get("content-length", math.inf)
if isinstance(c_length, str):
c_length = int(c_length)
if max_size < c_length:
return HttpResponse("Too big/not specified",
status=413)
fp = NamedTemporaryFile(
suffix='.upload',
dir=settings.FILE_UPLOAD_TEMP_DIR)
written_size = 0
for chunk in resp.iter_content(fp.DEFAULT_CHUNK_SIZE):
written_size += fp.write(chunk)
self.update_used_space(written_size, "remote")
# saves object
cached_content.file.save("", File(fp))
except requests.exceptions.SSLError as exc:
logger.info(
"referrer: \"%s\" has a broken ssl configuration",
self.url,
exc_info=exc)
return HttpResponse("ssl error", status=502)
except ValidationError as exc:
del fp
logging.info("Quota exceeded", exc_info=exc)
return HttpResponse("Quota", status=413)
except Exception as exc:
del fp
logging.info("file retrieval failed: \"%s\" failed",
self.url,
exc_info=exc)
return HttpResponse("other error", status=502)
ret = CbFileResponse(cached_content.file.open("rb"))
q = models.Q()
for i in kwargs["request"].POST.getlist("keyhash"):
q |= models.Q(target__info__contains="\x1epubkeyhash=%s" % i)
ret.refcopies = self.associated.smarttags.filter(q)
# ret["X-TYPE"] = kwargs["rtype"].name
ret["X-KEYLIST"] = json.dumps(self.quota_data["key_list"])
ret["X-KEYHASH-ALGO"] = self.free_data["hash_algorithm"]
return ret
def receive(self,
message_id,
outfp=None,
access_method=AccessMethod.view,
extra_key_hashes=None,
max_size=None):
if not self.ok:
raise NotReady()
merged_url, headers = self.merge_and_headers(self.url, raw="embed")
response = self.session.get(merge_get_url(self.url, raw="embed"),
headers=headers)
response.raise_for_status()
graph = Graph()
graph.parse(data=response.content, format="turtle")
self.retrieve_missing(graph, merged_url)
result = list(
graph.query(
"""
SELECT DISTINCT ?base ?hash_algorithm ?type
WHERE {
?base a <https://spkcspider.net/static/schemes/spkcgraph#spkc:Content> ;
spkc:type ?type ;
spkc:properties ?propalg , ?propid .
?propid spkc:name ?idname ;
spkc:value ?idvalue .
?propalg spkc:name ?algname ;
spkc:value ?hash_algorithm .
}
""", # noqa E501
initNs={"spkc": spkcgraph},
initBindings={
"idvalue": Literal(message_id),
"algname": Literal("hash_algorithm", datatype=XSD.string),
"idname": Literal("id", datatype=XSD.string),
}))
if not result or result[0].type.toPython() not in {
"WebReference", "MessageContent"
}:
raise SrcException("No Message")
# every object has it's own copy of the hash algorithm, used
hash_algo = getattr(hashes, result[0].hash_algorithm.upper())()
if not outfp:
outfp = tempfile.TempFile()
if hash_algo == self.hash_algo:
pub_key_hashalg = "%s=%s" % (hash_algo.name,
self.hash_key_public.hex())
else:
digest = hashes.Hash(hash_algo, backend=default_backend())
digest.update(self.pem_key_public)
pub_key_hashalg = "%s=%s" % (hash_algo.name,
digest.finalize().hex())
key_hashes = list()
if extra_key_hashes:
extra_key_hashes = set(extra_key_hashes)
extra_key_hashes.discard(pub_key_hashalg)
for key in self.client_list:
if key[0] in extra_key_hashes:
if hash_algo == self.hash_algo:
key_hashalg = "%s=%s" % (hash_algo.name, key[0])
else:
digest = hashes.Hash(hash_algo,
backend=default_backend())
digest.update(key[1])
key_hashalg = "%s=%s" % (hash_algo.name,
digest.finalize().hex())
key_hashes.append(key_hashalg)
if access_method == AccessMethod.view:
key_hashes.append(pub_key_hashalg)
retrieve_url, headers = self.merge_and_headers(
replace_action(
result[0].base, "bypass/" if
(access_method == AccessMethod.bypass) else "message/"))
data = {}
if access_method != AccessMethod.bypass:
data.update({"max_size": max_size or "", "keyhash": key_hashes})
response = self.session.post(retrieve_url,
stream=True,
headers=headers,
data=data)
try:
response.raise_for_status()
except Exception as exc:
raise DestException("Message retrieval failed",
response.text) from exc
key_list = json.loads(response.headers["X-KEYLIST"])
key = key_list.get(pub_key_hashalg, None)
if not key:
raise WrongRecipient("message not for me")
decrypted_key = self.priv_key.decrypt(
base64.b64decode(key),
padding.OAEP(mgf=padding.MGF1(algorithm=hash_algo),
algorithm=hash_algo,
label=None))
headblock = b""
fdecryptor = None
eparser = emailparser.BytesFeedParser(policy=policy.default)
headers = None
for chunk in response.iter_content(chunk_size=256):
blob = None
if not fdecryptor:
headblock = b"%b%b" % (headblock, chunk)
if b"\0" in headblock:
nonce, headblock = headblock.split(b"\0", 1)
nonce = base64.b64decode(nonce)
fdecryptor = Cipher(algorithms.AES(decrypted_key),
modes.GCM(nonce),
backend=default_backend()).decryptor()
blob = fdecryptor.update(headblock[:-16])
headblock = headblock[-16:]
else:
continue
else:
blob = fdecryptor.update(b"%b%b" % (headblock, chunk[:-16]))
headblock = chunk[-16:]
if not headers:
if b"\n\n" not in blob:
eparser.feed(blob)
continue
headersrest, blob = blob.split(b"\n\n", 1)
eparser.feed(headersrest)
headers = eparser.close()
# check what to do
t = headers.get("SPKC-Type", MessageType.email)
if t == MessageType.email:
outfp.write(
headers.as_bytes(unixfrom=True, policy=policy.SMTP))
outfp.write(blob)
outfp.write(fdecryptor.finalize_with_tag(headblock))
return outfp, headers, decrypted_key
def refer_with_post(self, context, token):
# application/x-www-form-urlencoded is best here,
# for beeing compatible to most webservers
# client side rdf is no problem
# NOTE: csrf must be disabled or use csrf token from GET,
# here is no way to know the token value
h = hashlib.sha256(context["referrer"].encode("utf8")).hexdigest()
def h_fun(*a):
return h
# rate limit on errors
if ratelimit.get_ratelimit(request=self.request,
group="refer_with_post.refer_with_post",
key=h_fun,
rate=settings.SPIDER_DOMAIN_ERROR_RATE,
inc=False)["request_limit"] > 0:
return HttpResponseRedirect(
redirect_to=merge_get_url(context["referrer"],
status="post_failed",
error="error_rate_limit"))
d = {
"token": token.token,
"hash_algorithm": settings.SPIDER_HASH_ALGORITHM.name,
"action": context["action"],
}
if context["payload"] is not None:
d["payload"] = context["payload"]
params, inline_domain = get_requests_params(context["referrer"])
if inline_domain:
response = Client().post(
context["referrer"],
data=d,
Connection="close",
Referer=merge_get_url(
"%s%s" % (context["hostpart"], self.request.path)
# sending full url not required anymore, payload
),
SERVER_NAME=inline_domain)
if response.status_code != 200:
return HttpResponseRedirect(redirect_to=merge_get_url(
context["referrer"], status="post_failed", error="other"))
else:
try:
with requests.post(
context["referrer"],
data=d,
headers={
"Referer":
merge_get_url(
"%s%s" %
(context["hostpart"], self.request.path)
# sending full url not required anymore, payload
),
"Connection":
"close"
},
**params) as resp:
resp.raise_for_status()
except requests.exceptions.SSLError as exc:
logger.info("referrer: \"%s\" has a broken ssl configuration",
context["referrer"],
exc_info=exc)
return HttpResponseRedirect(redirect_to=merge_get_url(
context["referrer"], status="post_failed", error="ssl"))
except (Exception, requests.exceptions.HTTPError) as exc:
apply_error_limit = False
if isinstance(exc, (requests.exceptions.ConnectionError,
requests.exceptions.Timeout)):
apply_error_limit = True
elif (isinstance(exc, requests.exceptions.HTTPError)
and exc.response.status_code >= 500):
apply_error_limit = True
if apply_error_limit:
ratelimit.get_ratelimit(
request=self.request,
group="refer_with_post",
key=h_fun,
rate=settings.SPIDER_DOMAIN_ERROR_RATE,
inc=True)
logger.info("post failed: \"%s\" failed",
context["referrer"],
exc_info=exc)
return HttpResponseRedirect(redirect_to=merge_get_url(
context["referrer"], status="post_failed", error="other"))
context["post_success"] = True
h = get_hashob()
h.update(token.token.encode("utf-8", "ignore"))
return HttpResponseRedirect(redirect_to=merge_get_url(
context["referrer"], status="success", hash=h.finalize().hex()))
def sign(self, confirm=False):
url, headers = self.merge_and_headers(self.url)
try:
check_result = self.simple_check(
url,
token=headers.get("X-TOKEN"),
session=self.session,
)
attestation = check_result["attestation"]
errored = set(map(lambda x: x[0], check_result["errors"]))
key_list = check_result["key_list"]
key_hashes = set(map(lambda x: x[0], key_list))
except CheckError as exc:
if not exc.key_list or not exc.errored or not exc.attestation:
raise exc
attestation = exc.attestation
errored = set(map(lambda x: x[0], exc.errored))
key_list = exc.key_list
key_hashes = set(map(lambda x: x[0], key_list))
if self.hash_key_public not in key_hashes:
raise CheckError("Key is not part of chain")
if not confirm:
return (self.hash_key_public in errored, key_list)
# change to update url
postbox_update = merge_get_url(replace_action(self.url, "update/"),
raw="embed",
search="\x1etype=PostBox\x1e")
# retrieve csrftoken
response = self.session.get(postbox_update, headers=headers)
graph = Graph()
graph.parse(data=response.content, format="html")
csrftoken = list(
graph.objects(predicate=spkcgraph["csrftoken"]))[0].toPython()
fields = dict(
map(
lambda x: (x[0].toPython(), x[1].toPython()),
graph.query(
"""
SELECT DISTINCT ?fieldname ?value
WHERE {
?base spkc:fieldname ?fieldname ;
spkc:value ?value .
}
""",
initNs={"spkc": spkcgraph},
)))
fields["signatures"] = []
own_signature = None
for key in self.client_list:
if key[0] != self.hash_key_public:
signature = key[2]
else:
# currently only one priv key is supported
signature = self.priv_key.sign(
attestation,
padding.PSS(mgf=padding.MGF1(self.hash_algo),
salt_length=padding.PSS.MAX_LENGTH),
self.hash_algo)
if signature:
fields["signatures"].append({
"hash":
f"{self.hash_algo.name}={key[0].hex()}",
"signature":
"{}={}".format(self.hash_algo.name,
base64.b64encode(signature).decode("ascii"))
})
if key[0] == self.hash_key_public:
own_signature = fields["signatures"][-1]["signature"]
fields["signatures"] = json.dumps(fields["signatures"])
# update
response = self.session.post(postbox_update,
data=fields,
headers={
"X-CSRFToken": csrftoken,
**headers
})
try:
response.raise_for_status()
graph = Graph()
graph.parse(data=response.content, format="html")
vals = set(extract_property(graph, "signature").values())
breakpoint()
if own_signature not in vals:
raise SrcException("could not update signature", own_signature)
except SrcException as exc:
raise exc
except Exception as exc:
raise SrcException("could not update signature",
own_signature) from exc
return (self.hash_key_public in errored, key_list)
def validate(ob, hostpart, task=None, info_filters=None):
dvfile = None
source = None
if not info_filters:
info_filters = []
info_filters = set(info_filters)
g = Graph()
g.namespace_manager.bind("spkc", spkcgraph, replace=True)
with requests.session() as session:
view_url = None
if isinstance(ob, tuple):
current_size = ob[1]
with open(ob[0], "rb") as f:
retrieve_object(f, [current_size], graph=g, session=session)
if ob[2]:
try:
os.unlink(ob[0])
except FileNotFoundError:
pass
else:
current_size = 0
view_url = ob
retrieve_object(ob, [current_size], graph=g, session=session)
tmp = list(g.query(
"""
SELECT ?base ?scope ?pages ?view
WHERE {
?base spkc:scope ?scope ;
spkc:pages.num_pages ?pages ;
spkc:pages.current_page ?current_page ;
spkc:action:view ?view .
}
""",
initNs={"spkc": spkcgraph},
initBindings={
"current_page": Literal(1, datatype=XSD.positiveInteger)
}
))
if len(tmp) != 1:
raise exceptions.ValidationError(
_('Invalid graph'),
code="invalid_graph"
)
tmp = tmp[0]
start = tmp.base
pages = tmp.pages.toPython()
# scope = tmp.scope.toPython()
if not view_url:
view_url = tmp.view.toPython()
# normalize view_url
splitted = view_url.split("?", 1)
if len(splitted) != 2:
splitted = [splitted[0], ""]
# update/create source object
source = VerifySourceObject.objects.update_or_create(
url=splitted[0], defaults={"get_params": splitted[1]}
)[0]
# create internal info filter (only those contents are checked)
# differs from info field
_filter_info = ""
for filt in info_filters:
fchar = filt[0]
if fchar == "\x1e":
_filter_info = \
f"{_filter_info}FILTER CONTAINS(?info, {filt})\n"
elif fchar == "!" and not filt[1] == "!":
_filter_info = \
f"{_filter_info}FILTER NOT CONTAINS(?info, {filt[1:]})\n"
else:
raise ValueError("Invalid filter")
if task:
task.update_state(
state='RETRIEVING',
meta={
'page': 1,
'num_pages': pages
}
)
# retrieve further pages
for page in range(2, pages+1):
url = merge_get_url(
view_url, raw="embed", page=str(page)
)
retrieve_object(url, [current_size], graph=g, session=session)
if task:
task.update_state(
state='RETRIEVING',
meta={
'page': page,
'num_pages': pages
}
)
# check and clean graph
data_type = get_settings_func(
"VERIFIER_CLEAN_GRAPH",
"spkcspider.apps.verifier.functions.clean_graph"
)(g, start, source, hostpart)
if not data_type:
raise exceptions.ValidationError(
_('Invalid graph (Verification failed)'),
code="graph_failed"
)
g.remove((None, spkcgraph["csrftoken"], None))
hashable_nodes = g.query(
f"""
SELECT DISTINCT ?base ?info ?type ?name ?value
WHERE {{
?base spkc:type ?type ;
spkc:properties ?pinfo, ?pval .
?pinfo spkc:name "info"^^xsd:string ;
spkc:value ?info .
?pval spkc:hashable "true"^^xsd:boolean ;
spkc:name ?name ;
spkc:value ?value .
OPTIONAL {{
?value spkc:properties ?prop2 ;
spkc:name "info"^^xsd:string ;
spkc:value ?val_info .
}}
{_filter_info}
}}
""",
initNs={"spkc": spkcgraph}
)
if task:
task.update_state(
state='HASHING',
meta={
'hashable_nodes_checked': 0
}
)
# make sure triples are linked to start
# (user can provide arbitary data)
g.remove((start, spkcgraph["hashed"], None))
# MAYBE: think about logic for incoperating hashes
g.remove((start, spkcgraph["hash"], None))
nodes = {}
resources_with_hash = {}
for count, val in enumerate(hashable_nodes, start=1):
if isinstance(val.value, URIRef):
assert(val.val_info)
h = get_hashob()
# should always hash with xsd.string
h.update(XSD.string.encode("utf8"))
# don't strip id, as some contents seperate only by id
h.update(str(val.val_info).encode("utf8"))
_hash = h.finalize()
elif val.value.datatype == spkcgraph["hashableURI"]:
_hash = resources_with_hash.get(val.value.value)
if not _hash:
url = merge_get_url(val.value.value, raw="embed")
if not get_settings_func(
"SPIDER_URL_VALIDATOR",
"spkcspider.apps.spider.functions.validate_url_default"
)(url):
raise exceptions.ValidationError(
_('invalid url: %(url)s'),
params={"url": url},
code="invalid_url"
)
_hash = retrieve_object(
url, [current_size], session=session
)
# do not use add as it could be corrupted by user
# (user can provide arbitary data)
_uri = URIRef(val.value.value)
g.set((
_uri,
spkcgraph["hash"],
Literal(_hash.hex())
))
else:
h = get_hashob()
if val.value.datatype == XSD.base64Binary:
h.update(val.value.datatype.encode("utf8"))
h.update(val.value.toPython())
elif val.value.datatype:
h.update(val.value.datatype.encode("utf8"))
h.update(val.value.encode("utf8"))
else:
h.update(XSD.string.encode("utf8"))
h.update(val.value.encode("utf8"))
_hash = h.finalize()
h = get_hashob()
h.update(val.name.toPython().encode("utf8"))
h.update(_hash)
base = str(val.base)
nodes.setdefault(base, ([], val.type))
nodes[base][0].append(h.finalize())
if task:
task.update_state(
state='HASHING',
meta={
'hashable_nodes_checked': count
}
)
if task:
task.update_state(
state='HASHING',
meta={
'hashable_nodes_checked': "all"
}
)
# first sort hashes per node and create hash over sorted hashes
# de-duplicate super-hashes (means: nodes are identical)
hashes = set()
for val, _type in nodes.values():
h = get_hashob()
for _hob in sorted(val):
h.update(_hob)
h.update(_type.encode("utf8"))
hashes.add(h.finalize())
# then create hash over sorted de-duplicated node hashes
h = get_hashob()
for i in sorted(hashes):
h.update(i)
# do not use add as it could be corrupted by user
# (user can provide arbitary data)
digest = h.finalize().hex()
g.set((
start,
spkcgraph["hash"],
Literal(digest)
))
with tempfile.NamedTemporaryFile(delete=True) as dvfile:
# save in temporary file
g.serialize(
dvfile, format="turtle"
)
result, created = DataVerificationTag.objects.get_or_create(
defaults={
"dvfile": File(dvfile),
"source": source,
"data_type": data_type
},
hash=digest
)
update_fields = set()
# and source, cannot remove source without replacement
if not created and source and source != result.source:
result.source = source
update_fields.add("source")
if data_type != result.data_type:
result.data_type = data_type
update_fields.add("data_type")
result.save(update_fields=update_fields)
verify_tag(result, task=task, ffrom="validate")
if task:
task.update_state(
state='SUCCESS'
)
return result
def render_to_response(self, context):
if self.scope != "export" and "raw" not in self.request.GET:
return super().render_to_response(context)
embed = (self.scope == "export"
or self.request.GET.get("raw", "") == "embed")
session_dict = {
"request": self.request,
"context": context,
"scope": self.scope,
"expires": None,
"hostpart": context["hostpart"],
"uc_namespace": spkcgraph["components"],
"sourceref": URIRef(context["hostpart"] + self.request.path)
}
g = Graph()
g.namespace_manager.bind("spkc", spkcgraph, replace=True)
if embed:
# embed empty components
per_page = getattr(settings, "SPIDER_SERIALIZED_PER_PAGE",
settings.SPIDER_OBJECTS_PER_PAGE) // 2
p = [
paginate_stream(self.get_queryset_contents(), per_page,
settings.SPIDER_MAX_EMBED_DEPTH),
paginate_stream(
context["object_list"], # empty components
per_page,
settings.SPIDER_MAX_EMBED_DEPTH)
]
else:
p = [
paginate_stream(
context["object_list"],
getattr(settings, "SPIDER_SERIALIZED_PER_PAGE",
settings.SPIDER_OBJECTS_PER_PAGE),
settings.SPIDER_MAX_EMBED_DEPTH)
]
page = 1
try:
page = int(self.request.GET.get("page", "1"))
except Exception:
pass
if page <= 1:
g.add((session_dict["sourceref"], spkcgraph["scope"],
Literal(context["scope"], datatype=XSD.string)))
g.add((session_dict["sourceref"], spkcgraph["strength"],
Literal(self.source_strength)))
token = getattr(session_dict["request"], "auth_token", None)
if token:
token = token.token
url2 = merge_get_url(str(session_dict["sourceref"]), token=token)
g.add((session_dict["sourceref"], spkcgraph["action:view"],
Literal(url2, datatype=XSD.anyURI)))
serialize_stream(g,
p,
session_dict,
page=page,
embed=embed,
restrict_embed=(self.source_strength == 10),
restrict_inclusion=(self.source_strength == 10))
ret = HttpResponse(g.serialize(format="turtle"),
content_type="text/turtle;charset=utf-8")
ret["Access-Control-Allow-Origin"] = "*"
return ret
def render_serialize(self, **kwargs):
from ..models import AssignedContent
# ** creates copy of dict, so it is safe to overwrite kwargs here
session_dict = {
"request": kwargs["request"],
"context": kwargs,
"scope": kwargs["scope"],
"hostpart": kwargs["hostpart"],
"ac_namespace": spkcgraph["contents"],
"sourceref": URIRef(urljoin(
kwargs["hostpart"], kwargs["request"].path
))
}
g = Graph()
g.namespace_manager.bind("spkc", spkcgraph, replace=True)
p = paginate_stream(
AssignedContent.objects.filter(id=self.associated_id),
getattr(
settings, "SPIDER_SERIALIZED_PER_PAGE",
settings.SPIDER_OBJECTS_PER_PAGE
),
settings.SPIDER_MAX_EMBED_DEPTH
)
page = 1
try:
page = int(session_dict["request"].GET.get("page", "1"))
except Exception:
pass
serialize_stream(
g, p, session_dict,
page=page,
embed=True
)
if hasattr(kwargs["request"], "token_expires"):
session_dict["expires"] = kwargs["request"].token_expires.strftime(
"%a, %d %b %Y %H:%M:%S %z"
)
if page <= 1:
source = kwargs.get("source", self)
# "expires" (string) different from "token_expires" (datetime)
if session_dict.get("expires"):
add_property(
g, "token_expires", ob=session_dict["request"],
ref=session_dict["sourceref"]
)
if kwargs.get("machine_variants"):
assert kwargs["request"].is_owner
ucref = URIRef(urljoin(
kwargs["hostpart"],
source.associated.usercomponent.get_absolute_url()
))
for machinec in kwargs["machine_variants"]:
g.add((
ucref,
spkcgraph["create:name"],
Literal(machinec, datatype=XSD.string)
))
g.add((
session_dict["sourceref"],
spkcgraph["scope"],
Literal(kwargs["scope"], datatype=XSD.string)
))
uc = kwargs.get("source", self.associated.usercomponent)
g.add((
session_dict["sourceref"], spkcgraph["strength"],
Literal(uc.strength, datatype=XSD.integer)
))
token = getattr(session_dict["request"], "auth_token", None)
if token:
token = token.token
url2 = merge_get_url(str(session_dict["sourceref"]), token=token)
g.add(
(
session_dict["sourceref"],
spkcgraph["action:view"],
URIRef(url2)
)
)
if kwargs["referrer"]:
g.add((
session_dict["sourceref"],
spkcgraph["referrer"],
Literal(kwargs["referrer"], datatype=XSD.anyURI)
))
if kwargs["token_strength"]:
add_property(
g, "token_strength", ref=session_dict["sourceref"],
literal=kwargs["token_strength"]
)
add_property(
g, "intentions", ref=session_dict["sourceref"],
literal=kwargs["intentions"], datatype=XSD.string,
iterate=True
)
ret = HttpResponse(
g.serialize(format="turtle"),
content_type="text/turtle;charset=utf-8"
)
if session_dict.get("expires", None):
ret['X-Token-Expires'] = session_dict["expires"]
# allow cors requests for raw
ret["Access-Control-Allow-Origin"] = "*"
return ret
