def mock_schema(properties_and_types_dict, id, context=None):
properties_schema = {}
type2datatype = {'string': 'xsd:string',
'integer': 'xsd:integer',
'date': 'xsd:date',
'datetime': 'xsd:dateTime',
'boolean': 'xsd:boolean'}
for property_name, type_value in properties_and_types_dict.items():
property_uri = expand_uri(property_name, context=context)
if type_value in ["string_uri", "array_string_uri"]:
properties_schema[property_uri] = {
'range': {'type': 'string', 'format': 'uri'}
}
if type_value == "string_uri":
properties_schema[property_uri]["type"] = "string"
else:
properties_schema[property_uri]["type"] = "array"
properties_schema[property_uri]['items'] = {'type': 'string', 'format': 'uri'}
else:
properties_schema[property_uri] = {'type': type_value, 'datatype': expand_uri(type2datatype[type_value], context=context)}
schema = {'properties': properties_schema}
if id is not None:
schema["id"] = id
return schema
def cast_item(item, property_to_type):
"""
Casts values of properties, according to the mapping provided.
Example:
item = {
"team": "Semantic Team",
"grade": "10"
}
property_to_type = {
"team": str,
"grade": int
}
cast_item(item, property_to_type)
Returns:
{
"team": "Semantic Team",
"grade": 10
}
"""
new_item = {}
for property_, value in item.items():
type_ = property_to_type.get(expand_uri(property_))
if not type_ is None:
if isinstance(value, list):
value = [type_(v) for v in value]
elif type_ is bool:
value = bool(int(value))
else:
value = type_(value)
if isinstance(value, list):
value = sorted(value)
new_item[property_] = value
return new_item
def build_json(items_list, query_params):
class_url = build_class_url(query_params)
schema_url = unquote(build_schema_url_for_instance(query_params, class_url))
class_properties = get_class.get_cached_schema(query_params)["properties"]
items_list = cast_items_values(items_list, class_properties)
json = {
'_schema_url': schema_url,
'pattern': '',
'_class_prefix': expand_uri(query_params['class_prefix']),
'_base_url': remove_last_slash(query_params.base_url),
'items': items_list,
'@context': {"@language": query_params.get("lang")},
'@id': query_params['class_uri']
}
def calculate_total_items():
result_dict = query_count_filter_instances(query_params)
total_items = int(get_one_value(result_dict, 'total'))
return total_items
decorate_dict_with_pagination(json, query_params, calculate_total_items)
return json
def filter_instances(query_params):
if not class_exists(query_params):
error_message = u"Class {0} in graph {1} does not exist".format(
query_params["class_uri"], query_params["graph_uri"])
raise HTTPError(404, log_message=error_message)
keymap = {
"label": "title",
"subject": "@id",
"sort_object": shorten_uri(query_params["sort_by"]),
}
for p, o, index in extract_po_tuples(query_params):
keymap[o[1:]] = shorten_uri(p)
result_dict = query_filter_instances(query_params)
if not result_dict or not result_dict['results']['bindings']:
return None
items_list = compress_keys_and_values(
result_dict,
keymap=keymap,
ignore_keys=["total"],
do_expand_uri=query_params['expand_uri'] == u"1")
items_list = merge_by_id(items_list)
for item in items_list:
uri = item["@id"]
item["instance_prefix"] = u"{0}/".format(uri.rsplit('/', 1)[0])
item["class_prefix"] = expand_uri(query_params['class_prefix'])
decorate_with_resource_id(items_list)
return build_json(items_list, query_params)
def _override_with(self, handler):
"Override this dictionary with values whose keys are present in the request"
# order is critical below because *_uri should be set before *_prefix
for key in self.arguments:
if (key not in self) and (not self._matches_dynamic_pattern(key)):
raise InvalidParam(key)
value = self.arguments.get(key, None)
if value is not None:
try:
self[key] = expand_uri(value)
except KeyError as ex:
raise RequiredParamMissing(unicode(ex))
def _override_with(self, handler):
"Override this dictionary with values whose keys are present in the request"
# order is critical below because *_uri should be set before *_prefix
for key in self.arguments:
if (key not in self) and (not self._matches_dynamic_pattern(key)):
raise InvalidParam(key)
value = self.arguments.get(key, None)
if value is not None:
try:
self[key] = expand_uri(value)
except KeyError as ex:
raise RequiredParamMissing(unicode(ex))
def mock_schema(properties_and_types_dict, id, context=None):
properties_schema = {}
type2datatype = {
'string': 'xsd:string',
'integer': 'xsd:integer',
'date': 'xsd:date',
'datetime': 'xsd:dateTime',
'boolean': 'xsd:boolean'
}
for property_name, type_value in properties_and_types_dict.items():
property_uri = expand_uri(property_name, context=context)
if type_value in ["string_uri", "array_string_uri"]:
properties_schema[property_uri] = {
'range': {
'type': 'string',
'format': 'uri'
}
}
if type_value == "string_uri":
properties_schema[property_uri]["type"] = "string"
else:
properties_schema[property_uri]["type"] = "array"
properties_schema[property_uri]['items'] = {
'type': 'string',
'format': 'uri'
}
else:
properties_schema[property_uri] = {
'type': type_value,
'datatype': expand_uri(type2datatype[type_value],
context=context)
}
schema = {'properties': properties_schema}
if id is not None:
schema["id"] = id
return schema
def compress_keys_and_values(result_dict,
keymap={},
ignore_keys=[],
context=None,
do_expand_uri=False):
"""
Return a list of compressed items of the 'bindings' list of a Virtuoso response dict.
Usage:
>>> result_dict = {'results': {'bindings': [{'key': {'type': 'some type', 'value': 'some value'}}, \
{'key': {'type': 'some type', 'value': 'another value'}}]}}
>>> compress_keys_and_values(result_dict)
[{'key': 'some value'}, {'key': 'another value'}]
Optional params:
- keymap = {'key': 'renamed_key'}: renames resulting dict 'key' by 'renamed_key'
- ignore_keys = ['key']: list of keys that shouldn't be returned
- context (instance of MemorizeContext): shortens URIs according to provided context
>>> compress_keys_and_values(result_dict, keymap={'key': 'renamed_key'})
[{'renamed_key': 'some value'}, {'renamed_key': 'another value'}]
>>> compress_keys_and_values(result_dict, ignore_keys=['key'])
[]
>>> from brainiak.prefixes import MemorizeContext
>>> result_dict = {'results': {'bindings': [{'key': {'type': 'uri', 'value': 'http://xmlns.com/foaf/0.1/value'}}]}}
>>> context = MemorizeContext()
>>> compress_keys_and_values(result_dict, context=context)
[{'key': 'foaf:value'}]
"""
result_list = []
for item in result_dict['results']['bindings']:
row = {}
for key in item:
if key not in ignore_keys:
value = item[key]['value']
effective_key = keymap.get(key, key)
if item[key][
'type'] == 'uri' and context and effective_key != '@id' and not do_expand_uri:
value = context.shorten_uri(value)
if do_expand_uri:
effective_key = expand_uri(effective_key)
row[effective_key] = value
result_list.append(row)
return result_list
def decorate_with_resource_id(list_of_dicts):
"""Adds to each entry in the input parameter list_of_dicts
a new key`resource_id` created from the last path segment given by the key `@id`"""
for dict_item in list_of_dicts:
try:
id_key = expand_uri(dict_item["@id"])
title = dict_item.get("title")
if title == ROOT_CONTEXT:
resource_id = u''
elif id_key.endswith("/"):
resource_id = id_key.rsplit("/")[-2]
else:
resource_id = id_key.rsplit("/")[-1]
dict_item['resource_id'] = unicode(resource_id)
except KeyError as ex:
raise TypeError(u"dict missing key {0:s} while processing decorate_with_resource_id()".format(ex))
def decorate_with_resource_id(list_of_dicts):
"""Adds to each entry in the input parameter list_of_dicts
a new key`resource_id` created from the last path segment given by the key `@id`"""
for dict_item in list_of_dicts:
try:
id_key = expand_uri(dict_item["@id"])
title = dict_item.get("title")
if title == ROOT_CONTEXT:
resource_id = u''
elif id_key.endswith("/"):
resource_id = id_key.rsplit("/")[-2]
else:
resource_id = id_key.rsplit("/")[-1]
dict_item['resource_id'] = str(resource_id)
except KeyError as ex:
raise TypeError(
u"dict missing key {0:s} while processing decorate_with_resource_id()"
.format(ex))
def compress_keys_and_values(result_dict, keymap={}, ignore_keys=[], context=None, do_expand_uri=False):
"""
Return a list of compressed items of the 'bindings' list of a Virtuoso response dict.
Usage:
>>> result_dict = {'results': {'bindings': [{'key': {'type': 'some type', 'value': 'some value'}}, \
{'key': {'type': 'some type', 'value': 'another value'}}]}}
>>> compress_keys_and_values(result_dict)
[{'key': 'some value'}, {'key': 'another value'}]
Optional params:
- keymap = {'key': 'renamed_key'}: renames resulting dict 'key' by 'renamed_key'
- ignore_keys = ['key']: list of keys that shouldn't be returned
- context (instance of MemorizeContext): shortens URIs according to provided context
>>> compress_keys_and_values(result_dict, keymap={'key': 'renamed_key'})
[{'renamed_key': 'some value'}, {'renamed_key': 'another value'}]
>>> compress_keys_and_values(result_dict, ignore_keys=['key'])
[]
>>> from brainiak.prefixes import MemorizeContext
>>> result_dict = {'results': {'bindings': [{'key': {'type': 'uri', 'value': 'http://xmlns.com/foaf/0.1/value'}}]}}
>>> context = MemorizeContext()
>>> compress_keys_and_values(result_dict, context=context)
[{'key': 'foaf:value'}]
"""
result_list = []
for item in result_dict['results']['bindings']:
row = {}
for key in item:
if key not in ignore_keys:
value = item[key]['value']
effective_key = keymap.get(key, key)
if item[key]['type'] == 'uri' and context and effective_key != '@id' and not do_expand_uri:
value = context.shorten_uri(value)
if do_expand_uri:
effective_key = expand_uri(effective_key)
row[effective_key] = value
result_list.append(row)
return result_list
def normalize_term(term, language=""):
"""
Provided a query term (literal, variable, expanded uri or
compressed uri), and language (to be applied in literals),
return the term in the form so it can be used inside a
SPARQL Query.
examples:
(1) http://expanded.predicate -> <http://expanded.predicate>
(2) compressed:predicate -> compressed:predicate
(3) "some literal" -> '"some literal"@lang'
(4) ?variable -> ?variable
"""
language_tag = u"@%s" % language if language else u""
if (not term.startswith("?")):
if (":" in term):
expanded_term = expand_uri(term)
if expanded_term != term or is_uri(expanded_term):
term = u"<%s>" % expanded_term
else:
term = u'"%s"%s' % (term, language_tag)
return term
def normalize_term(term, language=""):
"""
Provided a query term (literal, variable, expanded uri or
compressed uri), and language (to be applied in literals),
return the term in the form so it can be used inside a
SPARQL Query.
examples:
(1) http://expanded.predicate -> <http://expanded.predicate>
(2) compressed:predicate -> compressed:predicate
(3) "some literal" -> '"some literal"@lang'
(4) ?variable -> ?variable
"""
language_tag = u"@%s" % language if language else u""
if (not term.startswith("?")):
if (":" in term):
expanded_term = expand_uri(term)
if expanded_term != term or is_uri(expanded_term):
term = u"<%s>" % expanded_term
else:
term = u'"%s"%s' % (term, language_tag)
return term
def normalize_class_prefix(class_prefix, should_expand_uri):
return expand_uri(class_prefix) if should_expand_uri else shorten_uri(
class_prefix)
def test_expand_uri_whatever(self):
self.assertEqual("http://schema.org/whatever",
expand_uri("schema:whatever"))
... '"1"^^<http://www.w3.org/2001/XMLSchema#int>'
"""
assert is_uri(predicate_datatype)
template = u'"{0}"^^<{1}>'
return template.format(value, predicate_datatype)
SPARQLFY_MAP = {
"rdf:XMLLiteral": sparqlfy_string,
"rdfs:Literal": sparqlfy_string,
"xsd:string": sparqlfy_string,
"xsd:boolean": sparqlfy_boolean
}
SPARQLFY_MAP_EXPANDED = {expand_uri(k): v for k, v in SPARQLFY_MAP.items()}
def sparqlfy(value, predicate_datatype):
"""
Create SPARQL-friendly string representation of the value, based on the
predicate_datatype.
Examples:
>>> sparqlfy("http://expanded.uri", "http://www.w3.org/2001/XMLSchema#string")
... "http://expanded.uri"
>>> sparqlfy(True, "http://www.w3.org/2001/XMLSchema#boolean")
... '"true"^^<http://www.w3.org/2001/XMLSchema#boolean>'
"xsd:nonNegativeInteger": "integer",
"xsd:negativeInteger": "integer",
"xsd:positiveInteger": "integer",
"xsd:long": "integer",
"xsd:int": "integer",
"xsd:short": "integer",
"xsd:byte": "integer",
"xsd:decimal": "integer",
"xsd:unsignedLong": "integer",
"xsd:unsignedInt": "integer",
"xsd:unsignedShort": "integer",
"xsd:unsignedByte": "integer",
"xsd:boolean": "boolean"
}
_MAP_EXPAND_XSD_TO_JSON_TYPE = {
expand_uri(k): v
for k, v in _MAP_XSD_TO_JSON_TYPE.items()
}
_MAP_JSON_TO_XSD_TYPE = {
"string": "xsd:string",
"number": "xsd:double",
"integer": "xsd:integer",
"boolean": "xsd:boolean"
}
_MAP_JSON_TO_EXPAND_XSD_TYPE = {
k: expand_uri(v)
for k, v in _MAP_XSD_TO_JSON_TYPE.items()
}
# TODO: support other JSON types: "object", "array"
def __setitem__(self, key, value):
"""Process collateral effects in params that are related.
Changes in *_prefix should reflect in *_uri.
"""
def _key_is_undefined(key):
try:
value = dict.__getitem__(self, key)
return not value or value == '_'
except KeyError:
return True
if key == 'graph_uri':
graph_uri_value = safe_slug_to_prefix(value)
dict.__setitem__(self, key, graph_uri_value)
with safe_split():
if graph_uri_value and _key_is_undefined('context_name'):
# FIXME: the code below should disappear after #10602 - Normalização no tratamento de parâmetros no Brainiak
context_name_value = _MAP_PREFIX_TO_SLUG.get(graph_uri_value, '_')
dict.__setitem__(self, 'context_name', context_name_value)
elif key == 'class_uri':
class_uri_value = expand_uri(value)
dict.__setitem__(self, key, class_uri_value)
with safe_split():
# FIXME: the code below should disappear after #10602 - Normalização no tratamento de parâmetros no Brainiak
if class_uri_value and _key_is_undefined('class_name'):
class_name_value = class_uri_value.split("/")[-1]
dict.__setitem__(self, 'class_name', class_name_value)
# FIXME: the code below should disappear after #10602 - Normalização no tratamento de parâmetros no Brainiak
# class_prefix must be set when class_uri is defined, because it is used by json-schema logic.
if class_uri_value:
class_prefix_value = "/".join(class_uri_value.split("/")[:-1]) + "/"
dict.__setitem__(self, 'class_prefix', class_prefix_value)
elif key == "context_name":
dict.__setitem__(self, key, value)
uri = safe_slug_to_prefix(value)
dict.__setitem__(self, "graph_uri", uri)
dict.__setitem__(self, "class_prefix", uri)
elif key == "class_name":
dict.__setitem__(self, key, value)
class_prefix = self["class_prefix"]
if not class_prefix.endswith('/'):
class_prefix += "/"
dict.__setitem__(self, "class_uri", "{0}{1}".format(class_prefix, self["class_name"]))
elif key == "instance_id":
dict.__setitem__(self, key, value)
dict.__setitem__(self, "instance_uri", u"{0}{1}/{2}".format(self["class_prefix"], self["class_name"], self["instance_id"]))
dict.__setitem__(self, "instance_prefix", extract_prefix(self["instance_uri"]))
elif key == "class_prefix":
dict.__setitem__(self, key, safe_slug_to_prefix(value))
dict.__setitem__(self, "class_uri", u"{0}{1}".format(self["class_prefix"], self["class_name"]))
elif key == "instance_prefix":
dict.__setitem__(self, key, safe_slug_to_prefix(value))
dict.__setitem__(self, "instance_uri", u"{0}{1}".format(self["instance_prefix"], self["instance_id"]))
elif key == "instance_uri":
dict.__setitem__(self, key, value)
dict.__setitem__(self, "instance_prefix", extract_prefix(value))
else:
dict.__setitem__(self, key, value)
def test_expand_uri_that_is_already_a_uri(self):
self.assertEqual("http://oi", expand_uri("http://oi"))
def add_prefix(items_list, class_prefix):
for item in items_list:
uri = item["@id"]
item["instance_prefix"] = extract_prefix(uri)
item["class_prefix"] = expand_uri(class_prefix)
def test_expand_uri(self):
self.assertEqual("http://www.w3.org/2003/01/geo/wgs84_pos#Brasil",
expand_uri("geo:Brasil"))
def __setitem__(self, key, value):
"""Process collateral effects in params that are related.
Changes in *_prefix should reflect in *_uri.
"""
def _key_is_undefined(key):
try:
value = dict.__getitem__(self, key)
return not value or value == '_'
except KeyError:
return True
if key == 'graph_uri':
graph_uri_value = safe_slug_to_prefix(value)
dict.__setitem__(self, key, graph_uri_value)
with safe_split():
if graph_uri_value and _key_is_undefined('context_name'):
# FIXME: the code below should disappear after #10602 - Normalização no tratamento de parâmetros no Brainiak
context_name_value = graph_uri_value.split("/")[-2]
dict.__setitem__(self, 'context_name', context_name_value)
elif key == 'class_uri':
class_uri_value = expand_uri(value)
dict.__setitem__(self, key, class_uri_value)
with safe_split():
# FIXME: the code below should disappear after #10602 - Normalização no tratamento de parâmetros no Brainiak
if class_uri_value and _key_is_undefined('class_name'):
class_name_value = class_uri_value.split("/")[-1]
dict.__setitem__(self, 'class_name', class_name_value)
# FIXME: the code below should disappear after #10602 - Normalização no tratamento de parâmetros no Brainiak
# class_prefix must be set when class_uri is defined, because it is used by json-schema logic.
if class_uri_value:
class_prefix_value = "/".join(
class_uri_value.split("/")[:-1]) + "/"
dict.__setitem__(self, 'class_prefix', class_prefix_value)
elif key == "context_name":
dict.__setitem__(self, key, value)
uri = safe_slug_to_prefix(value)
dict.__setitem__(self, "graph_uri", uri)
dict.__setitem__(self, "class_prefix", uri)
elif key == "class_name":
dict.__setitem__(self, key, value)
class_prefix = self["class_prefix"]
if not class_prefix.endswith('/'):
class_prefix += "/"
dict.__setitem__(self, "class_uri",
"{0}{1}".format(class_prefix, self["class_name"]))
elif key == "instance_id":
dict.__setitem__(self, key, value)
dict.__setitem__(
self, "instance_uri",
u"{0}{1}/{2}".format(self["class_prefix"], self["class_name"],
self["instance_id"]))
dict.__setitem__(self, "instance_prefix",
extract_prefix(self["instance_uri"]))
elif key == "class_prefix":
dict.__setitem__(self, key, safe_slug_to_prefix(value))
dict.__setitem__(
self, "class_uri", u"{0}{1}".format(self["class_prefix"],
self["class_name"]))
elif key == "instance_prefix":
dict.__setitem__(self, key, safe_slug_to_prefix(value))
dict.__setitem__(
self, "instance_uri", u"{0}{1}".format(self["instance_prefix"],
self["instance_id"]))
elif key == "instance_uri":
dict.__setitem__(self, key, value)
dict.__setitem__(self, "instance_prefix", extract_prefix(value))
else:
dict.__setitem__(self, key, value)
def test_expand_uri_that_is_already_a_uri_with_https(self):
self.assertEqual("https://secure", expand_uri("https://secure"))
def test_expand_uri_with_value_string_containing_double_colon(self):
misleading_value = "Some value: this is no CURIE"
self.assertEqual(misleading_value, expand_uri(misleading_value))
def test_expand_uri_with_None(self):
self.assertEqual('', expand_uri(None))
... '"1"^^<http://www.w3.org/2001/XMLSchema#int>'
"""
assert is_uri(predicate_datatype)
template = u'"{0}"^^<{1}>'
return template.format(value, predicate_datatype)
SPARQLFY_MAP = {
"rdf:XMLLiteral": sparqlfy_string,
"rdfs:Literal": sparqlfy_string,
"xsd:string": sparqlfy_string,
"xsd:boolean": sparqlfy_boolean
}
SPARQLFY_MAP_EXPANDED = {expand_uri(k): v for k, v in SPARQLFY_MAP.items()}
def sparqlfy(value, predicate_datatype):
"""
Create SPARQL-friendly string representation of the value, based on the
predicate_datatype.
Examples:
>>> sparqlfy("http://expanded.uri", "http://www.w3.org/2001/XMLSchema#string")
... "http://expanded.uri"
>>> sparqlfy(True, "http://www.w3.org/2001/XMLSchema#boolean")
... '"true"^^<http://www.w3.org/2001/XMLSchema#boolean>'
"xsd:nonPositiveInteger": "integer",
"xsd:nonNegativeInteger": "integer",
"xsd:negativeInteger": "integer",
"xsd:positiveInteger": "integer",
"xsd:long": "integer",
"xsd:int": "integer",
"xsd:short": "integer",
"xsd:byte": "integer",
"xsd:decimal": "integer",
"xsd:unsignedLong": "integer",
"xsd:unsignedInt": "integer",
"xsd:unsignedShort": "integer",
"xsd:unsignedByte": "integer",
"xsd:boolean": "boolean"
}
_MAP_EXPAND_XSD_TO_JSON_TYPE = {expand_uri(k): v for k, v in _MAP_XSD_TO_JSON_TYPE.items()}
_MAP_JSON_TO_XSD_TYPE = {
"string": "xsd:string",
"number": "xsd:double",
"integer": "xsd:integer",
"boolean": "xsd:boolean"
}
_MAP_JSON_TO_EXPAND_XSD_TYPE = {k: expand_uri(v) for k, v in _MAP_XSD_TO_JSON_TYPE.items()}
# TODO: support other JSON types: "object", "array"
#_MAP_JSON_TO_PYTHON = {
# "string": unicode,
# "number": float,
# "integer": int,
