def mock_schema(properties_and_types_dict, id, context=None): properties_schema = {} type2datatype = {'string': 'xsd:string', 'integer': 'xsd:integer', 'date': 'xsd:date', 'datetime': 'xsd:dateTime', 'boolean': 'xsd:boolean'} for property_name, type_value in properties_and_types_dict.items(): property_uri = expand_uri(property_name, context=context) if type_value in ["string_uri", "array_string_uri"]: properties_schema[property_uri] = { 'range': {'type': 'string', 'format': 'uri'} } if type_value == "string_uri": properties_schema[property_uri]["type"] = "string" else: properties_schema[property_uri]["type"] = "array" properties_schema[property_uri]['items'] = {'type': 'string', 'format': 'uri'} else: properties_schema[property_uri] = {'type': type_value, 'datatype': expand_uri(type2datatype[type_value], context=context)} schema = {'properties': properties_schema} if id is not None: schema["id"] = id return schema
def cast_item(item, property_to_type): """ Casts values of properties, according to the mapping provided. Example: item = { "team": "Semantic Team", "grade": "10" } property_to_type = { "team": str, "grade": int } cast_item(item, property_to_type) Returns: { "team": "Semantic Team", "grade": 10 } """ new_item = {} for property_, value in item.items(): type_ = property_to_type.get(expand_uri(property_)) if not type_ is None: if isinstance(value, list): value = [type_(v) for v in value] elif type_ is bool: value = bool(int(value)) else: value = type_(value) if isinstance(value, list): value = sorted(value) new_item[property_] = value return new_item
def build_json(items_list, query_params): class_url = build_class_url(query_params) schema_url = unquote(build_schema_url_for_instance(query_params, class_url)) class_properties = get_class.get_cached_schema(query_params)["properties"] items_list = cast_items_values(items_list, class_properties) json = { '_schema_url': schema_url, 'pattern': '', '_class_prefix': expand_uri(query_params['class_prefix']), '_base_url': remove_last_slash(query_params.base_url), 'items': items_list, '@context': {"@language": query_params.get("lang")}, '@id': query_params['class_uri'] } def calculate_total_items(): result_dict = query_count_filter_instances(query_params) total_items = int(get_one_value(result_dict, 'total')) return total_items decorate_dict_with_pagination(json, query_params, calculate_total_items) return json
def filter_instances(query_params): if not class_exists(query_params): error_message = u"Class {0} in graph {1} does not exist".format( query_params["class_uri"], query_params["graph_uri"]) raise HTTPError(404, log_message=error_message) keymap = { "label": "title", "subject": "@id", "sort_object": shorten_uri(query_params["sort_by"]), } for p, o, index in extract_po_tuples(query_params): keymap[o[1:]] = shorten_uri(p) result_dict = query_filter_instances(query_params) if not result_dict or not result_dict['results']['bindings']: return None items_list = compress_keys_and_values( result_dict, keymap=keymap, ignore_keys=["total"], do_expand_uri=query_params['expand_uri'] == u"1") items_list = merge_by_id(items_list) for item in items_list: uri = item["@id"] item["instance_prefix"] = u"{0}/".format(uri.rsplit('/', 1)[0]) item["class_prefix"] = expand_uri(query_params['class_prefix']) decorate_with_resource_id(items_list) return build_json(items_list, query_params)
def _override_with(self, handler): "Override this dictionary with values whose keys are present in the request" # order is critical below because *_uri should be set before *_prefix for key in self.arguments: if (key not in self) and (not self._matches_dynamic_pattern(key)): raise InvalidParam(key) value = self.arguments.get(key, None) if value is not None: try: self[key] = expand_uri(value) except KeyError as ex: raise RequiredParamMissing(unicode(ex))
def _override_with(self, handler): "Override this dictionary with values whose keys are present in the request" # order is critical below because *_uri should be set before *_prefix for key in self.arguments: if (key not in self) and (not self._matches_dynamic_pattern(key)): raise InvalidParam(key) value = self.arguments.get(key, None) if value is not None: try: self[key] = expand_uri(value) except KeyError as ex: raise RequiredParamMissing(unicode(ex))
def mock_schema(properties_and_types_dict, id, context=None): properties_schema = {} type2datatype = { 'string': 'xsd:string', 'integer': 'xsd:integer', 'date': 'xsd:date', 'datetime': 'xsd:dateTime', 'boolean': 'xsd:boolean' } for property_name, type_value in properties_and_types_dict.items(): property_uri = expand_uri(property_name, context=context) if type_value in ["string_uri", "array_string_uri"]: properties_schema[property_uri] = { 'range': { 'type': 'string', 'format': 'uri' } } if type_value == "string_uri": properties_schema[property_uri]["type"] = "string" else: properties_schema[property_uri]["type"] = "array" properties_schema[property_uri]['items'] = { 'type': 'string', 'format': 'uri' } else: properties_schema[property_uri] = { 'type': type_value, 'datatype': expand_uri(type2datatype[type_value], context=context) } schema = {'properties': properties_schema} if id is not None: schema["id"] = id return schema
def compress_keys_and_values(result_dict, keymap={}, ignore_keys=[], context=None, do_expand_uri=False): """ Return a list of compressed items of the 'bindings' list of a Virtuoso response dict. Usage: >>> result_dict = {'results': {'bindings': [{'key': {'type': 'some type', 'value': 'some value'}}, \ {'key': {'type': 'some type', 'value': 'another value'}}]}} >>> compress_keys_and_values(result_dict) [{'key': 'some value'}, {'key': 'another value'}] Optional params: - keymap = {'key': 'renamed_key'}: renames resulting dict 'key' by 'renamed_key' - ignore_keys = ['key']: list of keys that shouldn't be returned - context (instance of MemorizeContext): shortens URIs according to provided context >>> compress_keys_and_values(result_dict, keymap={'key': 'renamed_key'}) [{'renamed_key': 'some value'}, {'renamed_key': 'another value'}] >>> compress_keys_and_values(result_dict, ignore_keys=['key']) [] >>> from brainiak.prefixes import MemorizeContext >>> result_dict = {'results': {'bindings': [{'key': {'type': 'uri', 'value': 'http://xmlns.com/foaf/0.1/value'}}]}} >>> context = MemorizeContext() >>> compress_keys_and_values(result_dict, context=context) [{'key': 'foaf:value'}] """ result_list = [] for item in result_dict['results']['bindings']: row = {} for key in item: if key not in ignore_keys: value = item[key]['value'] effective_key = keymap.get(key, key) if item[key][ 'type'] == 'uri' and context and effective_key != '@id' and not do_expand_uri: value = context.shorten_uri(value) if do_expand_uri: effective_key = expand_uri(effective_key) row[effective_key] = value result_list.append(row) return result_list
def decorate_with_resource_id(list_of_dicts): """Adds to each entry in the input parameter list_of_dicts a new key`resource_id` created from the last path segment given by the key `@id`""" for dict_item in list_of_dicts: try: id_key = expand_uri(dict_item["@id"]) title = dict_item.get("title") if title == ROOT_CONTEXT: resource_id = u'' elif id_key.endswith("/"): resource_id = id_key.rsplit("/")[-2] else: resource_id = id_key.rsplit("/")[-1] dict_item['resource_id'] = unicode(resource_id) except KeyError as ex: raise TypeError(u"dict missing key {0:s} while processing decorate_with_resource_id()".format(ex))
def decorate_with_resource_id(list_of_dicts): """Adds to each entry in the input parameter list_of_dicts a new key`resource_id` created from the last path segment given by the key `@id`""" for dict_item in list_of_dicts: try: id_key = expand_uri(dict_item["@id"]) title = dict_item.get("title") if title == ROOT_CONTEXT: resource_id = u'' elif id_key.endswith("/"): resource_id = id_key.rsplit("/")[-2] else: resource_id = id_key.rsplit("/")[-1] dict_item['resource_id'] = str(resource_id) except KeyError as ex: raise TypeError( u"dict missing key {0:s} while processing decorate_with_resource_id()" .format(ex))
def compress_keys_and_values(result_dict, keymap={}, ignore_keys=[], context=None, do_expand_uri=False): """ Return a list of compressed items of the 'bindings' list of a Virtuoso response dict. Usage: >>> result_dict = {'results': {'bindings': [{'key': {'type': 'some type', 'value': 'some value'}}, \ {'key': {'type': 'some type', 'value': 'another value'}}]}} >>> compress_keys_and_values(result_dict) [{'key': 'some value'}, {'key': 'another value'}] Optional params: - keymap = {'key': 'renamed_key'}: renames resulting dict 'key' by 'renamed_key' - ignore_keys = ['key']: list of keys that shouldn't be returned - context (instance of MemorizeContext): shortens URIs according to provided context >>> compress_keys_and_values(result_dict, keymap={'key': 'renamed_key'}) [{'renamed_key': 'some value'}, {'renamed_key': 'another value'}] >>> compress_keys_and_values(result_dict, ignore_keys=['key']) [] >>> from brainiak.prefixes import MemorizeContext >>> result_dict = {'results': {'bindings': [{'key': {'type': 'uri', 'value': 'http://xmlns.com/foaf/0.1/value'}}]}} >>> context = MemorizeContext() >>> compress_keys_and_values(result_dict, context=context) [{'key': 'foaf:value'}] """ result_list = [] for item in result_dict['results']['bindings']: row = {} for key in item: if key not in ignore_keys: value = item[key]['value'] effective_key = keymap.get(key, key) if item[key]['type'] == 'uri' and context and effective_key != '@id' and not do_expand_uri: value = context.shorten_uri(value) if do_expand_uri: effective_key = expand_uri(effective_key) row[effective_key] = value result_list.append(row) return result_list
def normalize_term(term, language=""): """ Provided a query term (literal, variable, expanded uri or compressed uri), and language (to be applied in literals), return the term in the form so it can be used inside a SPARQL Query. examples: (1) http://expanded.predicate -> <http://expanded.predicate> (2) compressed:predicate -> compressed:predicate (3) "some literal" -> '"some literal"@lang' (4) ?variable -> ?variable """ language_tag = u"@%s" % language if language else u"" if (not term.startswith("?")): if (":" in term): expanded_term = expand_uri(term) if expanded_term != term or is_uri(expanded_term): term = u"<%s>" % expanded_term else: term = u'"%s"%s' % (term, language_tag) return term
def normalize_term(term, language=""): """ Provided a query term (literal, variable, expanded uri or compressed uri), and language (to be applied in literals), return the term in the form so it can be used inside a SPARQL Query. examples: (1) http://expanded.predicate -> <http://expanded.predicate> (2) compressed:predicate -> compressed:predicate (3) "some literal" -> '"some literal"@lang' (4) ?variable -> ?variable """ language_tag = u"@%s" % language if language else u"" if (not term.startswith("?")): if (":" in term): expanded_term = expand_uri(term) if expanded_term != term or is_uri(expanded_term): term = u"<%s>" % expanded_term else: term = u'"%s"%s' % (term, language_tag) return term
def normalize_class_prefix(class_prefix, should_expand_uri): return expand_uri(class_prefix) if should_expand_uri else shorten_uri( class_prefix)
def test_expand_uri_whatever(self): self.assertEqual("http://schema.org/whatever", expand_uri("schema:whatever"))
... '"1"^^<http://www.w3.org/2001/XMLSchema#int>' """ assert is_uri(predicate_datatype) template = u'"{0}"^^<{1}>' return template.format(value, predicate_datatype) SPARQLFY_MAP = { "rdf:XMLLiteral": sparqlfy_string, "rdfs:Literal": sparqlfy_string, "xsd:string": sparqlfy_string, "xsd:boolean": sparqlfy_boolean } SPARQLFY_MAP_EXPANDED = {expand_uri(k): v for k, v in SPARQLFY_MAP.items()} def sparqlfy(value, predicate_datatype): """ Create SPARQL-friendly string representation of the value, based on the predicate_datatype. Examples: >>> sparqlfy("http://expanded.uri", "http://www.w3.org/2001/XMLSchema#string") ... "http://expanded.uri" >>> sparqlfy(True, "http://www.w3.org/2001/XMLSchema#boolean") ... '"true"^^<http://www.w3.org/2001/XMLSchema#boolean>'
"xsd:nonNegativeInteger": "integer", "xsd:negativeInteger": "integer", "xsd:positiveInteger": "integer", "xsd:long": "integer", "xsd:int": "integer", "xsd:short": "integer", "xsd:byte": "integer", "xsd:decimal": "integer", "xsd:unsignedLong": "integer", "xsd:unsignedInt": "integer", "xsd:unsignedShort": "integer", "xsd:unsignedByte": "integer", "xsd:boolean": "boolean" } _MAP_EXPAND_XSD_TO_JSON_TYPE = { expand_uri(k): v for k, v in _MAP_XSD_TO_JSON_TYPE.items() } _MAP_JSON_TO_XSD_TYPE = { "string": "xsd:string", "number": "xsd:double", "integer": "xsd:integer", "boolean": "xsd:boolean" } _MAP_JSON_TO_EXPAND_XSD_TYPE = { k: expand_uri(v) for k, v in _MAP_XSD_TO_JSON_TYPE.items() } # TODO: support other JSON types: "object", "array"
def __setitem__(self, key, value): """Process collateral effects in params that are related. Changes in *_prefix should reflect in *_uri. """ def _key_is_undefined(key): try: value = dict.__getitem__(self, key) return not value or value == '_' except KeyError: return True if key == 'graph_uri': graph_uri_value = safe_slug_to_prefix(value) dict.__setitem__(self, key, graph_uri_value) with safe_split(): if graph_uri_value and _key_is_undefined('context_name'): # FIXME: the code below should disappear after #10602 - Normalização no tratamento de parâmetros no Brainiak context_name_value = _MAP_PREFIX_TO_SLUG.get(graph_uri_value, '_') dict.__setitem__(self, 'context_name', context_name_value) elif key == 'class_uri': class_uri_value = expand_uri(value) dict.__setitem__(self, key, class_uri_value) with safe_split(): # FIXME: the code below should disappear after #10602 - Normalização no tratamento de parâmetros no Brainiak if class_uri_value and _key_is_undefined('class_name'): class_name_value = class_uri_value.split("/")[-1] dict.__setitem__(self, 'class_name', class_name_value) # FIXME: the code below should disappear after #10602 - Normalização no tratamento de parâmetros no Brainiak # class_prefix must be set when class_uri is defined, because it is used by json-schema logic. if class_uri_value: class_prefix_value = "/".join(class_uri_value.split("/")[:-1]) + "/" dict.__setitem__(self, 'class_prefix', class_prefix_value) elif key == "context_name": dict.__setitem__(self, key, value) uri = safe_slug_to_prefix(value) dict.__setitem__(self, "graph_uri", uri) dict.__setitem__(self, "class_prefix", uri) elif key == "class_name": dict.__setitem__(self, key, value) class_prefix = self["class_prefix"] if not class_prefix.endswith('/'): class_prefix += "/" dict.__setitem__(self, "class_uri", "{0}{1}".format(class_prefix, self["class_name"])) elif key == "instance_id": dict.__setitem__(self, key, value) dict.__setitem__(self, "instance_uri", u"{0}{1}/{2}".format(self["class_prefix"], self["class_name"], self["instance_id"])) dict.__setitem__(self, "instance_prefix", extract_prefix(self["instance_uri"])) elif key == "class_prefix": dict.__setitem__(self, key, safe_slug_to_prefix(value)) dict.__setitem__(self, "class_uri", u"{0}{1}".format(self["class_prefix"], self["class_name"])) elif key == "instance_prefix": dict.__setitem__(self, key, safe_slug_to_prefix(value)) dict.__setitem__(self, "instance_uri", u"{0}{1}".format(self["instance_prefix"], self["instance_id"])) elif key == "instance_uri": dict.__setitem__(self, key, value) dict.__setitem__(self, "instance_prefix", extract_prefix(value)) else: dict.__setitem__(self, key, value)
def test_expand_uri_that_is_already_a_uri(self): self.assertEqual("http://oi", expand_uri("http://oi"))
def add_prefix(items_list, class_prefix): for item in items_list: uri = item["@id"] item["instance_prefix"] = extract_prefix(uri) item["class_prefix"] = expand_uri(class_prefix)
def test_expand_uri(self): self.assertEqual("http://www.w3.org/2003/01/geo/wgs84_pos#Brasil", expand_uri("geo:Brasil"))
def __setitem__(self, key, value): """Process collateral effects in params that are related. Changes in *_prefix should reflect in *_uri. """ def _key_is_undefined(key): try: value = dict.__getitem__(self, key) return not value or value == '_' except KeyError: return True if key == 'graph_uri': graph_uri_value = safe_slug_to_prefix(value) dict.__setitem__(self, key, graph_uri_value) with safe_split(): if graph_uri_value and _key_is_undefined('context_name'): # FIXME: the code below should disappear after #10602 - Normalização no tratamento de parâmetros no Brainiak context_name_value = graph_uri_value.split("/")[-2] dict.__setitem__(self, 'context_name', context_name_value) elif key == 'class_uri': class_uri_value = expand_uri(value) dict.__setitem__(self, key, class_uri_value) with safe_split(): # FIXME: the code below should disappear after #10602 - Normalização no tratamento de parâmetros no Brainiak if class_uri_value and _key_is_undefined('class_name'): class_name_value = class_uri_value.split("/")[-1] dict.__setitem__(self, 'class_name', class_name_value) # FIXME: the code below should disappear after #10602 - Normalização no tratamento de parâmetros no Brainiak # class_prefix must be set when class_uri is defined, because it is used by json-schema logic. if class_uri_value: class_prefix_value = "/".join( class_uri_value.split("/")[:-1]) + "/" dict.__setitem__(self, 'class_prefix', class_prefix_value) elif key == "context_name": dict.__setitem__(self, key, value) uri = safe_slug_to_prefix(value) dict.__setitem__(self, "graph_uri", uri) dict.__setitem__(self, "class_prefix", uri) elif key == "class_name": dict.__setitem__(self, key, value) class_prefix = self["class_prefix"] if not class_prefix.endswith('/'): class_prefix += "/" dict.__setitem__(self, "class_uri", "{0}{1}".format(class_prefix, self["class_name"])) elif key == "instance_id": dict.__setitem__(self, key, value) dict.__setitem__( self, "instance_uri", u"{0}{1}/{2}".format(self["class_prefix"], self["class_name"], self["instance_id"])) dict.__setitem__(self, "instance_prefix", extract_prefix(self["instance_uri"])) elif key == "class_prefix": dict.__setitem__(self, key, safe_slug_to_prefix(value)) dict.__setitem__( self, "class_uri", u"{0}{1}".format(self["class_prefix"], self["class_name"])) elif key == "instance_prefix": dict.__setitem__(self, key, safe_slug_to_prefix(value)) dict.__setitem__( self, "instance_uri", u"{0}{1}".format(self["instance_prefix"], self["instance_id"])) elif key == "instance_uri": dict.__setitem__(self, key, value) dict.__setitem__(self, "instance_prefix", extract_prefix(value)) else: dict.__setitem__(self, key, value)
def test_expand_uri_that_is_already_a_uri_with_https(self): self.assertEqual("https://secure", expand_uri("https://secure"))
def test_expand_uri_with_value_string_containing_double_colon(self): misleading_value = "Some value: this is no CURIE" self.assertEqual(misleading_value, expand_uri(misleading_value))
def test_expand_uri_with_None(self): self.assertEqual('', expand_uri(None))
... '"1"^^<http://www.w3.org/2001/XMLSchema#int>' """ assert is_uri(predicate_datatype) template = u'"{0}"^^<{1}>' return template.format(value, predicate_datatype) SPARQLFY_MAP = { "rdf:XMLLiteral": sparqlfy_string, "rdfs:Literal": sparqlfy_string, "xsd:string": sparqlfy_string, "xsd:boolean": sparqlfy_boolean } SPARQLFY_MAP_EXPANDED = {expand_uri(k): v for k, v in SPARQLFY_MAP.items()} def sparqlfy(value, predicate_datatype): """ Create SPARQL-friendly string representation of the value, based on the predicate_datatype. Examples: >>> sparqlfy("http://expanded.uri", "http://www.w3.org/2001/XMLSchema#string") ... "http://expanded.uri" >>> sparqlfy(True, "http://www.w3.org/2001/XMLSchema#boolean") ... '"true"^^<http://www.w3.org/2001/XMLSchema#boolean>'
"xsd:nonPositiveInteger": "integer", "xsd:nonNegativeInteger": "integer", "xsd:negativeInteger": "integer", "xsd:positiveInteger": "integer", "xsd:long": "integer", "xsd:int": "integer", "xsd:short": "integer", "xsd:byte": "integer", "xsd:decimal": "integer", "xsd:unsignedLong": "integer", "xsd:unsignedInt": "integer", "xsd:unsignedShort": "integer", "xsd:unsignedByte": "integer", "xsd:boolean": "boolean" } _MAP_EXPAND_XSD_TO_JSON_TYPE = {expand_uri(k): v for k, v in _MAP_XSD_TO_JSON_TYPE.items()} _MAP_JSON_TO_XSD_TYPE = { "string": "xsd:string", "number": "xsd:double", "integer": "xsd:integer", "boolean": "xsd:boolean" } _MAP_JSON_TO_EXPAND_XSD_TYPE = {k: expand_uri(v) for k, v in _MAP_XSD_TO_JSON_TYPE.items()} # TODO: support other JSON types: "object", "array" #_MAP_JSON_TO_PYTHON = { # "string": unicode, # "number": float, # "integer": int,