def edit_instance(query_params, instance_data): if not are_there_label_properties_in(instance_data): raise HTTPError( 400, log_message= _(u"Label properties like rdfs:label or its subproperties are required" )) if must_retrieve_graph_and_class_uri(query_params): triplestore_response = get_class_and_graph(query_params) bindings = triplestore_response['results']['bindings'] query_params['graph_uri'] = extract_graph_uri(bindings) query_params['class_uri'] = extract_class_uri(bindings) try: instance_uri = query_params['instance_uri'] graph_uri = query_params['graph_uri'] class_uri = query_params['class_uri'] except KeyError as ex: raise HTTPError( 404, log_message=_( u"Parameter <{0:s}> is missing in order to update instance.". format(ex))) class_object = get_cached_schema(query_params) try: triples = create_explicit_triples(instance_uri, instance_data, class_object, graph_uri, query_params) except InstanceError, exception: raise HTTPError(400, log_message=exception.message)
def create_explicit_triples(instance_uri, instance_data, class_object, graph_uri, query_params): class_id = class_object["id"] copy_instance_data = instance_data.copy() predicate_object_tuples = unpack_tuples(copy_instance_data) triples = [] errors = [] template_msg = _(u'Incorrect value for property ({1}). A value compatible with a ({2}) was expected, but ({0}) was given.') for (predicate_uri, object_value) in predicate_object_tuples: if not is_reserved_attribute(predicate_uri): predicate_has_error = False try: predicate_datatype = get_predicate_datatype(class_object, predicate_uri) except KeyError: template = _(u'Inexistent property ({0}) in the schema ({1}), used to create instance ({2})') msg = template.format(predicate_uri, class_id, instance_uri) errors.append(msg) predicate_datatype = None predicate_has_error = True else: if predicate_datatype is None: # ObjectProperty try: object_ = sparqlfy_object(object_value) except InstanceError: msg = template_msg.format(object_value, predicate_uri, "owl:ObjectProperty") errors.append(msg) predicate_has_error = True else: if is_instance(object_value, predicate_datatype): object_ = sparqlfy(object_value, predicate_datatype) else: msg = template_msg.format(object_value, predicate_uri, predicate_datatype) errors.append(msg) predicate_has_error = True if not errors: instance = sparqlfy_object(instance_uri) predicate = sparqlfy_object(predicate_uri) triple = (instance, predicate, object_) triples.append(triple) if not predicate_has_error: if property_must_map_a_unique_value(class_object, predicate_uri): if is_value_already_used(instance_uri, object_, predicate_uri, class_object, graph_uri, query_params): template = _(u"The property ({0}) defined in the schema ({1}) must map a unique value. The value provided ({2}) is already used by another instance.") msg = template.format(predicate_uri, class_id, object_value) errors.append(msg) undefined_obligatory_properties = find_undefined_obligatory_properties(class_object, instance_data) template = _(u"The property ({0}) is obligatory according to the definition of the class ({1}). A value must be provided for this field in order to create or edit ({2}).") for property_ in undefined_obligatory_properties: msg = template.format(property_, class_id, instance_uri) errors.append(msg) if errors: error_msg = json.dumps(errors) raise InstanceError(error_msg) return triples
def create_explicit_triples(instance_uri, instance_data, class_object, graph_uri, query_params): class_id = class_object["id"] copy_instance_data = instance_data.copy() predicate_object_tuples = unpack_tuples(copy_instance_data) triples = [] errors = [] template_msg = _(u'Incorrect value for property ({1}). A value compatible with a ({2}) was expected, but ({0}) was given.') for (predicate_uri, object_value) in predicate_object_tuples: if not is_reserved_attribute(predicate_uri): predicate_has_error = False try: predicate_datatype = get_predicate_datatype(class_object, predicate_uri) except KeyError: template = _(u'Inexistent property ({0}) in the schema ({1}), used to create instance ({2})') msg = template.format(predicate_uri, class_id, instance_uri) errors.append(msg) predicate_datatype = None predicate_has_error = True else: if predicate_datatype is None: # ObjectProperty try: object_ = sparqlfy_object(object_value) except InstanceError: msg = template_msg.format(object_value, predicate_uri, "owl:ObjectProperty") errors.append(msg) predicate_has_error = True else: if is_instance(object_value, predicate_datatype): object_ = sparqlfy(object_value, predicate_datatype) else: msg = template_msg.format(object_value, predicate_uri, predicate_datatype) errors.append(msg) predicate_has_error = True if not errors: instance = sparqlfy_object(instance_uri) predicate = sparqlfy_object(predicate_uri) triple = (instance, predicate, object_) triples.append(triple) if not predicate_has_error: if property_must_map_a_unique_value(class_object, predicate_uri): if is_value_already_used(instance_uri, object_, predicate_uri, class_object, graph_uri, query_params): template = _(u"The property ({0}) defined in the schema ({1}) must map a unique value. The value provided ({2}) is already used by another instance.") msg = template.format(predicate_uri, class_id, object_value) errors.append(msg) undefined_obligatory_properties = find_undefined_obligatory_properties(class_object, instance_data) template = _(u"The property ({0}) is obligatory according to the definition of the class ({1}). A value must be provided for this field in order to create or edit ({2}).") for property_ in undefined_obligatory_properties: msg = template.format(property_, class_id, instance_uri) errors.append(msg) if errors: error_msg = json.dumps(errors) raise InstanceError(error_msg) return triples
def build_items_dict(bindings, class_uri, expand_object_properties, class_schema): super_predicates = get_super_properties(bindings) items_dict = {} for item in bindings: is_object_blank_node = int( item.get("is_object_blank", {}).get("value", "0")) if not is_object_blank_node: predicate_uri = item["predicate"]["value"] object_value = item["object"]["value"] object_type = item["object"].get("type") object_label = item.get("object_label", {}).get("value") if expand_object_properties and object_type == "uri": if object_label: value = {"@id": object_value, "title": object_label} else: msg = _(u"The predicate {0} refers to an object {1} which doesn't have a label.").format(predicate_uri, object_value) + \ _(" Set expand_object_properties=0 if you don't care about this ontological inconsistency.") value = {"@id": object_value} logger.debug(msg) # raise Exception(msg) else: value = _convert_to_python(object_value, class_schema, predicate_uri) if predicate_uri in items_dict: if not isinstance(items_dict[predicate_uri], list): value_list = [items_dict[predicate_uri]] else: value_list = items_dict[predicate_uri] value_list.append(value) items_dict[predicate_uri] = value_list else: base_uri = None if predicate_uri in class_schema["properties"]: base_uri = predicate_uri # Here we IGNORE predicate/object values that exist in the triplestore # but the predicate is not in the class schema if base_uri: if class_schema["properties"][base_uri][ u'type'] == u'array': items_dict[predicate_uri] = [value] else: items_dict[predicate_uri] = value remove_super_properties(items_dict, super_predicates) if not class_uri is None: items_dict[ "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] = class_uri return items_dict
def post(self, context_name, class_name): valid_params = CLASS_PARAMS with safe_params(valid_params): self.query_params = ParamDict(self, context_name=context_name, class_name=class_name, **valid_params) del context_name del class_name try: schema = schema_resource.get_cached_schema(self.query_params) except SchemaNotFound: schema = None if schema is None: class_uri = self.query_params["class_uri"] graph_uri = self.query_params["graph_uri"] raise HTTPError( 404, log_message=_( u"Class {0} doesn't exist in context {1}.").format( class_uri, graph_uri)) try: instance_data = json.loads(self.request.body) except ValueError: raise HTTPError(400, log_message=_(u"No JSON object could be decoded")) instance_data = normalize_all_uris_recursively(instance_data) try: (instance_uri, instance_id) = create_instance(self.query_params, instance_data) except InstanceError as ex: raise HTTPError(500, log_message=unicode(ex)) instance_url = self.build_resource_url(instance_id) self.set_header("location", instance_url) self.set_header("X-Brainiak-Resource-URI", instance_uri) self.query_params["instance_uri"] = instance_uri self.query_params["instance_id"] = instance_id self.query_params["expand_object_properties"] = "1" instance_data = get_instance(self.query_params) if settings.NOTIFY_BUS: self._notify_bus(action="POST", instance_data=instance_data) self.finalize(201)
def wrapper(*params): try: response = function(*params) except exceptions: log.logger.error(_(u"CacheError: First try returned {0}").format(traceback.format_exc())) try: global redis_client redis_client = connect() response = function(*params) except exceptions: log.logger.error(_(u"CacheError: Second try returned {0}").format(traceback.format_exc())) response = None return response
def do_suggest(query_params, suggest_params): search_params = suggest_params["search"] range_result = _get_predicate_ranges(query_params, search_params) if is_result_empty(range_result): message = _( u"Either the predicate {0} does not exists or it does not have any rdfs:range defined in the triplestore" ) message = message.format(search_params["target"]) raise_no_results(message) classes = _validate_class_restriction(query_params, range_result) graphs = _validate_graph_restriction(query_params, range_result) indexes = ["semantica." + uri_to_slug(graph) for graph in graphs] search_fields = list( set( _get_search_fields(query_params, suggest_params) + LABEL_PROPERTIES)) response_params = suggest_params.get("response", {}) response_fields = _get_response_fields(query_params, response_params, classes, LABEL_PROPERTIES) request_body = _build_body_query(query_params, search_params, classes, search_fields, response_fields) # Sorting in ES is done using memory. From the docs [1]: # "When sorting, the relevant sorted field values are loaded into memory. # This means that per shard, there should be enough memory to contain them" # Currently Globo.com ES servers don't have enough memory to load all data # During the 30th October 2013, a query using sort caused Split-brain and all ES shards were down. # Therefore, we need to think twice and use sort cleverly to avoid problems # [1] http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-sort.html assert not "sort" in request_body # Read comments above elasticsearch_result = run_search(request_body, indexes=indexes) if elasticsearch_result is None: message = _(u"There were no search results.") raise_no_results(message) class_fields = response_params.get("class_fields", []) total_items = elasticsearch_result["hits"]["total"] if total_items: items = _build_items(query_params, elasticsearch_result, LABEL_PROPERTIES, class_fields) response = build_json(items, total_items, query_params) else: response = {} return response
def purge(pattern): keys_with_pattern = keys(pattern) or [] log.logger.debug(_(u"Cache: key(s) to be deleted: {0}").format(keys_with_pattern)) log_details = _(u"{0} key(s), matching the pattern: {1}").format(len(keys_with_pattern), pattern) response = 1 for key in keys_with_pattern: response *= delete(key) if response and keys_with_pattern: log.logger.info(_(u"Cache: purged with success {0}").format(log_details)) elif not keys_with_pattern: log.logger.info(_(u"Cache: {0}").format(log_details)) else: log.logger.info(_(u"Cache: failed purging {0}").format(log_details))
def _extract_cardinalities(bindings, predicate_dict): cardinalities = {} for binding in bindings: property_ = binding["predicate"]["value"] try: range_ = binding["range"]["value"] except KeyError: try: range_ = predicate_dict[property_]["range"]["value"] except KeyError: msg = _(u"The property {0} is not defined properly").format( property_) raise InstanceError(msg) if not property_ in cardinalities: cardinalities[property_] = {} if not range_ in cardinalities[property_] and not range_.startswith( "nodeID://"): cardinalities[property_][range_] = {} current_property = cardinalities[property_] if "min" in binding: min_value = binding["min"]["value"] try: min_value = int(min_value) except ValueError: msg = _( u"The property {0} defines a non-integer owl:minQualifiedCardinality {1}" ).format(property_, min_value) raise InstanceError(msg) else: current_property[range_].update({"minItems": min_value}) if min_value: current_property[range_].update({"required": True}) if "max" in binding: max_value = binding["max"]["value"] try: max_value = int(max_value) except ValueError: msg = _( u"The property {0} defines a non-integer owl:maxQualifiedCardinality {1}" ).format(property_, max_value) raise InstanceError(msg) else: current_property[range_].update({"maxItems": max_value}) return cardinalities
def build_items_dict(bindings, class_uri, expand_object_properties, class_schema): super_predicates = get_super_properties(bindings) items_dict = {} for item in bindings: is_object_blank_node = int(item.get("is_object_blank", {}).get("value", "0")) if not is_object_blank_node: predicate_uri = item["predicate"]["value"] object_value = item["object"]["value"] object_type = item["object"].get("type") object_label = item.get("object_label", {}).get("value") if expand_object_properties and object_type == "uri": if object_label: value = {"@id": object_value, "title": object_label} else: msg = _(u"The predicate {0} refers to an object {1} which doesn't have a label.").format(predicate_uri, object_value) + \ _(" Set expand_object_properties=0 if you don't care about this ontological inconsistency.") value = {"@id": object_value} logger.debug(msg) # raise Exception(msg) else: value = _convert_to_python(object_value, class_schema, predicate_uri) if predicate_uri in items_dict: if not isinstance(items_dict[predicate_uri], list): value_list = [items_dict[predicate_uri]] else: value_list = items_dict[predicate_uri] value_list.append(value) items_dict[predicate_uri] = value_list else: base_uri = None if predicate_uri in class_schema["properties"]: base_uri = predicate_uri # Here we IGNORE predicate/object values that exist in the triplestore # but the predicate is not in the class schema if base_uri: if class_schema["properties"][base_uri][u'type'] == u'array': items_dict[predicate_uri] = [value] else: items_dict[predicate_uri] = value remove_super_properties(items_dict, super_predicates) if not class_uri is None: items_dict["http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] = class_uri return items_dict
def get_json_request_as_dict(json_request_body): try: raw_body_params = json.loads(json_request_body) except ValueError: error_message = _("JSON malformed. Received: {0}.") raise HTTPError(400, log_message=error_message.format(json_request_body)) return raw_body_params
def delete(self, context_name, class_name, instance_id): valid_params = INSTANCE_PARAMS with safe_params(valid_params): self.query_params = ParamDict(self, context_name=context_name, class_name=class_name, instance_id=instance_id, **valid_params) del context_name del class_name del instance_id deleted = delete_instance(self.query_params) if deleted: response = 204 if settings.NOTIFY_BUS: self._notify_bus(action="DELETE") cache.purge_an_instance(self.query_params['instance_uri']) else: msg = _(u"Instance ({0}) of class ({1}) in graph ({2}) was not found.") error_message = msg.format(self.query_params["instance_uri"], self.query_params["class_uri"], self.query_params["graph_uri"]) raise HTTPError(404, log_message=error_message) self.finalize(response)
def post(self): valid_params = PAGING_PARAMS with safe_params(valid_params): try: raw_body_params = json.loads(self.request.body) except ValueError: error_message = _("JSON malformed. Received: {0}.") raise HTTPError(400, log_message=error_message.format( self.request.body)) body_params = normalize_all_uris_recursively(raw_body_params) if '@context' in body_params: del body_params['@context'] validate_json_schema(body_params, SUGGEST_PARAM_SCHEMA) self.query_params = ParamDict(self, **valid_params) response = do_suggest(self.query_params, body_params) if self.query_params['expand_uri'] == "0": response = normalize_all_uris_recursively(response, mode=SHORTEN) self.finalize(response)
def delete(self, context_name, class_name, instance_id): valid_params = INSTANCE_PARAMS with safe_params(valid_params): self.query_params = ParamDict(self, context_name=context_name, class_name=class_name, instance_id=instance_id, **valid_params) del context_name del class_name del instance_id deleted = delete_instance(self.query_params) if deleted: response = 204 if settings.NOTIFY_BUS: self._notify_bus(action="DELETE") cache.purge_an_instance(self.query_params['instance_uri']) else: msg = _(u"Instance ({0}) of class ({1}) in graph ({2}) was not found.") error_message = msg.format(self.query_params["instance_uri"], self.query_params["class_uri"], self.query_params["graph_uri"]) raise HTTPError(404, log_message=error_message) self.finalize(response)
def decode_boolean(object_value): if object_value == "0": return False elif object_value == "1": return True else: raise InstanceError(_(u"Could not decode boolean using {0}".format(object_value)))
def convert_bindings_dict(context, bindings, cardinalities, superclasses): super_predicates = get_super_properties(bindings) assembled_predicates = {} for binding_row in bindings: predicate_uri = binding_row['predicate']['value'] # super_predicate is when we use rdfs:subPropertyOf # this case does not consider inherited predicates if predicate_uri in super_predicates.keys(): continue predicate = assemble_predicate(predicate_uri, binding_row, cardinalities, context) existing_predicate = assembled_predicates.get(predicate_uri, False) if existing_predicate: if 'datatype' in existing_predicate and 'datatype' in predicate: assembled_predicates[predicate_uri] = most_specialized_predicate(superclasses, existing_predicate, predicate) elif existing_predicate != predicate: assembled_predicates[predicate_uri] = join_predicates(existing_predicate, predicate) else: msg = _(u"The property {0} seems to be duplicated in class {1}") raise InstanceError(msg.format(predicate_uri, predicate["class"])) else: assembled_predicates[predicate_uri] = predicate if "unique_value" in binding_row and binding_row["unique_value"]["value"] == "1": assembled_predicates[predicate_uri]["unique_value"] = True return assembled_predicates
def finalize(self, response): # FIXME: handle cache policy uniformly self.set_header("Cache-control", "private") self.set_header("max-age", "0") if response is None: # TODO separate filter message logic (e.g. if response is None and ("p" in self.query_params or "o" in self.query_params)) filter_message = [] po_tuples = extract_po_tuples(self.query_params) sorted_po_tuples = sorted(po_tuples, key=lambda po: po[2]) for (p, o, index) in sorted_po_tuples: if not index: index = '' if not p.startswith("?"): filter_message.append(u" with p{0}=({1})".format(index, p)) if not o.startswith("?"): filter_message.append(u" with o{0}=({1})".format(index, o)) self.query_params["filter_message"] = "".join(filter_message) self.query_params["page"] = int(self.query_params["page"]) + 1 # Showing real page in response msg = _(u"Instances of class ({class_uri}) in graph ({graph_uri}){filter_message}, language=({lang}) and in page=({page}) were not found.") response = { "warning": msg.format(**self.query_params), "items": [] } self.write(response) elif isinstance(response, int): # status code self.set_status(response) else: self.write(response) url_schema = build_schema_url(self.query_params, propagate_params=True) self.set_header("Content-Type", content_type_profile(url_schema))
def finalize(self, response): if response is None: # TODO separate filter message logic (e.g. if response is None and ("p" in self.query_params or "o" in self.query_params)) filter_message = [] po_tuples = extract_po_tuples(self.query_params) sorted_po_tuples = sorted(po_tuples, key=lambda po: po[2]) for (p, o, index) in sorted_po_tuples: if not index: index = '' if not p.startswith("?"): filter_message.append(u" with p{0}=({1})".format(index, p)) if not o.startswith("?"): filter_message.append(u" with o{0}=({1})".format(index, o)) self.query_params["filter_message"] = "".join(filter_message) self.query_params["page"] = int(self.query_params["page"]) + 1 # Showing real page in response msg = _(u"Instances of class ({class_uri}) in graph ({graph_uri}){filter_message}, language=({lang}) and in page=({page}) were not found.") response = { "warning": msg.format(**self.query_params), "items": [] } self.write(response) elif isinstance(response, int): # status code self.set_status(response) else: self.write(response) self.set_header("Content-Type", content_type_profile(build_schema_url(self.query_params)))
def decode_boolean(object_value): if object_value == "0": return False elif object_value == "1": return True else: raise InstanceError(_(u"Could not decode boolean using {0}".format(object_value)))
def do_suggest(query_params, suggest_params): search_params = suggest_params["search"] range_result = _get_predicate_ranges(query_params, search_params) if is_result_empty(range_result): message = _(u"Either the predicate {0} does not exists or it does not have any rdfs:range defined in the triplestore") message = message.format(search_params["target"]) raise HTTPError(400, message) classes = _validate_class_restriction(query_params, range_result) graphs = _validate_graph_restriction(query_params, range_result) indexes = ["semantica." + uri_to_slug(graph) for graph in graphs] search_fields = list(set(_get_search_fields(query_params, suggest_params) + LABEL_PROPERTIES)) response_params = suggest_params.get("response", {}) response_fields = _get_response_fields( query_params, response_params, classes, LABEL_PROPERTIES) # request_body = _build_body_query( # query_params, # search_params, # classes, # search_fields, # response_fields) analyze_response = run_analyze(search_params["pattern"], settings.ES_ANALYZER, indexes) tokens = analyze_response["tokens"] request_body = _build_body_query_compatible_with_uatu_and_es_19_in_envs( query_params, tokens, classes, search_fields, response_fields, search_params["pattern"] ) # Sorting in ES is done using memory. From the docs [1]: # "When sorting, the relevant sorted field values are loaded into memory. # This means that per shard, there should be enough memory to contain them" # Currently Globo.com ES servers don't have enough memory to load all data # During the 30th October 2013, a query using sort caused Split-brain and all ES shards were down. # Therefore, we need to think twice and use sort cleverly to avoid problems # [1] http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-sort.html assert not "sort" in request_body # Read comments above elasticsearch_result = run_search(request_body, indexes=indexes) class_fields = response_params.get("class_fields", []) total_items = elasticsearch_result["hits"]["total"] if total_items: items = _build_items(query_params, elasticsearch_result, LABEL_PROPERTIES, class_fields) response = build_json(items, total_items, query_params) else: response = {} return response
def purge(pattern): keys_with_pattern = keys(pattern) or [] log.logger.debug( _(u"Cache: key(s) to be deleted: {0}").format(keys_with_pattern)) log_details = _(u"{0} key(s), matching the pattern: {1}").format( len(keys_with_pattern), pattern) response = 1 for key in keys_with_pattern: response *= delete(key) if response and keys_with_pattern: log.logger.info( _(u"Cache: purged with success {0}").format(log_details)) elif not keys_with_pattern: log.logger.info(_(u"Cache: {0}").format(log_details)) else: log.logger.info(_(u"Cache: failed purging {0}").format(log_details))
def get_json_request_as_dict(json_request_body): try: raw_body_params = json.loads(json_request_body) except ValueError: error_message = _("JSON malformed. Received: {0}") raise HTTPError(400, log_message=error_message.format(json_request_body)) return raw_body_params
def wrapper(*params): try: response = function(*params) except exceptions: log.logger.error( _(u"CacheError: First try returned {0}").format( traceback.format_exc())) try: global redis_client redis_client = connect() response = function(*params) except exceptions: log.logger.error( _(u"CacheError: Second try returned {0}").format( traceback.format_exc())) response = None return response
def post(self, context_name, class_name): valid_params = CLASS_PARAMS with safe_params(valid_params): self.query_params = ParamDict(self, context_name=context_name, class_name=class_name, **valid_params) del context_name del class_name try: schema = schema_resource.get_cached_schema(self.query_params) except SchemaNotFound: schema = None if schema is None: class_uri = self.query_params["class_uri"] graph_uri = self.query_params["graph_uri"] raise HTTPError(404, log_message=_(u"Class {0} doesn't exist in context {1}.").format(class_uri, graph_uri)) try: instance_data = json.loads(self.request.body) except ValueError: raise HTTPError(400, log_message=_(u"No JSON object could be decoded")) instance_data = normalize_all_uris_recursively(instance_data) try: (instance_uri, instance_id) = create_instance(self.query_params, instance_data) except InstanceError as ex: raise HTTPError(500, log_message=unicode(ex)) instance_url = self.build_resource_url(instance_id) self.set_header("location", instance_url) self.set_header("X-Brainiak-Resource-URI", instance_uri) self.query_params["instance_uri"] = instance_uri self.query_params["instance_id"] = instance_id self.query_params["expand_object_properties"] = "1" instance_data = get_instance(self.query_params) if settings.NOTIFY_BUS: self._notify_bus(action="POST", instance_data=instance_data) self.finalize(201)
def _validate_graph_restriction(search_params, range_result): graphs = set(filter_values(range_result, "range_graph")) if "graphs" in search_params: graphs_set = set(search_params["graphs"]) graphs_not_in_range = list(graphs_set.difference(graphs)) if graphs_not_in_range: raise HTTPError(400, _(u"Classes in the range of predicate '{0}' are not in graphs {1}".format(search_params["target"], graphs_not_in_range))) graphs = graphs_set graphs = graphs.difference(set(settings.GRAPHS_WITHOUT_INSTANCES)) if not graphs: raise HTTPError(400, _(u"Classes in the range of predicate '{0}' are in graphs without instances, such as: {1}".format( search_params["target"], settings.GRAPHS_WITHOUT_INSTANCES))) return list(graphs)
def get(self, query_id): stored_query = get_stored_query(query_id) if stored_query is not None: self.finalize(stored_query) else: not_found_message = _("The stored query with id '{0}' was not found").format(query_id) raise HTTPError(404, log_message=not_found_message)
def purge(self, **kargs): if settings.ENABLE_CACHE: path = self.get_cache_path() recursive = int(self.request.headers.get('X-Cache-Recursive', '0')) cache.purge_by_path(path, recursive) else: raise HTTPError(405, log_message=_("Cache is disabled (Brainaik's settings.ENABLE_CACHE is set to False)"))
def get(self, query_id): stored_query = get_stored_query(query_id) if stored_query is not None: self.finalize(stored_query) else: not_found_message = _("The stored query with id '{0}' was not found").format(query_id) raise HTTPError(404, log_message=not_found_message)
def purge(self, **kargs): if settings.ENABLE_CACHE: path = self.get_cache_path() recursive = int(self.request.headers.get('X-Cache-Recursive', '0')) cache.purge_by_path(path, recursive) else: raise HTTPError(405, log_message=_("Cache is disabled (Brainaik's settings.ENABLE_CACHE is set to False)"))
def finalize(self, response): if response is None: msg = _("There were no search results.") raise HTTPError(404, log_message=msg) elif isinstance(response, dict): self.write(response) self.set_header("Content-Type", content_type_profile(build_schema_url(self.query_params))) elif isinstance(response, int): # status code self.set_status(response)
def purge(self): if settings.ENABLE_CACHE: valid_params = PAGING_PARAMS with safe_params(valid_params): self.query_params = ParamDict(self, **valid_params) recursive = int(self.request.headers.get('X-Cache-Recursive', '0')) cache.purge_root(recursive) else: raise HTTPError(405, log_message=_("Cache is disabled (Brainaik's settings.ENABLE_CACHE is set to False)"))
def _set_triplestore_config(self, request): auth_client_id = request.headers.get(CLIENT_ID_HEADER, 'default') try: self.triplestore_config = parse_section(section=auth_client_id) except ConfigParserNoSectionError: raise HTTPError( 404, _(u"Client-Id provided at '{0}' ({1}) is not known").format( CLIENT_ID_HEADER, auth_client_id))
def validate_json_schema(request_json, schema): try: validate(request_json, schema) except ValidationError as ex: raise HTTPError( 400, log_message=_( u"JSON not according to JSON schema definition.\n {0:s}"). format(ex))
def purge(self): if settings.ENABLE_CACHE: valid_params = PAGING_PARAMS with safe_params(valid_params): self.query_params = ParamDict(self, **valid_params) recursive = int(self.request.headers.get('X-Cache-Recursive', '0')) cache.purge_root(recursive) else: raise HTTPError(405, log_message=_("Cache is disabled (Brainaik's settings.ENABLE_CACHE is set to False)"))
def assemble_predicate(predicate_uri, binding_row, cardinalities, context): predicate_graph = binding_row["predicate_graph"]['value'] predicate_type = binding_row['type']['value'] range_uri = binding_row['range']['value'] range_graph = binding_row.get('range_graph', {}).get('value', "") range_label = binding_row.get('range_label', {}).get('value', "") class_uri = binding_row["domain_class"]['value'] # build up predicate dictionary predicate = { "class": class_uri, "graph": predicate_graph, "title": binding_row["title"]['value'] } if "predicate_comment" in binding_row: predicate["description"] = binding_row["predicate_comment"]['value'] if predicate_type == OBJECT_PROPERTY: context.add_object_property(predicate_uri, range_uri) predicate["range"] = {'@id': range_uri, 'graph': range_graph, 'title': range_label, 'type': 'string', 'format': 'uri'} max_items = cardinalities.get(predicate_uri, {}).get(range_uri, {}).get('maxItems', 2) min_items = cardinalities.get(predicate_uri, {}).get(range_uri, {}).get('minItems', 2) if (min_items > 1) or (max_items > 1) or (not min_items and not max_items): predicate["type"] = "array" predicate["items"] = {"type": "string", "format": "uri"} else: predicate["type"] = "string" predicate["format"] = "uri" elif predicate_type == DATATYPE_PROPERTY: max_items = cardinalities.get(predicate_uri, {}).get(range_uri, {}).get('maxItems', 1) min_items = cardinalities.get(predicate_uri, {}).get(range_uri, {}).get('minItems', 1) # add predicate['type'] and (optional) predicate['format'] predicate.update(items_from_range(range_uri, min_items, max_items)) else: # TODO: owl:AnnotationProperty msg = _(u"Predicates of type {0} are not supported yet").format(predicate_type) raise InstanceError(msg) if predicate["type"] == "array": if (predicate_uri in cardinalities) and (range_uri in cardinalities[predicate_uri]): predicate_restriction = cardinalities[predicate_uri] predicate.update(predicate_restriction[range_uri]) else: required = cardinalities.get(predicate_uri, {}).get(range_uri, {}).get('required', False) if required: predicate['required'] = True return predicate
def put(self, context_name, class_name, instance_id): valid_params = INSTANCE_PARAMS with safe_params(valid_params): self.query_params = ParamDict(self, context_name=context_name, class_name=class_name, instance_id=instance_id, **valid_params) del context_name del class_name del instance_id instance_data = get_json_request_as_dict(self.request.body) instance_data = normalize_all_uris_recursively(instance_data) rdf_type_error = is_rdf_type_invalid(self.query_params, instance_data) if rdf_type_error: raise HTTPError(400, log_message=rdf_type_error) try: if not instance_exists(self.query_params): try: schema = schema_resource.get_cached_schema( self.query_params) except SchemaNotFound: schema = None if schema is None: msg = _(u"Class {0} doesn't exist in graph {1}.") raise HTTPError(404, log_message=msg.format( self.query_params["class_uri"], self.query_params["graph_uri"])) instance_uri, instance_id = create_instance( self.query_params, instance_data, self.query_params["instance_uri"]) resource_url = self.request.full_url() status = 201 self.set_header("location", resource_url) self.set_header("X-Brainiak-Resource-URI", instance_uri) else: edit_instance(self.query_params, instance_data) status = 200 except InstanceError as ex: raise HTTPError(400, log_message=str(ex)) except SchemaNotFound as ex: raise HTTPError(404, log_message=str(ex)) cache.purge_an_instance(self.query_params['instance_uri']) self.query_params["expand_object_properties"] = "1" instance_data = get_instance(self.query_params) if instance_data and settings.NOTIFY_BUS: self.query_params["instance_uri"] = instance_data["@id"] self._notify_bus(action="PUT", instance_data=instance_data) self.finalize(status)
def _extract_cardinalities(bindings, predicate_dict): cardinalities = {} for binding in bindings: property_ = binding["predicate"]["value"] try: range_ = binding["range"]["value"] except KeyError: try: range_ = predicate_dict[property_]["range"]["value"] except KeyError: msg = _(u"The property {0} is not defined properly").format(property_) raise InstanceError(msg) if not property_ in cardinalities: cardinalities[property_] = {} if not range_ in cardinalities[property_] and not range_.startswith("nodeID://"): cardinalities[property_][range_] = {} current_property = cardinalities[property_] if "min" in binding: min_value = binding["min"]["value"] try: min_value = int(min_value) except ValueError: msg = _(u"The property {0} defines a non-integer owl:minQualifiedCardinality {1}").format(property_, min_value) raise InstanceError(msg) else: current_property[range_].update({"minItems": min_value}) if min_value: current_property[range_].update({"required": True}) if "max" in binding: max_value = binding["max"]["value"] try: max_value = int(max_value) except ValueError: msg = _(u"The property {0} defines a non-integer owl:maxQualifiedCardinality {1}").format(property_, max_value) raise InstanceError(msg) else: current_property[range_].update({"maxItems": max_value}) return cardinalities
def safe_params(valid_params=None, body_params=None): try: yield except InvalidParam as ex: msg = _(u"Argument {0:s} is not supported.").format(ex) if valid_params is not None: params_msg = ", ".join(sorted(set(valid_params.keys() + DEFAULT_PARAMS.keys()))) msg += _(u" The supported querystring arguments are: {0}.").format(params_msg) else: params_msg = ", ".join(sorted(DEFAULT_PARAMS.keys())) msg += _(u" The supported querystring arguments are: {0}.").format(params_msg) if body_params is not None: body_msg = ", ".join(body_params) msg += _(u" The supported body arguments are: {0}.").format(body_msg) raise HTTPError(400, log_message=msg) except RequiredParamMissing as ex: msg = _(u"Required parameter ({0:s}) was not given.").format(ex) raise HTTPError(400, log_message=unicode(msg))
def get_query(stored_query, querystring_params): # template existence is validated in stored query creation/modification template = stored_query["sparql_template"] try: # .arguments is a dict with decoded querystring params query = template % querystring_params.arguments return query except KeyError as key_error: missing_key_message = _("Missing key '{0}' in querystring.\n Template: {1}").format(key_error.args[0], template) raise HTTPError(400, log_message=missing_key_message)
def purge(self, context_name, class_name): if settings.ENABLE_CACHE: with safe_params(): self.query_params = ParamDict(self, context_name=context_name, class_name=class_name) path = cache.build_key_for_class(self.query_params) cache.purge_by_path(path, False) else: raise HTTPError(405, log_message=_("Cache is disabled (Brainaik's settings.ENABLE_CACHE is set to False)"))
def get_cached_schema(query_params, include_meta=False): schema_key = build_key_for_class(query_params) class_object = memoize(query_params, get_schema, query_params, key=schema_key) if class_object is None or not class_object["body"]: msg = _(u"The class definition for {0} was not found in graph {1}") raise SchemaNotFound(msg.format(query_params['class_uri'], query_params['graph_uri'])) if include_meta: return class_object else: return class_object['body']
def _validate_class_restriction(search_params, range_result): classes = set(filter_values(range_result, "range")) if "classes" in search_params: classes_not_in_range = list(set(search_params["classes"]).difference(classes)) if classes_not_in_range: raise HTTPError(400, _(u"Classes {0} are not in the range of predicate '{1}'".format(classes_not_in_range, search_params["target"]))) classes = search_params["classes"] return list(classes)
def purge(self, context_name, class_name): if settings.ENABLE_CACHE: with safe_params(): self.query_params = ParamDict(self, context_name=context_name, class_name=class_name) path = cache.build_key_for_class(self.query_params) cache.purge_by_path(path, False) else: raise HTTPError(405, log_message=_("Cache is disabled (Brainaik's settings.ENABLE_CACHE is set to False)"))
def safe_params(valid_params=None, body_params=None): try: yield except InvalidParam as ex: msg = _(u"Argument {0:s} is not supported.").format(ex) if valid_params is not None: params_msg = ", ".join(sorted(set(valid_params.keys() + DEFAULT_PARAMS.keys()))) msg += _(u" The supported querystring arguments are: {0}.").format(params_msg) else: params_msg = ", ".join(sorted(DEFAULT_PARAMS.keys())) msg += _(u" The supported querystring arguments are: {0}.").format(params_msg) if body_params is not None: body_msg = ", ".join(body_params) msg += _(u" The supported body arguments are: {0}.").format(body_msg) raise HTTPError(400, log_message=msg) except RequiredParamMissing as ex: msg = _(u"Required parameter ({0:s}) was not given.").format(ex) raise HTTPError(400, log_message=unicode(msg))
def patch(self, context_name, class_name, instance_id): valid_params = INSTANCE_PARAMS with safe_params(valid_params): self.query_params = ParamDict(self, context_name=context_name, class_name=class_name, instance_id=instance_id, **valid_params) del context_name del class_name del instance_id try: patch_list = json.loads(self.request.body) except ValueError: raise HTTPError(400, log_message=_("No JSON object could be decoded")) # Retrieve original data instance_data = memoize(self.query_params, get_instance, key=build_instance_key(self.query_params), function_arguments=self.query_params) try: instance_data = instance_data['body'] except TypeError: raise HTTPError(404, log_message=_("Inexistent instance")) instance_data.pop('http://www.w3.org/1999/02/22-rdf-syntax-ns#type', None) # compute patch changed_data = apply_patch(instance_data, patch_list) # Try to put edit_instance(self.query_params, changed_data) status = 200 # Clear cache cache.purge_an_instance(self.query_params['instance_uri']) self.finalize(status)
def finalize(self, response): if response is None: msg = _("There were no search results.") raise HTTPError(404, log_message=msg) elif isinstance(response, dict): self.write(response) self.set_header( "Content-Type", content_type_profile(build_schema_url(self.query_params))) elif isinstance(response, int): # status code self.set_status(response)
def get(self, context_name): valid_params = LIST_PARAMS + GRAPH_PARAMS with safe_params(valid_params): self.query_params = ParamDict(self, context_name=context_name, **valid_params) del context_name response = list_classes(self.query_params) if response is None: raise HTTPError(404, log_message=_(u"Context {0} not found").format(self.query_params['graph_uri'])) self.finalize(response)
def get(self, context_name): valid_params = LIST_PARAMS + GRAPH_PARAMS with safe_params(valid_params): self.query_params = ParamDict(self, context_name=context_name, **valid_params) del context_name response = list_classes(self.query_params) if response is None: raise HTTPError(404, log_message=_(u"Context {0} not found").format(self.query_params['graph_uri'])) self.finalize(response)
def _convert_to_python(object_value, class_schema, predicate_uri): if predicate_uri in class_schema["properties"]: schema_type = get_predicate_datatype(class_schema, predicate_uri) python_type = MAP_RDF_EXPANDED_TYPE_TO_PYTHON.get(schema_type) if python_type is None: msg = _(u"The property {0} is unknown according to the schema definitions {1}").format(predicate_uri, class_schema) logger.debug(msg) converted_value = object_value elif python_type == bool: converted_value = decode_boolean(object_value) else: msg = _(u"The property {0} is mapped to a inconsistent value {1}").format(predicate_uri, object_value) try: converted_value = python_type(object_value) except ValueError: raise Exception(msg) return converted_value else: # values with predicate_uri not in schema will be ignored anyway return object_value
def get_query(stored_query, querystring_params): # template existence is validated in stored query creation/modification template = stored_query["sparql_template"] try: # .arguments is a dict with decoded querystring params query = template % querystring_params.arguments return query except KeyError as key_error: missing_key_message = _( "Missing key '{0}' in querystring.\n Template: {1}").format( key_error.args[0], template) raise HTTPError(400, log_message=missing_key_message)
def delete_instance(query_params): dependants_result_dict = query_dependants(query_params) if not is_result_empty(dependants_result_dict): values = [item['dependant']['value'] for item in dependants_result_dict['results']['bindings']] str_values = ", ".join(values) raise HTTPError(409, log_message=_(u"Cannot exclude instance because of the dependencies: {0}").format(str_values)) query_result_dict = query_delete(query_params) if some_triples_deleted(query_result_dict, query_params['graph_uri']): return True
def _validate_graph_restriction(search_params, range_result): graphs = set(filter_values(range_result, "range_graph")) if "graphs" in search_params: graphs_set = set(search_params["graphs"]) graphs_not_in_range = list(graphs_set.difference(graphs)) if graphs_not_in_range: raise HTTPError( 400, _(u"Classes in the range of predicate '{0}' are not in graphs {1}" .format(search_params["target"], graphs_not_in_range))) graphs = graphs_set graphs = graphs.difference(set(settings.GRAPHS_WITHOUT_INSTANCES)) if not graphs: raise HTTPError( 400, _(u"Classes in the range of predicate '{0}' are in graphs without instances, such as: {1}" .format(search_params["target"], settings.GRAPHS_WITHOUT_INSTANCES))) return list(graphs)
def patch(self, context_name, class_name, instance_id): valid_params = INSTANCE_PARAMS with safe_params(valid_params): self.query_params = ParamDict(self, context_name=context_name, class_name=class_name, instance_id=instance_id, **valid_params) del context_name del class_name del instance_id try: patch_list = json.loads(self.request.body) except ValueError: raise HTTPError(400, log_message=_("No JSON object could be decoded")) # Retrieve original data instance_data = memoize(self.query_params, get_instance, key=build_instance_key(self.query_params), function_arguments=self.query_params) try: instance_data = instance_data['body'] except TypeError: raise HTTPError(404, log_message=_("Inexistent instance")) instance_data.pop('http://www.w3.org/1999/02/22-rdf-syntax-ns#type', None) # compute patch changed_data = apply_patch(instance_data, patch_list) # Try to put edit_instance(self.query_params, changed_data) status = 200 # Clear cache cache.purge_an_instance(self.query_params['instance_uri']) self.finalize(status)
def _handle_request_exception(self, e): if hasattr(e, "status_code"): # and e.code in httplib.responses: status_code = e.status_code else: status_code = 500 error_message = u"[{0}] on {1}".format(status_code, self._request_summary()) if isinstance(e, NotificationFailure): message = unicode(e) logger.error(message) self.send_error(status_code, message=message) elif isinstance(e, HTTPClientError): message = _(u"Access to backend service failed. {0:s}.").format( unicode(e)) extra_messages = check_messages_when_port_is_mentioned(unicode(e)) if extra_messages: for msg in extra_messages: message += msg if hasattr(e, "response") and e.response is not None and \ hasattr(e.response, "body") and e.response.body is not None: message += _(u"\nResponse:\n") + unicode( str(e.response.body).decode("utf-8")) logger.error(message) self.send_error(status_code, message=message) elif isinstance(e, InstanceError): logger.error( _(u"Ontology inconsistency: {0}\n").format(error_message)) self.send_error(status_code, message=e.message) elif isinstance(e, HTTPError): try: possible_list = json.loads(e.log_message) except (TypeError, ValueError): pass else: if isinstance(possible_list, list): self.send_error(status_code, errors_list=possible_list) return if e.log_message: error_message += u"\n {0}".format(e.log_message) if status_code == 500: logger.error( _(u"Unknown HTTP error [{0}]:\n {1}\n").format( e.status_code, error_message)) self.send_error(status_code, exc_info=sys.exc_info(), message=e.log_message) else: logger.error(_(u"HTTP error: {0}\n").format(error_message)) self.send_error(status_code, message=e.log_message) else: logger.error(_(u"Uncaught exception: {0}\n").format(error_message), exc_info=True) self.send_error(status_code, exc_info=sys.exc_info())
def get(self): triplestore_status = triplestore.status() event_bus_status = event_bus.status() output = [] if "SUCCEED" not in triplestore_status: output.append(triplestore_status) if "FAILED" in event_bus_status: output.append(event_bus_status) if output: response = "\n".join(output) else: response = _(u"WORKING") self.write(response)
def check_and_clean_rdftype(instance_type, items): """Validate actual type and remove rdf:type from the instance to be returned""" if 'rdf:type' in items: rdftype = 'rdf:type' elif 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' in items: rdftype = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' else: rdftype = None if rdftype is not None: if instance_type != items[rdftype]: msg = _(u"The type specified={0} is not the same informed from the triplestore={1}") raise Exception(msg.format(instance_type, items[rdftype])) del items[rdftype]
def purge(self, context_name, class_name, instance_id): if settings.ENABLE_CACHE: optional_params = INSTANCE_PARAMS with safe_params(optional_params): self.query_params = ParamDict(self, context_name=context_name, class_name=class_name, instance_id=instance_id, **optional_params) path = build_instance_key(self.query_params) cache.purge_by_path(path, False) else: raise HTTPError(405, log_message=_("Cache is disabled (Brainaik's settings.ENABLE_CACHE is set to False)"))
def get(self): triplestore_status = triplestore.status() event_bus_status = event_bus.status() output = [] if "SUCCEED" not in triplestore_status: output.append(triplestore_status) if "FAILED" in event_bus_status: output.append(event_bus_status) if output: response = "\n".join(output) else: response = _(u"WORKING") self.write(response)
def _validate_class_restriction(search_params, range_result): classes = set(filter_values(range_result, "range")) if "classes" in search_params: classes_not_in_range = list( set(search_params["classes"]).difference(classes)) if classes_not_in_range: raise HTTPError( 400, _(u"Classes {0} are not in the range of predicate '{1}'". format(classes_not_in_range, search_params["target"]))) classes = search_params["classes"] return list(classes)
def purge(self, context_name, class_name, instance_id): if settings.ENABLE_CACHE: optional_params = INSTANCE_PARAMS with safe_params(optional_params): self.query_params = ParamDict(self, context_name=context_name, class_name=class_name, instance_id=instance_id, **optional_params) path = build_instance_key(self.query_params) cache.purge_by_path(path, False) else: raise HTTPError(405, log_message=_("Cache is disabled (Brainaik's settings.ENABLE_CACHE is set to False)"))