def edit_instance(query_params, instance_data):
if not are_there_label_properties_in(instance_data):
raise HTTPError(
400,
log_message=
_(u"Label properties like rdfs:label or its subproperties are required"
))
if must_retrieve_graph_and_class_uri(query_params):
triplestore_response = get_class_and_graph(query_params)
bindings = triplestore_response['results']['bindings']
query_params['graph_uri'] = extract_graph_uri(bindings)
query_params['class_uri'] = extract_class_uri(bindings)
try:
instance_uri = query_params['instance_uri']
graph_uri = query_params['graph_uri']
class_uri = query_params['class_uri']
except KeyError as ex:
raise HTTPError(
404,
log_message=_(
u"Parameter <{0:s}> is missing in order to update instance.".
format(ex)))
class_object = get_cached_schema(query_params)
try:
triples = create_explicit_triples(instance_uri, instance_data,
class_object, graph_uri,
query_params)
except InstanceError, exception:
raise HTTPError(400, log_message=exception.message)
def create_explicit_triples(instance_uri, instance_data, class_object, graph_uri, query_params):
class_id = class_object["id"]
copy_instance_data = instance_data.copy()
predicate_object_tuples = unpack_tuples(copy_instance_data)
triples = []
errors = []
template_msg = _(u'Incorrect value for property ({1}). A value compatible with a ({2}) was expected, but ({0}) was given.')
for (predicate_uri, object_value) in predicate_object_tuples:
if not is_reserved_attribute(predicate_uri):
predicate_has_error = False
try:
predicate_datatype = get_predicate_datatype(class_object, predicate_uri)
except KeyError:
template = _(u'Inexistent property ({0}) in the schema ({1}), used to create instance ({2})')
msg = template.format(predicate_uri, class_id, instance_uri)
errors.append(msg)
predicate_datatype = None
predicate_has_error = True
else:
if predicate_datatype is None: # ObjectProperty
try:
object_ = sparqlfy_object(object_value)
except InstanceError:
msg = template_msg.format(object_value, predicate_uri, "owl:ObjectProperty")
errors.append(msg)
predicate_has_error = True
else:
if is_instance(object_value, predicate_datatype):
object_ = sparqlfy(object_value, predicate_datatype)
else:
msg = template_msg.format(object_value, predicate_uri, predicate_datatype)
errors.append(msg)
predicate_has_error = True
if not errors:
instance = sparqlfy_object(instance_uri)
predicate = sparqlfy_object(predicate_uri)
triple = (instance, predicate, object_)
triples.append(triple)
if not predicate_has_error:
if property_must_map_a_unique_value(class_object, predicate_uri):
if is_value_already_used(instance_uri, object_, predicate_uri, class_object, graph_uri, query_params):
template = _(u"The property ({0}) defined in the schema ({1}) must map a unique value. The value provided ({2}) is already used by another instance.")
msg = template.format(predicate_uri, class_id, object_value)
errors.append(msg)
undefined_obligatory_properties = find_undefined_obligatory_properties(class_object, instance_data)
template = _(u"The property ({0}) is obligatory according to the definition of the class ({1}). A value must be provided for this field in order to create or edit ({2}).")
for property_ in undefined_obligatory_properties:
msg = template.format(property_, class_id, instance_uri)
errors.append(msg)
if errors:
error_msg = json.dumps(errors)
raise InstanceError(error_msg)
return triples
def create_explicit_triples(instance_uri, instance_data, class_object, graph_uri, query_params):
class_id = class_object["id"]
copy_instance_data = instance_data.copy()
predicate_object_tuples = unpack_tuples(copy_instance_data)
triples = []
errors = []
template_msg = _(u'Incorrect value for property ({1}). A value compatible with a ({2}) was expected, but ({0}) was given.')
for (predicate_uri, object_value) in predicate_object_tuples:
if not is_reserved_attribute(predicate_uri):
predicate_has_error = False
try:
predicate_datatype = get_predicate_datatype(class_object, predicate_uri)
except KeyError:
template = _(u'Inexistent property ({0}) in the schema ({1}), used to create instance ({2})')
msg = template.format(predicate_uri, class_id, instance_uri)
errors.append(msg)
predicate_datatype = None
predicate_has_error = True
else:
if predicate_datatype is None: # ObjectProperty
try:
object_ = sparqlfy_object(object_value)
except InstanceError:
msg = template_msg.format(object_value, predicate_uri, "owl:ObjectProperty")
errors.append(msg)
predicate_has_error = True
else:
if is_instance(object_value, predicate_datatype):
object_ = sparqlfy(object_value, predicate_datatype)
else:
msg = template_msg.format(object_value, predicate_uri, predicate_datatype)
errors.append(msg)
predicate_has_error = True
if not errors:
instance = sparqlfy_object(instance_uri)
predicate = sparqlfy_object(predicate_uri)
triple = (instance, predicate, object_)
triples.append(triple)
if not predicate_has_error:
if property_must_map_a_unique_value(class_object, predicate_uri):
if is_value_already_used(instance_uri, object_, predicate_uri, class_object, graph_uri, query_params):
template = _(u"The property ({0}) defined in the schema ({1}) must map a unique value. The value provided ({2}) is already used by another instance.")
msg = template.format(predicate_uri, class_id, object_value)
errors.append(msg)
undefined_obligatory_properties = find_undefined_obligatory_properties(class_object, instance_data)
template = _(u"The property ({0}) is obligatory according to the definition of the class ({1}). A value must be provided for this field in order to create or edit ({2}).")
for property_ in undefined_obligatory_properties:
msg = template.format(property_, class_id, instance_uri)
errors.append(msg)
if errors:
error_msg = json.dumps(errors)
raise InstanceError(error_msg)
return triples
def build_items_dict(bindings, class_uri, expand_object_properties,
class_schema):
super_predicates = get_super_properties(bindings)
items_dict = {}
for item in bindings:
is_object_blank_node = int(
item.get("is_object_blank", {}).get("value", "0"))
if not is_object_blank_node:
predicate_uri = item["predicate"]["value"]
object_value = item["object"]["value"]
object_type = item["object"].get("type")
object_label = item.get("object_label", {}).get("value")
if expand_object_properties and object_type == "uri":
if object_label:
value = {"@id": object_value, "title": object_label}
else:
msg = _(u"The predicate {0} refers to an object {1} which doesn't have a label.").format(predicate_uri, object_value) + \
_(" Set expand_object_properties=0 if you don't care about this ontological inconsistency.")
value = {"@id": object_value}
logger.debug(msg)
# raise Exception(msg)
else:
value = _convert_to_python(object_value, class_schema,
predicate_uri)
if predicate_uri in items_dict:
if not isinstance(items_dict[predicate_uri], list):
value_list = [items_dict[predicate_uri]]
else:
value_list = items_dict[predicate_uri]
value_list.append(value)
items_dict[predicate_uri] = value_list
else:
base_uri = None
if predicate_uri in class_schema["properties"]:
base_uri = predicate_uri
# Here we IGNORE predicate/object values that exist in the triplestore
# but the predicate is not in the class schema
if base_uri:
if class_schema["properties"][base_uri][
u'type'] == u'array':
items_dict[predicate_uri] = [value]
else:
items_dict[predicate_uri] = value
remove_super_properties(items_dict, super_predicates)
if not class_uri is None:
items_dict[
"http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] = class_uri
return items_dict
def post(self, context_name, class_name):
valid_params = CLASS_PARAMS
with safe_params(valid_params):
self.query_params = ParamDict(self,
context_name=context_name,
class_name=class_name,
**valid_params)
del context_name
del class_name
try:
schema = schema_resource.get_cached_schema(self.query_params)
except SchemaNotFound:
schema = None
if schema is None:
class_uri = self.query_params["class_uri"]
graph_uri = self.query_params["graph_uri"]
raise HTTPError(
404,
log_message=_(
u"Class {0} doesn't exist in context {1}.").format(
class_uri, graph_uri))
try:
instance_data = json.loads(self.request.body)
except ValueError:
raise HTTPError(400,
log_message=_(u"No JSON object could be decoded"))
instance_data = normalize_all_uris_recursively(instance_data)
try:
(instance_uri,
instance_id) = create_instance(self.query_params, instance_data)
except InstanceError as ex:
raise HTTPError(500, log_message=unicode(ex))
instance_url = self.build_resource_url(instance_id)
self.set_header("location", instance_url)
self.set_header("X-Brainiak-Resource-URI", instance_uri)
self.query_params["instance_uri"] = instance_uri
self.query_params["instance_id"] = instance_id
self.query_params["expand_object_properties"] = "1"
instance_data = get_instance(self.query_params)
if settings.NOTIFY_BUS:
self._notify_bus(action="POST", instance_data=instance_data)
self.finalize(201)
def wrapper(*params):
try:
response = function(*params)
except exceptions:
log.logger.error(_(u"CacheError: First try returned {0}").format(traceback.format_exc()))
try:
global redis_client
redis_client = connect()
response = function(*params)
except exceptions:
log.logger.error(_(u"CacheError: Second try returned {0}").format(traceback.format_exc()))
response = None
return response
def do_suggest(query_params, suggest_params):
search_params = suggest_params["search"]
range_result = _get_predicate_ranges(query_params, search_params)
if is_result_empty(range_result):
message = _(
u"Either the predicate {0} does not exists or it does not have any rdfs:range defined in the triplestore"
)
message = message.format(search_params["target"])
raise_no_results(message)
classes = _validate_class_restriction(query_params, range_result)
graphs = _validate_graph_restriction(query_params, range_result)
indexes = ["semantica." + uri_to_slug(graph) for graph in graphs]
search_fields = list(
set(
_get_search_fields(query_params, suggest_params) +
LABEL_PROPERTIES))
response_params = suggest_params.get("response", {})
response_fields = _get_response_fields(query_params, response_params,
classes, LABEL_PROPERTIES)
request_body = _build_body_query(query_params, search_params, classes,
search_fields, response_fields)
# Sorting in ES is done using memory. From the docs [1]:
# "When sorting, the relevant sorted field values are loaded into memory.
# This means that per shard, there should be enough memory to contain them"
# Currently Globo.com ES servers don't have enough memory to load all data
# During the 30th October 2013, a query using sort caused Split-brain and all ES shards were down.
# Therefore, we need to think twice and use sort cleverly to avoid problems
# [1] http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-sort.html
assert not "sort" in request_body # Read comments above
elasticsearch_result = run_search(request_body, indexes=indexes)
if elasticsearch_result is None:
message = _(u"There were no search results.")
raise_no_results(message)
class_fields = response_params.get("class_fields", [])
total_items = elasticsearch_result["hits"]["total"]
if total_items:
items = _build_items(query_params, elasticsearch_result,
LABEL_PROPERTIES, class_fields)
response = build_json(items, total_items, query_params)
else:
response = {}
return response
def purge(pattern):
keys_with_pattern = keys(pattern) or []
log.logger.debug(_(u"Cache: key(s) to be deleted: {0}").format(keys_with_pattern))
log_details = _(u"{0} key(s), matching the pattern: {1}").format(len(keys_with_pattern), pattern)
response = 1
for key in keys_with_pattern:
response *= delete(key)
if response and keys_with_pattern:
log.logger.info(_(u"Cache: purged with success {0}").format(log_details))
elif not keys_with_pattern:
log.logger.info(_(u"Cache: {0}").format(log_details))
else:
log.logger.info(_(u"Cache: failed purging {0}").format(log_details))
def _extract_cardinalities(bindings, predicate_dict):
cardinalities = {}
for binding in bindings:
property_ = binding["predicate"]["value"]
try:
range_ = binding["range"]["value"]
except KeyError:
try:
range_ = predicate_dict[property_]["range"]["value"]
except KeyError:
msg = _(u"The property {0} is not defined properly").format(
property_)
raise InstanceError(msg)
if not property_ in cardinalities:
cardinalities[property_] = {}
if not range_ in cardinalities[property_] and not range_.startswith(
"nodeID://"):
cardinalities[property_][range_] = {}
current_property = cardinalities[property_]
if "min" in binding:
min_value = binding["min"]["value"]
try:
min_value = int(min_value)
except ValueError:
msg = _(
u"The property {0} defines a non-integer owl:minQualifiedCardinality {1}"
).format(property_, min_value)
raise InstanceError(msg)
else:
current_property[range_].update({"minItems": min_value})
if min_value:
current_property[range_].update({"required": True})
if "max" in binding:
max_value = binding["max"]["value"]
try:
max_value = int(max_value)
except ValueError:
msg = _(
u"The property {0} defines a non-integer owl:maxQualifiedCardinality {1}"
).format(property_, max_value)
raise InstanceError(msg)
else:
current_property[range_].update({"maxItems": max_value})
return cardinalities
def build_items_dict(bindings, class_uri, expand_object_properties, class_schema):
super_predicates = get_super_properties(bindings)
items_dict = {}
for item in bindings:
is_object_blank_node = int(item.get("is_object_blank", {}).get("value", "0"))
if not is_object_blank_node:
predicate_uri = item["predicate"]["value"]
object_value = item["object"]["value"]
object_type = item["object"].get("type")
object_label = item.get("object_label", {}).get("value")
if expand_object_properties and object_type == "uri":
if object_label:
value = {"@id": object_value, "title": object_label}
else:
msg = _(u"The predicate {0} refers to an object {1} which doesn't have a label.").format(predicate_uri, object_value) + \
_(" Set expand_object_properties=0 if you don't care about this ontological inconsistency.")
value = {"@id": object_value}
logger.debug(msg)
# raise Exception(msg)
else:
value = _convert_to_python(object_value, class_schema, predicate_uri)
if predicate_uri in items_dict:
if not isinstance(items_dict[predicate_uri], list):
value_list = [items_dict[predicate_uri]]
else:
value_list = items_dict[predicate_uri]
value_list.append(value)
items_dict[predicate_uri] = value_list
else:
base_uri = None
if predicate_uri in class_schema["properties"]:
base_uri = predicate_uri
# Here we IGNORE predicate/object values that exist in the triplestore
# but the predicate is not in the class schema
if base_uri:
if class_schema["properties"][base_uri][u'type'] == u'array':
items_dict[predicate_uri] = [value]
else:
items_dict[predicate_uri] = value
remove_super_properties(items_dict, super_predicates)
if not class_uri is None:
items_dict["http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] = class_uri
return items_dict
def get_json_request_as_dict(json_request_body):
try:
raw_body_params = json.loads(json_request_body)
except ValueError:
error_message = _("JSON malformed. Received: {0}.")
raise HTTPError(400, log_message=error_message.format(json_request_body))
return raw_body_params
def delete(self, context_name, class_name, instance_id):
valid_params = INSTANCE_PARAMS
with safe_params(valid_params):
self.query_params = ParamDict(self,
context_name=context_name,
class_name=class_name,
instance_id=instance_id,
**valid_params)
del context_name
del class_name
del instance_id
deleted = delete_instance(self.query_params)
if deleted:
response = 204
if settings.NOTIFY_BUS:
self._notify_bus(action="DELETE")
cache.purge_an_instance(self.query_params['instance_uri'])
else:
msg = _(u"Instance ({0}) of class ({1}) in graph ({2}) was not found.")
error_message = msg.format(self.query_params["instance_uri"],
self.query_params["class_uri"],
self.query_params["graph_uri"])
raise HTTPError(404, log_message=error_message)
self.finalize(response)
def post(self):
valid_params = PAGING_PARAMS
with safe_params(valid_params):
try:
raw_body_params = json.loads(self.request.body)
except ValueError:
error_message = _("JSON malformed. Received: {0}.")
raise HTTPError(400,
log_message=error_message.format(
self.request.body))
body_params = normalize_all_uris_recursively(raw_body_params)
if '@context' in body_params:
del body_params['@context']
validate_json_schema(body_params, SUGGEST_PARAM_SCHEMA)
self.query_params = ParamDict(self, **valid_params)
response = do_suggest(self.query_params, body_params)
if self.query_params['expand_uri'] == "0":
response = normalize_all_uris_recursively(response, mode=SHORTEN)
self.finalize(response)
def delete(self, context_name, class_name, instance_id):
valid_params = INSTANCE_PARAMS
with safe_params(valid_params):
self.query_params = ParamDict(self,
context_name=context_name,
class_name=class_name,
instance_id=instance_id,
**valid_params)
del context_name
del class_name
del instance_id
deleted = delete_instance(self.query_params)
if deleted:
response = 204
if settings.NOTIFY_BUS:
self._notify_bus(action="DELETE")
cache.purge_an_instance(self.query_params['instance_uri'])
else:
msg = _(u"Instance ({0}) of class ({1}) in graph ({2}) was not found.")
error_message = msg.format(self.query_params["instance_uri"],
self.query_params["class_uri"],
self.query_params["graph_uri"])
raise HTTPError(404, log_message=error_message)
self.finalize(response)
def decode_boolean(object_value):
if object_value == "0":
return False
elif object_value == "1":
return True
else:
raise InstanceError(_(u"Could not decode boolean using {0}".format(object_value)))
def convert_bindings_dict(context, bindings, cardinalities, superclasses):
super_predicates = get_super_properties(bindings)
assembled_predicates = {}
for binding_row in bindings:
predicate_uri = binding_row['predicate']['value']
# super_predicate is when we use rdfs:subPropertyOf
# this case does not consider inherited predicates
if predicate_uri in super_predicates.keys():
continue
predicate = assemble_predicate(predicate_uri, binding_row, cardinalities, context)
existing_predicate = assembled_predicates.get(predicate_uri, False)
if existing_predicate:
if 'datatype' in existing_predicate and 'datatype' in predicate:
assembled_predicates[predicate_uri] = most_specialized_predicate(superclasses,
existing_predicate,
predicate)
elif existing_predicate != predicate:
assembled_predicates[predicate_uri] = join_predicates(existing_predicate, predicate)
else:
msg = _(u"The property {0} seems to be duplicated in class {1}")
raise InstanceError(msg.format(predicate_uri, predicate["class"]))
else:
assembled_predicates[predicate_uri] = predicate
if "unique_value" in binding_row and binding_row["unique_value"]["value"] == "1":
assembled_predicates[predicate_uri]["unique_value"] = True
return assembled_predicates
def finalize(self, response):
# FIXME: handle cache policy uniformly
self.set_header("Cache-control", "private")
self.set_header("max-age", "0")
if response is None:
# TODO separate filter message logic (e.g. if response is None and ("p" in self.query_params or "o" in self.query_params))
filter_message = []
po_tuples = extract_po_tuples(self.query_params)
sorted_po_tuples = sorted(po_tuples, key=lambda po: po[2])
for (p, o, index) in sorted_po_tuples:
if not index:
index = ''
if not p.startswith("?"):
filter_message.append(u" with p{0}=({1})".format(index, p))
if not o.startswith("?"):
filter_message.append(u" with o{0}=({1})".format(index, o))
self.query_params["filter_message"] = "".join(filter_message)
self.query_params["page"] = int(self.query_params["page"]) + 1 # Showing real page in response
msg = _(u"Instances of class ({class_uri}) in graph ({graph_uri}){filter_message}, language=({lang}) and in page=({page}) were not found.")
response = {
"warning": msg.format(**self.query_params),
"items": []
}
self.write(response)
elif isinstance(response, int): # status code
self.set_status(response)
else:
self.write(response)
url_schema = build_schema_url(self.query_params, propagate_params=True)
self.set_header("Content-Type", content_type_profile(url_schema))
def finalize(self, response):
if response is None:
# TODO separate filter message logic (e.g. if response is None and ("p" in self.query_params or "o" in self.query_params))
filter_message = []
po_tuples = extract_po_tuples(self.query_params)
sorted_po_tuples = sorted(po_tuples, key=lambda po: po[2])
for (p, o, index) in sorted_po_tuples:
if not index:
index = ''
if not p.startswith("?"):
filter_message.append(u" with p{0}=({1})".format(index, p))
if not o.startswith("?"):
filter_message.append(u" with o{0}=({1})".format(index, o))
self.query_params["filter_message"] = "".join(filter_message)
self.query_params["page"] = int(self.query_params["page"]) + 1 # Showing real page in response
msg = _(u"Instances of class ({class_uri}) in graph ({graph_uri}){filter_message}, language=({lang}) and in page=({page}) were not found.")
response = {
"warning": msg.format(**self.query_params),
"items": []
}
self.write(response)
elif isinstance(response, int): # status code
self.set_status(response)
else:
self.write(response)
self.set_header("Content-Type", content_type_profile(build_schema_url(self.query_params)))
def decode_boolean(object_value):
if object_value == "0":
return False
elif object_value == "1":
return True
else:
raise InstanceError(_(u"Could not decode boolean using {0}".format(object_value)))
def do_suggest(query_params, suggest_params):
search_params = suggest_params["search"]
range_result = _get_predicate_ranges(query_params, search_params)
if is_result_empty(range_result):
message = _(u"Either the predicate {0} does not exists or it does not have any rdfs:range defined in the triplestore")
message = message.format(search_params["target"])
raise HTTPError(400, message)
classes = _validate_class_restriction(query_params, range_result)
graphs = _validate_graph_restriction(query_params, range_result)
indexes = ["semantica." + uri_to_slug(graph) for graph in graphs]
search_fields = list(set(_get_search_fields(query_params, suggest_params) + LABEL_PROPERTIES))
response_params = suggest_params.get("response", {})
response_fields = _get_response_fields(
query_params,
response_params,
classes,
LABEL_PROPERTIES)
# request_body = _build_body_query(
# query_params,
# search_params,
# classes,
# search_fields,
# response_fields)
analyze_response = run_analyze(search_params["pattern"], settings.ES_ANALYZER, indexes)
tokens = analyze_response["tokens"]
request_body = _build_body_query_compatible_with_uatu_and_es_19_in_envs(
query_params,
tokens,
classes,
search_fields,
response_fields,
search_params["pattern"]
)
# Sorting in ES is done using memory. From the docs [1]:
# "When sorting, the relevant sorted field values are loaded into memory.
# This means that per shard, there should be enough memory to contain them"
# Currently Globo.com ES servers don't have enough memory to load all data
# During the 30th October 2013, a query using sort caused Split-brain and all ES shards were down.
# Therefore, we need to think twice and use sort cleverly to avoid problems
# [1] http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-sort.html
assert not "sort" in request_body # Read comments above
elasticsearch_result = run_search(request_body, indexes=indexes)
class_fields = response_params.get("class_fields", [])
total_items = elasticsearch_result["hits"]["total"]
if total_items:
items = _build_items(query_params, elasticsearch_result, LABEL_PROPERTIES, class_fields)
response = build_json(items, total_items, query_params)
else:
response = {}
return response
def purge(pattern):
keys_with_pattern = keys(pattern) or []
log.logger.debug(
_(u"Cache: key(s) to be deleted: {0}").format(keys_with_pattern))
log_details = _(u"{0} key(s), matching the pattern: {1}").format(
len(keys_with_pattern), pattern)
response = 1
for key in keys_with_pattern:
response *= delete(key)
if response and keys_with_pattern:
log.logger.info(
_(u"Cache: purged with success {0}").format(log_details))
elif not keys_with_pattern:
log.logger.info(_(u"Cache: {0}").format(log_details))
else:
log.logger.info(_(u"Cache: failed purging {0}").format(log_details))
def get_json_request_as_dict(json_request_body):
try:
raw_body_params = json.loads(json_request_body)
except ValueError:
error_message = _("JSON malformed. Received: {0}")
raise HTTPError(400,
log_message=error_message.format(json_request_body))
return raw_body_params
def wrapper(*params):
try:
response = function(*params)
except exceptions:
log.logger.error(
_(u"CacheError: First try returned {0}").format(
traceback.format_exc()))
try:
global redis_client
redis_client = connect()
response = function(*params)
except exceptions:
log.logger.error(
_(u"CacheError: Second try returned {0}").format(
traceback.format_exc()))
response = None
return response
def post(self, context_name, class_name):
valid_params = CLASS_PARAMS
with safe_params(valid_params):
self.query_params = ParamDict(self,
context_name=context_name,
class_name=class_name,
**valid_params)
del context_name
del class_name
try:
schema = schema_resource.get_cached_schema(self.query_params)
except SchemaNotFound:
schema = None
if schema is None:
class_uri = self.query_params["class_uri"]
graph_uri = self.query_params["graph_uri"]
raise HTTPError(404, log_message=_(u"Class {0} doesn't exist in context {1}.").format(class_uri, graph_uri))
try:
instance_data = json.loads(self.request.body)
except ValueError:
raise HTTPError(400, log_message=_(u"No JSON object could be decoded"))
instance_data = normalize_all_uris_recursively(instance_data)
try:
(instance_uri, instance_id) = create_instance(self.query_params, instance_data)
except InstanceError as ex:
raise HTTPError(500, log_message=unicode(ex))
instance_url = self.build_resource_url(instance_id)
self.set_header("location", instance_url)
self.set_header("X-Brainiak-Resource-URI", instance_uri)
self.query_params["instance_uri"] = instance_uri
self.query_params["instance_id"] = instance_id
self.query_params["expand_object_properties"] = "1"
instance_data = get_instance(self.query_params)
if settings.NOTIFY_BUS:
self._notify_bus(action="POST", instance_data=instance_data)
self.finalize(201)
def _validate_graph_restriction(search_params, range_result):
graphs = set(filter_values(range_result, "range_graph"))
if "graphs" in search_params:
graphs_set = set(search_params["graphs"])
graphs_not_in_range = list(graphs_set.difference(graphs))
if graphs_not_in_range:
raise HTTPError(400,
_(u"Classes in the range of predicate '{0}' are not in graphs {1}".format(search_params["target"], graphs_not_in_range)))
graphs = graphs_set
graphs = graphs.difference(set(settings.GRAPHS_WITHOUT_INSTANCES))
if not graphs:
raise HTTPError(400,
_(u"Classes in the range of predicate '{0}' are in graphs without instances, such as: {1}".format(
search_params["target"], settings.GRAPHS_WITHOUT_INSTANCES)))
return list(graphs)
def get(self, query_id):
stored_query = get_stored_query(query_id)
if stored_query is not None:
self.finalize(stored_query)
else:
not_found_message = _("The stored query with id '{0}' was not found").format(query_id)
raise HTTPError(404,
log_message=not_found_message)
def purge(self, **kargs):
if settings.ENABLE_CACHE:
path = self.get_cache_path()
recursive = int(self.request.headers.get('X-Cache-Recursive', '0'))
cache.purge_by_path(path, recursive)
else:
raise HTTPError(405, log_message=_("Cache is disabled (Brainaik's settings.ENABLE_CACHE is set to False)"))
def get(self, query_id):
stored_query = get_stored_query(query_id)
if stored_query is not None:
self.finalize(stored_query)
else:
not_found_message = _("The stored query with id '{0}' was not found").format(query_id)
raise HTTPError(404,
log_message=not_found_message)
def purge(self, **kargs):
if settings.ENABLE_CACHE:
path = self.get_cache_path()
recursive = int(self.request.headers.get('X-Cache-Recursive', '0'))
cache.purge_by_path(path, recursive)
else:
raise HTTPError(405, log_message=_("Cache is disabled (Brainaik's settings.ENABLE_CACHE is set to False)"))
def finalize(self, response):
if response is None:
msg = _("There were no search results.")
raise HTTPError(404, log_message=msg)
elif isinstance(response, dict):
self.write(response)
self.set_header("Content-Type", content_type_profile(build_schema_url(self.query_params)))
elif isinstance(response, int): # status code
self.set_status(response)
def purge(self):
if settings.ENABLE_CACHE:
valid_params = PAGING_PARAMS
with safe_params(valid_params):
self.query_params = ParamDict(self, **valid_params)
recursive = int(self.request.headers.get('X-Cache-Recursive', '0'))
cache.purge_root(recursive)
else:
raise HTTPError(405, log_message=_("Cache is disabled (Brainaik's settings.ENABLE_CACHE is set to False)"))
def _set_triplestore_config(self, request):
auth_client_id = request.headers.get(CLIENT_ID_HEADER, 'default')
try:
self.triplestore_config = parse_section(section=auth_client_id)
except ConfigParserNoSectionError:
raise HTTPError(
404,
_(u"Client-Id provided at '{0}' ({1}) is not known").format(
CLIENT_ID_HEADER, auth_client_id))
def validate_json_schema(request_json, schema):
try:
validate(request_json, schema)
except ValidationError as ex:
raise HTTPError(
400,
log_message=_(
u"JSON not according to JSON schema definition.\n {0:s}").
format(ex))
def purge(self):
if settings.ENABLE_CACHE:
valid_params = PAGING_PARAMS
with safe_params(valid_params):
self.query_params = ParamDict(self, **valid_params)
recursive = int(self.request.headers.get('X-Cache-Recursive', '0'))
cache.purge_root(recursive)
else:
raise HTTPError(405, log_message=_("Cache is disabled (Brainaik's settings.ENABLE_CACHE is set to False)"))
def assemble_predicate(predicate_uri, binding_row, cardinalities, context):
predicate_graph = binding_row["predicate_graph"]['value']
predicate_type = binding_row['type']['value']
range_uri = binding_row['range']['value']
range_graph = binding_row.get('range_graph', {}).get('value', "")
range_label = binding_row.get('range_label', {}).get('value', "")
class_uri = binding_row["domain_class"]['value']
# build up predicate dictionary
predicate = {
"class": class_uri,
"graph": predicate_graph,
"title": binding_row["title"]['value']
}
if "predicate_comment" in binding_row:
predicate["description"] = binding_row["predicate_comment"]['value']
if predicate_type == OBJECT_PROPERTY:
context.add_object_property(predicate_uri, range_uri)
predicate["range"] = {'@id': range_uri,
'graph': range_graph,
'title': range_label,
'type': 'string',
'format': 'uri'}
max_items = cardinalities.get(predicate_uri, {}).get(range_uri, {}).get('maxItems', 2)
min_items = cardinalities.get(predicate_uri, {}).get(range_uri, {}).get('minItems', 2)
if (min_items > 1) or (max_items > 1) or (not min_items and not max_items):
predicate["type"] = "array"
predicate["items"] = {"type": "string", "format": "uri"}
else:
predicate["type"] = "string"
predicate["format"] = "uri"
elif predicate_type == DATATYPE_PROPERTY:
max_items = cardinalities.get(predicate_uri, {}).get(range_uri, {}).get('maxItems', 1)
min_items = cardinalities.get(predicate_uri, {}).get(range_uri, {}).get('minItems', 1)
# add predicate['type'] and (optional) predicate['format']
predicate.update(items_from_range(range_uri, min_items, max_items))
else: # TODO: owl:AnnotationProperty
msg = _(u"Predicates of type {0} are not supported yet").format(predicate_type)
raise InstanceError(msg)
if predicate["type"] == "array":
if (predicate_uri in cardinalities) and (range_uri in cardinalities[predicate_uri]):
predicate_restriction = cardinalities[predicate_uri]
predicate.update(predicate_restriction[range_uri])
else:
required = cardinalities.get(predicate_uri, {}).get(range_uri, {}).get('required', False)
if required:
predicate['required'] = True
return predicate
def put(self, context_name, class_name, instance_id):
valid_params = INSTANCE_PARAMS
with safe_params(valid_params):
self.query_params = ParamDict(self,
context_name=context_name,
class_name=class_name,
instance_id=instance_id,
**valid_params)
del context_name
del class_name
del instance_id
instance_data = get_json_request_as_dict(self.request.body)
instance_data = normalize_all_uris_recursively(instance_data)
rdf_type_error = is_rdf_type_invalid(self.query_params, instance_data)
if rdf_type_error:
raise HTTPError(400, log_message=rdf_type_error)
try:
if not instance_exists(self.query_params):
try:
schema = schema_resource.get_cached_schema(
self.query_params)
except SchemaNotFound:
schema = None
if schema is None:
msg = _(u"Class {0} doesn't exist in graph {1}.")
raise HTTPError(404,
log_message=msg.format(
self.query_params["class_uri"],
self.query_params["graph_uri"]))
instance_uri, instance_id = create_instance(
self.query_params, instance_data,
self.query_params["instance_uri"])
resource_url = self.request.full_url()
status = 201
self.set_header("location", resource_url)
self.set_header("X-Brainiak-Resource-URI", instance_uri)
else:
edit_instance(self.query_params, instance_data)
status = 200
except InstanceError as ex:
raise HTTPError(400, log_message=str(ex))
except SchemaNotFound as ex:
raise HTTPError(404, log_message=str(ex))
cache.purge_an_instance(self.query_params['instance_uri'])
self.query_params["expand_object_properties"] = "1"
instance_data = get_instance(self.query_params)
if instance_data and settings.NOTIFY_BUS:
self.query_params["instance_uri"] = instance_data["@id"]
self._notify_bus(action="PUT", instance_data=instance_data)
self.finalize(status)
def _extract_cardinalities(bindings, predicate_dict):
cardinalities = {}
for binding in bindings:
property_ = binding["predicate"]["value"]
try:
range_ = binding["range"]["value"]
except KeyError:
try:
range_ = predicate_dict[property_]["range"]["value"]
except KeyError:
msg = _(u"The property {0} is not defined properly").format(property_)
raise InstanceError(msg)
if not property_ in cardinalities:
cardinalities[property_] = {}
if not range_ in cardinalities[property_] and not range_.startswith("nodeID://"):
cardinalities[property_][range_] = {}
current_property = cardinalities[property_]
if "min" in binding:
min_value = binding["min"]["value"]
try:
min_value = int(min_value)
except ValueError:
msg = _(u"The property {0} defines a non-integer owl:minQualifiedCardinality {1}").format(property_, min_value)
raise InstanceError(msg)
else:
current_property[range_].update({"minItems": min_value})
if min_value:
current_property[range_].update({"required": True})
if "max" in binding:
max_value = binding["max"]["value"]
try:
max_value = int(max_value)
except ValueError:
msg = _(u"The property {0} defines a non-integer owl:maxQualifiedCardinality {1}").format(property_, max_value)
raise InstanceError(msg)
else:
current_property[range_].update({"maxItems": max_value})
return cardinalities
def safe_params(valid_params=None, body_params=None):
try:
yield
except InvalidParam as ex:
msg = _(u"Argument {0:s} is not supported.").format(ex)
if valid_params is not None:
params_msg = ", ".join(sorted(set(valid_params.keys() + DEFAULT_PARAMS.keys())))
msg += _(u" The supported querystring arguments are: {0}.").format(params_msg)
else:
params_msg = ", ".join(sorted(DEFAULT_PARAMS.keys()))
msg += _(u" The supported querystring arguments are: {0}.").format(params_msg)
if body_params is not None:
body_msg = ", ".join(body_params)
msg += _(u" The supported body arguments are: {0}.").format(body_msg)
raise HTTPError(400, log_message=msg)
except RequiredParamMissing as ex:
msg = _(u"Required parameter ({0:s}) was not given.").format(ex)
raise HTTPError(400, log_message=unicode(msg))
def get_query(stored_query, querystring_params):
# template existence is validated in stored query creation/modification
template = stored_query["sparql_template"]
try:
# .arguments is a dict with decoded querystring params
query = template % querystring_params.arguments
return query
except KeyError as key_error:
missing_key_message = _("Missing key '{0}' in querystring.\n Template: {1}").format(key_error.args[0], template)
raise HTTPError(400, log_message=missing_key_message)
def purge(self, context_name, class_name):
if settings.ENABLE_CACHE:
with safe_params():
self.query_params = ParamDict(self,
context_name=context_name,
class_name=class_name)
path = cache.build_key_for_class(self.query_params)
cache.purge_by_path(path, False)
else:
raise HTTPError(405, log_message=_("Cache is disabled (Brainaik's settings.ENABLE_CACHE is set to False)"))
def get_cached_schema(query_params, include_meta=False):
schema_key = build_key_for_class(query_params)
class_object = memoize(query_params, get_schema, query_params, key=schema_key)
if class_object is None or not class_object["body"]:
msg = _(u"The class definition for {0} was not found in graph {1}")
raise SchemaNotFound(msg.format(query_params['class_uri'], query_params['graph_uri']))
if include_meta:
return class_object
else:
return class_object['body']
def _validate_class_restriction(search_params, range_result):
classes = set(filter_values(range_result, "range"))
if "classes" in search_params:
classes_not_in_range = list(set(search_params["classes"]).difference(classes))
if classes_not_in_range:
raise HTTPError(400,
_(u"Classes {0} are not in the range of predicate '{1}'".format(classes_not_in_range, search_params["target"])))
classes = search_params["classes"]
return list(classes)
def purge(self, context_name, class_name):
if settings.ENABLE_CACHE:
with safe_params():
self.query_params = ParamDict(self,
context_name=context_name,
class_name=class_name)
path = cache.build_key_for_class(self.query_params)
cache.purge_by_path(path, False)
else:
raise HTTPError(405, log_message=_("Cache is disabled (Brainaik's settings.ENABLE_CACHE is set to False)"))
def safe_params(valid_params=None, body_params=None):
try:
yield
except InvalidParam as ex:
msg = _(u"Argument {0:s} is not supported.").format(ex)
if valid_params is not None:
params_msg = ", ".join(sorted(set(valid_params.keys() + DEFAULT_PARAMS.keys())))
msg += _(u" The supported querystring arguments are: {0}.").format(params_msg)
else:
params_msg = ", ".join(sorted(DEFAULT_PARAMS.keys()))
msg += _(u" The supported querystring arguments are: {0}.").format(params_msg)
if body_params is not None:
body_msg = ", ".join(body_params)
msg += _(u" The supported body arguments are: {0}.").format(body_msg)
raise HTTPError(400, log_message=msg)
except RequiredParamMissing as ex:
msg = _(u"Required parameter ({0:s}) was not given.").format(ex)
raise HTTPError(400, log_message=unicode(msg))
def patch(self, context_name, class_name, instance_id):
valid_params = INSTANCE_PARAMS
with safe_params(valid_params):
self.query_params = ParamDict(self,
context_name=context_name,
class_name=class_name,
instance_id=instance_id,
**valid_params)
del context_name
del class_name
del instance_id
try:
patch_list = json.loads(self.request.body)
except ValueError:
raise HTTPError(400,
log_message=_("No JSON object could be decoded"))
# Retrieve original data
instance_data = memoize(self.query_params,
get_instance,
key=build_instance_key(self.query_params),
function_arguments=self.query_params)
try:
instance_data = instance_data['body']
except TypeError:
raise HTTPError(404, log_message=_("Inexistent instance"))
instance_data.pop('http://www.w3.org/1999/02/22-rdf-syntax-ns#type',
None)
# compute patch
changed_data = apply_patch(instance_data, patch_list)
# Try to put
edit_instance(self.query_params, changed_data)
status = 200
# Clear cache
cache.purge_an_instance(self.query_params['instance_uri'])
self.finalize(status)
def finalize(self, response):
if response is None:
msg = _("There were no search results.")
raise HTTPError(404, log_message=msg)
elif isinstance(response, dict):
self.write(response)
self.set_header(
"Content-Type",
content_type_profile(build_schema_url(self.query_params)))
elif isinstance(response, int): # status code
self.set_status(response)
def get(self, context_name):
valid_params = LIST_PARAMS + GRAPH_PARAMS
with safe_params(valid_params):
self.query_params = ParamDict(self, context_name=context_name, **valid_params)
del context_name
response = list_classes(self.query_params)
if response is None:
raise HTTPError(404, log_message=_(u"Context {0} not found").format(self.query_params['graph_uri']))
self.finalize(response)
def get(self, context_name):
valid_params = LIST_PARAMS + GRAPH_PARAMS
with safe_params(valid_params):
self.query_params = ParamDict(self, context_name=context_name, **valid_params)
del context_name
response = list_classes(self.query_params)
if response is None:
raise HTTPError(404, log_message=_(u"Context {0} not found").format(self.query_params['graph_uri']))
self.finalize(response)
def _convert_to_python(object_value, class_schema, predicate_uri):
if predicate_uri in class_schema["properties"]:
schema_type = get_predicate_datatype(class_schema, predicate_uri)
python_type = MAP_RDF_EXPANDED_TYPE_TO_PYTHON.get(schema_type)
if python_type is None:
msg = _(u"The property {0} is unknown according to the schema definitions {1}").format(predicate_uri, class_schema)
logger.debug(msg)
converted_value = object_value
elif python_type == bool:
converted_value = decode_boolean(object_value)
else:
msg = _(u"The property {0} is mapped to a inconsistent value {1}").format(predicate_uri, object_value)
try:
converted_value = python_type(object_value)
except ValueError:
raise Exception(msg)
return converted_value
else: # values with predicate_uri not in schema will be ignored anyway
return object_value
def get_query(stored_query, querystring_params):
# template existence is validated in stored query creation/modification
template = stored_query["sparql_template"]
try:
# .arguments is a dict with decoded querystring params
query = template % querystring_params.arguments
return query
except KeyError as key_error:
missing_key_message = _(
"Missing key '{0}' in querystring.\n Template: {1}").format(
key_error.args[0], template)
raise HTTPError(400, log_message=missing_key_message)
def delete_instance(query_params):
dependants_result_dict = query_dependants(query_params)
if not is_result_empty(dependants_result_dict):
values = [item['dependant']['value'] for item in dependants_result_dict['results']['bindings']]
str_values = ", ".join(values)
raise HTTPError(409, log_message=_(u"Cannot exclude instance because of the dependencies: {0}").format(str_values))
query_result_dict = query_delete(query_params)
if some_triples_deleted(query_result_dict, query_params['graph_uri']):
return True
def _validate_graph_restriction(search_params, range_result):
graphs = set(filter_values(range_result, "range_graph"))
if "graphs" in search_params:
graphs_set = set(search_params["graphs"])
graphs_not_in_range = list(graphs_set.difference(graphs))
if graphs_not_in_range:
raise HTTPError(
400,
_(u"Classes in the range of predicate '{0}' are not in graphs {1}"
.format(search_params["target"], graphs_not_in_range)))
graphs = graphs_set
graphs = graphs.difference(set(settings.GRAPHS_WITHOUT_INSTANCES))
if not graphs:
raise HTTPError(
400,
_(u"Classes in the range of predicate '{0}' are in graphs without instances, such as: {1}"
.format(search_params["target"],
settings.GRAPHS_WITHOUT_INSTANCES)))
return list(graphs)
def patch(self, context_name, class_name, instance_id):
valid_params = INSTANCE_PARAMS
with safe_params(valid_params):
self.query_params = ParamDict(self,
context_name=context_name,
class_name=class_name,
instance_id=instance_id,
**valid_params)
del context_name
del class_name
del instance_id
try:
patch_list = json.loads(self.request.body)
except ValueError:
raise HTTPError(400, log_message=_("No JSON object could be decoded"))
# Retrieve original data
instance_data = memoize(self.query_params,
get_instance,
key=build_instance_key(self.query_params),
function_arguments=self.query_params)
try:
instance_data = instance_data['body']
except TypeError:
raise HTTPError(404, log_message=_("Inexistent instance"))
instance_data.pop('http://www.w3.org/1999/02/22-rdf-syntax-ns#type', None)
# compute patch
changed_data = apply_patch(instance_data, patch_list)
# Try to put
edit_instance(self.query_params, changed_data)
status = 200
# Clear cache
cache.purge_an_instance(self.query_params['instance_uri'])
self.finalize(status)
def _handle_request_exception(self, e):
if hasattr(e, "status_code"): # and e.code in httplib.responses:
status_code = e.status_code
else:
status_code = 500
error_message = u"[{0}] on {1}".format(status_code,
self._request_summary())
if isinstance(e, NotificationFailure):
message = unicode(e)
logger.error(message)
self.send_error(status_code, message=message)
elif isinstance(e, HTTPClientError):
message = _(u"Access to backend service failed. {0:s}.").format(
unicode(e))
extra_messages = check_messages_when_port_is_mentioned(unicode(e))
if extra_messages:
for msg in extra_messages:
message += msg
if hasattr(e, "response") and e.response is not None and \
hasattr(e.response, "body") and e.response.body is not None:
message += _(u"\nResponse:\n") + unicode(
str(e.response.body).decode("utf-8"))
logger.error(message)
self.send_error(status_code, message=message)
elif isinstance(e, InstanceError):
logger.error(
_(u"Ontology inconsistency: {0}\n").format(error_message))
self.send_error(status_code, message=e.message)
elif isinstance(e, HTTPError):
try:
possible_list = json.loads(e.log_message)
except (TypeError, ValueError):
pass
else:
if isinstance(possible_list, list):
self.send_error(status_code, errors_list=possible_list)
return
if e.log_message:
error_message += u"\n {0}".format(e.log_message)
if status_code == 500:
logger.error(
_(u"Unknown HTTP error [{0}]:\n {1}\n").format(
e.status_code, error_message))
self.send_error(status_code,
exc_info=sys.exc_info(),
message=e.log_message)
else:
logger.error(_(u"HTTP error: {0}\n").format(error_message))
self.send_error(status_code, message=e.log_message)
else:
logger.error(_(u"Uncaught exception: {0}\n").format(error_message),
exc_info=True)
self.send_error(status_code, exc_info=sys.exc_info())
def get(self):
triplestore_status = triplestore.status()
event_bus_status = event_bus.status()
output = []
if "SUCCEED" not in triplestore_status:
output.append(triplestore_status)
if "FAILED" in event_bus_status:
output.append(event_bus_status)
if output:
response = "\n".join(output)
else:
response = _(u"WORKING")
self.write(response)
def check_and_clean_rdftype(instance_type, items):
"""Validate actual type and remove rdf:type from the instance to be returned"""
if 'rdf:type' in items:
rdftype = 'rdf:type'
elif 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' in items:
rdftype = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'
else:
rdftype = None
if rdftype is not None:
if instance_type != items[rdftype]:
msg = _(u"The type specified={0} is not the same informed from the triplestore={1}")
raise Exception(msg.format(instance_type, items[rdftype]))
del items[rdftype]
def purge(self, context_name, class_name, instance_id):
if settings.ENABLE_CACHE:
optional_params = INSTANCE_PARAMS
with safe_params(optional_params):
self.query_params = ParamDict(self,
context_name=context_name,
class_name=class_name,
instance_id=instance_id,
**optional_params)
path = build_instance_key(self.query_params)
cache.purge_by_path(path, False)
else:
raise HTTPError(405, log_message=_("Cache is disabled (Brainaik's settings.ENABLE_CACHE is set to False)"))
def get(self):
triplestore_status = triplestore.status()
event_bus_status = event_bus.status()
output = []
if "SUCCEED" not in triplestore_status:
output.append(triplestore_status)
if "FAILED" in event_bus_status:
output.append(event_bus_status)
if output:
response = "\n".join(output)
else:
response = _(u"WORKING")
self.write(response)
def _validate_class_restriction(search_params, range_result):
classes = set(filter_values(range_result, "range"))
if "classes" in search_params:
classes_not_in_range = list(
set(search_params["classes"]).difference(classes))
if classes_not_in_range:
raise HTTPError(
400,
_(u"Classes {0} are not in the range of predicate '{1}'".
format(classes_not_in_range, search_params["target"])))
classes = search_params["classes"]
return list(classes)
def purge(self, context_name, class_name, instance_id):
if settings.ENABLE_CACHE:
optional_params = INSTANCE_PARAMS
with safe_params(optional_params):
self.query_params = ParamDict(self,
context_name=context_name,
class_name=class_name,
instance_id=instance_id,
**optional_params)
path = build_instance_key(self.query_params)
cache.purge_by_path(path, False)
else:
raise HTTPError(405, log_message=_("Cache is disabled (Brainaik's settings.ENABLE_CACHE is set to False)"))
