def instance_from_full_uri(self,
uri,
cls=None,
use_cache=True,
deprecated=False,
api="query",
scope="released",
resolved=False):
# 'deprecated=True' means 'returns an instance even if that instance is deprecated'
# should perhaps be called 'show_deprecated' or 'include_deprecated'
logger.debug(
"Retrieving instance from {}, api='{}' use_cache={}".format(
uri, api, use_cache))
if use_cache and uri in self.cache:
logger.debug("Retrieving instance {} from cache".format(uri))
instance = self.cache[uri]
elif api == "nexus":
instance = Instance(Instance.extract_id_from_url(
uri, self._instance_repo.path),
data=self._instance_repo._http_client.get(uri),
root_path=Instance.path)
if instance and instance.data and "@id" in instance.data:
if deprecated is False and instance.data["nxv:deprecated"]:
instance = None
logger.debug("Not returning deprecated instance")
else:
self.cache[instance.data["@id"]] = instance
logger.debug("Retrieved instance from KG Nexus" +
str(instance.data))
else:
instance = None
elif api == "query":
if cls and hasattr(cls, "query_id") and cls.query_id is not None:
if resolved:
query_id = cls.query_id_resolved
else:
query_id = cls.query_id
response = self._kg_query_client.get(
"{}/{}/instances?databaseScope={}&id={}".format(
cls.path, query_id, SCOPE_MAP[scope], uri))
if response and len(response["results"]) > 0:
instance = Instance(cls.path, response["results"][0],
Instance.path)
self.cache[instance.data["@id"]] = instance
logger.debug("Retrieved instance from KG Query" +
str(instance.data))
else:
logger.warning(
"Instance not found at {} using KG Query API".format(
uri))
instance = None
else:
raise NotImplementedError(
"No query id available: cls={}".format(str(cls)))
else:
raise ValueError("'api' must be either 'nexus' or 'query'")
return instance
def query_kgquery(self,
path,
query_id,
filter,
from_index=0,
size=100,
scope="released"):
template = "{}/{}/instances?start={{}}&size={}&databaseScope={}".format(
path, query_id, size, SCOPE_MAP[scope])
if filter:
for key, value in filter.items():
if hasattr(value, "iri"):
filter[key] = value.iri
template += "&" + "&".join(
"{}={}".format(k, quote_plus(v.encode("utf-8")))
for k, v in filter.items())
if scope not in SCOPE_MAP:
raise ValueError("'scope' must be either '{}'".format(
"' or '".join(list(SCOPE_MAP))))
start = from_index
#url = quote_plus(template.format(start).encode("utf-8"))
url = template.format(start)
try:
response = self._kg_query_client.get(url)
except HTTPError as err:
if err.response.status_code == 403:
response = None
else:
raise
if response and "results" in response:
instances = [
Instance(path, data, Instance.path)
for data in response["results"]
]
start += response["size"]
while start < min(response["total"], size):
#url = quote_plus(template.format(start).encode("utf-8"))
url = template.format(start)
response = self._kg_query_client.get(url)
instances.extend([
Instance(path, data, Instance.path)
for data in response["results"]
])
start += response["size"]
else:
instances = []
for instance in instances:
self.cache[instance.data["@id"]] = instance
instance.data["fg:api"] = "query"
return instances
def by_name(self,
cls,
name,
match="equals",
all=False,
api="query",
scope="released",
resolved=False):
"""Retrieve an object based on the value of schema:name"""
# todo: allow non-exact searches
if api not in ("query", "nexus"):
raise ValueError("'api' must be either 'nexus' or 'query'")
valid_match_methods = {
#"query": ("starts_with", "ends_with", "contains", "equals", "regex"),
"query": ("contains", "equals"),
"nexus": ("equals")
}
if match not in valid_match_methods[api]:
raise ValueError("'match' must be one of {}".format(
valid_match_methods[api]))
if api == "nexus":
op = {"equals": "eq", "contains": "in"}[match]
context = {"schema": "http://schema.org/"}
query_filter = {"path": "schema:name", "op": op, "value": name}
instances = self.query_nexus(cls.path, query_filter, context)
else:
assert api == "query"
if hasattr(cls, "query_id") and cls.query_id is not None:
if resolved:
query_id = cls.query_id_resolved
else:
query_id = cls.query_id
response = self._kg_query_client.get(
"{}/{}{}/instances?databaseScope={}&name={}".format(
cls.path,
query_id,
match == "contains" and "_name_contains"
or "", # workaround
SCOPE_MAP[scope],
name))
instances = [
Instance(cls.path, result, Instance.path)
for result in response["results"]
]
else:
raise NotImplementedError(
"Coming soon. For now, please use api='nexus'")
if instances:
if all:
return [
cls.from_kg_instance(inst, self, resolved=resolved)
for inst in instances
]
else: # return only the first result
return cls.from_kg_instance(instances[0],
self,
resolved=resolved)
else:
return None
def test_round_trip(self, kg_client):
trace1 = Trace("example001",
data_location=Distribution(
"http://example.com/example.csv",
content_type="text/tab-separated-values"),
generated_by=MockKGObject(
id="http://fake_uuid_abc123",
type=PatchClampExperiment.type),
generation_metadata=MockKGObject(
id="http://fake_uuid_def456",
type=QualifiedTraceGeneration.type),
channel=42,
data_unit="mV",
time_step=QuantitativeValue(0.1, "ms"),
part_of=MockKGObject(id="http://fake_uuid_ghi789",
type=Dataset.type))
instance = Instance(Trace.path, trace1._build_data(kg_client),
Instance.path)
instance.data["@id"] = "http://fake_uuid_6a5d6ecf87"
instance.data["@type"] = Trace.type
trace2 = Trace.from_kg_instance(instance, kg_client)
for field in ("name", "data_location", "channel", "data_unit",
"time_step"):
assert getattr(trace1, field) == getattr(trace2, field)
for field in ("generated_by", "generation_metadata", "part_of"):
obj1 = getattr(trace1, field)
obj2 = getattr(trace2, field)
assert isinstance(obj2, KGProxy)
assert obj1.id == obj2.id
assert obj1.type == obj2.type
def test_round_trip(self, kg_client):
p1 = Person("Hamilton", "Margaret", "*****@*****.**",
KGProxy(Organization, "http://fake_uuid_855fead8"))
instance = Instance(Person.path, p1._build_data(kg_client), Instance.path)
instance.data["@id"] = "http://fake_uuid_8ab3dc739b"
instance.data["@type"] = Person.type
p2 = Person.from_kg_instance(instance, kg_client)
for field in ("family_name", "given_name", "email", "affiliation", "full_name"):
assert getattr(p1, field) == getattr(p2, field)
def test_round_trip(self, kg_client):
obj1 = Organization(name="NeuroPSI",
address=Address(locality="Saclay", country="France"),
parent=KGProxy(Organization, "http://fake_uuid_00481be7a1"))
instance = Instance(Organization.path, obj1._build_data(kg_client), Instance.path)
instance.data["@id"] = "http://fake_uuid_7bb3c1e78b"
instance.data["@type"] = Organization.type
obj2 = Organization.from_kg_instance(instance, kg_client)
for field in ("name", "address", "parent"):
assert getattr(obj1, field) == getattr(obj2, field)
def instance_from_full_uri(self, uri):
if uri in self.cache:
return self.cache[uri]
else:
instance = Instance(Instance.extract_id_from_url(
uri, self._instance_repo.path),
data=self._instance_repo._http_client.get(uri),
root_path=Instance.path)
self.cache[instance.data["@id"]] = instance
return instance
def test_round_trip(self, kg_client):
obj1 = Subject(name="Mickey", species=Species("Mus musculus"),
strain=Strain("129/Sv"), sex=Sex("male"),
age=Age(QuantitativeValue(20, "days"), "Post-natal"),
death_date=datetime(1960, 1, 1))
instance = Instance(Subject.path, obj1._build_data(kg_client), Instance.path)
instance.data["@id"] = "http://fake_uuid_9ab2227fe1"
instance.data["@type"] = Subject.type
obj2 = Subject.from_kg_instance(instance, kg_client)
for field in ("name", "species", "strain", "sex", "age", "death_date"):
assert getattr(obj1, field) == getattr(obj2, field)
def instance_from_full_uri(self, uri, use_cache=True):
if use_cache and uri in self.cache:
logger.debug("Retrieving instance from cache")
return self.cache[uri]
else:
instance = Instance(Instance.extract_id_from_url(
uri, self._instance_repo.path),
data=self._instance_repo._http_client.get(uri),
root_path=Instance.path)
self.cache[instance.data["@id"]] = instance
logger.debug("Retrieved instance from KG " + str(instance.data))
return instance
def test_round_trip_with_morphology_file(self, kg_client):
cls = self.class_under_test
obj1 = cls("test_morph", morphology_file="http://example.com/test.asc")
instance = Instance(cls.path, obj1._build_data(kg_client),
Instance.path)
instance.data["@id"] = random_uuid()
instance.data["@type"] = cls.type
obj2 = cls.from_kg_instance(instance, kg_client)
for field in cls.fields:
if field.intrinsic:
val1 = getattr(obj1, field.name)
val2 = getattr(obj2, field.name)
if issubclass(field.types[0], KGObject):
assert isinstance(val1, MockKGObject)
assert isinstance(val2, KGProxy)
assert val1.type == val2.cls.type
else:
assert val1 == val2
assert obj1.morphology_file == obj2.morphology_file
def test_round_trip(self, kg_client):
cell1 = PatchedCell("example001",
brain_location=BrainRegion("primary auditory cortex"),
collection=None,
cell_type=CellType("pyramidal cell"),
experiments=None,
pipette_id=31,
seal_resistance=QuantitativeValue(1.2, "GΩ"),
pipette_resistance=QuantitativeValue(1.5, "MΩ"),
liquid_junction_potential=QuantitativeValue(5.0, "mV"),
labeling_compound="0.1% biocytin ",
reversal_potential_cl=QuantitativeValue(-65, "mV"))
instance = Instance(PatchedCell.path, cell1._build_data(kg_client), Instance.path)
instance.data["@id"] = "http://fake_uuid_93f9cd9a9b"
instance.data["@type"] = PatchedCell.type
cell2 = PatchedCell.from_kg_instance(instance, kg_client)
for field in ("name", "brain_location", "cell_type",
"pipette_id", "seal_resistance", "pipette_resistance",
"liquid_junction_potential", "labeling_compound",
"reversal_potential_cl"):
assert getattr(cell1, field) == getattr(cell2, field)
def test_round_trip_minimal_random(self, kg_client):
cls = self.class_under_test
if cls.fields:
obj1 = generate_random_object(cls, all_fields=False)
instance = Instance(cls.path, obj1._build_data(kg_client),
Instance.path)
instance.data["@id"] = random_uuid()
instance.data["@type"] = cls.type
obj2 = cls.from_kg_instance(instance, kg_client)
for field in cls.fields:
if field.intrinsic and field.required:
val1 = getattr(obj1, field.name)
val2 = getattr(obj2, field.name)
if issubclass(field.types[0], KGObject):
assert isinstance(val1, MockKGObject)
assert isinstance(val2, KGProxy)
assert val1.type == val2.cls.type
elif date in field.types:
assert dates_equal(val1, val2)
else:
assert val1 == val2
def build_kg_object(cls, data, resolved=False, client=None):
"""
Build a KGObject, a KGProxy, or a list of such, based on the data provided.
This takes care of the JSON-LD quirk that you get a list if there are multiple
objects, but you get the object directly if there is only one.
Returns `None` if data is None.
"""
if data is None:
return None
if not isinstance(data, list):
if not isinstance(data, dict):
raise ValueError("data must be a list or dict")
if "@list" in data:
assert len(data) == 1
data = data["@list"]
else:
data = [data]
objects = []
for item in data:
if cls is None:
# note that if cls is None, then the class can be different for each list item
# therefore we need to use a new variable kg_cls inside the loop
if "@type" in item:
try:
kg_cls = lookup_type(item["@type"])
except KeyError:
kg_cls = lookup_type(compact_uri(item["@type"], standard_context))
elif "label" in item:
kg_cls = lookup_by_iri(item["@id"])
# todo: add lookup by @id
else:
raise ValueError("Cannot determine type. Item was: {}".format(item))
else:
kg_cls = cls
if issubclass(kg_cls, StructuredMetadata):
obj = kg_cls.from_jsonld(item)
elif issubclass(kg_cls, KGObject):
if "@id" in item and item["@id"].startswith("http"):
# here is where we check the "resolved" keyword,
# and return an actual object if we have the data
# or resolve the proxy if we don't
if resolved:
if kg_cls.namespace is None:
kg_cls.namespace = namespace_from_id(item["@id"])
try:
instance = Instance(kg_cls.path, item, Instance.path)
obj = kg_cls.from_kg_instance(instance, client, resolved=resolved)
except (ValueError, KeyError) as err:
# to add: emit a warning
logger.warning("Error in building {}: {}".format(kg_cls.__name__, err))
obj = KGProxy(kg_cls, item["@id"]).resolve(
client,
api=item.get("fg:api", "query"))
else:
obj = KGProxy(kg_cls, item["@id"])
else:
# todo: add a logger.warning that we have dud data
obj = None
else:
raise ValueError("cls must be a subclass of KGObject or StructuredMetadata")
if obj is not None:
objects.append(obj)
if len(objects) == 1:
return objects[0]
else:
return objects
def create_new_instance(self, path, data):
instance = Instance(path, data, Instance.path)
entity = self._nexus_client.instances.create(instance)
entity.data.update(data)
return entity
def resolve(self, search_result):
identifier = Entity.extract_id_from_url(search_result.self_link,
self.path)
data = self._read(identifier)
return Instance(identifier, data,
self.path) if data is not None else None
def read(self, organization, domain, schema, version, uuid, revision=None):
identifier = Instance.create_id(organization, domain, schema,
version) + "/" + uuid
data = self._read(identifier, revision)
return Instance(identifier, data,
self.path) if data is not None else None
def read_by_full_id(self, full_id, revision=None):
data = self._read(full_id, revision)
return Instance(full_id, data, self.path) if data is not None else None