def _prepare_incremental(self): """ Look in the resource registry for any resources that have a preload ID on them so that they can be referenced under this preload ID during this load run. """ log.debug("Loading prior preloaded resources for reference") access_args = create_access_args("SUPERUSER", ["SUPERUSER"]) res_objs, res_keys = self.rr.find_resources_ext(alt_id_ns="PRE", id_only=False, access_args=access_args) res_preload_ids = [key['alt_id'] for key in res_keys] res_ids = [obj._id for obj in res_objs] log.debug("Found %s previously preloaded resources", len(res_objs)) res_assocs = self.rr.find_associations(predicate="*", id_only=False) [self.resource_assocs.setdefault(assoc["p"], []).append(assoc) for assoc in res_assocs] log.debug("Found %s existing associations", len(res_assocs)) existing_resources = dict(zip(res_preload_ids, res_objs)) if len(existing_resources) != len(res_objs): log.error("Stored preload IDs are NOT UNIQUE!!! This causes random links to existing resources") res_id_mapping = dict(zip(res_preload_ids, res_ids)) self.resource_ids.update(res_id_mapping) res_obj_mapping = dict(zip(res_preload_ids, res_objs)) self.resource_objs.update(res_obj_mapping)
def execute_query(self, discovery_query, id_only=True, query_args=None, query_params=None): try: if "QUERYEXP" in discovery_query: ds_query, ds_name = discovery_query, discovery_query["query_args"].get("datastore", DataStore.DS_RESOURCES) else: log.info("DatastoreDiscovery.execute_query(): discovery_query=\n%s", pprint.pformat(discovery_query)) ds_query, ds_name = self._build_ds_query(discovery_query, id_only=id_only) current_actor_id=get_ion_actor_id(self.process) ds_query.setdefault("query_params", {}) if query_params: ds_query["query_params"].update(query_params) ds_query["query_params"]["current_actor"] = current_actor_id log.debug("DatastoreDiscovery.execute_query(): ds_query=\n%s", pprint.pformat(ds_query)) ds = self._get_datastore(ds_name) access_args = create_access_args(current_actor_id=current_actor_id, superuser_actor_ids=self.container.resource_registry.get_superuser_actors()) query_results = ds.find_by_query(ds_query, access_args=access_args) log.info("Datastore discovery query resulted in %s rows", len(query_results)) if query_args and query_args.get("query_info", False): query_info = dict(_query_info=True, query=ds_query, access_args=access_args, ds_name=ds_name) query_info.update(ds_query.get("_result", {})) query_results.append(query_info) return query_results except Exception as ex: log.exception("DatastoreDiscovery.execute_query() failed") return []
def find_resources_ext(self, restype='', lcstate='', name='', keyword='', nested_type='', attr_name='', attr_value='', alt_id='', alt_id_ns='', limit=0, skip=0, descending=False, id_only=False, query=''): access_args = create_access_args(current_actor_id=get_ion_actor_id(self._process), superuser_actor_ids=self._rr.get_superuser_actors()) return self._rr.find_resources_ext(restype=restype, lcstate=lcstate, name=name, keyword=keyword, nested_type=nested_type, attr_name=attr_name, attr_value=attr_value, alt_id=alt_id, alt_id_ns=alt_id_ns, limit=limit, skip=skip, descending=descending, id_only=id_only, query=query, access_args=access_args)
def execute_query(self, discovery_query, id_only=True): try: if "QUERYEXP" in discovery_query: ds_query, ds_name = discovery_query, discovery_query["query_exp"].get("datastore", DataStore.DS_RESOURCES) else: log.info("DatastoreDiscovery.execute_query(): discovery_query=\n%s", pprint.pformat(discovery_query)) ds_query, ds_name = self._build_ds_query(discovery_query, id_only=id_only) log.debug("DatastoreDiscovery.execute_query(): ds_query=\n%s", pprint.pformat(ds_query)) ds = self._get_datastore(ds_name) access_args = create_access_args(current_actor_id=get_ion_actor_id(self.process), superuser_actor_ids=self.container.resource_registry.get_superuser_actors()) res = ds.find_by_query(ds_query, access_args=access_args) log.info("Datastore discovery query resulted in %s rows", len(res)) return res except Exception as ex: log.exception("DatastoreDiscovery.execute_query() failed") return []
def test_datastore_query(self): data_store = self.ds_class(datastore_name='ion_test_ds', profile=DataStore.DS_PROFILE.RESOURCES, scope=get_sys_name()) # Just in case previous run failed without cleaning up, delete data store try: data_store.delete_datastore() except NotFound: pass data_store.create_datastore() self.data_store = data_store self.resources = {} from interface.objects import GeospatialIndex, ResourceVisibilityEnum, GeospatialLocation # Create a few resources plat1_obj_id = self._create_resource(RT.TestPlatform, 'Buoy1', description='My Platform') aid1_obj_id = self._create_resource(RT.ActorIdentity, 'Actor1') plat2_obj_id = self._create_resource(RT.TestPlatform, 'Buoy2', visibility=ResourceVisibilityEnum.OWNER) self._create_association(plat2_obj_id, PRED.hasOwner, aid1_obj_id) plat3_obj_id = self._create_resource(RT.TestPlatform, 'Buoy3', visibility=ResourceVisibilityEnum.OWNER) dp1_obj_id = self._create_resource(RT.TestSite, 'Site1', location=GeospatialLocation(latitude=1.0, longitude=2.0)) # Queries qb = DatastoreQueryBuilder() qb.build_query(where=qb.or_(qb.and_(qb.eq(qb.RA_NAME, "Buoy1"), qb.eq(qb.RA_NAME, "Buoy1")), qb.eq(qb.RA_NAME, "Buoy1"))) res = data_store.find_by_query(qb.get_query()) self.assertEquals(len(res), 1) qb = DatastoreQueryBuilder() qb.build_query(where=qb.and_(qb.like(qb.RA_NAME, "Si%"), qb.overlaps_bbox(qb.RA_GEOM, 1, -1.2, 4, 4))) res = data_store.find_by_query(qb.get_query()) self.assertEquals(len(res), 1) qb = DatastoreQueryBuilder() qb.build_query(where=qb.attr_like("description", "My%")) res = data_store.find_by_query(qb.get_query()) self.assertEquals(len(res), 1) # two tests: first should NOT have above Site1 in radius, second should qb = DatastoreQueryBuilder(where=qb.overlaps_geom(qb.RA_GEOM,'POINT(2.0 2.0)',0.5)) qb.build_query() res = data_store.find_by_query(qb.get_query()) self.assertEquals(len(res), 0) # -- additional 0.001 is to compensate for outer edge NOT being considered an overlap/intersect qb = DatastoreQueryBuilder(where=qb.overlaps_geom(qb.RA_GEOM,'POINT(2.0 2.0)',1.001)) qb.build_query() res = data_store.find_by_query(qb.get_query()) self.assertEquals(len(res), 1) # Access tests qb = DatastoreQueryBuilder() qb.build_query(where=qb.or_(qb.and_(qb.like(qb.RA_NAME, "Buoy%"), qb.eq(qb.ATT_TYPE, RT.TestPlatform)))) res = data_store.find_by_query(qb.get_query()) self.assertEquals(len(res), 1) access_args = create_access_args(current_actor_id=aid1_obj_id) res = data_store.find_by_query(qb.get_query(), access_args=access_args) self.assertEquals(len(res), 2) access_args = create_access_args(current_actor_id=aid1_obj_id, superuser_actor_ids=[aid1_obj_id]) res = data_store.find_by_query(qb.get_query(), access_args=access_args) self.assertEquals(len(res), 3) # Clean up self.data_store.delete_mult([plat1_obj_id, plat2_obj_id, plat3_obj_id, aid1_obj_id, dp1_obj_id])
def test_visibility(self): res_objs = [ (IonObject(RT.ActorIdentity, name="system"), ), (IonObject(RT.ActorIdentity, name="AI1"), ), (IonObject(RT.ActorIdentity, name="AI2"), ), (IonObject(RT.ActorIdentity, name="AI3"), ), (IonObject(RT.Org, name="Org1"), ), (IonObject(RT.Org, name="Org2"), ), (IonObject(RT.TestInstrument, name="ID1a", visibility=ResourceVisibilityEnum.PUBLIC, availability=AS.AVAILABLE), "AI1"), (IonObject(RT.TestInstrument, name="ID1b", visibility=ResourceVisibilityEnum.PUBLIC, availability=AS.PRIVATE), "AI1"), (IonObject(RT.TestInstrument, name="ID1c", visibility=ResourceVisibilityEnum.PUBLIC, availability=AS.AVAILABLE), "AI2"), (IonObject(RT.TestInstrument, name="ID2a", visibility=ResourceVisibilityEnum.REGISTERED, availability=AS.AVAILABLE), "AI1"), (IonObject(RT.TestInstrument, name="ID2b", visibility=ResourceVisibilityEnum.REGISTERED, availability=AS.AVAILABLE), "AI2"), (IonObject(RT.TestInstrument, name="ID2c", visibility=ResourceVisibilityEnum.REGISTERED, availability=AS.AVAILABLE), ), (IonObject(RT.TestInstrument, name="ID3a", visibility=ResourceVisibilityEnum.FACILITY, availability=AS.AVAILABLE), "AI1"), (IonObject(RT.TestInstrument, name="ID3b", visibility=ResourceVisibilityEnum.FACILITY, availability=AS.AVAILABLE), "AI2"), (IonObject(RT.TestInstrument, name="ID3c", visibility=ResourceVisibilityEnum.FACILITY, availability=AS.AVAILABLE), ), (IonObject(RT.TestInstrument, name="ID4a", visibility=ResourceVisibilityEnum.OWNER, availability=AS.AVAILABLE), "AI1"), (IonObject(RT.TestInstrument, name="ID4b", visibility=ResourceVisibilityEnum.OWNER, availability=AS.AVAILABLE), "AI2"), (IonObject(RT.TestInstrument, name="ID4c", visibility=ResourceVisibilityEnum.OWNER, availability=AS.AVAILABLE), ), (IonObject(RT.TestDataset, name="DP1a", visibility=ResourceVisibilityEnum.PUBLIC, availability=AS.AVAILABLE), "AI1"), (IonObject(RT.TestDataset, name="DP1b", visibility=ResourceVisibilityEnum.PUBLIC, availability=AS.PRIVATE), "AI1"), (IonObject(RT.TestDataset, name="DP1c", visibility=ResourceVisibilityEnum.PUBLIC, availability=AS.AVAILABLE), "AI2"), (IonObject(RT.TestDataset, name="DP2a", visibility=ResourceVisibilityEnum.REGISTERED, availability=AS.AVAILABLE), "AI1"), (IonObject(RT.TestDataset, name="DP2b", visibility=ResourceVisibilityEnum.REGISTERED, availability=AS.AVAILABLE), "AI2"), (IonObject(RT.TestDataset, name="DP2c", visibility=ResourceVisibilityEnum.REGISTERED, availability=AS.AVAILABLE), ), (IonObject(RT.TestDataset, name="DP3a", visibility=ResourceVisibilityEnum.FACILITY, availability=AS.AVAILABLE), "AI1"), (IonObject(RT.TestDataset, name="DP3b", visibility=ResourceVisibilityEnum.FACILITY, availability=AS.AVAILABLE), "AI2"), (IonObject(RT.TestDataset, name="DP3c", visibility=ResourceVisibilityEnum.FACILITY, availability=AS.AVAILABLE), ), (IonObject(RT.TestDataset, name="DP4a", visibility=ResourceVisibilityEnum.OWNER, availability=AS.AVAILABLE), "AI1"), (IonObject(RT.TestDataset, name="DP4b", visibility=ResourceVisibilityEnum.OWNER, availability=AS.AVAILABLE), "AI2"), (IonObject(RT.TestDataset, name="DP4c", visibility=ResourceVisibilityEnum.OWNER, availability=AS.AVAILABLE), ), ] assocs = [ ("Org1", PRED.hasMember, "AI2"), ("Org1", PRED.hasResource, "ID3a"), ("Org1", PRED.hasResource, "ID3b"), ("Org1", PRED.hasResource, "ID3c"), ("ID1a", PRED.hasTestDataset, "DP1a"), ("ID1b", PRED.hasTestDataset, "DP1b"), ("ID1c", PRED.hasTestDataset, "DP1c"), ("ID2a", PRED.hasTestDataset, "DP2a"), ("ID2b", PRED.hasTestDataset, "DP2b"), ("ID2c", PRED.hasTestDataset, "DP2c"), ("ID3a", PRED.hasTestDataset, "DP3a"), ("ID3b", PRED.hasTestDataset, "DP3b"), ("ID3c", PRED.hasTestDataset, "DP3c"), ("ID4a", PRED.hasTestDataset, "DP4a"), ("ID4b", PRED.hasTestDataset, "DP4b"), ("ID4c", PRED.hasTestDataset, "DP4c"), ("DP1a", PRED.hasTestSource, "ID1a"), ("DP1b", PRED.hasTestSource, "ID1b"), ("DP1c", PRED.hasTestSource, "ID1c"), ("DP2a", PRED.hasTestSource, "ID2a"), ("DP2b", PRED.hasTestSource, "ID2b"), ("DP2c", PRED.hasTestSource, "ID2c"), ("DP3a", PRED.hasTestSource, "ID3a"), ("DP3b", PRED.hasTestSource, "ID3b"), ("DP3c", PRED.hasTestSource, "ID3c"), ("DP4a", PRED.hasTestSource, "ID4a"), ("DP4b", PRED.hasTestSource, "ID4b"), ("DP4c", PRED.hasTestSource, "ID4c"), ] res_by_name = {} for res_entry in res_objs: res_obj = res_entry[0] res_name = res_obj.name res_obj.alt_ids.append("TEST:%s" % res_name) actor_id = res_by_name[res_entry[1]] if len(res_entry) > 1 else None rid, _ = self.rr.create(res_obj, actor_id=actor_id) res_by_name[res_name] = rid for assoc in assocs: sname, p, oname = assoc s, o = res_by_name[sname], res_by_name[oname] self.rr.create_association(s, p, o) # Finds with different caller actors # - Anonymous call (expects only PUBLIC) rids, _ = self.rr.find_resources(restype=RT.TestInstrument, id_only=True) self.assertEquals(len(rids), 3) for rname in ["ID1a", "ID1b", "ID1c"]: self.assertIn(res_by_name[rname], rids) # - Registered user call (expects to see all REGISTERED and owned resources) access_args = create_access_args(current_actor_id=res_by_name["AI1"]) rids, _ = self.rr.find_resources(restype=RT.TestInstrument, id_only=True, access_args=access_args) self.assertEquals(len(rids), 8) for rname in ["ID1a", "ID1b", "ID1c", "ID2a", "ID2b", "ID2c", "ID3a", "ID4a"]: self.assertIn(res_by_name[rname], rids, "Resource %s" % rname) # - Facility access_args = create_access_args(current_actor_id=res_by_name["AI2"]) rids, _ = self.rr.find_resources(restype=RT.TestInstrument, id_only=True, access_args=access_args) #self.assertEquals(len(rids), 9) for rname in ["ID1a", "ID1b", "ID1c", "ID2a", "ID2b", "ID2c", "ID3a", "ID3b", "ID4b"]: self.assertIn(res_by_name[rname], rids, "Resource %s" % rname) # - Superuser call (expects to see all) access_args = create_access_args(current_actor_id=res_by_name["AI1"], superuser_actor_ids=[res_by_name["AI1"]]) rids, _ = self.rr.find_resources(restype=RT.TestInstrument, id_only=True, access_args=access_args) self.assertEquals(len(rids), 12) # Find by association # - Anonymous rids, _ = self.rr.find_subjects(subject_type=RT.TestInstrument, predicate=PRED.hasTestDataset, object=res_by_name["DP1a"], id_only=True) self.assertEquals(len(rids), 1) self.assertEquals(rids[0], res_by_name["ID1a"]) rids, _ = self.rr.find_objects(subject=res_by_name["DP1a"], predicate=PRED.hasTestSource, object_type=RT.TestInstrument, id_only=True) self.assertEquals(len(rids), 1) self.assertEquals(rids[0], res_by_name["ID1a"]) # - Owner rids, _ = self.rr.find_subjects(subject_type=RT.TestInstrument, predicate=PRED.hasTestDataset, object=res_by_name["DP4a"], id_only=True) self.assertEquals(len(rids), 0) access_args = create_access_args(current_actor_id=res_by_name["AI1"]) rids, _ = self.rr.find_subjects(subject_type=RT.TestInstrument, predicate=PRED.hasTestDataset, object=res_by_name["DP4a"], id_only=True, access_args=access_args) self.assertEquals(len(rids), 1) self.assertEquals(rids[0], res_by_name["ID4a"]) rids, _ = self.rr.find_objects(subject=res_by_name["DP4a"], predicate=PRED.hasTestSource, object_type=RT.TestInstrument, id_only=True) self.assertEquals(len(rids), 0) rids, _ = self.rr.find_objects(subject=res_by_name["DP4a"], predicate=PRED.hasTestSource, object_type=RT.TestInstrument, id_only=True, access_args=access_args) self.assertEquals(len(rids), 1) self.assertEquals(rids[0], res_by_name["ID4a"]) # - Superuser access_args = create_access_args(current_actor_id=res_by_name["AI3"], superuser_actor_ids=[res_by_name["AI3"]]) rids, _ = self.rr.find_objects(subject=res_by_name["DP4a"], predicate=PRED.hasTestSource, object_type=RT.TestInstrument, id_only=True, access_args=access_args) self.assertEquals(len(rids), 1) self.assertEquals(rids[0], res_by_name["ID4a"]) # - Facility rids, _ = self.rr.find_subjects(subject_type=RT.TestInstrument, predicate=PRED.hasTestDataset, object=res_by_name["DP3a"], id_only=True) self.assertEquals(len(rids), 0) access_args = create_access_args(current_actor_id=res_by_name["AI2"]) rids, _ = self.rr.find_subjects(subject_type=RT.TestInstrument, predicate=PRED.hasTestDataset, object=res_by_name["DP3a"], id_only=True, access_args=access_args) self.assertEquals(len(rids), 1) self.assertEquals(rids[0], res_by_name["ID3a"]) rids, _ = self.rr.find_objects(subject=res_by_name["DP3a"], predicate=PRED.hasTestSource, object_type=RT.TestInstrument, id_only=True) self.assertEquals(len(rids), 0) rids, _ = self.rr.find_objects(subject=res_by_name["DP3a"], predicate=PRED.hasTestSource, object_type=RT.TestInstrument, id_only=True, access_args=access_args) self.assertEquals(len(rids), 1) self.assertEquals(rids[0], res_by_name["ID3a"]) #from pyon.util.breakpoint import breakpoint #breakpoint() self.rr.rr_store.delete_mult(res_by_name.values())
def find_resources(self, restype="", lcstate="", name="", id_only=False): access_args = create_access_args(current_actor_id=get_ion_actor_id(self._process), superuser_actor_ids=self._rr.get_superuser_actors()) return self._rr.find_resources(restype=restype, lcstate=lcstate, name=name, id_only=id_only, access_args=access_args)
def find_subjects_mult(self, objects=None, id_only=False, predicate=""): access_args = create_access_args(current_actor_id=get_ion_actor_id(self._process), superuser_actor_ids=self._rr.get_superuser_actors()) return self._rr.find_subjects_mult(objects=objects, id_only=id_only, predicate=predicate, access_args=access_args)
def find_subjects(self, subject_type="", predicate="", object="", id_only=False, limit=0, skip=0, descending=False): access_args = create_access_args(current_actor_id=get_ion_actor_id(self._process), superuser_actor_ids=self._rr.get_superuser_actors()) return self._rr.find_subjects(subject_type=subject_type, predicate=predicate, object=object, id_only=id_only, limit=limit, skip=skip, descending=descending, access_args=access_args)