def _crawl_related_resources(self, resource_id, resource_types=None, predicate_set=None): """ An internal helper function to generate a unique list of related resources @return: """ resource_types = resource_types if resource_types is not None else [] predicate_set = predicate_set if predicate_set is not None else {} r = RelatedResourcesCrawler() test_real_fn = r.generate_get_related_resources_fn(self.clients.resource_registry, resource_whitelist=resource_types, predicate_dictionary=predicate_set) related_objs = test_real_fn(resource_id) unique_ids = [] for i in related_objs: if i.o not in unique_ids: unique_ids.append(i.o) if i.s not in unique_ids: unique_ids.append(i.s) return unique_ids
def _crawl_related_resources(self, resource_id, resource_types=None, predicate_set=None): """ An internal helper function to generate a unique list of related resources @return: """ resource_types = resource_types if resource_types is not None else [] predicate_set = predicate_set if predicate_set is not None else {} r = RelatedResourcesCrawler() test_real_fn = r.generate_get_related_resources_fn(self.clients.resource_registry, resource_whitelist=resource_types, predicate_dictionary=predicate_set) related_objs = test_real_fn(resource_id) unique_ids = [] for i in related_objs: if i.o not in unique_ids: unique_ids.append(i.o) if i.s not in unique_ids: unique_ids.append(i.s) return unique_ids
def find_related_frames_of_reference(self, input_resource_id='', output_resource_type_list=None): # use the related resources crawler finder = RelatedResourcesCrawler() # generate the partial function (cached association list) get_assns = finder.generate_related_resources_partial(self.RR, [PRED.hasSite]) # run 2 searches allowing all site-based resource types: one down (subj-obj), one up (obj-subj) full_crawllist = [RT.InstrumentSite, RT.PlatformSite, RT.Subsite, RT.Observatory] search_down = get_assns({PRED.hasSite: (True, False)}, full_crawllist) search_up = get_assns({PRED.hasSite: (False, True)}, full_crawllist) # the searches return a list of association objects, so compile all the ids by extracting them retval_ids = set([]) # we want only those IDs that are not the input resource id for a in search_down(input_resource_id, -1) + search_up(input_resource_id, -1): if a.o not in retval_ids and a.o != input_resource_id: retval_ids.add(a.o) if a.s not in retval_ids and a.s != input_resource_id: retval_ids.add(a.s) log.trace("converting retrieved ids to objects = %s" % retval_ids) #initialize the dict retval = dict((restype, []) for restype in output_resource_type_list) #workaround for read_mult problem all_res = [] if retval_ids: all_res = self.RR.read_mult(list(retval_ids)) #all_res = self.RR.read_mult(retval_ids) # put resources in the slot based on their type for resource in all_res: typename = type(resource).__name__ if typename in output_resource_type_list: retval[typename].append(resource) # display a count of how many resources we retrieved log.debug("got these resources: %s", dict([(k, len(v)) for k, v in retval.iteritems()])) return retval
def _get_site_extension_plus(self, site_id='', ext_associations=None, ext_exclude=None, user_id=''): # the "plus" means "plus all sub-site objects" extended_site, RR2 = self._get_site_extension(site_id, ext_associations, ext_exclude, user_id) # use the related resources crawler finder = RelatedResourcesCrawler() get_assns = finder.generate_related_resources_partial(RR2, [PRED.hasSite]) full_crawllist = [RT.InstrumentSite, RT.PlatformSite, RT.Subsite] search_down = get_assns({PRED.hasSite: (True, False)}, full_crawllist) # the searches return a list of association objects, so compile all the ids by extracting them subsite_ids = set([]) # we want only those IDs that are not the input resource id for a in search_down(site_id, -1): if a.o != site_id: subsite_ids.add(a.o) log.trace("converting retrieved ids to objects = %s" % subsite_ids) subsite_objs = RR2.read_mult(list(subsite_ids)) # filtered subsites def fs(resource_type, filter_fn): both = lambda s: ((resource_type == s._get_type()) and filter_fn(s)) return filter(both, subsite_objs) def pfs(filter_fn): return fs(RT.PlatformSite, filter_fn) def ifs(filter_fn): return fs(RT.InstrumentSite, filter_fn) extended_site.computed.platform_station_sites = pfs(lambda s: "StationSite" == s.alt_resource_type) extended_site.computed.platform_component_sites = pfs(lambda s: "PlatformComponentSite" == s.alt_resource_type) extended_site.computed.platform_assembly_sites = pfs(lambda s: "PlatformAssemblySite" == s.alt_resource_type) extended_site.computed.instrument_sites = ifs(lambda _: True) return extended_site, RR2, subsite_objs
def test_related_resource_crawler(self): """ """ self.create_dummy_structure() r = RelatedResourcesCrawler() # test the basic forward-backward searches for (st, p, ot) in self.expected_associations: rst = self.realtype[st] rot = self.realtype[ot] s = self.care[st] o = self.care[ot] test_sto_fn = r.generate_get_related_resources_fn(self.RR, [rot], {p: (True, False)}) sto_crawl = test_sto_fn(s, 1) # depth of 1 if 2 < len(sto_crawl): # we get 2 because of care/dontcare self.fail("got %s" % self.describe_assn_graph(self.simplify_assn_resource_ids(sto_crawl))) self.assertIn(o, [t.o for t in sto_crawl]) test_ots_fn = r.generate_get_related_resources_fn(self.RR, [rst], {p: (False, True)}) ots_crawl = test_ots_fn(o, 1) # depth of 1 if 1 != len(ots_crawl): self.fail("got %s" % self.describe_assn_graph(self.simplify_assn_resource_ids(ots_crawl))) # test a nontrivial lookup, in which we extract resources related to an instrument device rw = [] pd = {} # we want things related to an instrument device rw.append(RT.PlatformModel) rw.append(RT.InstrumentModel) rw.append(RT.PlatformDevice) rw.append(RT.InstrumentSite) rw.append(RT.PlatformSite) rw.append(RT.Subsite) rw.append(RT.Observatory) rw.append(RT.InstrumentDevice) pd[PRED.hasModel] = (True, True) pd[PRED.hasDevice] = (False, True) pd[PRED.hasSite] = (False, True) test_real_fn = r.generate_get_related_resources_fn(self.RR, resource_whitelist=rw, predicate_dictionary=pd) related = test_real_fn(self.care[RT.InstrumentDevice]) log.debug("========= Result is:") for l in self.describe_assn_graph(self.simplify_assn_resource_ids(related)): log.debug(" %s", l) # check that we only got things we care about for a in related: # special case for platform model, because we don't care about the top-level platform's model # so it will blow up if we don't ignore it. if we got an extra platform model, we'd have an # extra platform anyway... so this special case is safe. if a.st != RT.PlatformModel: self.assertIn(a.s, self.care.values(), "%s %s not cared about" % (a.st, a.s)) if a.ot != RT.PlatformModel: self.assertIn(a.o, self.care.values(), "%s %s not cared about" % (a.ot, a.o))
def test_related_resource_crawler(self): """ """ self.create_dummy_structure() r = RelatedResourcesCrawler() # test the basic forward-backward searches for (st, p, ot) in self.expected_associations: rst = self.realtype[st] rot = self.realtype[ot] s = self.care[st] o = self.care[ot] test_sto_fn = r.generate_get_related_resources_fn( self.RR, [rot], {p: (True, False)}) sto_crawl = test_sto_fn(s, 1) # depth of 1 if 2 < len(sto_crawl): # we get 2 because of care/dontcare self.fail("got %s" % self.describe_assn_graph( self.simplify_assn_resource_ids(sto_crawl))) self.assertIn(o, [t.o for t in sto_crawl]) test_ots_fn = r.generate_get_related_resources_fn( self.RR, [rst], {p: (False, True)}) ots_crawl = test_ots_fn(o, 1) # depth of 1 if 1 != len(ots_crawl): self.fail("got %s" % self.describe_assn_graph( self.simplify_assn_resource_ids(ots_crawl))) # test a nontrivial lookup, in which we extract resources related to an instrument device rw = [] pd = {} # we want things related to an instrument device rw.append(RT.PlatformModel) rw.append(RT.InstrumentModel) rw.append(RT.PlatformDevice) rw.append(RT.InstrumentSite) rw.append(RT.PlatformSite) rw.append(RT.Subsite) rw.append(RT.Observatory) rw.append(RT.InstrumentDevice) pd[PRED.hasModel] = (True, True) pd[PRED.hasDevice] = (False, True) pd[PRED.hasSite] = (False, True) test_real_fn = r.generate_get_related_resources_fn( self.RR, resource_whitelist=rw, predicate_dictionary=pd) related = test_real_fn(self.care[RT.InstrumentDevice]) log.debug("========= Result is:") for l in self.describe_assn_graph( self.simplify_assn_resource_ids(related)): log.debug(" %s", l) # check that we only got things we care about for a in related: # special case for platform model, because we don't care about the top-level platform's model # so it will blow up if we don't ignore it. if we got an extra platform model, we'd have an # extra platform anyway... so this special case is safe. if a.st != RT.PlatformModel: self.assertIn(a.s, self.care.values(), "%s %s not cared about" % (a.st, a.s)) if a.ot != RT.PlatformModel: self.assertIn(a.o, self.care.values(), "%s %s not cared about" % (a.ot, a.o))