def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.org_management_service = OrgManagementServiceClient(
node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#print 'TestObservatoryManagementServiceIntegration: started services'
self.event_publisher = EventPublisher()
def __init__(self, clients):
self.clients = clients
self.RR2 = EnhancedResourceRegistryClient(self.clients.resource_registry)
self.agent_instance_obj = None
self.associated_objects = None
self.last_id = None
self.will_launch = False
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.client = DataProductManagementServiceClient(
node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
def __init__(self,
clients,
deployment_obj,
allow_children,
include_children,
RR2=None):
"""
@param clients dict of clients from a service
@param deployment_obj the deployment to activate
@param allow_children whether to allow children of a device to be provided
@param include_children whether to search for child devices from root device
@param RR2 a reference to an enhanced RR client
"""
self.clients = clients
self.RR2 = RR2
#sanity
if include_children: assert allow_children
self.allow_children = allow_children
self.include_children = include_children
if None is self.RR2:
self.RR2 = EnhancedResourceRegistryClient(
self.clients.resource_registry)
if not isinstance(self.RR2, EnhancedResourceRegistryClient):
raise AssertionError(
"Type of self.RR2 is %s not %s" %
(type(self.RR2), type(EnhancedResourceRegistryClient)))
self.deployment_obj = deployment_obj
self.on_init()
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dmpsclient = DataProductManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.psmsclient = PubsubManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.c = DotDict()
self.c.resource_registry = self.rrclient
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
self.dsmsclient = DataProcessManagementServiceClient(node=self.container.node)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dsmsclient.deactivate_data_process(proc_id)
self.dsmsclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def __init__(self, clients=None, enhanced_rr=None):
self.clients = clients
self.enhanced_rr = enhanced_rr
if not enhanced_rr:
self.enhanced_rr = EnhancedResourceRegistryClient(self.clients.resource_registry)
self.outil = ObservatoryUtil(self, enhanced_rr=self.enhanced_rr)
def setUp(self):
# Start container by calling parent's setUp
super(TestAssembly, self).setUp()
# Now create client to DataProductManagementService
self.client = DotDict()
self.client.DAMS = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.client.DPMS = DataProductManagementServiceClient(
node=self.container.node)
self.client.IMS = InstrumentManagementServiceClient(
node=self.container.node)
self.client.OMS = ObservatoryManagementServiceClient(
node=self.container.node)
self.client.PSMS = PubsubManagementServiceClient(
node=self.container.node)
self.client.DPRS = DataProcessManagementServiceClient(
node=self.container.node)
self.client.RR = ResourceRegistryServiceClient(
node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.client.RR)
self.dataset_management = DatasetManagementServiceClient()
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.client.RR.find_resources(
RT.DataProcess, None, None, True)[0]:
self.client.DPRS.deactivate_data_process(proc_id)
self.client.DPRS.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def __init__(self, process=None):
"""
the process should be the "self" of a service instance
"""
assert process
self.process = process
self.RR2 = EnhancedResourceRegistryClient(
self.process.clients.resource_registry)
def setUp(self):
self._start_container()
self._pp = pprint.PrettyPrinter()
log.debug("oms_uri = %s", OMS_URI)
self.oms = CIOMSClientFactory.create_instance(OMS_URI)
self._get_platform_attributes()
url = OmsTestMixin.start_http_server()
log.info("TestPlatformInstrument:setup http url %s", url)
result = self.oms.event.register_event_listener(url)
log.info(
"TestPlatformInstrument:setup register_event_listener result %s",
result)
# response = self.oms.port.get_platform_ports('LPJBox_CI_Ben_Hall')
# log.info("TestPlatformInstrument:setup get_platform_ports %s", response)
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
self.org_id = self.RR2.create(any_old(RT.Org))
log.debug("Org created: %s", self.org_id)
# see _set_receive_timeout
self._receive_timeout = 177
self.instrument_device = ''
self.platform_device = ''
self.platform_agent_instance_id = ''
self._pa_client = ''
def done():
CIOMSClientFactory.destroy_instance(self.oms)
event_notifications = OmsTestMixin.stop_http_server()
log.info("event_notifications = %s" % str(event_notifications))
self.addCleanup(done)
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.generic_int_helper_rr = ResourceRegistryServiceClient(
node=self.container.node)
self.generic_int_helper_rr2 = EnhancedResourceRegistryClient(
self.generic_int_helper_rr)
def setUp(self):
self.clients = DotDict()
self.rr = Mock()
self.RR2 = EnhancedResourceRegistryClient(self.rr)
self.clients.resource_registry = self.rr
self.policy = ResourceLCSPolicy(self.clients)
def override_clients(self, new_clients):
"""
Replaces the service clients with a new set of them... and makes sure they go to the right places
"""
self.RR2 = EnhancedResourceRegistryClient(
self.clients.resource_registry)
#shortcut names for the import sub-services
if hasattr(self.clients, "resource_registry"):
self.RR = self.clients.resource_registry
def __init__(self, process=None):
"""
the process should be the "self" of a service instance
"""
assert process
self.process = process
self.dtm = DriverTypingMethod.ByRR
self.RR2 = None
# make an internal pointer to this function so we can Mock it for testing
self._get_agent_client = ResourceAgentClient
if DriverTypingMethod.ByRR == self.dtm:
self.RR2 = EnhancedResourceRegistryClient(process.clients.resource_registry)
def _start_raw_ingestion(self):
dpsc_cli = DataProductManagementServiceClient()
rrclient = ResourceRegistryServiceClient()
RR2 = EnhancedResourceRegistryClient(rrclient)
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
log.info("Create data product... raw stream id: %s",
self._raw_stream_id)
dp_id = dpsc_cli.create_data_product_(data_product=dp_obj)
dataset_id = self.create_dataset(self._raw_stream_pdict_id)
RR2.assign_stream_definition_to_data_product_with_has_stream_definition(
self._raw_stream_def_id, dp_id)
RR2.assign_stream_to_data_product_with_has_stream(
self._raw_stream_id, dp_id)
RR2.assign_dataset_to_data_product_with_has_dataset(dataset_id, dp_id)
self._raw_dataset_id = dataset_id
log.info("Create data product...Complete")
# Assert that the data product has an associated stream at this stage
stream_ids, _ = rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream,
True)
self.assertNotEquals(len(stream_ids), 0)
# Assert that the data product has an associated stream def at this stage
stream_ids, _ = rrclient.find_objects(dp_id, PRED.hasStreamDefinition,
RT.StreamDefinition, True)
self.assertNotEquals(len(stream_ids), 0)
log.info("Activate data product persistence")
dpsc_cli.activate_data_product_persistence(dp_id)
log.info("Read data product")
dp_obj = dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Created data product %s', dp_obj)
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
log.debug("Start rel from url")
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.DPMS = DataProductManagementServiceClient()
self.RR = ResourceRegistryServiceClient()
self.RR2 = EnhancedResourceRegistryClient(self.RR)
self.DAMS = DataAcquisitionManagementServiceClient()
self.PSMS = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.PD = ProcessDispatcherServiceClient()
self.DSMS = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
log.debug("get datastore")
datastore_name = CACHE_DATASTORE_NAME
self.db = self.container.datastore_manager.get_datastore(
datastore_name)
self.stream_def_id = self.PSMS.create_stream_definition(
name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(
name='ingestion worker')
ingestion_worker_definition.executable = {
'module':
'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class': 'ScienceGranuleIngestionWorker'
}
process_definition_id = self.PD.create_process_definition(
process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
self.addCleanup(self.cleaning_up)
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
self._setup_statuses()
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.resource_registry = self.container.resource_registry
self.RR2 = EnhancedResourceRegistryClient(self.resource_registry)
self.data_acquisition_management = DataAcquisitionManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.instrument_management = InstrumentManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.user_notification = UserNotificationServiceClient()
self.workflow_management = WorkflowManagementServiceClient()
self.visualization = VisualizationServiceClient()
def __init__(self, clients, RR2=None):
self.clients = clients
self.RR2 = RR2
if self.RR2 is None:
log.warn("Creating new RR2")
self.RR2 = EnhancedResourceRegistryClient(
self.clients.resource_registry)
if not isinstance(self.RR2, EnhancedResourceRegistryClient):
raise AssertionError(
"Type of self.RR2 is %s not %s" %
(type(self.RR2), type(EnhancedResourceRegistryClient)))
self.agent_instance_obj = None
self.associated_objects = None
self.last_id = None
self.will_launch = False
self.generated_config = False
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient()
self.RR2 = EnhancedResourceRegistryClient(self.RR)
self.OMS = ObservatoryManagementServiceClient()
self.org_management_service = OrgManagementServiceClient()
self.IMS = InstrumentManagementServiceClient()
self.dpclient = DataProductManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self._load_stage = 0
self._resources = {}
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.resource_registry = self.container.resource_registry
self.RR2 = EnhancedResourceRegistryClient(self.resource_registry)
self.data_acquisition_management = DataAcquisitionManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.instrument_management = InstrumentManagementServiceClient()
self.discovery = DiscoveryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.user_notification = UserNotificationServiceClient()
self.observatory_management = ObservatoryManagementServiceClient()
self.visualization = VisualizationServiceClient()
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
self.ctd_count = 0
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dmpsclient = DataProductManagementServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.psmsclient = PubsubManagementServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.c = DotDict()
self.c.resource_registry = self.rrclient
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
# create missing data process definition
self.dsmsclient = DataProcessManagementServiceClient(
node=self.container.node)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name=LOGICAL_TRANSFORM_DEFINITION_NAME,
description="normally in preload",
module='ion.processes.data.transforms.logical_transform',
class_name='logical_transform')
self.dsmsclient.create_data_process_definition(dpd_obj)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(
RT.DataProcess, None, None, True)[0]:
self.dsmsclient.deactivate_data_process(proc_id)
self.dsmsclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def setUp(self):
# Start container by calling parent's setUp
super(TestAssembly, self).setUp()
# Now create client to DataProductManagementService
self.client = DotDict()
self.client.DAMS = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.client.DPMS = DataProductManagementServiceClient(
node=self.container.node)
self.client.IMS = InstrumentManagementServiceClient(
node=self.container.node)
self.client.OMS = ObservatoryManagementServiceClient(
node=self.container.node)
self.client.PSMS = PubsubManagementServiceClient(
node=self.container.node)
self.client.DPRS = DataProcessManagementServiceClient(
node=self.container.node)
self.client.RR = ResourceRegistryServiceClient(
node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.client.RR)
self.dataset_management = DatasetManagementServiceClient()
dpd_obj = IonObject(
RT.DataProcessDefinition,
name=LOGICAL_TRANSFORM_DEFINITION_NAME,
description="normally in preload",
module='ion.processes.data.transforms.logical_transform',
class_name='logical_transform')
self.client.DPRS.create_data_process_definition(dpd_obj)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.client.RR.find_resources(
RT.DataProcess, None, None, True)[0]:
self.client.DPRS.deactivate_data_process(proc_id)
self.client.DPRS.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
unittest # suppress an pycharm inspector error if all unittest.skip references are commented out
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
print 'started services'
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient()
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
# get serialized version for the configuration:
self._network_definition_ser = NetworkUtil.serialize_network_definition(
self._network_definition)
if log.isEnabledFor(logging.TRACE):
log.trace("NetworkDefinition serialization:\n%s",
self._network_definition_ser)
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
def setUp(self):
self.rr = Mock()
self.RR2 = EnhancedResourceRegistryClient(self.rr)
def freeze():
if isinstance(resource_registry_client,
EnhancedResourceRegistryClient):
RR2 = resource_registry_client
else:
RR2 = EnhancedResourceRegistryClient(resource_registry_client)
for p in predicate_list:
if not RR2.has_cached_predicate(p):
RR2.cache_predicate(p)
def get_related_resources_partial_fn(predicate_dictionary,
resource_whitelist):
"""
This function generates a resource crawler from 2 data structures representing desired crawl behavior
The predicate dictionary is keyed on a predicate type, whose value is a 2-tuple of booleans
the first boolean is whether to crawl subject-object, the second boolean for object-subject
For example: dict([(PRED.hasModel, (False, True)]) would generate a crawler that could find
platforms or instruments with a given model
The resource whitelist is a list of resource types that will be crawled.
The return value of this function is a function that accepts a resource id and returns a list
of associations related (based on crawl behavior)
"""
log.trace(
"get_related_resources_partial_fn predicate_dict=%s rsrc_whitelist=%s",
predicate_dictionary, resource_whitelist)
# assertions on data types
assert type({}) == type(predicate_dictionary)
for v in predicate_dictionary.values():
assert type((True, True)) == type(v)
assert type([]) == type(resource_whitelist)
for rt in resource_whitelist:
RR2.cache_resources(rt)
def lookup_fn(resource_id):
"""
return a dict of related resources as dictated by the pred dict and whitelist
- the key is the next resource id to crawl
- the value is the entire association
"""
retval = {}
sto_match = lambda assn: assn.s == resource_id and assn.ot in resource_whitelist
ots_match = lambda assn: assn.o == resource_id and assn.st in resource_whitelist
for p, (search_sto,
search_ots) in predicate_dictionary.iteritems():
if search_sto:
for a in RR2.filter_cached_associations(
p, sto_match):
log.trace("lookup_fn matched %s object", a.ot)
retval[a.o] = a
if search_ots:
for a in RR2.filter_cached_associations(
p, ots_match):
log.trace("lookup_fn matched %s subject", a.st)
retval[a.s] = a
return retval
def get_related_resources_h(accum, input_resource_id,
recursion_limit):
"""
This is a recursive helper function that does the work of crawling for related resources
The accum is a tuple: (set of associations that are deemed "Related", set of "seen" resources)
The input resource id is the current resource being crawled
The recursion limit decrements with each recursive call, ending at 0. So -1 for infinity.
The return value is a list of associations
"""
if 0 == recursion_limit:
return accum
if -1000 > recursion_limit:
log.warn(
"Terminating related resource recursion, hit -1000"
)
return accum
acc, seen = accum
matches = lookup_fn(input_resource_id)
log.trace("get_related_resources_h got matches %s", [
dict((k, "%s %s %s" % (a.st, a.p, a.ot))
for k, a in matches.iteritems())
])
unseen = set(matches.keys()) - seen
seen.add(input_resource_id)
acc = acc | set(matches.values())
#if log.isEnabledFor(logging.TRACE):
# summary = {}
# for a in acc:
# label = "%s %s %s" % (a.st, a.p, a.ot)
# if not label in summary: summary[label] = 0
# summary[label] += 1
# log.trace("acc2 is now %s", ["%s x%d" % (k, v) for k, v in summary.iteritems()])
def looper(acc2, input_rsrc_id):
return get_related_resources_h(acc2, input_rsrc_id,
recursion_limit - 1)
h_ret = reduce(looper, unseen, (acc, seen))
#h_ret = reduce(looper, unseen, (acc, seen))
#(h_ret_acc, h_ret_seen) = h_ret
#log.trace("h_ret is %s", ["%s %s %s" % (a.st, a.p, a.ot) for a in h_ret_acc])
return h_ret
def get_related_resources_fn(input_resource_id,
recursion_limit=1024):
"""
This is the function that finds related resources.
input_resource_id and recursion_limit are self explanatory
The return value is a list of associations.
"""
retval, _ = get_related_resources_h(
(set([]), set([])), input_resource_id, recursion_limit)
log.trace("final_ret is %s",
["%s %s %s" % (a.st, a.p, a.ot) for a in retval])
return list(retval)
return get_related_resources_fn # retval of get_related_resources_partial_fn
return get_related_resources_partial_fn # retval of freeze()