def _do_test_find_objects_mult(self):
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.resource_registry_service.create(dp)
transform_id, _ = self.resource_registry_service.create(transform)
pd_id, _ = self.resource_registry_service.create(pd)
self.resource_registry_service.create_association(
subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.resource_registry_service.create_association(
subject=transform_id,
object=pd_id,
predicate=PRED.hasProcessDefinition)
results, _ = self.resource_registry_service.find_objects_mult(
subjects=[dp_id], id_only=True)
self.assertTrue(results == [transform_id])
results, _ = self.resource_registry_service.find_objects_mult(
subjects=[dp_id, transform_id], id_only=True)
results.sort()
correct = [transform_id, pd_id]
correct.sort()
self.assertTrue(results == correct)
def test_iterative_associative_searching(self):
#--------------------------------------------------------------------------------
# Tests the ability to limit the iterations
#--------------------------------------------------------------------------------
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
search_string = "belongs to '%s' depth 1" % dp_id
results = self.poll(5, self.discovery.parse,search_string)
results = list([i._id for i in results])
correct = [transform_id]
self.assertTrue(results == correct, '%s' % results)
search_string = "belongs to '%s' depth 2" % dp_id
results = self.poll(5, self.discovery.parse,search_string)
results = list([i._id for i in results])
results.sort()
correct = [transform_id, pd_id]
correct.sort()
self.assertTrue(results == correct)
def create_data_process2(self, data_process_definition_id='', in_data_product_ids=None, out_data_product_ids=None, configuration=None):
'''
Creates a DataProcess resource and launches the process.
A DataProcess is a process that receives one (or more) data products and produces one (or more) data products.
@param data_process_definition_id : The Data Process Definition to use, if none is specified the standard TransformDataProcess is used
@param in_data_product_ids : A list of input data product identifiers
@param out_data_product_ids : A list of output data product identifiers
@param configuration : The configuration dictionary for the process, and the routing table:
The routing table is defined as such:
{ (in_data_product_id, out_data_product_id) : actor }
Routes are specified in the configuration dictionary under the item "routes"
actor is either None (for ParameterFunctions) or a valid TransformFunction identifier
'''
configuration = configuration or DotDict()
routes = configuration.get_safe('process.routes', {})
self.validate_compatibility(in_data_product_ids, out_data_product_ids)
configuration.process.routes = self._manage_routes(routes)
dproc = DataProcess()
dproc.name = 'data_process_%s' % self.get_unique_id()
dproc.configuration = configuration
dproc_id, rev = self.clients.resource_registry.create(dproc)
dproc._id = dproc_id
dproc._rev = rev
self._manage_producers(dproc_id, out_data_product_ids)
self._manage_attachments()
queue_name = self._create_subscription(dproc, in_data_product_ids)
pid = self._launch_data_process(
queue_name=queue_name,
data_process_definition_id=data_process_definition_id,
out_data_product_ids=out_data_product_ids,
configuration=configuration)
self.clients.resource_registry.create_association(subject=dproc_id, predicate=PRED.hasProcess, object=pid)
return dproc_id
def test_events_search(self):
# Create a resource to force a new event
dp = DataProcess()
dp_id, rev = self.rr.create(dp)
search_string = "SEARCH 'origin' IS '%s' FROM 'events_index'" % dp_id
results = self.poll(9, self.discovery.parse,search_string)
origin_type = results[0]['_source'].origin_type
origin_id = results[0]['_source'].origin
self.assertTrue(origin_type == RT.DataProcess)
self.assertTrue(origin_id == dp_id)
def test_traversal(self):
dp = DataProcess()
transform = Transform()
pd = ProcessDefinition()
dp_id, _ = self.rr.create(dp)
transform_id, _ = self.rr.create(transform)
pd_id, _ = self.rr.create(pd)
self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)
results = self.discovery.traverse(dp_id)
results.sort()
correct = [pd_id, transform_id]
correct.sort()
self.assertTrue(results == correct, '%s' % results)
def create_data_process(self, data_process_definition_id='', in_data_product_ids=None, out_data_product_ids=None, configuration=None):
'''
Creates a DataProcess resource and launches the process.
A DataProcess is a process that receives one (or more) data products and produces one (or more) data products.
@param data_process_definition_id : The Data Process Definition to use, if none is specified the standard TransformDataProcess is used
@param in_data_product_ids : A list of input data product identifiers
@param out_data_product_ids : A list of output data product identifiers
@param configuration : The configuration dictionary for the process, and the routing table:
The routing table is defined as such:
{ in_data_product_id: {out_data_product_id : actor }}
Routes are specified in the configuration dictionary under the item "routes"
actor is either None (for ParameterFunctions) or a valid TransformFunction identifier
'''
configuration = DotDict(configuration or {})
in_data_product_ids = in_data_product_ids or []
out_data_product_ids = out_data_product_ids or []
routes = configuration.get_safe('process.routes', {})
if not routes and (1==len(in_data_product_ids)==len(out_data_product_ids)):
routes = {in_data_product_ids[0]: {out_data_product_ids[0]:None}}
# Routes are not supported for processes with discrete data process definitions
elif not routes and not data_process_definition_id:
raise BadRequest('No valid route defined for this data process.')
self.validate_compatibility(data_process_definition_id, in_data_product_ids, out_data_product_ids, routes)
routes = self._manage_routes(routes)
configuration.process.input_products = in_data_product_ids
configuration.process.output_products = out_data_product_ids
configuration.process.routes = routes
if 'lookup_docs' in configuration.process:
configuration.process.lookup_docs.extend(self._get_lookup_docs(in_data_product_ids, out_data_product_ids))
else:
configuration.process.lookup_docs = self._get_lookup_docs(in_data_product_ids, out_data_product_ids)
dproc = DataProcess()
dproc.name = 'data_process_%s' % self.get_unique_id()
dproc.configuration = configuration
dproc_id, rev = self.clients.resource_registry.create(dproc)
dproc._id = dproc_id
dproc._rev = rev
for data_product_id in in_data_product_ids:
self.clients.resource_registry.create_association(subject=dproc_id, predicate=PRED.hasInputProduct, object=data_product_id)
if data_process_definition_id:
self.clients.resource_registry.create_association(data_process_definition_id, PRED.hasDataProcess ,dproc_id)
self._manage_producers(dproc_id, out_data_product_ids)
self._manage_attachments()
queue_name = self._create_subscription(dproc, in_data_product_ids)
pid = self._launch_data_process(
queue_name=queue_name,
data_process_definition_id=data_process_definition_id,
out_data_product_ids=out_data_product_ids,
configuration=configuration)
self.clients.resource_registry.create_association(subject=dproc_id, predicate=PRED.hasProcess, object=pid)
return dproc_id
def test_get_data_product_provenance_report(self):
#Create a test device
device_obj = Device(name='Device1', description='test instrument site')
device_id, _ = self.rrclient.create(device_obj)
self.addCleanup(self.rrclient.delete, device_id)
#Create a test DataProduct
data_product1_obj = DataProduct(name='DataProduct1',
description='test data product 1')
data_product1_id, _ = self.rrclient.create(data_product1_obj)
self.addCleanup(self.rrclient.delete, data_product1_id)
#Create a test DataProcess
data_process_obj = DataProcess(name='DataProcess',
description='test data process')
data_process_id, _ = self.rrclient.create(data_process_obj)
self.addCleanup(self.rrclient.delete, data_process_id)
#Create a second test DataProduct
data_product2_obj = DataProduct(name='DataProduct2',
description='test data product 2')
data_product2_id, _ = self.rrclient.create(data_product2_obj)
self.addCleanup(self.rrclient.delete, data_product2_id)
#Create a test DataProducer
data_producer_obj = DataProducer(name='DataProducer',
description='test data producer')
data_producer_id, rev = self.rrclient.create(data_producer_obj)
#Link the DataProcess to the second DataProduct manually
assoc_id, _ = self.rrclient.create_association(
subject=data_process_id,
predicate=PRED.hasInputProduct,
object=data_product2_id)
self.addCleanup(self.rrclient.delete_association, assoc_id)
# Register the instrument and process. This links the device and the data process
# with their own producers
self.damsclient.register_instrument(device_id)
self.addCleanup(self.damsclient.unregister_instrument, device_id)
self.damsclient.register_process(data_process_id)
self.addCleanup(self.damsclient.unregister_process, data_process_id)
#Manually link the first DataProduct with the test DataProducer
assoc_id, _ = self.rrclient.create_association(
subject=data_product1_id,
predicate=PRED.hasDataProducer,
object=data_producer_id)
#Get the DataProducer linked to the DataProcess (created in register_process above)
#Associate that with with DataProduct1's DataProducer
data_process_producer_ids, _ = self.rrclient.find_objects(
subject=data_process_id,
predicate=PRED.hasDataProducer,
object_type=RT.DataProducer,
id_only=True)
assoc_id, _ = self.rrclient.create_association(
subject=data_process_producer_ids[0],
predicate=PRED.hasParent,
object=data_producer_id)
self.addCleanup(self.rrclient.delete_association, assoc_id)
#Get the DataProducer linked to the Device (created in register_instrument
#Associate that with the DataProcess's DataProducer
device_producer_ids, _ = self.rrclient.find_objects(
subject=device_id,
predicate=PRED.hasDataProducer,
object_type=RT.DataProducer,
id_only=True)
assoc_id, _ = self.rrclient.create_association(
subject=data_producer_id,
predicate=PRED.hasParent,
object=device_producer_ids[0])
#Create the links between the Device, DataProducts, DataProcess, and all DataProducers
self.damsclient.assign_data_product(input_resource_id=device_id,
data_product_id=data_product1_id)
self.addCleanup(self.damsclient.unassign_data_product, device_id,
data_product1_id)
self.damsclient.assign_data_product(input_resource_id=data_process_id,
data_product_id=data_product2_id)
self.addCleanup(self.damsclient.unassign_data_product, data_process_id,
data_product2_id)
#Traverse through the relationships to get the links between objects
res = self.dpmsclient.get_data_product_provenance_report(
data_product2_id)
#Make sure there are four keys
self.assertEqual(len(res.keys()), 4)
parent_count = 0
config_count = 0
for v in res.itervalues():
if 'parent' in v:
parent_count += 1
if 'config' in v:
config_count += 1
#Make sure there are three parents and four configs
self.assertEqual(parent_count, 3)
self.assertEqual(config_count, 4)
def create_data_process(self,
data_process_definition_id='',
in_data_product_ids=None,
out_data_product_ids=None,
configuration=None):
'''
Creates a DataProcess resource and launches the process.
A DataProcess is a process that receives one (or more) data products and produces one (or more) data products.
@param data_process_definition_id : The Data Process Definition to use, if none is specified the standard TransformDataProcess is used
@param in_data_product_ids : A list of input data product identifiers
@param out_data_product_ids : A list of output data product identifiers
@param configuration : The configuration dictionary for the process, and the routing table:
The routing table is defined as such:
{ in_data_product_id: {out_data_product_id : actor }}
Routes are specified in the configuration dictionary under the item "routes"
actor is either None (for ParameterFunctions) or a valid TransformFunction identifier
'''
configuration = DotDict(configuration or {})
in_data_product_ids = in_data_product_ids or []
out_data_product_ids = out_data_product_ids or []
routes = configuration.get_safe('process.routes', {})
if not routes and (1 == len(in_data_product_ids) ==
len(out_data_product_ids)):
routes = {in_data_product_ids[0]: {out_data_product_ids[0]: None}}
# Routes are not supported for processes with discrete data process definitions
elif not routes and not data_process_definition_id:
raise BadRequest('No valid route defined for this data process.')
self.validate_compatibility(data_process_definition_id,
in_data_product_ids, out_data_product_ids,
routes)
routes = self._manage_routes(routes)
configuration.process.input_products = in_data_product_ids
configuration.process.output_products = out_data_product_ids
configuration.process.routes = routes
if 'lookup_docs' in configuration.process:
configuration.process.lookup_docs.extend(
self._get_lookup_docs(in_data_product_ids,
out_data_product_ids))
else:
configuration.process.lookup_docs = self._get_lookup_docs(
in_data_product_ids, out_data_product_ids)
dproc = DataProcess()
dproc.name = 'data_process_%s' % self.get_unique_id()
dproc.configuration = configuration
dproc_id, rev = self.clients.resource_registry.create(dproc)
dproc._id = dproc_id
dproc._rev = rev
for data_product_id in in_data_product_ids:
self.clients.resource_registry.create_association(
subject=dproc_id,
predicate=PRED.hasInputProduct,
object=data_product_id)
if data_process_definition_id:
self.clients.resource_registry.create_association(
data_process_definition_id, PRED.hasDataProcess, dproc_id)
self._manage_producers(dproc_id, out_data_product_ids)
self._manage_attachments()
queue_name = self._create_subscription(dproc, in_data_product_ids)
pid = self._launch_data_process(
queue_name=queue_name,
data_process_definition_id=data_process_definition_id,
out_data_product_ids=out_data_product_ids,
configuration=configuration)
self.clients.resource_registry.create_association(
subject=dproc_id, predicate=PRED.hasProcess, object=pid)
return dproc_id