def create_data_process2(self, data_process_definition_id='', in_data_product_ids=None, out_data_product_ids=None, configuration=None):
'''
Creates a DataProcess resource and launches the process.
A DataProcess is a process that receives one (or more) data products and produces one (or more) data products.
@param data_process_definition_id : The Data Process Definition to use, if none is specified the standard TransformDataProcess is used
@param in_data_product_ids : A list of input data product identifiers
@param out_data_product_ids : A list of output data product identifiers
@param configuration : The configuration dictionary for the process, and the routing table:
The routing table is defined as such:
{ (in_data_product_id, out_data_product_id) : actor }
Routes are specified in the configuration dictionary under the item "routes"
actor is either None (for ParameterFunctions) or a valid TransformFunction identifier
'''
configuration = configuration or DotDict()
routes = configuration.get_safe('process.routes', {})
self.validate_compatibility(in_data_product_ids, out_data_product_ids)
configuration.process.routes = self._manage_routes(routes)
dproc = DataProcess()
dproc.name = 'data_process_%s' % self.get_unique_id()
dproc.configuration = configuration
dproc_id, rev = self.clients.resource_registry.create(dproc)
dproc._id = dproc_id
dproc._rev = rev
self._manage_producers(dproc_id, out_data_product_ids)
self._manage_attachments()
queue_name = self._create_subscription(dproc, in_data_product_ids)
pid = self._launch_data_process(
queue_name=queue_name,
data_process_definition_id=data_process_definition_id,
out_data_product_ids=out_data_product_ids,
configuration=configuration)
self.clients.resource_registry.create_association(subject=dproc_id, predicate=PRED.hasProcess, object=pid)
return dproc_id
def create_data_process(self, data_process_definition_id='', in_data_product_ids=None, out_data_product_ids=None, configuration=None):
'''
Creates a DataProcess resource and launches the process.
A DataProcess is a process that receives one (or more) data products and produces one (or more) data products.
@param data_process_definition_id : The Data Process Definition to use, if none is specified the standard TransformDataProcess is used
@param in_data_product_ids : A list of input data product identifiers
@param out_data_product_ids : A list of output data product identifiers
@param configuration : The configuration dictionary for the process, and the routing table:
The routing table is defined as such:
{ in_data_product_id: {out_data_product_id : actor }}
Routes are specified in the configuration dictionary under the item "routes"
actor is either None (for ParameterFunctions) or a valid TransformFunction identifier
'''
configuration = DotDict(configuration or {})
in_data_product_ids = in_data_product_ids or []
out_data_product_ids = out_data_product_ids or []
routes = configuration.get_safe('process.routes', {})
if not routes and (1==len(in_data_product_ids)==len(out_data_product_ids)):
routes = {in_data_product_ids[0]: {out_data_product_ids[0]:None}}
# Routes are not supported for processes with discrete data process definitions
elif not routes and not data_process_definition_id:
raise BadRequest('No valid route defined for this data process.')
self.validate_compatibility(data_process_definition_id, in_data_product_ids, out_data_product_ids, routes)
routes = self._manage_routes(routes)
configuration.process.input_products = in_data_product_ids
configuration.process.output_products = out_data_product_ids
configuration.process.routes = routes
if 'lookup_docs' in configuration.process:
configuration.process.lookup_docs.extend(self._get_lookup_docs(in_data_product_ids, out_data_product_ids))
else:
configuration.process.lookup_docs = self._get_lookup_docs(in_data_product_ids, out_data_product_ids)
dproc = DataProcess()
dproc.name = 'data_process_%s' % self.get_unique_id()
dproc.configuration = configuration
dproc_id, rev = self.clients.resource_registry.create(dproc)
dproc._id = dproc_id
dproc._rev = rev
for data_product_id in in_data_product_ids:
self.clients.resource_registry.create_association(subject=dproc_id, predicate=PRED.hasInputProduct, object=data_product_id)
if data_process_definition_id:
self.clients.resource_registry.create_association(data_process_definition_id, PRED.hasDataProcess ,dproc_id)
self._manage_producers(dproc_id, out_data_product_ids)
self._manage_attachments()
queue_name = self._create_subscription(dproc, in_data_product_ids)
pid = self._launch_data_process(
queue_name=queue_name,
data_process_definition_id=data_process_definition_id,
out_data_product_ids=out_data_product_ids,
configuration=configuration)
self.clients.resource_registry.create_association(subject=dproc_id, predicate=PRED.hasProcess, object=pid)
return dproc_id
def create_data_process(self,
data_process_definition_id='',
in_data_product_ids=None,
out_data_product_ids=None,
configuration=None):
'''
Creates a DataProcess resource and launches the process.
A DataProcess is a process that receives one (or more) data products and produces one (or more) data products.
@param data_process_definition_id : The Data Process Definition to use, if none is specified the standard TransformDataProcess is used
@param in_data_product_ids : A list of input data product identifiers
@param out_data_product_ids : A list of output data product identifiers
@param configuration : The configuration dictionary for the process, and the routing table:
The routing table is defined as such:
{ in_data_product_id: {out_data_product_id : actor }}
Routes are specified in the configuration dictionary under the item "routes"
actor is either None (for ParameterFunctions) or a valid TransformFunction identifier
'''
configuration = DotDict(configuration or {})
in_data_product_ids = in_data_product_ids or []
out_data_product_ids = out_data_product_ids or []
routes = configuration.get_safe('process.routes', {})
if not routes and (1 == len(in_data_product_ids) ==
len(out_data_product_ids)):
routes = {in_data_product_ids[0]: {out_data_product_ids[0]: None}}
# Routes are not supported for processes with discrete data process definitions
elif not routes and not data_process_definition_id:
raise BadRequest('No valid route defined for this data process.')
self.validate_compatibility(data_process_definition_id,
in_data_product_ids, out_data_product_ids,
routes)
routes = self._manage_routes(routes)
configuration.process.input_products = in_data_product_ids
configuration.process.output_products = out_data_product_ids
configuration.process.routes = routes
if 'lookup_docs' in configuration.process:
configuration.process.lookup_docs.extend(
self._get_lookup_docs(in_data_product_ids,
out_data_product_ids))
else:
configuration.process.lookup_docs = self._get_lookup_docs(
in_data_product_ids, out_data_product_ids)
dproc = DataProcess()
dproc.name = 'data_process_%s' % self.get_unique_id()
dproc.configuration = configuration
dproc_id, rev = self.clients.resource_registry.create(dproc)
dproc._id = dproc_id
dproc._rev = rev
for data_product_id in in_data_product_ids:
self.clients.resource_registry.create_association(
subject=dproc_id,
predicate=PRED.hasInputProduct,
object=data_product_id)
if data_process_definition_id:
self.clients.resource_registry.create_association(
data_process_definition_id, PRED.hasDataProcess, dproc_id)
self._manage_producers(dproc_id, out_data_product_ids)
self._manage_attachments()
queue_name = self._create_subscription(dproc, in_data_product_ids)
pid = self._launch_data_process(
queue_name=queue_name,
data_process_definition_id=data_process_definition_id,
out_data_product_ids=out_data_product_ids,
configuration=configuration)
self.clients.resource_registry.create_association(
subject=dproc_id, predicate=PRED.hasProcess, object=pid)
return dproc_id
