def test_create_event_process(self):
"""
Test creating an event process
"""
process_definition = ProcessDefinition(name='test')
process_definition.definition = ''
rrc = ResourceRegistryServiceClient(node = self.container.node)
process_definition_id = rrc.create(process_definition)
self.mock_rr_client.find_objects = Mock()
self.mock_rr_client.find_objects.return_value = ['stream_id_1'], 'obj_assoc_1'
# self.mock_pd_client.schedule_process = Mock()
# self.mock_pd_client.schedule_process.return_value = 'process_id'
self.mock_rr_client.create_association = mocksignature(self.mock_rr_client.create_association)
pid = self.event_management.create_event_process(process_definition_id=process_definition_id,
event_types=['type_1', 'type_2'],
sub_types=['subtype_1', 'subtype_2'],
origins=['or_1', 'or_2'],
origin_types=['t1', 't2'],
out_data_products={'conductivity': 'id1'}
)
def notification_worker(self, process, config):
# user notifications
notification_module = config.get_safe('bootstrap.processes.user_notification.module','ion.processes.data.transforms.notification_worker')
notification_class = config.get_safe('bootstrap.processes.user_notification.class' ,'NotificationWorker')
notification_workers = config.get_safe('bootstrap.processes.user_notification.workers', 1)
#--------------------------------------------------------------------------------
# Create notification workers
#--------------------------------------------------------------------------------
# set up the process definition
process_definition_uns = ProcessDefinition(
name='notification_worker_process',
description='Worker transform process for user notifications')
process_definition_uns.executable['module']= notification_module
process_definition_uns.executable['class'] = notification_class
uns_procdef_id = self.pds_client.create_process_definition(process_definition=process_definition_uns)
config = DotDict()
config.process.type = 'simple'
for i in xrange(notification_workers):
config.process.name = 'notification_worker_%s' % i
config.process.queue_name = 'notification_worker_queue'
self.pds_client.schedule_process(process_definition_id=uns_procdef_id, configuration=config)
def ingestion_worker(self, process, config):
# ingestion
ingestion_module = config.get_safe('bootstrap.processes.ingestion.module','ion.processes.data.ingestion.science_granule_ingestion_worker')
ingestion_class = config.get_safe('bootstrap.processes.ingestion.class' ,'ScienceGranuleIngestionWorker')
ingestion_datastore = config.get_safe('bootstrap.processes.ingestion.datastore_name', 'datasets')
ingestion_queue = config.get_safe('bootstrap.processes.ingestion.queue' , 'science_granule_ingestion')
ingestion_workers = config.get_safe('bootstrap.processes.ingestion.workers', 1)
#--------------------------------------------------------------------------------
# Create ingestion workers
#--------------------------------------------------------------------------------
process_definition = ProcessDefinition(
name='ingestion_worker_process',
description='Worker transform process for ingestion of datasets')
process_definition.executable['module']= ingestion_module
process_definition.executable['class'] = ingestion_class
ingestion_procdef_id = self.pds_client.create_process_definition(process_definition=process_definition)
#--------------------------------------------------------------------------------
# Simulate a HA ingestion worker by creating two of them
#--------------------------------------------------------------------------------
config = DotDict()
config.process.datastore_name = ingestion_datastore
config.process.queue_name = ingestion_queue
for i in xrange(ingestion_workers):
self.pds_client.schedule_process(process_definition_id=ingestion_procdef_id, configuration=config)
def launch_platform(self, agt_id, agent_config, timeout_spawn=30):
"""
Launches a platform agent.
@param agt_id Some ID mainly used for logging
@param agent_config Agent configuration
@param timeout_spawn Timeout in secs for the SPAWN event (by
default 30). If None or zero, no wait is performed.
@return process ID
"""
log.debug("launch platform: agt_id=%r, timeout_spawn=%s", agt_id, timeout_spawn)
name = 'PlatformAgent_%s' % agt_id
pdef = ProcessDefinition(name=name)
pdef.executable = {
'module': 'ion.agents.platform.platform_agent',
'class': 'PlatformAgent'
}
pdef_id = self._pd_client.create_process_definition(process_definition=pdef)
pid = self._agent_launcher.launch(agent_config, pdef_id)
if timeout_spawn:
self._agent_launcher.await_launch(timeout_spawn)
return pid
def create_data_process_definition(self, data_process_definition=None):
data_process_definition_id = self.RR2.create(data_process_definition, RT.DataProcessDefinition)
# -------------------------------
# Process Definition
# -------------------------------
# Create the underlying process definition
process_definition = ProcessDefinition()
process_definition.name = data_process_definition.name
process_definition.description = data_process_definition.description
process_definition.executable = {
"module": data_process_definition.module,
"class": data_process_definition.class_name,
}
process_definition_id = self.clients.process_dispatcher.create_process_definition(
process_definition=process_definition
)
self.RR2.assign_process_definition_to_data_process_definition_with_has_process_definition(
process_definition_id, data_process_definition_id
)
return data_process_definition_id
def create_event_process_definition(self, version='', module='', class_name='', uri='', arguments=None, event_types = None, sub_types = None, origin_types = None):
"""
Create a resource which defines the processing of events.
@param version str
@param module str
@param class_name str
@param uri str
@param arguments list
@return procdef_id str
"""
# Create the event process detail object
event_process_definition_detail = EventProcessDefinitionDetail()
event_process_definition_detail.event_types = event_types
event_process_definition_detail.sub_types = sub_types
event_process_definition_detail.origin_types = origin_types
# Create the process definition
process_definition = ProcessDefinition(name=create_unique_identifier('event_process'))
process_definition.executable = {
'module':module,
'class': class_name,
'url': uri
}
process_definition.version = version
process_definition.arguments = arguments
process_definition.definition = event_process_definition_detail
procdef_id = self.clients.process_dispatcher.create_process_definition(process_definition=process_definition)
return procdef_id
def launch_instrument(self, agt_id, agent_config, timeout_spawn=None):
"""
Launches an instrument agent.
@param agt_id Some ID mainly used for logging
@param agent_config Agent configuration
@param timeout_spawn Timeout in secs for the RUNNING event (by
default, the value given in constructor).
If None or zero, no wait is performed.
@return process ID
"""
timeout_spawn = timeout_spawn or self._timeout_spawn
log.debug("launch_instrument: agt_id=%r, timeout_spawn=%s", agt_id, timeout_spawn)
name = 'InstrumentAgent_%s' % agt_id
pdef = ProcessDefinition(name=name)
pdef.executable = {
'module': 'ion.agents.instrument.instrument_agent',
'class': 'InstrumentAgent'
}
pdef_id = self._pd_client.create_process_definition(process_definition=pdef)
pid = self._agent_launcher.launch(agent_config, pdef_id)
if timeout_spawn:
log.debug("launch_instrument: agt_id=%r: waiting for RUNNING", agt_id)
self._agent_launcher.await_launch(timeout_spawn)
log.debug("launch_instrument: agt_id=%r: RUNNING", agt_id)
return pid
def test_presf_L1(self):
'''
Test that packets are processed by the ctd_L1_pressure transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='PresfL1Transform',
description='For testing PresfL1Transform')
process_definition.executable['module']= 'ion.processes.data.transforms.ctd.presf_L1'
process_definition.executable['class'] = 'PresfL1Transform'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition('pres_stream_def', parameter_dictionary_id=pdict_id)
pres_stream_id, _ = self.pubsub.create_stream('test_pressure',
stream_definition_id=stream_def_id,
exchange_point='science_data')
config.process.publish_streams.seafloor_pressure = pres_stream_id
# Schedule the process
self.process_dispatcher.schedule_process(process_definition_id=ctd_transform_proc_def_id, configuration=config)
def dispatch_process(self, upid, spec, subscribers, constraints=None,
immediate=False):
name = spec.get('name')
self.event_pub.publish_event(event_type="ProcessLifecycleEvent",
origin=name, origin_type="DispatchedHAProcess",
state=ProcessStateEnum.SPAWN)
process_def = ProcessDefinition(name=name)
process_def.executable = {'module': spec.get('module'),
'class': spec.get('class')}
process_def_id = self.real_client.create_process_definition(process_def)
pid = self.real_client.create_process(process_def_id)
process_schedule = ProcessSchedule()
sched_pid = self.real_client.schedule_process(process_def_id,
process_schedule, configuration={}, process_id=pid)
proc = self.real_client.read_process(sched_pid)
dict_proc = {'upid': proc.process_id,
'state': self.state_map.get(proc.process_state, self.unknown_state),
}
return dict_proc
def start_worker(self):
proc_def = ProcessDefinition()
proc_def.executable['module'] = 'ion.processes.data.last_update_cache'
proc_def.executable['class'] = 'LastUpdateCache'
proc_def_id = self.pd_cli.create_process_definition(process_definition=proc_def)
subscription_id = self.pubsub_cli.create_subscription(query=ExchangeQuery(), exchange_name='ingestion_aggregate')
config = {
'couch_storage' : {
'datastore_name' :self.datastore_name,
'datastore_profile' : 'SCIDATA'
}
}
transform_id = self.tms_cli.create_transform(
name='last_update_cache',
description='LastUpdate that compiles an aggregate of metadata',
in_subscription_id=subscription_id,
process_definition_id=proc_def_id,
configuration=config
)
self.tms_cli.activate_transform(transform_id=transform_id)
transform = self.rr_cli.read(transform_id)
pid = transform.process_id
handle = self.container.proc_manager.procs[pid]
return handle
def test_event_in_stream_out_transform(self):
"""
Test the event-in/stream-out transform
"""
stream_id, _ = self.pubsub.create_stream('test_stream', exchange_point='science_data')
self.exchange_cleanup.append('science_data')
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='EventToStreamTransform',
description='For testing an event-in/stream-out transform')
process_definition.executable['module']= 'ion.processes.data.transforms.event_in_stream_out_transform'
process_definition.executable['class'] = 'EventToStreamTransform'
proc_def_id = self.process_dispatcher.create_process_definition(process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = 'test_queue'
config.process.exchange_point = 'science_data'
config.process.publish_streams.output = stream_id
config.process.event_type = 'ExampleDetectableEvent'
config.process.variables = ['voltage', 'temperature' ]
# Schedule the process
pid = self.process_dispatcher.schedule_process(process_definition_id=proc_def_id, configuration=config)
self.addCleanup(self.process_dispatcher.cancel_process,pid)
#---------------------------------------------------------------------------------------------
# Create a subscriber for testing
#---------------------------------------------------------------------------------------------
ar_cond = gevent.event.AsyncResult()
def subscriber_callback(m, r, s):
ar_cond.set(m)
sub = StandaloneStreamSubscriber('sub', subscriber_callback)
self.addCleanup(sub.stop)
sub_id = self.pubsub.create_subscription('subscription_cond',
stream_ids=[stream_id],
exchange_name='sub')
self.pubsub.activate_subscription(sub_id)
self.queue_cleanup.append(sub.xn.queue)
sub.start()
gevent.sleep(4)
#---------------------------------------------------------------------------------------------
# Publish an event. The transform has been configured to receive this event
#---------------------------------------------------------------------------------------------
event_publisher = EventPublisher("ExampleDetectableEvent")
event_publisher.publish_event(origin = 'fake_origin', voltage = '5', temperature = '273')
# Assert that the transform processed the event and published data on the output stream
result_cond = ar_cond.get(timeout=10)
self.assertTrue(result_cond)
def __init__(self, *args, **kwargs):
super(TransformExampleLauncher,self).__init__(*args,**kwargs)
#-------------------------------
# Process Definitions
#-------------------------------
transform_example_definition = ProcessDefinition(name='transform_example_definition')
transform_example_definition.executable['module'] = 'ion.services.dm.transformation.transform_example'
transform_example_definition.executable['class'] = 'TransformExample'
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
datastore_name = CACHE_DATASTORE_NAME
self.db = self.container.datastore_manager.get_datastore(datastore_name)
self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(name='ingestion worker')
ingestion_worker_definition.executable = {
'module':'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class' :'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def replay_defs(self, process, config):
replay_module = config.get_safe('bootstrap.processes.replay.module', 'ion.processes.data.replay.replay_process')
replay_class = config.get_safe('bootstrap.processes.replay.class' , 'ReplayProcess')
#--------------------------------------------------------------------------------
# Create replay process definition
#--------------------------------------------------------------------------------
process_definition = ProcessDefinition(name=DataRetrieverService.REPLAY_PROCESS, description='Process for the replay of datasets')
process_definition.executable['module']= replay_module
process_definition.executable['class'] = replay_class
self.pds_client.create_process_definition(process_definition=process_definition)
def eoi_services(self,process,config):
eoi_module = config.get_safe('bootstrap.processes.registration.module', 'ion.processes.data.registration.eoi_registration_process')
eoi_class = config.get_safe('bootstrap.processes.registration.class', 'EOIRegistrationProcess')
process_definition = ProcessDefinition(
name = 'eoi_server',
description = 'Process for eoi data sources')
process_definition.executable['module'] = eoi_module
process_definition.executable['class'] = eoi_class
self._create_and_launch(process_definition)
def test_execute_transform(self):
# set up
process_definition = ProcessDefinition(name='procdef_execute')
process_definition.executable['module'] = 'ion.processes.data.transforms.transform_example'
process_definition.executable['class'] = 'ReverseTransform'
data = [1,2,3]
process_definition_id, _ = self.rr_cli.create(process_definition)
retval = self.tms_cli.execute_transform(process_definition_id,data)
self.assertEquals(retval,[3,2,1])
def test_execute_transform(self):
# set up
process_definition = ProcessDefinition(name="procdef_execute")
process_definition.executable["module"] = "ion.processes.data.transforms.transform_example"
process_definition.executable["class"] = "ReverseTransform"
data = [1, 2, 3]
process_definition_id, _ = self.rr_cli.create(process_definition)
retval = self.tms_cli.execute_transform(process_definition_id, data)
self.assertEquals(retval, [3, 2, 1])
def create_definition(self, definition_id, definition_type, executable,
name=None, description=None):
if name is None:
raise BadRequest("create_definition must have a name supplied")
# note: we lose the description
definition = ProcessDefinition(name=name)
definition.executable = {'module': executable.get('module'),
'class': executable.get('class')}
definition.definition_type = definition_type
created_definition = self.real_client.create_process_definition(
definition, definition_id)
def create_ingestion_configuration(self, exchange_point_id='', couch_storage=None, hdf_storage=None,number_of_workers=0):
"""
@brief Setup ingestion workers to ingest all the data from a single exchange point.
@param exchange_point_id is the resource id for the exchagne point to ingest from
@param couch_storage is the specification of the couch database to use
@param hdf_storage is the specification of the filesystem to use for hdf data files
@param number_of_workers is the number of ingestion workers to create
"""
if self.process_definition_id is None:
process_definition = ProcessDefinition(name='ingestion_worker_process', description='Worker transform process for ingestion of datasets')
process_definition.executable['module']='ion.processes.data.ingestion.ingestion_worker'
process_definition.executable['class'] = 'IngestionWorker'
self.process_definition_id = self.clients.process_dispatcher.create_process_definition(process_definition=process_definition)
# Give each ingestion configuration its own queue name to receive data on
exchange_name = 'ingestion_queue'
##------------------------------------------------------------------------------------
## declare our intent to subscribe to all messages on the exchange point
query = ExchangeQuery()
subscription_id = self.clients.pubsub_management.create_subscription(query=query,\
exchange_name=exchange_name, name='Ingestion subscription', description='Subscription for ingestion workers')
##------------------------------------------------------------------------------------------
# create an ingestion_configuration instance and update the registry
# @todo: right now sending in the exchange_point_id as the name...
ingestion_configuration = IngestionConfiguration( name = self.XP)
ingestion_configuration.description = '%s exchange point ingestion configuration' % self.XP
ingestion_configuration.number_of_workers = number_of_workers
if hdf_storage is not None:
ingestion_configuration.hdf_storage.update(hdf_storage)
if couch_storage is not None:
ingestion_configuration.couch_storage.update(couch_storage)
ingestion_configuration_id, _ = self.clients.resource_registry.create(ingestion_configuration)
self._launch_transforms(
ingestion_configuration.number_of_workers,
subscription_id,
ingestion_configuration_id,
ingestion_configuration,
self.process_definition_id
)
return ingestion_configuration_id
def on_start(self):
super(IngestionManagementService,self).on_start()
self.event_publisher = EventPublisher(event_type="DatasetIngestionConfigurationEvent")
#########################################################################################################
# The code for process_definition may not really belong here, but we do not have a different way so
# far to preload the process definitions. This will later probably be part of a set of predefinitions
# for processes.
#########################################################################################################
process_definition = ProcessDefinition(name='ingestion_worker_process', description='Worker transform process for ingestion of datasets')
process_definition.executable['module']='ion.processes.data.ingestion.ingestion_worker'
process_definition.executable['class'] = 'IngestionWorker'
self.process_definition_id = self.clients.process_dispatcher.create_process_definition(process_definition=process_definition)
def _get_process_definition(self, data_process_definition_id=""):
process_definition_id = ""
if data_process_definition_id:
process_definitions, _ = self.clients.resource_registry.find_objects(
subject=data_process_definition_id, predicate=PRED.hasProcessDefinition, id_only=True
)
if process_definitions:
process_definition_id = process_definitions[0]
else:
process_definition = ProcessDefinition()
process_definition.name = "transform_data_process"
process_definition.executable["module"] = "ion.processes.data.transforms.transform_prime"
process_definition.executable["class"] = "TransformPrime"
process_definition_id = self.clients.process_dispatcher.create_process_definition(process_definition)
else:
process_definitions, _ = self.clients.resource_registry.find_resources(
name="transform_data_process", restype=RT.ProcessDefinition, id_only=True
)
if process_definitions:
process_definition_id = process_definitions[0]
else:
process_definition = ProcessDefinition()
process_definition.name = "transform_data_process"
process_definition.executable["module"] = "ion.processes.data.transforms.transform_prime"
process_definition.executable["class"] = "TransformPrime"
process_definition_id = self.clients.process_dispatcher.create_process_definition(process_definition)
return process_definition_id
def pydap_server(self, process, config):
pydap_module = config.get_safe('bootstrap.processes.pydap.module', 'ion.processes.data.externalization.lightweight_pydap')
pydap_class = config.get_safe('bootstrap.processes.pydap.class', 'LightweightPyDAP')
use_pydap = config.get_safe('bootstrap.launch_pydap', False)
process_definition = ProcessDefinition(
name = 'pydap_server',
description = 'Lightweight WSGI Server for PyDAP')
process_definition.executable['module'] = pydap_module
process_definition.executable['class'] = pydap_class
self._create_and_launch(process_definition,use_pydap)
def run_external_transform(self):
'''
This example script illustrates how a transform can interact with the an outside process (very basic)
it launches an external_transform example which uses the operating system command 'bc' to add 1 to the input
Producer -> A -> 'FS.TEMP/transform_output'
A is an external transform that spawns an OS process to increment the input by 1
'''
pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
tms_cli = TransformManagementServiceClient(node=self.container.node)
procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
#-------------------------------
# Process Definition
#-------------------------------
process_definition = ProcessDefinition(name='external_transform_definition')
process_definition.executable['module'] = 'ion.processes.data.transforms.transform_example'
process_definition.executable['class'] = 'ExternalTransform'
process_definition_id = procd_cli.create_process_definition(process_definition=process_definition)
#-------------------------------
# Streams
#-------------------------------
input_stream_id = pubsub_cli.create_stream(name='input_stream', original=True)
#-------------------------------
# Subscription
#-------------------------------
query = StreamQuery(stream_ids=[input_stream_id])
input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue')
#-------------------------------
# Launch Transform
#-------------------------------
transform_id = tms_cli.create_transform(name='external_transform',
in_subscription_id=input_subscription_id,
process_definition_id=process_definition_id,
configuration={})
tms_cli.activate_transform(transform_id)
#-------------------------------
# Launch Producer
#-------------------------------
id_p = self.container.spawn_process('myproducer', 'ion.processes.data.transforms.transform_example', 'TransformExampleProducer', {'process':{'type':'stream_process','publish_streams':{'out_stream':input_stream_id}},'stream_producer':{'interval':4000}})
self.container.proc_manager.procs[id_p].start()
def _do_launch_gate(self, platform_id, agent_config, timeout_spawn):
"""
The method for when using the ProcessStateGate pattern, which is the
one used by test_oms_launch2 to launch the root platform.
"""
log.debug("_do_launch_gate: platform_id=%r, timeout_spawn=%s",
platform_id, timeout_spawn)
pa_name = 'PlatformAgent_%s' % platform_id
pdef = ProcessDefinition(name=pa_name)
pdef.executable = {
'module': PA_MOD,
'class': PA_CLS
}
pdef_id = self._pd_client.create_process_definition(process_definition=pdef)
log.debug("using schedule_process directly %r", platform_id)
pid = self._pd_client.schedule_process(process_definition_id=pdef_id,
schedule=None,
configuration=agent_config)
if timeout_spawn:
# ProcessStateGate used as indicated in its pydoc (9/21/12)
gate = ProcessStateGate(self._pd_client.read_process, pid, ProcessStateEnum.RUNNING)
err_msg = None
try:
if not gate.await(timeout_spawn):
err_msg = "The platform agent instance did not spawn in " \
"%s seconds. gate.wait returned false. " % \
timeout_spawn
log.error(err_msg)
except Exception as e:
log.error("Exception while waiting for platform agent instance "
"(platform_id=%r) "
"to spawn in %s seconds: %s",
platform_id, timeout_spawn, str(e)) #,exc_Info=True)
if err_msg:
raise PlatformException(err_msg)
log.debug("_do_launch_gate: platform_id=%r: agent spawned, pid=%r "
"(ProcessStateGate pattern used)",
platform_id, pid)
return pid
def create_logger(self, name, stream_id=""):
# logger process
producer_definition = ProcessDefinition(name=name + "_logger")
producer_definition.executable = {
"module": "ion.processes.data.stream_granule_logger",
"class": "StreamGranuleLogger",
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {"process": {"stream_id": stream_id}}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id, configuration=configuration
)
return pid
def post_process_dispatcher(self, config):
process_definition = ProcessDefinition(
name='ingestion_worker_process',
description='Worker transform process for ingestion of datasets')
process_definition.executable['module']='ion.processes.data.ingestion.ingestion_worker'
process_definition.executable['class'] = 'IngestionWorker'
self.clients.process_dispatcher.create_process_definition(process_definition=process_definition)
self.process_definition = ProcessDefinition(
name='data_replay_process',
description='Process for the replay of datasets')
self.process_definition.executable['module']='ion.processes.data.replay_process'
self.process_definition.executable['class'] = 'ReplayProcess'
self.clients.process_dispatcher.create_process_definition(process_definition=self.process_definition)
def create_process(name= '', module = '', class_name = '', configuration = None):
'''
A helper method to create a process
'''
producer_definition = ProcessDefinition(name=name)
producer_definition.executable = {
'module':module,
'class': class_name
}
process_dispatcher = ProcessDispatcherServiceClient()
procdef_id = process_dispatcher.create_process_definition(process_definition=producer_definition)
pid = process_dispatcher.schedule_process(process_definition_id= procdef_id, configuration=configuration)
return pid
def ingestion_worker(self, process, config):
# ingestion
ingestion_module = config.get_safe('bootstrap.processes.ingestion.module','ion.processes.data.ingestion.science_granule_ingestion_worker')
ingestion_class = config.get_safe('bootstrap.processes.ingestion.class' ,'ScienceGranuleIngestionWorker')
ingestion_datastore = config.get_safe('bootstrap.processes.ingestion.datastore_name', 'datasets')
ingestion_queue = config.get_safe('bootstrap.processes.ingestion.queue' , 'science_granule_ingestion')
ingestion_workers = config.get_safe('bootstrap.processes.ingestion.workers', 1)
#--------------------------------------------------------------------------------
# Create ingestion workers
#--------------------------------------------------------------------------------
process_definition = ProcessDefinition(
name='ingestion_worker_process',
description='Worker transform process for ingestion of datasets')
process_definition.executable['module']= ingestion_module
process_definition.executable['class'] = ingestion_class
ingestion_procdef_id = self.pds_client.create_process_definition(process_definition=process_definition)
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id= logger_procdef_id, configuration=configuration)
return pid
def registration_worker(self, process, config):
res, meta = self.resource_registry.find_resources(name='registration_worker', restype=RT.ProcessDefinition)
if len(res):
return
registration_module = config.get_safe('bootstrap.processes.registration.module', 'ion.processes.data.registration.registration_process')
registration_class = config.get_safe('bootstrap.processes.registration.class', 'RegistrationProcess')
use_pydap = config.get_safe('bootstrap.use_pydap', False)
process_definition = ProcessDefinition(
name='registration_worker',
description='For registering datasets with ERDDAP')
process_definition.executable['module'] = registration_module
process_definition.executable['class'] = registration_class
self._create_and_launch(process_definition, use_pydap)
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient()
self.rrclient = ResourceRegistryServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
datastore_name = CACHE_DATASTORE_NAME
self.db = self.container.datastore_manager.get_datastore(
datastore_name)
self.stream_def_id = self.pubsubcli.create_stream_definition(
name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(
name='ingestion worker')
ingestion_worker_definition.executable = {
'module':
'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class': 'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(
process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(
self.process_definitions['ingestion_worker'], configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
self.pd_cli = ProcessDispatcherServiceClient(
to_name="process_dispatcher")
self.process_definition_id = uuid4().hex
self.process_definition = ProcessDefinition(
name='test',
executable={
'module': 'ion.agents.cei.test.test_haagent',
'class': 'TestProcess'
})
self.pd_cli.create_process_definition(self.process_definition,
self.process_definition_id)
http_port = 8919
http_port = self._start_webserver(port=http_port)
self.resource_id = "haagent_4567"
self._haa_name = "high_availability_agent"
self._haa_config = {
'server': {
'trafficsentinel': {
'host': 'localhost',
'port': http_port,
'protocol': 'http',
'username': '******',
'password': '******'
}
},
'highavailability': {
'policy': {
'interval': 1,
'name': 'sensor',
'parameters': {
'metric': 'app_attributes:ml',
'sample_period': 600,
'sample_function': 'Average',
'cooldown_period': 5,
'scale_up_threshold': 2.0,
'scale_up_n_processes': 1,
'scale_down_threshold': 1.0,
'scale_down_n_processes': 1,
'maximum_processes': 5,
'minimum_processes': 1,
}
},
'process_definition_id': self.process_definition_id,
"process_dispatchers": ['process_dispatcher']
},
'agent': {
'resource_id': self.resource_id
},
}
self._base_procs = self.pd_cli.list_processes()
self.waiter = ProcessStateWaiter()
self.waiter.start()
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._haa_pid = self.container_client.spawn_process(
name=self._haa_name,
module="ion.agents.cei.high_availability_agent",
cls="HighAvailabilityAgent",
config=self._haa_config)
# Start a resource agent client to talk with the instrument agent.
self._haa_pyon_client = SimpleResourceAgentClient(
self.resource_id, process=FakeProcess())
log.info('Got haa client %s.', str(self._haa_pyon_client))
self.haa_client = HighAvailabilityAgentClient(self._haa_pyon_client)
