def on_initial_bootstrap(self, process, config, **kwargs):
pds_client = ProcessDispatcherServiceProcessClient(process=process)
ingestion_module = config.get_safe('bootstrap.processes.ingestion.module','ion.processes.data.ingestion.science_granule_ingestion_worker')
ingestion_class = config.get_safe('bootstrap.processes.ingestion.class' ,'ScienceGranuleIngestionWorker')
ingestion_datastore = config.get_safe('bootstrap.processes.ingestion.datastore_name', 'datasets')
ingestion_queue = config.get_safe('bootstrap.processes.ingestion.queue' , 'science_granule_ingestion')
ingestion_workers = config.get_safe('bootstrap.processes.ingestion.workers', 2)
replay_module = config.get_safe('bootstrap.processes.replay.module', 'ion.processes.data.replay.replay_process')
replay_class = config.get_safe('bootstrap.processes.replay.class' , 'ReplayProcess')
process_definition = ProcessDefinition(
name='ingestion_worker_process',
description='Worker transform process for ingestion of datasets')
process_definition.executable['module']= ingestion_module
process_definition.executable['class'] = ingestion_class
ingestion_procdef_id = pds_client.create_process_definition(process_definition=process_definition)
#--------------------------------------------------------------------------------
# Simulate a HA ingestion worker by creating two of them
#--------------------------------------------------------------------------------
config = DotDict()
config.process.datastore_name = ingestion_datastore
config.process.queue_name = ingestion_queue
for i in xrange(ingestion_workers):
pds_client.schedule_process(process_definition_id=ingestion_procdef_id, configuration=config)
process_definition = ProcessDefinition(name='data_replay_process', description='Process for the replay of datasets')
process_definition.executable['module']= replay_module
process_definition.executable['class'] = replay_class
pds_client.create_process_definition(process_definition=process_definition)
class BootstrapQCPostProcessor(BootstrapPlugin):
'''
Sets up one QC Post Processing worker and initiates
the Scheduler Service's interval every 24 hours.
'''
def on_initial_bootstrap(self, process, config, **kwargs):
if self.process_exists(process, 'qc_post_processor'):
# Short circuit the bootstrap to make sure not more than one is ever started
return
self.scheduler_service = SchedulerServiceProcessClient(process=process)
self.process_dispatcher = ProcessDispatcherServiceProcessClient(process=process)
interval_key = uuid4().hex # Unique identifier for this process
config = DotDict()
config.process.interval_key = interval_key
process_definition = ProcessDefinition(name='qc_post_processor',
executable={'module':'ion.processes.data.transforms.qc_post_processing', 'class':'QCPostProcessing'})
process_definition_id = self.process_dispatcher.create_process_definition(process_definition)
process_id = self.process_dispatcher.create_process(process_definition_id)
self.process_dispatcher.schedule_process(process_definition_id, process_id=process_id, configuration=config)
timer_id = self.scheduler_service.create_interval_timer(start_time=time.time(),
end_time=-1, #Run FOREVER
interval=3600*24,
event_origin=interval_key)
def process_exists(self, process, name):
proc_ids, meta = process.container.resource_registry.find_resources(restype=RT.Process, id_only=True)
return any([name in p['name'] for p in meta])
class BootstrapQCPostProcessor(BootstrapPlugin):
'''
Sets up one QC Post Processing worker and initiates
the Scheduler Service's interval every 24 hours.
'''
def on_initial_bootstrap(self, process, config, **kwargs):
if os.environ.get('PYCC_MODE'):
# This environment is an ion integration test
log.info('PYCC_MODE: skipping qc_post_processor launch')
return
if self.process_exists(process, 'qc_post_processor'):
# Short circuit the bootstrap to make sure not more than one is ever started
return
self.scheduler_service = SchedulerServiceProcessClient(process=process)
self.process_dispatcher = ProcessDispatcherServiceProcessClient(
process=process)
self.run_interval = CFG.get_safe('service.qc_processing.run_interval',
24)
interval_key = uuid4().hex # Unique identifier for this process
config = DotDict()
config.process.interval_key = interval_key
process_definition = ProcessDefinition(
name='qc_post_processor',
executable={
'module': 'ion.processes.data.transforms.qc_post_processing',
'class': 'QCPostProcessing'
})
process_definition_id = self.process_dispatcher.create_process_definition(
process_definition)
process_id = self.process_dispatcher.create_process(
process_definition_id)
self.process_dispatcher.schedule_process(process_definition_id,
process_id=process_id,
configuration=config)
timer_id = self.scheduler_service.create_interval_timer(
start_time=str(time.time()),
end_time='-1', #Run FOREVER
interval=3600 * self.run_interval,
event_origin=interval_key)
def process_exists(self, process, name):
proc_ids, meta = process.container.resource_registry.find_resources(
restype=RT.Process, id_only=True)
return any([name in p['name'] for p in meta if p['name']])
class BootstrapQCPostProcessor(BootstrapPlugin):
"""
Sets up one QC Post Processing worker and initiates
the Scheduler Service's interval every 24 hours.
"""
def on_initial_bootstrap(self, process, config, **kwargs):
if os.environ.get("PYCC_MODE"):
# This environment is an ion integration test
log.info("PYCC_MODE: skipping qc_post_processor launch")
return
if self.process_exists(process, "qc_post_processor"):
# Short circuit the bootstrap to make sure not more than one is ever started
return
self.scheduler_service = SchedulerServiceProcessClient(process=process)
self.process_dispatcher = ProcessDispatcherServiceProcessClient(process=process)
self.run_interval = CFG.get_safe("service.qc_processing.run_interval", 24)
interval_key = uuid4().hex # Unique identifier for this process
config = DotDict()
config.process.interval_key = interval_key
process_definition = ProcessDefinition(
name="qc_post_processor",
executable={"module": "ion.processes.data.transforms.qc_post_processing", "class": "QCPostProcessing"},
)
process_definition_id = self.process_dispatcher.create_process_definition(process_definition)
process_id = self.process_dispatcher.create_process(process_definition_id)
self.process_dispatcher.schedule_process(process_definition_id, process_id=process_id, configuration=config)
timer_id = self.scheduler_service.create_interval_timer(
start_time=str(time.time()),
end_time="-1", # Run FOREVER
interval=3600 * self.run_interval,
event_origin=interval_key,
)
def process_exists(self, process, name):
proc_ids, meta = process.container.resource_registry.find_resources(restype=RT.Process, id_only=True)
return any([name in p["name"] for p in meta if p["name"]])
class BootstrapQCPostProcessor(BootstrapPlugin):
'''
Sets up one QC Post Processing worker and initiates
the Scheduler Service's interval every 24 hours.
'''
def on_initial_bootstrap(self, process, config, **kwargs):
# TODO: Temporary skip while refactoring QC work for M088
return
if os.environ.get('PYCC_MODE'):
# This environment is an ion integration test
log.info('PYCC_MODE: skipping qc_post_processor launch')
return
if self.process_exists(process, 'qc_post_processor'):
# Short circuit the bootstrap to make sure not more than one is ever started
return
self.scheduler_service = SchedulerServiceProcessClient(process=process)
self.process_dispatcher = ProcessDispatcherServiceProcessClient(process=process)
self.run_interval = CFG.get_safe('service.qc_processing.run_interval', 24)
interval_key = uuid4().hex # Unique identifier for this process
config = DotDict()
config.process.interval_key = interval_key
process_definition = ProcessDefinition(name='qc_post_processor',
executable={'module':'ion.processes.data.transforms.qc_post_processing', 'class':'QCPostProcessing'})
process_definition_id = self.process_dispatcher.create_process_definition(process_definition)
process_id = self.process_dispatcher.create_process(process_definition_id)
self.process_dispatcher.schedule_process(process_definition_id, process_id=process_id, configuration=config)
timer_id = self.scheduler_service.create_interval_timer(start_time=str(time.time()),
end_time='-1', #Run FOREVER
interval=3600*self.run_interval,
event_origin=interval_key)
def process_exists(self, process, name):
proc_ids, meta = process.container.resource_registry.find_resources(restype=RT.Process, id_only=True)
return any([name in p['name'] for p in meta if p['name']])
class BootstrapProcessDispatcher(BootstrapPlugin):
"""
Bootstrap process for process dispatcher.
"""
def on_initial_bootstrap(self, process, config, **kwargs):
self.pds_client = ProcessDispatcherServiceProcessClient(
process=process)
self.resource_registry = ResourceRegistryServiceProcessClient(
process=process)
self.ingestion_worker(process, config)
self.replay_defs(process, config)
self.notification_worker(process, config)
self.registration_worker(process, config)
self.pydap_server(process, config)
self.eoi_services(process, config)
def eoi_services(self, process, config):
eoi_module = config.get_safe(
'bootstrap.processes.registration.module',
'ion.processes.data.registration.eoi_registration_process')
eoi_class = config.get_safe('bootstrap.processes.registration.class',
'EOIRegistrationProcess')
process_definition = ProcessDefinition(
name='eoi_server', description='Process for eoi data sources')
process_definition.executable['module'] = eoi_module
process_definition.executable['class'] = eoi_class
self._create_and_launch(process_definition)
def pydap_server(self, process, config):
pydap_module = config.get_safe(
'bootstrap.processes.pydap.module',
'ion.processes.data.externalization.lightweight_pydap')
pydap_class = config.get_safe('bootstrap.processes.pydap.class',
'LightweightPyDAP')
use_pydap = config.get_safe('bootstrap.launch_pydap', False)
process_definition = ProcessDefinition(
name='pydap_server',
description='Lightweight WSGI Server for PyDAP')
process_definition.executable['module'] = pydap_module
process_definition.executable['class'] = pydap_class
self._create_and_launch(process_definition, use_pydap)
def registration_worker(self, process, config):
res, meta = self.resource_registry.find_resources(
name='registration_worker', restype=RT.ProcessDefinition)
if len(res):
return
registration_module = config.get_safe(
'bootstrap.processes.registration.module',
'ion.processes.data.registration.registration_process')
registration_class = config.get_safe(
'bootstrap.processes.registration.class', 'RegistrationProcess')
use_pydap = True
process_definition = ProcessDefinition(
name='registration_worker',
description='For registering datasets with ERDDAP')
process_definition.executable['module'] = registration_module
process_definition.executable['class'] = registration_class
self._create_and_launch(process_definition, use_pydap)
def _create_and_launch(self, process_definition, conditional=True):
proc_def_id = self.pds_client.create_process_definition(
process_definition=process_definition)
if conditional:
process_res_id = self.pds_client.create_process(
process_definition_id=proc_def_id)
self.pds_client.schedule_process(process_definition_id=proc_def_id,
process_id=process_res_id)
def ingestion_worker(self, process, config):
# ingestion
ingestion_module = config.get_safe(
'bootstrap.processes.ingestion.module',
'ion.processes.data.ingestion.science_granule_ingestion_worker')
ingestion_class = config.get_safe(
'bootstrap.processes.ingestion.class',
'ScienceGranuleIngestionWorker')
ingestion_datastore = config.get_safe(
'bootstrap.processes.ingestion.datastore_name', 'datasets')
ingestion_queue = config.get_safe(
'bootstrap.processes.ingestion.queue', 'science_granule_ingestion')
ingestion_workers = config.get_safe(
'bootstrap.processes.ingestion.workers', 1)
#--------------------------------------------------------------------------------
# Create ingestion workers
#--------------------------------------------------------------------------------
process_definition = ProcessDefinition(
name='ingestion_worker_process',
description='Worker transform process for ingestion of datasets')
process_definition.executable['module'] = ingestion_module
process_definition.executable['class'] = ingestion_class
ingestion_procdef_id = self.pds_client.create_process_definition(
process_definition=process_definition)
#--------------------------------------------------------------------------------
# Simulate a HA ingestion worker by creating two of them
#--------------------------------------------------------------------------------
# config = DotDict()
# config.process.datastore_name = ingestion_datastore
# config.process.queue_name = ingestion_queue
#
# for i in xrange(ingestion_workers):
# self.pds_client.schedule_process(process_definition_id=ingestion_procdef_id, configuration=config)
def notification_worker(self, process, config):
# user notifications
notification_module = config.get_safe(
'bootstrap.processes.user_notification.module',
'ion.processes.data.transforms.notification_worker')
notification_class = config.get_safe(
'bootstrap.processes.user_notification.class',
'NotificationWorker')
notification_workers = config.get_safe(
'bootstrap.processes.user_notification.workers', 1)
#--------------------------------------------------------------------------------
# Create notification workers
#--------------------------------------------------------------------------------
# set up the process definition
process_definition_uns = ProcessDefinition(
name='notification_worker_process',
description='Worker transform process for user notifications')
process_definition_uns.executable['module'] = notification_module
process_definition_uns.executable['class'] = notification_class
uns_procdef_id = self.pds_client.create_process_definition(
process_definition=process_definition_uns)
config = DotDict()
config.process.type = 'simple'
for i in xrange(notification_workers):
config.process.name = 'notification_worker_%s' % i
config.process.queue_name = 'notification_worker_queue'
self.pds_client.schedule_process(
process_definition_id=uns_procdef_id, configuration=config)
def replay_defs(self, process, config):
replay_module = config.get_safe(
'bootstrap.processes.replay.module',
'ion.processes.data.replay.replay_process')
replay_class = config.get_safe('bootstrap.processes.replay.class',
'ReplayProcess')
#--------------------------------------------------------------------------------
# Create replay process definition
#--------------------------------------------------------------------------------
process_definition = ProcessDefinition(
name=DataRetrieverService.REPLAY_PROCESS,
description='Process for the replay of datasets')
process_definition.executable['module'] = replay_module
process_definition.executable['class'] = replay_class
self.pds_client.create_process_definition(
process_definition=process_definition)
def on_restart(self, process, config, **kwargs):
pass
class BootstrapProcessDispatcher(BootstrapPlugin):
"""
Bootstrap process for process dispatcher.
"""
def on_initial_bootstrap(self, process, config, **kwargs):
self.pds_client = ProcessDispatcherServiceProcessClient(process=process)
self.resource_registry = ResourceRegistryServiceProcessClient(process=process)
self.ingestion_worker(process,config)
self.replay_defs(process,config)
self.notification_worker(process,config)
self.registration_worker(process,config)
self.pydap_server(process,config)
def pydap_server(self, process, config):
pydap_module = config.get_safe('bootstrap.processes.pydap.module', 'ion.processes.data.externalization.lightweight_pydap')
pydap_class = config.get_safe('bootstrap.processes.pydap.class', 'LightweightPyDAP')
use_pydap = config.get_safe('bootstrap.launch_pydap', False)
process_definition = ProcessDefinition(
name = 'pydap_server',
description = 'Lightweight WSGI Server for PyDAP')
process_definition.executable['module'] = pydap_module
process_definition.executable['class'] = pydap_class
self._create_and_launch(process_definition,use_pydap)
def registration_worker(self, process, config):
res, meta = self.resource_registry.find_resources(name='registration_worker', restype=RT.ProcessDefinition)
if len(res):
return
registration_module = config.get_safe('bootstrap.processes.registration.module', 'ion.processes.data.registration.registration_process')
registration_class = config.get_safe('bootstrap.processes.registration.class', 'RegistrationProcess')
use_pydap = config.get_safe('bootstrap.use_pydap', False)
process_definition = ProcessDefinition(
name='registration_worker',
description='For registering datasets with ERDDAP')
process_definition.executable['module'] = registration_module
process_definition.executable['class'] = registration_class
self._create_and_launch(process_definition, use_pydap)
def _create_and_launch(self, process_definition, conditional=True):
proc_def_id = self.pds_client.create_process_definition(process_definition=process_definition)
if conditional:
process_res_id = self.pds_client.create_process(process_definition_id=proc_def_id)
self.pds_client.schedule_process(process_definition_id=proc_def_id, process_id=process_res_id)
def ingestion_worker(self, process, config):
# ingestion
ingestion_module = config.get_safe('bootstrap.processes.ingestion.module','ion.processes.data.ingestion.science_granule_ingestion_worker')
ingestion_class = config.get_safe('bootstrap.processes.ingestion.class' ,'ScienceGranuleIngestionWorker')
ingestion_datastore = config.get_safe('bootstrap.processes.ingestion.datastore_name', 'datasets')
ingestion_queue = config.get_safe('bootstrap.processes.ingestion.queue' , 'science_granule_ingestion')
ingestion_workers = config.get_safe('bootstrap.processes.ingestion.workers', 1)
#--------------------------------------------------------------------------------
# Create ingestion workers
#--------------------------------------------------------------------------------
process_definition = ProcessDefinition(
name='ingestion_worker_process',
description='Worker transform process for ingestion of datasets')
process_definition.executable['module']= ingestion_module
process_definition.executable['class'] = ingestion_class
ingestion_procdef_id = self.pds_client.create_process_definition(process_definition=process_definition)
#--------------------------------------------------------------------------------
# Simulate a HA ingestion worker by creating two of them
#--------------------------------------------------------------------------------
# config = DotDict()
# config.process.datastore_name = ingestion_datastore
# config.process.queue_name = ingestion_queue
#
# for i in xrange(ingestion_workers):
# self.pds_client.schedule_process(process_definition_id=ingestion_procdef_id, configuration=config)
def notification_worker(self, process, config):
# user notifications
notification_module = config.get_safe('bootstrap.processes.user_notification.module','ion.processes.data.transforms.notification_worker')
notification_class = config.get_safe('bootstrap.processes.user_notification.class' ,'NotificationWorker')
notification_workers = config.get_safe('bootstrap.processes.user_notification.workers', 1)
#--------------------------------------------------------------------------------
# Create notification workers
#--------------------------------------------------------------------------------
# set up the process definition
process_definition_uns = ProcessDefinition(
name='notification_worker_process',
description='Worker transform process for user notifications')
process_definition_uns.executable['module']= notification_module
process_definition_uns.executable['class'] = notification_class
uns_procdef_id = self.pds_client.create_process_definition(process_definition=process_definition_uns)
config = DotDict()
config.process.type = 'simple'
for i in xrange(notification_workers):
config.process.name = 'notification_worker_%s' % i
config.process.queue_name = 'notification_worker_queue'
self.pds_client.schedule_process(process_definition_id=uns_procdef_id, configuration=config)
def replay_defs(self, process, config):
replay_module = config.get_safe('bootstrap.processes.replay.module', 'ion.processes.data.replay.replay_process')
replay_class = config.get_safe('bootstrap.processes.replay.class' , 'ReplayProcess')
#--------------------------------------------------------------------------------
# Create replay process definition
#--------------------------------------------------------------------------------
process_definition = ProcessDefinition(name=DataRetrieverService.REPLAY_PROCESS, description='Process for the replay of datasets')
process_definition.executable['module']= replay_module
process_definition.executable['class'] = replay_class
self.pds_client.create_process_definition(process_definition=process_definition)
def on_restart(self, process, config, **kwargs):
pass
