def on_initial_bootstrap(self, process, config, **kwargs): pds_client = ProcessDispatcherServiceProcessClient(process=process) ingestion_module = config.get_safe('bootstrap.processes.ingestion.module','ion.processes.data.ingestion.science_granule_ingestion_worker') ingestion_class = config.get_safe('bootstrap.processes.ingestion.class' ,'ScienceGranuleIngestionWorker') ingestion_datastore = config.get_safe('bootstrap.processes.ingestion.datastore_name', 'datasets') ingestion_queue = config.get_safe('bootstrap.processes.ingestion.queue' , 'science_granule_ingestion') ingestion_workers = config.get_safe('bootstrap.processes.ingestion.workers', 2) replay_module = config.get_safe('bootstrap.processes.replay.module', 'ion.processes.data.replay.replay_process') replay_class = config.get_safe('bootstrap.processes.replay.class' , 'ReplayProcess') process_definition = ProcessDefinition( name='ingestion_worker_process', description='Worker transform process for ingestion of datasets') process_definition.executable['module']= ingestion_module process_definition.executable['class'] = ingestion_class ingestion_procdef_id = pds_client.create_process_definition(process_definition=process_definition) #-------------------------------------------------------------------------------- # Simulate a HA ingestion worker by creating two of them #-------------------------------------------------------------------------------- config = DotDict() config.process.datastore_name = ingestion_datastore config.process.queue_name = ingestion_queue for i in xrange(ingestion_workers): pds_client.schedule_process(process_definition_id=ingestion_procdef_id, configuration=config) process_definition = ProcessDefinition(name='data_replay_process', description='Process for the replay of datasets') process_definition.executable['module']= replay_module process_definition.executable['class'] = replay_class pds_client.create_process_definition(process_definition=process_definition)
class BootstrapQCPostProcessor(BootstrapPlugin): ''' Sets up one QC Post Processing worker and initiates the Scheduler Service's interval every 24 hours. ''' def on_initial_bootstrap(self, process, config, **kwargs): if self.process_exists(process, 'qc_post_processor'): # Short circuit the bootstrap to make sure not more than one is ever started return self.scheduler_service = SchedulerServiceProcessClient(process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(process=process) interval_key = uuid4().hex # Unique identifier for this process config = DotDict() config.process.interval_key = interval_key process_definition = ProcessDefinition(name='qc_post_processor', executable={'module':'ion.processes.data.transforms.qc_post_processing', 'class':'QCPostProcessing'}) process_definition_id = self.process_dispatcher.create_process_definition(process_definition) process_id = self.process_dispatcher.create_process(process_definition_id) self.process_dispatcher.schedule_process(process_definition_id, process_id=process_id, configuration=config) timer_id = self.scheduler_service.create_interval_timer(start_time=time.time(), end_time=-1, #Run FOREVER interval=3600*24, event_origin=interval_key) def process_exists(self, process, name): proc_ids, meta = process.container.resource_registry.find_resources(restype=RT.Process, id_only=True) return any([name in p['name'] for p in meta])
class BootstrapQCPostProcessor(BootstrapPlugin): ''' Sets up one QC Post Processing worker and initiates the Scheduler Service's interval every 24 hours. ''' def on_initial_bootstrap(self, process, config, **kwargs): if os.environ.get('PYCC_MODE'): # This environment is an ion integration test log.info('PYCC_MODE: skipping qc_post_processor launch') return if self.process_exists(process, 'qc_post_processor'): # Short circuit the bootstrap to make sure not more than one is ever started return self.scheduler_service = SchedulerServiceProcessClient(process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient( process=process) self.run_interval = CFG.get_safe('service.qc_processing.run_interval', 24) interval_key = uuid4().hex # Unique identifier for this process config = DotDict() config.process.interval_key = interval_key process_definition = ProcessDefinition( name='qc_post_processor', executable={ 'module': 'ion.processes.data.transforms.qc_post_processing', 'class': 'QCPostProcessing' }) process_definition_id = self.process_dispatcher.create_process_definition( process_definition) process_id = self.process_dispatcher.create_process( process_definition_id) self.process_dispatcher.schedule_process(process_definition_id, process_id=process_id, configuration=config) timer_id = self.scheduler_service.create_interval_timer( start_time=str(time.time()), end_time='-1', #Run FOREVER interval=3600 * self.run_interval, event_origin=interval_key) def process_exists(self, process, name): proc_ids, meta = process.container.resource_registry.find_resources( restype=RT.Process, id_only=True) return any([name in p['name'] for p in meta if p['name']])
class BootstrapQCPostProcessor(BootstrapPlugin): """ Sets up one QC Post Processing worker and initiates the Scheduler Service's interval every 24 hours. """ def on_initial_bootstrap(self, process, config, **kwargs): if os.environ.get("PYCC_MODE"): # This environment is an ion integration test log.info("PYCC_MODE: skipping qc_post_processor launch") return if self.process_exists(process, "qc_post_processor"): # Short circuit the bootstrap to make sure not more than one is ever started return self.scheduler_service = SchedulerServiceProcessClient(process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(process=process) self.run_interval = CFG.get_safe("service.qc_processing.run_interval", 24) interval_key = uuid4().hex # Unique identifier for this process config = DotDict() config.process.interval_key = interval_key process_definition = ProcessDefinition( name="qc_post_processor", executable={"module": "ion.processes.data.transforms.qc_post_processing", "class": "QCPostProcessing"}, ) process_definition_id = self.process_dispatcher.create_process_definition(process_definition) process_id = self.process_dispatcher.create_process(process_definition_id) self.process_dispatcher.schedule_process(process_definition_id, process_id=process_id, configuration=config) timer_id = self.scheduler_service.create_interval_timer( start_time=str(time.time()), end_time="-1", # Run FOREVER interval=3600 * self.run_interval, event_origin=interval_key, ) def process_exists(self, process, name): proc_ids, meta = process.container.resource_registry.find_resources(restype=RT.Process, id_only=True) return any([name in p["name"] for p in meta if p["name"]])
class BootstrapQCPostProcessor(BootstrapPlugin): ''' Sets up one QC Post Processing worker and initiates the Scheduler Service's interval every 24 hours. ''' def on_initial_bootstrap(self, process, config, **kwargs): # TODO: Temporary skip while refactoring QC work for M088 return if os.environ.get('PYCC_MODE'): # This environment is an ion integration test log.info('PYCC_MODE: skipping qc_post_processor launch') return if self.process_exists(process, 'qc_post_processor'): # Short circuit the bootstrap to make sure not more than one is ever started return self.scheduler_service = SchedulerServiceProcessClient(process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(process=process) self.run_interval = CFG.get_safe('service.qc_processing.run_interval', 24) interval_key = uuid4().hex # Unique identifier for this process config = DotDict() config.process.interval_key = interval_key process_definition = ProcessDefinition(name='qc_post_processor', executable={'module':'ion.processes.data.transforms.qc_post_processing', 'class':'QCPostProcessing'}) process_definition_id = self.process_dispatcher.create_process_definition(process_definition) process_id = self.process_dispatcher.create_process(process_definition_id) self.process_dispatcher.schedule_process(process_definition_id, process_id=process_id, configuration=config) timer_id = self.scheduler_service.create_interval_timer(start_time=str(time.time()), end_time='-1', #Run FOREVER interval=3600*self.run_interval, event_origin=interval_key) def process_exists(self, process, name): proc_ids, meta = process.container.resource_registry.find_resources(restype=RT.Process, id_only=True) return any([name in p['name'] for p in meta if p['name']])
class BootstrapProcessDispatcher(BootstrapPlugin): """ Bootstrap process for process dispatcher. """ def on_initial_bootstrap(self, process, config, **kwargs): self.pds_client = ProcessDispatcherServiceProcessClient( process=process) self.resource_registry = ResourceRegistryServiceProcessClient( process=process) self.ingestion_worker(process, config) self.replay_defs(process, config) self.notification_worker(process, config) self.registration_worker(process, config) self.pydap_server(process, config) self.eoi_services(process, config) def eoi_services(self, process, config): eoi_module = config.get_safe( 'bootstrap.processes.registration.module', 'ion.processes.data.registration.eoi_registration_process') eoi_class = config.get_safe('bootstrap.processes.registration.class', 'EOIRegistrationProcess') process_definition = ProcessDefinition( name='eoi_server', description='Process for eoi data sources') process_definition.executable['module'] = eoi_module process_definition.executable['class'] = eoi_class self._create_and_launch(process_definition) def pydap_server(self, process, config): pydap_module = config.get_safe( 'bootstrap.processes.pydap.module', 'ion.processes.data.externalization.lightweight_pydap') pydap_class = config.get_safe('bootstrap.processes.pydap.class', 'LightweightPyDAP') use_pydap = config.get_safe('bootstrap.launch_pydap', False) process_definition = ProcessDefinition( name='pydap_server', description='Lightweight WSGI Server for PyDAP') process_definition.executable['module'] = pydap_module process_definition.executable['class'] = pydap_class self._create_and_launch(process_definition, use_pydap) def registration_worker(self, process, config): res, meta = self.resource_registry.find_resources( name='registration_worker', restype=RT.ProcessDefinition) if len(res): return registration_module = config.get_safe( 'bootstrap.processes.registration.module', 'ion.processes.data.registration.registration_process') registration_class = config.get_safe( 'bootstrap.processes.registration.class', 'RegistrationProcess') use_pydap = True process_definition = ProcessDefinition( name='registration_worker', description='For registering datasets with ERDDAP') process_definition.executable['module'] = registration_module process_definition.executable['class'] = registration_class self._create_and_launch(process_definition, use_pydap) def _create_and_launch(self, process_definition, conditional=True): proc_def_id = self.pds_client.create_process_definition( process_definition=process_definition) if conditional: process_res_id = self.pds_client.create_process( process_definition_id=proc_def_id) self.pds_client.schedule_process(process_definition_id=proc_def_id, process_id=process_res_id) def ingestion_worker(self, process, config): # ingestion ingestion_module = config.get_safe( 'bootstrap.processes.ingestion.module', 'ion.processes.data.ingestion.science_granule_ingestion_worker') ingestion_class = config.get_safe( 'bootstrap.processes.ingestion.class', 'ScienceGranuleIngestionWorker') ingestion_datastore = config.get_safe( 'bootstrap.processes.ingestion.datastore_name', 'datasets') ingestion_queue = config.get_safe( 'bootstrap.processes.ingestion.queue', 'science_granule_ingestion') ingestion_workers = config.get_safe( 'bootstrap.processes.ingestion.workers', 1) #-------------------------------------------------------------------------------- # Create ingestion workers #-------------------------------------------------------------------------------- process_definition = ProcessDefinition( name='ingestion_worker_process', description='Worker transform process for ingestion of datasets') process_definition.executable['module'] = ingestion_module process_definition.executable['class'] = ingestion_class ingestion_procdef_id = self.pds_client.create_process_definition( process_definition=process_definition) #-------------------------------------------------------------------------------- # Simulate a HA ingestion worker by creating two of them #-------------------------------------------------------------------------------- # config = DotDict() # config.process.datastore_name = ingestion_datastore # config.process.queue_name = ingestion_queue # # for i in xrange(ingestion_workers): # self.pds_client.schedule_process(process_definition_id=ingestion_procdef_id, configuration=config) def notification_worker(self, process, config): # user notifications notification_module = config.get_safe( 'bootstrap.processes.user_notification.module', 'ion.processes.data.transforms.notification_worker') notification_class = config.get_safe( 'bootstrap.processes.user_notification.class', 'NotificationWorker') notification_workers = config.get_safe( 'bootstrap.processes.user_notification.workers', 1) #-------------------------------------------------------------------------------- # Create notification workers #-------------------------------------------------------------------------------- # set up the process definition process_definition_uns = ProcessDefinition( name='notification_worker_process', description='Worker transform process for user notifications') process_definition_uns.executable['module'] = notification_module process_definition_uns.executable['class'] = notification_class uns_procdef_id = self.pds_client.create_process_definition( process_definition=process_definition_uns) config = DotDict() config.process.type = 'simple' for i in xrange(notification_workers): config.process.name = 'notification_worker_%s' % i config.process.queue_name = 'notification_worker_queue' self.pds_client.schedule_process( process_definition_id=uns_procdef_id, configuration=config) def replay_defs(self, process, config): replay_module = config.get_safe( 'bootstrap.processes.replay.module', 'ion.processes.data.replay.replay_process') replay_class = config.get_safe('bootstrap.processes.replay.class', 'ReplayProcess') #-------------------------------------------------------------------------------- # Create replay process definition #-------------------------------------------------------------------------------- process_definition = ProcessDefinition( name=DataRetrieverService.REPLAY_PROCESS, description='Process for the replay of datasets') process_definition.executable['module'] = replay_module process_definition.executable['class'] = replay_class self.pds_client.create_process_definition( process_definition=process_definition) def on_restart(self, process, config, **kwargs): pass
class BootstrapProcessDispatcher(BootstrapPlugin): """ Bootstrap process for process dispatcher. """ def on_initial_bootstrap(self, process, config, **kwargs): self.pds_client = ProcessDispatcherServiceProcessClient(process=process) self.resource_registry = ResourceRegistryServiceProcessClient(process=process) self.ingestion_worker(process,config) self.replay_defs(process,config) self.notification_worker(process,config) self.registration_worker(process,config) self.pydap_server(process,config) def pydap_server(self, process, config): pydap_module = config.get_safe('bootstrap.processes.pydap.module', 'ion.processes.data.externalization.lightweight_pydap') pydap_class = config.get_safe('bootstrap.processes.pydap.class', 'LightweightPyDAP') use_pydap = config.get_safe('bootstrap.launch_pydap', False) process_definition = ProcessDefinition( name = 'pydap_server', description = 'Lightweight WSGI Server for PyDAP') process_definition.executable['module'] = pydap_module process_definition.executable['class'] = pydap_class self._create_and_launch(process_definition,use_pydap) def registration_worker(self, process, config): res, meta = self.resource_registry.find_resources(name='registration_worker', restype=RT.ProcessDefinition) if len(res): return registration_module = config.get_safe('bootstrap.processes.registration.module', 'ion.processes.data.registration.registration_process') registration_class = config.get_safe('bootstrap.processes.registration.class', 'RegistrationProcess') use_pydap = config.get_safe('bootstrap.use_pydap', False) process_definition = ProcessDefinition( name='registration_worker', description='For registering datasets with ERDDAP') process_definition.executable['module'] = registration_module process_definition.executable['class'] = registration_class self._create_and_launch(process_definition, use_pydap) def _create_and_launch(self, process_definition, conditional=True): proc_def_id = self.pds_client.create_process_definition(process_definition=process_definition) if conditional: process_res_id = self.pds_client.create_process(process_definition_id=proc_def_id) self.pds_client.schedule_process(process_definition_id=proc_def_id, process_id=process_res_id) def ingestion_worker(self, process, config): # ingestion ingestion_module = config.get_safe('bootstrap.processes.ingestion.module','ion.processes.data.ingestion.science_granule_ingestion_worker') ingestion_class = config.get_safe('bootstrap.processes.ingestion.class' ,'ScienceGranuleIngestionWorker') ingestion_datastore = config.get_safe('bootstrap.processes.ingestion.datastore_name', 'datasets') ingestion_queue = config.get_safe('bootstrap.processes.ingestion.queue' , 'science_granule_ingestion') ingestion_workers = config.get_safe('bootstrap.processes.ingestion.workers', 1) #-------------------------------------------------------------------------------- # Create ingestion workers #-------------------------------------------------------------------------------- process_definition = ProcessDefinition( name='ingestion_worker_process', description='Worker transform process for ingestion of datasets') process_definition.executable['module']= ingestion_module process_definition.executable['class'] = ingestion_class ingestion_procdef_id = self.pds_client.create_process_definition(process_definition=process_definition) #-------------------------------------------------------------------------------- # Simulate a HA ingestion worker by creating two of them #-------------------------------------------------------------------------------- # config = DotDict() # config.process.datastore_name = ingestion_datastore # config.process.queue_name = ingestion_queue # # for i in xrange(ingestion_workers): # self.pds_client.schedule_process(process_definition_id=ingestion_procdef_id, configuration=config) def notification_worker(self, process, config): # user notifications notification_module = config.get_safe('bootstrap.processes.user_notification.module','ion.processes.data.transforms.notification_worker') notification_class = config.get_safe('bootstrap.processes.user_notification.class' ,'NotificationWorker') notification_workers = config.get_safe('bootstrap.processes.user_notification.workers', 1) #-------------------------------------------------------------------------------- # Create notification workers #-------------------------------------------------------------------------------- # set up the process definition process_definition_uns = ProcessDefinition( name='notification_worker_process', description='Worker transform process for user notifications') process_definition_uns.executable['module']= notification_module process_definition_uns.executable['class'] = notification_class uns_procdef_id = self.pds_client.create_process_definition(process_definition=process_definition_uns) config = DotDict() config.process.type = 'simple' for i in xrange(notification_workers): config.process.name = 'notification_worker_%s' % i config.process.queue_name = 'notification_worker_queue' self.pds_client.schedule_process(process_definition_id=uns_procdef_id, configuration=config) def replay_defs(self, process, config): replay_module = config.get_safe('bootstrap.processes.replay.module', 'ion.processes.data.replay.replay_process') replay_class = config.get_safe('bootstrap.processes.replay.class' , 'ReplayProcess') #-------------------------------------------------------------------------------- # Create replay process definition #-------------------------------------------------------------------------------- process_definition = ProcessDefinition(name=DataRetrieverService.REPLAY_PROCESS, description='Process for the replay of datasets') process_definition.executable['module']= replay_module process_definition.executable['class'] = replay_class self.pds_client.create_process_definition(process_definition=process_definition) def on_restart(self, process, config, **kwargs): pass