def _get_param_dicts_from_streams(self): pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_def_cond = pubsub.read_stream_definition( stream_id=self.cond_stream) self.stream_def_pres = pubsub.read_stream_definition( stream_id=self.pres_stream) self.stream_def_temp = pubsub.read_stream_definition( stream_id=self.temp_stream) self.params = {} self.params['conductivity'] = self.stream_def_cond._id self.params['pressure'] = self.stream_def_pres._id self.params['temperature'] = self.stream_def_temp._id
def on_start(self): ''' Starts the process ''' log.info('Replay Process Started') super(ReplayProcess,self).on_start() dsm_cli = DatasetManagementServiceProcessClient(process=self) pubsub = PubsubManagementServiceProcessClient(process=self) self.dataset_id = self.CFG.get_safe('process.dataset_id', None) self.delivery_format = self.CFG.get_safe('process.delivery_format',{}) self.start_time = self.CFG.get_safe('process.query.start_time', None) self.end_time = self.CFG.get_safe('process.query.end_time', None) self.stride_time = self.CFG.get_safe('process.query.stride_time', None) self.parameters = self.CFG.get_safe('process.query.parameters',None) self.publish_limit = self.CFG.get_safe('process.query.publish_limit', 10) self.tdoa = self.CFG.get_safe('process.query.tdoa',None) self.stream_id = self.CFG.get_safe('process.publish_streams.output', '') self.stream_def = pubsub.read_stream_definition(stream_id=self.stream_id) self.stream_def_id = self.stream_def._id self.replay_thread = None self.publishing.clear() self.play.set() self.end.clear() if self.dataset_id is None: raise BadRequest('dataset_id not specified') self.dataset = dsm_cli.read_dataset(self.dataset_id) self.pubsub = PubsubManagementServiceProcessClient(process=self)
def on_start(self): self.pdict = None stream_id = self.CFG.get_safe('process.stream_id') pubsub_cli = PubsubManagementServiceProcessClient(process=self) self.stream_def = pubsub_cli.read_stream_definition( stream_id=stream_id) super(BetterDataProducer, self).on_start()
def on_start(self): ''' Starts the process ''' log.info('Replay Process Started') super(ReplayProcess,self).on_start() dsm_cli = DatasetManagementServiceProcessClient(process=self) pubsub = PubsubManagementServiceProcessClient(process=self) self.dataset_id = self.CFG.get_safe('process.dataset_id', None) self.delivery_format = self.CFG.get_safe('process.delivery_format',{}) self.start_time = self.CFG.get_safe('process.query.start_time', None) self.end_time = self.CFG.get_safe('process.query.end_time', None) self.stride_time = self.CFG.get_safe('process.query.stride_time', None) self.parameters = self.CFG.get_safe('process.query.parameters',None) self.publish_limit = self.CFG.get_safe('process.query.publish_limit', 10) self.tdoa = self.CFG.get_safe('process.query.tdoa',None) self.stream_id = self.CFG.get_safe('process.publish_streams.output', '') self.stream_def = pubsub.read_stream_definition(stream_id=self.stream_id) self.stream_def_id = self.stream_def._id self.publishing.clear() self.play.set() self.end.clear() if self.dataset_id is None: raise BadRequest('dataset_id not specified') self.dataset = dsm_cli.read_dataset(self.dataset_id) self.pubsub = PubsubManagementServiceProcessClient(process=self)
def on_start(self): super(SalinityTransform, self).on_start() self.sal_stream = self.CFG.process.publish_streams.values()[0] # Read the parameter dict from the stream def of the stream pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_definition = pubsub.read_stream_definition(stream_id=self.sal_stream)
def on_start(self): super(EventTriggeredTransform_A, self).on_start() self.awake = False self.cond_stream = self.CFG.process.publish_streams.values()[0] # Read the parameter dict from the stream def of the stream pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_definition = pubsub.read_stream_definition(stream_id=self.cond_stream)
def on_start(self): super(EventTriggeredTransform_A, self).on_start() self.awake = False self.cond_stream = self.CFG.process.publish_streams.values()[0] # Read the parameter dict from the stream def of the stream pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_definition = pubsub.read_stream_definition( stream_id=self.cond_stream)
def on_start(self): super(CTDL1ConductivityTransform, self).on_start() if not self.CFG.process.publish_streams.has_key('conductivity'): raise BadRequest("For CTD transforms, please send the stream_id using " "a special keyword (ex: conductivity)") self.cond_stream = self.CFG.process.publish_streams.conductivity # Read the parameter dict from the stream def of the stream pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_definition = pubsub.read_stream_definition(stream_id=self.cond_stream)
def on_start(self): super(DensityTransform, self).on_start() if not self.CFG.process.publish_streams.has_key('density'): raise BadRequest("For CTD transforms, please send the stream_id " "using a special keyword (ex: density)") self.dens_stream = self.CFG.process.publish_streams.density # Read the parameter dict from the stream def of the stream pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_definition = pubsub.read_stream_definition(stream_id=self.dens_stream)
def on_start(self): super(CTDL1TemperatureTransform, self).on_start() if not self.CFG.process.publish_streams.has_key('temperature'): raise BadRequest("For CTD transforms, please send the stream_id using a " "special keyword (ex: temperature)") self.temp_stream = self.CFG.process.publish_streams.temperature # Read the parameter dict from the stream def of the stream pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_definition = pubsub.read_stream_definition(stream_id=self.temp_stream)
class VizTransformGoogleDT(TransformDataProcess): """ This class is used for converting incoming data from CDM format to JSON style Google DataTables Note: One behaviour that this class is expected to achieve specifically is to determine if its supposed to work as a realtime transform (exists indefinitely and maintains a sliding window of data) or as a replay transform (one-shot). [2] This transform behaves as an instantaneous forwarder. There is no waiting for the entire stream to create the complete datatable. As the granules come in, they are translated to the datatable 'components'. Components, because we are not creating the actual datatable in this code. That's the job of the viz service to put all the components of a datatable together in JSON format before sending it to the client [3] The time stamp in the incoming stream can't be converted to the datetime object here because the Raw stream definition only expects regular primitives (strings, floats, ints etc) Usage: https://gist.github.com/3834918 """ output_bindings = ['google_dt_components'] def __init__(self): super(VizTransformGoogleDT, self).__init__() def on_start(self): self.pubsub_management = PubsubManagementServiceProcessClient(process=self) self.stream_info = self.CFG.get_safe('process.publish_streams', {}) self.stream_names = self.stream_info.keys() self.stream_ids = self.stream_info.values() if not self.stream_names: raise BadRequest('Google DT Transform has no output streams.') super(VizTransformGoogleDT,self).on_start() def recv_packet(self, packet, in_stream_route, in_stream_id): log.info('Received packet') outgoing = VizTransformGoogleDTAlgorithm.execute(packet, params=self.get_stream_definition()) for stream_name in self.stream_names: publisher = getattr(self, stream_name) publisher.publish(outgoing) def get_stream_definition(self): stream_id = self.stream_ids[0] self.stream_def = self.pubsub_management.read_stream_definition(stream_id=stream_id) return self.stream_def._id
def on_start(self): super(PresfL1Transform, self).on_start() if not self.CFG.process.publish_streams.has_key('seafloor_pressure'): raise BadRequest("For the PresfL1Transform, please send the stream_id using " "a special keyword (ex: seafloor_pressure)") self.pres_stream = self.CFG.process.publish_streams.seafloor_pressure # Read the parameter dict from the stream def of the stream pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_definition = pubsub.read_stream_definition(stream_id=self.pres_stream)
def on_start(self): super(CTDL1PressureTransform, self).on_start() if not self.CFG.process.publish_streams.has_key('pressure'): raise BadRequest("For CTD transforms, please send the stream_id using " "a special keyword (ex: pressure)") self.pres_stream = self.CFG.process.publish_streams.pressure # Read the parameter dict from the stream def of the stream pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_definition = pubsub.read_stream_definition(stream_id=self.pres_stream)
def on_start(self): super(CTDBP_L0_all, self).on_start() self.L0_stream_id = self.CFG.process.publish_streams.values()[0] log.debug("the output stream: %s", self.L0_stream_id) pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_def_L0 = pubsub.read_stream_definition(stream_id=self.L0_stream_id) self.params = {'L0_stream' : self.stream_def_L0._id } log.debug("the params: %s", self.params)
def on_start(self): super(CTDBP_L1_Transform, self).on_start() self.L1_stream_id = self.CFG.process.publish_streams.values()[0] # Read the parameter dict from the stream def of the stream pubsub = PubsubManagementServiceProcessClient(process=self) stream_def = pubsub.read_stream_definition(stream_id=self.L1_stream_id) self.stream_definition_id = stream_def._id self.temp_calibration_coeffs = self.CFG.process.calibration_coeffs['temp_calibration_coeffs'] self.pres_calibration_coeffs = self.CFG.process.calibration_coeffs['pres_calibration_coeffs'] self.cond_calibration_coeffs = self.CFG.process.calibration_coeffs['cond_calibration_coeffs']
def on_start(self): pubsub = PubsubManagementServiceProcessClient(process=self) if self.CFG.process.publish_streams.has_key("salinity"): self.sal_stream = self.CFG.process.publish_streams.salinity elif self.CFG.process.publish_streams.has_key("output"): self.sal_stream = self.CFG.process.publish_streams.output self.salinity = self.output self.CFG.process.stream_id = self.sal_stream self.stream_id = self.sal_stream self.stream_def = pubsub.read_stream_definition(stream_id=self.sal_stream) super(SalinityDoubler, self).on_start()
class VizTransformGoogleDT(TransformDataProcess): """ This class is used for converting incoming data from CDM format to JSON style Google DataTables Note: One behaviour that this class is expected to achieve specifically is to determine if its supposed to work as a realtime transform (exists indefinitely and maintains a sliding window of data) or as a replay transform (one-shot). [2] This transform behaves as an instantaneous forwarder. There is no waiting for the entire stream to create the complete datatable. As the granules come in, they are translated to the datatable 'components'. Components, because we are not creating the actual datatable in this code. That's the job of the viz service to put all the components of a datatable together in JSON format before sending it to the client [3] The time stamp in the incoming stream can't be converted to the datetime object here because the Raw stream definition only expects regular primitives (strings, floats, ints etc) Usage: https://gist.github.com/3834918 """ output_bindings = ['google_dt_components'] def __init__(self): super(VizTransformGoogleDT, self).__init__() def on_start(self): self.pubsub_management = PubsubManagementServiceProcessClient( process=self) self.stream_info = self.CFG.get_safe('process.publish_streams', {}) self.stream_names = self.stream_info.keys() self.stream_ids = self.stream_info.values() if not self.stream_names: raise BadRequest('Google DT Transform has no output streams.') super(VizTransformGoogleDT, self).on_start() def recv_packet(self, packet, in_stream_route, in_stream_id): log.info('Received packet') outgoing = VizTransformGoogleDTAlgorithm.execute( packet, params=self.get_stream_definition()) for stream_name in self.stream_names: publisher = getattr(self, stream_name) publisher.publish(outgoing) def get_stream_definition(self): stream_id = self.stream_ids[0] self.stream_def = self.pubsub_management.read_stream_definition( stream_id=stream_id) return self.stream_def._id
def on_start(self): super(PresfL0Splitter, self).on_start() if not self.CFG.process.publish_streams.has_key('absolute_pressure'): raise BadRequest( "For the PresfL0Splitter, please send the stream_id using " "a special keyword (ex: absolute_pressure)") self.pres_stream = self.CFG.process.publish_streams.absolute_pressure # Read the parameter dict from the stream def of the stream pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_definition = pubsub.read_stream_definition( stream_id=self.pres_stream)
def on_start(self): super(CTDBP_L0_all, self).on_start() self.L0_stream_id = self.CFG.process.publish_streams.values()[0] log.debug("the output stream: %s", self.L0_stream_id) pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_def_L0 = pubsub.read_stream_definition( stream_id=self.L0_stream_id) self.params = {'L0_stream': self.stream_def_L0._id} log.debug("the params: %s", self.params)
def on_start(self): super(EventTriggeredTransform_B, self).on_start() self.awake = False if not self.CFG.process.publish_streams.has_key('output'): raise BadRequest("For event triggered transform, please send the stream_id " "using the special keyword, output") self.output = self.CFG.process.publish_streams.output # Read the parameter dict from the stream def of the stream pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_definition = pubsub.read_stream_definition(stream_id=self.output)
def on_start(self): pubsub = PubsubManagementServiceProcessClient(process=self) if self.CFG.process.publish_streams.has_key('salinity'): self.sal_stream = self.CFG.process.publish_streams.salinity elif self.CFG.process.publish_streams.has_key('output'): self.sal_stream = self.CFG.process.publish_streams.output self.salinity = self.output self.CFG.process.stream_id = self.sal_stream self.stream_id = self.sal_stream self.stream_def = pubsub.read_stream_definition( stream_id=self.sal_stream) super(SalinityDoubler, self).on_start()
def on_start(self): super(SimpleCtdPublisher,self).on_start() pubsub_cli = PubsubManagementServiceProcessClient(process=self) self.stream_id = self.CFG.get_safe('process.stream_id',{}) self.interval = self.CFG.get_safe('process.interval', 1.0) #self.last_time = self.CFG.get_safe('process.last_time', 0) self.stream_def = pubsub_cli.read_stream_definition(stream_id=self.stream_id) self.pdict = self.stream_def.parameter_dictionary self.finished = gevent.event.Event() self.greenlet = gevent.spawn(self.publish_loop) self._stats['publish_count'] = 0 log.info('SimpleCTDPublisher started, publishing to %s', self.publisher.stream_route.__dict__)
def on_start(self): super(CTDBP_L0_all, self).on_start() if not self.CFG.process.publish_streams.has_key('L0_stream'): raise BadRequest("For CTD transforms, please send the stream_id for the L0_stream using " "a special keyword (L0_stream)") self.L0_stream_id = self.CFG.process.publish_streams.L0_stream log.debug("the output stream: %s", self.L0_stream_id) pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_def_L0 = pubsub.read_stream_definition(stream_id=self.L0_stream_id) self.params = {'L0_stream' : self.stream_def_L0._id } log.debug("the params: %s", self.params)
def on_start(self): super(SimpleCtdPublisher, self).on_start() pubsub_cli = PubsubManagementServiceProcessClient(process=self) self.stream_id = self.CFG.get_safe('process.stream_id', {}) self.interval = self.CFG.get_safe('process.interval', 1.0) #self.last_time = self.CFG.get_safe('process.last_time', 0) self.stream_def = pubsub_cli.read_stream_definition( stream_id=self.stream_id) self.pdict = self.stream_def.parameter_dictionary self.finished = gevent.event.Event() self.greenlet = gevent.spawn(self.publish_loop) self._stats['publish_count'] = 0 log.info('SimpleCTDPublisher started, publishing to %s', self.publisher.stream_route.__dict__)
def on_start(self): super(EventTriggeredTransform_B, self).on_start() self.awake = False if not self.CFG.process.publish_streams.has_key('output'): raise BadRequest( "For event triggered transform, please send the stream_id " "using the special keyword, output") self.output = self.CFG.process.publish_streams.output # Read the parameter dict from the stream def of the stream pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_definition = pubsub.read_stream_definition( stream_id=self.output)
def on_start(self): super(CTDBP_L1_Transform, self).on_start() self.L1_stream_id = self.CFG.process.publish_streams.values()[0] # Read the parameter dict from the stream def of the stream pubsub = PubsubManagementServiceProcessClient(process=self) stream_def = pubsub.read_stream_definition(stream_id=self.L1_stream_id) self.stream_definition_id = stream_def._id self.temp_calibration_coeffs = self.CFG.process.calibration_coeffs[ 'temp_calibration_coeffs'] self.pres_calibration_coeffs = self.CFG.process.calibration_coeffs[ 'pres_calibration_coeffs'] self.cond_calibration_coeffs = self.CFG.process.calibration_coeffs[ 'cond_calibration_coeffs']
def on_start(self): super(CTDBP_DensityTransform, self).on_start() self.dens_stream_id = self.CFG.process.publish_streams.values()[0] lat = self.CFG.get_safe('process.lat',None) if lat is None: raise BadRequest('Latitude is required to determine density') lon = self.CFG.get_safe('process.lon',None) if lon is None: raise BadRequest('Longitude is required to determine density') # Read the parameter dict from the stream def of the stream pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_definition = pubsub.read_stream_definition(stream_id=self.dens_stream_id) self.params = {'stream_def' : self.stream_definition._id, 'lat': lat, 'lon' : lon}
def on_start(self): super(CTDBP_L1_Transform, self).on_start() # Validate the CFG used to launch the transform has all the required fields if not self.CFG.process.publish_streams.has_key('L1_stream'): raise BadRequest("For CTDBP transforms, please send the stream_id for the L1_stream using " "a special keyword (L1_stream)") self.L1_stream_id = self.CFG.process.publish_streams.L1_stream # Read the parameter dict from the stream def of the stream pubsub = PubsubManagementServiceProcessClient(process=self) stream_def = pubsub.read_stream_definition(stream_id=self.L1_stream_id) self.stream_definition_id = stream_def._id self.temp_calibration_coeffs = self.CFG.process.calibration_coeffs['temp_calibration_coeffs'] self.pres_calibration_coeffs = self.CFG.process.calibration_coeffs['pres_calibration_coeffs'] self.cond_calibration_coeffs = self.CFG.process.calibration_coeffs['cond_calibration_coeffs']
def on_start(self): super(CTDBP_L1_Transform, self).on_start() # Validate the CFG used to launch the transform has all the required fields if not self.CFG.process.publish_streams.has_key('L1_stream'): raise BadRequest("For CTDBP transforms, please send the stream_id for the L1_stream using " "a special keyword (L1_stream)") self.L1_stream_id = self.CFG.process.publish_streams.L1_stream calibration_coeffs= self.CFG.process.calibration_coeffs # Read the parameter dict from the stream def of the stream pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_definition = pubsub.read_stream_definition(stream_id=self.L1_stream_id) self.params = {} self.params['stream_def_id'] = self.stream_definition._id self.params['calibration_coeffs'] = calibration_coeffs
def on_start(self): super(CTDBP_DensityTransform, self).on_start() if not self.CFG.process.publish_streams.has_key('density'): raise BadRequest("For CTD transforms, please send the stream_id using a special keyword (ex: density)") self.dens_stream_id = self.CFG.process.publish_streams.density lat = self.CFG.get_safe('process.lat',None) if lat is None: raise BadRequest('Latitude is required to determine density') lon = self.CFG.get_safe('process.lon',None) if lon is None: raise BadRequest('Lonitude is required to determine density') # Read the parameter dict from the stream def of the stream pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_definition = pubsub.read_stream_definition(stream_id=self.dens_stream_id) self.params = {'stream_def' : self.stream_definition._id, 'lat': lat, 'lon' : lon}
def on_start(self): super(CTDBP_DensityTransform, self).on_start() if not self.CFG.process.publish_streams.has_key('density'): raise BadRequest("For CTD transforms, please send the stream_id " "using a special keyword (ex: density)") self.dens_stream_id = self.CFG.process.publish_streams.density lat = self.CFG.get_safe('process.lat',None) if lat is None: raise BadRequest('Latitude is required to determine density') lon = self.CFG.get_safe('process.lon',None) if lon is None: raise BadRequest('Lonitude is required to determine density') # Read the parameter dict from the stream def of the stream pubsub = PubsubManagementServiceProcessClient(process=self) self.stream_definition = pubsub.read_stream_definition(stream_id=self.dens_stream_id) self.params = {'stream_def' : self.stream_definition._id, 'lat': lat, 'lon' : lon}
class TransformPrime(TransformDataProcess): binding=['output'] ''' Transforms which have an incoming stream and an outgoing stream. Parameters: process.stream_id Outgoing stream identifier. process.exchange_point Route's exchange point. process.routing_key Route's routing key. process.queue_name Name of the queue to listen on. process.routes streams,actor for each route {(stream_input_id, stream_output_id):actor} Either the stream_id or both the exchange_point and routing_key need to be provided. ''' def on_start(self): TransformDataProcess.on_start(self) self.pubsub_management = PubsubManagementServiceProcessClient(process=self) @memoize_lru(100) def read_stream_def(self,stream_id): return self.pubsub_management.read_stream_definition(stream_id=stream_id) def recv_packet(self, msg, stream_route, stream_id): process_routes = self.CFG.get_safe('process.routes', {}) for streams,actor in process_routes.iteritems(): stream_in_id, stream_out_id = streams if stream_id == stream_in_id: if actor is None: rdt_out = self._execute_transform(msg, streams) else: rdt_out = self._execute_actor(msg, actor, streams) self.publish(rdt_out.to_granule(), stream_out_id) def publish(self, msg, stream_out_id): publisher = getattr(self, stream_out_id) publisher.publish(msg) def _execute_actor(self, msg, actor, streams): stream_in_id,stream_out_id = streams stream_def_out = self._read_stream_def(stream_out_id) #do the stuff with the actor rdt_out = RecordDictionaryTool(stream_definition_id=stream_def_out._id) return rdt_out def _execute_transform(self, msg, streams): stream_in_id,stream_out_id = streams stream_def_in = self.read_stream_def(stream_in_id) stream_def_out = self.read_stream_def(stream_out_id) incoming_pdict_dump = stream_def_in.parameter_dictionary outgoing_pdict_dump = stream_def_out.parameter_dictionary incoming_pdict = ParameterDictionary.load(incoming_pdict_dump) outgoing_pdict = ParameterDictionary.load(outgoing_pdict_dump) merged_pdict = ParameterDictionary() for k,v in incoming_pdict.iteritems(): ordinal, v = v if k not in merged_pdict: merged_pdict.add_context(v) for k,v in outgoing_pdict.iteritems(): ordinal, v = v if k not in merged_pdict: merged_pdict.add_context(v) rdt_temp = RecordDictionaryTool(param_dictionary=merged_pdict) rdt_in = RecordDictionaryTool.load_from_granule(msg) for field in rdt_temp.fields: if not isinstance(rdt_temp._pdict.get_context(field).param_type, ParameterFunctionType): try: rdt_temp[field] = rdt_in[field] except KeyError: pass for field in rdt_temp.fields: if isinstance(rdt_temp._pdict.get_context(field).param_type, ParameterFunctionType): rdt_temp[field] = rdt_temp[field] rdt_out = RecordDictionaryTool(stream_definition_id=stream_def_out._id) for field in rdt_out.fields: rdt_out[field] = rdt_temp[field] return rdt_out
def on_start(self): self.pdict = None stream_id = self.CFG.get_safe('process.stream_id') pubsub_cli = PubsubManagementServiceProcessClient(process=self) self.stream_def = pubsub_cli.read_stream_definition(stream_id=stream_id) super(CCDataProducer,self).on_start()
class TransformPrime(TransformDataProcess): binding=['output'] ''' Transforms which have an incoming stream and an outgoing stream. Parameters: process.stream_id Outgoing stream identifier. process.exchange_point Route's exchange point. process.routing_key Route's routing key. process.queue_name Name of the queue to listen on. process.routes streams,actor for each route {(stream_input_id, stream_output_id):actor} Either the stream_id or both the exchange_point and routing_key need to be provided. ''' def on_start(self): TransformDataProcess.on_start(self) self.pubsub_management = PubsubManagementServiceProcessClient(process=self) @memoize_lru(maxsize=100) def _read_stream_def(self, stream_id): return self.pubsub_management.read_stream_definition(stream_id=stream_id) def recv_packet(self, msg, stream_route, stream_id): process_routes = self.CFG.get_safe('process.routes', {}) for streams,actor in process_routes.iteritems(): stream_in_id, stream_out_id = streams if stream_id == stream_in_id: if actor is None: rdt_out = self._execute_transform(msg, streams) else: rdt_out = self._execute_actor(msg, actor, streams) self.publish(rdt_out.to_granule(), stream_out_id) def publish(self, msg, stream_out_id): publisher = getattr(self, stream_out_id) publisher.publish(msg) def _execute_actor(self, msg, actor, streams): stream_in_id,stream_out_id = streams stream_def_out = self._read_stream_def(stream_out_id) #do the stuff with the actor rdt_out = RecordDictionaryTool(stream_definition_id=stream_def_out._id) return rdt_out def _execute_transform(self, msg, streams): stream_in_id,stream_out_id = streams stream_def_in = self._read_stream_def(stream_in_id) stream_def_out = self._read_stream_def(stream_out_id) incoming_pdict_dump = stream_def_in.parameter_dictionary outgoing_pdict_dump = stream_def_out.parameter_dictionary incoming_pdict = ParameterDictionary.load(incoming_pdict_dump) outgoing_pdict = ParameterDictionary.load(outgoing_pdict_dump) merged_pdict = dict([(k,v) for k,v in incoming_pdict.iteritems()] + [(k,v) for k,v in outgoing_pdict.iteritems()]) rdt_in = RecordDictionaryTool.load_from_granule(msg) rdt_out = RecordDictionaryTool(stream_definition_id=stream_def_out._id) #modify the shape of the rdt out since we are using _rd then _shp will never get set rdt_out._shp = rdt_in._shp if rdt_out._available_fields is None: rdt_out._available_fields = [] if rdt_in._available_fields is None: rdt_in._available_fields = [] for key,pctup in merged_pdict.iteritems(): n,pc = pctup #if function then a transform is applied to calculate values if isinstance(pc.param_type, ParameterFunctionType): #apply transform pv = get_value_class(pc.param_type, rdt_in.domain) #recursive function applies values def pval_callback(name, slice_): result = None #search for input data...first level input if name in rdt_in._available_fields: result = rdt_in[name] else: #not first level data so continue to evaluate n,pc = merged_pdict[name] pv = get_value_class(pc.param_type, rdt_in.domain) pv._pval_callback = pval_callback result = pv[:] return result #set the evaluation callback so it can find values in the input stream pv._pval_callback = pval_callback if key in rdt_out._available_fields: #rdt to and from granule wraps result in a paramval so no need #paramval = rdt_out.get_paramval(pc.param_type, rdt_in.domain, pv[:]) #paramval._pval_callback = pval_callback #rdt_out._rd[key] = paramval rdt_out._rd[key] = pv[:] else: #field exists in both the in and the out stream so pass it along to the output stream if key in rdt_in._available_fields and key in rdt_out._available_fields: #pass through rdt_out._rd[key] = rdt_in._rd[key][:] return rdt_out
class TransformPrime(TransformDataProcess): binding=['output'] ''' Transforms which have an incoming stream and an outgoing stream. Parameters: process.stream_id Outgoing stream identifier. process.exchange_point Route's exchange point. process.routing_key Route's routing key. process.queue_name Name of the queue to listen on. process.routes streams,actor for each route {(stream_input_id, stream_output_id):actor} Either the stream_id or both the exchange_point and routing_key need to be provided. ''' def on_start(self): TransformDataProcess.on_start(self) self.pubsub_management = PubsubManagementServiceProcessClient(process=self) @memoize_lru(100) def read_stream_def(self,stream_id): return self.pubsub_management.read_stream_definition(stream_id=stream_id) def recv_packet(self, msg, stream_route, stream_id): process_routes = self.CFG.get_safe('process.routes', {}) for stream_in_id,routes in process_routes.iteritems(): if stream_id == stream_in_id: for stream_out_id, actor in routes.iteritems(): if actor is None: rdt_out = self._execute_transform(msg, (stream_in_id, stream_out_id)) self.publish(rdt_out.to_granule(), stream_out_id) else: outgoing = self._execute_actor(msg, actor, (stream_in_id, stream_out_id)) self.publish(outgoing, stream_out_id) def publish(self, msg, stream_out_id): publisher = getattr(self, stream_out_id) publisher.publish(msg) def _load_actor(self, actor): ''' Returns callable execute method if it exists, otherwise it raises a BadRequest ''' try: module = __import__(actor['module'], fromlist=['']) except ImportError: log.exception('Actor could not be loaded') raise try: cls = getattr(module, actor['class']) except AttributeError: log.exception('Module %s does not have class %s', repr(module), actor['class']) raise try: execute = getattr(cls,'execute') except AttributeError: log.exception('Actor class does not contain execute method') raise return execute def _execute_actor(self, msg, actor, streams): stream_in_id,stream_out_id = streams stream_def_out = self.read_stream_def(stream_out_id) params = self.CFG.get_safe('process.params', {}) config = self.CFG.get_safe('process') #do the stuff with the actor params['stream_def'] = stream_def_out._id executor = self._load_actor(actor) try: rdt_out = executor(msg, None, config, params, None) except: log.exception('Error running actor for %s', self.id) raise return rdt_out def _execute_transform(self, msg, streams): stream_in_id,stream_out_id = streams stream_def_in = self.read_stream_def(stream_in_id) stream_def_out = self.read_stream_def(stream_out_id) incoming_pdict_dump = stream_def_in.parameter_dictionary outgoing_pdict_dump = stream_def_out.parameter_dictionary incoming_pdict = ParameterDictionary.load(incoming_pdict_dump) outgoing_pdict = ParameterDictionary.load(outgoing_pdict_dump) merged_pdict = ParameterDictionary() for k,v in incoming_pdict.iteritems(): ordinal, v = v if k not in merged_pdict: merged_pdict.add_context(v) for k,v in outgoing_pdict.iteritems(): ordinal, v = v if k not in merged_pdict: merged_pdict.add_context(v) rdt_temp = RecordDictionaryTool(param_dictionary=merged_pdict) rdt_in = RecordDictionaryTool.load_from_granule(msg) for field in rdt_temp.fields: if not isinstance(rdt_temp._pdict.get_context(field).param_type, ParameterFunctionType): try: rdt_temp[field] = rdt_in[field] except KeyError: pass for field in rdt_temp.fields: if isinstance(rdt_temp._pdict.get_context(field).param_type, ParameterFunctionType): rdt_temp[field] = rdt_temp[field] rdt_out = RecordDictionaryTool(stream_definition_id=stream_def_out._id) for field in rdt_out.fields: rdt_out[field] = rdt_temp[field] return rdt_out
class VizTransformMatplotlibGraphs(TransformStreamPublisher, TransformEventListener, TransformStreamListener): """ This class is used for instantiating worker processes that have subscriptions to data streams and convert incoming data from CDM format to Matplotlib graphs """ output_bindings = ['graph_image_param_dict'] event_timer_interval = None def on_start(self): #print ">>>>>>>>>>>>>>>>>>>>>> MPL CFG = ", self.CFG self.pubsub_management = PubsubManagementServiceProcessClient(process=self) self.ssclient = SchedulerServiceProcessClient(process=self) self.rrclient = ResourceRegistryServiceProcessClient(process=self) self.data_retriever_client = DataRetrieverServiceProcessClient(process=self) self.dsm_client = DatasetManagementServiceProcessClient(process=self) self.pubsub_client = PubsubManagementServiceProcessClient(process = self) self.stream_info = self.CFG.get_safe('process.publish_streams',{}) self.stream_names = self.stream_info.keys() self.stream_ids = self.stream_info.values() if not self.stream_names: raise BadRequest('MPL Transform has no output streams.') graph_time_periods= self.CFG.get_safe('graph_time_periods') # If this is meant to be an event driven process, schedule an event to be generated every few minutes/hours self.event_timer_interval = self.CFG.get_safe('graph_gen_interval') if self.event_timer_interval: event_origin = "Interval_Timer_Matplotlib" sub = EventSubscriber(event_type="ResourceEvent", callback=self.interval_timer_callback, origin=event_origin) sub.start() self.interval_timer_id = self.ssclient.create_interval_timer(start_time="now" , interval=self._str_to_secs(self.event_timer_interval), event_origin=event_origin, event_subtype="") super(VizTransformMatplotlibGraphs,self).on_start() # when tranform is used as a data process def recv_packet(self, packet, in_stream_route, in_stream_id): #Check to see if the class instance was set up as a event triggered transform. If yes, skip the packet if self.event_timer_interval: return log.info('Received packet') mpl_data_granule = VizTransformMatplotlibGraphsAlgorithm.execute(packet, params=self.get_stream_definition()) for stream_name in self.stream_names: publisher = getattr(self, stream_name) publisher.publish(mpl_data_granule) def get_stream_definition(self): stream_id = self.stream_ids[0] stream_def = self.pubsub_management.read_stream_definition(stream_id=stream_id) return stream_def._id def process_event(self, msg, headers): return def interval_timer_callback(self, *args, **kwargs): #Find out the input data product to this process in_dp_id = self.CFG.get_safe('in_dp_id') print " >>>>>>>>>>>>>> IN DP ID from cfg : ", in_dp_id # get the dataset_id associated with the data_product. Need it to do the data retrieval ds_ids,_ = self.rrclient.find_objects(in_dp_id, PRED.hasDataset, RT.Dataset, True) if ds_ids is None or not ds_ids: return None # retrieve data for the specified time interval. Setup up query from pass config first query = {} param_list_str = self.CFG.get_safe('parameters') if param_list_str: query['parameters'] = param_list_str.split(', ') # append time if not present in list of parameters if not 'time' in query['parameters']: query['parameters'].append('time') query['start_time'] = query['end_time'] = ntplib.system_to_ntp_time(time.time()) # Now query['stride_time'] = 1 if self.CFG.get_safe('graph_time_period'): query['start_time'] = query['end_time'] - self._str_to_secs(self.CFG.get_safe('graph_time_period')) #print " >>>>>>>>>>>>>> QUERY = ", query #retrieved_granule = self.data_retriever_client.retrieve(ds_ids[0],{'start_time':start_time,'end_time':end_time}) retrieved_granule = self.data_retriever_client.retrieve(ds_ids[0], query=query) # add extra parameters to query passed in config that are not needed by data retrieval if self.CFG.get_safe('resolution'): query['resolution'] = self.CFG.get_safe('resolution') # send the granule through the Algorithm code to get the matplotlib graphs mpl_pdict_id = self.dsm_client.read_parameter_dictionary_by_name('graph_image_param_dict',id_only=True) mpl_stream_def = self.pubsub_client.create_stream_definition('mpl', parameter_dictionary_id=mpl_pdict_id) fileName = self.CFG.get_safe('graph_time_period') mpl_data_granule = VizTransformMatplotlibGraphsAlgorithm.execute(retrieved_granule, config=query, params=mpl_stream_def, fileName=fileName) if mpl_data_granule == None: return None # publish on all specified output streams for stream_name in self.stream_names: publisher = getattr(self, stream_name) publisher.publish(mpl_data_granule) return def _str_to_secs(self, time_period): # this method converts commonly used time periods to its actual seconds counterpart #separate alpha and numeric parts of the time period time_n = time_period.lower().rstrip('abcdefghijklmnopqrstuvwxyz ') time_a = time_period.lower().lstrip('0123456789. ') # determine if user specified, secs, mins, hours, days, weeks, months, years factor = None if time_a == 'sec' or time_a == "secs" or time_a == 'second' or time_a == "seconds": factor = 1 if time_a == "min" or time_a == "mins" or time_a == "minute" or time_a == "minutes": factor = 60 if time_a == "hr" or time_a == "hrs" or time_a == "hour" or time_a == "hours": factor = 60 * 60 if time_a == "day" or time_a == "days": factor = 60 * 60 * 24 if time_a == "wk" or time_a == "wks" or time_a == "week" or time_a == "weeks": factor = 60 * 60 * 24 * 7 if time_a == "mon" or time_a == "mons" or time_a == "month" or time_a == "months": factor = 60 * 60 * 24 * 30 if time_a == "yr" or time_a == "yrs" or time_a == "year" or time_a == "years": factor = 60 * 60 * 24 * 365 time_period_secs = float(time_n) * factor return time_period_secs def on_quit(self): #Cancel the timer if hasattr(self, 'interval_timer_id'): self.ssclient.cancel_timer(self.interval_timer_id) super(VizTransformMatplotlibGraphs,self).on_quit()
class VizTransformMatplotlibGraphs(TransformStreamPublisher, TransformEventListener, TransformStreamListener): """ This class is used for instantiating worker processes that have subscriptions to data streams and convert incoming data from CDM format to Matplotlib graphs """ output_bindings = ['graph_image_param_dict'] event_timer_interval = None def on_start(self): #print ">>>>>>>>>>>>>>>>>>>>>> MPL CFG = ", self.CFG self.pubsub_management = PubsubManagementServiceProcessClient( process=self) self.ssclient = SchedulerServiceProcessClient(process=self) self.rrclient = ResourceRegistryServiceProcessClient(process=self) self.data_retriever_client = DataRetrieverServiceProcessClient( process=self) self.dsm_client = DatasetManagementServiceProcessClient(process=self) self.pubsub_client = PubsubManagementServiceProcessClient(process=self) self.stream_info = self.CFG.get_safe('process.publish_streams', {}) self.stream_names = self.stream_info.keys() self.stream_ids = self.stream_info.values() if not self.stream_names: raise BadRequest('MPL Transform has no output streams.') graph_time_periods = self.CFG.get_safe('graph_time_periods') # If this is meant to be an event driven process, schedule an event to be generated every few minutes/hours self.event_timer_interval = self.CFG.get_safe('graph_gen_interval') if self.event_timer_interval: event_origin = "Interval_Timer_Matplotlib" sub = EventSubscriber(event_type="ResourceEvent", callback=self.interval_timer_callback, origin=event_origin) sub.start() self.interval_timer_id = self.ssclient.create_interval_timer( start_time="now", interval=self._str_to_secs(self.event_timer_interval), event_origin=event_origin, event_subtype="") super(VizTransformMatplotlibGraphs, self).on_start() # when tranform is used as a data process def recv_packet(self, packet, in_stream_route, in_stream_id): #Check to see if the class instance was set up as a event triggered transform. If yes, skip the packet if self.event_timer_interval: return log.info('Received packet') mpl_data_granule = VizTransformMatplotlibGraphsAlgorithm.execute( packet, params=self.get_stream_definition()) for stream_name in self.stream_names: publisher = getattr(self, stream_name) publisher.publish(mpl_data_granule) def get_stream_definition(self): stream_id = self.stream_ids[0] stream_def = self.pubsub_management.read_stream_definition( stream_id=stream_id) return stream_def._id def process_event(self, msg, headers): return def interval_timer_callback(self, *args, **kwargs): #Find out the input data product to this process in_dp_id = self.CFG.get_safe('in_dp_id') print " >>>>>>>>>>>>>> IN DP ID from cfg : ", in_dp_id # get the dataset_id associated with the data_product. Need it to do the data retrieval ds_ids, _ = self.rrclient.find_objects(in_dp_id, PRED.hasDataset, RT.Dataset, True) if ds_ids is None or not ds_ids: return None # retrieve data for the specified time interval. Setup up query from pass config first query = {} param_list_str = self.CFG.get_safe('parameters') if param_list_str: query['parameters'] = param_list_str.split(', ') # append time if not present in list of parameters if not 'time' in query['parameters']: query['parameters'].append('time') query['start_time'] = query['end_time'] = ntplib.system_to_ntp_time( time.time()) # Now query['stride_time'] = 1 if self.CFG.get_safe('graph_time_period'): query['start_time'] = query['end_time'] - self._str_to_secs( self.CFG.get_safe('graph_time_period')) #print " >>>>>>>>>>>>>> QUERY = ", query #retrieved_granule = self.data_retriever_client.retrieve(ds_ids[0],{'start_time':start_time,'end_time':end_time}) retrieved_granule = self.data_retriever_client.retrieve(ds_ids[0], query=query) # add extra parameters to query passed in config that are not needed by data retrieval if self.CFG.get_safe('resolution'): query['resolution'] = self.CFG.get_safe('resolution') # send the granule through the Algorithm code to get the matplotlib graphs mpl_pdict_id = self.dsm_client.read_parameter_dictionary_by_name( 'graph_image_param_dict', id_only=True) mpl_stream_def = self.pubsub_client.create_stream_definition( 'mpl', parameter_dictionary_id=mpl_pdict_id) fileName = self.CFG.get_safe('graph_time_period') mpl_data_granule = VizTransformMatplotlibGraphsAlgorithm.execute( retrieved_granule, config=query, params=mpl_stream_def, fileName=fileName) if mpl_data_granule == None: return None # publish on all specified output streams for stream_name in self.stream_names: publisher = getattr(self, stream_name) publisher.publish(mpl_data_granule) return def _str_to_secs(self, time_period): # this method converts commonly used time periods to its actual seconds counterpart #separate alpha and numeric parts of the time period time_n = time_period.lower().rstrip('abcdefghijklmnopqrstuvwxyz ') time_a = time_period.lower().lstrip('0123456789. ') # determine if user specified, secs, mins, hours, days, weeks, months, years factor = None if time_a == 'sec' or time_a == "secs" or time_a == 'second' or time_a == "seconds": factor = 1 if time_a == "min" or time_a == "mins" or time_a == "minute" or time_a == "minutes": factor = 60 if time_a == "hr" or time_a == "hrs" or time_a == "hour" or time_a == "hours": factor = 60 * 60 if time_a == "day" or time_a == "days": factor = 60 * 60 * 24 if time_a == "wk" or time_a == "wks" or time_a == "week" or time_a == "weeks": factor = 60 * 60 * 24 * 7 if time_a == "mon" or time_a == "mons" or time_a == "month" or time_a == "months": factor = 60 * 60 * 24 * 30 if time_a == "yr" or time_a == "yrs" or time_a == "year" or time_a == "years": factor = 60 * 60 * 24 * 365 time_period_secs = float(time_n) * factor return time_period_secs def on_quit(self): #Cancel the timer if hasattr(self, 'interval_timer_id'): self.ssclient.cancel_timer(self.interval_timer_id) super(VizTransformMatplotlibGraphs, self).on_quit()
class TransformPrime(TransformDataProcess): binding=['output'] ''' Transforms which have an incoming stream and an outgoing stream. Parameters: process.stream_id Outgoing stream identifier. process.exchange_point Route's exchange point. process.routing_key Route's routing key. process.queue_name Name of the queue to listen on. process.routes streams,actor for each route {(stream_input_id, stream_output_id):actor} Either the stream_id or both the exchange_point and routing_key need to be provided. ''' def on_start(self): TransformDataProcess.on_start(self) self.pubsub_management = PubsubManagementServiceProcessClient(process=self) self.stored_values = StoredValueManager(self.container) self.input_data_product_ids = self.CFG.get_safe('process.input_products', []) self.output_data_product_ids = self.CFG.get_safe('process.output_products', []) self.lookup_docs = self.CFG.get_safe('process.lookup_docs',[]) self.new_lookups = Queue() self.lookup_monitor = EventSubscriber(event_type=OT.ExternalReferencesUpdatedEvent,callback=self._add_lookups, auto_delete=True) self.lookup_monitor.start() def on_quit(self): self.lookup_monitor.stop() TransformDataProcess.on_quit(self) def _add_lookups(self, event, *args, **kwargs): if event.origin in self.input_data_product_ids + self.output_data_product_ids: if isinstance(event.reference_keys, list): self.new_lookups.put(event.reference_keys) @memoize_lru(100) def read_stream_def(self,stream_id): return self.pubsub_management.read_stream_definition(stream_id=stream_id) def recv_packet(self, msg, stream_route, stream_id): process_routes = self.CFG.get_safe('process.routes', {}) for stream_in_id,routes in process_routes.iteritems(): if stream_id == stream_in_id: for stream_out_id, actor in routes.iteritems(): if actor is None: rdt_out = self._execute_transform(msg, (stream_in_id, stream_out_id)) self.publish(rdt_out.to_granule(), stream_out_id) else: outgoing = self._execute_actor(msg, actor, (stream_in_id, stream_out_id)) self.publish(outgoing, stream_out_id) def publish(self, msg, stream_out_id): publisher = getattr(self, stream_out_id) publisher.publish(msg) def _load_actor(self, actor): ''' Returns callable execute method if it exists, otherwise it raises a BadRequest ''' try: module = __import__(actor['module'], fromlist=['']) except ImportError: log.exception('Actor could not be loaded') raise try: cls = getattr(module, actor['class']) except AttributeError: log.exception('Module %s does not have class %s', repr(module), actor['class']) raise try: execute = getattr(cls,'execute') except AttributeError: log.exception('Actor class does not contain execute method') raise return execute def _execute_actor(self, msg, actor, streams): stream_in_id,stream_out_id = streams stream_def_out = self.read_stream_def(stream_out_id) params = self.CFG.get_safe('process.params', {}) config = self.CFG.get_safe('process') #do the stuff with the actor params['stream_def'] = stream_def_out._id executor = self._load_actor(actor) try: rdt_out = executor(msg, None, config, params, None) except: log.exception('Error running actor for %s', self.id) raise return rdt_out def _merge_pdicts(self, pdict1, pdict2): incoming_pdict = ParameterDictionary.load(pdict1) outgoing_pdict = ParameterDictionary.load(pdict2) merged_pdict = ParameterDictionary() for k,v in incoming_pdict.iteritems(): ordinal, v = v if k not in merged_pdict: merged_pdict.add_context(v) for k,v in outgoing_pdict.iteritems(): ordinal, v = v if k not in merged_pdict: merged_pdict.add_context(v) return merged_pdict def _merge_rdt(self, stream_def_in, stream_def_out): incoming_pdict_dump = stream_def_in.parameter_dictionary outgoing_pdict_dump = stream_def_out.parameter_dictionary merged_pdict = self._merge_pdicts(incoming_pdict_dump, outgoing_pdict_dump) rdt_temp = RecordDictionaryTool(param_dictionary=merged_pdict) return rdt_temp def _get_lookup_value(self, lookup_value): if not self.new_lookups.empty(): new_values = self.new_lookups.get() self.lookup_docs = new_values + self.lookup_docs lookup_value_document_keys = self.lookup_docs for key in lookup_value_document_keys: try: document = self.stored_values.read_value(key) if lookup_value in document: return document[lookup_value] except NotFound: log.warning('Specified lookup document does not exist') return None def _execute_transform(self, msg, streams): stream_in_id,stream_out_id = streams stream_def_in = self.read_stream_def(stream_in_id) stream_def_out = self.read_stream_def(stream_out_id) rdt_temp = self._merge_rdt(stream_def_in, stream_def_out) rdt_in = RecordDictionaryTool.load_from_granule(msg) for field in rdt_temp.fields: if not isinstance(rdt_temp._pdict.get_context(field).param_type, ParameterFunctionType): try: rdt_temp[field] = rdt_in[field] except KeyError: pass rdt_temp.fetch_lookup_values() for lookup_field in rdt_temp.lookup_values(): s = lookup_field stored_value = self._get_lookup_value(rdt_temp.context(s).lookup_value) if stored_value is not None: rdt_temp[s] = stored_value for field in rdt_temp.fields: if isinstance(rdt_temp._pdict.get_context(field).param_type, ParameterFunctionType): rdt_temp[field] = rdt_temp[field] rdt_out = RecordDictionaryTool(stream_definition_id=stream_def_out._id) for field in rdt_out.fields: rdt_out[field] = rdt_temp[field] return rdt_out
class TransformPrime(TransformDataProcess): binding = ['output'] ''' Transforms which have an incoming stream and an outgoing stream. Parameters: process.stream_id Outgoing stream identifier. process.exchange_point Route's exchange point. process.routing_key Route's routing key. process.queue_name Name of the queue to listen on. process.routes streams,actor for each route {(stream_input_id, stream_output_id):actor} Either the stream_id or both the exchange_point and routing_key need to be provided. ''' def on_start(self): TransformDataProcess.on_start(self) self.pubsub_management = PubsubManagementServiceProcessClient( process=self) self.stored_values = StoredValueManager(self.container) self.input_data_product_ids = self.CFG.get_safe( 'process.input_products', []) self.output_data_product_ids = self.CFG.get_safe( 'process.output_products', []) self.lookup_docs = self.CFG.get_safe('process.lookup_docs', []) self.new_lookups = Queue() self.lookup_monitor = EventSubscriber( event_type=OT.ExternalReferencesUpdatedEvent, callback=self._add_lookups, auto_delete=True) self.lookup_monitor.start() def on_quit(self): self.lookup_monitor.stop() TransformDataProcess.on_quit(self) def _add_lookups(self, event, *args, **kwargs): if event.origin in self.input_data_product_ids + self.output_data_product_ids: if isinstance(event.reference_keys, list): self.new_lookups.put(event.reference_keys) @memoize_lru(100) def read_stream_def(self, stream_id): return self.pubsub_management.read_stream_definition( stream_id=stream_id) def recv_packet(self, msg, stream_route, stream_id): process_routes = self.CFG.get_safe('process.routes', {}) for stream_in_id, routes in process_routes.iteritems(): if stream_id == stream_in_id: for stream_out_id, actor in routes.iteritems(): if actor is None: rdt_out = self._execute_transform( msg, (stream_in_id, stream_out_id)) self.publish(rdt_out.to_granule(), stream_out_id) else: outgoing = self._execute_actor( msg, actor, (stream_in_id, stream_out_id)) self.publish(outgoing, stream_out_id) def publish(self, msg, stream_out_id): publisher = getattr(self, stream_out_id) publisher.publish(msg) def _load_actor(self, actor): ''' Returns callable execute method if it exists, otherwise it raises a BadRequest ''' try: module = __import__(actor['module'], fromlist=['']) except ImportError: log.exception('Actor could not be loaded') raise try: cls = getattr(module, actor['class']) except AttributeError: log.exception('Module %s does not have class %s', repr(module), actor['class']) raise try: execute = getattr(cls, 'execute') except AttributeError: log.exception('Actor class does not contain execute method') raise return execute def _execute_actor(self, msg, actor, streams): stream_in_id, stream_out_id = streams stream_def_out = self.read_stream_def(stream_out_id) params = self.CFG.get_safe('process.params', {}) config = self.CFG.get_safe('process') #do the stuff with the actor params['stream_def'] = stream_def_out._id executor = self._load_actor(actor) try: rdt_out = executor(msg, None, config, params, None) except: log.exception('Error running actor for %s', self.id) raise return rdt_out def _merge_pdicts(self, pdict1, pdict2): incoming_pdict = ParameterDictionary.load(pdict1) outgoing_pdict = ParameterDictionary.load(pdict2) merged_pdict = ParameterDictionary() for k, v in incoming_pdict.iteritems(): ordinal, v = v if k not in merged_pdict: merged_pdict.add_context(v) for k, v in outgoing_pdict.iteritems(): ordinal, v = v if k not in merged_pdict: merged_pdict.add_context(v) return merged_pdict def _merge_rdt(self, stream_def_in, stream_def_out): incoming_pdict_dump = stream_def_in.parameter_dictionary outgoing_pdict_dump = stream_def_out.parameter_dictionary merged_pdict = self._merge_pdicts(incoming_pdict_dump, outgoing_pdict_dump) rdt_temp = RecordDictionaryTool(param_dictionary=merged_pdict) return rdt_temp def _get_lookup_value(self, lookup_value): if not self.new_lookups.empty(): new_values = self.new_lookups.get() self.lookup_docs = new_values + self.lookup_docs lookup_value_document_keys = self.lookup_docs for key in lookup_value_document_keys: try: document = self.stored_values.read_value(key) if lookup_value in document: return document[lookup_value] except NotFound: log.warning('Specified lookup document does not exist') return None def _execute_transform(self, msg, streams): stream_in_id, stream_out_id = streams stream_def_in = self.read_stream_def(stream_in_id) stream_def_out = self.read_stream_def(stream_out_id) rdt_temp = self._merge_rdt(stream_def_in, stream_def_out) rdt_in = RecordDictionaryTool.load_from_granule(msg) for field in rdt_temp.fields: if not isinstance( rdt_temp._pdict.get_context(field).param_type, ParameterFunctionType): try: rdt_temp[field] = rdt_in[field] except KeyError: pass rdt_temp.fetch_lookup_values() for lookup_field in rdt_temp.lookup_values(): s = lookup_field stored_value = self._get_lookup_value( rdt_temp.context(s).lookup_value) if stored_value is not None: rdt_temp[s] = stored_value for field in rdt_temp.fields: if isinstance( rdt_temp._pdict.get_context(field).param_type, ParameterFunctionType): rdt_temp[field] = rdt_temp[field] rdt_out = RecordDictionaryTool(stream_definition_id=stream_def_out._id) for field in rdt_out.fields: rdt_out[field] = rdt_temp[field] return rdt_out
class VizTransformMatplotlibGraphs(TransformDataProcess, TransformEventListener): """ This class is used for instantiating worker processes that have subscriptions to data streams and convert incoming data from CDM format to Matplotlib graphs """ output_bindings = ['graph_image_param_dict'] def on_start(self): self.pubsub_management = PubsubManagementServiceProcessClient(process=self) self.ssclient = SchedulerServiceProcessClient(node=self.container.node, process=self) self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=self) self.stream_info = self.CFG.get_safe('process.publish_streams',{}) self.stream_names = self.stream_info.keys() self.stream_ids = self.stream_info.values() if not self.stream_names: raise BadRequest('MPL Transform has no output streams.') graph_time_periods= self.CFG.get_safe('graph_time_periods') # If this is meant to be an event driven process, schedule an event to be generated every few minutes/hours event_timer_interval = self.CFG.get_safe('event_timer_interval') if event_timer_interval: event_origin = "Interval_Timer_Matplotlib" sub = EventSubscriber(event_type="ResourceEvent", callback=self.interval_timer_callback, origin=event_origin) sub.start() self.interval_timer_id = self.ssclient.create_interval_timer(start_time="now" , interval=event_timer_interval, event_origin=event_origin, event_subtype="") super(VizTransformMatplotlibGraphs,self).on_start() def recv_packet(self, packet, in_stream_route, in_stream_id): log.info('Received packet') outgoing = VizTransformMatplotlibGraphsAlgorithm.execute(packet, params=self.get_stream_definition()) for stream_name in self.stream_names: publisher = getattr(self, stream_name) publisher.publish(outgoing) def get_stream_definition(self): stream_id = self.stream_ids[0] stream_def = self.pubsub_management.read_stream_definition(stream_id=stream_id) return stream_def._id def process_event(self, msg, headers): return def interval_timer_callback(self, *args, **kwargs): #Find out the input data product to this process # retrieve data for every case of the output graph return def on_quit(self): #Cancel the timer if hasattr(self, 'interval_timer_id'): self.ssclient.cancel_timer(self.interval_timer_id) super(VizTransformMatplotlibGraphs,self).on_quit()