def _gen(self, stream_in): # IMPORTANT: Messages may contain pointers that are shared with # other streams. Transforms that modify their input # messages should only manipulate copies. log.info('Running StatefulTransform [%s]' % self.get_hash()) for message in stream_in: # we only handle TRADE events. if (hasattr(message, 'type') and message.type not in ( DATASOURCE_TYPE.TRADE, DATASOURCE_TYPE.CUSTOM)): # TODO: this should be yielding the original message # instead of swallowing it. Will be an issue when we # have a transaction source from brokers etc. continue # allow upstream generators to yield None to avoid # blocking. if message is None: continue assert_sort_unframe_protocol(message) tnfm_value = self.state.update(message) out_message = message out_message[self.namestring] = tnfm_value yield out_message log.info('Finished StatefulTransform [%s]' % self.get_hash())
def _gen(self, stream_in): # IMPORTANT: Messages may contain pointers that are shared with # other streams. Transforms that modify their input # messages should only manipulate copies. log.info('Running StatefulTransform [%s]' % self.get_hash()) for message in stream_in: # we only handle TRADE events. if (hasattr(message, 'type') and message.type not in (DATASOURCE_TYPE.TRADE, DATASOURCE_TYPE.CUSTOM)): # TODO: this should be yielding the original message # instead of swallowing it. Will be an issue when we # have a transaction source from brokers etc. continue # allow upstream generators to yield None to avoid # blocking. if message is None: continue assert_sort_unframe_protocol(message) tnfm_value = self.state.update(message) out_message = message out_message[self.namestring] = tnfm_value yield out_message log.info('Finished StatefulTransform [%s]' % self.get_hash())
def _gen(self, stream_in): # IMPORTANT: Messages may contain pointers that are shared with # other streams. Transforms that modify their input # messages should only manipulate copies. log.debug('Running StatefulTransform [%s]' % self.get_hash()) for message in stream_in: # we only handle TRADE events. if (hasattr(message, 'type') and message.type not in ( DATASOURCE_TYPE.TRADE, DATASOURCE_TYPE.CUSTOM)): yield message # allow upstream generators to yield None to avoid # blocking. if message is None: continue assert_sort_unframe_protocol(message) tnfm_value = self.state.update(message) out_message = message out_message[self.namestring] = tnfm_value yield out_message log.debug('Finished StatefulTransform [%s]' % self.get_hash())
def _gen(self, stream_in): # IMPORTANT: Messages may contain pointers that are shared with # other streams. Transforms that modify their input # messages should only manipulate copies. log.debug('Running StatefulTransform [%s]' % self.get_hash()) for message in stream_in: # we only handle TRADE events. if (hasattr(message, 'type') and message.type not in (DATASOURCE_TYPE.TRADE, DATASOURCE_TYPE.CUSTOM)): yield message continue # allow upstream generators to yield None to avoid # blocking. if message is None: continue assert_sort_unframe_protocol(message) tnfm_value = self.state.update(message) out_message = message out_message[self.namestring] = tnfm_value yield out_message log.debug('Finished StatefulTransform [%s]' % self.get_hash())
def _gen(self, stream_in): # IMPORTANT: Messages may contain pointers that are shared with # other streams. Transforms that modify their input # messages should only manipulate copies. log.info('Running StatefulTransform [%s]' % self.get_hash()) for message in stream_in: # allow upstream generators to yield None to avoid # blocking. if message is None: continue assert_sort_unframe_protocol(message) # This flag is set by by merged_transforms to ensure # isolation of messages. if self.merged: message = deepcopy(message) tnfm_value = self.state.update(message) # PASSTHROUGH flag means we want to keep all original # values, plus append tnfm_id and tnfm_value. Used for # preserving the original event fields when our output # will be fed into a merge. Currently only Passthrough # uses this flag. if self.passthrough and self.merged: out_message = message out_message.tnfm_id = self.namestring out_message.tnfm_value = tnfm_value yield out_message # If the merged flag is set, we create a new message # containing just the tnfm_id, the event's datetime, and # the calculated tnfm_value. This is the default behavior # for a non-passthrough transform being fed into a merge. elif self.merged: out_message = ndict() out_message.tnfm_id = self.namestring out_message.tnfm_value = tnfm_value out_message.dt = message.dt yield out_message # Sequential flag should be used to add a single new # key-value pair to the event. The new key is this # transform's namestring, and its value is the value # returned by state.update(event). This is almost # identical to the behavior of FORWARDER, except we # compress the two calculated values (tnfm_id, and # tnfm_value) into a single field. This mode is used by # the sequential_transforms composite and is the default # if no behavior is specified by the internal state class. elif self.sequential: out_message = message out_message[self.namestring] = tnfm_value yield out_message log.info('Finished StatefulTransform [%s]' % self.get_hash())
def _gen(self, stream_in): # IMPORTANT: Messages may contain pointers that are shared with # other streams. Transforms that modify their input # messages should only manipulate copies. log.info('Running StatefulTransform [%s]' % self.get_hash()) for message in stream_in: # allow upstream generators to yield None to avoid # blocking. if message is None: continue assert_sort_unframe_protocol(message) # This flag is set by by merged_transforms to ensure # isolation of messages. if self.merged: message = deepcopy(message) tnfm_value = self.state.update(message) # PASSTHROUGH flag means we want to keep all original # values, plus append tnfm_id and tnfm_value. Used for # preserving the original event fields when our output # will be fed into a merge. Currently only Passthrough # uses this flag. if self.passthrough and self.merged: out_message = message out_message.tnfm_id = self.namestring out_message.tnfm_value = tnfm_value yield out_message # If the merged flag is set, we create a new message # containing just the tnfm_id, the event's datetime, and # the calculated tnfm_value. This is the default behavior # for a non-passthrough transform being fed into a merge. elif self.merged: out_message = ndict() out_message.tnfm_id = self.namestring out_message.tnfm_value = tnfm_value out_message.dt = message.dt yield out_message # Sequential flag should be used to add a single new # key-value pair to the event. The new key is this # transform's namestring, and its value is the value # returned by state.update(event). This is almost # identical to the behavior of FORWARDER, except we # compress the two calculated values (tnfm_id, and # tnfm_value) into a single field. This mode is used by # the sequential_transforms composite and is the default # if no behavior is specified by the internal state class. elif self.sequential: out_message = message out_message[self.namestring] = tnfm_value yield out_message log.info('Finished StatefulTransform [%s]' % self.get_hash())
def _gen(self, stream_in): # IMPORTANT: Messages may contain pointers that are shared with # other streams. Transforms that modify their input # messages should only manipulate copies. log.info('Running StatefulTransform [%s]' % self.get_hash()) for message in stream_in: # allow upstream generators to yield None to avoid # blocking. if message is None: continue assert_sort_unframe_protocol(message) tnfm_value = self.state.update(message) out_message = message out_message[self.namestring] = tnfm_value yield out_message log.info('Finished StatefulTransform [%s]' % self.get_hash())
def _gen(self, stream_in): # IMPORTANT: Messages may contain pointers that are shared with # other streams. Transforms that modify their input # messages should only manipulate copies. for message in stream_in: # we only handle TRADE and CUSTOM events. if (hasattr(message, 'type') and message.type not in ( DATASOURCE_TYPE.TRADE, DATASOURCE_TYPE.CUSTOM)): yield message continue # allow upstream generators to yield None to avoid # blocking. if message is None: continue assert_sort_unframe_protocol(message) try: tnfm_value = self.state.update(message) except WrongDataForTransform: # Transform classes should raise WrongDataForTransform if they # are unable to process the event BEFORE performing any state # modifications, because we continue the simulation if a # WrongDataForTransform is raised on a CUSTOM event. if message.type == DATASOURCE_TYPE.CUSTOM: # Pass through custom events that are not applicable to # this transform. yield message continue else: # If a TRADE event raises a WrongDataForTransform, # something bad has happend. raise out_message = message out_message[self.namestring] = tnfm_value yield out_message
def _gen(self, stream_in): # IMPORTANT: Messages may contain pointers that are shared with # other streams. Transforms that modify their input # messages should only manipulate copies. for message in stream_in: # we only handle TRADE and CUSTOM events. if hasattr(message, 'type') and \ message.type not in (DATASOURCE_TYPE.TRADE, DATASOURCE_TYPE.CUSTOM): yield message continue # allow upstream generators to yield None to avoid # blocking. if message is None: continue assert_sort_unframe_protocol(message) try: tnfm_value = self.state.update(message) except WrongDataForTransform: # Transform classes should raise WrongDataForTransform if they # are unable to process the event BEFORE performing any state # modifications, because we continue the simulation if a # WrongDataForTransform is raised on a CUSTOM event. if message.type == DATASOURCE_TYPE.CUSTOM: # Pass through custom events that are not applicable to # this transform. yield message continue else: # If a TRADE event raises a WrongDataForTransform, # something bad has happend. raise out_message = message out_message[self.namestring] = tnfm_value yield out_message