def _gen(self, stream_in):
# IMPORTANT: Messages may contain pointers that are shared with
# other streams. Transforms that modify their input
# messages should only manipulate copies.
log.info('Running StatefulTransform [%s]' % self.get_hash())
for message in stream_in:
# we only handle TRADE events.
if (hasattr(message, 'type')
and message.type not in (
DATASOURCE_TYPE.TRADE,
DATASOURCE_TYPE.CUSTOM)):
# TODO: this should be yielding the original message
# instead of swallowing it. Will be an issue when we
# have a transaction source from brokers etc.
continue
# allow upstream generators to yield None to avoid
# blocking.
if message is None:
continue
assert_sort_unframe_protocol(message)
tnfm_value = self.state.update(message)
out_message = message
out_message[self.namestring] = tnfm_value
yield out_message
log.info('Finished StatefulTransform [%s]' % self.get_hash())
def _gen(self, stream_in):
# IMPORTANT: Messages may contain pointers that are shared with
# other streams. Transforms that modify their input
# messages should only manipulate copies.
log.info('Running StatefulTransform [%s]' % self.get_hash())
for message in stream_in:
# we only handle TRADE events.
if (hasattr(message, 'type') and message.type
not in (DATASOURCE_TYPE.TRADE, DATASOURCE_TYPE.CUSTOM)):
# TODO: this should be yielding the original message
# instead of swallowing it. Will be an issue when we
# have a transaction source from brokers etc.
continue
# allow upstream generators to yield None to avoid
# blocking.
if message is None:
continue
assert_sort_unframe_protocol(message)
tnfm_value = self.state.update(message)
out_message = message
out_message[self.namestring] = tnfm_value
yield out_message
log.info('Finished StatefulTransform [%s]' % self.get_hash())
def _gen(self, stream_in):
# IMPORTANT: Messages may contain pointers that are shared with
# other streams. Transforms that modify their input
# messages should only manipulate copies.
log.debug('Running StatefulTransform [%s]' % self.get_hash())
for message in stream_in:
# we only handle TRADE events.
if (hasattr(message, 'type')
and message.type not in (
DATASOURCE_TYPE.TRADE,
DATASOURCE_TYPE.CUSTOM)):
yield message
# allow upstream generators to yield None to avoid
# blocking.
if message is None:
continue
assert_sort_unframe_protocol(message)
tnfm_value = self.state.update(message)
out_message = message
out_message[self.namestring] = tnfm_value
yield out_message
log.debug('Finished StatefulTransform [%s]' % self.get_hash())
def _gen(self, stream_in):
# IMPORTANT: Messages may contain pointers that are shared with
# other streams. Transforms that modify their input
# messages should only manipulate copies.
log.debug('Running StatefulTransform [%s]' % self.get_hash())
for message in stream_in:
# we only handle TRADE events.
if (hasattr(message, 'type') and message.type
not in (DATASOURCE_TYPE.TRADE, DATASOURCE_TYPE.CUSTOM)):
yield message
continue
# allow upstream generators to yield None to avoid
# blocking.
if message is None:
continue
assert_sort_unframe_protocol(message)
tnfm_value = self.state.update(message)
out_message = message
out_message[self.namestring] = tnfm_value
yield out_message
log.debug('Finished StatefulTransform [%s]' % self.get_hash())
def _gen(self, stream_in):
# IMPORTANT: Messages may contain pointers that are shared with
# other streams. Transforms that modify their input
# messages should only manipulate copies.
log.info('Running StatefulTransform [%s]' % self.get_hash())
for message in stream_in:
# allow upstream generators to yield None to avoid
# blocking.
if message is None:
continue
assert_sort_unframe_protocol(message)
# This flag is set by by merged_transforms to ensure
# isolation of messages.
if self.merged:
message = deepcopy(message)
tnfm_value = self.state.update(message)
# PASSTHROUGH flag means we want to keep all original
# values, plus append tnfm_id and tnfm_value. Used for
# preserving the original event fields when our output
# will be fed into a merge. Currently only Passthrough
# uses this flag.
if self.passthrough and self.merged:
out_message = message
out_message.tnfm_id = self.namestring
out_message.tnfm_value = tnfm_value
yield out_message
# If the merged flag is set, we create a new message
# containing just the tnfm_id, the event's datetime, and
# the calculated tnfm_value. This is the default behavior
# for a non-passthrough transform being fed into a merge.
elif self.merged:
out_message = ndict()
out_message.tnfm_id = self.namestring
out_message.tnfm_value = tnfm_value
out_message.dt = message.dt
yield out_message
# Sequential flag should be used to add a single new
# key-value pair to the event. The new key is this
# transform's namestring, and its value is the value
# returned by state.update(event). This is almost
# identical to the behavior of FORWARDER, except we
# compress the two calculated values (tnfm_id, and
# tnfm_value) into a single field. This mode is used by
# the sequential_transforms composite and is the default
# if no behavior is specified by the internal state class.
elif self.sequential:
out_message = message
out_message[self.namestring] = tnfm_value
yield out_message
log.info('Finished StatefulTransform [%s]' % self.get_hash())
def _gen(self, stream_in):
# IMPORTANT: Messages may contain pointers that are shared with
# other streams. Transforms that modify their input
# messages should only manipulate copies.
log.info('Running StatefulTransform [%s]' % self.get_hash())
for message in stream_in:
# allow upstream generators to yield None to avoid
# blocking.
if message is None:
continue
assert_sort_unframe_protocol(message)
# This flag is set by by merged_transforms to ensure
# isolation of messages.
if self.merged:
message = deepcopy(message)
tnfm_value = self.state.update(message)
# PASSTHROUGH flag means we want to keep all original
# values, plus append tnfm_id and tnfm_value. Used for
# preserving the original event fields when our output
# will be fed into a merge. Currently only Passthrough
# uses this flag.
if self.passthrough and self.merged:
out_message = message
out_message.tnfm_id = self.namestring
out_message.tnfm_value = tnfm_value
yield out_message
# If the merged flag is set, we create a new message
# containing just the tnfm_id, the event's datetime, and
# the calculated tnfm_value. This is the default behavior
# for a non-passthrough transform being fed into a merge.
elif self.merged:
out_message = ndict()
out_message.tnfm_id = self.namestring
out_message.tnfm_value = tnfm_value
out_message.dt = message.dt
yield out_message
# Sequential flag should be used to add a single new
# key-value pair to the event. The new key is this
# transform's namestring, and its value is the value
# returned by state.update(event). This is almost
# identical to the behavior of FORWARDER, except we
# compress the two calculated values (tnfm_id, and
# tnfm_value) into a single field. This mode is used by
# the sequential_transforms composite and is the default
# if no behavior is specified by the internal state class.
elif self.sequential:
out_message = message
out_message[self.namestring] = tnfm_value
yield out_message
log.info('Finished StatefulTransform [%s]' % self.get_hash())
def _gen(self, stream_in):
# IMPORTANT: Messages may contain pointers that are shared with
# other streams. Transforms that modify their input
# messages should only manipulate copies.
log.info('Running StatefulTransform [%s]' % self.get_hash())
for message in stream_in:
# allow upstream generators to yield None to avoid
# blocking.
if message is None:
continue
assert_sort_unframe_protocol(message)
tnfm_value = self.state.update(message)
out_message = message
out_message[self.namestring] = tnfm_value
yield out_message
log.info('Finished StatefulTransform [%s]' % self.get_hash())
def _gen(self, stream_in):
# IMPORTANT: Messages may contain pointers that are shared with
# other streams. Transforms that modify their input
# messages should only manipulate copies.
for message in stream_in:
# we only handle TRADE and CUSTOM events.
if (hasattr(message, 'type')
and message.type not in (
DATASOURCE_TYPE.TRADE,
DATASOURCE_TYPE.CUSTOM)):
yield message
continue
# allow upstream generators to yield None to avoid
# blocking.
if message is None:
continue
assert_sort_unframe_protocol(message)
try:
tnfm_value = self.state.update(message)
except WrongDataForTransform:
# Transform classes should raise WrongDataForTransform if they
# are unable to process the event BEFORE performing any state
# modifications, because we continue the simulation if a
# WrongDataForTransform is raised on a CUSTOM event.
if message.type == DATASOURCE_TYPE.CUSTOM:
# Pass through custom events that are not applicable to
# this transform.
yield message
continue
else:
# If a TRADE event raises a WrongDataForTransform,
# something bad has happend.
raise
out_message = message
out_message[self.namestring] = tnfm_value
yield out_message
def _gen(self, stream_in):
# IMPORTANT: Messages may contain pointers that are shared with
# other streams. Transforms that modify their input
# messages should only manipulate copies.
for message in stream_in:
# we only handle TRADE and CUSTOM events.
if hasattr(message, 'type') and \
message.type not in (DATASOURCE_TYPE.TRADE,
DATASOURCE_TYPE.CUSTOM):
yield message
continue
# allow upstream generators to yield None to avoid
# blocking.
if message is None:
continue
assert_sort_unframe_protocol(message)
try:
tnfm_value = self.state.update(message)
except WrongDataForTransform:
# Transform classes should raise WrongDataForTransform if they
# are unable to process the event BEFORE performing any state
# modifications, because we continue the simulation if a
# WrongDataForTransform is raised on a CUSTOM event.
if message.type == DATASOURCE_TYPE.CUSTOM:
# Pass through custom events that are not applicable to
# this transform.
yield message
continue
else:
# If a TRADE event raises a WrongDataForTransform,
# something bad has happend.
raise
out_message = message
out_message[self.namestring] = tnfm_value
yield out_message
