def process(self):
""" Processes all data that is provided by the input node
Returns a generator that yields the data after being processed by this
node.
"""
assert (self.input_node != None), "No input node specified!"
# Assert that this node has already been trained
assert (not self.is_trainable()
or self.get_remaining_train_phase() == 0), "Node not trained!"
if self.window_definition is None:
if self.is_training(
) and self.windower_spec_file_train is not None:
self.window_definition = \
Windower._load_window_spec(self.windower_spec_file_train,
self.local_window_conf)
else:
self.window_definition = \
Windower._load_window_spec(self.windower_spec_file,
self.local_window_conf)
data_generator = \
itertools.imap(lambda (data, label):
(self.execute(data), label),
self.input_node.process())
self.client = TimeSeriesClient(ts_stream=data_generator)
self.client.set_window_defs(self.window_definition)
self.client.connect()
self.marker_windower = MarkerWindower(
data_client=self.client,
windowdefs=self.window_definition,
nullmarker_stride_ms=self.nullmarker_stride_ms)
if self.marker_windower == None:
self.window_stream()
# Create a generator that emits the windows
test_data_generator = ((sample, label) \
for (sample, label) in self.marker_windower)
self.data_for_testing = MemoizeGenerator(test_data_generator)
# Return a fresh copy of the generator
return self.data_for_testing.fresh()
def request_data_for_testing(self):
""" Returns the data for testing of subsequent nodes
.. todo:: to document
"""
if self.data_for_testing is None:
# set window definition for test phase windower file
self.window_definition = \
Windower._load_window_spec(self.windower_spec_file,
self.local_window_conf)
test_data = list(self.input_node.request_data_for_testing())
# create stream of windows
self.window_stream(test_data)
# Create a generator that emits the windows
test_data_generator = ((sample, label) \
for (sample, label) in self.marker_windower)
self.data_for_testing = MemoizeGenerator(test_data_generator)
# Return a fresh copy of the generator
return self.data_for_testing.fresh()
else:
return self.data_for_testing.fresh()
def request_data_for_testing(self):
""" Returns the data for testing of subsequent nodes
.. todo:: to document
"""
if self.data_for_testing is None:
# set window definition for test phase windower file
self.window_definition = \
Windower._load_window_spec(self.windower_spec_file,
self.local_window_conf)
test_data = list(self.input_node.request_data_for_testing())
# create stream of windows
self.window_stream(test_data)
# Create a generator that emits the windows
test_data_generator = ((sample, label) \
for (sample, label) in self.marker_windower)
self.data_for_testing = MemoizeGenerator(test_data_generator)
# Return a fresh copy of the generator
return self.data_for_testing.fresh()
else:
return self.data_for_testing.fresh()
def __init__(self, windower_spec_file=None, local_window_conf=False,
nullmarker_stride_ms=1000, no_overlap=False,
continuous=False,
data_consistency_check=False, **kwargs):
super(Stream2TimeSeriesSourceNode, self).__init__(**kwargs)
assert not(nullmarker_stride_ms is None and windower_spec_file is None),\
"No segmentation parameters specified!"
if windower_spec_file is None:
no_overlap = True
continuous = True
wdefs = WindowFactory.default_windower_spec(
endoffsetms=nullmarker_stride_ms)
else:
wdefs = Windower._load_window_spec(windower_spec_file,
local_window_conf)
self.set_permanent_attributes(
window_definition=wdefs,
nullmarker_stride_ms=nullmarker_stride_ms,
no_overlap=no_overlap,
data_consistency_check=data_consistency_check,
dataset=None,
continuous=continuous)
def __init__(self,
windower_spec_file=None,
local_window_conf=False,
nullmarker_stride_ms=1000,
no_overlap=False,
continuous=False,
data_consistency_check=False,
**kwargs):
super(Stream2TimeSeriesSourceNode, self).__init__(**kwargs)
assert not(nullmarker_stride_ms is None and windower_spec_file is None),\
"No segmentation parameters specified!"
if windower_spec_file is None:
no_overlap = True
continuous = True
wdefs = WindowFactory.default_windower_spec(
endoffsetms=nullmarker_stride_ms)
else:
wdefs = Windower._load_window_spec(windower_spec_file,
local_window_conf)
self.set_permanent_attributes(
window_definition=wdefs,
nullmarker_stride_ms=nullmarker_stride_ms,
no_overlap=no_overlap,
data_consistency_check=data_consistency_check,
dataset=None,
continuous=continuous)
def __init__(self, windower_spec_file=None, local_window_conf=False,
nullmarker_stride_ms=None, no_overlap=False,
continuous=False,
data_consistency_check=False, **kwargs):
super(Stream2TimeSeriesSourceNode, self).__init__(**kwargs)
assert not(nullmarker_stride_ms is None and windower_spec_file is None),\
"No segmentation parameters specified!"
if windower_spec_file is None:
windower_spec_file = ""
no_overlap = True
continuous = True
elif nullmarker_stride_ms is None:
nullmarker_stride_ms = 1000
self.set_permanent_attributes(
input_types=["stream"],
window_definition=Windower._load_window_spec(
windower_spec_file, local_window_conf),
nullmarker_stride_ms=nullmarker_stride_ms,
no_overlap=no_overlap,
data_consistency_check=data_consistency_check,
dataset=None,
continuous=continuous)
def process(self):
""" Processes all data that is provided by the input node
Returns a generator that yields the data after being processed by this
node.
"""
assert(self.input_node != None), "No input node specified!"
# Assert that this node has already been trained
assert(not self.is_trainable() or
self.get_remaining_train_phase() == 0), "Node not trained!"
if self.window_definition is None:
if self.is_training() and self.windower_spec_file_train is not None:
self.window_definition = \
Windower._load_window_spec(self.windower_spec_file_train,
self.local_window_conf)
else:
self.window_definition = \
Windower._load_window_spec(self.windower_spec_file,
self.local_window_conf)
data_generator = \
itertools.imap(lambda (data, label):
(self.execute(data), label),
self.input_node.process())
self.client = TimeSeriesClient(ts_stream = data_generator)
self.client.set_window_defs(self.window_definition)
self.client.connect()
self.marker_windower = MarkerWindower(data_client=self.client,
windowdefs=self.window_definition,
nullmarker_stride_ms=self.nullmarker_stride_ms)
if self.marker_windower == None:
self.window_stream()
# Create a generator that emits the windows
test_data_generator = ((sample, label) \
for (sample, label) in self.marker_windower)
self.data_for_testing = MemoizeGenerator(test_data_generator)
# Return a fresh copy of the generator
return self.data_for_testing.fresh()
def request_data_for_training(self, use_test_data):
""" Returns the data that can be used for training of subsequent nodes
.. todo:: to document
"""
# set window definition for train phase windower file
self.window_definition = \
Windower._load_window_spec(self.windower_spec_file_train,
self.local_window_conf)
self._log("Requesting train data...")
if self.data_for_training is None:
if not use_test_data:
# Get training and test data (with labels)
train_data = \
list(self.input_node.request_data_for_training(use_test_data=use_test_data))
# If training or test data is an empty list
if train_data == []:
self.data_for_training = MemoizeGenerator(
(x for x in [].__iter__()), caching=True)
return self.data_for_training.fresh()
# create stream of
self.window_stream(train_data)
# Create a generator that emits the windows
train_data_generator = ((sample, label)
for (sample,
label) in self.marker_windower)
self.data_for_training = MemoizeGenerator(train_data_generator,
caching=True)
return self.data_for_training.fresh()
else:
# Return the test data as there is no additional data that
# was dedicated for training
self.data_for_training = self.request_data_for_testing()
return self.data_for_training
else:
return self.data_for_training.fresh()
def request_data_for_training(self, use_test_data):
""" Returns the data that can be used for training of subsequent nodes
.. todo:: to document
"""
# set window definition for train phase windower file
self.window_definition = \
Windower._load_window_spec(self.windower_spec_file_train,
self.local_window_conf)
self._log("Requesting train data...")
if self.data_for_training is None:
if not use_test_data:
# Get training and test data (with labels)
train_data = \
list(self.input_node.request_data_for_training(use_test_data=use_test_data))
# If training or test data is an empty list
if train_data == []:
self.data_for_training=MemoizeGenerator(
(x for x in [].__iter__()), caching=True)
return self.data_for_training.fresh()
# create stream of
self.window_stream(train_data)
# Create a generator that emits the windows
train_data_generator = ((sample, label) for (sample, label)
in self.marker_windower)
self.data_for_training = MemoizeGenerator(train_data_generator,
caching=True)
return self.data_for_training.fresh()
else:
# Return the test data as there is no additional data that
# was dedicated for training
self.data_for_training = self.request_data_for_testing()
return self.data_for_training.fresh()
else:
return self.data_for_training.fresh()