def process_current_split(self):
"""
Compute the results of this sink node for the current split of the data
into train and test data
"""
index = 0
# Compute the time series for the data used for training
for time_series, label in self.input_node.request_data_for_training(
False):
# Do lazy initialization of the class
if self.time_series_collection == None:
self.time_series_collection = \
TimeSeriesDataset(sort_string=self.sort_string)
if index < self.max_num_stored_objects:
# Add sample
self.time_series_collection.add_sample(
time_series,
label=label,
train=True,
split=self.current_split,
run=self.run_number)
index += 1
# Compute the time series for the data used for testing
index = 0
for time_series, label in self.input_node.request_data_for_testing():
# Do lazy initialization of the class
# (maybe there were no training examples)
if self.time_series_collection == None:
self.time_series_collection = \
TimeSeriesDataset(sort_string=self.sort_string)
if index < self.max_num_stored_objects:
# Add sample
self.time_series_collection.add_sample(
time_series,
label=label,
train=False,
split=self.current_split,
run=self.run_number)
index += 1
def get_result_dataset(self):
""" Return the result """
# Merges all timeseries inside the collection if merge flag is set to true
if self.merge:
merged_time_series = self.merge_time_series(
self.time_series_collection)
self.time_series_collection = None
self.time_series_collection = \
TimeSeriesDataset(sort_string=self.sort_string)
self.time_series_collection.add_sample(merged_time_series,
label='Window',
train=False)
return self.time_series_collection
def generate_data_set(self):
""" Generate a dataset using the given generators """
self.dataset = TimeSeriesDataset()
# generate a set of dummy labels to know which class is used later
label_sequence = numpy.hstack(
(numpy.ones(self.ir_items), numpy.zeros(self.nir_items)))
if self.shuffle:
random.shuffle(label_sequence)
ts_generator = TestTimeSeriesGenerator()
current_item = 0 # count produced data objects for drift
for label in label_sequence:
if label == 1:
#generate a data item using the ir_generator
data_item = \
ts_generator.generate_test_data(
channels=len(self.channel_names),
time_points=self.time_points,
function=self.ir_generator,
sampling_frequency=self.sampling_frequency,
channel_order=True,
channel_names=self.channel_names,
dtype=numpy.float)
# Drift:
data_item = data_item + current_item * self.ir_drift_vector
self.dataset.add_sample(data_item, self.ir_label, False)
else:
#generate a data item using the nir_generator
data_item = \
ts_generator.generate_test_data(
channels=len(self.channel_names),
time_points=self.time_points,
function=self.nir_generator,
sampling_frequency=self.sampling_frequency,
channel_order=True,
channel_names=self.channel_names,
dtype=numpy.float)
# Drift:
data_item = data_item + current_item * self.nir_drift_vector
self.dataset.add_sample(data_item, self.nir_label, False)
current_item += 1. / (self.ir_items + self.nir_items)
def prepare_training(self,
training_files,
potentials,
operation,
nullmarker_stride_ms=None):
""" Prepares pyspace live for training.
Prepares everything for training of pyspace live,
i.e. creates flows based on the dataflow specs
and configures them.
"""
online_logger.info("Preparing Training")
self.potentials = potentials
self.operation = operation
self.nullmarker_stride_ms = nullmarker_stride_ms
if self.nullmarker_stride_ms == None:
online_logger.warn(
'Nullmarker stride interval is %s. You can specify it in your parameter file.'
% self.nullmarker_stride_ms)
else:
online_logger.info('Nullmarker stride interval is set to %s ms ' %
self.nullmarker_stride_ms)
online_logger.info("Creating flows..")
for key in self.potentials.keys():
spec_base = self.potentials[key]["configuration"].spec_dir
if self.operation == "train":
self.potentials[key]["node_chain"] = os.path.join(
spec_base, self.potentials[key]["node_chain"])
online_logger.info("node_chain_spec:" +
self.potentials[key]["node_chain"])
elif self.operation in ("prewindowing", "prewindowing_offline"):
self.potentials[key]["prewindowing_flow"] = os.path.join(
spec_base, self.potentials[key]["prewindowing_flow"])
online_logger.info("prewindowing_dataflow_spec: " +
self.potentials[key]["prewindowing_flow"])
elif self.operation == "prewindowed_train":
self.potentials[key]["postprocess_flow"] = os.path.join(
spec_base, self.potentials[key]["postprocess_flow"])
online_logger.info("postprocessing_dataflow_spec: " +
self.potentials[key]["postprocess_flow"])
self.training_active_potential[key] = multiprocessing.Value(
"b", False)
online_logger.info("Path variables set for NodeChains")
# check if multiple potentials are given for training
if isinstance(training_files, list):
self.training_data = training_files
else:
self.training_data = [training_files]
# Training is done in separate processes, we send the time series
# windows to these threads via two queues
online_logger.info("Initializing Queues")
for key in self.potentials.keys():
self.queue[key] = multiprocessing.Queue()
def flow_generator(key):
"""create a generator to yield all the abri flow windows"""
# Yield all windows until a None item is found in the queue
while True:
window = self.queue[key].get(block=True, timeout=None)
if window == None: break
yield window
# Create the actual data flows
for key in self.potentials.keys():
if self.operation == "train":
self.node_chains[key] = NodeChainFactory.flow_from_yaml(
Flow_Class=NodeChain,
flow_spec=file(self.potentials[key]["node_chain"]))
self.node_chains[key][0].set_generator(flow_generator(key))
flow = open(self.potentials[key]["node_chain"])
elif self.operation in ("prewindowing", "prewindowing_offline"):
online_logger.info("loading prewindowing flow..")
online_logger.info(
"file: " + str(self.potentials[key]["prewindowing_flow"]))
self.node_chains[key] = NodeChainFactory.flow_from_yaml(
Flow_Class=NodeChain,
flow_spec=file(self.potentials[key]["prewindowing_flow"]))
self.node_chains[key][0].set_generator(flow_generator(key))
flow = open(self.potentials[key]["prewindowing_flow"])
elif self.operation == "prewindowed_train":
self.node_chains[key] = NodeChainFactory.flow_from_yaml(
Flow_Class=NodeChain,
flow_spec=file(self.potentials[key]["postprocess_flow"]))
replace_start_and_end_markers = False
final_collection = TimeSeriesDataset()
final_collection_path = os.path.join(
self.prewindowed_data_directory, key, "all_train_data")
# delete previous training collection
if os.path.exists(final_collection_path):
online_logger.info(
"deleting old training data collection for " + key)
shutil.rmtree(final_collection_path)
# load all prewindowed collections and
# append data to the final collection
prewindowed_sets = \
glob.glob(os.path.join(self.prewindowed_data_directory, key, "*"))
if len(prewindowed_sets) == 0:
online_logger.error(
"Couldn't find data, please do prewindowing first!")
raise Exception
online_logger.info("concatenating prewindowed data from " +
str(prewindowed_sets))
for s, d in enumerate(prewindowed_sets):
collection = BaseDataset.load(d)
data = collection.get_data(0, 0, "train")
for d, (sample, label) in enumerate(data):
if replace_start_and_end_markers:
# in case we concatenate multiple 'Window' labeled
# sets we have to remove every start- and endmarker
for k in sample.marker_name.keys():
# find '{S,s} 8' or '{S,s} 9'
m = re.match("^s\s{0,2}[8,9]{1}$", k,
re.IGNORECASE)
if m is not None:
online_logger.info(
str("remove %s from %d %d" %
(m.group(), s, d)))
del (sample.marker_name[m.group()])
if s == len(prewindowed_sets)-1 and \
d == len(data)-1:
# insert endmarker
sample.marker_name["S 9"] = [0.0]
online_logger.info("added endmarker" + str(s) +
" " + str(d))
if s == 0 and d == 0:
# insert startmarker
sample.marker_name["S 8"] = [0.0]
online_logger.info("added startmarker" +
str(s) + " " + str(d))
final_collection.add_sample(sample, label, True)
# save final collection (just for debugging)
os.mkdir(final_collection_path)
final_collection.store(final_collection_path)
online_logger.info("stored final collection at " +
final_collection_path)
# load final collection again for training
online_logger.info("loading data from " +
final_collection_path)
self.prewindowed_data[key] = BaseDataset.load(
final_collection_path)
self.node_chains[key][0].set_input_dataset(
self.prewindowed_data[key])
flow = open(self.potentials[key]["postprocess_flow"])
# create window_stream for every potential
if self.operation in ("prewindowing"):
window_spec_file = os.path.join(
spec_base, "node_chains", "windower",
self.potentials[key]["windower_spec_path_train"])
self.window_stream[key] = \
self.stream_manager.request_window_stream(window_spec_file,
nullmarker_stride_ms = self.nullmarker_stride_ms)
elif self.operation in ("prewindowing_offline"):
pass
elif self.operation in ("train"):
pass
self.node_chain_definitions[key] = yaml.load(flow)
flow.close()
# TODO: check if the prewindowing flow is still needed when using the stream mode!
if self.operation in ("train"):
online_logger.info("Removing old flows...")
try:
shutil.rmtree(self.flow_storage)
except:
online_logger.info("Could not delete flow storage directory")
os.mkdir(self.flow_storage)
elif self.operation in ("prewindowing", "prewindowing_offline"):
# follow this policy:
# - delete prewindowed data older than 12 hours
# - always delete trained/stored flows
now = datetime.datetime.now()
then = now - datetime.timedelta(hours=12)
if not os.path.exists(self.prewindowed_data_directory):
os.mkdir(self.prewindowed_data_directory)
if not os.path.exists(self.flow_storage):
os.mkdir(self.flow_storage)
for key in self.potentials.keys():
found = self.find_files_older_than(then, \
os.path.join(self.prewindowed_data_directory, key))
if found is not None:
for f in found:
online_logger.info(
str("recursively deleting files in \'%s\'" % f))
try:
shutil.rmtree(os.path.abspath(f))
except Exception as e:
# TODO: find a smart solution for this!
pass # dir was probably already deleted..
if os.path.exists(
os.path.join(self.prewindowed_data_directory, key,
"all_train_data")):
shutil.rmtree(
os.path.join(self.prewindowed_data_directory, key,
"all_train_data"))
online_logger.info(
"deleted concatenated training data for " + key)
online_logger.info("Training preparations finished")
return 0
