def process(self, data_object):
'''
Process a given file
:param data_object: MetricManager instance
:return: Updated data_object reference
'''
analysis_metrics = data_object.analyze()
# merge the data sets
for group_key, metric_key, metric_obj in metric_iter(analysis_metrics):
existing = metric_get(group_key, metric_key, self.merged_data)
# new entry?
if existing is None:
metric_add(metric_obj, self.merged_data, group_key, metric_key)
continue
# merge existing metrics
existing.merge(metric_obj)
# merge config
for config, config_value in data_object.get_config().items():
# new entry
if config not in self.merged_config:
self.merged_config[config] = config_value
continue
# merge existing entry, keep common values
if self.merged_config[config] != config_value:
self.merged_config[config] = None
return data_object
def _process_metrics(self, metric_seq, folder_path, file_filter):
# check if we already have the data for each metric
metric_data = {}
not_loaded_seq = []
# try graphs first
for g_name, m_name, metric in metric_seq:
# data was found
if self._have_data(g_name, m_name) and metric.load(
self._get_data(g_name, m_name)):
logging.debug('Loading existing data for %s:%s', g_name,
m_name)
# check if we have graph data
if hasattr(metric, 'create_graph'):
if self._get_graph(g_name, m_name) is None:
logging.debug(
'Generating graph data from existing data')
self._set_graph(metric.create_graph(), g_name, m_name)
if hasattr(metric, 'create_summation'):
if metric.force_summation() or self._get_sum(
g_name, m_name) is None:
logging.debug(
'Generating metric data from existing data')
self._set_sum(metric.create_summation(), g_name,
m_name)
metric_add(metric, metric_data, g_name, m_name)
else:
# no existing data found, will need to calculate it later
not_loaded_seq.append((g_name, m_name, metric))
# run the missing graphs
metrics_list = [m_inst for _, _, m_inst in not_loaded_seq]
if len(metrics_list) > 0:
file_reader = JSONFileReader(metrics_list)
finder = FileFinder([file_reader])
finder.process(folder_path, file_filter)
# save the results of running the metrics
for g_name, m_name, metric_obj in not_loaded_seq:
self._set_data(metric_obj.to_string(), g_name, m_name)
if hasattr(metric_obj, 'create_graph'):
self._set_graph(metric_obj.create_graph(), g_name, m_name)
if hasattr(metric_obj, 'create_summation'):
self._set_sum(metric_obj.create_summation(), g_name,
m_name)
metric_add(metric_obj, metric_data, g_name, m_name)
return metric_data
def process(self, data_object):
'''
Process a given file
:param data_object: MetricManager instance
:return: Updated data_object reference
'''
self.add_column('experiment_name')
# ensure we have all the data
data_object.analyze()
# check columns first
for group_key, _, metric_obj in metric_iter(
data_object.metrics['summations']):
for metric in metric_obj:
metric_name = metric['full_name']
exists = metric_get(group_key, metric_name, self.metric_map)
# adding new column
if exists is None:
self.add_column(metric_name)
position = len(self.data_frame.columns) + \
len(self._columns_to_add) - 1
metric_add(position, self.metric_map, group_key,
metric_name)
# populate the values for the table row
row = [
None for _ in range(
0,
len(self.data_frame.columns) + len(self._columns_to_add))
]
row[0] = Configuration.get_hash_name(
data_object.get_config(), [self.experiment_variable, 'repeat'])
for group_key, _, metric_obj in metric_iter(
data_object.metrics['summations']):
for metric in metric_obj:
metric_name = metric['full_name']
position = metric_get(group_key, metric_name, self.metric_map)
if position is None:
raise Exception('Something very bad happened')
row[position] = metric['value']
self.add_row(row)
return data_object
def _set_sum(self, value, *args):
self.is_dirty = True
metric_add(value, self.metrics['summations'], *args)
def _set_graph(self, value, *args):
self.is_dirty = True
metric_add(value, self.metrics['graphs'], *args)
def _set_data(self, value, *args):
self.is_dirty = True
metric_add(value, self.metrics['data'], *args)