def event_matches_target(self, message):
metric_name, data_type = pyromq.unpack_name(message['name'])
if self.target_metric != "*" and self.target_metric != metric_name:
return False
if self.target_data != "*" and self.target_data != data_type:
return False
return True
def metric_events(self, message):
"""
Metric events are a bit more complex than training events simply because
a metric might evaluate each of the classes separately (as is the case
for the IndividualF1 measure).
More specifically, there are two sitautions corresponding to two ways
that metric events can emit information:
1. message is of format {"name": metric_name, "value": numeric_value}
2. message is of format {"name": metric_name, "value": a_dictionary}
In the second case, the dictionary is a mapping from specific evaluations
to numeric values:
{specific_name: specific_value, ...}
:param message: see the primary portion of this docstring.
:type message: dict
"""
if not self.enabled:
return
metric_name, data_type = pyromq.unpack_name(message['name'])
writer_name = '{}/{}'.format(metric_name, data_type)
self.epoch_averages[writer_name].append(message['value'])
writer_name += '/long'
if isinstance(message['value'], dict):
self.writer.add_scalars(writer_name, message['value'].items(),
self.metric_event_counter)
else:
self.writer.add_scalar(writer_name, message['value'],
self.metric_event_counter)
self.metric_event_counter += 1
def metric_events(self, message):
metric_name, data_type = pyromq.unpack_name(message['name'])
if data_type == 'val' and metric_name == self._target_metric_name:
self._cached_validation_values.append(message['value'])
def metric_events(self, message):
"""
Metric events are a bit more complex than training events simply because
a metric might evaluate each of the classes separately (as is the case
for the IndividualF1 measure).
More specifically, there are two sitautions corresponding to two ways
that metric events can emit information:
1. message is of format {"name": metric_name, "value": numeric_value}
2. message is of format {"name": metric_name, "value": a_dictionary}
In the second case, the dictionary is a mapping from specific evaluations
to numeric values:
{specific_name: specific_value, ...}
:param message: see the primary portion of this docstring.
:type message: dict
"""
metric_name, data_type = pyromq.unpack_name(message['name'])
if isinstance(message['value'], dict):
# this is the individual F1 case. Custom handling here, plans to
# streamline in next iteration of code condensing.
entries = {
"experiment": [],
"metric_name": [],
"data_type": [],
"value": [],
"timestamp": [],
"epoch": []
}
stamp = datetime.now()
for specific_metric_name, value in message['value'].items():
entries['experiment'].append(self.experiment_uid)
entries['metric_name'].append(metric_name + "." +
specific_metric_name.lower())
entries['data_type'].append(data_type)
entries['value'].append(value)
entries['timestamp'].append(stamp)
entries['epoch'].append(self.epoch)
else:
entries = {
"experiment": [self.experiment_uid],
"metric_name": [metric_name],
"data_type": [data_type],
"value": [message['value']],
"timestamp": [datetime.now()],
'epoch': [self.epoch]
}
for _ in range(constants.NUM_SQL_RETRIES):
try:
pd.DataFrame(entries).to_sql("metric_events",
self.conn,
if_exists="append")
return
except (DatabaseError, sqlite3.OperationalError):
time.sleep(1)
pd.DataFrame(entries).to_sql("metric_events",
self.conn,
if_exists="append")