def _compute_scores(metrics, real_data, synthetic_data, metadata, output):
metrics = get_metrics(metrics, metadata)
metric_args = _prepare_metric_args(real_data, synthetic_data, metadata)
scores = []
output['scores'] = scores
for metric_name, metric in metrics.items():
scores.append({
'metric': metric_name,
'error': 'Metric Timeout',
})
output['scores'] = scores # re-inject list to multiprocessing output
error = None
score = None
start = datetime.utcnow()
try:
LOGGER.info('Computing %s on dataset %s', metric_name, metadata._metadata['name'])
score = metric.compute(*metric_args)
except Exception:
LOGGER.exception('Metric %s failed on dataset %s. Skipping.',
metric_name, metadata._metadata['name'])
_, error = format_exception()
scores[-1].update({
'score': score,
'error': error,
'metric_time': (datetime.utcnow() - start).total_seconds()
})
output['scores'] = scores # re-inject list to multiprocessing output
def _score(synthesizer,
metadata,
metrics,
iteration,
output=None,
max_rows=None):
if output is None:
output = {}
name = synthesizer['name']
output['timeout'] = True # To be deleted if there is no error
output['error'] = 'Load Timeout' # To be deleted if there is no error
try:
real_data = load_tables(metadata, max_rows)
LOGGER.info('Running %s on %s dataset %s; iteration %s; %s', name,
metadata.modality, metadata._metadata['name'], iteration,
used_memory())
output[
'error'] = 'Synthesizer Timeout' # To be deleted if there is no error
synthetic_data, model_time = _synthesize(synthesizer, real_data.copy(),
metadata)
output['synthetic_data'] = synthetic_data
output['model_time'] = model_time.total_seconds()
LOGGER.info('Scoring %s on %s dataset %s; iteration %s; %s', name,
metadata.modality, metadata._metadata['name'], iteration,
used_memory())
del output[
'error'] # No error so far. _compute_scores tracks its own errors by metric
_compute_scores(metrics, real_data, synthetic_data, metadata, output)
output['timeout'] = False # There was no timeout
except Exception:
LOGGER.exception('Error running %s on dataset %s; iteration %s', name,
metadata._metadata['name'], iteration)
exception, error = format_exception()
output['exception'] = exception
output['error'] = error
output['timeout'] = False # There was no timeout
finally:
LOGGER.info('Finished %s on dataset %s; iteration %s; %s', name,
metadata._metadata['name'], iteration, used_memory())
return output