def create_dataset(dataset_type, dir):
dataset = models.DataSet()
dataset.type = dataset_type
dataset.family_type = '{}_family'.format(dataset_type)
dataset.name = '{}/dataset'.format(dir)
models.db.session.add(dataset)
metric = models.Metric()
metric.type = '{}2ed'.format(dataset_type)
metric.dataset = dataset
models.db.session.add(metric)
return dataset, metric
def create_dataset(dataset_type):
dataset = models.DataSet()
dataset.type = dataset_type
dataset.family_type = '{}_family'.format(dataset_type)
dataset.name = '/path/to/dataset_{}'.format(dataset_type)
models.db.session.add(dataset)
metric = models.Metric()
metric.type = '{}2ed'.format(dataset_type)
metric.duration = datetime.timedelta(seconds=9.0001)
metric.dataset = dataset
models.db.session.add(metric)
return dataset, metric
def collect_metric(task_type, job, dataset_uid):
begin = datetime.datetime.utcnow()
yield
end = datetime.datetime.utcnow()
try:
dataset = models.DataSet.find_by_uid(dataset_uid)
metric = models.Metric()
metric.job = job
metric.dataset = dataset
metric.type = task_type
metric.duration = end - begin
models.db.session.add(metric)
models.db.session.commit()
except:
logger = logging.getLogger(__name__)
logger.exception('unable to persist Metrics data: ')
