def __init__(self, SUFFIX=SUFFIX, **kwargs):
self._x_values = []
self._results = OrderedDict()
self._filename = None
self._loaded_from = None
self._absolute = False
self._raw_values = {}
self._raw_keys = None
self.metadata = SeparatorDict(kwargs, sep=":")
self.SUFFIX = SUFFIX
self.t0 = None
if 'TIME' not in self.metadata or self.metadata['TIME'] is None:
self.metadata['TIME'] = datetime.utcnow()
if 'NAME' not in self.metadata or self.metadata['NAME'] is None:
raise RuntimeError("Missing name for resultset")
if 'DATA_FILENAME' not in self.metadata \
or self.metadata['DATA_FILENAME'] is None:
self.metadata['DATA_FILENAME'] = self.dump_filename
if not self.metadata['DATA_FILENAME'].endswith(self.SUFFIX):
self.metadata['DATA_FILENAME'] += self.SUFFIX
self._filename = self.metadata['DATA_FILENAME']
self._label = None
if 'TITLE' in self.metadata and self.metadata['TITLE']:
self.title = "%s - %s" % (self.metadata['NAME'],
self.metadata['TITLE'])
self.long_title = "%s - %s" % (
self.title,
format_date(self.metadata['TIME'], fmt="%Y-%m-%d %H:%M:%S"))
else:
self.title = "%s - %s" % (self.metadata['NAME'],
format_date(self.metadata['TIME'],
fmt="%Y-%m-%d %H:%M:%S"))
self.long_title = self.title
def __init__(self, SUFFIX=SUFFIX, **kwargs):
self._x_values = []
self._results = OrderedDict()
self._filename = None
self._loaded_from = None
self._absolute = False
self._raw_values = {}
self._raw_keys = None
self.metadata = SeparatorDict(kwargs, sep=":")
self.SUFFIX = SUFFIX
self.t0 = None
if 'TIME' not in self.metadata or self.metadata['TIME'] is None:
self.metadata['TIME'] = datetime.utcnow()
if 'NAME' not in self.metadata or self.metadata['NAME'] is None:
raise RuntimeError("Missing name for resultset")
if 'DATA_FILENAME' not in self.metadata \
or self.metadata['DATA_FILENAME'] is None:
self.metadata['DATA_FILENAME'] = self.dump_filename
if not self.metadata['DATA_FILENAME'].endswith(self.SUFFIX):
self.metadata['DATA_FILENAME'] += self.SUFFIX
self._filename = self.metadata['DATA_FILENAME']
self._label = None
if 'TITLE' in self.metadata and self.metadata['TITLE']:
self.title = "%s - %s" % (self.metadata['NAME'],
self.metadata['TITLE'])
self.long_title = "%s - %s" % (self.title, format_date(
self.metadata['TIME'], fmt="%Y-%m-%d %H:%M:%S"))
else:
self.title = "%s - %s" % (self.metadata['NAME'],
format_date(self.metadata['TIME'],
fmt="%Y-%m-%d %H:%M:%S"))
self.long_title = self.title
def run(self):
if self.settings.INPUT:
return self.load_input(self.settings)
elif self.settings.BATCH_NAMES:
start_time = self.settings.TIME
sys.stderr.write("Started batch sequence at %s.\n" % format_date(start_time, fmt="%Y-%m-%d %H:%M:%S"))
if len(self.settings.BATCH_NAMES) == 1 and self.settings.BATCH_NAMES[0] == 'ALL':
sys.stderr.write("Running all batches.\n")
batches = self.batches.keys()
else:
batches = self.settings.BATCH_NAMES
total_time = sum([self.get_batch_runtime(b) for b in batches])
if total_time > 0:
sys.stderr.write("Estimated total runtime: %s\n" % timedelta(seconds=total_time))
for b in batches:
try:
sys.stderr.write("Running batch '%s'.\n" % b)
self.run_batch(b)
except RuntimeError:
raise
except Exception as e:
if self.settings.DEBUG_ERROR:
raise
raise RuntimeError("Error while running batch '%s': %r." % (b, e))
end_time = datetime.utcnow()
sys.stderr.write("Ended batch sequence at %s. %s %d tests in %s.\n" % (format_date(end_time, fmt="%Y-%m-%d %H:%M:%S"),
"Ran" if not self.settings.BATCH_DRY else 'Would have run',
self.tests_run, (end_time - start_time)))
return True
else:
return self.run_test(self.settings, self.settings.DATA_DIR, True)
def _gen_filename(self):
if self._filename is not None:
return self._filename
if 'TITLE' in self.metadata and self.metadata['TITLE']:
return "%s-%s.%s%s" % (self.metadata['NAME'],
format_date(self.metadata['TIME']).replace(":", ""),
re.sub("[^A-Za-z0-9]", "_", self.metadata['TITLE'])[:50],
self.SUFFIX)
else:
return "%s-%s%s" % (self.metadata['NAME'], format_date(self.metadata['TIME']).replace(":", ""), self.SUFFIX)
def _gen_filename(self):
if self._filename is not None:
return self._filename
if 'TITLE' in self.metadata and self.metadata['TITLE']:
return "%s-%s.%s%s" % (self.metadata['NAME'],
format_date(self.metadata['TIME']).replace(":", ""),
re.sub("[^A-Za-z0-9]", "_", self.metadata['TITLE'])[:50],
self.SUFFIX)
else:
return "%s-%s%s" % (self.metadata['NAME'], format_date(self.metadata['TIME']).replace(":", ""), self.SUFFIX)
def _gen_filename(self):
if self._filename is not None:
return self._filename
if 'TITLE' in self.metadata and self.metadata['TITLE']:
name = "%s-%s.%%s%s" % (self.metadata['NAME'],
format_date(self.metadata['TIME']).replace(":", ""),
self.SUFFIX)
title_len = MAX_FILENAME_LEN - len(name) + 2
return name % re.sub("[^A-Za-z0-9]", "_", self.metadata['TITLE'])[:title_len]
else:
return "%s-%s%s" % (self.metadata['NAME'], format_date(self.metadata['TIME']).replace(":", ""), self.SUFFIX)
def _gen_filename(self):
if self._filename is not None:
return self._filename
if 'TITLE' in self.metadata and self.metadata['TITLE']:
name = "%s-%s.%%s%s" % (self.metadata['NAME'],
format_date(self.metadata['TIME']).replace(
":", ""), self.SUFFIX)
title_len = MAX_FILENAME_LEN - len(name) + 2
return name % re.sub("[^A-Za-z0-9-]", "_",
self.metadata['TITLE'])[:title_len]
else:
return "%s-%s%s" % (self.metadata['NAME'],
format_date(self.metadata['TIME']).replace(
":", ""), self.SUFFIX)
def _annotate_plot(self, skip_title=False):
titles = []
title_y=1
if self.override_title:
self.title_obj = self.figure.suptitle(self.override_title,
fontsize=14, y=title_y)
titles.append(self.title_obj)
self.title = self.override_title
elif self.print_title:
plot_title = self.description
if 'description' in self.config:
plot_title += "\n" + self.config['description']
if self.metadata['TITLE'] and not skip_title:
plot_title += "\n" + self.metadata['TITLE']
if 'description' in self.config and self.metadata['TITLE'] and not skip_title:
title_y=1.00
self.title_obj = self.figure.suptitle(plot_title, fontsize=14, y=title_y)
titles.append(self.title_obj)
self.title = plot_title
else:
self.title_obj = None
if self.annotate:
annotation_string = "Local/remote: %s/%s - Time: %s - Length/step: %ds/%.2fs" % (
self.metadata['LOCAL_HOST'], self.metadata['HOST'],
format_date(self.metadata['TIME']),
self.metadata['LENGTH'], self.metadata['STEP_SIZE'])
self.annotation_obj = self.figure.text(0.5, 0.0, annotation_string,
horizontalalignment='center',
verticalalignment='bottom',
fontsize=8)
else:
self.annotation_obj = None
return titles
def run(self):
if self.settings.INPUT:
return self.load_input(self.settings)
elif self.settings.BATCH_NAMES:
start_time = self.settings.TIME
self.settings.BATCH_UUID = str(uuid.uuid4())
self.settings.LOG_FILE = None # batch run will generate logs
logger.info("Started batch run %s at %s.",
self.settings.BATCH_UUID,
format_date(start_time, fmt="%Y-%m-%d %H:%M:%S"))
if len(self.settings.BATCH_NAMES) == 1 and \
self.settings.BATCH_NAMES[0] == 'ALL':
logger.info("Running all batches.")
batches = self.batches.keys()
else:
batches = self.settings.BATCH_NAMES
runtimes = [self.get_batch_runtime(b) for b in batches]
total_time, total_n = map(sum, zip(*runtimes))
if total_time > 0:
logger.info("Estimated total runtime: %s (%d tests)",
timedelta(seconds=total_time), total_n)
for b in batches:
try:
logger.info("Running batch '%s'.", b)
self.run_batch(b)
except RuntimeError:
raise
except Exception as e:
if self.settings.DEBUG_ERROR:
raise
raise RuntimeError("Error while running batch '%s': %r." %
(b, e))
end_time = datetime.utcnow()
logger.info(
"Ended batch sequence at %s. %s %d tests in %s.",
format_date(end_time, fmt="%Y-%m-%d %H:%M:%S"),
"Ran" if not self.settings.BATCH_DRY else
'Would have run', # noqa: E501
self.tests_run,
(end_time - start_time))
return True
else:
return self.run_test(self.settings, self.settings.DATA_DIR, True)
def run(self):
if self.settings.INPUT:
return self.load_input(self.settings)
elif self.settings.BATCH_NAMES:
start_time = self.settings.TIME
self.settings.BATCH_UUID = str(uuid.uuid4())
self.settings.LOG_FILE = None # batch run will generate logs
logger.info("Started batch run %s at %s.",
self.settings.BATCH_UUID,
format_date(start_time, fmt="%Y-%m-%d %H:%M:%S"))
if len(self.settings.BATCH_NAMES) == 1 and \
self.settings.BATCH_NAMES[0] == 'ALL':
logger.info("Running all batches.")
batches = self.batches.keys()
else:
batches = self.settings.BATCH_NAMES
runtimes = [self.get_batch_runtime(b) for b in batches]
total_time, total_n = map(sum, zip(*runtimes))
if total_time > 0:
logger.info("Estimated total runtime: %s (%d tests)",
timedelta(seconds=total_time),
total_n)
for b in batches:
try:
logger.info("Running batch '%s'.", b)
self.run_batch(b)
except RuntimeError:
raise
except Exception as e:
if self.settings.DEBUG_ERROR:
raise
raise RuntimeError("Error while running batch '%s': %r."
% (b, e))
end_time = datetime.utcnow()
logger.info("Ended batch sequence at %s. %s %d tests in %s.",
format_date(end_time, fmt="%Y-%m-%d %H:%M:%S"),
"Ran" if not self.settings.BATCH_DRY else 'Would have run', # noqa: E501
self.tests_run,
(end_time - start_time))
return True
else:
return self.run_test(self.settings, self.settings.DATA_DIR, True)
def run(self):
if self.settings.INPUT:
return self.load_input(self.settings)
elif self.settings.BATCH_NAMES:
start_time = self.settings.TIME
sys.stderr.write("Started batch sequence at %s.\n" %
format_date(start_time, fmt="%Y-%m-%d %H:%M:%S"))
if len(self.settings.BATCH_NAMES
) == 1 and self.settings.BATCH_NAMES[0] == 'ALL':
sys.stderr.write("Running all batches.\n")
batches = self.batches.keys()
else:
batches = self.settings.BATCH_NAMES
total_time = sum([self.get_batch_runtime(b) for b in batches])
if total_time > 0:
sys.stderr.write("Estimated total runtime: %s\n" %
timedelta(seconds=total_time))
for b in batches:
try:
sys.stderr.write("Running batch '%s'.\n" % b)
self.run_batch(b)
except RuntimeError:
raise
except Exception as e:
if self.settings.DEBUG_ERROR:
raise
raise RuntimeError("Error while running batch '%s': %r." %
(b, e))
end_time = datetime.utcnow()
sys.stderr.write(
"Ended batch sequence at %s. %s %d tests in %s.\n" %
(format_date(end_time, fmt="%Y-%m-%d %H:%M:%S"),
"Ran" if not self.settings.BATCH_DRY else 'Would have run',
self.tests_run, (end_time - start_time)))
return True
else:
return self.run_test(self.settings, self.settings.DATA_DIR, True)
def expand_argsets(self,
batch,
argsets,
batch_time,
batch_name,
print_status=True,
no_shuffle=False):
sets = itertools.product(*argsets)
if self.settings.BATCH_SHUFFLE and not no_shuffle:
sets = list(sets)
random.shuffle(sets)
for argset in sets:
rep = argset[-1]
argset = argset[:-1]
settings = self.settings.copy()
if print_status:
logger.info(" args:%s rep:%02d%s", ",".join(argset), rep,
" (dry run)" if settings.BATCH_DRY else "")
settings.FORMAT = 'null'
settings.BATCH_NAME = batch_name
settings.BATCH_TIME = batch_time
settings.TIME = datetime.utcnow()
expand_vars = {
'repetition':
"%02d" % rep,
'batch_time':
format_date(settings.BATCH_TIME, fmt="%Y-%m-%dT%H%M%S")
}
for arg in argset:
if arg not in self.args:
raise RuntimeError("Invalid arg: '%s'." % arg)
expand_vars.update(self.args[arg])
b = self.apply_args(batch, expand_vars, settings)
if 'test_name' not in b:
raise RuntimeError("Missing test name.")
settings.load_rcvalues(b.items(), override=True)
settings.NAME = b['test_name']
settings.load_test(informational=True)
settings.DATA_FILENAME = self.gen_filename(settings, b, argset,
rep)
yield b, settings
def expand_argsets(self,
batch,
argsets,
batch_time,
batch_name,
print_status=True):
for argset in itertools.product(*argsets):
rep = argset[-1]
argset = argset[:-1]
settings = self.settings.copy()
if print_status:
sys.stderr.write(" args:%s rep:%02d" % (",".join(argset), rep))
if settings.BATCH_DRY:
sys.stderr.write(" (dry run)")
sys.stderr.write(".\n")
settings.FORMAT = 'null'
settings.BATCH_NAME = batch_name
settings.BATCH_TIME = batch_time
settings.TIME = datetime.utcnow()
expand_vars = {
'repetition':
"%02d" % rep,
'batch_time':
format_date(settings.BATCH_TIME, fmt="%Y-%m-%dT%H%M%S")
}
for arg in argset:
if not arg in self.args:
raise RuntimeError("Invalid arg: '%s'." % arg)
expand_vars.update(self.args[arg])
b = self.apply_args(batch, expand_vars, settings)
if not 'test_name' in b:
raise RuntimeError("Missing test name.")
settings.load_rcvalues(b.items(), override=True)
settings.NAME = b['test_name']
settings.load_test(informational=settings.BATCH_DRY)
settings.DATA_FILENAME = self.gen_filename(settings, b, argset,
rep)
yield b, settings
def save_intermediate(self, new_results, config, orig_meta):
orig_meta = orig_meta.copy()
del orig_meta['TITLE']
if self.save_dir:
t = datetime.utcnow()
series = config['series']
# Can't serialise 'source' Glob objects
for s in series:
if 'source' in s:
del s['source']
for i, r in enumerate(new_results):
r.meta().update(orig_meta)
r.meta("FROM_COMBINER", self.__class__.__name__)
r.meta("COMBINER_SERIES", series)
r.meta("COMBINER_PLOT", config['plot_name'])
r._filename = "%s-%s-%s-%02d%s" % (
config['plot_name'], self.__class__.__name__,
format_date(t).replace(":", ""), i, r.SUFFIX)
r.dump_dir(self.save_dir)
def expand_argsets(self, batch, argsets, batch_time, batch_name,
print_status=True, no_shuffle=False):
sets = itertools.product(*argsets)
if self.settings.BATCH_SHUFFLE and not no_shuffle:
sets = list(sets)
random.shuffle(sets)
for argset in sets:
rep = argset[-1]
argset = argset[:-1]
settings = self.settings.copy()
if print_status:
logger.info(" args:%s rep:%02d%s", ",".join(argset), rep,
" (dry run)" if settings.BATCH_DRY else "")
settings.FORMAT = 'null'
settings.BATCH_NAME = batch_name
settings.BATCH_TIME = batch_time
settings.TIME = datetime.utcnow()
expand_vars = {'repetition': "%02d" % rep,
'batch_time': format_date(settings.BATCH_TIME,
fmt="%Y-%m-%dT%H%M%S")}
for arg in argset:
if arg not in self.args:
raise RuntimeError("Invalid arg: '%s'." % arg)
expand_vars.update(self.args[arg])
b = self.apply_args(batch, expand_vars, settings)
if 'test_name' not in b:
raise RuntimeError("Missing test name.")
settings.load_rcvalues(b.items(), override=True)
settings.NAME = b['test_name']
settings.load_test(informational=True)
settings.DATA_FILENAME = self.gen_filename(settings, b, argset, rep)
yield b, settings
def save_intermediate(self, new_results, config, orig_meta):
orig_meta = orig_meta.copy()
del orig_meta['TITLE']
if self.save_dir:
t = datetime.utcnow()
series = config['series']
# Can't serialise 'source' Glob objects
for s in series:
if 'source' in s:
del s['source']
for i, r in enumerate(new_results):
r.meta().update(orig_meta)
r.meta("FROM_COMBINER", self.__class__.__name__)
r.meta("COMBINER_SERIES", series)
r.meta("COMBINER_PLOT", config['plot_name'])
r._filename = "%s-%s-%s-%02d%s" % (
config['plot_name'],
self.__class__.__name__,
format_date(t).replace(":", ""),
i,
r.SUFFIX)
r.dump_dir(self.save_dir)
def expand_argsets(self, batch, argsets, batch_time, batch_name,
print_status=True):
for argset in itertools.product(*argsets):
rep = argset[-1]
argset = argset[:-1]
settings = self.settings.copy()
if print_status:
sys.stderr.write(" args:%s rep:%02d" % (",".join(argset), rep))
if settings.BATCH_DRY:
sys.stderr.write(" (dry run)")
sys.stderr.write(".\n")
settings.FORMAT = 'null'
settings.BATCH_NAME = batch_name
settings.BATCH_TIME = batch_time
settings.TIME = datetime.utcnow()
expand_vars = {'repetition': "%02d" % rep,
'batch_time': format_date(settings.BATCH_TIME,
fmt="%Y-%m-%dT%H%M%S")}
for arg in argset:
if arg not in self.args:
raise RuntimeError("Invalid arg: '%s'." % arg)
expand_vars.update(self.args[arg])
b = self.apply_args(batch, expand_vars, settings)
if 'test_name' not in b:
raise RuntimeError("Missing test name.")
settings.load_rcvalues(b.items(), override=True)
settings.NAME = b['test_name']
settings.load_test(informational=settings.BATCH_DRY)
settings.DATA_FILENAME = self.gen_filename(settings, b, argset, rep)
yield b, settings
def serialise_metadata(self):
metadata = self.metadata.copy()
for t in TIME_SETTINGS:
if t in metadata and metadata[t] is not None:
metadata[t] = format_date(metadata[t], utc=True)
return metadata
def label(self):
return self._label or self.metadata["TITLE"] \
or format_date(self.metadata["TIME"])
def serialise_metadata(self):
metadata = self.metadata.copy()
for t in TIME_SETTINGS:
if t in metadata and metadata[t] is not None:
metadata[t] = format_date(metadata[t], utc=True)
return metadata
def label(self):
return self._label or self.metadata["TITLE"] \
or format_date(self.metadata["TIME"])
