def build_exploration_report(self, report_file):
""" Builds exploration report for inference and single device training.
"""
header = "%-20s %-10s" % ('Network', 'Device')
for batch in self.batches:
header = "%s %-10s" % (header, batch)
report = []
json_report = {'data': []}
for net in self.nets:
for device in self.devices:
profile = {
'net': net,
'device': device,
'time': [],
'throughput': []
}
profile_ok = False
for batch in self.batches:
key = '{0}_{1}_{2}'.format(net, device, batch)
batch_tm = throughput = -1
if key in self.cache and self.cache[key] > 0:
batch_tm = self.cache[key]
profile_ok = True
throughput = int(batch * (1000.0 / batch_tm))
json_profile = SummaryBuilder.default_json_profile(net, 'strong', batch)
json_profile['perf']['data']['1'] = batch_tm
json_report['data'].append(json_profile)
profile['time'].append(round(batch_tm, 3))
profile['throughput'].append(throughput)
if profile_ok:
report.append(profile)
SummaryBuilder.print_report_txt(BATCH_TM_TITLE, header, report, 'net', 'device', 'time')
SummaryBuilder.print_report_txt(IPS_TITLE, header, report, 'net', 'device', 'throughput')
DictUtils.dump_json_to_file(json_report, report_file)
def build_strong_scaling_report(self, jsonfile):
""" Builds strong scaling report for multi-GPU training.
"""
header = "%-20s %-10s" % ('Network', 'Batch')
for device in self.devices:
header = "%s %-10s" % (header, (1 + device.count(',')))
report = []
json_report = {'data': []}
for net in self.nets:
for batch in self.batches:
profile = {
'net': net,
'batch': batch,
'time': [],
'throughput': [],
'efficiency': [],
'speedup': []
}
json_profile = SummaryBuilder.default_json_profile(
net, 'strong', batch)
profile_ok = False
# device here is '0', '0,1', '0,1,2,3' ...
for device in self.devices:
key = '{0}_{1}_{2}'.format(net, device, batch)
batch_tm = throughput = efficiency = speedup = -1
num_devices = 1 + device.count(',')
if key in self.cache:
batch_tm = self.cache[key]
throughput = int(batch * (1000.0 / batch_tm))
json_profile['perf']['data'][str(
num_devices)] = batch_tm
if len(profile['throughput']) == 0:
speedup = 1
else:
speedup = 1.0 * throughput / profile['throughput'][
0]
if len(profile['efficiency']) == 0:
efficiency = 100.00
profile_ok = True
elif profile['time'][0] > 0:
efficiency = int(10000.0 * profile['time'][0] /
(num_devices * batch_tm)) / 100.0
profile_ok = True
profile['time'].append(batch_tm)
profile['throughput'].append(throughput)
profile['efficiency'].append(efficiency)
profile['speedup'].append(speedup)
if profile_ok:
report.append(profile)
json_report['data'].append(json_profile)
SummaryBuilder.print_report_txt(BATCH_TM_TITLE, header, report, 'net',
'batch', 'time')
SummaryBuilder.print_report_txt(IPS_TITLE, header, report, 'net',
'batch', 'throughput')
SummaryBuilder.print_report_txt(SPEEDUP_TITLE, header, report, 'net',
'batch', 'speedup')
SummaryBuilder.print_report_txt("Efficiency = 100% * t1 / (N * tN)",
header, report, 'net', 'batch',
'efficiency')
DictUtils.dump_json_to_file(json_report, jsonfile)
def build_weak_scaling_report(self, jsonfile):
""" Builds weak scaling report for multi-GPU training.
"""
header = "%-20s %-10s" % ('Network', 'Batch')
for device in self.devices:
header = "%s %-10d" % (header, (1 + device.count(',')))
report = []
json_report = {'data': []}
for net in self.nets:
for batch in self.batches:
# batch is the base 'batch size' i.e. for a one GPU
profile = {
'net': net, # network name
'batch': batch, # per device batch size
'time': [], # batch times
'throughput': [], # throughput
'efficiency': [], # efficiency
'speedup': [] # speedup
}
json_profile = SummaryBuilder.default_json_profile(net, 'weak', batch)
profile_ok = False
for device in self.devices:
# weak scaling: we want to find results for effective batch size
# which is N * batch
num_devices = 1 + device.count(',')
key = '{0}_{1}_{2}'.format(net, device, (batch*num_devices))
if num_devices == 1 and key not in self.cache:
# If we do not have data for one device, does not make sense
# to continue
break
batch_tm = throughput = efficiency = speedup = -1.0
if key in self.cache:
batch_tm = self.cache[key]
throughput = int((num_devices*batch) * (1000.0 / batch_tm))
json_profile['perf']['data'][str(num_devices)] = batch_tm
if len(profile['throughput']) == 0:
speedup = 1
else:
speedup = 1.0 * throughput / profile['throughput'][0]
if len(profile['efficiency']) == 0:
efficiency = 100.00
profile_ok = True
elif profile['time'][0] > 0:
efficiency = int(10000.0 * profile['time'][0] / batch_tm) / 100.0
profile_ok = True
profile['time'].append(batch_tm)
profile['throughput'].append(int(throughput))
profile['efficiency'].append(efficiency)
profile['speedup'].append(speedup)
if profile_ok:
report.append(profile)
json_report['data'].append(json_profile)
SummaryBuilder.print_report_txt(BATCH_TM_TITLE, header, report, 'net', 'batch', 'time')
SummaryBuilder.print_report_txt(IPS_TITLE, header, report, 'net', 'batch', 'throughput')
SummaryBuilder.print_report_txt(SPEEDUP_TITLE, header, report, 'net', 'batch', 'speedup')
SummaryBuilder.print_report_txt(
"Efficiency = 100% * t1 / tN",
header, report, 'net', 'batch', 'efficiency'
)
DictUtils.dump_json_to_file(json_report, jsonfile)
def filter_benchmarks(args):
"""Filter benchmarks by removing those that do not contain provided parameters.
:param argparse args: Command line arguments.
The following command line arguments are used:
* ``args.input_file`` A file with benchmark results.
* ``args.params`` Specification of mandatory parameters. For format,
read comments of ``get_params`` function
* ``args.output_file`` An output file with updated benchmark results.
"""
# Load benchmarks and parameters
input_benchmarks = load_json_file(args.input_file)['data']
params = get_params(args.params)
# Filter benchmarks
output_benchmarks = []
for input_benchmark in input_benchmarks:
keep = True
for key in params:
if key not in input_benchmark or not input_benchmark[key]:
keep = False
break
if keep:
output_benchmarks.append(input_benchmark)
# Report results and serialize
print("Number of input benchmarks: %d" % len(input_benchmarks))
print("Number of output benchmarks: %d" % len(output_benchmarks))
DictUtils.dump_json_to_file({"data": output_benchmarks}, args.output_file)
def update_benchmarks(args):
"""Update benchmarks by overriding parameters provided by a user.
:param argparse args: Command line arguments.
The following command line arguments are used:
* ``args.input_file`` A file with benchmark results.
* ``args.params`` Specification of mandatory parameters. For format,
read comments of ``get_params`` function
* ``args.output_file`` An output file with updated benchmark results.
"""
# Load benchmarks and parameters.
benchmarks = load_json_file(args.input_file)['data']
prefix = '__'
params = {prefix + k: v for k, v in get_params(args.params).items()}
# Add prefixed parameters to all benchmarks.
for benchmark in benchmarks:
benchmark.update(params)
# Process and compute variables
Processor().compute_variables(benchmarks)
# Replace prefix overwriting variables in case of a conflict
prefixed_keys = params.keys()
prefix_len = len(prefix)
output_benchmarks = []
for benchmark in benchmarks:
for k in prefixed_keys:
benchmark[k[prefix_len:]] = benchmark[k]
del benchmark[k]
if benchmark['exp.model'] != '':
output_benchmarks.append(benchmark)
benchmarks = output_benchmarks
# Serialize updated benchmarks.
DictUtils.dump_json_to_file({"data": benchmarks}, args.output_file)
def report_active(self, log_file):
if self.__file_name:
self.__progress['active_benchmark'] = {
'status': 'inprogress',
'start_time': str(datetime.datetime.now()),
'stop_time': None,
'log_file': log_file
}
DictUtils.dump_json_to_file(self.__progress, self.__file_name)
def build_plan(self, serialize=False):
"""Builds plan combining configuration, parameters and variables."""
self.plan = Builder.build(self.config, self.params, self.variables)
if serialize:
if self.plan_file:
DictUtils.dump_json_to_file(self.plan, self.plan_file)
else:
json.dump(self.plan, sys.stdout, indent=4)
print('')
def report_active_completed(self):
if self.__file_name:
self.__progress['active_benchmark']['stop_time'] = str(
datetime.datetime.now())
self.__progress['active_benchmark']['status'] = 'completed'
self.__progress['completed_benchmarks'].append(
self.__progress['active_benchmark'])
self.__progress['num_completed_benchmarks'] += 1
self.__progress["active_benchmark"] = {}
DictUtils.dump_json_to_file(self.__progress, self.__file_name)
def report(self, log_file, status, counts=True):
if self.__file_name:
self.__progress['completed_benchmarks'].append({
'status':
status,
'start_time':
str(datetime.datetime.now()),
'stop_time':
str(datetime.datetime.now()),
'log_file':
log_file
})
if counts:
self.__progress['num_completed_benchmarks'] += 1
DictUtils.dump_json_to_file(self.__progress, self.__file_name)
def report_active(self, log_file):
""" Report that new active benchmark has just started.
Args:
log_file (str): A log file for a currently active benchmark.
"""
self.__progress['active_benchmark'] = {
'exec_status': 'inprogress',
'status': None,
'start_time': datetime.datetime.now(),
'end_time': None,
'log_file': log_file
}
if self.__file_name:
DictUtils.dump_json_to_file(self.__progress, self.__file_name)
def main():
"""Does all log parsing work."""
parser = argparse.ArgumentParser()
parser.add_argument('--summary-file', type=str, required=False, default=None, help='Write summary of experiments into this JSON file.')
parser.add_argument('--log-dir', type=str, required=False, default=None, help='Scan this folder for *.log files. Scan recursively if --recursive is set.')
parser.add_argument('--recursive', required=False, default=False, action='store_true', help='Scan --log-dir folder recursively for log files.')
parser.add_argument('--keys', nargs='*', required=False, help='Parameters to extract from log files. If not set or empty, all parameters are returned.')
parser.add_argument('--strict', action='store_true', default=False, help='If set, serialzie only those results that contain all keys specified with --keys arg.')
parser.add_argument('log_files', nargs='*', help='Log files to parse')
args = parser.parse_args()
files = []
if len(args.log_files) > 0:
files = args.log_files
elif args.log_dir is not None:
files = IOUtils.find_files(args.log_dir, "*.log", args.recursive)
params = LogParser.parse_log_files(files, keys=args.keys)
if args.strict and len(args.keys) > 0:
filtered_params = []
for param in params:
param_ok = True
for key in args.keys:
if key not in param:
param_ok = False
#print ("skipping because missing field %s" % key)
break
if param_ok:
filtered_params.append(param)
params = filtered_params
summary = {"data": params}
if args.summary_file is None:
json.dump(summary, sys.stdout, indent=4, sort_keys=True)
print ("")
else:
DictUtils.dump_json_to_file(summary, args.summary_file)
def update_benchmarks(args):
"""Update benchmarks by overriding parameters provided by a user.
:param argparse args: Command line arguments.
The following command line arguments are used:
* ``args.input_file`` A file with benchmark results.
* ``args.params`` Specification of mandatory parameters. For format,
read comments of ``get_params`` function
* ``args.output_file`` An output file with updated benchmark results.
"""
# Load benchmarks and parameters.
benchmarks = load_json_file(args.input_file)['data']
params = get_params(args.params)
# Update benchmarks.
for benchmark in benchmarks:
#keys = [key for key in params if key not in benchmark]
#for key in keys:
# benchmark[key] = params[key]
for key in params:
benchmark[key] = params[key]
# Serialize updated benchmarks.
DictUtils.dump_json_to_file({"data": benchmarks}, args.output_file)
def report_all_completed(self):
if self.__file_name:
self.__progress['stop_time'] = str(datetime.datetime.now())
self.__progress['status'] = 'completed'
DictUtils.dump_json_to_file(self.__progress, self.__file_name)
def main():
"""Entry point when invoking this scrip from a command line."""
parser = argparse.ArgumentParser()
parser.add_argument('inputs',
nargs='*',
help='Log directory or a JSON file')
parser.add_argument(
'--recursive',
required=False,
default=False,
action='store_true',
help='If input is folder, scan it recursively for log files.')
parser.add_argument('--xparam',
type=str,
required=True,
default=None,
help='A parameter that is associated with x axis.')
parser.add_argument('--yparam',
type=str,
required=True,
default=None,
help='A parameter that is associated with y axis.')
parser.add_argument('--series',
type=str,
required=True,
default=None,
help='A json array with filters for series.')
parser.add_argument(
'--aggregation',
type=str,
required=True,
default="avg",
help=
'In case of multiple matches, use this to aggregate values (min, max, avg)'
)
parser.add_argument('--chart_file',
'--chart-file',
type=str,
required=False,
default=None,
help='If present, write chart into this file.')
parser.add_argument(
'--series_file',
'--series-file',
type=str,
required=False,
default=None,
help='If present, write series JSON data into this file.')
parser.add_argument(
'--chart_opts',
'--chart-opts',
type=str,
required=False,
default=None,
help='If present, a json object specifying chart options.')
parser.add_argument('--chart_type',
'--chart-type',
type=str,
required=False,
default='line',
help='Type of a chart ("line" or "bar").')
parser.add_argument(
'--baseline_xvalue',
'--baseline-xvalue',
type=str,
required=False,
default=None,
help=
"A value that's used to normalize one series. Useful to plot speedup charts."
)
parser.add_argument(
'--baseline_series',
'--baseline-series',
type=int,
required=False,
default=None,
help="An index of a baseline series to use to normalize all series.")
args = parser.parse_args()
if len(args.inputs) == 0:
raise ValueError("Must be at least one input ('--input')")
# Parse log files and load benchmark data
logfiles = [] # Original raw log files with benchmark data
benchmarks = [] # Parsed benchmarks
for input_path in args.inputs:
if os.path.isdir(input_path):
logfiles.extend(
IOUtils.find_files(input_path, "*.log", args.recursive))
elif os.path.isfile(input_path) and input_path.endswith(
('.json', '.json.gz')):
file_benchmarks = IOUtils.read_json(input_path)
if 'data' in file_benchmarks and isinstance(
file_benchmarks['data'], list):
benchmarks.extend(file_benchmarks['data'])
else:
logging.warn("Cannot parse file (%s). Invalid content.",
input_path)
else:
logging.warn("Cannot parse file (%s). Unknown extension. ",
input_path)
if len(logfiles) > 0:
benchmarks.extend(LogParser.parse_log_files(logfiles))
else:
logging.warn("No input log files have been found")
if len(benchmarks) == 0:
raise ValueError("No benchmarks have been loaded.")
# Build data for series
chart_data = SeriesBuilder.build(benchmarks, args)
# Write it
if args.series_file:
DictUtils.dump_json_to_file(chart_data, args)
# Plot it
if args.chart_file:
SeriesBuilder.plot(chart_data, args)
def report_all_completed(self):
"""Report all benchmarks have been done."""
self.__progress['end_time'] = datetime.datetime.now()
self.__progress['status'] = 'completed'
if self.__file_name:
DictUtils.dump_json_to_file(self.__progress, self.__file_name)
def report(self, log_file, exec_status, bench_status=None):
""" Report progress on done benchmark for 'disabled' or 'skipped' benchmarks.
Args:
log_file (str): A log file of the done benchmark.
exec_status(str): An execution status of a benchmark:
- 'inactive' User has decided not to run this benchmark.
- 'skipped' Benchmark for which its log file has been found, benchmark has not been conducted.
- 'completed' Benchmark has been conducted.
bench_status (str): Benchmark status like ok or failure. May be None.
"""
# 1. Verify that we have here expected execution status.
expected_statuses = ['inactive', 'skipped', 'completed']
if exec_status not in expected_statuses:
raise LogicError("Unexpected benchmark execution status (={}'). "
"Expecting one of {}.".format(
exec_status, expected_statuses))
# 2. Update benchmark status.
# For benchmarks for which we are supposed to have log files, check its runtime status. In theory,
# `inactive` benchmarks may also have log files, but user has explicitly disabled such benchmarks.
if exec_status == 'inactive':
bench_status = None
elif bench_status is None:
# Either benchmark has been run or log file has been found. I do not rely on `exp.status` parameter in log
# files because post-processing now is not good enough.
bench_status = BenchData.status(log_file)
# 3. Track information on this benchmark in a history array.
if exec_status in ['inactive', 'skipped']:
bench_info = {
'start_time': datetime.datetime.now(),
'log_file': log_file
}
else:
bench_info = self.__progress['active_benchmark']
self.__progress["active_benchmark"] = {}
bench_info.update({
'end_time': datetime.datetime.now(),
'exec_status': exec_status,
'status': bench_status
})
self.__progress['completed_benchmarks'].append(bench_info)
# 4. Update statistics
if exec_status == 'inactive':
self.__progress['num_inactive_benchmarks'] += 1
elif exec_status == 'skipped':
if bench_status == 'ok':
self.__progress['num_existing_successful_benchmarks'] += 1
else:
self.__progress['num_existing_failed_benchmarks'] += 1
else:
self.__progress['num_completed_benchmarks'] += 1
if bench_status == 'ok':
self.__progress['num_successful_benchmarks'] += 1
else:
self.__progress['num_failed_benchmarks'] += 1
# 5. If progress file has been provided, update that.
if self.__file_name:
DictUtils.dump_json_to_file(self.__progress, self.__file_name)