def test_validate(self, load, ra, swift, ceph, opsmgr):
file_path = 'path'
test_mod.validate(file_path)
load.assert_called_once_with(file_path)
ra.assert_called_once_with(load.return_value)
swift.assert_called_once_with(load.return_value)
ceph.assert_called_once_with(load.return_value)
opsmgr.assert_called_once_with(load.return_value)
def main(config_dir, output_dir):
"""Run the experiment."""
config, msg = validate(config_dir)
if config is None:
write_status(output_dir, False, msg)
return 1
try:
result = {}
config_iter = itertools.product(
config['models'],
config['targets'],
config['devices'])
for (model, target, device) in config_iter:
# TODO(weberlo): There has to be some idiom to get rid of this boilerplate.
if model not in result:
result[model] = {}
if target not in result[model]:
result[model][target] = {}
if device not in result[model][target]:
result[model][target][device] = {}
result[model][target][device] = run_single(model, target, device, config)
except Exception as e:
write_status(output_dir, False, 'Exception encountered:\n' + render_exception(e))
return 1
write_json(output_dir, 'data.json', result)
write_status(output_dir, True, 'success')
def main(config_dir, output_dir):
config, msg = validate(config_dir)
if config is None:
write_status(output_dir, False, msg)
return 1
if 'pt' not in config['frameworks']:
write_status(output_dir, True, 'PT not run')
return 0
datasets = config['datasets']
for dataset, max_idx in datasets:
success, msg = run_trials(
'pt',
'treelstm',
config['dry_run'],
config['n_times_per_input'],
config['n_inputs'],
treelstm_trial,
treelstm_setup,
treelstm_teardown, ['device', 'dataset', 'idx'],
[config['devices'], [dataset], [i for i in range(max_idx)]],
path_prefix=output_dir,
append_to_csv=True)
if not success:
write_status(output_dir, success, msg)
return 1
write_status(output_dir, True, 'success')
def main(data_dir, config_dir, output_dir):
try:
config, msg = validate(config_dir)
if config is None:
write_status(output_dir, False, msg)
return 1
summary = {}
for model in sorted(config['models']):
summary[model] = []
# the script will not be run if there is an error
cmd_id = 0
for _, _, exp_config in parse_commands(model, config):
for combo in unfold_settings(exp_config):
stats, msg = parse_data_file(exp_config['type'], model,
config, combo, data_dir,
cmd_id)
if stats is None:
write_status(output_dir, False, msg)
return 1
stats['command_id'] = cmd_id
summary[model].append(stats)
cmd_id += 1
write_json(output_dir, 'data.json', summary)
write_status(output_dir, True, 'success')
except Exception as e:
write_status(output_dir, False, render_exception(e))
def main(data_dir, config_dir, output_dir):
config, msg = validate(config_dir)
if config is None:
write_status(output_dir, False, msg)
return 1
all_data = sort_data(data_dir)
most_recent = all_data[-1]
most_recent = {k: v for (k, v) in most_recent.items() if k not in METADATA_KEYS}
summary = ''
for (model, targets) in most_recent.items():
# simulated target summary
sim_targets = {target: targets[target] for target in targets if target in SIM_TARGETS}
for (target, devices) in sim_targets.items():
for (device, stats) in devices.items():
summary += '_Stats on ({}, {}, {}) & _\n'.format(model, target.upper(), device.upper())
for (stat, val) in stats.items():
summary += '{}: {:.2E}\n'.format(stat, Decimal(val))
# physical target summary
phys_targets = {target: v for (target, v) in targets.items() if target in PHYS_TARGETS}
for (target, devices) in phys_targets.items():
for (device, mean_time) in devices.items():
summary += 'Time on ({}, {}, {}): {:.2f}\n'.format(
model, target.upper(), device.upper(), mean_time)
write_summary(output_dir, config['title'], summary)
write_status(output_dir, True, 'success')
def main(config_dir, output_dir):
config, msg = validate(config_dir)
if config is None:
write_status(output_dir, False, msg)
return 1
if not config['methods']:
write_status(output_dir, True, 'Nothing run')
return 0
for model in sorted(config['models']):
cmd_id = 0
for success, msg, processed_command in parse_commands(model, config):
if not success:
write_status(output_dir, False, msg)
return 1
else:
print(f'Running on command: {model}: {processed_command}')
success, msg = eval_command(model, processed_command, config,
config_dir, output_dir, cmd_id)
if not success:
write_status(output_dir, False, msg)
return 1
cmd_id += 1
write_status(output_dir, True, 'success')
def main(config_dir, output_dir):
"""Run the experiment."""
config, msg = validate(config_dir)
if config is None:
write_status(output_dir, False, msg)
return 1
try:
# the experiment involves RPC calls that could potentially hang so we have a timeout on our end too
killswitch = Timer(config.get('timeout', 300),
lambda: timeout_failure(output_dir))
killswitch.start()
result = {}
config_iter = itertools.product(config['models'], config['targets'],
config['devices'])
for (model, target, device) in config_iter:
# TODO(weberlo): There has to be some idiom to get rid of this boilerplate.
if model not in result:
result[model] = {}
if target not in result[model]:
result[model][target] = {}
if device not in result[model][target]:
result[model][target][device] = {}
result[model][target][device] = run_single(model, target, device,
config)
killswitch.cancel()
except Exception as e:
write_status(output_dir, False,
'Exception encountered:\n' + render_exception(e))
return 1
write_json(output_dir, 'data.json', result)
write_status(output_dir, True, 'success')
def main(data_dir, config_dir, output_dir):
config, msg = validate(config_dir)
if config is None:
write_status(output_dir, False, msg)
return 1
# No further analysis is required beyond the raw stats reported by the VTA
# simulator, so we just propagate the data to the next stage of the
# pipeline.
data = read_json(data_dir, 'data.json')
write_json(output_dir, 'data.json', data)
write_status(output_dir, True, 'success')
def main(data_dir, config_dir, output_dir):
try:
config, msg = validate(config_dir)
if config is None:
write_status(output_dir, False, msg)
return 1
all_data = sort_data(data_dir)
most_recent = all_data[-1]
success, msg = render_graph(config, most_recent, output_dir)
write_status(output_dir, success, msg)
except Exception as e:
write_status(output_dir, False,
'Exception encountered: ' + render_exception(e))
return 1
finally:
plt.close()
def main(data_dir, config_dir, output_dir):
config, msg = validate(config_dir)
if config is None:
write_status(output_dir, False, msg)
return 1
# read in data, output graphs of most recent data, and output longitudinal graphs
all_data = sort_data(data_dir)
most_recent = all_data[-1]
try:
generate_longitudinal_comparisons(all_data, output_dir)
generate_arm_vta_comparisons(most_recent, output_dir)
except Exception as e:
write_status(output_dir, False, 'Exception encountered:\n' + render_exception(e))
return 1
write_status(output_dir, True, 'success')
def main(data_dir, config_dir, output_dir):
try:
config, msg = validate(config_dir)
if config is None:
write_status(output_dir, False, msg)
return 1
all_data = sort_data(data_dir)
most_recent = all_data[-1]
summary = summarize(config, most_recent)
write_summary(output_dir, 'Pareto Curve Trial', summary)
write_status(output_dir, True, 'success')
except Exception as e:
write_status(output_dir, False,
'Exception encountered: ' + render_exception(e))
return 1
def main(config_dir, output_dir, method, dataset):
config, msg = validate(config_dir)
if config is None:
write_status(output_dir, False, msg)
return 1
if 'relay' not in config['frameworks']:
write_status(output_dir, True, 'Relay not run')
return 0
if method not in config['relay_methods']:
write_status(output_dir, True, '{} not run'.format(method))
return 0
datasets = config['datasets']
max_idx = -1
for pair in config['datasets']:
if pair[0] == dataset:
max_idx = pair[1]
break
# dataset is not included in the config, so skip
if max_idx == -1:
write_status(output_dir, True, 'Dataset {} not run'.format(dataset))
return 0
success, msg = run_trials(
'relay',
'treelstm',
config['dry_run'],
config['n_times_per_input'],
config['n_inputs'],
treelstm_trial,
treelstm_setup,
treelstm_teardown, ['device', 'method', 'dataset', 'idx'],
[config['devices'], [method], [dataset], [i for i in range(max_idx)]],
path_prefix=output_dir,
append_to_csv=True)
if not success:
write_status(output_dir, success, msg)
return 1
write_status(output_dir, True, 'success')
def main(config_dir,
experiment_mode,
model_name,
input_idx,
params_file,
out_file,
trial_run=False,
trial_run_outfile=None):
config, msg = validate(config_dir)
if config is None:
print(msg)
return 1
use_dtr = (experiment_mode == 'dtr')
i = int(input_idx)
is_trial = trial_run == 'True'
if config['set_seed']:
torch.manual_seed(config['seed'] + i)
random.seed(config['seed'] + i)
cwd = os.getcwd()
# handle specific params, esp. for DTR
specific_params = read_json(cwd, params_file)
assert 'batch_size' in specific_params
if use_dtr:
assert 'memory_budget' in specific_params
if specific_params['memory_budget'] > 0:
print(f'Setting budget to {int(specific_params["memory_budget"])}')
torch.set_memory_budget(int(specific_params['memory_budget']))
if is_trial:
timing_loop(model_name, i, config, use_dtr, specific_params, None,
True, trial_run_outfile)
return
with open(out_file, 'a', newline='') as csvfile:
writer = create_csv_writer(csvfile, specific_params)
timing_loop(model_name, i, config, use_dtr, specific_params, writer)
def main(config_dir, output_dir, device):
config, msg = validate(config_dir)
if config is None:
write_status(output_dir, False, msg)
return 1
if 'tf' not in config['frameworks']:
write_status(output_dir, True, 'TF not run')
return 0
if device not in config['devices']:
write_status(output_dir, True, 'TF not run on {}'.format(device))
return 0
configure_seed(config)
enable_xla = [False]
if config['use_xla']:
enable_xla.append(True)
success, msg = run_trials(
'tf',
'cnn_comp',
config['dry_run'],
config['n_times_per_input'],
config['n_inputs'],
cnn_trial,
cnn_setup,
cnn_teardown, ['network', 'device', 'batch_size', 'enable_xla'],
[config['networks'], [device], config['batch_sizes'], enable_xla],
path_prefix=output_dir,
append_to_csv=True)
write_status(output_dir, success, msg)
if not success:
return 1
from config import CONFIG from snekrouter import SnekRouter import time, threading, socket from http.server import HTTPServer from validate_config import validate validate() sock = socket.socket (socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) addr = (CONFIG['HOST'],CONFIG['PORT']) number_threads = CONFIG['NUMBER_THREADS'] sock.bind(addr) sock.listen(number_threads) class Thread(threading.Thread): def __init__(self, i): threading.Thread.__init__(self) self.i, self.daemon = i, True self.start() def run(self): httpd = HTTPServer(addr, SnekRouter, False) httpd.socket = sock httpd.server_bind = self.server_close = lambda self: None httpd.serve_forever() def main():
