n=st.n_range[sl][-1])))
else:
print("WARNING Couldn't save {},file not found.".format(stdout_file))
# run nvprof on n_range[2]
for conf, proj_dir in zip(configurations, project_dirs):
main_arg = ''
if proj_dir in ['cuda_standalone', 'GeNNworkspace']:
if proj_dir == 'GeNNworkspace':
main_arg = 'test {time} 1'.format(time=st.duration/second)
ns = st.n_range[sl]
idx = 2
max_runtime = 20
conf_name = conf.__name__
print("Rerunning {} with n = {} for nvprof profiling".format(conf_name, st.n_range[idx]))
tb, res, runtime, prof_info = results(conf, st, st.n_range[idx], maximum_run_time=maximum_run_time)
if not isinstance(res, Exception) and runtime < max_runtime:
option = '--profile-from-start-off' if proj_dir == 'cuda_standalone' else ''
cmd = 'cd {proj_dir} && nvprof {opt} --log-file ../{log_file} ./main {arg}'.format(
proj_dir=proj_dir, arg=main_arg, opt=option,
log_file=os.path.join(prof_dir, 'nvprof_{st}_{conf}_{n}.log'.format(
st=name, conf=conf_name, n=st.n_range[idx])))
prof_start = datetime.datetime.fromtimestamp(time.time()).strftime(time_format)
print(cmd)
x = os.system(cmd)
if x:
print('nvprof failed with {}'.format(x))
prof_end = datetime.datetime.fromtimestamp(time.time()).strftime(time_format)
prof_diff = datetime.datetime.strptime(prof_end, time_format) - datetime.datetime.strptime(prof_start, time_format)
print("Profiling took {} for runtime of {}".format(prof_diff, runtime))
finally:
length = len(ft.n_range)
sl_idcs = sl.indices(length)
max_idx = sl_idcs[1] - sl_idcs[2]
n_init = ft.n_range[max_idx]
n = n_prev = n_init
#n = ft.n_range[0]
has_failed = False
for i in range(10):
if name.startswith('STDPCUDA'):
n = (n // 1000) * 1000
start = time.time()
script_start = datetime.datetime.fromtimestamp(start).strftime(
time_format)
print "LOG RUNNING {} with n={} (start at {})".format(
name, n, script_start)
tb, res, runtime, prof_info = results(CUDAStandaloneConfiguration, ft,
n)
print "LOG FINISHED {} with n={} in {:.2f}s".format(
name, n,
time.time() - start)
codes.append((n, res, tb))
diff = np.abs(n - n_prev)
n_prev = n
if isinstance(res, Exception):
print "LOG FAILED:\n", res, tb
has_failed = True
if i == 0:
print(
"LOG FAILED at first run ({}) n={}\n\terror={}\nt\ttb={}".
format(name, n, res, tb))
break
# assuming the first run passes, when we fail we always go down half the abs distance
n_last_fail = sl[1]
n_init = int(n_last_success + (n_last_fail - n_last_success) / 2)
know_last_fail = True
n = n_prev = n_init
#n = ft.n_range[0]
has_failed = False
for i in range(3):
if name.startswith('STDPCUDA'):
n = (n//1000) * 1000
start = time.time()
script_start = datetime.datetime.fromtimestamp(start).strftime(time_format)
append_message = print_flushed(
f"LOG RUNNING {name} with n={n} (start at {script_start})",
new_reply=True
)
tb, res, runtime, prof_info = results(config, ft, n)
took = datetime.timedelta(seconds=int(time.time() - start))
print_flushed(f"LOG FINISHED {name} with n={n} in {took}")
codes.append((n, res, tb))
diff = np.abs(n - n_prev)
n_prev = n
if not know_last_fail:
if isinstance(res, Exception):
print_flushed(f"LOG FAILED:\n{res}, {tb}")
if i == 0:
print_flushed(f"LOG FAILED at first run ({name}) n={n}\n\terror={res}\nt\ttb={tb}")
#print_flushed(f"LOG FAILED at first run ({name}) n={n}\n\terror={res}\nTraceback printed above.")
config.delete_project_dir(feature_name=ft.__name__, n=n_prev)
break
# assuming the first run passes, when we fail we always go down half the abs distance
n -= int(0.5 * diff)