n=st.n_range[sl][-1]))) else: print("WARNING Couldn't save {},file not found.".format(stdout_file)) # run nvprof on n_range[2] for conf, proj_dir in zip(configurations, project_dirs): main_arg = '' if proj_dir in ['cuda_standalone', 'GeNNworkspace']: if proj_dir == 'GeNNworkspace': main_arg = 'test {time} 1'.format(time=st.duration/second) ns = st.n_range[sl] idx = 2 max_runtime = 20 conf_name = conf.__name__ print("Rerunning {} with n = {} for nvprof profiling".format(conf_name, st.n_range[idx])) tb, res, runtime, prof_info = results(conf, st, st.n_range[idx], maximum_run_time=maximum_run_time) if not isinstance(res, Exception) and runtime < max_runtime: option = '--profile-from-start-off' if proj_dir == 'cuda_standalone' else '' cmd = 'cd {proj_dir} && nvprof {opt} --log-file ../{log_file} ./main {arg}'.format( proj_dir=proj_dir, arg=main_arg, opt=option, log_file=os.path.join(prof_dir, 'nvprof_{st}_{conf}_{n}.log'.format( st=name, conf=conf_name, n=st.n_range[idx]))) prof_start = datetime.datetime.fromtimestamp(time.time()).strftime(time_format) print(cmd) x = os.system(cmd) if x: print('nvprof failed with {}'.format(x)) prof_end = datetime.datetime.fromtimestamp(time.time()).strftime(time_format) prof_diff = datetime.datetime.strptime(prof_end, time_format) - datetime.datetime.strptime(prof_start, time_format) print("Profiling took {} for runtime of {}".format(prof_diff, runtime)) finally:
length = len(ft.n_range) sl_idcs = sl.indices(length) max_idx = sl_idcs[1] - sl_idcs[2] n_init = ft.n_range[max_idx] n = n_prev = n_init #n = ft.n_range[0] has_failed = False for i in range(10): if name.startswith('STDPCUDA'): n = (n // 1000) * 1000 start = time.time() script_start = datetime.datetime.fromtimestamp(start).strftime( time_format) print "LOG RUNNING {} with n={} (start at {})".format( name, n, script_start) tb, res, runtime, prof_info = results(CUDAStandaloneConfiguration, ft, n) print "LOG FINISHED {} with n={} in {:.2f}s".format( name, n, time.time() - start) codes.append((n, res, tb)) diff = np.abs(n - n_prev) n_prev = n if isinstance(res, Exception): print "LOG FAILED:\n", res, tb has_failed = True if i == 0: print( "LOG FAILED at first run ({}) n={}\n\terror={}\nt\ttb={}". format(name, n, res, tb)) break # assuming the first run passes, when we fail we always go down half the abs distance
n_last_fail = sl[1] n_init = int(n_last_success + (n_last_fail - n_last_success) / 2) know_last_fail = True n = n_prev = n_init #n = ft.n_range[0] has_failed = False for i in range(3): if name.startswith('STDPCUDA'): n = (n//1000) * 1000 start = time.time() script_start = datetime.datetime.fromtimestamp(start).strftime(time_format) append_message = print_flushed( f"LOG RUNNING {name} with n={n} (start at {script_start})", new_reply=True ) tb, res, runtime, prof_info = results(config, ft, n) took = datetime.timedelta(seconds=int(time.time() - start)) print_flushed(f"LOG FINISHED {name} with n={n} in {took}") codes.append((n, res, tb)) diff = np.abs(n - n_prev) n_prev = n if not know_last_fail: if isinstance(res, Exception): print_flushed(f"LOG FAILED:\n{res}, {tb}") if i == 0: print_flushed(f"LOG FAILED at first run ({name}) n={n}\n\terror={res}\nt\ttb={tb}") #print_flushed(f"LOG FAILED at first run ({name}) n={n}\n\terror={res}\nTraceback printed above.") config.delete_project_dir(feature_name=ft.__name__, n=n_prev) break # assuming the first run passes, when we fail we always go down half the abs distance n -= int(0.5 * diff)