def _update(param_file, exe_path, out_path):
entry = shelve.open(param_file, writeback=True)
finished = 0
parameters = entry.iteritems()
num_trials = 0
errors = 0
running = 0
waiting = 0
for k, v in parameters:
if k == 'default_script' or k == 'default_results':
continue
num_trials += 1
if v[Trial.STATE] != Trial.SUBMITTED:
if v[Trial.STATE] == Trial.FINISHED:
finished += 1
elif v[Trial.STATE] == Trial.ERROR:
errors += 1
elif v[Trial.STATE] == Trial.WAITING:
waiting += 1
else:
print v[Trial.STATE]
continue
if v[Trial.QUEUE] == Trial.SHARCNET:
if SharcNetTrial._default_connection == None:
C = SharcNetConnection(username='******')
SharcNetTrial._default_connection = C
T = SharcNetTrial(params=eval(k), exe_path=exe_path,
out_path=out_path)
elif v[Trial.QUEUE] == Trial.COALITION:
if CoalitionTrial._default_connection == None:
C = CoalitionConnection()
CoalitionTrial._default_connection = C
T = CoalitionTrial(params=eval(k), exe_path=exe_path,
out_path=out_path)
elif v[Trial.QUEUE] == Trial.LOCAL:
if BaseTrial._default_connection == None:
C = Connection()
BaseTrial._default_connection = C
T = BaseTrial(params=eval(k), exe_path=exe_path,
out_path=out_path)
else:
print 'Unknown Queue ', v[Trial.QUEUE]
continue
new_state = T.get_state()
if not new_state:
running += 1
continue
else:
v[Trial.STATE] = new_state
if new_state == Trial.FINISHED:
finished += 1
elif new_state == Trial.ERROR:
errors += 1
entry[k] = v
entry.close()
return (num_trials - finished, finished, num_trials, errors, running, waiting)
def rehash(exe_path, out_path, server):
if server == Trial.SHARCNET:
T = SharcNetTrial(params=[], exe_path=exe_path, out_path=out_path)
elif server == Trial.COALITION:
T = CoalitionTrial(params=[], exe_path=exe_path, out_path=out_path)
else:
print 'unknown server ', server
return
for folder in T.connection.listdir(T.out_path):
config = path(folder) / path('config.yml')
files = T.connection.listdir(T.out_path / path(folder))
if '.finished' not in files:
print 'Script not done'
continue
try:
cfile = T.connection.open(T.out_path / config)
except:
print "couldn't open:", T.out_path / config
continue
# This is a fix for unsorted parameters.
data = yaml.load(cfile)
sort_params = sorted(data[1])
new_path = T.out_path / T._hash(data[0], sort_params)
cfile.close()
if new_path == T.out_path / folder:
print folder, ' already in right spot'
continue
# rewrite the sorted parameters into the config file
cfile = T.connection.open(T.out_path / config, 'w')
cfile.write(yaml.dump(data))
cfile.close()
T.connection.rename(T.out_path / folder, new_path)
uoft = [path(u.strip()) for u in open("./uoft.txt")]
SharcNetTrial._default_connection = SharcNetConnection()
nets = mcnc + uoft
net_info = yaml.load(path('/var/www/pyvpr_results/common/'\
'anneal-fast-mean_outer_iterations_and_runtimes.yml').bytes())
for net in nets:
for seed in range(10):
# Convert runtime to minutes with a minimum of 15 and
# factoring in cpu speed differences.
runtime = int(15 + (net_info[net.namebase]['runtime'] * 3 / 60))
run_count = net_info[net.namebase]['outer iter']
np = '$BENCHMARK_PATH/' + net
S = SharcNetTrial(params=[('netlist_file', np), ('arch_file', '$BENCHMARK_PATH/k4-n1.xml'),
('seed', seed), ('run_count', run_count), ('inner_num', 1)], time=runtime,
priority=1, out_path='$PYVPR_RESULTS/96',
exe_path='$PYVPR_EXPERIMENTS/96/freeze_annealer.py')
output_path = S.out_path / S.hash_path
try:
if '.finished' in S.connection.sftp.listdir(str(output_path)):
print '''skipping %s, seed=%d since '.finished' file exists '''\
'''in %s''' % (np, seed, output_path)
continue
except IOError:
# Directory listing failed since output directory
# doesn't exist yet.
pass
S.make_output_dir()
S.write_config()
S.submit()