def listMissingResults(exp, runs=1):
idx = 0
for path in listResultsPaths(exp, runs):
archive = getArchiveName(path)
if not os.path.exists(path) and not inArchive(archive, path):
yield idx
idx += 1
def listMissingResults(exp: ExperimentDescription, runs: int = 1):
idx = 0
for path in listResultsPaths(exp, runs):
archive = getArchiveName(path)
if not os.path.exists(path) and not inArchive(archive, path):
yield idx
idx += 1
def test_listResultsPaths(self):
exp = RLExperiment({
'agent': 'test_listResultsPaths',
'environment': 'gridworld',
'metaParameters': {
'alpha': [0.01, 0.02],
'lambda': [1.0, 0.99],
}
})
got = list(listResultsPaths(exp, 2))
expected = [
'.tmp/test_listResultsPaths/gridworld/alpha-0.01_lambda-1.0/0',
'.tmp/test_listResultsPaths/gridworld/alpha-0.02_lambda-1.0/0',
'.tmp/test_listResultsPaths/gridworld/alpha-0.01_lambda-0.99/0',
'.tmp/test_listResultsPaths/gridworld/alpha-0.02_lambda-0.99/0',
'.tmp/test_listResultsPaths/gridworld/alpha-0.01_lambda-1.0/1',
'.tmp/test_listResultsPaths/gridworld/alpha-0.02_lambda-1.0/1',
'.tmp/test_listResultsPaths/gridworld/alpha-0.01_lambda-0.99/1',
'.tmp/test_listResultsPaths/gridworld/alpha-0.02_lambda-0.99/1',
]
self.assertListEqual(got, expected)
def count(gen):
c = 0
for _ in gen:
c += 1
return c
experiment_paths = sys.argv[2:]
total = 0
total_missing = 0
for path in experiment_paths:
print(path)
exp = Experiment.load(path)
size = exp.numPermutations()
paths = listResultsPaths(exp, 1)
indices = generateMissing(paths)
missing = count(indices)
total_missing += missing
total += size
print(missing, size, missing / size)
print('total:', total_missing, total)
# Scheduling logic
# ----------------
for path in experiment_paths:
print(path)
# load the experiment json file
exp = Experiment.load(path)
# load the slurm config file
slurm = Slurm.fromFile(slurm_path)
if exp.agent in SLOW_ALGS:
slurm.sequential = 1
# figure out how many indices to use
size = exp.numPermutations() * runs
paths = listResultsPaths(exp, runs)
res_path = first(paths)
data = []
data_path = f'{res_path}/returns.csv'
if os.path.exists(data_path):
f = open(data_path, 'r')
data = f.readlines()
f.close()
indices = listIndices(exp, runs)
# get all of the indices corresponding to missing results
indices = generateMissing(exp, indices, data)
indices = printProgress(size, indices)
# compute how many "tasks" to clump into each job
def loadResults(exp: ExperimentDescription, result_file: str, ResultClass=Result) -> Generator[Result, Any, Any]:
for i, path in enumerate(listResultsPaths(exp)):
summary_path = path + '/' + result_file
yield ResultClass(summary_path, exp, i)
pool = Pool()
runs = sys.argv[2]
args = Args.ArgsModel({
'experiment_paths': sys.argv[4:],
'base_path': sys.argv[3],
'runs': int(runs),
'executable': "python " + sys.argv[1],
})
cmds = []
for path in args.experiment_paths:
exp = Experiment.load(path)
paths = listResultsPaths(exp, args.runs)
res_path = first(paths)
data = []
raise NotImplementedError(
'Make sure to change the expected result file!!')
data_path = f'{res_path}/TODO-CHANGE-ME.csv'
if os.path.exists(data_path):
f = open(data_path, 'r')
data = f.readlines()
f.close()
indices = listIndices(exp, args.runs)
# get all of the indices corresponding to missing results
indices = generateMissing(exp, indices, data)
def processResultPath(new_base, resultAndPath):
result, path = resultAndPath
mean, stderr, count = result
# sampled_mean = windowAverage(mean)
# sampled_stderr = windowAverage(stderr)
sampled_mean = everyN(mean, EVERY)
sampled_stderr = everyN(stderr, EVERY)
sampled = [sampled_mean, sampled_stderr, count]
new_path = new_base + '/' + rest(path)
os.makedirs(up(new_path), exist_ok=True)
np.save(new_path, sampled)
if __name__ == '__main__':
pool = Pool()
new_base = sys.argv[1]
exp_paths = sys.argv[2:]
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
result_paths = listResultsPaths(exp)
results = map(loadResults, result_paths)
pool.map(partial(processResultPath, new_base), results)
def loadResults(exp, result_file):
for i, path in enumerate(listResultsPaths(exp)):
summary_path = path + '/' + result_file
yield Result(summary_path, exp, i)
