def listMissingResults(exp, runs=1): idx = 0 for path in listResultsPaths(exp, runs): archive = getArchiveName(path) if not os.path.exists(path) and not inArchive(archive, path): yield idx idx += 1
def listMissingResults(exp: ExperimentDescription, runs: int = 1): idx = 0 for path in listResultsPaths(exp, runs): archive = getArchiveName(path) if not os.path.exists(path) and not inArchive(archive, path): yield idx idx += 1
def test_listResultsPaths(self): exp = RLExperiment({ 'agent': 'test_listResultsPaths', 'environment': 'gridworld', 'metaParameters': { 'alpha': [0.01, 0.02], 'lambda': [1.0, 0.99], } }) got = list(listResultsPaths(exp, 2)) expected = [ '.tmp/test_listResultsPaths/gridworld/alpha-0.01_lambda-1.0/0', '.tmp/test_listResultsPaths/gridworld/alpha-0.02_lambda-1.0/0', '.tmp/test_listResultsPaths/gridworld/alpha-0.01_lambda-0.99/0', '.tmp/test_listResultsPaths/gridworld/alpha-0.02_lambda-0.99/0', '.tmp/test_listResultsPaths/gridworld/alpha-0.01_lambda-1.0/1', '.tmp/test_listResultsPaths/gridworld/alpha-0.02_lambda-1.0/1', '.tmp/test_listResultsPaths/gridworld/alpha-0.01_lambda-0.99/1', '.tmp/test_listResultsPaths/gridworld/alpha-0.02_lambda-0.99/1', ] self.assertListEqual(got, expected)
def count(gen): c = 0 for _ in gen: c += 1 return c experiment_paths = sys.argv[2:] total = 0 total_missing = 0 for path in experiment_paths: print(path) exp = Experiment.load(path) size = exp.numPermutations() paths = listResultsPaths(exp, 1) indices = generateMissing(paths) missing = count(indices) total_missing += missing total += size print(missing, size, missing / size) print('total:', total_missing, total)
# Scheduling logic # ---------------- for path in experiment_paths: print(path) # load the experiment json file exp = Experiment.load(path) # load the slurm config file slurm = Slurm.fromFile(slurm_path) if exp.agent in SLOW_ALGS: slurm.sequential = 1 # figure out how many indices to use size = exp.numPermutations() * runs paths = listResultsPaths(exp, runs) res_path = first(paths) data = [] data_path = f'{res_path}/returns.csv' if os.path.exists(data_path): f = open(data_path, 'r') data = f.readlines() f.close() indices = listIndices(exp, runs) # get all of the indices corresponding to missing results indices = generateMissing(exp, indices, data) indices = printProgress(size, indices) # compute how many "tasks" to clump into each job
def loadResults(exp: ExperimentDescription, result_file: str, ResultClass=Result) -> Generator[Result, Any, Any]: for i, path in enumerate(listResultsPaths(exp)): summary_path = path + '/' + result_file yield ResultClass(summary_path, exp, i)
pool = Pool() runs = sys.argv[2] args = Args.ArgsModel({ 'experiment_paths': sys.argv[4:], 'base_path': sys.argv[3], 'runs': int(runs), 'executable': "python " + sys.argv[1], }) cmds = [] for path in args.experiment_paths: exp = Experiment.load(path) paths = listResultsPaths(exp, args.runs) res_path = first(paths) data = [] raise NotImplementedError( 'Make sure to change the expected result file!!') data_path = f'{res_path}/TODO-CHANGE-ME.csv' if os.path.exists(data_path): f = open(data_path, 'r') data = f.readlines() f.close() indices = listIndices(exp, args.runs) # get all of the indices corresponding to missing results indices = generateMissing(exp, indices, data)
def processResultPath(new_base, resultAndPath): result, path = resultAndPath mean, stderr, count = result # sampled_mean = windowAverage(mean) # sampled_stderr = windowAverage(stderr) sampled_mean = everyN(mean, EVERY) sampled_stderr = everyN(stderr, EVERY) sampled = [sampled_mean, sampled_stderr, count] new_path = new_base + '/' + rest(path) os.makedirs(up(new_path), exist_ok=True) np.save(new_path, sampled) if __name__ == '__main__': pool = Pool() new_base = sys.argv[1] exp_paths = sys.argv[2:] for exp_path in exp_paths: exp = loadExperiment(exp_path) result_paths = listResultsPaths(exp) results = map(loadResults, result_paths) pool.map(partial(processResultPath, new_base), results)
def loadResults(exp, result_file): for i, path in enumerate(listResultsPaths(exp)): summary_path = path + '/' + result_file yield Result(summary_path, exp, i)