def test_group(self): arr = [1, 2, 3, 4, 5, 6, 7, 8] got = list(group(arr, 3)) expected = [ [1, 2, 3], [4, 5, 6], [7, 8], # last group may not be same size as rest ] self.assertEqual(got, expected)
data = [] data_path = f'{res_path}/returns.csv' if os.path.exists(data_path): f = open(data_path, 'r') data = f.readlines() f.close() indices = listIndices(exp, runs) # get all of the indices corresponding to missing results indices = generateMissing(exp, indices, data) indices = printProgress(size, indices) # compute how many "tasks" to clump into each job groupSize = slurm.cores * slurm.sequential for g in group(indices, groupSize): l = list(g) print("scheduling:", path, l) # build the executable string runner = f'python {executable} {path} ' # generate the gnu-parallel command for dispatching to many CPUs across server nodes parallel = Slurm.buildParallel( runner, l, { 'ntasks': slurm.cores, 'nodes-per-process': 1, 'threads-per-process': 1, # <-- if you need multiple threads for a single process, change this number (NOTE: the rest of the scheduling logic does not know how to handle this yet) })
total_hours = int(hours) + (int(minutes) / 60) + (int(seconds) / 3600) # gather missing and sum up cost missing, cost = gatherMissing(experiment_paths, runs, groupSize, slurm.cores, total_hours) print( f"Expected to use {cost[0]:.2f} core years, which is {cost[1]:.4f}% of our annual allocation" ) input("Press Enter to confirm or ctrl+c to exit") for path in missing: # reload this because we do bad mutable things later on slurm = Slurm.fromFile(slurm_path) for g in group(missing[path], groupSize): l = list(g) print("scheduling:", path, l) # build the executable string runner = f'python {executable} {path} ' # generate the gnu-parallel command for dispatching to many CPUs across server nodes parallel = Parallel.build({ 'executable': runner, 'cores': slurm.cores, 'tasks': l, }) # generate the bash script which will be scheduled script = getJobScript(parallel)
def windowAverage(arr, window): for g in group(arr, window): yield np.mean(g)