def test_group(self):
arr = [1, 2, 3, 4, 5, 6, 7, 8]
got = list(group(arr, 3))
expected = [
[1, 2, 3],
[4, 5, 6],
[7, 8], # last group may not be same size as rest
]
self.assertEqual(got, expected)
data = []
data_path = f'{res_path}/returns.csv'
if os.path.exists(data_path):
f = open(data_path, 'r')
data = f.readlines()
f.close()
indices = listIndices(exp, runs)
# get all of the indices corresponding to missing results
indices = generateMissing(exp, indices, data)
indices = printProgress(size, indices)
# compute how many "tasks" to clump into each job
groupSize = slurm.cores * slurm.sequential
for g in group(indices, groupSize):
l = list(g)
print("scheduling:", path, l)
# build the executable string
runner = f'python {executable} {path} '
# generate the gnu-parallel command for dispatching to many CPUs across server nodes
parallel = Slurm.buildParallel(
runner,
l,
{
'ntasks': slurm.cores,
'nodes-per-process': 1,
'threads-per-process':
1, # <-- if you need multiple threads for a single process, change this number (NOTE: the rest of the scheduling logic does not know how to handle this yet)
})
total_hours = int(hours) + (int(minutes) / 60) + (int(seconds) / 3600)
# gather missing and sum up cost
missing, cost = gatherMissing(experiment_paths, runs, groupSize, slurm.cores,
total_hours)
print(
f"Expected to use {cost[0]:.2f} core years, which is {cost[1]:.4f}% of our annual allocation"
)
input("Press Enter to confirm or ctrl+c to exit")
for path in missing:
# reload this because we do bad mutable things later on
slurm = Slurm.fromFile(slurm_path)
for g in group(missing[path], groupSize):
l = list(g)
print("scheduling:", path, l)
# build the executable string
runner = f'python {executable} {path} '
# generate the gnu-parallel command for dispatching to many CPUs across server nodes
parallel = Parallel.build({
'executable': runner,
'cores': slurm.cores,
'tasks': l,
})
# generate the bash script which will be scheduled
script = getJobScript(parallel)
def windowAverage(arr, window):
for g in group(arr, window):
yield np.mean(g)