def split_commands(commands):
"""Split the commands into N cluster jobs."""
verbose("There are {0} commands".format(len(commands)))
if WORK_DATA["job_size"]: # size of cluster job is pre-defined
job_size = WORK_DATA["job_size"]
else: # if was not defined - compute the size of cluster job
job_size = (len(commands) // WORK_DATA["jobs_num"]) + 1
verbose(f"Split commands with size of {job_size} for each cluster job")
batch = parts(commands, n=job_size)
verbose("There are {} cluster jobs".format(len(batch)))
return batch
def save_jobs(filled_buckets, bucket_jobs_num, jobs_dir):
"""Save cesar calls in the dir assigned."""
os.mkdir(jobs_dir) if not os.path.isdir(jobs_dir) else None
file_num, to_combine = 0, []
for bucket_id, jobs in filled_buckets.items():
num_of_files = bucket_jobs_num[bucket_id]
# just in case
num_of_files = len(jobs) if num_of_files >= len(jobs) else num_of_files
size_of_file = len(jobs) // num_of_files
# size_of_file = size_of_file + 1 if len(jobs) % num_of_files != 0 else size_of_file
jobs_split = parts(jobs, n=size_of_file)
for part in jobs_split:
file_num += 1
file_name = f"cesar_job_{file_num}_{bucket_id}"
file_path = os.path.join(jobs_dir, file_name)
f = open(file_path, "w")
f.write("\n".join(part) + "\n")
f.close()
to_combine.append(file_path)
return to_combine