def _set_run_dirs(self):
"""
Sets the relative run directories as instance
variables for all runs
"""
def get_run_number(number):
return str(number).zfill(5)
def get_shard_dir(shard_number):
first_run = self.shard_size * (shard_number - 1) + 1
last_run = self.shard_size * (shard_number)
return 'runs-%s-%s' % (get_run_number(first_run),
get_run_number(last_run))
current_run = 0
shards = tools.divide_list(self.runs, self.shard_size)
for shard_number, shard in enumerate(shards, start=1):
shard_dir = os.path.join(self.base_dir,
get_shard_dir(shard_number))
tools.overwrite_dir(shard_dir)
for run in shard:
current_run += 1
rel_dir = os.path.join(get_shard_dir(shard_number),
get_run_number(current_run))
abs_dir = os.path.join(self.base_dir, rel_dir)
run.dir = abs_dir
def _build_main_script(self):
"""
Generates the main script
"""
num_tasks = math.ceil(len(self.runs) / float(self.runs_per_task))
current_dir = os.path.dirname(os.path.abspath(__file__))
job_params = {
'logfile':
os.path.join(current_dir, self.exp_name, self.exp_name + '.log'),
'errfile':
os.path.join(current_dir, self.exp_name, self.exp_name + '.err'),
'driver_timeout':
self.timeout + 30,
'num_tasks':
num_tasks,
'queue':
self.queue,
'priority':
self.priority,
}
script_template = open('data/gkigrid-job-header-template').read()
script = script_template % job_params
script += '\n'
run_groups = tools.divide_list(self.runs, self.runs_per_task)
for task_id, run_group in enumerate(run_groups, start=1):
script += 'if [[ $SGE_TASK_ID == %s ]]; then\n' % task_id
for run in run_group:
# Change into the run dir
script += ' cd %s\n' % run.dir
script += ' ./run\n'
script += 'fi\n'
name = self.exp_name
filename = name if name.endswith('.q') else name + '.q'
filename = self._get_abs_path(filename)
with open(filename, 'w') as file:
file.write(script)
def _set_run_dirs(self):
"""
Sets the relative run directories as instance variables for all runs
"""
def run_number(number):
return str(number).zfill(5)
def get_shard_dir(shard_number):
first_run = self.shard_size * (shard_number - 1) + 1
last_run = self.shard_size * (shard_number)
return 'runs-%s-%s' % (run_number(first_run), run_number(last_run))
current_run = 0
shards = tools.divide_list(self.runs, self.shard_size)
for shard_number, shard in enumerate(shards, start=1):
shard_dir = os.path.join(self.path, get_shard_dir(shard_number))
for run in shard:
current_run += 1
rel_dir = os.path.join(get_shard_dir(shard_number),
run_number(current_run))
run.dir = self._get_abs_path(rel_dir)