def build(self, overwrite=False, only_main_script=False, no_main_script=False):
"""Apply all the actions to the filesystem.
If *overwrite* is True and the experiment directory exists, it is
overwritten without prior confirmation.
"""
logging.info('Exp Dir: "%s"' % self.path)
self._set_run_dirs()
# TODO: Currently no_main_script is always False.
if not no_main_script:
# This is the first part where we only write the main script.
# We only overwrite the exp dir in the first part.
if os.path.exists(self.path):
runs_exist = any(path.startswith('runs')
for path in os.listdir(self.path))
logging.info('The directory "%s" contains run directories: %s' %
(self.path, runs_exist))
# Overwrite if overwrite is True or if no runs exist.
tools.overwrite_dir(self.path, overwrite or not runs_exist)
else:
tools.makedirs(self.path)
self._build_main_script()
if only_main_script:
return
# This is the second part where we write everything else
self._build_resources()
self._build_runs()
self._build_properties_file()
def build(self,
overwrite=False,
only_main_script=False,
no_main_script=False):
"""Apply all the actions to the filesystem.
If *overwrite* is True and the experiment directory exists, it is
overwritten without prior confirmation.
"""
logging.info('Exp Dir: "%s"' % self.path)
self._set_run_dirs()
# TODO: Currently no_main_script is always False.
if not no_main_script:
# This is the first part where we only write the main script.
# We only overwrite the exp dir in the first part.
if os.path.exists(self.path):
runs_exist = any(
path.startswith('runs') for path in os.listdir(self.path))
logging.info(
'The directory "%s" contains run directories: %s' %
(self.path, runs_exist))
# Overwrite if overwrite is True or if no runs exist.
tools.overwrite_dir(self.path, overwrite or not runs_exist)
else:
tools.makedirs(self.path)
self._build_main_script()
if only_main_script:
return
# This is the second part where we write everything else
self._build_resources()
self._build_runs()
self._build_properties_file()
def run_steps(self, steps):
timestamp = datetime.datetime.now().isoformat()
job_dir = os.path.join(self.exp.cache_dir,
'grid-steps',
timestamp + '-' + self.exp.name)
tools.overwrite_dir(job_dir)
# Build the job files before submitting the other jobs.
logging.info('Building job scripts')
for step in steps:
if step._funcname == 'build':
script_step = step.copy()
script_step.kwargs['only_main_script'] = True
script_step()
prev_job_name = None
for number, step in enumerate(steps, start=1):
job_name = self._get_job_name(step)
# We cannot submit a job from within the grid, so we submit it
# directly.
if step._funcname == 'run':
self.__wait_for_job_name = prev_job_name
self._job_name = job_name
step()
else:
step.is_last_step = (number == len(steps))
with open(os.path.join(job_dir, job_name), 'w') as f:
f.write(self._get_job(step))
submit = ['qsub']
if prev_job_name:
submit.extend(['-hold_jid', prev_job_name])
submit.append(job_name)
tools.run_command(submit, cwd=job_dir)
prev_job_name = job_name
def run_steps(self, steps):
timestamp = datetime.datetime.now().isoformat()
job_dir = os.path.join(self.exp.cache_dir, 'grid-steps',
timestamp + '-' + self.exp.name)
tools.overwrite_dir(job_dir)
# Build the job files before submitting the other jobs.
logging.info('Building job scripts')
for step in steps:
if step._funcname == 'build':
script_step = step.copy()
script_step.kwargs['only_main_script'] = True
script_step()
prev_job_name = None
for number, step in enumerate(steps, start=1):
job_name = self._get_job_name(step)
# We cannot submit a job from within the grid, so we submit it
# directly.
if step._funcname == 'run':
self.__wait_for_job_name = prev_job_name
self._job_name = job_name
step()
else:
step.is_last_step = (number == len(steps))
with open(os.path.join(job_dir, job_name), 'w') as f:
f.write(self._get_job(step))
submit = ['qsub']
if prev_job_name:
submit.extend(['-hold_jid', prev_job_name])
submit.append(job_name)
tools.run_command(submit, cwd=job_dir)
prev_job_name = job_name
def build(self):
"""
After having made all the necessary adjustments with the methods above,
this method can be used to write everything to the disk.
"""
assert self.path
tools.overwrite_dir(self.path)
# We need to build the linked resources before the run script.
# Only this way we have all resources in self.resources
# (linked ones too).
# We need to build the run script before the resources, because the run
# script is a resource.
self._build_linked_resources()
self._build_run_script()
self._build_resources()
self._build_properties_file()
def build(self):
"""
After having made all the necessary adjustments with the methods above,
this method can be used to write everything to the disk.
"""
assert self.path
tools.overwrite_dir(self.path)
# We need to build the linked resources before the run script.
# Only this way we have all resources in self.resources
# (linked ones too).
# We need to build the run script before the resources, because the run
# script is a resource.
self._build_linked_resources()
self._build_run_script()
self._build_resources()
self._build_properties_file()
