def job_info(job: Job):
""" Prints information about given job to stdout
:param job(Job):
"""
from sisyphus import tools
print("Job id: %s" % job._sis_id())
print("Arguments:")
for k, v in job._sis_kwargs.items():
print(" %s : %s" % (k, str(v)))
print("Inputs:")
for name, value in job.__dict__.items():
if not name.startswith('_sis_'):
paths = tools.extract_paths(value)
for path in paths:
if path.creator is not job:
if path.creator is None:
print(" %s : %s" % (name, path.path))
else:
print(" %s : %s %s" %
(name, path.creator._sis_id(), path.path))
print("Outputs:")
for name, value in job.__dict__.items():
if not name.startswith('_sis_'):
paths = tools.extract_paths(value)
for path in paths:
if path.creator is job:
print(" %s : %s" % (name, path.path))
print("Job dir: %s" % os.path.abspath(job._sis_path()))
print("Work dir: %s" % job._sis_path(gs.WORK_DIR))
def __init__(self, name, inputs):
"""
:param str name:
:param inputs:
"""
self._required = extract_paths(inputs)
self.required_full_list = sorted(list(self._required))
self.name = name
async def async_run(obj: Any):
"""
Run and setup all jobs that are contained inside object and all jobs that are necessary.
:param obj:
:param quiet: Do not forward job output do stdout
:return:
"""
sis_graph.add_target(graph.OutputTarget(name='async_run', inputs=obj))
all_paths = {p for p in extract_paths(obj) if not p.available()}
with async_context():
while all_paths:
await asyncio.sleep(gs.WAIT_PERIOD_BETWEEN_CHECKS)
all_paths = {p for p in all_paths if not p.available()}
def _sis_init(self, args, kwargs, parsed_args):
for key, arg in parsed_args.items():
if isinstance(arg, Job):
logging.warning(
"A Job instance was used as argument \"%s\" in \"%s\", this might result in undesired behavior"
% (key, self.__class__))
self._sis_aliases = None
self._sis_alias_prefixes = set()
self._sis_vis_name = None
self._sis_output_dirs = set()
self._sis_outputs = {}
self._sis_keep_value = None
self._sis_hold_job = False
self._sis_blocks = set()
self._sis_kwargs = parsed_args
self._sis_task_rqmt_overwrite = {}
self._sis_job_lock = Job.get_lock()
self._sis_is_finished = False
self._sis_setup_since_restart = False
self._sis_environment = None
if gs.CLEANUP_ENVIRONMENT:
self._sis_environment = tools.EnvironmentModifier()
self._sis_environment.keep(gs.DEFAULT_ENVIRONMENT_KEEP)
self._sis_environment.set(gs.DEFAULT_ENVIRONMENT_SET)
if gs.AUTO_SET_JOB_INIT_ATTRIBUTES:
self.set_attrs(parsed_args)
self._sis_inputs = set()
self.__init__(*args, **kwargs)
self._sis_inputs.update({
p
for p in tools.extract_paths(self.__dict__) if p.creator != self
})
for i in self._sis_inputs:
i.add_user(self)
self._sis_quiet = False
self._sis_cleanable_cache = False
self._sis_cleaned_or_not_cleanable = False
self._sis_needed_for_which_targets = set()
self._sis_stacktrace = []
def _sis_init(self, args, kwargs, parsed_args):
self._sis_aliases = None
self._sis_alias_prefixes = set()
self._sis_vis_name = None
self._sis_output_dirs = set()
self._sis_outputs = {}
self._sis_keep_value = None
self._sis_blocks = set()
self._sis_tracebacks = set()
self._sis_kwargs = parsed_args
self._sis_task_rqmt_overwrite = {}
self._sis_work_dir = gs.WORK_DIR
self._sis_job_lock = multiprocessing.Lock()
self._sis_is_finished = False
if gs.CLEANUP_ENVIRONMENT:
from sisyphus.toolkit import EnvironmentModifier
self._sis_environment = EnvironmentModifier()
self._sis_environment.keep(gs.DEFAULT_ENVIRONMENT_KEEP)
self._sis_environment.set(gs.DEFAULT_ENVIRONMENT_SET)
if gs.AUTO_SET_JOB_INIT_ATTRIBUTES:
self.set_attrs(parsed_args)
self._sis_inputs = set()
self.__init__(*args, **kwargs)
self._sis_inputs.update({p for p in tools.extract_paths(self.__dict__) if p.creator != self})
for i in self._sis_inputs:
i.add_user(self)
self._sis_quiet = False
self._sis_cleanable_cache = False
self._sis_needed_for_which_targets = set()
def __call__(cls, *args, **kwargs):
""" Implemented to ensure that each job is created only once """
try:
if 'sis_tags' in kwargs:
tags = kwargs['sis_tags']
if tags is not None:
for tag in tags:
for char in tag:
assert char in '-.0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz'
del kwargs['sis_tags']
else:
tags = None
parsed_args = get_args(cls.__init__, args, kwargs)
except TypeError as e:
logging.error(
'Wrong input arguments or missing __init__ function?\n'
'Class: %s\nArguments: %s %s' %
(str(cls), str(args), str(kwargs)))
raise
# create key
sis_hash = cls._sis_hash_static(parsed_args)
module_name = cls.__module__
recipe_prefix = gs.RECIPE_PREFIX + '.'
assert module_name.startswith(recipe_prefix)
sis_name = os.path.join(
module_name[len(recipe_prefix):].replace('.', os.path.sep),
cls.__name__)
sis_id = "%s.%s" % (sis_name, sis_hash)
# Update tags
if tags is None:
tags = set()
for p in tools.extract_paths(parsed_args):
tags.update(p.tags)
# check cache
if sis_id in created_jobs:
job = created_jobs[sis_id]
else:
# create new object
job = super(Job, cls).__new__(cls)
assert isinstance(job, Job)
job._sis_tags = tags
# store _sis_id
job._sis_id_cache = sis_id
job._sis_init(*copy.deepcopy((args, kwargs, parsed_args)))
created_jobs[sis_id] = job
# Add block
# skip in worker mode
if block.active_blocks:
for b in block.active_blocks:
b.add_job(job)
job._sis_add_block(b)
# Update alias prefixes
job._sis_alias_prefixes.add(gs.ALIAS_AND_OUTPUT_SUBDIR)
# add stacktrace information, if set to None or -1 use full stack
stack_depth = gs.JOB_ADD_STACKTRACE_WITH_DEPTH + 1 if gs.JOB_ADD_STACKTRACE_WITH_DEPTH >= 0 else None
if stack_depth > 1 or None:
# add +1 for the traceback command itself, and remove it later
stacktrace = traceback.extract_stack(limit=stack_depth)[:-1]
job._sis_stacktrace.append(stacktrace)
return job
def add_hidden_input(self, input):
"""in case a Path has to be converted to a string that is then added to the network"""
self.hidden_inputs.update(extract_paths(input))
def update_values(self, report_values):
if report_values is not None:
self._report_values = report_values
self._required = extract_paths(self._report_values)
self.required_full_list = sorted(list(self._required))
def __call__(cls, *args, **kwargs):
""" Implemented to ensure that each job is created only once """
try:
if 'sis_tags' in kwargs:
tags = kwargs['sis_tags']
if tags is not None:
for tag in tags:
for char in tag:
assert char in '-.0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz'
del kwargs['sis_tags']
else:
tags = None
parsed_args = get_args(cls.__init__, args, kwargs)
except TypeError as e:
logging.error(
'Wrong input arguments or missing __init__ function?\n'
'Class: %s\nArguments: %s %s' %
(str(cls), str(args), str(kwargs)))
raise
# create key
sis_hash = cls._sis_hash_static(parsed_args)
module_name = cls.__module__
recipe_prefix = gs.RECIPE_DIR + '.'
assert module_name.startswith(recipe_prefix)
sis_name = os.path.join(module_name[len(recipe_prefix):].replace('.', os.path.sep), cls.__name__)
sis_id = "%s.%s" % (sis_name, sis_hash)
# Update tags
if tags is None:
tags = set()
for p in tools.extract_paths(parsed_args):
tags.update(p.tags)
# check cache
if sis_id in created_jobs:
job = created_jobs[sis_id]
else:
# create new object
job = super(Job, cls).__new__(cls)
assert isinstance(job, Job)
job._sis_tags = tags
# store _sis_id
job._sis_id_cache = sis_id
job._sis_init(*copy.deepcopy((args, kwargs, parsed_args)))
created_jobs[sis_id] = job
# Add block
# skip in worker mode
if block.active_blocks:
for b in block.active_blocks:
b.add_job(job)
job._sis_add_block(b)
if gs.LOG_TRACEBACKS:
# 6:-1 to remove Sisyphus related traces
job._sis_tracebacks.add(tuple(traceback.format_stack()[6:-1]))
# Update alias prefixes
job._sis_alias_prefixes.add(gs.ALIAS_AND_OUTPUT_SUBDIR)
return job
