def prepare_stageout(self, tmp_dir):
# get the sandbox stage-out mask
stageout_mask = self.task.sandbox_stageout()
if not stageout_mask:
return None
# determine outputs as seen from outside and within the sandbox
outputs = self.task.output()
with patch_object(os, "environ", self.task.env, lock=True):
sandbox_outputs = self.task.output()
# apply the mask to both structs
outputs = mask_struct(stageout_mask, outputs)
sandbox_outputs = mask_struct(stageout_mask, sandbox_outputs)
if not outputs:
return None
# define the stage-out directory
cfg = Config.instance()
section = self.sandbox_inst.get_config_section()
stageout_dir = tmp_dir.child(cfg.get_expanded(section, "stageout_dir"),
type="d")
stageout_dir.touch()
# create a lookup for input -> sandbox input
sandbox_targets = dict(zip(flatten(outputs), flatten(sandbox_outputs)))
return StageInfo(outputs, stageout_dir, sandbox_targets)
def stagein(self, tmp_dir):
# check if the stage-in dir is set
cfg = Config.instance()
section = self.sandbox_inst.get_config_section()
stagein_dir_name = cfg.get_expanded(section, "stagein_dir_name")
if not stagein_dir_name:
return None
# get the sandbox stage-in mask
stagein_mask = self.task.sandbox_stagein()
if not stagein_mask:
return None
# determine inputs as seen from outside and within the sandbox
inputs = self.task.input()
with patch_object(os, "environ", self.task.env, lock=True):
sandbox_inputs = self.task.input()
# apply the mask to both structs
inputs = mask_struct(stagein_mask, inputs)
sandbox_inputs = mask_struct(stagein_mask, sandbox_inputs)
if not inputs:
return None
# create a lookup for input -> sandbox input
sandbox_targets = dict(zip(flatten(inputs), flatten(sandbox_inputs)))
# create the stage-in directory
stagein_dir = tmp_dir.child(stagein_dir_name, type="d")
stagein_dir.touch()
# create the structure of staged inputs
def stagein_target(target):
sandbox_target = sandbox_targets[target]
staged_target = make_staged_target(stagein_dir, sandbox_target)
logger.debug("stage-in {} to {}".format(target.path,
staged_target.path))
target.copy_to_local(staged_target)
return staged_target
def map_collection(func, collection, **kwargs):
map_struct(func, collection.targets, **kwargs)
staged_inputs = map_struct(
stagein_target,
inputs,
custom_mappings={TargetCollection: map_collection})
logger.info("staged-in {} file(s)".format(len(stagein_dir.listdir())))
return StageInfo(inputs, stagein_dir, staged_inputs)
def _staged_input(self):
# get the original inputs
inputs = self.__getattribute__("input", proxy=False)()
# create the struct of staged inputs
staged_inputs = make_staged_target_struct(_sandbox_stagein_dir, inputs)
# apply the stage-in mask
return mask_struct(self.sandbox_stagein(), staged_inputs, inputs)
def _staged_output(self):
# get the original outputs
outputs = self.__getattribute__("output", proxy=False)()
# create the struct of staged outputs
staged_outputs = make_staged_target_struct(_sandbox_stageout_dir,
outputs)
# apply the stage-out mask
return mask_struct(self.sandbox_stageout(), staged_outputs, outputs)
def _staged_output(self):
outputs = self.__getattribute__("output", proxy=False)()
# create the struct of staged inputs and use the mask to deeply select between the two
def map_targets(target):
return make_staged_target(_sandbox_stageout_dir, target)
staged_outputs = map_struct(map_targets, outputs)
outputs = mask_struct(self.sandbox_stageout_mask(), staged_outputs, outputs)
return outputs
def _staged_output(self):
if not _sandbox_stageout_dir:
raise Exception("LAW_SANDBOX_STAGEOUT_DIR must not be empty in a sandbox when target "
"stage-out is required")
# get the original outputs
outputs = self.__getattribute__("output", proxy=False)()
# create the struct of staged outputs
staged_outputs = make_staged_target_struct(_sandbox_stageout_dir, outputs)
# apply the stage-out mask
return mask_struct(self.sandbox_stageout(), staged_outputs, outputs)
def _staged_input(self):
if not _sandbox_stagein_dir:
raise Exception("LAW_SANDBOX_STAGEIN_DIR must not be empty in a sandbox when target "
"stage-in is required")
# get the original inputs
inputs = self.__getattribute__("input", proxy=False)()
# create the struct of staged inputs
staged_inputs = make_staged_target_struct(_sandbox_stagein_dir, inputs)
# apply the stage-in mask
return mask_struct(self.sandbox_stagein(), staged_inputs, inputs)
def prepare_stageout(self):
outputs = mask_struct(self.task.sandbox_stageout_mask(), self.task.output())
if not outputs:
return None
# create a tmp dir
tmp_dir = LocalDirectoryTarget(is_tmp=True)
tmp_dir.touch()
# map output files to local local targets in tmp_dir
def map_target(target):
return make_staged_target(tmp_dir, target)
stage_outputs = map_struct(map_target, outputs)
return StageInfo(outputs, tmp_dir, stage_outputs)
def stagein(self):
inputs = mask_struct(self.task.sandbox_stagein_mask(), self.task.input())
if not inputs:
return None
# create a tmp dir
tmp_dir = LocalDirectoryTarget(is_tmp=True)
tmp_dir.touch()
# copy input files and map to local targets in tmp_dir
def map_target(target):
tmp_target = make_staged_target(tmp_dir, target)
target.copy(tmp_target)
return tmp_target
stage_inputs = map_struct(map_target, inputs)
return StageInfo(inputs, tmp_dir, stage_inputs)
