def prepare_stageout(self, tmp_dir): # get the sandbox stage-out mask stageout_mask = self.task.sandbox_stageout() if not stageout_mask: return None # determine outputs as seen from outside and within the sandbox outputs = self.task.output() with patch_object(os, "environ", self.task.env, lock=True): sandbox_outputs = self.task.output() # apply the mask to both structs outputs = mask_struct(stageout_mask, outputs) sandbox_outputs = mask_struct(stageout_mask, sandbox_outputs) if not outputs: return None # define the stage-out directory cfg = Config.instance() section = self.sandbox_inst.get_config_section() stageout_dir = tmp_dir.child(cfg.get_expanded(section, "stageout_dir"), type="d") stageout_dir.touch() # create a lookup for input -> sandbox input sandbox_targets = dict(zip(flatten(outputs), flatten(sandbox_outputs))) return StageInfo(outputs, stageout_dir, sandbox_targets)
def stagein(self, tmp_dir): # check if the stage-in dir is set cfg = Config.instance() section = self.sandbox_inst.get_config_section() stagein_dir_name = cfg.get_expanded(section, "stagein_dir_name") if not stagein_dir_name: return None # get the sandbox stage-in mask stagein_mask = self.task.sandbox_stagein() if not stagein_mask: return None # determine inputs as seen from outside and within the sandbox inputs = self.task.input() with patch_object(os, "environ", self.task.env, lock=True): sandbox_inputs = self.task.input() # apply the mask to both structs inputs = mask_struct(stagein_mask, inputs) sandbox_inputs = mask_struct(stagein_mask, sandbox_inputs) if not inputs: return None # create a lookup for input -> sandbox input sandbox_targets = dict(zip(flatten(inputs), flatten(sandbox_inputs))) # create the stage-in directory stagein_dir = tmp_dir.child(stagein_dir_name, type="d") stagein_dir.touch() # create the structure of staged inputs def stagein_target(target): sandbox_target = sandbox_targets[target] staged_target = make_staged_target(stagein_dir, sandbox_target) logger.debug("stage-in {} to {}".format(target.path, staged_target.path)) target.copy_to_local(staged_target) return staged_target def map_collection(func, collection, **kwargs): map_struct(func, collection.targets, **kwargs) staged_inputs = map_struct( stagein_target, inputs, custom_mappings={TargetCollection: map_collection}) logger.info("staged-in {} file(s)".format(len(stagein_dir.listdir()))) return StageInfo(inputs, stagein_dir, staged_inputs)
def _staged_input(self): # get the original inputs inputs = self.__getattribute__("input", proxy=False)() # create the struct of staged inputs staged_inputs = make_staged_target_struct(_sandbox_stagein_dir, inputs) # apply the stage-in mask return mask_struct(self.sandbox_stagein(), staged_inputs, inputs)
def _staged_output(self): # get the original outputs outputs = self.__getattribute__("output", proxy=False)() # create the struct of staged outputs staged_outputs = make_staged_target_struct(_sandbox_stageout_dir, outputs) # apply the stage-out mask return mask_struct(self.sandbox_stageout(), staged_outputs, outputs)
def _staged_output(self): outputs = self.__getattribute__("output", proxy=False)() # create the struct of staged inputs and use the mask to deeply select between the two def map_targets(target): return make_staged_target(_sandbox_stageout_dir, target) staged_outputs = map_struct(map_targets, outputs) outputs = mask_struct(self.sandbox_stageout_mask(), staged_outputs, outputs) return outputs
def _staged_output(self): if not _sandbox_stageout_dir: raise Exception("LAW_SANDBOX_STAGEOUT_DIR must not be empty in a sandbox when target " "stage-out is required") # get the original outputs outputs = self.__getattribute__("output", proxy=False)() # create the struct of staged outputs staged_outputs = make_staged_target_struct(_sandbox_stageout_dir, outputs) # apply the stage-out mask return mask_struct(self.sandbox_stageout(), staged_outputs, outputs)
def _staged_input(self): if not _sandbox_stagein_dir: raise Exception("LAW_SANDBOX_STAGEIN_DIR must not be empty in a sandbox when target " "stage-in is required") # get the original inputs inputs = self.__getattribute__("input", proxy=False)() # create the struct of staged inputs staged_inputs = make_staged_target_struct(_sandbox_stagein_dir, inputs) # apply the stage-in mask return mask_struct(self.sandbox_stagein(), staged_inputs, inputs)
def prepare_stageout(self): outputs = mask_struct(self.task.sandbox_stageout_mask(), self.task.output()) if not outputs: return None # create a tmp dir tmp_dir = LocalDirectoryTarget(is_tmp=True) tmp_dir.touch() # map output files to local local targets in tmp_dir def map_target(target): return make_staged_target(tmp_dir, target) stage_outputs = map_struct(map_target, outputs) return StageInfo(outputs, tmp_dir, stage_outputs)
def stagein(self): inputs = mask_struct(self.task.sandbox_stagein_mask(), self.task.input()) if not inputs: return None # create a tmp dir tmp_dir = LocalDirectoryTarget(is_tmp=True) tmp_dir.touch() # copy input files and map to local targets in tmp_dir def map_target(target): tmp_target = make_staged_target(tmp_dir, target) target.copy(tmp_target) return tmp_target stage_inputs = map_struct(map_target, inputs) return StageInfo(inputs, tmp_dir, stage_inputs)