def build_task_descriptor(cls, task_descriptor, parent_task_record): # Convert task_private to a reference in here. task_private_id = ("%s:_private" % task_descriptor["task_id"]) task_private_ref = ref_from_object(task_descriptor["task_private"], BaseExecutor.TASK_PRIVATE_ENCODING, task_private_id) parent_task_record.publish_ref(task_private_ref) task_descriptor["task_private"] = task_private_ref task_descriptor["dependencies"].append(task_private_ref)
def build_task_descriptor(cls, task_descriptor, parent_task_record, args, n_outputs, is_tail_spawn=False, handler_name=None): # This is needed to work around the fact that stdinout has its own implementation of build_task_descriptor, so # we can't rely using cls.handler_name to find the actual executor. if handler_name is None: handler_name = cls.handler_name if is_tail_spawn and len( task_descriptor["expected_outputs"]) != n_outputs: raise BlameUserException( "SimpleExecutor being built with delegated outputs %s but n_outputs=%d" % (task_descriptor["expected_outputs"], n_outputs)) # Throw early if the args are bad cls.check_args_valid(args, n_outputs) # Discover required ref IDs for this executor reqd_refs = cls.get_required_refs(args) task_descriptor["dependencies"].extend(reqd_refs) sha = hashlib.sha1() hash_update_with_structure(sha, [args, n_outputs]) name_prefix = "%s:%s:" % (handler_name, sha.hexdigest()) # Name our outputs if not is_tail_spawn: task_descriptor["expected_outputs"] = [ "%s%d" % (name_prefix, i) for i in range(n_outputs) ] # Add the args dict args_name = "%ssimple_exec_args" % name_prefix args_ref = ref_from_object(args, "pickle", args_name) parent_task_record.publish_ref(args_ref) task_descriptor["dependencies"].append(args_ref) task_descriptor["task_private"]["simple_exec_args"] = args_ref BaseExecutor.build_task_descriptor(task_descriptor, parent_task_record) if is_tail_spawn: return None else: return [ SW2_FutureReference(x) for x in task_descriptor["expected_outputs"] ]
def build_task_descriptor(cls, task_descriptor, parent_task_record, args, n_outputs, is_tail_spawn=False, handler_name=None): # This is needed to work around the fact that stdinout has its own implementation of build_task_descriptor, so # we can't rely using cls.handler_name to find the actual executor. if handler_name is None: handler_name = cls.handler_name if is_tail_spawn and len(task_descriptor["expected_outputs"]) != n_outputs: raise BlameUserException("SimpleExecutor being built with delegated outputs %s but n_outputs=%d" % (task_descriptor["expected_outputs"], n_outputs)) # Throw early if the args are bad cls.check_args_valid(args, n_outputs) # Discover required ref IDs for this executor reqd_refs = cls.get_required_refs(args) task_descriptor["dependencies"].extend(reqd_refs) sha = hashlib.sha1() hash_update_with_structure(sha, [args, n_outputs]) name_prefix = "%s:%s:" % (handler_name, sha.hexdigest()) # Name our outputs if not is_tail_spawn: task_descriptor["expected_outputs"] = ["%s%d" % (name_prefix, i) for i in range(n_outputs)] # Add the args dict args_name = "%ssimple_exec_args" % name_prefix args_ref = ref_from_object(args, "pickle", args_name) parent_task_record.publish_ref(args_ref) task_descriptor["dependencies"].append(args_ref) task_descriptor["task_private"]["simple_exec_args"] = args_ref BaseExecutor.build_task_descriptor(task_descriptor, parent_task_record) if is_tail_spawn: return None else: return [SW2_FutureReference(x) for x in task_descriptor["expected_outputs"]]
def _execute(self): reflist = [ self.task_record.retrieve_ref(x) for x in self.args["inputs"] ] self.output_refs[0] = ref_from_object(reflist, "json", self.output_ids[0])
def _execute(self): reflist = [self.task_record.retrieve_ref(x) for x in self.args["inputs"]] self.output_refs[0] = ref_from_object(reflist, "json", self.output_ids[0])