def get_completed_ref(self): completed_ref = self.output_ctx.get_completed_ref() if isinstance(completed_ref, SW2_ConcreteReference) and self.make_local_sweetheart: completed_ref = SW2_SweetheartReference.from_concrete( completed_ref, get_own_netloc()) return completed_ref
def publish_fetched_ref(self, fetch): completed_ref = fetch.get_completed_ref() if completed_ref is None: ciel.log("Cancelling async fetch %s (chunk %d)" % (fetch.ref.id, fetch.chunk_size), "EXEC", logging.DEBUG) else: if fetch.make_sweetheart: completed_ref = SW2_SweetheartReference.from_concrete(completed_ref, get_own_netloc()) self.task_record.publish_ref(completed_ref)
def open_ref_async(self, ref, chunk_size, sole_consumer=False, make_sweetheart=False, must_block=False, fd_socket_name=None): if not sendmsg_enabled: fd_socket_name = None ciel.log("Not using FDs directly: module 'sendmsg' not available", "EXEC", logging.DEBUG) real_ref = self.task_record.retrieve_ref(ref) new_fetch = OngoingFetch(real_ref, chunk_size, self.task_record, sole_consumer, make_sweetheart, must_block, can_accept_fd=(fd_socket_name is not None)) ret = {"sending_fd": False} ret_fd = None if fd_socket_name is not None: fd, fd_blocking = new_fetch.get_fd() if fd is not None: ret["sending_fd"] = True ret["blocking"] = fd_blocking ret_fd = fd if not ret["sending_fd"]: filename, file_blocking = new_fetch.get_filename() ret["filename"] = filename ret["blocking"] = file_blocking if not new_fetch.done: self.context_manager.add_context(new_fetch) self.ongoing_fetches.append(new_fetch) else: self.publish_fetched_ref(new_fetch) # Definitions here: "done" means we're already certain that the producer has completed successfully. # "blocking" means that EOF, as and when it arrives, means what it says. i.e. it's a regular file and done, or a pipe-like thing. ret.update({"done": new_fetch.done, "size": new_fetch.bytes}) ciel.log( "Async fetch %s (chunk %d): initial status %d bytes, done=%s, blocking=%s, sending_fd=%s" % (real_ref, chunk_size, ret["size"], ret["done"], ret["blocking"], ret["sending_fd"]), "EXEC", logging.DEBUG) # XXX: adding this because the OngoingFetch isn't publishing the sweetheart correctly. if make_sweetheart: self.task_record.publish_ref( SW2_SweetheartReference(ref.id, get_own_netloc())) if new_fetch.done: if not new_fetch.success: ciel.log("Async fetch %s failed early" % ref, "EXEC", logging.WARNING) ret["error"] = "EFAILED" return (ret, ret_fd)
def publish_fetched_ref(self, fetch): completed_ref = fetch.get_completed_ref() if completed_ref is None: ciel.log( "Cancelling async fetch %s (chunk %d)" % (fetch.ref.id, fetch.chunk_size), "EXEC", logging.DEBUG) else: if fetch.make_sweetheart: completed_ref = SW2_SweetheartReference.from_concrete( completed_ref, get_own_netloc()) self.task_record.publish_ref(completed_ref)
def open_ref(self, ref, accept_string=False, make_sweetheart=False): """Fetches a reference if it is available, and returns a filename for reading it. Options to do with eagerness, streaming, etc. If reference is unavailable, raises a ReferenceUnavailableException.""" ref = self.task_record.retrieve_ref(ref) if not accept_string: ctx = retrieve_filename_for_ref(ref, self.task_record, return_ctx=True) else: ctx = retrieve_file_or_string_for_ref(ref, self.task_record) if ctx.completed_ref is not None: if make_sweetheart: ctx.completed_ref = SW2_SweetheartReference.from_concrete(ctx.completed_ref, get_own_netloc()) self.task_record.publish_ref(ctx.completed_ref) return ctx.to_safe_dict()
def open_ref(self, ref, accept_string=False, make_sweetheart=False): """Fetches a reference if it is available, and returns a filename for reading it. Options to do with eagerness, streaming, etc. If reference is unavailable, raises a ReferenceUnavailableException.""" ref = self.task_record.retrieve_ref(ref) if not accept_string: ctx = retrieve_filename_for_ref(ref, self.task_record, return_ctx=True) else: ctx = retrieve_file_or_string_for_ref(ref, self.task_record) if ctx.completed_ref is not None: if make_sweetheart: ctx.completed_ref = SW2_SweetheartReference.from_concrete( ctx.completed_ref, get_own_netloc()) self.task_record.publish_ref(ctx.completed_ref) return ctx.to_safe_dict()
def get_completed_ref(self): completed_ref = self.output_ctx.get_completed_ref() if isinstance(completed_ref, SW2_ConcreteReference) and self.make_local_sweetheart: completed_ref = SW2_SweetheartReference.from_concrete(completed_ref, get_own_netloc()) return completed_ref