Ejemplo n.º 1
0
 def get_completed_ref(self):
     completed_ref = self.output_ctx.get_completed_ref()
     if isinstance(completed_ref,
                   SW2_ConcreteReference) and self.make_local_sweetheart:
         completed_ref = SW2_SweetheartReference.from_concrete(
             completed_ref, get_own_netloc())
     return completed_ref
Ejemplo n.º 2
0
Archivo: proc.py Proyecto: jepst/ciel
 def publish_fetched_ref(self, fetch):
     completed_ref = fetch.get_completed_ref()
     if completed_ref is None:
         ciel.log("Cancelling async fetch %s (chunk %d)" % (fetch.ref.id, fetch.chunk_size), "EXEC", logging.DEBUG)
     else:
         if fetch.make_sweetheart:
             completed_ref = SW2_SweetheartReference.from_concrete(completed_ref, get_own_netloc())
         self.task_record.publish_ref(completed_ref)
Ejemplo n.º 3
0
    def open_ref_async(self,
                       ref,
                       chunk_size,
                       sole_consumer=False,
                       make_sweetheart=False,
                       must_block=False,
                       fd_socket_name=None):
        if not sendmsg_enabled:
            fd_socket_name = None
            ciel.log("Not using FDs directly: module 'sendmsg' not available",
                     "EXEC", logging.DEBUG)
        real_ref = self.task_record.retrieve_ref(ref)

        new_fetch = OngoingFetch(real_ref,
                                 chunk_size,
                                 self.task_record,
                                 sole_consumer,
                                 make_sweetheart,
                                 must_block,
                                 can_accept_fd=(fd_socket_name is not None))
        ret = {"sending_fd": False}
        ret_fd = None
        if fd_socket_name is not None:
            fd, fd_blocking = new_fetch.get_fd()
            if fd is not None:
                ret["sending_fd"] = True
                ret["blocking"] = fd_blocking
                ret_fd = fd
        if not ret["sending_fd"]:
            filename, file_blocking = new_fetch.get_filename()
            ret["filename"] = filename
            ret["blocking"] = file_blocking
        if not new_fetch.done:
            self.context_manager.add_context(new_fetch)
            self.ongoing_fetches.append(new_fetch)
        else:
            self.publish_fetched_ref(new_fetch)
        # Definitions here: "done" means we're already certain that the producer has completed successfully.
        # "blocking" means that EOF, as and when it arrives, means what it says. i.e. it's a regular file and done, or a pipe-like thing.
        ret.update({"done": new_fetch.done, "size": new_fetch.bytes})
        ciel.log(
            "Async fetch %s (chunk %d): initial status %d bytes, done=%s, blocking=%s, sending_fd=%s"
            % (real_ref, chunk_size, ret["size"], ret["done"], ret["blocking"],
               ret["sending_fd"]), "EXEC", logging.DEBUG)

        # XXX: adding this because the OngoingFetch isn't publishing the sweetheart correctly.
        if make_sweetheart:
            self.task_record.publish_ref(
                SW2_SweetheartReference(ref.id, get_own_netloc()))

        if new_fetch.done:
            if not new_fetch.success:
                ciel.log("Async fetch %s failed early" % ref, "EXEC",
                         logging.WARNING)
                ret["error"] = "EFAILED"
        return (ret, ret_fd)
Ejemplo n.º 4
0
 def publish_fetched_ref(self, fetch):
     completed_ref = fetch.get_completed_ref()
     if completed_ref is None:
         ciel.log(
             "Cancelling async fetch %s (chunk %d)" %
             (fetch.ref.id, fetch.chunk_size), "EXEC", logging.DEBUG)
     else:
         if fetch.make_sweetheart:
             completed_ref = SW2_SweetheartReference.from_concrete(
                 completed_ref, get_own_netloc())
         self.task_record.publish_ref(completed_ref)
Ejemplo n.º 5
0
Archivo: proc.py Proyecto: jepst/ciel
 def open_ref(self, ref, accept_string=False, make_sweetheart=False):
     """Fetches a reference if it is available, and returns a filename for reading it.
     Options to do with eagerness, streaming, etc.
     If reference is unavailable, raises a ReferenceUnavailableException."""
     ref = self.task_record.retrieve_ref(ref)
     if not accept_string:   
         ctx = retrieve_filename_for_ref(ref, self.task_record, return_ctx=True)
     else:
         ctx = retrieve_file_or_string_for_ref(ref, self.task_record)
     if ctx.completed_ref is not None:
         if make_sweetheart:
             ctx.completed_ref = SW2_SweetheartReference.from_concrete(ctx.completed_ref, get_own_netloc())
         self.task_record.publish_ref(ctx.completed_ref)
     return ctx.to_safe_dict()
Ejemplo n.º 6
0
 def open_ref(self, ref, accept_string=False, make_sweetheart=False):
     """Fetches a reference if it is available, and returns a filename for reading it.
     Options to do with eagerness, streaming, etc.
     If reference is unavailable, raises a ReferenceUnavailableException."""
     ref = self.task_record.retrieve_ref(ref)
     if not accept_string:
         ctx = retrieve_filename_for_ref(ref,
                                         self.task_record,
                                         return_ctx=True)
     else:
         ctx = retrieve_file_or_string_for_ref(ref, self.task_record)
     if ctx.completed_ref is not None:
         if make_sweetheart:
             ctx.completed_ref = SW2_SweetheartReference.from_concrete(
                 ctx.completed_ref, get_own_netloc())
         self.task_record.publish_ref(ctx.completed_ref)
     return ctx.to_safe_dict()
Ejemplo n.º 7
0
 def get_completed_ref(self):
     completed_ref = self.output_ctx.get_completed_ref()
     if isinstance(completed_ref, SW2_ConcreteReference) and self.make_local_sweetheart:
         completed_ref = SW2_SweetheartReference.from_concrete(completed_ref, get_own_netloc())
     return completed_ref