def get_completed_ref(self):
completed_ref = self.output_ctx.get_completed_ref()
if isinstance(completed_ref,
SW2_ConcreteReference) and self.make_local_sweetheart:
completed_ref = SW2_SweetheartReference.from_concrete(
completed_ref, get_own_netloc())
return completed_ref
def publish_fetched_ref(self, fetch):
completed_ref = fetch.get_completed_ref()
if completed_ref is None:
ciel.log("Cancelling async fetch %s (chunk %d)" % (fetch.ref.id, fetch.chunk_size), "EXEC", logging.DEBUG)
else:
if fetch.make_sweetheart:
completed_ref = SW2_SweetheartReference.from_concrete(completed_ref, get_own_netloc())
self.task_record.publish_ref(completed_ref)
def open_ref_async(self,
ref,
chunk_size,
sole_consumer=False,
make_sweetheart=False,
must_block=False,
fd_socket_name=None):
if not sendmsg_enabled:
fd_socket_name = None
ciel.log("Not using FDs directly: module 'sendmsg' not available",
"EXEC", logging.DEBUG)
real_ref = self.task_record.retrieve_ref(ref)
new_fetch = OngoingFetch(real_ref,
chunk_size,
self.task_record,
sole_consumer,
make_sweetheart,
must_block,
can_accept_fd=(fd_socket_name is not None))
ret = {"sending_fd": False}
ret_fd = None
if fd_socket_name is not None:
fd, fd_blocking = new_fetch.get_fd()
if fd is not None:
ret["sending_fd"] = True
ret["blocking"] = fd_blocking
ret_fd = fd
if not ret["sending_fd"]:
filename, file_blocking = new_fetch.get_filename()
ret["filename"] = filename
ret["blocking"] = file_blocking
if not new_fetch.done:
self.context_manager.add_context(new_fetch)
self.ongoing_fetches.append(new_fetch)
else:
self.publish_fetched_ref(new_fetch)
# Definitions here: "done" means we're already certain that the producer has completed successfully.
# "blocking" means that EOF, as and when it arrives, means what it says. i.e. it's a regular file and done, or a pipe-like thing.
ret.update({"done": new_fetch.done, "size": new_fetch.bytes})
ciel.log(
"Async fetch %s (chunk %d): initial status %d bytes, done=%s, blocking=%s, sending_fd=%s"
% (real_ref, chunk_size, ret["size"], ret["done"], ret["blocking"],
ret["sending_fd"]), "EXEC", logging.DEBUG)
# XXX: adding this because the OngoingFetch isn't publishing the sweetheart correctly.
if make_sweetheart:
self.task_record.publish_ref(
SW2_SweetheartReference(ref.id, get_own_netloc()))
if new_fetch.done:
if not new_fetch.success:
ciel.log("Async fetch %s failed early" % ref, "EXEC",
logging.WARNING)
ret["error"] = "EFAILED"
return (ret, ret_fd)
def publish_fetched_ref(self, fetch):
completed_ref = fetch.get_completed_ref()
if completed_ref is None:
ciel.log(
"Cancelling async fetch %s (chunk %d)" %
(fetch.ref.id, fetch.chunk_size), "EXEC", logging.DEBUG)
else:
if fetch.make_sweetheart:
completed_ref = SW2_SweetheartReference.from_concrete(
completed_ref, get_own_netloc())
self.task_record.publish_ref(completed_ref)
def open_ref(self, ref, accept_string=False, make_sweetheart=False):
"""Fetches a reference if it is available, and returns a filename for reading it.
Options to do with eagerness, streaming, etc.
If reference is unavailable, raises a ReferenceUnavailableException."""
ref = self.task_record.retrieve_ref(ref)
if not accept_string:
ctx = retrieve_filename_for_ref(ref, self.task_record, return_ctx=True)
else:
ctx = retrieve_file_or_string_for_ref(ref, self.task_record)
if ctx.completed_ref is not None:
if make_sweetheart:
ctx.completed_ref = SW2_SweetheartReference.from_concrete(ctx.completed_ref, get_own_netloc())
self.task_record.publish_ref(ctx.completed_ref)
return ctx.to_safe_dict()
def open_ref(self, ref, accept_string=False, make_sweetheart=False):
"""Fetches a reference if it is available, and returns a filename for reading it.
Options to do with eagerness, streaming, etc.
If reference is unavailable, raises a ReferenceUnavailableException."""
ref = self.task_record.retrieve_ref(ref)
if not accept_string:
ctx = retrieve_filename_for_ref(ref,
self.task_record,
return_ctx=True)
else:
ctx = retrieve_file_or_string_for_ref(ref, self.task_record)
if ctx.completed_ref is not None:
if make_sweetheart:
ctx.completed_ref = SW2_SweetheartReference.from_concrete(
ctx.completed_ref, get_own_netloc())
self.task_record.publish_ref(ctx.completed_ref)
return ctx.to_safe_dict()
def get_completed_ref(self):
completed_ref = self.output_ctx.get_completed_ref()
if isinstance(completed_ref, SW2_ConcreteReference) and self.make_local_sweetheart:
completed_ref = SW2_SweetheartReference.from_concrete(completed_ref, get_own_netloc())
return completed_ref