def get_stream_ref(self):
if ciel.runtime.tcp_server.tcp_server_active():
return SW2_SocketStreamReference(
self.refid, get_own_netloc(),
ciel.runtime.tcp_server.aux_listen_port)
else:
return SW2_StreamReference(self.refid,
location_hints=[get_own_netloc()])
def build_task_descriptor(cls,
task_descriptor,
parent_task_record,
process_record_id=None,
is_fixed=False,
command=None,
proc_pargs=[],
proc_kwargs={},
force_n_outputs=None,
n_extra_outputs=0,
extra_dependencies=[],
is_tail_spawn=False,
accept_ref_list_for_single=False):
#if process_record_id is None and start_command is None:
# raise BlameUserException("ProcExecutor tasks must specify either process_record_id or start_command")
if process_record_id is not None:
task_descriptor["task_private"]["id"] = process_record_id
if command is not None:
task_descriptor["task_private"]["command"] = command
task_descriptor["task_private"]["proc_pargs"] = proc_pargs
task_descriptor["task_private"]["proc_kwargs"] = proc_kwargs
task_descriptor["dependencies"].extend(extra_dependencies)
task_private_id = ("%s:_private" % task_descriptor["task_id"])
if is_fixed:
task_private_ref = SW2_FixedReference(task_private_id,
get_own_netloc())
write_fixed_ref_string(
pickle.dumps(task_descriptor["task_private"]),
task_private_ref)
else:
task_private_ref = ref_from_string(
pickle.dumps(task_descriptor["task_private"]), task_private_id)
parent_task_record.publish_ref(task_private_ref)
task_descriptor["task_private"] = task_private_ref
task_descriptor["dependencies"].append(task_private_ref)
if force_n_outputs is not None:
if "expected_outputs" in task_descriptor and len(
task_descriptor["expected_outputs"]) > 0:
raise BlameUserException(
"Task already had outputs, but force_n_outputs is set")
task_descriptor["expected_outputs"] = [
"%s:out:%d" % (task_descriptor["task_id"], i)
for i in range(force_n_outputs)
]
if not is_tail_spawn:
if len(task_descriptor["expected_outputs"]
) == 1 and not accept_ref_list_for_single:
return SW2_FutureReference(
task_descriptor["expected_outputs"][0])
else:
return [
SW2_FutureReference(refid)
for refid in task_descriptor["expected_outputs"]
]
def get_completed_ref(self):
completed_ref = self.output_ctx.get_completed_ref()
if isinstance(completed_ref,
SW2_ConcreteReference) and self.make_local_sweetheart:
completed_ref = SW2_SweetheartReference.from_concrete(
completed_ref, get_own_netloc())
return completed_ref
def thread_main(self):
try:
with self.lock:
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.should_close = True
ciel.log("Connecting %s:%s" % (self.otherend_hostname, self.ref.socket_port), "TCP_FETCH", logging.DEBUG)
subscribe_remote_output_nopost(self.ref.id, self)
self.sock.connect((self.otherend_hostname, self.ref.socket_port))
self.sock.sendall("%s %s %d\n" % (self.ref.id, get_own_netloc(), self.chunk_size))
ciel.log("%s:%s connected: requesting %s (chunk size %d)" % (self.otherend_hostname, self.ref.socket_port, self.ref.id, self.chunk_size), "TCP_FETCH", logging.DEBUG)
fp = self.sock.makefile("r", bufsize=0)
response = fp.readline().strip()
fp.close()
with self.lock:
self.should_close = False
if response.find("GO") != -1:
ciel.log("TCP-fetch %s: transfer started" % self.ref.id, "TCP_FETCH", logging.DEBUG)
new_fd = os.dup(self.sock.fileno())
self.sock.close()
self.fetch_ctx.set_fd(new_fd, True)
else:
ciel.log("TCP-fetch %s: request failed: other end said '%s'" % (self.ref.id, response), "TCP_FETCH", logging.WARNING)
unsubscribe_remote_output_nopost(self.ref.id)
self.done = True
self.sock.close()
self.fetch_ctx.result(False)
except Exception as e:
unsubscribe_remote_output_nopost(self.ref.id)
ciel.log("TCP-fetch %s: failed due to exception %s" % (self.ref.id, repr(e)), "TCP_FETCH", logging.ERROR)
with self.lock:
if self.should_close:
self.sock.close()
self.done = True
self.should_close = False
self.fetch_ctx.result(False)
def fetch_ref_async(ref,
result_callback,
reset_callback,
start_filename_callback,
start_fd_callback=None,
string_callback=None,
progress_callback=None,
chunk_size=67108864,
may_pipe=False,
sole_consumer=False,
must_block=False,
task_record=None):
if isinstance(ref, SWErrorReference):
raise ErrorReferenceError(ref)
if isinstance(ref, SW2_FixedReference):
assert ref.fixed_netloc == get_own_netloc()
new_client = FetchInProgress(ref, result_callback, reset_callback,
start_filename_callback, start_fd_callback,
string_callback, progress_callback,
chunk_size, may_pipe, sole_consumer,
must_block, task_record)
new_client.start_fetch()
return new_client
def unsubscribe_remote_output(refid):
unsubscribe_remote_output_nopost(refid)
netloc = get_own_netloc()
post_data = simplejson.dumps({"netloc": netloc})
post_string_noreturn(
"http://%s/control/streamstat/%s/unsubscribe" % (netloc, refid),
post_data)
def publish_fetched_ref(self, fetch):
completed_ref = fetch.get_completed_ref()
if completed_ref is None:
ciel.log("Cancelling async fetch %s (chunk %d)" % (fetch.ref.id, fetch.chunk_size), "EXEC", logging.DEBUG)
else:
if fetch.make_sweetheart:
completed_ref = SW2_SweetheartReference.from_concrete(completed_ref, get_own_netloc())
self.task_record.publish_ref(completed_ref)
def open_ref_async(self,
ref,
chunk_size,
sole_consumer=False,
make_sweetheart=False,
must_block=False,
fd_socket_name=None):
if not sendmsg_enabled:
fd_socket_name = None
ciel.log("Not using FDs directly: module 'sendmsg' not available",
"EXEC", logging.DEBUG)
real_ref = self.task_record.retrieve_ref(ref)
new_fetch = OngoingFetch(real_ref,
chunk_size,
self.task_record,
sole_consumer,
make_sweetheart,
must_block,
can_accept_fd=(fd_socket_name is not None))
ret = {"sending_fd": False}
ret_fd = None
if fd_socket_name is not None:
fd, fd_blocking = new_fetch.get_fd()
if fd is not None:
ret["sending_fd"] = True
ret["blocking"] = fd_blocking
ret_fd = fd
if not ret["sending_fd"]:
filename, file_blocking = new_fetch.get_filename()
ret["filename"] = filename
ret["blocking"] = file_blocking
if not new_fetch.done:
self.context_manager.add_context(new_fetch)
self.ongoing_fetches.append(new_fetch)
else:
self.publish_fetched_ref(new_fetch)
# Definitions here: "done" means we're already certain that the producer has completed successfully.
# "blocking" means that EOF, as and when it arrives, means what it says. i.e. it's a regular file and done, or a pipe-like thing.
ret.update({"done": new_fetch.done, "size": new_fetch.bytes})
ciel.log(
"Async fetch %s (chunk %d): initial status %d bytes, done=%s, blocking=%s, sending_fd=%s"
% (real_ref, chunk_size, ret["size"], ret["done"], ret["blocking"],
ret["sending_fd"]), "EXEC", logging.DEBUG)
# XXX: adding this because the OngoingFetch isn't publishing the sweetheart correctly.
if make_sweetheart:
self.task_record.publish_ref(
SW2_SweetheartReference(ref.id, get_own_netloc()))
if new_fetch.done:
if not new_fetch.success:
ciel.log("Async fetch %s failed early" % ref, "EXEC",
logging.WARNING)
ret["error"] = "EFAILED"
return (ret, ret_fd)
def result(self, success):
self.completed = True
del active_http_transfers[self.ref.id]
if success:
ref = SW2_ConcreteReference(self.ref.id, self.last_progress, [get_own_netloc()])
self.bs_ctx.commit()
else:
ref = None
for l in self.listeners:
l.result(success, ref)
def result(self, success):
self.completed = True
del active_http_transfers[self.ref.id]
if success:
ref = SW2_ConcreteReference(self.ref.id, self.last_progress,
[get_own_netloc()])
self.bs_ctx.commit()
else:
ref = None
for l in self.listeners:
l.result(success, ref)
def subscribe_remote_output(refid, remote_netloc, chunk_size, subscriber):
subscribe_remote_output_nopost(refid, subscriber)
post_data = simplejson.dumps({
"netloc": get_own_netloc(),
"chunk_size": chunk_size
})
post_string_noreturn(
"http://%s/control/streamstat/%s/subscribe" % (remote_netloc, refid),
post_data,
result_callback=(
lambda success, url: subscribe_result(refid, success, url)))
def publish_fetched_ref(self, fetch):
completed_ref = fetch.get_completed_ref()
if completed_ref is None:
ciel.log(
"Cancelling async fetch %s (chunk %d)" %
(fetch.ref.id, fetch.chunk_size), "EXEC", logging.DEBUG)
else:
if fetch.make_sweetheart:
completed_ref = SW2_SweetheartReference.from_concrete(
completed_ref, get_own_netloc())
self.task_record.publish_ref(completed_ref)
def get_completed_ref(self):
if not self.closed:
raise Exception(
"FileOutputContext for ref %s must be closed before it is realised as a concrete reference"
% self.refid)
if self.direct_write_filename is not None or self.direct_write_fd is not None:
return SW2_CompletedReference(self.refid)
completed_file = producer_filename(self.refid)
if self.current_size < 1024:
with open(completed_file, "r") as fp:
return SWDataValue(self.refid, encode_datavalue(fp.read()))
else:
return SW2_ConcreteReference(self.refid,
size_hint=self.current_size,
location_hints=[get_own_netloc()])
def open_ref(self, ref, accept_string=False, make_sweetheart=False):
"""Fetches a reference if it is available, and returns a filename for reading it.
Options to do with eagerness, streaming, etc.
If reference is unavailable, raises a ReferenceUnavailableException."""
ref = self.task_record.retrieve_ref(ref)
if not accept_string:
ctx = retrieve_filename_for_ref(ref,
self.task_record,
return_ctx=True)
else:
ctx = retrieve_file_or_string_for_ref(ref, self.task_record)
if ctx.completed_ref is not None:
if make_sweetheart:
ctx.completed_ref = SW2_SweetheartReference.from_concrete(
ctx.completed_ref, get_own_netloc())
self.task_record.publish_ref(ctx.completed_ref)
return ctx.to_safe_dict()
def open_ref_async(self, ref, chunk_size, sole_consumer=False, make_sweetheart=False, must_block=False, fd_socket_name=None):
if not sendmsg_enabled:
fd_socket_name = None
ciel.log("Not using FDs directly: module 'sendmsg' not available", "EXEC", logging.DEBUG)
real_ref = self.task_record.retrieve_ref(ref)
new_fetch = OngoingFetch(real_ref, chunk_size, self.task_record, sole_consumer, make_sweetheart, must_block, can_accept_fd=(fd_socket_name is not None))
ret = {"sending_fd": False}
ret_fd = None
if fd_socket_name is not None:
fd, fd_blocking = new_fetch.get_fd()
if fd is not None:
ret["sending_fd"] = True
ret["blocking"] = fd_blocking
ret_fd = fd
if not ret["sending_fd"]:
filename, file_blocking = new_fetch.get_filename()
ret["filename"] = filename
ret["blocking"] = file_blocking
if not new_fetch.done:
self.context_manager.add_context(new_fetch)
self.ongoing_fetches.append(new_fetch)
else:
self.publish_fetched_ref(new_fetch)
# Definitions here: "done" means we're already certain that the producer has completed successfully.
# "blocking" means that EOF, as and when it arrives, means what it says. i.e. it's a regular file and done, or a pipe-like thing.
ret.update({"done": new_fetch.done, "size": new_fetch.bytes})
ciel.log("Async fetch %s (chunk %d): initial status %d bytes, done=%s, blocking=%s, sending_fd=%s" % (real_ref, chunk_size, ret["size"], ret["done"], ret["blocking"], ret["sending_fd"]), "EXEC", logging.DEBUG)
# XXX: adding this because the OngoingFetch isn't publishing the sweetheart correctly.
if make_sweetheart:
self.task_record.publish_ref(SW2_SweetheartReference(ref.id, get_own_netloc()))
if new_fetch.done:
if not new_fetch.success:
ciel.log("Async fetch %s failed early" % ref, "EXEC", logging.WARNING)
ret["error"] = "EFAILED"
return (ret, ret_fd)
def build_task_descriptor(cls, task_descriptor, parent_task_record,
process_record_id=None, is_fixed=False, command=None, proc_pargs=[], proc_kwargs={}, force_n_outputs=None,
n_extra_outputs=0, extra_dependencies=[], is_tail_spawn=False, accept_ref_list_for_single=False):
#if process_record_id is None and start_command is None:
# raise BlameUserException("ProcExecutor tasks must specify either process_record_id or start_command")
if process_record_id is not None:
task_descriptor["task_private"]["id"] = process_record_id
if command is not None:
task_descriptor["task_private"]["command"] = command
task_descriptor["task_private"]["proc_pargs"] = proc_pargs
task_descriptor["task_private"]["proc_kwargs"] = proc_kwargs
task_descriptor["dependencies"].extend(extra_dependencies)
task_private_id = ("%s:_private" % task_descriptor["task_id"])
if is_fixed:
task_private_ref = SW2_FixedReference(task_private_id, get_own_netloc())
write_fixed_ref_string(pickle.dumps(task_descriptor["task_private"]), task_private_ref)
else:
task_private_ref = ref_from_string(pickle.dumps(task_descriptor["task_private"]), task_private_id)
parent_task_record.publish_ref(task_private_ref)
task_descriptor["task_private"] = task_private_ref
task_descriptor["dependencies"].append(task_private_ref)
if force_n_outputs is not None:
if "expected_outputs" in task_descriptor and len(task_descriptor["expected_outputs"]) > 0:
raise BlameUserException("Task already had outputs, but force_n_outputs is set")
task_descriptor["expected_outputs"] = ["%s:out:%d" % (task_descriptor["task_id"], i) for i in range(force_n_outputs)]
if not is_tail_spawn:
if len(task_descriptor["expected_outputs"]) == 1 and not accept_ref_list_for_single:
return SW2_FutureReference(task_descriptor["expected_outputs"][0])
else:
return [SW2_FutureReference(refid) for refid in task_descriptor["expected_outputs"]]
def fetch_ref_async(
ref,
result_callback,
reset_callback,
start_filename_callback,
start_fd_callback=None,
string_callback=None,
progress_callback=None,
chunk_size=67108864,
may_pipe=False,
sole_consumer=False,
must_block=False,
task_record=None,
):
if isinstance(ref, SWErrorReference):
raise ErrorReferenceError(ref)
if isinstance(ref, SW2_FixedReference):
assert ref.fixed_netloc == get_own_netloc()
new_client = FetchInProgress(
ref,
result_callback,
reset_callback,
start_filename_callback,
start_fd_callback,
string_callback,
progress_callback,
chunk_size,
may_pipe,
sole_consumer,
must_block,
task_record,
)
new_client.start_fetch()
return new_client
def subscribe_remote_output(refid, remote_netloc, chunk_size, subscriber):
subscribe_remote_output_nopost(refid, subscriber)
post_data = simplejson.dumps({"netloc": get_own_netloc(), "chunk_size": chunk_size})
post_string_noreturn("http://%s/control/streamstat/%s/subscribe" % (remote_netloc, refid), post_data, result_callback=(lambda success, url: subscribe_result(refid, success, url)))
def open_ref(self, ref, accept_string=False, make_sweetheart=False):
"""Fetches a reference if it is available, and returns a filename for reading it.
Options to do with eagerness, streaming, etc.
If reference is unavailable, raises a ReferenceUnavailableException."""
ref = self.task_record.retrieve_ref(ref)
if not accept_string:
ctx = retrieve_filename_for_ref(ref, self.task_record, return_ctx=True)
else:
ctx = retrieve_file_or_string_for_ref(ref, self.task_record)
if ctx.completed_ref is not None:
if make_sweetheart:
ctx.completed_ref = SW2_SweetheartReference.from_concrete(ctx.completed_ref, get_own_netloc())
self.task_record.publish_ref(ctx.completed_ref)
return ctx.to_safe_dict()
def unsubscribe_remote_output(refid):
unsubscribe_remote_output_nopost(refid)
netloc = get_own_netloc()
post_data = simplejson.dumps({"netloc": netloc})
post_string_noreturn("http://%s/control/streamstat/%s/unsubscribe"
% (netloc, refid), post_data)
def get_completed_ref(self):
completed_ref = self.output_ctx.get_completed_ref()
if isinstance(completed_ref, SW2_ConcreteReference) and self.make_local_sweetheart:
completed_ref = SW2_SweetheartReference.from_concrete(completed_ref, get_own_netloc())
return completed_ref
def get_stream_ref(self):
if ciel.runtime.tcp_server.tcp_server_active():
return SW2_SocketStreamReference(self.refid, get_own_netloc(), ciel.runtime.tcp_server.aux_listen_port)
else:
return SW2_StreamReference(self.refid, location_hints=[get_own_netloc()])
def get_completed_ref(self):
if not self.closed:
raise Exception("FileOutputContext for ref %s must be closed before it is realised as a concrete reference" % self.refid)
if self.direct_write_filename is not None or self.direct_write_fd is not None:
return SW2_CompletedReference(self.refid)
completed_file = producer_filename(self.refid)
if self.current_size < 1024:
with open(completed_file, "r") as fp:
return SWDataValue(self.refid, encode_datavalue(fp.read()))
else:
return SW2_ConcreteReference(self.refid, size_hint=self.current_size, location_hints=[get_own_netloc()])
