def get_completed_ref(self):
if not self.closed:
raise Exception("FileOutputContext for ref %s must be closed before it is realised as a concrete reference" % self.refid)
if self.direct_write_filename is not None or self.direct_write_fd is not None:
return SW2_CompletedReference(self.refid)
completed_file = producer_filename(self.refid)
if self.current_size < 1024:
with open(completed_file, "r") as fp:
return SWDataValue(self.refid, encode_datavalue(fp.read()))
else:
return SW2_ConcreteReference(self.refid, size_hint=self.current_size, location_hints=[get_own_netloc()])
def make_local_output(id, subscribe_callback=None, may_pipe=False, can_use_fd=False):
'''
Creates a file-in-progress in the block store directory.
'''
if subscribe_callback is None:
subscribe_callback = fwt.create_watch
ciel.log.error('Creating file for output %s' % id, 'BLOCKSTORE', logging.DEBUG)
new_ctx = FileOutputContext(id, subscribe_callback, may_pipe=may_pipe, can_use_fd=can_use_fd)
dot_filename = producer_filename(id)
open(dot_filename, 'wb').close()
streaming_producers[id] = new_ctx
return new_ctx
def close(self):
if not self.closed:
del streaming_producers[self.refid]
with self.lock:
self.closed = True
self.succeeded = True
if self.direct_write_filename is None and self.direct_write_fd is None:
ciel.runtime.block_store.commit_producer(self.refid)
if self.file_watch is not None:
self.file_watch.cancel()
self.current_size = os.stat(producer_filename(self.refid)).st_size
for subscriber in self.subscriptions:
subscriber.progress(self.current_size)
subscriber.result(True)
def close(self):
if not self.closed:
del streaming_producers[self.refid]
with self.lock:
self.closed = True
self.succeeded = True
if self.direct_write_filename is None and self.direct_write_fd is None:
ciel.runtime.block_store.commit_producer(self.refid)
if self.file_watch is not None:
self.file_watch.cancel()
self.current_size = os.stat(producer_filename(self.refid)).st_size
for subscriber in self.subscriptions:
subscriber.progress(self.current_size)
subscriber.result(True)
def get_completed_ref(self):
if not self.closed:
raise Exception(
"FileOutputContext for ref %s must be closed before it is realised as a concrete reference"
% self.refid)
if self.direct_write_filename is not None or self.direct_write_fd is not None:
return SW2_CompletedReference(self.refid)
completed_file = producer_filename(self.refid)
if self.current_size < 1024:
with open(completed_file, "r") as fp:
return SWDataValue(self.refid, encode_datavalue(fp.read()))
else:
return SW2_ConcreteReference(self.refid,
size_hint=self.current_size,
location_hints=[get_own_netloc()])
def thread_main(self):
try:
fd_taken = self.socket_init()
if fd_taken is None:
return
elif fd_taken is True:
ciel.log(
"Incoming TCP connection for %s connected directly to producer"
% self.refid, "TCP_SERVER", logging.DEBUG)
self.sock_obj.close()
return
# Otherwise we'll get progress/result callbacks as we follow the producer's on-disk file.
os.close(self.write_fd)
self.read_filename = producer_filename(self.refid)
ciel.log(
"Auxiliary TCP connection for output %s (chunk %s) attached via push thread"
% (self.refid, self.chunk_size), "TCP_FETCH", logging.DEBUG)
with open(self.read_filename, "r") as input_fp:
while True:
while True:
buf = input_fp.read(4096)
self.sock_obj.sendall(buf)
self.bytes_copied += len(buf)
with self.lock:
if self.bytes_copied == self.bytes_available and self.fetch_done:
ciel.log(
"Socket-push for %s complete: wrote %d bytes"
% (self.refid, self.bytes_copied),
"TCP_SERVER", logging.DEBUG)
self.sock_obj.close()
return
if len(buf) < self.chunk_size:
# EOF, for now.
break
with self.lock:
self.pause_threshold = self.bytes_copied + self.chunk_size
while self.bytes_available < self.pause_threshold and not self.fetch_done:
self.cond.wait()
self.pause_threshold = None
except Exception as e:
ciel.log("Socket-push-thread died with exception %s" % repr(e),
"TCP_FETCH", logging.ERROR)
try:
self.sock_obj.close()
except:
pass
def attach_local_producer(self):
producer = get_producer_for_id(self.ref.id)
if producer is None:
raise PlanFailedError(
"Plan attach-local-producer failed for %s: not being produced here" % self.ref,
"BLOCKSTORE",
logging.DEBUG,
)
else:
is_pipe = producer.subscribe(self, try_direct=(self.may_pipe and self.sole_consumer))
if is_pipe:
ciel.log("Fetch-ref %s: attached to direct pipe!" % self.ref, "BLOCKSTORE", logging.DEBUG)
filename = producer.get_fifo_filename()
else:
ciel.log("Fetch-ref %s: following local producer's file" % self.ref, "BLOCKSTORE", logging.DEBUG)
filename = producer_filename(self.ref.id)
self.set_filename(filename, is_pipe)
def make_local_output(id,
subscribe_callback=None,
may_pipe=False,
can_use_fd=False):
'''
Creates a file-in-progress in the block store directory.
'''
if subscribe_callback is None:
subscribe_callback = fwt.create_watch
ciel.log.error('Creating file for output %s' % id, 'BLOCKSTORE',
logging.DEBUG)
new_ctx = FileOutputContext(id,
subscribe_callback,
may_pipe=may_pipe,
can_use_fd=can_use_fd)
dot_filename = producer_filename(id)
open(dot_filename, 'wb').close()
streaming_producers[id] = new_ctx
return new_ctx
def attach_local_producer(self):
producer = get_producer_for_id(self.ref.id)
if producer is None:
raise PlanFailedError(
"Plan attach-local-producer failed for %s: not being produced here"
% self.ref, "BLOCKSTORE", logging.DEBUG)
else:
is_pipe = producer.subscribe(self,
try_direct=(self.may_pipe
and self.sole_consumer))
if is_pipe:
ciel.log("Fetch-ref %s: attached to direct pipe!" % self.ref,
"BLOCKSTORE", logging.DEBUG)
filename = producer.get_fifo_filename()
else:
ciel.log(
"Fetch-ref %s: following local producer's file" % self.ref,
"BLOCKSTORE", logging.DEBUG)
filename = producer_filename(self.ref.id)
self.set_filename(filename, is_pipe)
def thread_main(self):
try:
fd_taken = self.socket_init()
if fd_taken is None:
return
elif fd_taken is True:
ciel.log("Incoming TCP connection for %s connected directly to producer" % self.refid, "TCP_SERVER", logging.DEBUG)
self.sock_obj.close()
return
# Otherwise we'll get progress/result callbacks as we follow the producer's on-disk file.
os.close(self.write_fd)
self.read_filename = producer_filename(self.refid)
ciel.log("Auxiliary TCP connection for output %s (chunk %s) attached via push thread" % (self.refid, self.chunk_size), "TCP_FETCH", logging.DEBUG)
with open(self.read_filename, "r") as input_fp:
while True:
while True:
buf = input_fp.read(4096)
self.sock_obj.sendall(buf)
self.bytes_copied += len(buf)
with self.lock:
if self.bytes_copied == self.bytes_available and self.fetch_done:
ciel.log("Socket-push for %s complete: wrote %d bytes" % (self.refid, self.bytes_copied), "TCP_SERVER", logging.DEBUG)
self.sock_obj.close()
return
if len(buf) < self.chunk_size:
# EOF, for now.
break
with self.lock:
self.pause_threshold = self.bytes_copied + self.chunk_size
while self.bytes_available < self.pause_threshold and not self.fetch_done:
self.cond.wait()
self.pause_threshold = None
except Exception as e:
ciel.log("Socket-push-thread died with exception %s" % repr(e), "TCP_FETCH", logging.ERROR)
try:
self.sock_obj.close()
except:
pass
def get_filename_or_fd(self):
if self.may_pipe:
with self.lock:
if self.direct_write_filename is None and self.direct_write_fd is None:
now = datetime.now()
if now < self.pipe_deadline:
wait_time = self.pipe_deadline - now
wait_secs = float(wait_time.seconds) + (float(wait_time.microseconds) / 10**6)
ciel.log("Producer for %s: waiting for a direct consumer" % self.refid, "BLOCKPIPE", logging.DEBUG)
self.cond.wait(wait_secs)
if self.direct_write_filename is not None:
ciel.log("Producer for %s: writing direct to filename %s" % (self.refid, self.direct_write_filename), "BLOCKPIPE", logging.DEBUG)
self.started = True
return (self.direct_write_filename, False)
elif self.direct_write_fd is not None:
ciel.log("Producer for %s: writing direct to consumer-supplied FD" % self.refid, "BLOCKPIPE", logging.DEBUG)
self.started = True
return (self.direct_write_fd, True)
elif self.started:
ciel.log("Producer for %s: kicked by a regular-file subscription; using conventional stream-file" % self.refid, "BLOCKPIPE", logging.DEBUG)
else:
self.started = True
ciel.log("Producer for %s: timed out waiting for a consumer; writing to local block store" % self.refid, "BLOCKPIPE", logging.DEBUG)
return (producer_filename(self.refid), False)
def get_filename_or_fd(self):
if self.may_pipe:
with self.lock:
if self.direct_write_filename is None and self.direct_write_fd is None:
now = datetime.now()
if now < self.pipe_deadline:
wait_time = self.pipe_deadline - now
wait_secs = float(wait_time.seconds) + (
float(wait_time.microseconds) / 10**6)
ciel.log(
"Producer for %s: waiting for a direct consumer" %
self.refid, "BLOCKPIPE", logging.DEBUG)
self.cond.wait(wait_secs)
if self.direct_write_filename is not None:
ciel.log(
"Producer for %s: writing direct to filename %s" %
(self.refid, self.direct_write_filename), "BLOCKPIPE",
logging.DEBUG)
self.started = True
return (self.direct_write_filename, False)
elif self.direct_write_fd is not None:
ciel.log(
"Producer for %s: writing direct to consumer-supplied FD"
% self.refid, "BLOCKPIPE", logging.DEBUG)
self.started = True
return (self.direct_write_fd, True)
elif self.started:
ciel.log(
"Producer for %s: kicked by a regular-file subscription; using conventional stream-file"
% self.refid, "BLOCKPIPE", logging.DEBUG)
else:
self.started = True
ciel.log(
"Producer for %s: timed out waiting for a consumer; writing to local block store"
% self.refid, "BLOCKPIPE", logging.DEBUG)
return (producer_filename(self.refid), False)