def tcp_fetch(self):
if (not self.may_pipe) or (not self.sole_consumer):
raise PlanFailedError(
"TCP-Fetch currently only capable of delivering a pipe")
self.producer = TcpTransferContext(self.ref, self.chunk_size, self)
self.producer.start()
def http_fetch(self):
self.producer = HttpTransferContext(self.ref, self)
self.producer.start()
class FetchInProgress:
def __init__(self, ref, result_callback, reset_callback,
start_filename_callback, start_fd_callback, string_callback,
progress_callback, chunk_size, may_pipe, sole_consumer,
must_block, task_record):
self.lock = threading.RLock()
self.result_callback = result_callback
self.reset_callback = reset_callback
self.start_filename_callback = start_filename_callback
self.start_fd_callback = start_fd_callback
self.string_callback = string_callback
self.progress_callback = progress_callback
self.chunk_size = chunk_size
self.may_pipe = may_pipe
self.sole_consumer = sole_consumer
self.must_block = must_block
self.task_record = task_record
self.pusher_thread = None
self.ref = ref
self.producer = None
self.cat_process = None
self.started = False
self.done = False
self.cancelled = False
self.success = None
self.form_plan()
def form_plan(self):
self.current_plan = 0
self.plans = []
if isinstance(self.ref, SWDataValue):
self.plans.append(self.resolve_dataval)
elif isinstance(self.ref, SW2_FetchReference):
self.plans.append(self.http_fetch)
else:
self.plans.append(self.use_local_file)
self.plans.append(self.attach_local_producer)
if isinstance(self.ref, SW2_ConcreteReference):
self.plans.append(self.http_fetch)
elif isinstance(self.ref, SW2_StreamReference):
if isinstance(self.ref, SW2_SocketStreamReference):
self.plans.append(self.tcp_fetch)
self.plans.append(self.http_fetch)
def start_fetch(self):
self.run_plans()
def run_plans(self):
while self.current_plan < len(self.plans):
try:
self.plans[self.current_plan]()
return
except PlanFailedError:
self.current_plan += 1
def run_next_plan(self):
self.current_plan += 1
self.run_plans()
def resolve_dataval(self):
if self.string_callback is not None:
decoded_dataval = decode_datavalue(self.ref)
self.string_callback(decoded_dataval)
else:
create_datavalue_file(self.ref)
self.set_filename(filename_for_ref(self.ref), True)
self.result(True, None)
def use_local_file(self):
filename = filename_for_ref(self.ref)
if os.path.exists(filename):
self.set_filename(filename, True)
self.result(True, None)
else:
raise PlanFailedError(
"Plan use-local-file failed for %s: no such file %s" %
(self.ref, filename), "BLOCKSTORE", logging.DEBUG)
def attach_local_producer(self):
producer = get_producer_for_id(self.ref.id)
if producer is None:
raise PlanFailedError(
"Plan attach-local-producer failed for %s: not being produced here"
% self.ref, "BLOCKSTORE", logging.DEBUG)
else:
is_pipe = producer.subscribe(self,
try_direct=(self.may_pipe
and self.sole_consumer))
if is_pipe:
ciel.log("Fetch-ref %s: attached to direct pipe!" % self.ref,
"BLOCKSTORE", logging.DEBUG)
filename = producer.get_fifo_filename()
else:
ciel.log(
"Fetch-ref %s: following local producer's file" % self.ref,
"BLOCKSTORE", logging.DEBUG)
filename = producer_filename(self.ref.id)
self.set_filename(filename, is_pipe)
def http_fetch(self):
self.producer = HttpTransferContext(self.ref, self)
self.producer.start()
def tcp_fetch(self):
if (not self.may_pipe) or (not self.sole_consumer):
raise PlanFailedError(
"TCP-Fetch currently only capable of delivering a pipe")
self.producer = TcpTransferContext(self.ref, self.chunk_size, self)
self.producer.start()
### Start callbacks from above
def result(self, success, result_ref=None):
with self.lock:
if not success:
if not self.started:
self.run_next_plan()
return
self.producer = None
self.done = True
self.success = success
if self.pusher_thread is not None:
self.pusher_thread.result(success)
self.result_callback(success, result_ref)
def reset(self):
if self.pusher_thread is not None:
self.pusher_thread.reset()
self.reset_callback()
def progress(self, bytes):
if self.pusher_thread is not None:
self.pusher_thread.progress(bytes)
if self.progress_callback is not None:
self.progress_callback(bytes)
def create_fifo(self):
fifo_name = tempfile.mktemp(prefix="ciel-socket-fifo")
os.mkfifo(fifo_name)
return fifo_name
def set_fd(self, fd, is_pipe):
# TODO: handle FDs that might point to regular files.
assert is_pipe
self.started = True
if self.start_fd_callback is not None:
self.start_fd_callback(fd, is_pipe)
else:
fifo_name = self.create_fifo()
self.cat_process = subprocess.Popen(["cat > %s" % fifo_name],
shell=True,
stdin=fd,
close_fds=True)
os.close(fd)
self.start_filename_callback(fifo_name, True)
def set_filename(self, filename, is_pipe):
self.started = True
if (not is_pipe) and self.must_block:
fifo_name = self.create_fifo()
self.pusher_thread = AsyncPushThread(self.ref, filename, fifo_name,
self)
self.pusher_thread.start()
else:
self.start_filename_callback(filename, is_pipe)
def cancel(self):
with self.lock:
self.cancelled = True
producer = self.producer
if producer is not None:
producer.unsubscribe(self)
if self.cat_process is not None:
try:
self.cat_process.kill()
except Exception as e:
ciel.log(
"Fetcher for %s failed to kill 'cat': %s" %
(self.ref.id, repr(e)), "FETCHER", logging.ERROR)
def tcp_fetch(self):
if (not self.may_pipe) or (not self.sole_consumer):
raise PlanFailedError("TCP-Fetch currently only capable of delivering a pipe")
self.producer = TcpTransferContext(self.ref, self.chunk_size, self)
self.producer.start()
def http_fetch(self):
self.producer = HttpTransferContext(self.ref, self)
self.producer.start()
class FetchInProgress:
def __init__(
self,
ref,
result_callback,
reset_callback,
start_filename_callback,
start_fd_callback,
string_callback,
progress_callback,
chunk_size,
may_pipe,
sole_consumer,
must_block,
task_record,
):
self.lock = threading.RLock()
self.result_callback = result_callback
self.reset_callback = reset_callback
self.start_filename_callback = start_filename_callback
self.start_fd_callback = start_fd_callback
self.string_callback = string_callback
self.progress_callback = progress_callback
self.chunk_size = chunk_size
self.may_pipe = may_pipe
self.sole_consumer = sole_consumer
self.must_block = must_block
self.task_record = task_record
self.pusher_thread = None
self.ref = ref
self.producer = None
self.cat_process = None
self.started = False
self.done = False
self.cancelled = False
self.success = None
self.form_plan()
def form_plan(self):
self.current_plan = 0
self.plans = []
if isinstance(self.ref, SWDataValue):
self.plans.append(self.resolve_dataval)
elif isinstance(self.ref, SW2_FetchReference):
self.plans.append(self.http_fetch)
else:
self.plans.append(self.use_local_file)
self.plans.append(self.attach_local_producer)
if isinstance(self.ref, SW2_ConcreteReference):
self.plans.append(self.http_fetch)
elif isinstance(self.ref, SW2_StreamReference):
if isinstance(self.ref, SW2_SocketStreamReference):
self.plans.append(self.tcp_fetch)
self.plans.append(self.http_fetch)
def start_fetch(self):
self.run_plans()
def run_plans(self):
while self.current_plan < len(self.plans):
try:
self.plans[self.current_plan]()
return
except PlanFailedError:
self.current_plan += 1
def run_next_plan(self):
self.current_plan += 1
self.run_plans()
def resolve_dataval(self):
if self.string_callback is not None:
decoded_dataval = decode_datavalue(self.ref)
self.string_callback(decoded_dataval)
else:
create_datavalue_file(self.ref)
self.set_filename(filename_for_ref(self.ref), True)
self.result(True, None)
def use_local_file(self):
filename = filename_for_ref(self.ref)
if os.path.exists(filename):
self.set_filename(filename, True)
self.result(True, None)
else:
raise PlanFailedError(
"Plan use-local-file failed for %s: no such file %s" % (self.ref, filename), "BLOCKSTORE", logging.DEBUG
)
def attach_local_producer(self):
producer = get_producer_for_id(self.ref.id)
if producer is None:
raise PlanFailedError(
"Plan attach-local-producer failed for %s: not being produced here" % self.ref,
"BLOCKSTORE",
logging.DEBUG,
)
else:
is_pipe = producer.subscribe(self, try_direct=(self.may_pipe and self.sole_consumer))
if is_pipe:
ciel.log("Fetch-ref %s: attached to direct pipe!" % self.ref, "BLOCKSTORE", logging.DEBUG)
filename = producer.get_fifo_filename()
else:
ciel.log("Fetch-ref %s: following local producer's file" % self.ref, "BLOCKSTORE", logging.DEBUG)
filename = producer_filename(self.ref.id)
self.set_filename(filename, is_pipe)
def http_fetch(self):
self.producer = HttpTransferContext(self.ref, self)
self.producer.start()
def tcp_fetch(self):
if (not self.may_pipe) or (not self.sole_consumer):
raise PlanFailedError("TCP-Fetch currently only capable of delivering a pipe")
self.producer = TcpTransferContext(self.ref, self.chunk_size, self)
self.producer.start()
### Start callbacks from above
def result(self, success, result_ref=None):
with self.lock:
if not success:
if not self.started:
self.run_next_plan()
return
self.producer = None
self.done = True
self.success = success
if self.pusher_thread is not None:
self.pusher_thread.result(success)
self.result_callback(success, result_ref)
def reset(self):
if self.pusher_thread is not None:
self.pusher_thread.reset()
self.reset_callback()
def progress(self, bytes):
if self.pusher_thread is not None:
self.pusher_thread.progress(bytes)
if self.progress_callback is not None:
self.progress_callback(bytes)
def create_fifo(self):
fifo_name = tempfile.mktemp(prefix="ciel-socket-fifo")
os.mkfifo(fifo_name)
return fifo_name
def set_fd(self, fd, is_pipe):
# TODO: handle FDs that might point to regular files.
assert is_pipe
self.started = True
if self.start_fd_callback is not None:
self.start_fd_callback(fd, is_pipe)
else:
fifo_name = self.create_fifo()
self.cat_process = subprocess.Popen(["cat > %s" % fifo_name], shell=True, stdin=fd, close_fds=True)
os.close(fd)
self.start_filename_callback(fifo_name, True)
def set_filename(self, filename, is_pipe):
self.started = True
if (not is_pipe) and self.must_block:
fifo_name = self.create_fifo()
self.pusher_thread = AsyncPushThread(self.ref, filename, fifo_name, self)
self.pusher_thread.start()
else:
self.start_filename_callback(filename, is_pipe)
def cancel(self):
with self.lock:
self.cancelled = True
producer = self.producer
if producer is not None:
producer.unsubscribe(self)
if self.cat_process is not None:
try:
self.cat_process.kill()
except Exception as e:
ciel.log("Fetcher for %s failed to kill 'cat': %s" % (self.ref.id, repr(e)), "FETCHER", logging.ERROR)