Esempio n. 1
0
 def http_fetch(self):
     self.producer = HttpTransferContext(self.ref, self)
     self.producer.start()
Esempio n. 2
0
 def tcp_fetch(self):
     if (not self.may_pipe) or (not self.sole_consumer):
         raise PlanFailedError("TCP-Fetch currently only capable of delivering a pipe")
     self.producer = TcpTransferContext(self.ref, self.chunk_size, self)
     self.producer.start()
Esempio n. 3
0
class FetchInProgress:

    def __init__(self, ref, result_callback, reset_callback, start_filename_callback, start_fd_callback, string_callback, progress_callback, chunk_size, may_pipe, sole_consumer, must_block, task_record):
        self.lock = threading.RLock()
        self.result_callback = result_callback
        self.reset_callback = reset_callback
        self.start_filename_callback = start_filename_callback
        self.start_fd_callback = start_fd_callback
        self.string_callback = string_callback
        self.progress_callback = progress_callback
        self.chunk_size = chunk_size
        self.may_pipe = may_pipe
        self.sole_consumer = sole_consumer
        self.must_block = must_block
        self.task_record = task_record
        self.pusher_thread = None
        self.ref = ref
        self.producer = None
        self.cat_process = None
        self.started = False
        self.done = False
        self.cancelled = False
        self.success = None
        self.form_plan()
        
    def form_plan(self):
        self.current_plan = 0
        self.plans = []
        if isinstance(self.ref, SWDataValue):
            self.plans.append(self.resolve_dataval)
        elif isinstance(self.ref, SW2_FetchReference):
            self.plans.append(self.http_fetch)
        else:
            self.plans.append(self.use_local_file)
            self.plans.append(self.attach_local_producer)
            if isinstance(self.ref, SW2_ConcreteReference):
                self.plans.append(self.http_fetch)
            elif isinstance(self.ref, SW2_StreamReference):
                if isinstance(self.ref, SW2_SocketStreamReference):
                    self.plans.append(self.tcp_fetch)
                self.plans.append(self.http_fetch)

    def start_fetch(self):
        self.run_plans()

    def run_plans(self):
        while self.current_plan < len(self.plans):
            try:
                self.plans[self.current_plan]()
                return
            except PlanFailedError:
                self.current_plan += 1

    def run_next_plan(self):
        self.current_plan += 1
        self.run_plans()

    def resolve_dataval(self):
        if self.string_callback is not None:
            decoded_dataval = decode_datavalue(self.ref)
            self.string_callback(decoded_dataval)
        else:
            create_datavalue_file(self.ref)
            self.set_filename(filename_for_ref(self.ref), True)
            self.result(True, None)

    def use_local_file(self):
        filename = filename_for_ref(self.ref)
        if os.path.exists(filename):
            self.set_filename(filename, True)
            self.result(True, None)
        else:
            raise PlanFailedError("Plan use-local-file failed for %s: no such file %s" % (self.ref, filename), "BLOCKSTORE", logging.INFO)

    def attach_local_producer(self):
        producer = get_producer_for_id(self.ref.id)
        if producer is None:
            raise PlanFailedError("Plan attach-local-producer failed for %s: not being produced here" % self.ref, "BLOCKSTORE", logging.INFO)
        else:
            is_pipe = producer.subscribe(self, try_direct=(self.may_pipe and self.sole_consumer))
            if is_pipe:
                ciel.log("Fetch-ref %s: attached to direct pipe!" % self.ref, "BLOCKSTORE", logging.INFO)
                filename = producer.get_fifo_filename()
            else:
                ciel.log("Fetch-ref %s: following local producer's file" % self.ref, "BLOCKSTORE", logging.INFO)
                filename = producer_filename(self.ref.id)
            self.set_filename(filename, is_pipe)

    def http_fetch(self):
        self.producer = HttpTransferContext(self.ref, self)
        self.producer.start()

    def tcp_fetch(self):
        if (not self.may_pipe) or (not self.sole_consumer):
            raise PlanFailedError("TCP-Fetch currently only capable of delivering a pipe")
        self.producer = TcpTransferContext(self.ref, self.chunk_size, self)
        self.producer.start()
                
    ### Start callbacks from above
    def result(self, success, result_ref=None):
        with self.lock:
            if not success:
                if not self.started:
                    self.run_next_plan()
                    return
            self.producer = None
            self.done = True
            self.success = success
        if self.pusher_thread is not None:
            self.pusher_thread.result(success)
        self.result_callback(success, result_ref)

    def reset(self):
        if self.pusher_thread is not None:
            self.pusher_thread.reset()
        self.reset_callback()

    def progress(self, bytes):
        if self.pusher_thread is not None:
            self.pusher_thread.progress(bytes)
        if self.progress_callback is not None:
            self.progress_callback(bytes)
            
    def create_fifo(self):
        fifo_name = tempfile.mktemp(prefix="ciel-socket-fifo")
        os.mkfifo(fifo_name)
        return fifo_name

    def set_fd(self, fd, is_pipe):
        # TODO: handle FDs that might point to regular files.
        assert is_pipe
        self.started = True
        if self.start_fd_callback is not None:
            self.start_fd_callback(fd, is_pipe)
        else:
            fifo_name = self.create_fifo()
            self.cat_process = subprocess.Popen(["cat > %s" % fifo_name], shell=True, stdin=fd, close_fds=True)
            os.close(fd)
            self.start_filename_callback(fifo_name, True)

    def set_filename(self, filename, is_pipe):
        self.started = True
        if (not is_pipe) and self.must_block:
            fifo_name = self.create_fifo()
            self.pusher_thread = AsyncPushThread(self.ref, filename, fifo_name, self)
            self.pusher_thread.start()
        else:
            self.start_filename_callback(filename, is_pipe)

    def cancel(self):
        with self.lock:
            self.cancelled = True
            producer = self.producer
        if producer is not None:
            producer.unsubscribe(self)
        if self.cat_process is not None:
            try:
                self.cat_process.kill()
            except Exception as e:
                ciel.log("Fetcher for %s failed to kill 'cat': %s" % (self.ref.id, repr(e)), "FETCHER", logging.ERROR)