def receive(self, msg): if ('serve', ANY, ANY) == msg: _, file_path, file_id = msg if not os.path.exists(file_path) and not os.path.isdir(file_path): err("attempt to publish a file that does not exist") elif file_id in self.published: err("Attempt to publish %r with ID %r but a file already exists with that ID" % (file_path, file_id)) else: self.published[file_id] = (file_path, datetime.datetime.now()) elif msg == ('request', ANY) or msg == ('request-local', ANY): request, file_id = msg if file_id not in self.published: err("attempt to get a file with ID %r which has not been published or is not available anymore" % (file_id,)) else: file_path, time_added = self.published[file_id] if request == 'request-local': self._touch_file(file_id) self.reply(('local-file', file_path)) else: response = self.spawn(Response.using(file=file_path, request=self.sender, threadpool=self.threadpool)) self.watch(response) self.responses[response] = file_id elif 'purge-old' == msg: self.send_later(60, 'purge-old') t = datetime.datetime.now() for file_id, (file_path, time_added) in self.published.items(): if (t - time_added).total_seconds() > constants.FILE_MAX_LIFETIME and file_id not in self.responses.values(): dbg("purging file %r at %r" % (file_id, file_path)) del self.published[file_id] elif ('terminated', IN(self.responses)) == msg: _, sender = msg self._touch_file(file_id=self.responses.pop(sender))
def transmit(self): """Puts all currently pending sent messages to the insock buffer of the recipient of the message. This is more useful than immediate "delivery" because it allows full flexibility of the order in which tests set up nodes and mock actors on those nodes, and of the order in which messages are sent out from a node. """ if not self.queue: return deliverable = [] for src, dst, msg in self.queue: _assert_valid_addr(src) _assert_valid_addr(dst) # assert (src, dst) in self.connections, "Hubs should only send messages to addresses they have previously connected to" if random.random() <= self._packet_loss.get((src, dst), 0.0): dbg("packet lost: %r %s → %s" % (msg, src, dst)) continue if dst not in self.listeners: pass # dbg(u"%r ⇝ ↴" % (_dumpmsg(msg),)) else: # dbg(u"%r → %s" % (_dumpmsg(msg), dst)) sock = self.listeners[dst] deliverable.append((msg, src, sock)) del self.queue[:] for msg, src, sock in deliverable: sock.gotMultipart(msg)
def connect(self, addr, endpoints): _assert_valid_addr(addr) for endpoint in endpoints: assert endpoint.type == 'connect', "Hubs should only connect MockOutSockets and not bind" _assert_valid_addr(endpoint.address) assert (addr, endpoint.address) not in self.connections dbg(u"%s → %s" % (addr, endpoint.address)) self.connections.add((addr, endpoint.address))
def connect(self, addr, endpoints): _assert_valid_addr(addr) for endpoint in endpoints: assert endpoint.type == "connect", "Hubs should only connect MockOutSockets and not bind" _assert_valid_addr(endpoint.address) assert (addr, endpoint.address) not in self.connections dbg(u"%s → %s" % (addr, endpoint.address)) self.connections.add((addr, endpoint.address))
def run(self): dbg("spawning ExampleActor") actor1 = self.spawn(ExampleActor) dbg("spawning ExampleProcess") self.spawn(ExampleProcess.using(other_actor=actor1)) self.get() # so that the entire app wouldn't exit immediately
def run(self): dbg("spawning ExampleActor") actor1 = self.spawn(ExampleActor) dbg("spawning ExampleProcess") self.spawn(ExampleProcess.using(other_actor=actor1)) yield self.get() # so that the entire app wouldn't exit immediately
def fetch(self, path, context=None): self._fetching[path] = Deferred() mkdir_p(os.path.dirname(path)) with self.open(context=context) as f: yield f.read_into(path) os.utime(path, (self.mtime, self.mtime)) dbg("fetched into", path, "with size", os.path.getsize(path)) self._fetching[path].callback(None) del self._fetching[path]
def receive(self, msg): if ('publish', ANY, ANY) == msg: _, file_path, pub_id = msg if not os.path.exists(file_path) and not os.path.isdir(file_path): err("attempt to publish a file that does not exist") return if pub_id in self.published: raise FileAlreadyPublished("Attempt to publish %r with ID %r but a file already exists with that ID" % (file_path, pub_id)) else: self.published[pub_id] = (file_path, datetime.datetime.now()) elif ('get-file', ANY, ANY) == msg: _, pub_id, send_to = msg if pub_id not in self.published: err("attempt to get a file with ID %r which has not been published or is not available anymore" % (pub_id,)) return self._touch_file(pub_id) file_path, time_added = self.published[pub_id] sender = self.watch(_Sender.using(service=self.ref, pub_id=pub_id, file=file_path, send_to=send_to)) self.senders[sender] = pub_id # self.senders[sender] = pub_id send_to << ('take-file', sender) elif ('touch-file', ANY) == msg: _, pub_id = msg if pub_id not in self.published: err("attempt to touch a file with ID %r which has not been published or is not available anymore" % (pub_id,)) return _, pub_id = msg self._touch_file(pub_id) elif 'purge-old' == msg: after(60.0).do(self.send, 'purge-old') t = datetime.datetime.now() for pub_id, (file_path, time_added) in self.published.items(): if (t - time_added).total_seconds() > FILE_MAX_LIFETIME and pub_id not in self.senders.values(): dbg("purging file %r at %r" % (pub_id, file_path)) del self.published[pub_id] elif ('terminated', IN(self.senders)) == msg: _, sender = msg del self.senders[sender]
def run(self, filename): md5 = get_checksum(filename) dbg("file checksum: ", md5) receivers = set() for _ in range(2): receiver = self.spawn(Receiver.using(self.ref)) self.spawn(Sender.using(filename=filename, receiver=receiver, checksum=md5)) self.watch(receiver) receivers.add(receiver) while receivers: _, receiver = yield self.get(('terminated', IN(receivers))) receivers.remove(receiver) dbg("done:", receiver)
def run(self, filename, receiver, checksum=None): if checksum is None: checksum = get_checksum(filename) dbg("checksum:", checksum) if isinstance(receiver, basestring): receiver = lookup(receiver) file = FileRef.publish(filename) receiver << checksum << file import os bytes = os.stat(filename).st_size t1 = time.time() self.watch(receiver) yield self.get(('terminated', receiver)) t2 = time.time() dt = t2 - t1 dbg("%s bytes transferred in %ss -- speed: %r MB/s" % (bytes, dt, round((bytes / dt) / 1024 / 1024, 2)))
def run(self, other_actor): other_actor = lookup(other_actor) if isinstance(other_actor, str) else other_actor while True: dbg("sending greeting to %r" % (other_actor,)) other_actor << ('hello!', self.ref) dbg("waiting for ack from %r" % (other_actor,)) yield with_timeout(5.0, self.get('ack')) dbg("got 'ack' from %r; now sleeping a bit..." % (other_actor,)) yield sleep(1.0)
def run(self): child = self.spawn(ExampleActor) while True: dbg("sending greeting to %r" % (child,)) child << ('hello!', self.ref) dbg("waiting for ack from %r" % (child,)) yield with_timeout(5.0, self.get('ack')) dbg("got 'ack' from %r; now sleeping a bit..." % (child,)) yield sleep(1.0)
def run(self): child = self.spawn(ExampleActor) while True: dbg("sending greeting to %r" % (child, )) child << 'hello!' dbg("waiting for ack from %r" % (child, )) with_timeout(5.0, self.get, 'ack') dbg("got 'ack' from %r; now sleeping a bit..." % (child, )) sleep(1.0)
def run(self, other_actor): if isinstance(other_actor, str): other_actor = self.node.lookup_str(other_actor) while True: dbg("sending greeting to %r" % (other_actor,)) other_actor << 'hello!' dbg("waiting for ack from %r" % (other_actor,)) self.get('ack', timeout=5.0) dbg("got 'ack' from %r; now sleeping a bit..." % (other_actor,)) sleep(1.0)
def run(self): checksum_orig = yield self.get() fileref = yield self.get() fh = fileref.get_handle() dbg("RECEIVED", fileref, "with checksum", checksum_orig) md5 = hashlib.md5() while True: chunk = yield fh.read(CHUNK_SIZE) dbg("READ CHUNK %r AT %r" % (hashlib.md5(chunk[:100]).hexdigest()[:8], str(time.time())[8:])) md5.update(chunk) if len(chunk) < CHUNK_SIZE: break checksum = md5.hexdigest() dbg("CHECKSUM", checksum) assert checksum_orig == checksum, (checksum, checksum_orig)
def receive(self, msg): dbg("%r from %r" % (msg, self.sender)) self.sender << 'ack'
def receive(self, msg): t = self.reactor.seconds() self.num_msgs_received += 1 dt = t - self.msg_receive_started if dt >= 1.0: dbg("%d msg/s" % (float(self.num_msgs_received) / dt)) self.msg_receive_started = t self.num_msgs_received = 0 self.num_msgs_received += 1 # dbg(t, msg) def logappend(*args): pass # self.log.append((t,) + args) # if len(self.log) > 50000: # dbg("trimming log") # self.log = self.log[10000:] # Monitoring if ('connect', ANY) == msg: _, sender = msg dbg("connect from", sender) sender << 'ack' elif ('up', ANY) == msg: _, sender = msg self.watch(sender) logappend('up', str(sender.uri)) self.up[sender] = ('up', t) elif ('terminated', IN(self.up)) == msg: _, sender = msg logappend('down', str(sender.uri)) self.up[sender] = ('down', t) elif ('state', ANY, ANY) == msg: _, sender, state = msg if sender in self.up: self.up[sender] = (self.up[sender][0], t) else: logappend('up', str(sender.uri)) self.up[sender] = ('up', t) logappend('state', str(sender.uri), state) self.state[sender] = state elif ('log', ANY, ANY) == msg: _, sender, logmsg = msg logappend('log', str(sender.uri), logmsg) # elif ('get-up', ANY, ANY) == msg: _, d, filter = msg msg[1].callback([(str(k.uri), v) for k, v in self.up.items() if k == filter]) elif ('get-state', ANY, ANY) == msg: _, d, filter = msg msg[1].callback([(str(k.uri), v) for k, v in self.state.items() if k == filter]) elif ('get-log', ANY, ANY) == msg: _, d, filter = msg msg[1].callback([x for x in self.log if x == filter]) elif ('get-all', ANY, ANY) == msg: _, d, filter = msg data = {} for client in self.up: uri = str(client.uri) if uri == filter: status, seen = self.up[client] state = self.state[client] if client in self.state else None data[uri] = (status, seen, state) msg[1].callback([(uri, status, seen, state) for uri, (status, seen, state) in data.items()]) else: raise Unhandled
def post_stop(self): dbg("stopping")
def pre_start(self): dbg("starting")
def receive(self, msg): content, sender = msg dbg("%r from %r" % (content, sender)) sender << 'ack'
def receive(self, fref): try: fref.fetch() except TransferInterrupted as e: dbg(e) failed.set()
def shutdown(self): dbg() self.owner.outsock_shutdown(self.addr, self.endpoint_addr)
def run(self): while True: dbg("sleeping...", gevent.getcurrent()) gevent.sleep(1.0)
def receive(self, msg): if ('serve', ANY, ANY) == msg: _, file_path, file_id = msg if not os.path.exists(file_path) and not os.path.isdir(file_path): err("attempt to publish a file that does not exist") elif file_id in self.published: err("Attempt to publish %r with ID %r but a file already exists with that ID" % (file_path, file_id)) else: self.published[file_id] = (file_path, datetime.datetime.now()) elif msg == ('request', ANY) or msg == ('request-local', ANY): request, file_id = msg if file_id not in self.published: # TODO: replace with a reply to the requestor, just like in the upload() case err("attempt to get a file with ID %r which has not been published or is not available anymore" % (file_id,)) else: file_path, time_added = self.published[file_id] if request == 'request-local': self._touch_file(file_id) self.reply(('local-file', file_path)) else: response = self.spawn(Response.using(file=file_path, request=self.sender, threadpool=self.threadpool)) self.watch(response) self.responses[response] = file_id elif 'purge-old' == msg: self.send_later(60, 'purge-old') t = datetime.datetime.now() for file_id, (file_path, time_added) in self.published.items(): if (t - time_added).total_seconds() > constants.FILE_MAX_LIFETIME and file_id not in self.responses.values(): dbg("purging file %r at %r" % (file_id, file_path)) del self.published[file_id] elif ('terminated', IN(self.responses)) == msg: _, sender = msg self._touch_file(file_id=self.responses.pop(sender)) elif ('upload', ANY, ANY, ANY, ANY) == msg: _, file_id, url, method, expect_response = msg if method not in ('POST', 'PUT'): self.reply((False, ('bad-method', None))) if file_id not in self.published: self.reply((False, ('file-not-found', None))) else: self._touch_file(file_id) file_path, _ = self.published[file_id] upload = requests.post if method == 'POST' else requests.put try: r = upload(url, files={'file': open(file_path, 'rb')}) except Exception as e: self.reply((False, ('exception', (type(e).__name__, e.message, traceback.format_exc())))) else: if r.status_code != expect_response: self.reply((False, ('bad-status-code', r.status_code))) else: self.reply((True, (r.status_code, dict(r.headers), r.text))) elif ('delete', ANY) == msg: _, file_id = msg if file_id not in self.published: self.reply((False, ('file-not-found', None))) else: file_path, _ = self.published[file_id] del self.published[file_id] try: os.unlink(file_path) except BaseException as e: self.reply((False, ('exception', (type(e).__name__, e.message, traceback.format_exc())))) else: self.reply((True, None))