def receive(self, msg):
if ('serve', ANY, ANY) == msg:
_, file_path, file_id = msg
if not os.path.exists(file_path) and not os.path.isdir(file_path):
err("attempt to publish a file that does not exist")
elif file_id in self.published:
err("Attempt to publish %r with ID %r but a file already exists with that ID" % (file_path, file_id))
else:
self.published[file_id] = (file_path, datetime.datetime.now())
elif msg == ('request', ANY) or msg == ('request-local', ANY):
request, file_id = msg
if file_id not in self.published:
err("attempt to get a file with ID %r which has not been published or is not available anymore" % (file_id,))
else:
file_path, time_added = self.published[file_id]
if request == 'request-local':
self._touch_file(file_id)
self.reply(('local-file', file_path))
else:
response = self.spawn(Response.using(file=file_path, request=self.sender, threadpool=self.threadpool))
self.watch(response)
self.responses[response] = file_id
elif 'purge-old' == msg:
self.send_later(60, 'purge-old')
t = datetime.datetime.now()
for file_id, (file_path, time_added) in self.published.items():
if (t - time_added).total_seconds() > constants.FILE_MAX_LIFETIME and file_id not in self.responses.values():
dbg("purging file %r at %r" % (file_id, file_path))
del self.published[file_id]
elif ('terminated', IN(self.responses)) == msg:
_, sender = msg
self._touch_file(file_id=self.responses.pop(sender))
def transmit(self):
"""Puts all currently pending sent messages to the insock buffer of the recipient of the message.
This is more useful than immediate "delivery" because it allows full flexibility of the order in which tests
set up nodes and mock actors on those nodes, and of the order in which messages are sent out from a node.
"""
if not self.queue:
return
deliverable = []
for src, dst, msg in self.queue:
_assert_valid_addr(src)
_assert_valid_addr(dst)
# assert (src, dst) in self.connections, "Hubs should only send messages to addresses they have previously connected to"
if random.random() <= self._packet_loss.get((src, dst), 0.0):
dbg("packet lost: %r %s → %s" % (msg, src, dst))
continue
if dst not in self.listeners:
pass # dbg(u"%r ⇝ ↴" % (_dumpmsg(msg),))
else:
# dbg(u"%r → %s" % (_dumpmsg(msg), dst))
sock = self.listeners[dst]
deliverable.append((msg, src, sock))
del self.queue[:]
for msg, src, sock in deliverable:
sock.gotMultipart(msg)
def transmit(self):
"""Puts all currently pending sent messages to the insock buffer of the recipient of the message.
This is more useful than immediate "delivery" because it allows full flexibility of the order in which tests
set up nodes and mock actors on those nodes, and of the order in which messages are sent out from a node.
"""
if not self.queue:
return
deliverable = []
for src, dst, msg in self.queue:
_assert_valid_addr(src)
_assert_valid_addr(dst)
# assert (src, dst) in self.connections, "Hubs should only send messages to addresses they have previously connected to"
if random.random() <= self._packet_loss.get((src, dst), 0.0):
dbg("packet lost: %r %s → %s" % (msg, src, dst))
continue
if dst not in self.listeners:
pass # dbg(u"%r ⇝ ↴" % (_dumpmsg(msg),))
else:
# dbg(u"%r → %s" % (_dumpmsg(msg), dst))
sock = self.listeners[dst]
deliverable.append((msg, src, sock))
del self.queue[:]
for msg, src, sock in deliverable:
sock.gotMultipart(msg)
def connect(self, addr, endpoints):
_assert_valid_addr(addr)
for endpoint in endpoints:
assert endpoint.type == 'connect', "Hubs should only connect MockOutSockets and not bind"
_assert_valid_addr(endpoint.address)
assert (addr, endpoint.address) not in self.connections
dbg(u"%s → %s" % (addr, endpoint.address))
self.connections.add((addr, endpoint.address))
def connect(self, addr, endpoints):
_assert_valid_addr(addr)
for endpoint in endpoints:
assert endpoint.type == "connect", "Hubs should only connect MockOutSockets and not bind"
_assert_valid_addr(endpoint.address)
assert (addr, endpoint.address) not in self.connections
dbg(u"%s → %s" % (addr, endpoint.address))
self.connections.add((addr, endpoint.address))
def run(self):
dbg("spawning ExampleActor")
actor1 = self.spawn(ExampleActor)
dbg("spawning ExampleProcess")
self.spawn(ExampleProcess.using(other_actor=actor1))
self.get() # so that the entire app wouldn't exit immediately
def run(self):
dbg("spawning ExampleActor")
actor1 = self.spawn(ExampleActor)
dbg("spawning ExampleProcess")
self.spawn(ExampleProcess.using(other_actor=actor1))
yield self.get() # so that the entire app wouldn't exit immediately
def fetch(self, path, context=None):
self._fetching[path] = Deferred()
mkdir_p(os.path.dirname(path))
with self.open(context=context) as f:
yield f.read_into(path)
os.utime(path, (self.mtime, self.mtime))
dbg("fetched into", path, "with size", os.path.getsize(path))
self._fetching[path].callback(None)
del self._fetching[path]
def receive(self, msg):
if ('publish', ANY, ANY) == msg:
_, file_path, pub_id = msg
if not os.path.exists(file_path) and not os.path.isdir(file_path):
err("attempt to publish a file that does not exist")
return
if pub_id in self.published:
raise FileAlreadyPublished("Attempt to publish %r with ID %r but a file already exists with that ID" % (file_path, pub_id))
else:
self.published[pub_id] = (file_path, datetime.datetime.now())
elif ('get-file', ANY, ANY) == msg:
_, pub_id, send_to = msg
if pub_id not in self.published:
err("attempt to get a file with ID %r which has not been published or is not available anymore" % (pub_id,))
return
self._touch_file(pub_id)
file_path, time_added = self.published[pub_id]
sender = self.watch(_Sender.using(service=self.ref, pub_id=pub_id, file=file_path, send_to=send_to))
self.senders[sender] = pub_id
# self.senders[sender] = pub_id
send_to << ('take-file', sender)
elif ('touch-file', ANY) == msg:
_, pub_id = msg
if pub_id not in self.published:
err("attempt to touch a file with ID %r which has not been published or is not available anymore" % (pub_id,))
return
_, pub_id = msg
self._touch_file(pub_id)
elif 'purge-old' == msg:
after(60.0).do(self.send, 'purge-old')
t = datetime.datetime.now()
for pub_id, (file_path, time_added) in self.published.items():
if (t - time_added).total_seconds() > FILE_MAX_LIFETIME and pub_id not in self.senders.values():
dbg("purging file %r at %r" % (pub_id, file_path))
del self.published[pub_id]
elif ('terminated', IN(self.senders)) == msg:
_, sender = msg
del self.senders[sender]
def run(self, filename):
md5 = get_checksum(filename)
dbg("file checksum: ", md5)
receivers = set()
for _ in range(2):
receiver = self.spawn(Receiver.using(self.ref))
self.spawn(Sender.using(filename=filename, receiver=receiver, checksum=md5))
self.watch(receiver)
receivers.add(receiver)
while receivers:
_, receiver = yield self.get(('terminated', IN(receivers)))
receivers.remove(receiver)
dbg("done:", receiver)
def run(self, filename):
md5 = get_checksum(filename)
dbg("file checksum: ", md5)
receivers = set()
for _ in range(2):
receiver = self.spawn(Receiver.using(self.ref))
self.spawn(Sender.using(filename=filename, receiver=receiver, checksum=md5))
self.watch(receiver)
receivers.add(receiver)
while receivers:
_, receiver = yield self.get(('terminated', IN(receivers)))
receivers.remove(receiver)
dbg("done:", receiver)
def run(self, filename, receiver, checksum=None):
if checksum is None:
checksum = get_checksum(filename)
dbg("checksum:", checksum)
if isinstance(receiver, basestring):
receiver = lookup(receiver)
file = FileRef.publish(filename)
receiver << checksum << file
import os
bytes = os.stat(filename).st_size
t1 = time.time()
self.watch(receiver)
yield self.get(('terminated', receiver))
t2 = time.time()
dt = t2 - t1
dbg("%s bytes transferred in %ss -- speed: %r MB/s" % (bytes, dt, round((bytes / dt) / 1024 / 1024, 2)))
def run(self, filename, receiver, checksum=None):
if checksum is None:
checksum = get_checksum(filename)
dbg("checksum:", checksum)
if isinstance(receiver, basestring):
receiver = lookup(receiver)
file = FileRef.publish(filename)
receiver << checksum << file
import os
bytes = os.stat(filename).st_size
t1 = time.time()
self.watch(receiver)
yield self.get(('terminated', receiver))
t2 = time.time()
dt = t2 - t1
dbg("%s bytes transferred in %ss -- speed: %r MB/s" % (bytes, dt, round((bytes / dt) / 1024 / 1024, 2)))
def run(self, other_actor):
other_actor = lookup(other_actor) if isinstance(other_actor, str) else other_actor
while True:
dbg("sending greeting to %r" % (other_actor,))
other_actor << ('hello!', self.ref)
dbg("waiting for ack from %r" % (other_actor,))
yield with_timeout(5.0, self.get('ack'))
dbg("got 'ack' from %r; now sleeping a bit..." % (other_actor,))
yield sleep(1.0)
def run(self):
child = self.spawn(ExampleActor)
while True:
dbg("sending greeting to %r" % (child,))
child << ('hello!', self.ref)
dbg("waiting for ack from %r" % (child,))
yield with_timeout(5.0, self.get('ack'))
dbg("got 'ack' from %r; now sleeping a bit..." % (child,))
yield sleep(1.0)
def run(self):
child = self.spawn(ExampleActor)
while True:
dbg("sending greeting to %r" % (child, ))
child << 'hello!'
dbg("waiting for ack from %r" % (child, ))
with_timeout(5.0, self.get, 'ack')
dbg("got 'ack' from %r; now sleeping a bit..." % (child, ))
sleep(1.0)
def run(self, other_actor):
if isinstance(other_actor, str):
other_actor = self.node.lookup_str(other_actor)
while True:
dbg("sending greeting to %r" % (other_actor,))
other_actor << 'hello!'
dbg("waiting for ack from %r" % (other_actor,))
self.get('ack', timeout=5.0)
dbg("got 'ack' from %r; now sleeping a bit..." % (other_actor,))
sleep(1.0)
def run(self):
checksum_orig = yield self.get()
fileref = yield self.get()
fh = fileref.get_handle()
dbg("RECEIVED", fileref, "with checksum", checksum_orig)
md5 = hashlib.md5()
while True:
chunk = yield fh.read(CHUNK_SIZE)
dbg("READ CHUNK %r AT %r" % (hashlib.md5(chunk[:100]).hexdigest()[:8], str(time.time())[8:]))
md5.update(chunk)
if len(chunk) < CHUNK_SIZE:
break
checksum = md5.hexdigest()
dbg("CHECKSUM", checksum)
assert checksum_orig == checksum, (checksum, checksum_orig)
def run(self):
checksum_orig = yield self.get()
fileref = yield self.get()
fh = fileref.get_handle()
dbg("RECEIVED", fileref, "with checksum", checksum_orig)
md5 = hashlib.md5()
while True:
chunk = yield fh.read(CHUNK_SIZE)
dbg("READ CHUNK %r AT %r" % (hashlib.md5(chunk[:100]).hexdigest()[:8], str(time.time())[8:]))
md5.update(chunk)
if len(chunk) < CHUNK_SIZE:
break
checksum = md5.hexdigest()
dbg("CHECKSUM", checksum)
assert checksum_orig == checksum, (checksum, checksum_orig)
def receive(self, msg):
dbg("%r from %r" % (msg, self.sender))
self.sender << 'ack'
def receive(self, msg):
t = self.reactor.seconds()
self.num_msgs_received += 1
dt = t - self.msg_receive_started
if dt >= 1.0:
dbg("%d msg/s" % (float(self.num_msgs_received) / dt))
self.msg_receive_started = t
self.num_msgs_received = 0
self.num_msgs_received += 1
# dbg(t, msg)
def logappend(*args):
pass
# self.log.append((t,) + args)
# if len(self.log) > 50000:
# dbg("trimming log")
# self.log = self.log[10000:]
# Monitoring
if ('connect', ANY) == msg:
_, sender = msg
dbg("connect from", sender)
sender << 'ack'
elif ('up', ANY) == msg:
_, sender = msg
self.watch(sender)
logappend('up', str(sender.uri))
self.up[sender] = ('up', t)
elif ('terminated', IN(self.up)) == msg:
_, sender = msg
logappend('down', str(sender.uri))
self.up[sender] = ('down', t)
elif ('state', ANY, ANY) == msg:
_, sender, state = msg
if sender in self.up:
self.up[sender] = (self.up[sender][0], t)
else:
logappend('up', str(sender.uri))
self.up[sender] = ('up', t)
logappend('state', str(sender.uri), state)
self.state[sender] = state
elif ('log', ANY, ANY) == msg:
_, sender, logmsg = msg
logappend('log', str(sender.uri), logmsg)
#
elif ('get-up', ANY, ANY) == msg:
_, d, filter = msg
msg[1].callback([(str(k.uri), v) for k, v in self.up.items() if k == filter])
elif ('get-state', ANY, ANY) == msg:
_, d, filter = msg
msg[1].callback([(str(k.uri), v) for k, v in self.state.items() if k == filter])
elif ('get-log', ANY, ANY) == msg:
_, d, filter = msg
msg[1].callback([x for x in self.log if x == filter])
elif ('get-all', ANY, ANY) == msg:
_, d, filter = msg
data = {}
for client in self.up:
uri = str(client.uri)
if uri == filter:
status, seen = self.up[client]
state = self.state[client] if client in self.state else None
data[uri] = (status, seen, state)
msg[1].callback([(uri, status, seen, state) for uri, (status, seen, state) in data.items()])
else:
raise Unhandled
def receive(self, msg):
t = self.reactor.seconds()
self.num_msgs_received += 1
dt = t - self.msg_receive_started
if dt >= 1.0:
dbg("%d msg/s" % (float(self.num_msgs_received) / dt))
self.msg_receive_started = t
self.num_msgs_received = 0
self.num_msgs_received += 1
# dbg(t, msg)
def logappend(*args):
pass
# self.log.append((t,) + args)
# if len(self.log) > 50000:
# dbg("trimming log")
# self.log = self.log[10000:]
# Monitoring
if ('connect', ANY) == msg:
_, sender = msg
dbg("connect from", sender)
sender << 'ack'
elif ('up', ANY) == msg:
_, sender = msg
self.watch(sender)
logappend('up', str(sender.uri))
self.up[sender] = ('up', t)
elif ('terminated', IN(self.up)) == msg:
_, sender = msg
logappend('down', str(sender.uri))
self.up[sender] = ('down', t)
elif ('state', ANY, ANY) == msg:
_, sender, state = msg
if sender in self.up:
self.up[sender] = (self.up[sender][0], t)
else:
logappend('up', str(sender.uri))
self.up[sender] = ('up', t)
logappend('state', str(sender.uri), state)
self.state[sender] = state
elif ('log', ANY, ANY) == msg:
_, sender, logmsg = msg
logappend('log', str(sender.uri), logmsg)
#
elif ('get-up', ANY, ANY) == msg:
_, d, filter = msg
msg[1].callback([(str(k.uri), v) for k, v in self.up.items()
if k == filter])
elif ('get-state', ANY, ANY) == msg:
_, d, filter = msg
msg[1].callback([(str(k.uri), v) for k, v in self.state.items()
if k == filter])
elif ('get-log', ANY, ANY) == msg:
_, d, filter = msg
msg[1].callback([x for x in self.log if x == filter])
elif ('get-all', ANY, ANY) == msg:
_, d, filter = msg
data = {}
for client in self.up:
uri = str(client.uri)
if uri == filter:
status, seen = self.up[client]
state = self.state[client] if client in self.state else None
data[uri] = (status, seen, state)
msg[1].callback([(uri, status, seen, state)
for uri, (status, seen, state) in data.items()])
else:
raise Unhandled
def post_stop(self):
dbg("stopping")
def pre_start(self):
dbg("starting")
def pre_start(self):
dbg("starting")
def receive(self, msg):
content, sender = msg
dbg("%r from %r" % (content, sender))
sender << 'ack'
def receive(self, fref):
try:
fref.fetch()
except TransferInterrupted as e:
dbg(e)
failed.set()
def receive(self, msg):
dbg("%r from %r" % (msg, self.sender))
self.sender << 'ack'
def shutdown(self):
dbg()
self.owner.outsock_shutdown(self.addr, self.endpoint_addr)
def run(self):
while True:
dbg("sleeping...", gevent.getcurrent())
gevent.sleep(1.0)
def receive(self, fref):
try:
fref.fetch()
except TransferInterrupted as e:
dbg(e)
failed.set()
def receive(self, msg):
if ('serve', ANY, ANY) == msg:
_, file_path, file_id = msg
if not os.path.exists(file_path) and not os.path.isdir(file_path):
err("attempt to publish a file that does not exist")
elif file_id in self.published:
err("Attempt to publish %r with ID %r but a file already exists with that ID" % (file_path, file_id))
else:
self.published[file_id] = (file_path, datetime.datetime.now())
elif msg == ('request', ANY) or msg == ('request-local', ANY):
request, file_id = msg
if file_id not in self.published:
# TODO: replace with a reply to the requestor, just like in the upload() case
err("attempt to get a file with ID %r which has not been published or is not available anymore" % (file_id,))
else:
file_path, time_added = self.published[file_id]
if request == 'request-local':
self._touch_file(file_id)
self.reply(('local-file', file_path))
else:
response = self.spawn(Response.using(file=file_path, request=self.sender, threadpool=self.threadpool))
self.watch(response)
self.responses[response] = file_id
elif 'purge-old' == msg:
self.send_later(60, 'purge-old')
t = datetime.datetime.now()
for file_id, (file_path, time_added) in self.published.items():
if (t - time_added).total_seconds() > constants.FILE_MAX_LIFETIME and file_id not in self.responses.values():
dbg("purging file %r at %r" % (file_id, file_path))
del self.published[file_id]
elif ('terminated', IN(self.responses)) == msg:
_, sender = msg
self._touch_file(file_id=self.responses.pop(sender))
elif ('upload', ANY, ANY, ANY, ANY) == msg:
_, file_id, url, method, expect_response = msg
if method not in ('POST', 'PUT'):
self.reply((False, ('bad-method', None)))
if file_id not in self.published:
self.reply((False, ('file-not-found', None)))
else:
self._touch_file(file_id)
file_path, _ = self.published[file_id]
upload = requests.post if method == 'POST' else requests.put
try:
r = upload(url, files={'file': open(file_path, 'rb')})
except Exception as e:
self.reply((False, ('exception', (type(e).__name__, e.message, traceback.format_exc()))))
else:
if r.status_code != expect_response:
self.reply((False, ('bad-status-code', r.status_code)))
else:
self.reply((True, (r.status_code, dict(r.headers), r.text)))
elif ('delete', ANY) == msg:
_, file_id = msg
if file_id not in self.published:
self.reply((False, ('file-not-found', None)))
else:
file_path, _ = self.published[file_id]
del self.published[file_id]
try:
os.unlink(file_path)
except BaseException as e:
self.reply((False, ('exception', (type(e).__name__, e.message, traceback.format_exc()))))
else:
self.reply((True, None))
def post_stop(self):
dbg("stopping")
