def run(self, service, pub_id, file, send_to):
self.watch(send_to)
seek_ptr = 0
while True:
try:
msg = yield with_timeout(OPEN_FILE_TIMEOUT, self.get())
except Timeout:
err("Sending of file at %r timed out" % (file,))
break
if ('next-chunk', ANY) == msg:
service << ('touch-file', pub_id)
_, chunk_size = msg
chunk = yield read_file_async(file, start=seek_ptr, end=seek_ptr + chunk_size)
seek_ptr += len(chunk)
more_coming = len(chunk) == chunk_size
send_to << ('chunk', chunk, more_coming)
if not more_coming:
break
elif ('terminated', send_to) == msg:
break
else:
self.unhandled(msg)
def start_actor():
if self._nodeid:
Events.log(Message("Setting up remoting; node ID = %s" % (self._nodeid,)))
try:
f1 = ZmqFactory()
insock = ZmqPullConnection(f1)
outsock = lambda: ZmqPushConnection(f1, linger=0)
hub = Hub(insock, outsock, nodeid=self._nodeid)
except Exception:
err("Could not set up remoting")
traceback.print_exc()
reactor.stop()
return
else:
Events.log(Message("No remoting requested; specify `--remoting/-r <nodeid>` (nodeid=host:port) to set up remoting"))
hub = HubWithNoRemoting()
supervision = {'stop': Stop, 'restart': Restart, 'resume': Resume}[self._supervise]
node = Node(hub=hub, root_supervision=supervision)
try:
self._wrapper = node.spawn(Wrapper.using(
self._actor_cls.using(**self._init_params),
spawn_at=self._name, keep_running=self._keep_running
), name='_runner')
except Exception:
panic("Failed to start wrapper for %s\n" % (actor_path,),
Failure().getTraceback())
reactor.stop()
else:
if self._initial_message is not _EMPTY:
self._wrapper << ('_forward', self._initial_message)
def naddr_to_zmq_endpoint(nid):
if '\0' in nid:
return None
try:
host, port = nid.split(':')
except ValueError:
return None
try:
return 'tcp://%s:%s' % (gethostbyname(host), port)
except socket.gaierror as e:
# XXX: perhaps we should retry in a few sec in case of EAI_ERRNO_TEMPORARY_FAILURE_IN_NAME_RESOLUTION?
if e.errno not in (socket.EAI_NONAME, EAI_ERRNO_TEMPORARY_FAILURE_IN_NAME_RESOLUTION):
err("%s\n%s" % (e, traceback.format_exc()))
return None
def naddr_to_zmq_endpoint(nid):
if '\0' in nid:
return None
try:
host, port = nid.split(':')
except ValueError:
return None
try:
return 'tcp://%s:%s' % (gethostbyname(host), port)
except socket.gaierror as e:
# XXX: perhaps we should retry in a few sec in case of EAI_ERRNO_TEMPORARY_FAILURE_IN_NAME_RESOLUTION?
if e.errno not in (socket.EAI_NONAME,
EAI_ERRNO_TEMPORARY_FAILURE_IN_NAME_RESOLUTION):
err("%s\n%s" % (e, traceback.format_exc()))
return None
def log(self, event, log_caller=False):
try:
(fail if isinstance(event, Error) else log)(event, caller=log_caller)
consumers = self.consumers.get(type(event))
if consumers:
consumer_d = consumers.pop(0)
consumer_d.callback(event)
return
subscriptions = self.subscriptions.get(type(event))
if subscriptions:
for fn in subscriptions:
try:
fn(event)
except Exception: # pragma: no cover
err("Error in event handler:\n", traceback.format_exc())
except Exception: # pragma: no cover
print("Events.log failed:\n", traceback.format_exc(), file=sys.stderr)
def receive(self, msg):
if ('serve', ANY, ANY) == msg:
_, file_path, file_id = msg
if not os.path.exists(file_path) and not os.path.isdir(file_path):
err("attempt to publish a file that does not exist")
elif file_id in self.published:
err("Attempt to publish %r with ID %r but a file already exists with that ID" % (file_path, file_id))
else:
self.published[file_id] = (file_path, datetime.datetime.now())
elif msg == ('request', ANY) or msg == ('request-local', ANY):
request, file_id = msg
if file_id not in self.published:
err("attempt to get a file with ID %r which has not been published or is not available anymore" % (file_id,))
else:
file_path, time_added = self.published[file_id]
if request == 'request-local':
self._touch_file(file_id)
self.reply(('local-file', file_path))
else:
response = self.spawn(Response.using(file=file_path, request=self.sender, threadpool=self.threadpool))
self.watch(response)
self.responses[response] = file_id
elif 'purge-old' == msg:
self.send_later(60, 'purge-old')
t = datetime.datetime.now()
for file_id, (file_path, time_added) in self.published.items():
if (t - time_added).total_seconds() > constants.FILE_MAX_LIFETIME and file_id not in self.responses.values():
dbg("purging file %r at %r" % (file_id, file_path))
del self.published[file_id]
elif ('terminated', IN(self.responses)) == msg:
_, sender = msg
self._touch_file(file_id=self.responses.pop(sender))
def log(self, event, log_caller=False):
try:
(fail if isinstance(event, Error) else log)(event,
caller=log_caller)
consumers = self.consumers.get(type(event))
if consumers:
consumer_d = consumers.pop(0)
consumer_d.set(event)
return
subscriptions = self.subscriptions.get(type(event))
if subscriptions:
for fn in subscriptions:
try:
fn(event)
except Exception: # pragma: no cover
err("Error in event handler:\n",
traceback.format_exc())
except Exception: # pragma: no cover
print("Events.log failed:\n",
traceback.format_exc(),
file=sys.stderr)
def receive(self, msg):
if ('publish', ANY, ANY) == msg:
_, file_path, pub_id = msg
if not os.path.exists(file_path) and not os.path.isdir(file_path):
err("attempt to publish a file that does not exist")
return
if pub_id in self.published:
raise FileAlreadyPublished("Attempt to publish %r with ID %r but a file already exists with that ID" % (file_path, pub_id))
else:
self.published[pub_id] = (file_path, datetime.datetime.now())
elif ('get-file', ANY, ANY) == msg:
_, pub_id, send_to = msg
if pub_id not in self.published:
err("attempt to get a file with ID %r which has not been published or is not available anymore" % (pub_id,))
return
self._touch_file(pub_id)
file_path, time_added = self.published[pub_id]
sender = self.watch(_Sender.using(service=self.ref, pub_id=pub_id, file=file_path, send_to=send_to))
self.senders[sender] = pub_id
# self.senders[sender] = pub_id
send_to << ('take-file', sender)
elif ('touch-file', ANY) == msg:
_, pub_id = msg
if pub_id not in self.published:
err("attempt to touch a file with ID %r which has not been published or is not available anymore" % (pub_id,))
return
_, pub_id = msg
self._touch_file(pub_id)
elif 'purge-old' == msg:
after(60.0).do(self.send, 'purge-old')
t = datetime.datetime.now()
for pub_id, (file_path, time_added) in self.published.items():
if (t - time_added).total_seconds() > FILE_MAX_LIFETIME and pub_id not in self.senders.values():
dbg("purging file %r at %r" % (pub_id, file_path))
del self.published[pub_id]
elif ('terminated', IN(self.senders)) == msg:
_, sender = msg
del self.senders[sender]
def receive(self, msg):
if ('serve', ANY, ANY) == msg:
_, file_path, file_id = msg
if not os.path.exists(file_path) and not os.path.isdir(file_path):
err("attempt to publish a file that does not exist")
elif file_id in self.published:
err("Attempt to publish %r with ID %r but a file already exists with that ID" % (file_path, file_id))
else:
self.published[file_id] = (file_path, datetime.datetime.now())
elif msg == ('request', ANY) or msg == ('request-local', ANY):
request, file_id = msg
if file_id not in self.published:
# TODO: replace with a reply to the requestor, just like in the upload() case
err("attempt to get a file with ID %r which has not been published or is not available anymore" % (file_id,))
else:
file_path, time_added = self.published[file_id]
if request == 'request-local':
self._touch_file(file_id)
self.reply(('local-file', file_path))
else:
response = self.spawn(Response.using(file=file_path, request=self.sender, threadpool=self.threadpool))
self.watch(response)
self.responses[response] = file_id
elif 'purge-old' == msg:
self.send_later(60, 'purge-old')
t = datetime.datetime.now()
for file_id, (file_path, time_added) in self.published.items():
if (t - time_added).total_seconds() > constants.FILE_MAX_LIFETIME and file_id not in self.responses.values():
dbg("purging file %r at %r" % (file_id, file_path))
del self.published[file_id]
elif ('terminated', IN(self.responses)) == msg:
_, sender = msg
self._touch_file(file_id=self.responses.pop(sender))
elif ('upload', ANY, ANY, ANY, ANY) == msg:
_, file_id, url, method, expect_response = msg
if method not in ('POST', 'PUT'):
self.reply((False, ('bad-method', None)))
if file_id not in self.published:
self.reply((False, ('file-not-found', None)))
else:
self._touch_file(file_id)
file_path, _ = self.published[file_id]
upload = requests.post if method == 'POST' else requests.put
try:
r = upload(url, files={'file': open(file_path, 'rb')})
except Exception as e:
self.reply((False, ('exception', (type(e).__name__, e.message, traceback.format_exc()))))
else:
if r.status_code != expect_response:
self.reply((False, ('bad-status-code', r.status_code)))
else:
self.reply((True, (r.status_code, dict(r.headers), r.text)))
elif ('delete', ANY) == msg:
_, file_id = msg
if file_id not in self.published:
self.reply((False, ('file-not-found', None)))
else:
file_path, _ = self.published[file_id]
del self.published[file_id]
try:
os.unlink(file_path)
except BaseException as e:
self.reply((False, ('exception', (type(e).__name__, e.message, traceback.format_exc()))))
else:
self.reply((True, None))
def console():
class _ImportFailed(Exception):
pass
class _InitError(Exception):
pass
argparser = argparse.ArgumentParser(
description="Runs the specified actor",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
argparser.add_argument('actor', metavar='ACTOR',
help="The actor to run")
argparser.add_argument('-using', '--using', metavar='USING', default={},
help="Parameters to pass to the actor")
argparser.add_argument('-msg', '--initial-message', metavar='MSG', nargs='*', default=[],
help="Messages to send to the actor. Specify multiple times to send multiple messages. These must be valid Python expressions")
argparser.add_argument('-keep', '--keep-running', action='store_true',
help="Whether to respawn the actor when it terminates")
argparser.add_argument('-name', '--name', metavar='NAME',
help="What to name the actor")
argparser.add_argument('-nid', '--node-id', metavar='NODEID',
help="What to name the actor")
argparser.add_argument('-relay', '--enable-relay', metavar='RELAY',
help="What to name the actor")
args = argparser.parse_args()
# where the spin command was invoked
sys.path.insert(0, os.getcwd())
try:
if '.' not in args.actor:
raise _InitError("Failed to import %s" % (args.actor,))
sys.exit(1)
else:
mod_name, cls_name = args.actor.rsplit('.', 1)
mod = __import__(mod_name, fromlist=[cls_name])
actor_cls = getattr(mod, cls_name)
eval_str = 'dict(%s)' % (args.using,)
try:
init_params = eval(eval_str)
except Exception:
raise _InitError("Failed to parse value: -init/--init-params")
initial_messages = []
for msg_raw in args.initial_message:
try:
msg = eval(msg_raw)
except Exception:
quote = '\'' if '"' in msg_raw else '"'
raise _InitError("Failde to parse value: -msg %s%s%s" % (quote, msg_raw, quote))
else:
initial_messages.append(msg)
except _InitError as exc:
err(exc.args[0])
sys.exit(exc.args[1] if len(exc.args) > 1 else 1)
spin(actor_cls, name=args.name, init_params=init_params, node_id=args.node_id,
initial_messages=initial_messages, keep_running=args.keep_running, enable_relay=args.enable_relay)
def console():
class _ImportFailed(Exception):
pass
class _InitError(Exception):
pass
argparser = argparse.ArgumentParser(
description="Runs the specified actor",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
argparser.add_argument('actor', metavar='ACTOR', help="The actor to run")
argparser.add_argument('-using',
'--using',
metavar='USING',
default={},
help="Parameters to pass to the actor")
argparser.add_argument(
'-msg',
'--initial-message',
metavar='MSG',
nargs='*',
default=[],
help=
"Messages to send to the actor. Specify multiple times to send multiple messages. These must be valid Python expressions"
)
argparser.add_argument(
'-keep',
'--keep-running',
action='store_true',
help="Whether to respawn the actor when it terminates")
argparser.add_argument('-name',
'--name',
metavar='NAME',
help="What to name the actor")
argparser.add_argument('-nid',
'--node-id',
metavar='NODEID',
help="What to name the actor")
argparser.add_argument('-relay',
'--enable-relay',
metavar='RELAY',
help="What to name the actor")
args = argparser.parse_args()
# where the spin command was invoked
sys.path.insert(0, os.getcwd())
try:
if '.' not in args.actor:
raise _InitError("Failed to import %s" % (args.actor, ))
sys.exit(1)
else:
mod_name, cls_name = args.actor.rsplit('.', 1)
mod = __import__(mod_name, fromlist=[cls_name])
actor_cls = getattr(mod, cls_name)
eval_str = 'dict(%s)' % (args.using, )
try:
init_params = eval(eval_str)
except Exception:
raise _InitError("Failed to parse value: -init/--init-params")
initial_messages = []
for msg_raw in args.initial_message:
try:
msg = eval(msg_raw)
except Exception:
quote = '\'' if '"' in msg_raw else '"'
raise _InitError("Failde to parse value: -msg %s%s%s" %
(quote, msg_raw, quote))
else:
initial_messages.append(msg)
except _InitError as exc:
err(exc.args[0])
sys.exit(exc.args[1] if len(exc.args) > 1 else 1)
spin(actor_cls,
name=args.name,
init_params=init_params,
node_id=args.node_id,
initial_messages=initial_messages,
keep_running=args.keep_running,
enable_relay=args.enable_relay)
