def start(self):
"""Starts a new child worker process"""
args = [control.get_process_command(), '--worker', '-f', '-s', '-P']
if self.threadpoolsize:
args.append('--threadpoolsize=%d' % self.threadpoolsize)
endpoint = ProcessEndpoint(reactor, control.get_process_command(),
args, os.environ)
factory = protocol.Factory()
factory.protocol = lambda: ProcessAMP(is_worker=False,
locator=JobHandler())
self.process = yield endpoint.connect(factory)
self.process.lost_handler = self._worker_died
self.started_at = datetime.datetime.now()
self._logger.debug("Started new worker %r", self)
if ipdevpoll_conf.getboolean("multiprocess",
"ping_workers",
fallback=True):
self._ping_loop.start(
interval=ipdevpoll_conf.getint("multiprocess",
"ping_interval",
fallback=30),
now=False,
)
returnValue(self)
def start():
from twisted.internet import reactor
ep = ProcessEndpoint(reactor,
RUST_PATH, [RUST_PATH],
childFDs={
0: 'w',
1: 'r',
2: 2
},
env=None)
factory = Factory.forProtocol(Responder)
rust = yield ep.connect(factory)
call1 = rust.callRemote(Sum, a=1, b=2)
call2 = rust.callRemote(Sum, a=4, b=-100)
dl = yield DeferredList([call1, call2])
print(dl)
d = Deferred()
reactor.callLater(.01, d.callback, None)
# Add small delay because it is hard to check when the request fd
# has been closed.
yield d
reactor.stop()
def __init__(self, *args, **kwargs):
"""
Ctor.
:param worker: The worker this endpoint is being used for.
:type worker: instance of WorkerProcess
"""
self._worker = kwargs.pop('worker')
ProcessEndpoint.__init__(self, *args, **kwargs)
def __init__(self, *args, **kwargs):
"""
Ctor.
:param worker: The worker this endpoint is being used for.
:type worker: instance of WorkerProcess
"""
self._worker = kwargs.pop('worker')
ProcessEndpoint.__init__(self, *args, **kwargs)
def start(self):
args = [control.get_process_command(), '--worker', '-f', '-s', '-P']
if self.threadpoolsize:
args.append('--threadpoolsize=%d' % self.threadpoolsize)
endpoint = ProcessEndpoint(reactor, control.get_process_command(),
args, os.environ)
factory = protocol.Factory()
factory.protocol = lambda: ProcessAMP(is_worker=False,
locator=JobHandler())
self.process = yield endpoint.connect(factory)
self.process.lost_handler = self._worker_died
returnValue(self)
def start(self):
"""Starts a new child worker process"""
args = [control.get_process_command(), '--worker', '-f', '-s', '-P']
if self.threadpoolsize:
args.append('--threadpoolsize=%d' % self.threadpoolsize)
endpoint = ProcessEndpoint(reactor, control.get_process_command(),
args, os.environ)
factory = protocol.Factory()
factory.protocol = lambda: ProcessAMP(is_worker=False,
locator=JobHandler())
self.process = yield endpoint.connect(factory)
self.process.lost_handler = self._worker_died
self.started_at = datetime.datetime.now()
self._logger.debug("Started new worker %r", self)
returnValue(self)
def start(self, cmd, lrCallback, clCallback):
endpoint = DisconnectedWorkaroundEndpoint(
ProcessEndpoint(reactor,
'stdbuf',
args=['-o0', '-e0', '-i0'] + cmd))
dfd = endpoint.connect(StreamingFactory(lrCallback, clCallback))
dfd.addCallback(self.getProtocol)
def applyPatch(patch, patchLevel="0", reactor=None):
"""
Apply a patch to the current git repository.
@param patch: Patch to apply
@type patch: L{str}
@param patchLevel: Number of directries to strip from paths in patch
"""
proto = AccumulatingProtocol()
done = Deferred()
proto.closedDeferred = done
def feedPatch(proto):
proto.transport.write(patch)
proto.transport.closeStdin()
connectProtocol(
ProcessEndpoint(reactor, "git",
("git", "apply", "--index", "-p", patchLevel)),
proto).addCallback(feedPatch)
def eb(_):
# Note, we can't print why git apply failed due to https://tm.tl/#6576
proto.closedReason.trap(ConnectionDone)
done.addCallback(eb)
return done
def _parse(self, reactor, directory, query, native_query=True, type=None, executable=None):
binary_name = self._binary_name(type)
if not executable:
executable = self._find_executable(binary_name)
if not executable:
raise ValueError("Failed to find command {:s}".format(binary_name))
args = (executable,)
if ast.literal_eval(str(native_query)):
args += ('-n',)
args += (directory, query)
return ProcessEndpoint(reactor, executable, args=list(map(fsencode, args)))
def zfs_command(reactor, arguments):
"""Run the ``zfs`` command-line tool with the given arguments.
:param reactor: A ``IReactorProcess`` provider.
:param arguments: A ``list`` of ``bytes``, command-line arguments to
``zfs``.
:return: A :class:`Deferred` firing with the bytes of the result (on
exit code 0), or errbacking with :class:`CommandFailed` or
:class:`BadArguments` depending on the exit code (1 or 2).
"""
endpoint = ProcessEndpoint(reactor, b"zfs", [b"zfs"] + arguments,
os.environ)
d = connectProtocol(endpoint, _AccumulatingProtocol())
d.addCallback(lambda protocol: protocol._result)
return d
def from_crawler(cls, crawler):
settings = crawler.settings
if crawler.settings.getbool('BROWSER_ENGINE_COOKIES_ENABLED', False):
if crawler.settings.getbool('COOKIES_ENABLED'):
logger.warning("Default cookies middleware enabled together "
"with browser engine aware cookies middleware. "
"Set COOKIES_ENABLED to False.")
cookies_mw = RemotelyAccessibleCookiesMiddleware(
debug=crawler.settings.getbool('COOKIES_DEBUG'))
else:
cookies_mw = None
server = settings.get('BROWSER_ENGINE_SERVER')
start_server = settings.getbool('BROWSER_ENGINE_START_SERVER', False)
if not (server or start_server):
raise NotConfigured("Must specify either BROWSER_ENGINE_SERVER or "
"BROWSER_ENGINE_START_SERVER")
if server and start_server:
raise NotConfigured("Must not specify both BROWSER_ENGINE_SERVER "
"and BROWSER_ENGINE_START_SERVER=True")
if server:
endpoint = clientFromString(reactor, server)
else:
# Twisted logs the process's stderr with INFO level.
logging.getLogger("twisted").setLevel(logging.INFO)
argv = [
sys.executable, "-m", "scrapy_qtwebkit.browser_engine", "stdio"
]
endpoint = ProcessEndpoint(reactor, argv[0], argv, env=None)
mw = cls(
crawler,
endpoint,
page_limit=settings.getint('BROWSER_ENGINE_PAGE_LIMIT', 4),
browser_options=settings.getdict('BROWSER_ENGINE_OPTIONS'),
cookies_middleware=cookies_mw,
)
crawler.signals.connect(mw._engine_stopped,
signal=signals.engine_stopped)
return mw
def askForPassword(self, reactor, prompt, title, description):
"""
The documentation appears to be here only:
https://github.com/gpg/pinentry/blob/287d40e879f767dbcb3d19b3629b872c08d39cf4/pinentry/pinentry.c#L1444-L1464
TODO: multiple backends for password-prompting.
"""
argv = self.argv()
assuan = yield (ProcessEndpoint(reactor, argv[0], argv,
os.environ.copy()).connect(
Factory.forProtocol(SimpleAssuan)))
try:
yield assuan.issueCommand(b"SETPROMPT", prompt.encode("utf-8"))
yield assuan.issueCommand(b"SETTITLE", title.encode("utf-8"))
yield assuan.issueCommand(b"SETDESC", description.encode("utf-8"))
response = yield assuan.issueCommand(b"GETPIN")
finally:
assuan.issueCommand(b"BYE")
returnValue(response.data.decode("utf-8"))
def from_crawler(cls, crawler):
settings = crawler.settings
if crawler.settings.getbool('BROWSER_ENGINE_COOKIES_ENABLED', False):
if crawler.settings.getbool('COOKIES_ENABLED'):
logger.warning("Default cookies middleware enabled together "
"with browser engine aware cookies middleware. "
"Set COOKIES_ENABLED to False.")
cookies_mw = RemotelyAccessbileCookiesMiddleware(
debug=crawler.settings.getbool('COOKIES_DEBUG'))
else:
cookies_mw = None
server = settings.get('BROWSER_ENGINE_SERVER')
if server:
endpoint = clientFromString(reactor, server)
else:
if settings.getbool('BROWSER_ENGINE_START_SERVER', False):
# Twisted logs the process's stderr with INFO level.
logging.getLogger("twisted").setLevel(logging.INFO)
argv = [
sys.executable, "-m", "scrapy_qtwebkit.browser_engine",
"stdio"
]
endpoint = ProcessEndpoint(reactor, argv[0], argv, env=None)
else:
raise NotConfigured(
"Must provide either BROWSER_ENGINE_SERVER "
"or BROWSER_ENGINE_START_SERVER")
ext = cls(
crawler,
endpoint,
page_limit=settings.getint('BROWSER_ENGINE_PAGE_LIMIT', 4),
cookies_middleware=cookies_mw,
)
return ext
def askForPassword(reactor, prompt, title, description):
"""
The documentation appears to be here only:
https://github.com/gpg/pinentry/blob/287d40e879f767dbcb3d19b3629b872c08d39cf4/pinentry/pinentry.c#L1444-L1464
TODO: multiple backends for password-prompting.
"""
executable = (
# It would be nice if there were a more general mechanism for this...
which('/usr/local/MacGPG2/libexec/pinentry-mac.app'
'/Contents/MacOS/pinentry-mac') + which('pinentry-mac') +
which('pinentry'))[0]
argv = [executable]
assuan = yield (ProcessEndpoint(reactor, executable, argv,
os.environ.copy()).connect(
Factory.forProtocol(SimpleAssuan)))
try:
yield assuan.issueCommand(b"SETPROMPT", prompt.encode("utf-8"))
yield assuan.issueCommand(b"SETTITLE", title.encode("utf-8"))
yield assuan.issueCommand(b"SETDESC", description.encode("utf-8"))
response = yield assuan.issueCommand(b"GETPIN")
finally:
assuan.issueCommand(b"BYE")
returnValue(response.data.decode("utf-8"))
def start_process(self, config):
"""
Management API for starting a new process on this node.
:param config: The process configuration.
:type config: dict
:returns: int -- The PID of the new process.
"""
if config['type'] in ['router', 'component.python']:
##
## start a Crossbar.io worker process
##
filename = pkg_resources.resource_filename('crossbar', 'worker.py')
args = [executable, "-u", filename]
if self.debug:
args.append('--debug')
if sys.platform == 'win32':
args.extend(['--logfile', 'test.log'])
ep = ProcessEndpoint(self._node._reactor,
executable,
args,
errFlag = StandardErrorBehavior.DROP,
env = os.environ)
else:
ep = ProcessEndpoint(self._node._reactor,
executable,
args,
childFDs = {0: 'w', 1: 'r', 2: 2}, # does not work on Windows
errFlag = StandardErrorBehavior.LOG,
env = os.environ)
ready = Deferred()
d = ep.connect(self._node._router_client_transport_factory)
def onconnect(res):
pid = res.transport.pid
self._processes[pid] = NodeControllerSession.NodeProcess('worker', pid, ready)
def onerror(err):
ready.errback(err)
d.addCallbacks(onconnect, onerror)
return ready
elif config['type'] == 'component.program':
##
## start a program process
##
from twisted.internet import protocol
from twisted.internet import reactor
from twisted.internet.error import ProcessDone, ProcessTerminated
import re, json
class ProgramWorkerProcess(protocol.ProcessProtocol):
def __init__(self):
self._pid = None
def connectionMade(self):
if 'stdin' in config and config['stdin'] == 'config' and 'config' in config:
## write process config from configuration to stdin
## of the forked process and close stdin
self.transport.write(json.dumps(config['config']))
self.transport.closeStdin()
def outReceived(self, data):
if 'stdout' in config and config['stdout'] == 'log':
try:
data = str(data).strip()
except:
data = "{} bytes".format(len(data))
log.msg("Worker {} (stdout): {}".format(self._pid, data))
def errReceived(self, data):
if 'stderr' in config and config['stderr'] == 'log':
try:
data = str(data).strip()
except:
data = "{} bytes".format(len(data))
log.msg("Worker {} (stderr): {}".format(self._pid, data))
def inConnectionLost(self):
pass
def outConnectionLost(self):
pass
def errConnectionLost(self):
pass
def processExited(self, reason):
pass
def processEnded(self, reason):
if isinstance(reason.value, ProcessDone):
log.msg("Worker {}: Ended cleanly.".format(self._pid))
elif isinstance(reason.value, ProcessTerminated):
log.msg("Worker {}: Ended with error {}".format(self._pid, reason.value.exitCode))
else:
## should not arrive here
pass
exe = config['executable']
args = [exe]
args.extend(config.get('arguments', []))
workdir = self._node._cbdir
if 'workdir' in config:
workdir = os.path.join(workdir, config['workdir'])
workdir = os.path.abspath(workdir)
ready = Deferred()
proto = ProgramWorkerProcess()
try:
trnsp = reactor.spawnProcess(proto, exe, args, path = workdir, env = os.environ)
except Exception as e:
log.msg("Worker: Program could not be started - {}".format(e))
ready.errback(e)
else:
pid = trnsp.pid
proto._pid = pid
self._processes[pid] = NodeControllerSession.NodeProcess('program', pid, ready)
log.msg("Worker {}: Program started.".format(pid))
ready.callback(pid)
return ready
else:
raise ApplicationError("wamp.error.invalid_argument", "Invalid process type '{}'".format(config['type']))
transport_factory.protocol = WorkerClientProtocol
executable = os.path.abspath(os.path.join(os.getcwd(), args.worker))
print("Starting worker {}".format(executable))
args = [executable]
if False:
from twisted.internet.endpoints import ProcessEndpoint, StandardErrorBehavior
ep = ProcessEndpoint(reactor,
executable,
args,
#childFDs = {0: 'w', 1: 'r', 2: 2}, # does not work on Windows
errFlag = StandardErrorBehavior.LOG,
env = os.environ)
else:
from crossbar.process import CustomProcessEndpoint
ep = CustomProcessEndpoint(reactor,
executable,
args,
name = "Worker",
env = os.environ)
d = ep.connect(transport_factory)
def onconnect(res):
def __init__(self, *args, **kwargs):
self._name = kwargs.pop("name", None)
ProcessEndpoint.__init__(self, *args, **kwargs)
