def start(self):
"""Starts a new child worker process"""
args = [control.get_process_command(), '--worker', '-f', '-s', '-P']
if self.threadpoolsize:
args.append('--threadpoolsize=%d' % self.threadpoolsize)
endpoint = ProcessEndpoint(reactor, control.get_process_command(),
args, os.environ)
factory = protocol.Factory()
factory.protocol = lambda: ProcessAMP(is_worker=False,
locator=JobHandler())
self.process = yield endpoint.connect(factory)
self.process.lost_handler = self._worker_died
self.started_at = datetime.datetime.now()
self._logger.debug("Started new worker %r", self)
if ipdevpoll_conf.getboolean("multiprocess",
"ping_workers",
fallback=True):
self._ping_loop.start(
interval=ipdevpoll_conf.getint("multiprocess",
"ping_interval",
fallback=30),
now=False,
)
returnValue(self)
def start(self, cmd, lrCallback, clCallback):
endpoint = DisconnectedWorkaroundEndpoint(
ProcessEndpoint(reactor,
'stdbuf',
args=['-o0', '-e0', '-i0'] + cmd))
dfd = endpoint.connect(StreamingFactory(lrCallback, clCallback))
dfd.addCallback(self.getProtocol)
def start():
from twisted.internet import reactor
ep = ProcessEndpoint(reactor,
RUST_PATH, [RUST_PATH],
childFDs={
0: 'w',
1: 'r',
2: 2
},
env=None)
factory = Factory.forProtocol(Responder)
rust = yield ep.connect(factory)
call1 = rust.callRemote(Sum, a=1, b=2)
call2 = rust.callRemote(Sum, a=4, b=-100)
dl = yield DeferredList([call1, call2])
print(dl)
d = Deferred()
reactor.callLater(.01, d.callback, None)
# Add small delay because it is hard to check when the request fd
# has been closed.
yield d
reactor.stop()
def applyPatch(patch, patchLevel="0", reactor=None):
"""
Apply a patch to the current git repository.
@param patch: Patch to apply
@type patch: L{str}
@param patchLevel: Number of directries to strip from paths in patch
"""
proto = AccumulatingProtocol()
done = Deferred()
proto.closedDeferred = done
def feedPatch(proto):
proto.transport.write(patch)
proto.transport.closeStdin()
connectProtocol(
ProcessEndpoint(reactor, "git",
("git", "apply", "--index", "-p", patchLevel)),
proto).addCallback(feedPatch)
def eb(_):
# Note, we can't print why git apply failed due to https://tm.tl/#6576
proto.closedReason.trap(ConnectionDone)
done.addCallback(eb)
return done
def start(self):
args = [control.get_process_command(), '--worker', '-f', '-s', '-P']
if self.threadpoolsize:
args.append('--threadpoolsize=%d' % self.threadpoolsize)
endpoint = ProcessEndpoint(reactor, control.get_process_command(),
args, os.environ)
factory = protocol.Factory()
factory.protocol = lambda: ProcessAMP(is_worker=False,
locator=JobHandler())
self.process = yield endpoint.connect(factory)
self.process.lost_handler = self._worker_died
returnValue(self)
def start(self):
"""Starts a new child worker process"""
args = [control.get_process_command(), '--worker', '-f', '-s', '-P']
if self.threadpoolsize:
args.append('--threadpoolsize=%d' % self.threadpoolsize)
endpoint = ProcessEndpoint(reactor, control.get_process_command(),
args, os.environ)
factory = protocol.Factory()
factory.protocol = lambda: ProcessAMP(is_worker=False,
locator=JobHandler())
self.process = yield endpoint.connect(factory)
self.process.lost_handler = self._worker_died
self.started_at = datetime.datetime.now()
self._logger.debug("Started new worker %r", self)
returnValue(self)
def _parse(self, reactor, directory, query, native_query=True, type=None, executable=None):
binary_name = self._binary_name(type)
if not executable:
executable = self._find_executable(binary_name)
if not executable:
raise ValueError("Failed to find command {:s}".format(binary_name))
args = (executable,)
if ast.literal_eval(str(native_query)):
args += ('-n',)
args += (directory, query)
return ProcessEndpoint(reactor, executable, args=list(map(fsencode, args)))
def zfs_command(reactor, arguments):
"""Run the ``zfs`` command-line tool with the given arguments.
:param reactor: A ``IReactorProcess`` provider.
:param arguments: A ``list`` of ``bytes``, command-line arguments to
``zfs``.
:return: A :class:`Deferred` firing with the bytes of the result (on
exit code 0), or errbacking with :class:`CommandFailed` or
:class:`BadArguments` depending on the exit code (1 or 2).
"""
endpoint = ProcessEndpoint(reactor, b"zfs", [b"zfs"] + arguments,
os.environ)
d = connectProtocol(endpoint, _AccumulatingProtocol())
d.addCallback(lambda protocol: protocol._result)
return d
def from_crawler(cls, crawler):
settings = crawler.settings
if crawler.settings.getbool('BROWSER_ENGINE_COOKIES_ENABLED', False):
if crawler.settings.getbool('COOKIES_ENABLED'):
logger.warning("Default cookies middleware enabled together "
"with browser engine aware cookies middleware. "
"Set COOKIES_ENABLED to False.")
cookies_mw = RemotelyAccessibleCookiesMiddleware(
debug=crawler.settings.getbool('COOKIES_DEBUG'))
else:
cookies_mw = None
server = settings.get('BROWSER_ENGINE_SERVER')
start_server = settings.getbool('BROWSER_ENGINE_START_SERVER', False)
if not (server or start_server):
raise NotConfigured("Must specify either BROWSER_ENGINE_SERVER or "
"BROWSER_ENGINE_START_SERVER")
if server and start_server:
raise NotConfigured("Must not specify both BROWSER_ENGINE_SERVER "
"and BROWSER_ENGINE_START_SERVER=True")
if server:
endpoint = clientFromString(reactor, server)
else:
# Twisted logs the process's stderr with INFO level.
logging.getLogger("twisted").setLevel(logging.INFO)
argv = [
sys.executable, "-m", "scrapy_qtwebkit.browser_engine", "stdio"
]
endpoint = ProcessEndpoint(reactor, argv[0], argv, env=None)
mw = cls(
crawler,
endpoint,
page_limit=settings.getint('BROWSER_ENGINE_PAGE_LIMIT', 4),
browser_options=settings.getdict('BROWSER_ENGINE_OPTIONS'),
cookies_middleware=cookies_mw,
)
crawler.signals.connect(mw._engine_stopped,
signal=signals.engine_stopped)
return mw
def askForPassword(self, reactor, prompt, title, description):
"""
The documentation appears to be here only:
https://github.com/gpg/pinentry/blob/287d40e879f767dbcb3d19b3629b872c08d39cf4/pinentry/pinentry.c#L1444-L1464
TODO: multiple backends for password-prompting.
"""
argv = self.argv()
assuan = yield (ProcessEndpoint(reactor, argv[0], argv,
os.environ.copy()).connect(
Factory.forProtocol(SimpleAssuan)))
try:
yield assuan.issueCommand(b"SETPROMPT", prompt.encode("utf-8"))
yield assuan.issueCommand(b"SETTITLE", title.encode("utf-8"))
yield assuan.issueCommand(b"SETDESC", description.encode("utf-8"))
response = yield assuan.issueCommand(b"GETPIN")
finally:
assuan.issueCommand(b"BYE")
returnValue(response.data.decode("utf-8"))
def from_crawler(cls, crawler):
settings = crawler.settings
if crawler.settings.getbool('BROWSER_ENGINE_COOKIES_ENABLED', False):
if crawler.settings.getbool('COOKIES_ENABLED'):
logger.warning("Default cookies middleware enabled together "
"with browser engine aware cookies middleware. "
"Set COOKIES_ENABLED to False.")
cookies_mw = RemotelyAccessbileCookiesMiddleware(
debug=crawler.settings.getbool('COOKIES_DEBUG'))
else:
cookies_mw = None
server = settings.get('BROWSER_ENGINE_SERVER')
if server:
endpoint = clientFromString(reactor, server)
else:
if settings.getbool('BROWSER_ENGINE_START_SERVER', False):
# Twisted logs the process's stderr with INFO level.
logging.getLogger("twisted").setLevel(logging.INFO)
argv = [
sys.executable, "-m", "scrapy_qtwebkit.browser_engine",
"stdio"
]
endpoint = ProcessEndpoint(reactor, argv[0], argv, env=None)
else:
raise NotConfigured(
"Must provide either BROWSER_ENGINE_SERVER "
"or BROWSER_ENGINE_START_SERVER")
ext = cls(
crawler,
endpoint,
page_limit=settings.getint('BROWSER_ENGINE_PAGE_LIMIT', 4),
cookies_middleware=cookies_mw,
)
return ext
def askForPassword(reactor, prompt, title, description):
"""
The documentation appears to be here only:
https://github.com/gpg/pinentry/blob/287d40e879f767dbcb3d19b3629b872c08d39cf4/pinentry/pinentry.c#L1444-L1464
TODO: multiple backends for password-prompting.
"""
executable = (
# It would be nice if there were a more general mechanism for this...
which('/usr/local/MacGPG2/libexec/pinentry-mac.app'
'/Contents/MacOS/pinentry-mac') + which('pinentry-mac') +
which('pinentry'))[0]
argv = [executable]
assuan = yield (ProcessEndpoint(reactor, executable, argv,
os.environ.copy()).connect(
Factory.forProtocol(SimpleAssuan)))
try:
yield assuan.issueCommand(b"SETPROMPT", prompt.encode("utf-8"))
yield assuan.issueCommand(b"SETTITLE", title.encode("utf-8"))
yield assuan.issueCommand(b"SETDESC", description.encode("utf-8"))
response = yield assuan.issueCommand(b"GETPIN")
finally:
assuan.issueCommand(b"BYE")
returnValue(response.data.decode("utf-8"))
