def open_urls(self, window_id=None):
urls = request.files.get('urls')
if urls is None:
return 'ERROR: Please provide urls file in the request'
urls = urls.stream.read().decode('utf8').splitlines()
mediator_logger.info('Open urls (window_id = %s): %s', window_id, urls)
return self.remote_api.open_urls(urls, window_id)
def send(self, command: dict) -> None:
encoded = self._encode(command)
mediator_logger.info('StdTransport SENDING: %s', command)
self._out.write(encoded['length'])
self._out.write(encoded['content'])
self._out.flush()
mediator_logger.info('StdTransport SENDING DONE: %s', command)
def _setup_routes(self) -> None:
mediator_logger.info('Starting mediator http server on %s:%s pid=%s',
self.host, self.port, self.pid)
self.app.register_error_handler(ConnectionError, self.error_handler)
self.app.register_error_handler(TimeoutError, self.error_handler)
self.app.register_error_handler(ValueError, self.error_handler)
self.app.register_error_handler(TransportError, self.error_handler)
self.app.route('/', methods=['GET'])(self.route_index)
self.app.route('/shutdown', methods=['GET'])(self.shutdown)
self.app.route('/list_tabs', methods=['GET'])(self.list_tabs)
self.app.route('/query_tabs/<query_info>',
methods=['GET'])(self.query_tabs)
self.app.route('/move_tabs/<query_info>',
methods=['GET'])(self.move_tabs)
self.app.route('/open_urls/<int:window_id>',
methods=['POST'])(self.open_urls)
self.app.route('/open_urls', methods=['POST'])(self.open_urls)
self.app.route('/close_tabs/<tab_ids>',
methods=['GET'])(self.close_tabs)
self.app.route('/new_tab/<query>', methods=['GET'])(self.new_tab)
self.app.route('/activate_tab/<int:tab_id>',
methods=['GET'])(self.activate_tab)
self.app.route('/get_active_tabs',
methods=['GET'])(self.get_active_tabs)
self.app.route('/get_words/<string:tab_id>',
methods=['GET'])(self.get_words)
self.app.route('/get_words', methods=['GET'])(self.get_words)
self.app.route('/get_text', methods=['GET'])(self.get_text)
self.app.route('/get_html', methods=['GET'])(self.get_html)
self.app.route('/get_pid', methods=['GET'])(self.get_pid)
self.app.route('/get_browser', methods=['GET'])(self.get_browser)
def activate_tab(self, tab_id: int, focused: bool):
mediator_logger.info('activating tab id: %s', tab_id)
command = {
'name': 'activate_tab',
'tab_id': tab_id,
'focused': focused
}
self._transport.send(command)
def close_tabs(self, tab_ids: str):
"""
:param tab_ids: Comma-separated list of tab IDs to close.
"""
int_tab_ids = [int(id_) for id_ in tab_ids.split(',')]
mediator_logger.info('closing tab ids: %s', int_tab_ids)
command = {'name': 'close_tabs', 'tab_ids': int_tab_ids}
self._transport.send(command)
return self._transport.recv()
def get_words(self, tab_id: str, match_regex: str, join_with: str):
mediator_logger.info('getting tab words: %s', tab_id)
command = {
'name': 'get_words',
'tab_id': tab_id,
'match_regex': match_regex,
'join_with': join_with,
}
self._transport.send(command)
return self._transport.recv()
def get_words(self, tab_id=None):
tab_id = int(tab_id) if is_valid_integer(tab_id) else None
match_regex = request.args.get('match_regex',
DEFAULT_GET_WORDS_MATCH_REGEX)
join_with = request.args.get('join_with', DEFAULT_GET_WORDS_JOIN_WITH)
words = self.remote_api.get_words(tab_id, decode_query(match_regex),
decode_query(join_with))
mediator_logger.info(
'words for tab_id %s (match_regex %s, join_with %s): %s', tab_id,
match_regex, join_with, words)
return '\n'.join(words)
def recv(self) -> dict:
mediator_logger.info('StdTransport RECEIVING')
raw_length = self._in.read(4)
if len(raw_length) == 0:
raise TransportError(
'StdTransport: cannot read, raw_length is empty')
message_length = struct.unpack('@I', raw_length)[0]
message = self._in.read(message_length).decode('utf8')
mediator_logger.info('StdTransport RECEIVED: %s',
message.encode('utf8'))
return json.loads(message)
def _watcher(self, running: Callable, parent_pid: int, interval: float) -> None:
mediator_logger.info('Watching parent process parent=%s current pid=%s',
parent_pid, os.getpid())
while True:
time.sleep(interval)
if not running(): # someone shutdown mediator, let's bail
break
if not pid_exists(parent_pid):
mediator_logger.info('Parent process died pid=%s, shutting down mediator', parent_pid)
self.shutdown(join=False)
break
def get_html(self, delimiter_regex: str, replace_with: str):
mediator_logger.info(
'getting html, delimiter_regex=%s, replace_with=%s',
delimiter_regex, replace_with)
command = {
'name': 'get_html',
'delimiter_regex': delimiter_regex,
'replace_with': replace_with,
}
self._transport.send(command)
return self._transport.recv()
def open_urls(self, urls: List[str], window_id=None):
"""
Open specified list of URLs in a window, specified by window_id.
If window_id is None, currently active window is used.
"""
mediator_logger.info('open urls: %s', urls)
command = {'name': 'open_urls', 'urls': urls}
if window_id is not None:
command['window_id'] = window_id
self._transport.send(command)
return self._transport.recv()
def move_tabs(self, move_triplets: str):
"""
:param move_triplets: Comma-separated list of:
<tabID> <windowID> <newIndex>
"""
mediator_logger.info('move_tabs, move_triplets: %s', move_triplets)
triplets = [
list(map(int, triplet.split(' ')))
for triplet in move_triplets.split(',')
]
mediator_logger.info('moving tab ids: %s', triplets)
command = {'name': 'move_tabs', 'move_triplets': triplets}
self._transport.send(command)
return self._transport.recv()
def __init__(self, prefix='a', port=None, remote_api=None):
mediator_logger.info('starting mediator pid=%s', os.getpid())
self.prefix = prefix
self.port = get_available_tcp_port() if port is None else port
input_r, input_w = os.pipe()
output_r, output_w = os.pipe()
self.transport_browser = transport_with_timeout(
output_r, input_w, 0.050)
self.transport_mediator = transport_with_timeout(
input_r, output_w, 0.050)
self.remote_api = default_remote_api(
self.transport_mediator) if remote_api is None else remote_api
self.server = MediatorHttpServer(DEFAULT_HTTP_IFACE,
self.port,
self.remote_api,
poll_interval=0.050)
self.thread = None
self.api = None
def run_as_child_process(self):
port = get_available_tcp_port()
mediator_logger.info('starting test pid=%s', os.getpid())
def run_threaded_mediator():
mediator = MockedPiperMediator(port=port)
mediator.start()
sig.setup(lambda: mediator.server.run.shutdown(join=False))
mediator.join()
mediator_process = Process(target=run_threaded_mediator)
mediator_process.start()
api = api_must_ready(port, 'mocked')
yield mediator_process
self.assertTrue(Waiter(api.pid_not_ready).wait(timeout=1.0))
mediator_process.join()
def mediator_main():
monkeypatch_socket_bind_allow_port_reuse()
disable_click_echo()
port_range = list(get_mediator_ports())
# transport = transport_with_timeout(sys.stdin.buffer, sys.stdout.buffer, DEFAULT_TRANSPORT_TIMEOUT)
transport = transport_with_timeout(sys.stdin.buffer, sys.stdout.buffer, 1.0)
remote_api = default_remote_api(transport)
host = DEFAULT_HTTP_IFACE
poll_interval = DEFAULT_SHUTDOWN_POLL_INTERVAL
for port in port_range:
mediator_logger.info('Starting mediator on %s:%s...', host, port)
if is_port_accepting_connections(port):
continue
try:
server = MediatorHttpServer(host, port, remote_api, poll_interval)
thread = server.run.in_thread()
sig.setup(lambda: server.run.shutdown(join=False))
server.run.parent_watcher(interval=1.0)
thread.join()
mediator_logger.info('Exiting mediator pid=%s on %s:%s...', os.getpid(), host, port)
break
except OSError as e:
# TODO: fixme: we won't get this if we run in a process
mediator_logger.info('Cannot bind on port %s: %s', port, e)
except BrokenPipeError as e:
# TODO: probably also won't work with processes, also a race
mediator_logger.exception('Pipe has been closed (%s)', e)
server.run.shutdown(join=True)
break
else:
mediator_logger.error('No TCP ports available for bind in range %s', port_range)
def test_when_parent_died(self):
port = get_available_tcp_port()
mediator_logger.info('starting test pid=%s', os.getpid())
kill_parent = Barrier(2)
def run_threaded_mediator():
mediator = MockedPiperMediator(port=port)
mediator.start()
mediator.join()
def run_doomed_parent_browser():
mediator_logger.info('doomed_parent_browser pid=%s', os.getpid())
mediator_process = Process(target=run_threaded_mediator)
mediator_process.start()
mediator_process.join()
def on_sig_child(signum, frame):
pid, status = os.wait()
mediator_logger.info('reaped child signum=%s pid=%s status=%s',
signum, pid, status)
def run_supervisor():
signal.signal(signal.SIGCHLD, on_sig_child)
doomed_parent_browser = Process(target=run_doomed_parent_browser)
doomed_parent_browser.start()
kill_parent.wait()
doomed_parent_browser.terminate()
doomed_parent_browser.join()
signal.signal(signal.SIGCHLD, on_sig_child)
supervisor = Process(target=run_supervisor)
supervisor.start()
api = api_must_ready(port, 'mocked')
# kill parent and expect mediator to terminate as well
kill_parent.wait()
self.assertTrue(Waiter(api.pid_not_ready).wait(timeout=1.0))
supervisor.join()
def shutdown(join: bool):
mediator_logger.info('Closing mediator http server on %s:%s', host,
port)
self.http_server.server_close()
mediator_logger.info('Shutting down mediator http server on %s:%s',
host, port)
thread = Thread(target=self.http_server.shutdown)
thread.daemon = True
thread.start()
if join:
thread.join()
mediator_logger.info(
'Done shutting down mediator (is_alive=%s) http server on %s:%s',
thread.is_alive(), host, port)
def new_tab(self, query: str):
url = "https://www.google.com/search?q=%s" % quote_plus(query)
mediator_logger.info('opening url: %s', url)
command = {'name': 'new_tab', 'url': url}
self._transport.send(command)
return self._transport.recv()
def run_doomed_parent_browser():
mediator_logger.info('doomed_parent_browser pid=%s', os.getpid())
mediator_process = Process(target=run_threaded_mediator)
mediator_process.start()
mediator_process.join()
def get_browser(self):
mediator_logger.info('getting browser name')
return self.remote_api.get_browser()
def here(self) -> None:
mediator_logger.info('Started mediator process, pid=%s', os.getpid())
disable_logging()
return self._serve()
def handler(signum, _frame):
mediator_logger.info('Got signal %s', signum)
shutdown()
def my_socket_bind(self, *args, **kwargs):
mediator_logger.info('Custom bind called: %s, %s', args, kwargs)
self.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
return socket.socket._bind(self, *args, **kwargs)
def serve():
mediator_logger.info('Serving mediator on %s:%s', host, port)
self.http_server.serve_forever(poll_interval=poll_interval)
def get_active_tabs(self) -> str:
mediator_logger.info('getting active tabs')
command = {'name': 'get_active_tabs'}
self._transport.send(command)
return self._transport.recv()
def get_browser(self):
mediator_logger.info('getting browser name')
command = {'name': 'get_browser'}
self._transport.send(command)
return self._transport.recv()
def shutdown(self, join: bool) -> None:
# TODO: break this to test ctrl-c
if not self._shutdown:
raise NotStarted('start the runner first')
mediator_logger.info('Runner: calling terminate (pid=%s): %s', os.getpid(), self._shutdown)
self._shutdown(join)
def query_tabs(self, query_info: str):
mediator_logger.info('query info: %s', query_info)
command = {'name': 'query_tabs', 'query_info': query_info}
self._transport.send(command)
return self._transport.recv()
def on_sig_child(signum, frame):
pid, status = os.wait()
mediator_logger.info('reaped child signum=%s pid=%s status=%s',
signum, pid, status)
def get_pid(self):
mediator_logger.info('getting pid')
return str(os.getpid())
