def test_index_no_arguments_ok(self):
self.mediator.transport.received.extend([
'mocked',
['1.1\ttitle\turl\tbody'],
])
sqlite_filename = in_temp_dir('tabs.sqlite')
tsv_filename = in_temp_dir('tabs.tsv')
assert_file_absent(sqlite_filename)
assert_file_absent(tsv_filename)
output = []
with patch('brotab.main.stdout_buffer_write', output.append):
self._run_commands(['index'])
assert self.mediator.transport.sent == [
{
'name': 'get_browser'
},
{
'delimiter_regex': '/\\n|\\r|\\t/g',
'name': 'get_text',
'replace_with': '" "'
},
]
assert_file_not_empty(sqlite_filename)
assert_file_not_empty(tsv_filename)
assert_file_contents(tsv_filename, 'a.1.1\ttitle\turl\tbody\n')
assert_sqlite3_table_contents(sqlite_filename, 'tabs',
'a.1.1\ttitle\turl\tbody')
def test_index_custom_filename(self):
self.mediator.transport.received.extend([
'mocked',
['1.1\ttitle\turl\tbody'],
])
sqlite_filename = in_temp_dir(uuid4().hex + '.sqlite')
tsv_filename = in_temp_dir(uuid4().hex + '.tsv')
assert_file_absent(sqlite_filename)
assert_file_absent(tsv_filename)
spit(tsv_filename, 'a.1.1\ttitle\turl\tbody\n')
output = []
with patch('brotab.main.stdout_buffer_write', output.append):
self._run_commands(
['index', '--sqlite', sqlite_filename, '--tsv', tsv_filename])
assert self.mediator.transport.sent == []
assert_file_not_empty(sqlite_filename)
assert_file_not_empty(tsv_filename)
assert_file_contents(tsv_filename, 'a.1.1\ttitle\turl\tbody\n')
assert_sqlite3_table_contents(sqlite_filename, 'tabs',
'a.1.1\ttitle\turl\tbody')
assert_file_absent(sqlite_filename)
assert_file_absent(tsv_filename)
def index_tabs(args):
if args.tsv is None:
args.tsv = in_temp_dir('tabs.tsv')
args.cleanup = True
brotab_logger.info(
'index_tabs: retrieving tabs from browser into file %s', args.tsv)
start = time.time()
get_text(args)
delta = time.time() - start
brotab_logger.info('getting text took %s', delta)
start = time.time()
index(args.sqlite, args.tsv)
delta = time.time() - start
brotab_logger.info('sqlite create took %s, size %s',
delta, get_file_size(args.sqlite))
import logging
from string import ascii_lowercase
from typing import Tuple, List
from brotab.inout import is_port_accepting_connections
from brotab.inout import get_mediator_ports
from brotab.inout import in_temp_dir
from brotab.api import SingleMediatorAPI, MultipleMediatorsAPI
FORMAT = '%(asctime)-15s %(levelname)-10s %(message)s'
logging.basicConfig(format=FORMAT,
filename=in_temp_dir('brotab.log'),
level=logging.DEBUG)
logger = logging.getLogger('brotab')
logger.info('Logger has been created')
def parse_target_hosts(target_hosts: str) -> Tuple[List[str], List[int]]:
"""
Input: localhost:2000,127.0.0.1:3000
Output: (['localhost', '127.0.0.1'], [2000, 3000])
"""
hosts, ports = [], []
for pair in target_hosts.split(','):
host, port = pair.split(':')
hosts.append(host)
ports.append(int(port))
return hosts, ports
def parse_args(args):
parser = ArgumentParser(
description='''
bt (brotab = Browser Tabs) is a command-line tool that helps you manage
browser tabs. It can help you list, close, reorder, open and activate
your tabs.
''')
parser.add_argument('--target', dest='target_hosts', default=None,
help='Target hosts IP:Port')
subparsers = parser.add_subparsers()
parser.set_defaults(func=partial(no_command, parser))
parser_move_tabs = subparsers.add_parser(
'move',
help='''
move tabs around. This command lists available tabs and runs
the editor. In the editor you can 1) reorder tabs -- tabs will
be moved in the browser 2) delete tabs -- tabs will be closed
3) change window ID of the tabs -- tabs will be moved to
specified windows
''')
parser_move_tabs.set_defaults(func=move_tabs)
parser_list_tabs = subparsers.add_parser(
'list',
help='''
list available tabs. The command will request all available clients
(browser plugins, mediators), and will display browser tabs in the
following format:
"<prefix>.<window_id>.<tab_id><Tab>Page title<Tab>URL"
''')
parser_list_tabs.set_defaults(func=list_tabs)
parser_close_tabs = subparsers.add_parser(
'close',
help='''
close specified tab IDs. Tab IDs should be in the following format:
"<prefix>.<window_id>.<tab_id>". You can use "list" command to obtain
tab IDs (first column)
''')
parser_close_tabs.set_defaults(func=close_tabs)
parser_close_tabs.add_argument('tab_ids', type=str, nargs='*',
help='Tab IDs to close')
parser_activate_tab = subparsers.add_parser(
'activate',
help='''
activate given tab ID. Tab ID should be in the following format:
"<prefix>.<window_id>.<tab_id>"
''')
parser_activate_tab.set_defaults(func=activate_tab)
parser_activate_tab.add_argument('tab_id', type=str, nargs=1,
help='Tab ID to activate')
parser_activate_tab.add_argument('--focused', action='store_const', const=True, default=None,
help='make browser focused after tab activation (default: False)')
parser_active_tab = subparsers.add_parser(
'active',
help='''
display active tab for each client/window in the following format:
"<prefix>.<window_id>.<tab_id>"
''')
parser_active_tab.set_defaults(func=show_active_tabs)
parser_search_tabs = subparsers.add_parser(
'search',
help='''
Search across your indexed tabs using sqlite fts5 plugin.
''')
parser_search_tabs.set_defaults(func=search_tabs)
parser_search_tabs.add_argument('--sqlite', type=str, default=in_temp_dir('tabs.sqlite'),
help='sqlite DB filename')
parser_search_tabs.add_argument('query', type=str, help='Search query')
parser_query_tabs = subparsers.add_parser(
'query',
help='''
Filter tabs using chrome.tabs api.
''',
prefix_chars='-+')
parser_query_tabs.set_defaults(func=query_tabs)
parser_query_tabs.add_argument('+active', action='store_const', const=True, default=None,
help='tabs are active in their windows')
parser_query_tabs.add_argument('-active', action='store_const', const=False, default=None,
help='tabs are not active in their windows')
parser_query_tabs.add_argument('+pinned', action='store_const', const=True, default=None,
help='tabs are pinned')
parser_query_tabs.add_argument('-pinned', action='store_const', const=False, default=None,
help='tabs are not pinned')
parser_query_tabs.add_argument('+audible', action='store_const', const=True, default=None,
help='tabs are audible')
parser_query_tabs.add_argument('-audible', action='store_const', const=False, default=None,
help='tabs are not audible')
parser_query_tabs.add_argument('+muted', action='store_const', const=True, default=None,
help='tabs are muted')
parser_query_tabs.add_argument('-muted', action='store_const', const=False, default=None,
help='tabs not are muted')
parser_query_tabs.add_argument('+highlighted', action='store_const', const=True, default=None,
help='tabs are highlighted')
parser_query_tabs.add_argument('-highlighted', action='store_const', const=False, default=None,
help='tabs not are highlighted')
parser_query_tabs.add_argument('+discarded', action='store_const', const=True, default=None,
help='tabs are discarded i.e. unloaded from memory but still visible in the tab strip.')
parser_query_tabs.add_argument('-discarded', action='store_const', const=False, default=None,
help='tabs are not discarded i.e. unloaded from memory but still visible in the tab strip.')
parser_query_tabs.add_argument('+autoDiscardable', action='store_const', const=True, default=None,
help='tabs can be discarded automatically by the browser when resources are low.')
parser_query_tabs.add_argument('-autoDiscardable', action='store_const', const=False, default=None,
help='tabs cannot be discarded automatically by the browser when resources are low.')
parser_query_tabs.add_argument('+currentWindow', action='store_const', const=True, default=None,
help='tabs are in the current window.')
parser_query_tabs.add_argument('-currentWindow', action='store_const', const=False, default=None,
help='tabs are not in the current window.')
parser_query_tabs.add_argument('+lastFocusedWindow', action='store_const', const=True, default=None,
help='tabs are in the last focused window.')
parser_query_tabs.add_argument('-lastFocusedWindow', action='store_const', const=False, default=None,
help='tabs are not in the last focused window.')
parser_query_tabs.add_argument('-status', type=str, choices=['loading', 'complete'],
help='whether the tabs have completed loading i.e. loading or complete.')
parser_query_tabs.add_argument('-title', type=str,
help='match page titles against a pattern.')
parser_query_tabs.add_argument('-url', type=str, action='append',
help='match tabs against one or more URL patterns. Fragment identifiers are not matched. see https://developer.chrome.com/extensions/match_patterns')
parser_query_tabs.add_argument('-windowId', type=int,
help='the ID of the parent window, or windows.WINDOW_ID_CURRENT for the current window.')
parser_query_tabs.add_argument('-windowType', type=str, choices=['normal', 'popup', 'panel', 'app', 'devtools'],
help='the type of window the tabs are in.')
parser_query_tabs.add_argument('-index', type=int,
help='the position of the tabs within their windows.')
parser_query_tabs.add_argument('-info', type=str,
help='''
the queryInfo parameter as outlined here: https://developer.chrome.com/extensions/tabs#method-query.
all other query arguments are ignored if this argument is present.
''')
parser_index_tabs = subparsers.add_parser(
'index',
help='''
Index the text from browser's tabs. Text is put into sqlite fts5 table.
''')
parser_index_tabs.set_defaults(func=index_tabs)
parser_index_tabs.add_argument('tab_ids', type=str, nargs='*',
help='Tab IDs to get text from')
parser_index_tabs.add_argument('--sqlite', type=str, default=in_temp_dir('tabs.sqlite'),
help='sqlite DB filename')
parser_index_tabs.add_argument('--tsv', type=str, default=None,
help='get text from tabs and index the results')
parser_index_tabs.add_argument(
'--delimiter-regex', type=str, default=DEFAULT_GET_TEXT_DELIMITER_REGEX,
help='Regex that is used to match delimiters in the page text')
parser_index_tabs.add_argument(
'--replace-with', type=str, default=DEFAULT_GET_TEXT_REPLACE_WITH,
help='String that is used to replaced matched delimiters')
parser_new_tab = subparsers.add_parser(
'new',
help='''
open new tab with the Google search results of the arguments that
follow. One positional argument is required:
<prefix>.<window_id> OR <client>. If window_id is not specified,
URL will be opened in the active window of the specifed client
''')
parser_new_tab.set_defaults(func=new_tab)
parser_new_tab.add_argument(
'prefix_window_id', type=str,
help='Client prefix and (optionally) window id, e.g. b.20')
parser_new_tab.add_argument('query', type=str, nargs='*',
help='Query to search for in Google')
parser_open_urls = subparsers.add_parser(
'open',
help='''
open URLs from the stdin (one URL per line). One positional argument is
required: <prefix>.<window_id> OR <client>. If window_id is not
specified, URL will be opened in the active window of the specifed
client
''')
parser_open_urls.set_defaults(func=open_urls)
parser_open_urls.add_argument(
'prefix_window_id', type=str,
help='Client prefix and (optionally) window id, e.g. b.20')
parser_get_words = subparsers.add_parser(
'words',
help='''
show sorted unique words from all active tabs of all clients. This is
a helper for webcomplete plugin that helps complete words from the
browser
''')
parser_get_words.set_defaults(func=get_words)
parser_get_words.add_argument('tab_ids', type=str, nargs='*',
help='Tab IDs to get words from')
parser_get_words.add_argument(
'--match-regex', type=str, default=DEFAULT_GET_WORDS_MATCH_REGEX,
help='Regex that is used to match words in the page text')
parser_get_words.add_argument(
'--join-with', type=str, default=DEFAULT_GET_WORDS_JOIN_WITH,
help='String that is used to join matched words')
parser_get_text = subparsers.add_parser(
'text',
help='''
show text form all tabs
''')
parser_get_text.set_defaults(func=get_text)
parser_get_text.add_argument('tab_ids', type=str, nargs='*',
help='Tab IDs to get text from')
parser_get_text.add_argument('--tsv', type=str, default=None,
help='tsv file to save results to')
parser_get_text.add_argument('--cleanup', action='store_true',
default=False,
help='force removal of extra whitespace')
parser_get_text.add_argument(
'--delimiter-regex', type=str, default=DEFAULT_GET_TEXT_DELIMITER_REGEX,
help='Regex that is used to match delimiters in the page text')
parser_get_text.add_argument(
'--replace-with', type=str, default=DEFAULT_GET_TEXT_REPLACE_WITH,
help='String that is used to replaced matched delimiters')
parser_get_html = subparsers.add_parser(
'html',
help='''
show html form all tabs
''')
parser_get_html.set_defaults(func=get_html)
parser_get_html.add_argument('tab_ids', type=str, nargs='*',
help='Tab IDs to get text from')
parser_get_html.add_argument('--tsv', type=str, default=None,
help='tsv file to save results to')
parser_get_html.add_argument('--cleanup', action='store_true',
default=False,
help='force removal of extra whitespace')
parser_get_html.add_argument(
'--delimiter-regex', type=str, default=DEFAULT_GET_HTML_DELIMITER_REGEX,
help='Regex that is used to match delimiters in the page text')
parser_get_html.add_argument(
'--replace-with', type=str, default=DEFAULT_GET_HTML_REPLACE_WITH,
help='String that is used to replaced matched delimiters')
parser_show_duplicates = subparsers.add_parser(
'dup',
help='''
display reminder on how to show duplicate tabs using command-line tools
''')
parser_show_duplicates.set_defaults(func=show_duplicates)
parser_show_windows = subparsers.add_parser(
'windows',
help='''
display available prefixes and window IDs, along with the number of
tabs in every window
''')
parser_show_windows.set_defaults(func=show_windows)
parser_show_clients = subparsers.add_parser(
'clients',
help='''
display available browser clients (mediators), their prefixes and
address (host:port), native app PIDs, and browser names
''')
parser_show_clients.set_defaults(func=show_clients)
parser_install_mediator = subparsers.add_parser(
'install',
help='''
configure browser settings to use bt mediator (native messaging app)
''')
parser_install_mediator.add_argument('--tests', action='store_true',
default=False,
help='install testing version of '
'manifest for chromium')
parser_install_mediator.set_defaults(func=install_mediator)
return parser.parse_args(args)
from brotab.inout import get_mediator_ports
from brotab.inout import is_port_accepting_connections
from brotab.inout import in_temp_dir
from brotab.const import \
DEFAULT_GET_WORDS_MATCH_REGEX, \
DEFAULT_GET_WORDS_JOIN_WITH, \
DEFAULT_GET_TEXT_DELIMITER_REGEX, \
DEFAULT_GET_TEXT_REPLACE_WITH, \
DEFAULT_GET_HTML_DELIMITER_REGEX, \
DEFAULT_GET_HTML_REPLACE_WITH
app = flask.Flask(__name__)
FORMAT = '%(asctime)-15s %(process)-5d %(levelname)-10s %(message)s'
MAX_LOG_SIZE = 50 * 1024 * 1024
LOG_FILENAME = in_temp_dir('brotab_mediator.log')
LOG_BACKUP_COUNT = 1
logger = logging.getLogger('brotab_mediator')
logger.setLevel(logging.DEBUG)
handler = logging.handlers.RotatingFileHandler(
filename=LOG_FILENAME,
maxBytes=MAX_LOG_SIZE,
backupCount=LOG_BACKUP_COUNT,
)
handler.setFormatter(logging.Formatter(FORMAT))
logger.addHandler(handler)
logger.info('Logger has been created')
DEFAULT_HTTP_IFACE = '127.0.0.1'
DEFAULT_MIN_HTTP_PORT = 4625
def init_mediator_logger(tag: str):
return _init_logger(tag, in_temp_dir('brotab_mediator.log'))
def init_brotab_logger(tag: str):
return _init_logger(tag, in_temp_dir('brotab.log'))
