def configure(args): args += ['--google-host', 'goagent-google-ip.fqrouter.com'] if config.read().get('youtube_scrambler_enabled', True): args += ['--enable-youtube-scrambler'] if config.read().get('goagent_public_servers_enabled', True): args += ['--proxy', 'dynamic,n=10,type=goagent,dns_record=goagent#n#.fqrouter.com,priority=1'] for server in config.list_goagent_private_servers(): proxy_config = 'goagent,appid=%s,path=%s,password=%s' % (server['appid'], server['path'], server['password']) args += ['--proxy', proxy_config] if config.read().get('shadowsocks_public_servers_enabled', True): args += ['--proxy', 'dynamic,n=4,type=ss,dns_record=ss#n#.fqrouter.com,priority=3'] for server in config.list_shadowsocks_private_servers(): proxy_config = 'ss,proxy_ip=%s,proxy_port=%s,password=%s,encrypt_method=%s' % ( server['host'], server['port'], server['password'], server['encryption_method']) args += ['--proxy', proxy_config] if config.read().get('http_proxy_public_servers_enabled', True): args += ['--proxy', 'dynamic,n=20,dns_record=proxy#n#.fqrouter.com,is_public=True,priority=4'] args += ['--proxy', 'dynamic,n=5,dns_record=proxy2#n#.fqrouter.com,priority=2'] for server in config.list_http_proxy_private_servers(): is_secured = 'True' if 'ssl' == server['transport_type'] else 'False' proxy_config = 'proxy_ip=%s,proxy_port=%s,username=%s,password=%s,is_secured=%s' % \ (server['host'], server['port'], server['username'], server['password'], is_secured) if 'http only' == server['traffic_type']: args += ['--proxy', 'http-relay,%s' % proxy_config] elif 'https only' == server['traffic_type']: args += ['--proxy', 'http-connect,%s' % proxy_config] else: args += ['--proxy', 'http-relay,%s' % proxy_config] args += ['--proxy', 'http-connect,%s' % proxy_config] for server in config.list_ssh_private_servers(): proxy_config = 'proxy_ip=%s,proxy_port=%s,username=%s,password=%s' % \ (server['host'], server['port'], server['username'], server['password']) for i in range(server['connections_count']): args += ['--proxy', 'ssh,%s' % proxy_config] return args
def handle_free_internet_connect(environ, start_response): components = [comp_dns, comp_scrambler, comp_proxy, comp_shortcut] if not config.read().get('tcp_scrambler_enabled', True): LOGGER.info('scrambler component disabled by config') components.remove(comp_scrambler) if not config.read().get('china_shortcut_enabled', True): LOGGER.info('shortcut component disabled by config china_short_enabled') components.remove(comp_shortcut) if not config.read().get('direct_access_enabled', True): LOGGER.info('shortcut component disabled by config direct_access_enabled') if comp_shortcut in components: components.remove(comp_shortcut) start_components(*components) start_response(httplib.OK, [('Content-Type', 'text/plain')]) return []
def run(args): # Load configuration conf = config.read(args.config, "chart", ChartConfig) conf.logger_prefix = "> " # Load videos and their data videos = saved_data.read_videos(conf) # Apply video filters if len(conf.filter_videos_ids) > 0: videos = [ video for video in videos if video["id"]["videoId"] in conf.filter_videos_ids ] if len(conf.filter_videos_titles) > 0: videos = [ video for video in videos if video["snippet"]["title"] in conf.filter_videos_titles ] conf.title_colors = [(title, color) for title, color in conf.title_colors if title in conf.filter_videos_titles] # Run compute() function in the proper module module = importlib.import_module(f"commands.chart.charts.{args.type}") module.__getattribute__("compute")(conf, videos)
def run(): skipped_components = [] LOGGER.info('environment: %s' % os.environ.items()) if not config.read().getboolean('fqrouter', 'BypassDirectlyEnabled'): LOGGER.info('scrambler component disabled by config') COMPONENTS.remove(comp_scrambler) for comp in COMPONENTS: try: shutdown_hook.add(comp.stop) handlers = comp.start() for method, url, handler in handlers or []: HANDLERS[(method, url)] = handler LOGGER.info('started component: %s' % comp.__name__) except: LOGGER.exception('failed to start component: %s' % comp.__name__) comp.stop() if getattr(comp, '__MANDATORY__', False): raise skipped_components.append(comp.__name__) LOGGER.info('all components started except: %s' % skipped_components) try: httpd = WSGIServer(('127.0.0.1', 8318), handle_request) LOGGER.info('serving HTTP on port 8318...') except: LOGGER.exception('failed to start HTTP server on port 8318') sys.exit(1) httpd.serve_forever()
def configure(args): args += ['--google-host', 'goagent-google-ip.fqrouter.com'] if config.read().get('full_google_play_enabled', True): args += ['--black-ip', 'android.clients.google.com'] if not config.read().get('auto_access_check_enabled', True): args += ['--disable-access-check'] if not config.read().get('china_shortcut_enabled', True): args += ['--disable-china-shortcut'] if not config.read().get('direct_access_enabled', True): args += ['--disable-direct-access'] if config.read().get('youtube_scrambler_enabled', True): args += ['--enable-youtube-scrambler'] if config.read().get('goagent_public_servers_enabled', True): args += ['--proxy', 'dynamic,n=10,type=goagent,dns_record=goagent#n#.fqrouter.com,priority=1'] for server in config.list_goagent_private_servers(): proxy_config = 'goagent,appid=%s,path=%s,password=%s' % (server['appid'], server['path'], server['password']) args += ['--proxy', proxy_config] if config.read().get('shadowsocks_public_servers_enabled', True): args += ['--proxy', 'dynamic,n=7,type=ss,dns_record=ss#n#.fqrouter.com,priority=2'] for server in config.list_shadowsocks_private_servers(): proxy_config = 'ss,proxy_ip=%s,proxy_port=%s,password=%s,encrypt_method=%s' % ( server['host'], server['port'], server['password'], server['encryption_method']) args += ['--proxy', proxy_config] if config.read().get('http_proxy_public_servers_enabled', True): args += ['--proxy', 'dynamic,n=20,dns_record=proxy#n#.fqrouter.com,is_public=True,priority=4'] args += ['--proxy', 'dynamic,n=5,dns_record=proxy2#n#.fqrouter.com,priority=2'] for server in config.list_http_proxy_private_servers(): if 'spdy (webvpn)' == server['transport_type']: proxy_config = 'proxy_ip=%s,proxy_port=%s,username=%s,password=%s,requested_spdy_version=%s' % \ (server['host'], server['port'], server['username'], server['password'], server['spdy_version']) for i in range(server['spdy_connections_count']): if 'http only' == server['traffic_type']: args += ['--proxy', 'spdy-relay,%s' % proxy_config] elif 'https only' == server['traffic_type']: args += ['--proxy', 'spdy-connect,%s' % proxy_config] else: args += ['--proxy', 'spdy-relay,%s' % proxy_config] args += ['--proxy', 'spdy-connect,%s' % proxy_config] else: is_secured = 'True' if 'ssl' == server['transport_type'] else 'False' proxy_config = 'proxy_ip=%s,proxy_port=%s,username=%s,password=%s,is_secured=%s' % \ (server['host'], server['port'], server['username'], server['password'], is_secured) if 'http only' == server['traffic_type']: args += ['--proxy', 'http-relay,%s' % proxy_config] elif 'https only' == server['traffic_type']: args += ['--proxy', 'http-connect,%s' % proxy_config] else: args += ['--proxy', 'http-relay,%s' % proxy_config] args += ['--proxy', 'http-connect,%s' % proxy_config] for server in config.list_ssh_private_servers(): proxy_config = 'proxy_ip=%s,proxy_port=%s,username=%s,password=%s' % \ (server['host'], server['port'], server['username'], server['password']) for i in range(server['connections_count']): args += ['--proxy', 'ssh,%s' % proxy_config] return args
def handle_start(environ, start_response): cfg = config.read() ssid = str(cfg['wifi_hotspot_ssid']) password = str(cfg['wifi_hotspot_password']) success, message = start_hotspot(ssid, password) status = httplib.OK if success else httplib.BAD_GATEWAY start_response(status, [('Content-Type', 'text/plain')]) yield message
def handle_free_internet_connect(environ, start_response): components = [comp_dns, comp_scrambler, comp_proxy, comp_shortcut] if not config.read()['tcp_scrambler_enabled']: LOGGER.info('scrambler component disabled by config') components.remove(comp_scrambler) start_components(*components) start_response(httplib.OK, [('Content-Type', 'text/plain')]) return []
def handle_start(environ, start_response): cfg = config.read() ssid = cfg.get('fqrouter', 'WifiHotspotSSID') password = cfg.get('fqrouter', 'WifiHotspotPassword') success, message = start_hotspot(ssid, password) status = httplib.OK if success else httplib.BAD_GATEWAY start_response(status, [('Content-Type', 'text/plain')]) yield message
def configure(args): args += ['--google-host', 'goagent-google-ip.fqrouter.com'] args += ['--google-host', 'goagent-google-ip2.fqrouter.com'] if not config.read().get('auto_access_check_enabled', True): args += ['--disable-access-check'] if not config.read().get('china_shortcut_enabled', True): args += ['--disable-china-shortcut'] if not config.read().get('direct_access_enabled', True): args += ['--disable-direct-access'] if config.read().get('youtube_scrambler_enabled', True): args += ['--enable-youtube-scrambler'] public_server_types = [] if config.read().get('goagent_public_servers_enabled', True): public_server_types.append('goagent') if config.read().get('shadowsocks_public_servers_enabled', True): public_server_types.append('ss') if config.read().get('http_proxy_public_servers_enabled', True): public_server_types.append('http-connect') public_server_types.append('http-relay') public_server_types.append('spdy-connect') public_server_types.append('spdy-relay') if public_server_types: args += ['--proxy', 'directory,src=proxies.fqrouter.com,%s' % ','.join(['%s=True' % t for t in public_server_types])] for server in config.list_goagent_private_servers(): proxy_config = 'goagent,appid=%s,path=%s,password=%s' % (server['appid'], server['path'], server['password']) args += ['--proxy', proxy_config] for server in config.list_shadowsocks_private_servers(): proxy_config = 'ss,proxy_host=%s,proxy_port=%s,password=%s,encrypt_method=%s' % ( server['host'], server['port'], server['password'], server['encryption_method']) args += ['--proxy', proxy_config] for server in config.list_http_proxy_private_servers(): if 'spdy (webvpn)' == server['transport_type']: proxy_config = 'proxy_host=%s,proxy_port=%s,username=%s,password=%s,requested_spdy_version=%s' % \ (server['host'], server['port'], server['username'], server['password'], server['spdy_version']) for i in range(server['spdy_connections_count']): if 'http only' == server['traffic_type']: args += ['--proxy', 'spdy-relay,%s' % proxy_config] elif 'https only' == server['traffic_type']: args += ['--proxy', 'spdy-connect,%s' % proxy_config] else: args += ['--proxy', 'spdy-relay,%s' % proxy_config] args += ['--proxy', 'spdy-connect,%s' % proxy_config] else: is_secured = 'True' if 'ssl' == server['transport_type'] else 'False' proxy_config = 'proxy_host=%s,proxy_port=%s,username=%s,password=%s,is_secured=%s' % \ (server['host'], server['port'], server['username'], server['password'], is_secured) if 'http only' == server['traffic_type']: args += ['--proxy', 'http-relay,%s' % proxy_config] elif 'https only' == server['traffic_type']: args += ['--proxy', 'http-connect,%s' % proxy_config] else: args += ['--proxy', 'http-relay,%s' % proxy_config] args += ['--proxy', 'http-connect,%s' % proxy_config] for server in config.list_ssh_private_servers(): proxy_config = 'proxy_host=%s,proxy_port=%s,username=%s,password=%s' % \ (server['host'], server['port'], server['username'], server['password']) for i in range(server['connections_count']): args += ['--proxy', 'ssh,%s' % proxy_config] return args
def start_fqsocks(): global fqsocks_process args = [ '--log-level', 'INFO', '--log-file', '/data/data/fq.router2/log/fqsocks.log', '--outbound-ip', '10.1.2.3', # send from 10.1.2.3 so we can skip redirecting those traffic '--listen', '10.1.2.3:8319' ] if config.read().get('tcp_scrambler_enabled', True): args += ['--http-request-mark', '0xbabe'] # trigger scrambler args = configure(args) fqsocks_process = shell.launch_python('fqsocks', args, on_exit=stop)
def handle_free_internet_connect(environ, start_response): components = [comp_dns, comp_scrambler, comp_proxy, comp_shortcut] if not config.read().get('comp_dns_enabled', True): LOGGER.info('dns component disabled by config') components.remove(comp_dns) if not config.read().get('comp_proxy_enabled', True): LOGGER.info('proxy component disabled by config') components.remove(comp_proxy) if not config.read().get('tcp_scrambler_enabled', True): LOGGER.info('scrambler component disabled by config') components.remove(comp_scrambler) if not config.read().get('china_shortcut_enabled', True): LOGGER.info('shortcut component disabled by config china_short_enabled') components.remove(comp_shortcut) if not config.read().get('direct_access_enabled', True): LOGGER.info('shortcut component disabled by config direct_access_enabled') if comp_shortcut in components: components.remove(comp_shortcut) start_components(*components) start_response(httplib.OK, [('Content-Type', 'text/plain')]) return []
def run(): iptables.init_fq_chains() shutdown_hook.add(iptables.flush_fq_chain) if config.read().get('comp_wifi_enabled', True): start_components(comp_wifi, comp_lan) else: LOGGER.info('wifi component disabled by config') comp_wifi.setup_lo_alias() start_components(comp_lan) httpd.HANDLERS[('GET', 'ping')] = handle_ping httpd.HANDLERS[('POST', 'free-internet/connect')] = handle_free_internet_connect httpd.HANDLERS[('POST', 'free-internet/disconnect')] = handle_free_internet_disconnect httpd.HANDLERS[('GET', 'free-internet/is-connected')] = handle_free_internet_is_connected httpd.serve_forever()
def run(): iptables.init_fq_chains() shutdown_hook.add(iptables.flush_fq_chain) if config.read().get('comp_wifi_enabled', True): start_components(comp_wifi, comp_lan) else: LOGGER.info('wifi component disabled by config') comp_wifi.setup_lo_alias() start_components(comp_lan) httpd.HANDLERS[('GET', 'ping')] = handle_ping httpd.HANDLERS[('POST', 'free-internet/connect')] = handle_free_internet_connect httpd.HANDLERS[('POST', 'free-internet/disconnect')] = handle_free_internet_disconnect httpd.HANDLERS[('GET', 'free-internet/is-connected')] = handle_free_internet_is_connected gevent.spawn(check_ping) httpd.serve_forever()
def start_fqsocks(): global fqsocks_process args = [ "--log-level", "INFO", "--log-file", "/data/data/fq.router2/log/fqsocks.log", "--outbound-ip", "10.1.2.3", # send from 10.1.2.3 so we can skip redirecting those traffic "--listen", "10.1.2.3:8319", ] if config.read().get("tcp_scrambler_enabled", True): args += ["--http-request-mark", "0xbabe"] # trigger scrambler args = configure(args) fqsocks_process = shell.launch_python("fqsocks", args, on_exit=stop)
def run(): skipped_components = [] LOGGER.info('environment: %s' % os.environ.items()) if not config.read()['tcp_scrambler_enabled']: LOGGER.info('scrambler component disabled by config') COMPONENTS.remove(comp_scrambler) for comp in COMPONENTS: try: shutdown_hook.add(comp.stop) handlers = comp.start() for method, url, handler in handlers or []: httpd.HANDLERS[(method, url)] = handler LOGGER.info('started component: %s' % comp.__name__) except: LOGGER.exception('failed to start component: %s' % comp.__name__) comp.stop() if getattr(comp, '__MANDATORY__', False): raise skipped_components.append(comp.__name__) LOGGER.info('all components started except: %s' % skipped_components) httpd.HANDLERS[('GET', 'ping')] = handle_ping httpd.serve_forever()
def read(secret, settings=None): """Returns a value of a key from Vault secret or configuration file. The input secret is a dictionary which contains the following fields: * path - path to the secret in Vault or config * field - the key to read from the secret * format (optional) - plain or base64 (defaults to plain) * version (optional) - Vault secret version to read * Note: if this is Vault secret and a v2 KV engine The input settings is an optional app-interface-settings object queried from app-interface. It is a dictionary containing `value: true` if Vault is to be used as the secret backend. Default vault setting is false, to allow using a config file without creating app-interface-settings. """ if settings and settings.get('vault'): return vault_client.read(secret) else: return config.read(secret)
def configure(args): args += ["--google-host", "goagent-google-ip.fqrouter.com"] if config.read().get("full_google_play_enabled", True): args += ["--black-ip", "android.clients.google.com"] if not config.read().get("auto_access_check_enabled", True): args += ["--disable-access-check"] if not config.read().get("china_shortcut_enabled", True): args += ["--disable-china-shortcut"] if not config.read().get("direct_access_enabled", True): args += ["--disable-direct-access"] if config.read().get("youtube_scrambler_enabled", True): args += ["--enable-youtube-scrambler"] if config.read().get("goagent_public_servers_enabled", True): args += ["--proxy", "dynamic,n=10,type=goagent,dns_record=goagent#n#.fqrouter.com,priority=1"] for server in config.list_goagent_private_servers(): proxy_config = "goagent,appid=%s,path=%s,password=%s" % (server["appid"], server["path"], server["password"]) args += ["--proxy", proxy_config] if config.read().get("shadowsocks_public_servers_enabled", True): args += ["--proxy", "dynamic,n=7,type=ss,dns_record=ss#n#.fqrouter.com,priority=2"] for server in config.list_shadowsocks_private_servers(): proxy_config = "ss,proxy_host=%s,proxy_port=%s,password=%s,encrypt_method=%s" % ( server["host"], server["port"], server["password"], server["encryption_method"], ) args += ["--proxy", proxy_config] if config.read().get("http_proxy_public_servers_enabled", True): args += ["--proxy", "dynamic,n=20,dns_record=proxy#n#.fqrouter.com,is_public=True,priority=4"] args += ["--proxy", "dynamic,n=5,dns_record=proxy2#n#.fqrouter.com,priority=2"] for server in config.list_http_proxy_private_servers(): if "spdy (webvpn)" == server["transport_type"]: proxy_config = "proxy_host=%s,proxy_port=%s,username=%s,password=%s,requested_spdy_version=%s" % ( server["host"], server["port"], server["username"], server["password"], server["spdy_version"], ) for i in range(server["spdy_connections_count"]): if "http only" == server["traffic_type"]: args += ["--proxy", "spdy-relay,%s" % proxy_config] elif "https only" == server["traffic_type"]: args += ["--proxy", "spdy-connect,%s" % proxy_config] else: args += ["--proxy", "spdy-relay,%s" % proxy_config] args += ["--proxy", "spdy-connect,%s" % proxy_config] else: is_secured = "True" if "ssl" == server["transport_type"] else "False" proxy_config = "proxy_host=%s,proxy_port=%s,username=%s,password=%s,is_secured=%s" % ( server["host"], server["port"], server["username"], server["password"], is_secured, ) if "http only" == server["traffic_type"]: args += ["--proxy", "http-relay,%s" % proxy_config] elif "https only" == server["traffic_type"]: args += ["--proxy", "http-connect,%s" % proxy_config] else: args += ["--proxy", "http-relay,%s" % proxy_config] args += ["--proxy", "http-connect,%s" % proxy_config] for server in config.list_ssh_private_servers(): proxy_config = "proxy_host=%s,proxy_port=%s,username=%s,password=%s" % ( server["host"], server["port"], server["username"], server["password"], ) for i in range(server["connections_count"]): args += ["--proxy", "ssh,%s" % proxy_config] return args
def run(args): # Load configuration conf = config.read(args.config, "stats", StatsConfig) conf.logger_prefix = "> " # Load videos and their data logger.info("Load video's data") videos = saved_data.read_videos(conf) # Compute statistics logger.info("Compute statistics") number_of_videos = len(videos) number_of_words = 0 wte_per_video = {} most_amount_in_video = 0 most_amount_in_video_name = "" most_average_per_seconds = 1000000 most_average_per_seconds_name = "" most_average_per_words = 1000000 most_average_per_words_name = "" words = {} for video, (_, word, _), pos, counter in clips.list_for(videos, var="timestamps"): video_id = video["id"]["videoId"] video_title = video["snippet"]["title"] number_of_words += 1 words[word] = words.get(word, 0) + 1 timestamps = video.get("data", {}).get("timestamps", []) time = video.get("data", {}).get("time", None) if time is not None: video_time = convert.str_to_sec(time) else: video_time = 0 if re.match(conf.word_to_extract, word): wte_per_video[video_id] = wte_per_video.get(video_id, 0) + 1 if wte_per_video[video_id] > most_amount_in_video: most_amount_in_video = wte_per_video[video_id] most_amount_in_video_name = video_title average_per_seconds = video_time / wte_per_video[video_id] average_per_words = len(timestamps) / wte_per_video[video_id] if average_per_seconds < most_average_per_seconds: most_average_per_seconds = average_per_seconds most_average_per_seconds_name = video_title if average_per_words < most_average_per_words: most_average_per_words = average_per_words most_average_per_words_name = video_title average_wte_per_video = 0 if number_of_words == 0 else sum( wte_per_video.values()) / number_of_words # Sort from high amount to low amount words_sorted = sorted([(amount, word) for word, amount in words.items()], reverse=True) words_first_10 = ", ".join( [f"{word} ({amount} times)" for amount, word in words_sorted[:10]]) number_of_wte = 0 position_of_wte = "" for i, (amount, word) in enumerate(words_sorted): if re.match(conf.word_to_extract, word): number_of_wte += amount if position_of_wte == "": position_of_wte = str(i + 1) else: position_of_wte += f", {str(i + 1)}" # Display statistics logger.info(f"{'=' * 21} Statistics {'=' * 21}") logger.info(f"Number of videos: {number_of_videos}") logger.info(f"Number of words: {number_of_words}") logger.info(f"First 10 words: {words_first_10}") logger.info(f"Amount: {number_of_wte}", prefix=f"{conf.word_to_extract} >> ") logger.info(f"Position: {position_of_wte}", prefix=f"{conf.word_to_extract} >> ") logger.info(f"Average per video: {average_wte_per_video}", prefix=f"{conf.word_to_extract} >> ") logger.info( f"Most amount in a video: {most_amount_in_video} times in '{most_amount_in_video_name}'", prefix=f"{conf.word_to_extract} >> ", ) logger.info( f"Most per seconds in a video: every {most_average_per_seconds} second times in '{most_average_per_seconds_name}'", prefix=f"{conf.word_to_extract} >> ", ) logger.info( f"Most per word in a video: every {most_average_per_words} word in '{most_average_per_words_name}'", prefix=f"{conf.word_to_extract} >> ", )