def SetUp(self): """Sets up the test harness and device before all tests are run.""" super(TestRunner, self).SetUp() if not self.adb.IsRootEnabled(): logging.warning( 'Unable to enable java asserts for %s, non rooted device', self.device) else: if self.adb.SetJavaAssertsEnabled( enable=not self.disable_assertions): self.adb.Reboot(full_reboot=False) # We give different default value to launch HTTP server based on shard index # because it may have race condition when multiple processes are trying to # launch lighttpd with same port at same time. http_server_ports = self.LaunchTestHttpServer( os.path.join(constants.CHROME_DIR), (constants.LIGHTTPD_RANDOM_PORT_FIRST + self.shard_index)) if self.ports_to_forward: port_pairs = [(port, port) for port in self.ports_to_forward] # We need to remember which ports the HTTP server is using, since the # forwarder will stomp on them otherwise. port_pairs.append(http_server_ports) self.forwarder = Forwarder(self.adb, self.build_type) self.forwarder.Run(port_pairs, self.tool, '127.0.0.1') self.CopyTestFilesOnce() self.flags.AddFlags(['--enable-test-intents'])
def test_submit_v1_service_checks(): f = Forwarder("api_key", DOMAIN) f.submit_v1_service_checks("data", None) t = get_transaction(f) assert t.endpoint == "/api/v1/check_run?api_key=api_key" assert t.payload == "data"
def init_dogstatsd(config): api_key = config['api_key'] recent_point_threshold = config.get('recent_point_threshold', None) server_host = config['dogstatsd']['bind_host'] dd_url = config['dd_url'] port = config['dogstatsd']['port'] forward_to_host = config['dogstatsd'].get('forward_host') forward_to_port = config['dogstatsd'].get('forward_port') non_local_traffic = config['dogstatsd'].get('non_local_traffic') so_rcvbuf = config['dogstatsd'].get('so_rcvbuf') utf8_decoding = config['dogstatsd'].get('utf8_decoding') interval = DOGSTATSD_FLUSH_INTERVAL aggregator_interval = DOGSTATSD_AGGREGATOR_BUCKET_SIZE hostname = get_hostname() # get proxy settings proxies = get_proxy() forwarder = Forwarder( api_key, dd_url, proxies=proxies, ) forwarder.start() aggregator = MetricsBucketAggregator( hostname, aggregator_interval, recent_point_threshold=recent_point_threshold, formatter=get_formatter(config), histogram_aggregates=config.get('histogram_aggregates'), histogram_percentiles=config.get('histogram_percentiles'), utf8_decoding=utf8_decoding ) # serializer serializer = Serializer( aggregator, forwarder, ) reporter = Reporter(interval, aggregator, serializer, api_key, use_watchdog=False, hostname=hostname) # NOTICE: when `non_local_traffic` is passed we need to bind to any interface on the box. The forwarder uses # Tornado which takes care of sockets creation (more than one socket can be used at once depending on the # network settings), so it's enough to just pass an empty string '' to the library. # In Dogstatsd we use a single, fullstack socket, so passing '' as the address doesn't work and we default to # '0.0.0.0'. If someone needs to bind Dogstatsd to the IPv6 '::', they need to turn off `non_local_traffic` and # use the '::' meta address as `bind_host`. if non_local_traffic: server_host = '0.0.0.0' server = Server(aggregator, server_host, port, forward_to_host=forward_to_host, forward_to_port=forward_to_port, so_rcvbuf=so_rcvbuf) return reporter, server, forwarder
def StartForwarder(self, port_pairs): """Starts TCP traffic forwarding for the given |port_pairs|. Args: host_port_pairs: A list of (device_port, local_port) tuples to forward. """ if self._forwarder: self._forwarder.Close() self._forwarder = Forwarder(self.adb, port_pairs, self.tool, '127.0.0.1', self.build_type)
def run(self): logging.info('Start running the thread!') self.wait_event.clear() self._GenerateCommandLineArguments() command = constants.CHROME_DIR if self.arguments['server-type'] == 'sync': command = [ os.path.join(command, 'sync', 'tools', 'testserver', 'sync_testserver.py') ] + self.command_line else: command = [ os.path.join(command, 'net', 'tools', 'testserver', 'testserver.py') ] + self.command_line logging.info('Running: %s', command) self.process = subprocess.Popen(command) if self.process: if self.pipe_out: self.is_ready = self._WaitToStartAndGetPortFromTestServer() else: self.is_ready = _CheckPortStatus(self.host_port, True) if self.is_ready: self._test_server_forwarder = Forwarder(self.adb, self.build_type) self._test_server_forwarder.Run([(0, self.host_port)], self.tool, '127.0.0.1') # Check whether the forwarder is ready on the device. self.is_ready = False device_port = self._test_server_forwarder.DevicePortForHostPort( self.host_port) if device_port: for timeout in range(1, 5): if ports.IsDevicePortUsed(self.adb, device_port, 'LISTEN'): self.is_ready = True self.forwarder_device_port = device_port break time.sleep(timeout) # Wake up the request handler thread. self.ready_event.set() # Keep thread running until Stop() gets called. while not self.stop_flag: time.sleep(1) if self.process.poll() is None: self.process.kill() if self._test_server_forwarder: self._test_server_forwarder.Close() self.process = None self.is_ready = False if self.pipe_out: os.close(self.pipe_in) os.close(self.pipe_out) self.pipe_in = None self.pipe_out = None logging.info('Test-server has died.') self.wait_event.set()
def __init__(self): self.API_TOKEN = "1700885261:AAETCokNpqNDk44x3d5XASfnQfzxiNOKWfI" ## os.environ["SECRET"] self.contatinhosSheet = "http://bit.ly/contatosbcc021" self.githubRepo = "https://github.com/lineuzinho-icmc/lineuzinho" self.usefulLinks = "Estamos adicionando todo mundo aos poucos. Se puder ajudar a achar o pessoal, passa o link do grupo na descrição!\n\nInscrição na semana de recepção: calouros.icmc.usp.br/\n\nGuia do Bixo: https://bit.ly/3c9mcUG\n\nContatinho de geral: {0}\n\n".format( self.contatinhosSheet) self.docsChannel = "https://t.me/docs21" self.greeter = Greeter() self.forwarder = Forwarder() self.beaner = Beaner()
class Lineuzinho: def __init__(self): self.API_TOKEN = "1700885261:AAETCokNpqNDk44x3d5XASfnQfzxiNOKWfI" ## os.environ["SECRET"] self.contatinhosSheet = "http://bit.ly/contatosbcc021" self.githubRepo = "https://github.com/lineuzinho-icmc/lineuzinho" self.usefulLinks = "Estamos adicionando todo mundo aos poucos. Se puder ajudar a achar o pessoal, passa o link do grupo na descrição!\n\nInscrição na semana de recepção: calouros.icmc.usp.br/\n\nGuia do Bixo: https://bit.ly/3c9mcUG\n\nContatinho de geral: {0}\n\n".format( self.contatinhosSheet) self.docsChannel = "https://t.me/docs21" self.greeter = Greeter() self.forwarder = Forwarder() self.beaner = Beaner() def start(self, update, context): update.message.reply_text("pó fala meu rei") def greet(self, update, context): newMembers = update.message.new_chat_members greetings = self.greeter.generateNewMembersGreetings(newMembers) context.bot.send_message(chat_id=update.effective_chat.id, text=greetings) def links(self, update, context): context.bot.send_message(chat_id=update.effective_chat.id, text=self.usefulLinks, disable_web_page_preview=True) def getRepo(self, update, context): context.bot.send_message(chat_id=update.effective_chat.id, text=self.githubRepo) def getContatinhos(self, update, context): context.bot.send_message(chat_id=update.effective_chat.id, text="CHAMA NOS CONTATINHO\n{0}".format( self.contatinhosSheet), disable_web_page_preview=True) def getDocsChannel(self, update, context): context.bot.send_message(chat_id=update.effective_chat.id, text=self.docsChannel) def save(self, update, context): self.forwarder.saveToDocs(update, context) def getBeanFlavor(self, update, context): beanFlavor = self.beaner.getFlavor() context.bot.send_message(chat_id=update.effective_chat.id, text=beanFlavor) def help(self, update, context): update.message.reply_text( "digita \"/\" no teclado pra dar uma olhada nos comandos disponíveis :V" )
def __init__(self): self.API_TOKEN = os.environ["SECRET"] self.contatinhosSheet = "http://bit.ly/contatosbcc021" self.githubRepo = "https://github.com/lineuzinho-icmc/lineuzinho" self.usefulLinks = "Estamos adicionando todo mundo aos poucos. Se puder ajudar a achar o pessoal, passa o link do grupo na descrição!\n\nInscrição na semana de recepção: calouros.icmc.usp.br/\n\nGuia do Bixo: https://bit.ly/3c9mcUG\n\nContatinho de geral: {0}\n\nEnquetes: https://t.me/joinchat/qrJ_MrnHDbE1ZmNh\n\n".format( self.contatinhosSheet) self.docsChannel = "https://t.me/docs21" self.greeter = Greeter() self.forwarder = Forwarder() self.beaner = Beaner() self.conn = Connection() self.last_pi_call = time.time()
def StartForwarder(self, port_pairs): """Starts TCP traffic forwarding for the given |port_pairs|. Args: host_port_pairs: A list of (device_port, local_port) tuples to forward. """ # Sometimes the forwarder device port may be already used. We have to kill # all forwarder processes to ensure that the forwarder can be started since # currently we can not associate the specified port to related pid. self.adb.KillAll('forwarder') if self._forwarder: self._forwarder.Close() self._forwarder = Forwarder(self.adb, port_pairs, self.tool, '127.0.0.1', self.build_type)
async def run(is_debug_mode): """ Runs the cluster discovery & forwarder. """ asyncio.create_task(_epsagon_conf_watcher(is_debug_mode)) events_manager = InMemoryEventsManager() epsagon_client = await EpsagonClient.create(EPSAGON_TOKEN) events_sender = EventsSender( epsagon_client, COLLECTOR_URL, CLUSTER_NAME, EPSAGON_TOKEN ) cluster_discovery = ClusterDiscovery( events_manager.write_event, should_collect_resources=SHOULD_COLLECT_RESOURCES, should_collect_events=SHOULD_COLLECT_EVENTS, ) forwarder = Forwarder( events_manager, events_sender ) while True: try: tasks = [ asyncio.create_task(forwarder.start()), asyncio.create_task(cluster_discovery.start()) ] await asyncio.gather(*tasks) except ( client_exceptions.ClientError, socket.gaierror, ConnectionRefusedError, EpsagonClientException ): logging.error( "Connection error, restarting agent in %d seconds", RESTART_WAIT_TIME_SECONDS ) _cancel_tasks(tasks) events_manager.clean() await asyncio.sleep(RESTART_WAIT_TIME_SECONDS) except Exception as exception: logging.error(str(exception)) logging.error(format_exc()) logging.info("Agent is exiting due to an unexpected error") _cancel_tasks(tasks) await epsagon_client.close() break
def SetUp(self): """Sets up the test harness and device before all tests are run.""" super(TestRunner, self).SetUp() if not self.adb.IsRootEnabled(): logging.warning( 'Unable to enable java asserts for %s, non rooted device', self.device) else: if self.adb.SetJavaAssertsEnabled(enable=True): self.adb.Reboot(full_reboot=False) # We give different default value to launch HTTP server based on shard index # because it may have race condition when multiple processes are trying to # launch lighttpd with same port at same time. # This line *must* come before the forwarding below, as it nukes all # the other forwarders. A more comprehensive fix might be to pull the # forwarder-killing line up to here, but that might violate assumptions # implicit in other places. self.LaunchTestHttpServer( os.path.join(constants.CHROME_DIR), (constants.LIGHTTPD_RANDOM_PORT_FIRST + self.shard_index)) if self.ports_to_forward: for port in self.ports_to_forward: self.forwarders.append( Forwarder(self.adb, [(port, port)], self.tool, '127.0.0.1', self.build_type)) self.CopyTestFilesOnce() self.flags.AddFlags(['--enable-test-intents'])
def LaunchChromeTestServerSpawner(self): """Launches test server spawner.""" server_ready = False error_msgs = [] # Try 3 times to launch test spawner server. for i in xrange(0, 3): # Do not allocate port for test server here. We will allocate # different port for individual test in TestServerThread. self.test_server_spawner_port = ports.AllocateTestServerPort() self._spawning_server = SpawningServer( self.test_server_spawner_port, self.adb, self.tool, self.build_type) self._spawning_server.Start() server_ready, error_msg = ports.IsHttpServerConnectable( '127.0.0.1', self.test_server_spawner_port, path='/ping', expected_read='ready') if server_ready: break else: error_msgs.append(error_msg) self._spawning_server.Stop() # Wait for 2 seconds then restart. time.sleep(2) if not server_ready: logging.error(';'.join(error_msgs)) raise Exception('Can not start the test spawner server.') self._PushTestServerPortInfoToDevice() self._spawner_forwarder = Forwarder( self.adb, [(self.test_server_spawner_port, self.test_server_spawner_port)], self.tool, '127.0.0.1', self.build_type)
async def handle_get(request: Request): url = get_server_location(request) try: f = Forwarder(url, 'GET', None, request.headers) resp, req = await f.forward(SELENIUM_TIMEOUT) except ClientTimeout: return error_on_timeout() return web.json_response(resp, status=req.status)
def ShutdownHelperToolsForTestSuite(self): """Shuts down the server and the forwarder.""" # Forwarders should be killed before the actual servers they're forwarding # to as they are clients potentially with open connections and to allow for # proper hand-shake/shutdown. Forwarder.KillDevice(self.adb) if self._http_server: self._http_server.ShutdownHttpServer() if self._spawning_server: self._spawning_server.Stop() self.flags.Restore()
def test_something(self): try: ff = Forwarder(concurrency=10) for _ in range(2000): ff.issue( Request('/dev/ram0', randint(0, 1), randint(0, 1024 * 1024 * 500), length=512, data=b'\xcd' * 512)) sleep(0.5) lat = 0 for i in range(2000): res = ff.res_Q.pop() lat = (lat * i + res.latency) / (i + 1) # print(res.addr, res.type, res.latency * 1000) print("\nmean latency = %f" % (lat * 1000)) self.assertTrue('mean latency = %f' % (lat * 1000)) except Exception as e: self.assertFalse(e.__traceback__)
async def delete_session(request: Request): """ DELETE /session/{session_id} :param request: :return: """ session_id = request.match_info['session_id'] url = get_server_location(request) f = Forwarder(url, 'DELETE', None, request.headers) try: resp, req = await f.forward(SELENIUM_TIMEOUT) except ClientTimeout: return error_on_timeout() if session_id in request.app['session_cache']: request.app['session_cache'].pop(session_id) return web.json_response(resp)
async def new_session(request): """ POST /session creates a new session :param request: :return: """ hub = await request.app['selenium_hubs'].select_hub() payload = await request.json() url = '{}/session'.format(hub.rstrip('/')) f = Forwarder(url, 'POST', payload, request.headers) try: resp, req = await f.forward(SELENIUM_TIMEOUT) except ClientTimeout: return error_on_timeout() if 'sessionId' in resp: sess_id = resp['sessionId'] request.app['session_cache'][sess_id] = (hub, dt.datetime.now()) return web.json_response(resp, status=req.status)
def test_submit_payload_(): f = Forwarder("api_key", DOMAIN) f._submit_payload("test", "data", {"test": 21}) t = get_transaction(f) assert t.payload == "data" assert t.domain == DOMAIN assert t.endpoint == "test?api_key=api_key" assert t.headers == {"test": 21, Forwarder.DD_API_HEADER: "api_key"} f._submit_payload("test", "data") t = get_transaction(f) assert t.payload == "data" assert t.domain == DOMAIN assert t.endpoint == "test?api_key=api_key" assert t.headers == {Forwarder.DD_API_HEADER: "api_key"}
class Proxy: forwarder = Forwarder() def __init__(self): raise Exception('Proxy is a static class, so can not be instantiated.') @staticmethod def invalidate(blk): if blk.typ == CacheConf.ram_blk: Tables.RAM.invalidate(blk.addr) elif blk.typ == CacheConf.ssd_blk: Tables.SSD.invalidate(blk.addr) @staticmethod def promote(blk, dst): pass @staticmethod def write(blk): Proxy.forwarder.issue(Request(CacheConf.get_dev(blk.typ), blk.typ, blk.addr, data=b'\x00' * blk.len)) @staticmethod def read(blk): Proxy.forwarder.issue(Request(CacheConf.get_dev(blk.typ), blk.typ, blk.addr, length=blk.len))
def test_forwarder_start_stop(): f = Forwarder("api_key", "https://datadog.com", 2) f.start() assert len(f.workers) == 2 assert f.workers[0].is_alive() assert f.workers[1].is_alive() assert f.retry_worker.is_alive() tmp_workers = f.workers tmp_retry_worker = f.retry_worker f.stop() assert len(f.workers) == 0 assert f.retry_worker is None assert not tmp_workers[0].is_alive() assert not tmp_workers[1].is_alive() assert not tmp_retry_worker.is_alive()
def _CreateAndRunForwarder(self, adb, port_pairs, tool, host_name, build_type): """Creates and run a forwarder.""" forwarder = Forwarder(adb, build_type) forwarder.Run(port_pairs, tool, host_name) return forwarder
class TestRunner(BaseTestRunner): """Responsible for running a series of tests connected to a single device.""" _DEVICE_DATA_DIR = 'chrome/test/data' _EMMA_JAR = os.path.join(os.environ.get('ANDROID_BUILD_TOP', ''), 'external/emma/lib/emma.jar') _COVERAGE_MERGED_FILENAME = 'unittest_coverage.es' _COVERAGE_WEB_ROOT_DIR = os.environ.get('EMMA_WEB_ROOTDIR') _COVERAGE_FILENAME = 'coverage.ec' _COVERAGE_RESULT_PATH = ( '/data/data/com.google.android.apps.chrome/files/' + _COVERAGE_FILENAME) _COVERAGE_META_INFO_PATH = os.path.join( os.environ.get('ANDROID_BUILD_TOP', ''), 'out/target/common/obj/APPS', 'Chrome_intermediates/coverage.em') _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + '/chrome-profile*') _DEVICE_HAS_TEST_FILES = {} def __init__(self, options, device, tests_iter, coverage, shard_index, apks, ports_to_forward): """Create a new TestRunner. Args: options: An options object with the following required attributes: - build_type: 'Release' or 'Debug'. - install_apk: Re-installs the apk if opted. - save_perf_json: Whether or not to save the JSON file from UI perf tests. - screenshot_failures: Take a screenshot for a test failure - tool: Name of the Valgrind tool. - wait_for_debugger: blocks until the debugger is connected. - disable_assertions: Whether to disable java assertions on the device. device: Attached android device. tests_iter: A list of tests to be run. coverage: Collects coverage information if opted. shard_index: shard # for this TestRunner, used to create unique port numbers. apks: A list of ApkInfo objects need to be installed. The first element should be the tests apk, the rests could be the apks used in test. The default is ChromeTest.apk. ports_to_forward: A list of port numbers for which to set up forwarders. Can be optionally requested by a test case. Raises: FatalTestException: if coverage metadata is not available. """ BaseTestRunner.__init__(self, device, options.tool, shard_index, options.build_type) if not apks: apks = [ apk_info.ApkInfo(options.test_apk_path, options.test_apk_jar_path) ] self.build_type = options.build_type self.install_apk = options.install_apk self.test_data = options.test_data self.save_perf_json = options.save_perf_json self.screenshot_failures = options.screenshot_failures self.wait_for_debugger = options.wait_for_debugger self.disable_assertions = options.disable_assertions self.tests_iter = tests_iter self.coverage = coverage self.apks = apks self.test_apk = apks[0] self.instrumentation_class_path = self.test_apk.GetPackageName() self.ports_to_forward = ports_to_forward self.test_results = TestResults() self.forwarder = None if self.coverage: if os.path.exists(TestRunner._COVERAGE_MERGED_FILENAME): os.remove(TestRunner._COVERAGE_MERGED_FILENAME) if not os.path.exists(TestRunner._COVERAGE_META_INFO_PATH): raise FatalTestException('FATAL ERROR in ' + sys.argv[0] + ' : Coverage meta info [' + TestRunner._COVERAGE_META_INFO_PATH + '] does not exist.') if (not TestRunner._COVERAGE_WEB_ROOT_DIR or not os.path.exists(TestRunner._COVERAGE_WEB_ROOT_DIR)): raise FatalTestException( 'FATAL ERROR in ' + sys.argv[0] + ' : Path specified in $EMMA_WEB_ROOTDIR [' + TestRunner._COVERAGE_WEB_ROOT_DIR + '] does not exist.') def _GetTestsIter(self): if not self.tests_iter: # multiprocessing.Queue can't be pickled across processes if we have it as # a member set during constructor. Grab one here instead. self.tests_iter = (BaseTestSharder.tests_container) assert self.tests_iter return self.tests_iter def CopyTestFilesOnce(self): """Pushes the test data files to the device. Installs the apk if opted.""" if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False): logging.warning( 'Already copied test files to device %s, skipping.', self.device) return for dest_host_pair in self.test_data: dst_src = dest_host_pair.split(':', 1) dst_layer = dst_src[0] host_src = dst_src[1] host_test_files_path = constants.CHROME_DIR + '/' + host_src if os.path.exists(host_test_files_path): self.adb.PushIfNeeded( host_test_files_path, self.adb.GetExternalStorage() + '/' + TestRunner._DEVICE_DATA_DIR + '/' + dst_layer) if self.install_apk: for apk in self.apks: self.adb.ManagedInstall(apk.GetApkPath(), package_name=apk.GetPackageName()) self.tool.CopyFiles() TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True def SaveCoverageData(self, test): """Saves the Emma coverage data before it's overwritten by the next test. Args: test: the test whose coverage data is collected. """ if not self.coverage: return if not self.adb.Adb().Pull(TestRunner._COVERAGE_RESULT_PATH, constants.CHROME_DIR): logging.error('ERROR: Unable to find file ' + TestRunner._COVERAGE_RESULT_PATH + ' on the device for test ' + test) pulled_coverage_file = os.path.join(constants.CHROME_DIR, TestRunner._COVERAGE_FILENAME) if os.path.exists(TestRunner._COVERAGE_MERGED_FILENAME): cmd = [ 'java', '-classpath', TestRunner._EMMA_JAR, 'emma', 'merge', '-in', pulled_coverage_file, '-in', TestRunner._COVERAGE_MERGED_FILENAME, '-out', TestRunner._COVERAGE_MERGED_FILENAME ] cmd_helper.RunCmd(cmd) else: shutil.copy(pulled_coverage_file, TestRunner._COVERAGE_MERGED_FILENAME) os.remove(pulled_coverage_file) def GenerateCoverageReportIfNeeded(self): """Uses the Emma to generate a coverage report and a html page.""" if not self.coverage: return cmd = [ 'java', '-classpath', TestRunner._EMMA_JAR, 'emma', 'report', '-r', 'html', '-in', TestRunner._COVERAGE_MERGED_FILENAME, '-in', TestRunner._COVERAGE_META_INFO_PATH ] cmd_helper.RunCmd(cmd) new_dir = os.path.join(TestRunner._COVERAGE_WEB_ROOT_DIR, time.strftime('Coverage_for_%Y_%m_%d_%a_%H:%M')) shutil.copytree('coverage', new_dir) latest_dir = os.path.join(TestRunner._COVERAGE_WEB_ROOT_DIR, 'Latest_Coverage_Run') if os.path.exists(latest_dir): shutil.rmtree(latest_dir) os.mkdir(latest_dir) webserver_new_index = os.path.join(new_dir, 'index.html') webserver_new_files = os.path.join(new_dir, '_files') webserver_latest_index = os.path.join(latest_dir, 'index.html') webserver_latest_files = os.path.join(latest_dir, '_files') # Setup new softlinks to last result. os.symlink(webserver_new_index, webserver_latest_index) os.symlink(webserver_new_files, webserver_latest_files) cmd_helper.RunCmd(['chmod', '755', '-R', latest_dir, new_dir]) def _GetInstrumentationArgs(self): ret = {} if self.coverage: ret['coverage'] = 'true' if self.wait_for_debugger: ret['debug'] = 'true' return ret def _TakeScreenshot(self, test): """Takes a screenshot from the device.""" screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png') logging.info('Taking screenshot named %s', screenshot_name) self.adb.TakeScreenshot(screenshot_name) def SetUp(self): """Sets up the test harness and device before all tests are run.""" super(TestRunner, self).SetUp() if not self.adb.IsRootEnabled(): logging.warning( 'Unable to enable java asserts for %s, non rooted device', self.device) else: if self.adb.SetJavaAssertsEnabled( enable=not self.disable_assertions): self.adb.Reboot(full_reboot=False) # We give different default value to launch HTTP server based on shard index # because it may have race condition when multiple processes are trying to # launch lighttpd with same port at same time. http_server_ports = self.LaunchTestHttpServer( os.path.join(constants.CHROME_DIR), (constants.LIGHTTPD_RANDOM_PORT_FIRST + self.shard_index)) if self.ports_to_forward: port_pairs = [(port, port) for port in self.ports_to_forward] # We need to remember which ports the HTTP server is using, since the # forwarder will stomp on them otherwise. port_pairs.append(http_server_ports) self.forwarder = Forwarder(self.adb, self.build_type) self.forwarder.Run(port_pairs, self.tool, '127.0.0.1') self.CopyTestFilesOnce() self.flags.AddFlags(['--enable-test-intents']) def TearDown(self): """Cleans up the test harness and saves outstanding data from test run.""" if self.forwarder: self.forwarder.Close() self.GenerateCoverageReportIfNeeded() super(TestRunner, self).TearDown() def TestSetup(self, test): """Sets up the test harness for running a particular test. Args: test: The name of the test that will be run. """ self.SetupPerfMonitoringIfNeeded(test) self._SetupIndividualTestTimeoutScale(test) self.tool.SetupEnvironment() # Make sure the forwarder is still running. self.RestartHttpServerForwarderIfNecessary() def _IsPerfTest(self, test): """Determines whether a test is a performance test. Args: test: The name of the test to be checked. Returns: Whether the test is annotated as a performance test. """ return _PERF_TEST_ANNOTATION in self.test_apk.GetTestAnnotations(test) def SetupPerfMonitoringIfNeeded(self, test): """Sets up performance monitoring if the specified test requires it. Args: test: The name of the test to be run. """ if not self._IsPerfTest(test): return self.adb.Adb().SendCommand( 'shell rm ' + TestRunner._DEVICE_PERF_OUTPUT_SEARCH_PREFIX) self.adb.StartMonitoringLogcat() def TestTeardown(self, test, test_result): """Cleans up the test harness after running a particular test. Depending on the options of this TestRunner this might handle coverage tracking or performance tracking. This method will only be called if the test passed. Args: test: The name of the test that was just run. test_result: result for this test. """ self.tool.CleanUpEnvironment() # The logic below relies on the test passing. if not test_result or test_result.GetStatusCode(): return self.TearDownPerfMonitoring(test) self.SaveCoverageData(test) def TearDownPerfMonitoring(self, test): """Cleans up performance monitoring if the specified test required it. Args: test: The name of the test that was just run. Raises: FatalTestException: if there's anything wrong with the perf data. """ if not self._IsPerfTest(test): return raw_test_name = test.split('#')[1] # Wait and grab annotation data so we can figure out which traces to parse regex = self.adb.WaitForLogMatch( re.compile('\*\*PERFANNOTATION\(' + raw_test_name + '\)\:(.*)'), None) # If the test is set to run on a specific device type only (IE: only # tablet or phone) and it is being run on the wrong device, the test # just quits and does not do anything. The java test harness will still # print the appropriate annotation for us, but will add --NORUN-- for # us so we know to ignore the results. # The --NORUN-- tag is managed by MainActivityTestBase.java if regex.group(1) != '--NORUN--': # Obtain the relevant perf data. The data is dumped to a # JSON formatted file. json_string = self.adb.GetFileContents( '/data/data/com.google.android.apps.chrome/files/PerfTestData.txt' ) if json_string: json_string = '\n'.join(json_string) else: raise FatalTestException( 'Perf file does not exist or is empty') if self.save_perf_json: json_local_file = '/tmp/chromium-android-perf-json-' + raw_test_name with open(json_local_file, 'w') as f: f.write(json_string) logging.info('Saving Perf UI JSON from test ' + test + ' to ' + json_local_file) raw_perf_data = regex.group(1).split(';') for raw_perf_set in raw_perf_data: if raw_perf_set: perf_set = raw_perf_set.split(',') if len(perf_set) != 3: raise FatalTestException( 'Unexpected number of tokens in ' 'perf annotation string: ' + raw_perf_set) # Process the performance data result = GetAverageRunInfoFromJSONString( json_string, perf_set[0]) PrintPerfResult(perf_set[1], perf_set[2], [result['average']], result['units']) def _SetupIndividualTestTimeoutScale(self, test): timeout_scale = self._GetIndividualTestTimeoutScale(test) valgrind_tools.SetChromeTimeoutScale(self.adb, timeout_scale) def _GetIndividualTestTimeoutScale(self, test): """Returns the timeout scale for the given |test|.""" annotations = self.apks[0].GetTestAnnotations(test) timeout_scale = 1 if 'TimeoutScale' in annotations: for annotation in annotations: scale_match = re.match('TimeoutScale:([0-9]+)', annotation) if scale_match: timeout_scale = int(scale_match.group(1)) if self.wait_for_debugger: timeout_scale *= 100 return timeout_scale def _GetIndividualTestTimeoutSecs(self, test): """Returns the timeout in seconds for the given |test|.""" annotations = self.apks[0].GetTestAnnotations(test) if 'Manual' in annotations: return 600 * 60 if 'External' in annotations: return 10 * 60 if 'LargeTest' in annotations or _PERF_TEST_ANNOTATION in annotations: return 5 * 60 if 'MediumTest' in annotations: return 3 * 60 return 1 * 60 def RunTests(self): """Runs the tests, generating the coverage if needed. Returns: A TestResults object. """ instrumentation_path = (self.instrumentation_class_path + '/android.test.InstrumentationTestRunner') instrumentation_args = self._GetInstrumentationArgs() for test in self._GetTestsIter(): test_result = None start_date_ms = None try: self.TestSetup(test) start_date_ms = int(time.time()) * 1000 args_with_filter = dict(instrumentation_args) args_with_filter['class'] = test # |test_results| is a list that should contain # a single TestResult object. logging.warn(args_with_filter) (test_results, _) = self.adb.Adb().StartInstrumentation( instrumentation_path=instrumentation_path, instrumentation_args=args_with_filter, timeout_time=(self._GetIndividualTestTimeoutSecs(test) * self._GetIndividualTestTimeoutScale(test) * self.tool.GetTimeoutScale())) duration_ms = int(time.time()) * 1000 - start_date_ms assert len(test_results) == 1 test_result = test_results[0] status_code = test_result.GetStatusCode() if status_code: log = test_result.GetFailureReason() if not log: log = 'No information.' if self.screenshot_failures or log.find( 'INJECT_EVENTS perm') >= 0: self._TakeScreenshot(test) self.test_results.failed += [ SingleTestResult(test, start_date_ms, duration_ms, log) ] else: result = [ SingleTestResult(test, start_date_ms, duration_ms) ] self.test_results.ok += result # Catch exceptions thrown by StartInstrumentation(). # See ../../third_party/android/testrunner/adb_interface.py except (errors.WaitForResponseTimedOutError, errors.DeviceUnresponsiveError, errors.InstrumentationError), e: if start_date_ms: duration_ms = int(time.time()) * 1000 - start_date_ms else: start_date_ms = int(time.time()) * 1000 duration_ms = 0 message = str(e) if not message: message = 'No information.' self.test_results.crashed += [ SingleTestResult(test, start_date_ms, duration_ms, message) ] test_result = None self.TestTeardown(test, test_result) return self.test_results
class Lineuzinho: def __init__(self): self.API_TOKEN = os.environ["SECRET"] self.contatinhosSheet = "http://bit.ly/contatosbcc021" self.githubRepo = "https://github.com/lineuzinho-icmc/lineuzinho" self.usefulLinks = "Estamos adicionando todo mundo aos poucos. Se puder ajudar a achar o pessoal, passa o link do grupo na descrição!\n\nInscrição na semana de recepção: calouros.icmc.usp.br/\n\nGuia do Bixo: https://bit.ly/3c9mcUG\n\nContatinho de geral: {0}\n\nEnquetes: https://t.me/joinchat/qrJ_MrnHDbE1ZmNh\n\n".format( self.contatinhosSheet) self.docsChannel = "https://t.me/docs21" self.greeter = Greeter() self.forwarder = Forwarder() self.beaner = Beaner() self.conn = Connection() self.last_pi_call = time.time() def start(self, update, context): update.message.reply_text("pó fala meu rei") def greet(self, update, context): newMembers = update.message.new_chat_members greetings = self.greeter.generateNewMembersGreetings(newMembers) context.bot.send_message(chat_id=update.effective_chat.id, text=greetings) def links(self, update, context): context.bot.send_message(chat_id=update.effective_chat.id, text=self.usefulLinks, disable_web_page_preview=True) def getRepo(self, update, context): context.bot.send_message(chat_id=update.effective_chat.id, text=self.githubRepo) def getContatinhos(self, update, context): context.bot.send_message(chat_id=update.effective_chat.id, text="CHAMA NOS CONTATINHO\n{0}".format( self.contatinhosSheet), disable_web_page_preview=True) def getDocsChannel(self, update, context): context.bot.send_message(chat_id=update.effective_chat.id, text=self.docsChannel) def save(self, update, context): self.forwarder.saveToDocs(update, context) def getBeanFlavor(self, update, context): beanFlavor = self.beaner.getFlavor() context.bot.send_message(chat_id=update.effective_chat.id, text=beanFlavor) def getPiRanking(self, update, context): if time.time() - self.last_pi_call > 86400: self.last_pi_call = time.time() pi_manager.generate_daily_ranking() context.bot.send_message(chat_id=update.effective_chat.id, text=pi_manager.get_daily_ranking()) def getUserPiRanking(self, update, context): if time.time() - self.last_pi_call > 86400: self.last_pi_call = time.time() pi_manager.generate_daily_ranking() context.bot.send_message(chat_id=update.effective_chat.id, text=pi_manager.get_member_ranking( update.message.from_user.username)) def help(self, update, context): update.message.reply_text( "digita \"/\" no teclado pra dar uma olhada nos comandos disponíveis :V" ) def randomActivityAlert(self, update, context): if random.randint(0, 500) == 250: update.message.reply_text( "Oi, desculpa atrapalhar, mas... já fez suas atividades de hoje? :)" )
def test_forwarder_creation(): f = Forwarder("api_key", DOMAIN) assert f.api_key == "api_key" assert f.domain == "https://app.datadoghq.com"
def start(): """ Dummy start until we have a collector """ init_agent() hostname = get_hostname() logging.info("Starting the agent, hostname: %s", hostname) # init Forwarder logging.info("Starting the Forwarder") api_key = config.get('api_key') dd_url = config.get('dd_url') if not dd_url: logging.error('No Datadog URL configured - cannot continue') sys.exit(1) if not api_key: logging.error('No API key configured - cannot continue') sys.exit(1) forwarder = Forwarder(api_key, dd_url) forwarder.start() # aggregator aggregator = MetricsAggregator( hostname, interval=config.get('aggregator_interval'), expiry_seconds=(config.get('min_collection_interval') + config.get('aggregator_expiry_seconds')), recent_point_threshold=config.get('recent_point_threshold'), histogram_aggregates=config.get('histogram_aggregates'), histogram_percentiles=config.get('histogram_percentiles'), ) # serializer serializer = Serializer( aggregator, forwarder, ) # instantiate collector collector = Collector(config, aggregator) collector.load_check_classes() collector.instantiate_checks() def signal_handler(signal, frame): logging.info("SIGINT received: stopping the agent") logging.info("Stopping the forwarder") forwarder.stop() logging.info("See you !") sys.exit(0) signal.signal(signal.SIGINT, signal_handler) # update the metadata periodically? metadata = get_metadata(hostname) serializer.submit_metadata(metadata) while True: collector.run_checks() serializer.serialize_and_push() time.sleep(config.get('min_collection_interval'))
def run(self): try: hostname = get_hostname() except HostnameException as e: logging.critical( "{} - You can define one in datadog.yaml or in your hosts file" .format(e)) sys.exit(1) logging.info("Starting the agent, hostname: %s", hostname) # init Forwarder logging.info("Starting the Forwarder") api_key = config.get('api_key') dd_url = config.get('dd_url') if not dd_url: logging.error('No Datadog URL configured - cannot continue') sys.exit(1) if not api_key: logging.error('No API key configured - cannot continue') sys.exit(1) # get proxy settings proxies = get_proxy() logging.debug('Proxy configuration used: %s', proxies) # get site url forwarder = Forwarder( api_key, get_site_url(dd_url, site=config.get('site')), proxies=proxies, ) forwarder.start() # agent aggregator aggregator = MetricsAggregator( hostname, interval=config.get('aggregator_interval'), expiry_seconds=(config.get('min_collection_interval') + config.get('aggregator_expiry_seconds')), recent_point_threshold=config.get('recent_point_threshold'), histogram_aggregates=config.get('histogram_aggregates'), histogram_percentiles=config.get('histogram_percentiles'), ) # serializer serializer = Serializer( aggregator, forwarder, ) # instantiate collector collector = Collector(config, aggregator) collector.load_check_classes() collector.instantiate_checks() # instantiate AgentRunner runner = AgentRunner(collector, serializer, config) # instantiate Dogstatsd reporter = None dsd_server = None dsd_enable = config['dogstatsd'].get('enable', False) if dsd_enable: reporter, dsd_server, _ = init_dogstatsd(config, forwarder=forwarder) dsd = DogstatsdRunner(dsd_server) # instantiate API status = { 'agent': aggregator.stats, 'forwarder': forwarder.stats, 'collector': collector.status, } if dsd_server: status['dogstatsd'] = dsd_server.aggregator.stats api = APIServer(config, status=status) handler = SignalHandler() # components handler.register('runner', runner) handler.register('forwarder', forwarder) handler.register('api', api) if dsd_enable: handler.register('reporter', reporter) handler.register('dsd_server', dsd_server) # signals handler.handle(signal.SIGTERM) handler.handle(signal.SIGINT) # start signal handler handler.start() runner.start() api.start() if dsd_enable: reporter.start() dsd.start() dsd.join() logging.info("Dogstatsd server done...") try: dsd.raise_for_status() except Exception as e: log.error("There was a problem with the dogstatsd server: %s", e) reporter.stop() runner.join() logging.info("Collector done...") api.join() logging.info("API done...") handler.stop() handler.join() logging.info("Signal handler done...") logging.info("Thank you for shopping at DataDog! Come back soon!") sys.exit(0)
class TestServerThread(threading.Thread): """A thread to run the test server in a separate process.""" def __init__(self, ready_event, arguments, adb, tool, build_type): """Initialize TestServerThread with the following argument. Args: ready_event: event which will be set when the test server is ready. arguments: dictionary of arguments to run the test server. adb: instance of AndroidCommands. tool: instance of runtime error detection tool. build_type: 'Release' or 'Debug'. """ threading.Thread.__init__(self) self.wait_event = threading.Event() self.stop_flag = False self.ready_event = ready_event self.ready_event.clear() self.arguments = arguments self.adb = adb self.tool = tool self.test_server_process = None self.is_ready = False self.host_port = self.arguments['port'] assert isinstance(self.host_port, int) self._test_server_forwarder = None # The forwarder device port now is dynamically allocated. self.forwarder_device_port = 0 # Anonymous pipe in order to get port info from test server. self.pipe_in = None self.pipe_out = None self.command_line = [] self.build_type = build_type def _WaitToStartAndGetPortFromTestServer(self): """Waits for the Python test server to start and gets the port it is using. The port information is passed by the Python test server with a pipe given by self.pipe_out. It is written as a result to |self.host_port|. Returns: Whether the port used by the test server was successfully fetched. """ assert self.host_port == 0 and self.pipe_out and self.pipe_in (in_fds, _, _) = select.select([ self.pipe_in, ], [], [], TEST_SERVER_STARTUP_TIMEOUT) if len(in_fds) == 0: logging.error( 'Failed to wait to the Python test server to be started.') return False # First read the data length as an unsigned 4-byte value. This # is _not_ using network byte ordering since the Python test server packs # size as native byte order and all Chromium platforms so far are # configured to use little-endian. # TODO(jnd): Change the Python test server and local_test_server_*.cc to # use a unified byte order (either big-endian or little-endian). data_length = os.read(self.pipe_in, struct.calcsize('=L')) if data_length: (data_length, ) = struct.unpack('=L', data_length) assert data_length if not data_length: logging.error('Failed to get length of server data.') return False port_json = os.read(self.pipe_in, data_length) if not port_json: logging.error('Failed to get server data.') return False logging.info('Got port json data: %s', port_json) port_json = json.loads(port_json) if port_json.has_key('port') and isinstance(port_json['port'], int): self.host_port = port_json['port'] return _CheckPortStatus(self.host_port, True) logging.error('Failed to get port information from the server data.') return False def _GenerateCommandLineArguments(self): """Generates the command line to run the test server. Note that all options are processed by following the definitions in testserver.py. """ if self.command_line: return # The following arguments must exist. type_cmd = _GetServerTypeCommandLine(self.arguments['server-type']) if type_cmd: self.command_line.append(type_cmd) self.command_line.append('--port=%d' % self.host_port) # Use a pipe to get the port given by the instance of Python test server # if the test does not specify the port. if self.host_port == 0: (self.pipe_in, self.pipe_out) = os.pipe() self.command_line.append('--startup-pipe=%d' % self.pipe_out) self.command_line.append('--host=%s' % self.arguments['host']) data_dir = self.arguments['data-dir'] or 'chrome/test/data' if not os.path.isabs(data_dir): data_dir = os.path.join(constants.CHROME_DIR, data_dir) self.command_line.append('--data-dir=%s' % data_dir) # The following arguments are optional depending on the individual test. if self.arguments.has_key('log-to-console'): self.command_line.append('--log-to-console') if self.arguments.has_key('auth-token'): self.command_line.append('--auth-token=%s' % self.arguments['auth-token']) if self.arguments.has_key('https'): self.command_line.append('--https') if self.arguments.has_key('cert-and-key-file'): self.command_line.append( '--cert-and-key-file=%s' % os.path.join(constants.CHROME_DIR, self.arguments['cert-and-key-file'])) if self.arguments.has_key('ocsp'): self.command_line.append('--ocsp=%s' % self.arguments['ocsp']) if self.arguments.has_key('https-record-resume'): self.command_line.append('--https-record-resume') if self.arguments.has_key('ssl-client-auth'): self.command_line.append('--ssl-client-auth') if self.arguments.has_key('tls-intolerant'): self.command_line.append('--tls-intolerant=%s' % self.arguments['tls-intolerant']) if self.arguments.has_key('ssl-client-ca'): for ca in self.arguments['ssl-client-ca']: self.command_line.append( '--ssl-client-ca=%s' % os.path.join(constants.CHROME_DIR, ca)) if self.arguments.has_key('ssl-bulk-cipher'): for bulk_cipher in self.arguments['ssl-bulk-cipher']: self.command_line.append('--ssl-bulk-cipher=%s' % bulk_cipher) def run(self): logging.info('Start running the thread!') self.wait_event.clear() self._GenerateCommandLineArguments() command = constants.CHROME_DIR if self.arguments['server-type'] == 'sync': command = [ os.path.join(command, 'sync', 'tools', 'testserver', 'sync_testserver.py') ] + self.command_line else: command = [ os.path.join(command, 'net', 'tools', 'testserver', 'testserver.py') ] + self.command_line logging.info('Running: %s', command) self.process = subprocess.Popen(command) if self.process: if self.pipe_out: self.is_ready = self._WaitToStartAndGetPortFromTestServer() else: self.is_ready = _CheckPortStatus(self.host_port, True) if self.is_ready: self._test_server_forwarder = Forwarder(self.adb, self.build_type) self._test_server_forwarder.Run([(0, self.host_port)], self.tool, '127.0.0.1') # Check whether the forwarder is ready on the device. self.is_ready = False device_port = self._test_server_forwarder.DevicePortForHostPort( self.host_port) if device_port: for timeout in range(1, 5): if ports.IsDevicePortUsed(self.adb, device_port, 'LISTEN'): self.is_ready = True self.forwarder_device_port = device_port break time.sleep(timeout) # Wake up the request handler thread. self.ready_event.set() # Keep thread running until Stop() gets called. while not self.stop_flag: time.sleep(1) if self.process.poll() is None: self.process.kill() if self._test_server_forwarder: self._test_server_forwarder.Close() self.process = None self.is_ready = False if self.pipe_out: os.close(self.pipe_in) os.close(self.pipe_out) self.pipe_in = None self.pipe_out = None logging.info('Test-server has died.') self.wait_event.set() def Stop(self): """Blocks until the loop has finished. Note that this must be called in another thread. """ if not self.process: return self.stop_flag = True self.wait_event.wait()
def SetUp(self): """Called before tests run.""" Forwarder.KillDevice(self.adb)
import threading from tqdm import tqdm from parser import Parser from forwarder import Forwarder if __name__ == '__main__': parsers = [ Parser('trace/NEW_2016031907-LUN1.csv'), Parser('trace/NEW_2016031907-LUN4.csv') ] forwarder = Forwarder(concurrency=10) llt = 0 for parser in parsers: llt += parser.cnt threading.Thread(target=parser.start_sending_requests, args=(forwarder.issue, ), daemon=True).start() lats = [] cnt = 0 with tqdm(total=llt) as pbar: try: while True: cnt += 1 if cnt == 500: pbar.update(cnt) cnt = 0 res = forwarder.res_Q.pop() lats.append(res.latency * 1000)
class BaseTestRunner(object): """Base class for running tests on a single device. A subclass should implement RunTests() with no parameter, so that calling the Run() method will set up tests, run them and tear them down. """ def __init__(self, device, tool, shard_index, build_type): """ Args: device: Tests will run on the device of this ID. shard_index: Index number of the shard on which the test suite will run. build_type: 'Release' or 'Debug'. """ self.device = device self.adb = android_commands.AndroidCommands(device=device) self.tool = CreateTool(tool, self.adb) self._http_server = None self._forwarder = None self._forwarder_device_port = 8000 self.forwarder_base_url = ('http://localhost:%d' % self._forwarder_device_port) self.flags = FlagChanger(self.adb) self.shard_index = shard_index self.flags.AddFlags(['--disable-fre']) self._spawning_server = None self._spawner_forwarder = None # We will allocate port for test server spawner when calling method # LaunchChromeTestServerSpawner and allocate port for test server when # starting it in TestServerThread. self.test_server_spawner_port = 0 self.test_server_port = 0 self.build_type = build_type def _PushTestServerPortInfoToDevice(self): """Pushes the latest port information to device.""" self.adb.SetFileContents( self.adb.GetExternalStorage() + '/' + NET_TEST_SERVER_PORT_INFO_FILE, '%d:%d' % (self.test_server_spawner_port, self.test_server_port)) def Run(self): """Calls subclass functions to set up tests, run them and tear them down. Returns: Test results returned from RunTests(). """ if not self.HasTests(): return True self.SetUp() try: return self.RunTests() finally: self.TearDown() def SetUp(self): """Called before tests run.""" pass def HasTests(self): """Whether the test suite has tests to run.""" return True def RunTests(self): """Runs the tests. Need to be overridden.""" raise NotImplementedError def TearDown(self): """Called when tests finish running.""" self.ShutdownHelperToolsForTestSuite() def CopyTestData(self, test_data_paths, dest_dir): """Copies |test_data_paths| list of files/directories to |dest_dir|. Args: test_data_paths: A list of files or directories relative to |dest_dir| which should be copied to the device. The paths must exist in |CHROME_DIR|. dest_dir: Absolute path to copy to on the device. """ for p in test_data_paths: self.adb.PushIfNeeded(os.path.join(constants.CHROME_DIR, p), os.path.join(dest_dir, p)) def LaunchTestHttpServer(self, document_root, port=None, extra_config_contents=None): """Launches an HTTP server to serve HTTP tests. Args: document_root: Document root of the HTTP server. port: port on which we want to the http server bind. extra_config_contents: Extra config contents for the HTTP server. """ self._http_server = lighttpd_server.LighttpdServer( document_root, port=port, extra_config_contents=extra_config_contents) if self._http_server.StartupHttpServer(): logging.info('http server started: http://localhost:%s', self._http_server.port) else: logging.critical('Failed to start http server') self.StartForwarderForHttpServer() def StartForwarder(self, port_pairs): """Starts TCP traffic forwarding for the given |port_pairs|. Args: host_port_pairs: A list of (device_port, local_port) tuples to forward. """ # Sometimes the forwarder device port may be already used. We have to kill # all forwarder processes to ensure that the forwarder can be started since # currently we can not associate the specified port to related pid. self.adb.KillAll('forwarder') if self._forwarder: self._forwarder.Close() self._forwarder = Forwarder(self.adb, port_pairs, self.tool, '127.0.0.1', self.build_type) def StartForwarderForHttpServer(self): """Starts a forwarder for the HTTP server. The forwarder forwards HTTP requests and responses between host and device. """ self.StartForwarder([(self._forwarder_device_port, self._http_server.port)]) def RestartHttpServerForwarderIfNecessary(self): """Restarts the forwarder if it's not open.""" # Checks to see if the http server port is being used. If not forwards the # request. # TODO(dtrainor): This is not always reliable because sometimes the port # will be left open even after the forwarder has been killed. if not ports.IsDevicePortUsed(self.adb, self._forwarder_device_port): self.StartForwarderForHttpServer() def ShutdownHelperToolsForTestSuite(self): """Shuts down the server and the forwarder.""" # Forwarders should be killed before the actual servers they're forwarding # to as they are clients potentially with open connections and to allow for # proper hand-shake/shutdown. if self._forwarder or self._spawner_forwarder: # Kill all forwarders on the device and then kill the process on the host # (if it exists) self.adb.KillAll('forwarder') if self._forwarder: self._forwarder.Close() if self._spawner_forwarder: self._spawner_forwarder.Close() if self._http_server: self._http_server.ShutdownHttpServer() if self._spawning_server: self._spawning_server.Stop() self.flags.Restore() def LaunchChromeTestServerSpawner(self): """Launches test server spawner.""" server_ready = False error_msgs = [] # Try 3 times to launch test spawner server. for i in xrange(0, 3): # Do not allocate port for test server here. We will allocate # different port for individual test in TestServerThread. self.test_server_spawner_port = ports.AllocateTestServerPort() self._spawning_server = SpawningServer( self.test_server_spawner_port, self.adb, self.tool, self.build_type) self._spawning_server.Start() server_ready, error_msg = ports.IsHttpServerConnectable( '127.0.0.1', self.test_server_spawner_port, path='/ping', expected_read='ready') if server_ready: break else: error_msgs.append(error_msg) self._spawning_server.Stop() # Wait for 2 seconds then restart. time.sleep(2) if not server_ready: logging.error(';'.join(error_msgs)) raise Exception('Can not start the test spawner server.') self._PushTestServerPortInfoToDevice() self._spawner_forwarder = Forwarder( self.adb, [(self.test_server_spawner_port, self.test_server_spawner_port)], self.tool, '127.0.0.1', self.build_type)
forwarders = [] def stop_forwarders(): for f in forwarders: f.finish() print 'Exiting forwarder daemon' if __name__ == '__main__': try: for binding in args.bindings: port_topic_pair = binding.split(':') forwarder = Forwarder(int(port_topic_pair[0]), port_topic_pair[1], zookeeper_host=args.zk) forwarders.append(forwarder) forwarder.operate() forwarder.run() for f in forwarders: f.join() except KeyboardInterrupt: print 'Interrupted' try: stop_forwarders() sys.exit(0) except SystemExit: os._exit(0)