def testSerialize(self): # pylint: disable=g-import-not-at-top try: import portpicker except ImportError: return with context.graph_mode(): worker_port = portpicker.pick_unused_port() ps_port = portpicker.pick_unused_port() cluster_dict = { "worker": ["localhost:%s" % worker_port], "ps": ["localhost:%s" % ps_port] } cs = server_lib.ClusterSpec(cluster_dict) worker = server_lib.Server( cs, job_name="worker", protocol="grpc", task_index=0, start=True) unused_ps = server_lib.Server( cs, job_name="ps", protocol="grpc", task_index=0, start=True) with ops.Graph().as_default(), session.Session(target=worker.target): with ops.device("/job:worker"): t = constant_op.constant([[1.0], [2.0]]) l = list_ops.tensor_list_from_tensor(t, element_shape=[1]) with ops.device("/job:ps"): l_ps = array_ops.identity(l) l_ps, e = list_ops.tensor_list_pop_back( l_ps, element_dtype=dtypes.float32) with ops.device("/job:worker"): worker_e = array_ops.identity(e) self.assertAllEqual(worker_e.eval(), [2.0])
def create_local_cluster(num_workers, num_ps, protocol="grpc"): """Create and start local servers and return the associated `Server` objects. Example: ```python workers, _ = tf.test.create_local_cluster(num_workers=2, num_ps=2) worker_sessions = [tf.Session(w.target) for w in workers] with tf.device("/job:ps/task:0"): ... with tf.device("/job:ps/task:1"): ... with tf.device("/job:worker/task:0"): ... with tf.device("/job:worker/task:1"): ... worker_sessions[0].run(...) ``` Args: num_workers: Number of worker servers to start. num_ps: Number of PS servers to start. protocol: Communication protocol. Allowed values are documented in the documentation of `tf.train.Server`. Returns: A tuple `(worker_servers, ps_servers)`. `worker_servers` is a list of `num_workers` objects of type `tf.train.Server` (all running locally); and `ps_servers` is a list of `num_ps` objects of similar type. Raises: ImportError: if portpicker module was not found at load time """ if _portpicker_import_error: raise _portpicker_import_error # pylint: disable=raising-bad-type worker_ports = [portpicker.pick_unused_port() for _ in range(num_workers)] ps_ports = [portpicker.pick_unused_port() for _ in range(num_ps)] cluster_dict = { "worker": ["localhost:%s" % port for port in worker_ports], "ps": ["localhost:%s" % port for port in ps_ports] } cs = server_lib.ClusterSpec(cluster_dict) workers = [ server_lib.Server( cs, job_name="worker", protocol=protocol, task_index=ix, start=True) for ix in range(num_workers) ] ps_servers = [ server_lib.Server( cs, job_name="ps", protocol=protocol, task_index=ix, start=True) for ix in range(num_ps) ] return workers, ps_servers
def _create_local_cluster(num_workers, num_ps, has_eval=False, protocol="grpc", worker_config=None, ps_config=None): if _portpicker_import_error: raise _portpicker_import_error # pylint: disable=raising-bad-type worker_ports = [portpicker.pick_unused_port() for _ in range(num_workers)] ps_ports = [portpicker.pick_unused_port() for _ in range(num_ps)] cluster_dict = { "worker": ["localhost:%s" % port for port in worker_ports], "ps": ["localhost:%s" % port for port in ps_ports] } if has_eval: cluster_dict["evaluator"] = ["localhost:%s" % portpicker.pick_unused_port()] cs = server_lib.ClusterSpec(cluster_dict) workers = [ server_lib.Server( cs, job_name="worker", protocol=protocol, task_index=ix, config=worker_config, start=True) for ix in range(num_workers) ] ps_servers = [ server_lib.Server( cs, job_name="ps", protocol=protocol, task_index=ix, config=ps_config, start=True) for ix in range(num_ps) ] if has_eval: evals = [ server_lib.Server( cs, job_name="evaluator", protocol=protocol, task_index=0, config=worker_config, start=True) ] else: evals = [] return workers, ps_servers, evals
def main(argv): del argv # Unused. if not flags.FLAGS.dest_server_config_path: raise ValueError("dest_server_config_path flag has to be provided.") if not flags.FLAGS.dest_client_config_path: raise ValueError("dest_client_config_path flag has to be provided.") admin_ui_port = portpicker.pick_unused_port() frontend_port = portpicker.pick_unused_port() source_server_config_path = package.ResourcePath( "grr-response-core", "install_data/etc/grr-server.yaml") config_lib.LoadConfig(config.CONFIG, source_server_config_path) config.CONFIG.SetWriteBack(flags.FLAGS.dest_server_config_path) # TODO(user): remove when AFF4 is gone. config.CONFIG.Set("Database.aff4_enabled", False) config.CONFIG.Set("Database.enabled", True) config.CONFIG.Set("Blobstore.implementation", "DbBlobStore") config.CONFIG.Set("Database.implementation", "MysqlDB") config.CONFIG.Set("Mysql.database", flags.FLAGS.config_mysql_database) if flags.FLAGS.config_mysql_username is not None: config.CONFIG.Set("Mysql.username", flags.FLAGS.config_mysql_username) if flags.FLAGS.config_mysql_password is not None: config.CONFIG.Set("Mysql.password", flags.FLAGS.config_mysql_password) config.CONFIG.Set("AdminUI.port", admin_ui_port) config.CONFIG.Set("AdminUI.headless", True) config.CONFIG.Set("Frontend.bind_address", "127.0.0.1") config.CONFIG.Set("Frontend.bind_port", frontend_port) config.CONFIG.Set("Server.initialized", True) config.CONFIG.Set("Client.poll_max", 1) config.CONFIG.Set("Client.server_urls", ["http://localhost:%d/" % frontend_port]) if flags.FLAGS.config_logging_path is not None: config.CONFIG.Set("Logging.path", flags.FLAGS.config_logging_path) config_updater_keys_util.GenerateKeys(config.CONFIG) config.CONFIG.Write() config_lib.SetPlatformArchContext() context = list(config.CONFIG.context) context.append("Client Context") deployer = build.ClientRepacker() config_data = deployer.GetClientConfig( context, validate=False, deploy_timestamp=False) with io.open(flags.FLAGS.dest_client_config_path, "w") as fd: fd.write(config_data)
def setUpClass(cls): gpu_memory_fraction_opt = ( "--gpu_memory_fraction=%f" % cls.PER_PROC_GPU_MEMORY_FRACTION) worker_port = portpicker.pick_unused_port() cluster_spec = "worker|localhost:%d" % worker_port tf_logging.info("cluster_spec: %s", cluster_spec) server_bin = test.test_src_dir_path("python/debug/grpc_tensorflow_server") cls.server_target = "grpc://localhost:%d" % worker_port cls.server_procs = {} cls.server_procs["worker"] = subprocess.Popen( [ server_bin, "--cluster_spec=%s" % cluster_spec, "--job_name=worker", "--task_id=0", gpu_memory_fraction_opt, ], stdout=sys.stdout, stderr=sys.stderr) # Start debug server in-process, on separate thread. (cls.debug_server_port, cls.debug_server_url, _, cls.debug_server_thread, cls.debug_server ) = grpc_debug_test_server.start_server_on_separate_thread( dump_to_filesystem=False) tf_logging.info("debug server url: %s", cls.debug_server_url) cls.session_config = config_pb2.ConfigProto( gpu_options=config_pb2.GPUOptions( per_process_gpu_memory_fraction=cls.PER_PROC_GPU_MEMORY_FRACTION))
def __init__(self, run_config, full_screen=False, **kwargs): self._proc = None self._sock = None self._controller = None self._tmp_dir = tempfile.mkdtemp(prefix="sc-", dir=run_config.tmp_dir) self._port = portpicker.pick_unused_port() self._check_exists(run_config.exec_path) args = [ run_config.exec_path, "-listen", "127.0.0.1", "-port", str(self._port), "-dataDir", os.path.join(run_config.data_dir, ""), "-tempDir", os.path.join(self._tmp_dir, ""), "-displayMode", "1" if full_screen else "0", ] try: self._proc = self._launch(run_config, args, **kwargs) self._sock = self._connect(self._port) client = protocol.StarcraftProtocol(self._sock) self._controller = remote_controller.RemoteController(client) with sw("startup"): self._controller.ping() except: self.close() raise
def run_benchmark_distributed(): ops = create_graph("/job:worker/task:0", "/job:worker/task:1") queues = [create_done_queue(0), create_done_queue(1)] # launch distributed service port0, port1 = [portpicker.pick_unused_port() for _ in range(2)] flags = " ".join(sys.argv) # pass parent flags to children def run_worker(w): my_env = os.environ.copy() if not FLAGS.verbose: my_env["CUDA_VISIBLE_DEVICES"] = "" my_env["TF_CPP_MIN_LOG_LEVEL"] = "2" if FLAGS.profile: my_env["LD_PRELOAD"]="/usr/lib/libtcmalloc_and_profiler.so.4" my_env["CPUPROFILE"]="/tmp/profile.out.%s"%(w) cmd = "python %s --task=%d --port0=%s --port1=%s"%(flags, w, port0, port1) subprocess.Popen(cmd, shell=True, stderr=subprocess.STDOUT, env=my_env) run_worker(0) run_worker(1) sess = tf.Session("grpc://%s:%s"%(host, port0), config=session_config()) rate = run_benchmark(sess, *ops) # bring down workers if FLAGS.verbose: print("Killing workers.") sess.run(queues[1].enqueue(1)) sess.run(queues[0].enqueue(1)) # bring down master last return rate
def _pick_unused_port(): """For some reason portpicker returns the same port sometimes.""" while True: p = portpicker.pick_unused_port() if p not in _PORTS: break _PORTS.add(p) return p
def start_server_on_separate_thread(dump_to_filesystem=True, server_start_delay_sec=0.0, poll_server=False, blocking=True, toggle_watch_on_core_metadata=None): """Create a test gRPC debug server and run on a separate thread. Args: dump_to_filesystem: (bool) whether the debug server will dump debug data to the filesystem. server_start_delay_sec: (float) amount of time (in sec) to delay the server start up for. poll_server: (bool) whether the server will be polled till success on startup. blocking: (bool) whether the server should be started in a blocking mode. toggle_watch_on_core_metadata: A list of (node_name, output_slot, debug_op) tuples to toggle the watchpoint status during the on_core_metadata calls (optional). Returns: server_port: (int) Port on which the server runs. debug_server_url: (str) grpc:// URL to the server. server_dump_dir: (str) The debug server's dump directory. server_thread: The server Thread object. server: The `EventListenerTestServicer` object. Raises: ValueError: If polling the server process for ready state is not successful within maximum polling count. """ server_port = portpicker.pick_unused_port() debug_server_url = "grpc://localhost:%d" % server_port server_dump_dir = tempfile.mkdtemp() if dump_to_filesystem else None server = EventListenerTestServicer( server_port=server_port, dump_dir=server_dump_dir, toggle_watch_on_core_metadata=toggle_watch_on_core_metadata) def delay_then_run_server(): time.sleep(server_start_delay_sec) server.run_server(blocking=blocking) server_thread = threading.Thread(target=delay_then_run_server) server_thread.start() if poll_server: if not _poll_server_till_success( 50, 0.2, debug_server_url, server_dump_dir, server, gpu_memory_fraction=0.1): raise ValueError( "Failed to start test gRPC debug server at port %d" % server_port) server.clear_data() return server_port, debug_server_url, server_dump_dir, server_thread, server
def __init__(self, emulator_cmd=None, deadline=10, start_options=(), silent=False): """Constructs a DatastoreEmulator. Clients should use DatastoreEmulatorFactory to construct DatastoreEmulator instances. Args: emulator_cmd: A string representing the path to an executable script that invokes emulator binary. deadline: A integer representing number of seconds to wait for the datastore to start. start_options: A list of additional command-line options to pass to the emulator 'start' command. silent: A bool indicates if emulator runs in silent mode. Raises: IOError: if the emulator failed to start within the deadline """ self._emulator_cmd = emulator_cmd self._http = httplib2.Http() self.__running = False self._silent = silent self._redirected_output = open(os.devnull, 'wb') if self._silent else None # Start the emulator and wait for it to start responding to requests. cmd = [self._emulator_cmd, 'start'] + _DEFAULT_EMULATOR_OPTIONS if start_options: cmd.extend(start_options) port = ParsePortFromOption(start_options or []) if not port: port = portpicker.pick_unused_port() cmd.append('--port=%d' % port) self._host = 'http://localhost:%d' % port cmd.append(tempfile.mkdtemp()) # On windows, cloud_datastore_emulator.bat always prompts up # 'Terminate batch job (Y/N)'. Passing nul to this .bat avoids self.Stop() # hang at this prompt up. if sys.platform.startswith('win'): cmd.append('<nul') popen_kwargs = {} if self._silent: popen_kwargs.update( stdout=self._redirected_output, stderr=self._redirected_output) self.emulator_proc = subprocess.Popen(cmd, **popen_kwargs) if not self._WaitForStartup(deadline): raise IOError('emulator did not respond within %ds' % deadline) self.__datastore = None self.__running = True
def test_basic(self): """Basic functionality test of START_PROCESS_REVERSE flavor.""" portpicker.pick_unused_port().AndReturn(2345) # As the lock is mocked out, this provides a mox expectation. with self.proxy._process_lock: safe_subprocess.start_process_file( args=['/runtime'], input_string=self.runtime_config.SerializeToString(), env={'foo': 'bar', 'PORT': '2345'}, cwd=self.tmpdir, stderr=subprocess.PIPE).AndReturn(self.process) self.proxy._stderr_tee = FakeTee('') self.mox.ReplayAll() self.proxy.start() self.assertEquals(2345, self.proxy._proxy._port) self.mox.VerifyAll()
def create_local_cluster(num_workers, num_ps, protocol="grpc"): """Create local GRPC servers and return their servers.""" worker_ports = [portpicker.pick_unused_port() for _ in range(num_workers)] ps_ports = [portpicker.pick_unused_port() for _ in range(num_ps)] cluster_dict = { "worker": ["localhost:%s" % port for port in worker_ports], "ps": ["localhost:%s" % port for port in ps_ports]} cs = tf.train.ClusterSpec(cluster_dict) workers = [ tf.train.Server( cs, job_name="worker", protocol=protocol, task_index=ix, start=True) for ix in range(num_workers)] ps_servers = [ tf.train.Server( cs, job_name="ps", protocol=protocol, task_index=ix, start=True) for ix in range(num_ps)] return workers, ps_servers
def __init__(self, tmux_window, job, task_id): self.tmux_window = tmux_window self.job = job self.ip = '127.0.0.1' # hostname/ip address self.id = task_id self.port = portpicker.pick_unused_port() self.connect_instructions = 'tmux a -t '+self.tmux_window self.last_stdout = '<unavailable>' # compatiblity with aws.py:Task self.last_stderr = '<unavailable>'
def testPickUnusedCanSuccessfullyUsePortServer(self): with mock.patch.object(portpicker, '_pick_unused_port_without_server'): portpicker._pick_unused_port_without_server.side_effect = ( Exception('eek!') ) # Since _PickUnusedPortWithoutServer() raises an exception, if we # can successfully obtain a port, the portserver must be working. port = portpicker.pick_unused_port() self.assertTrue(self.IsUnusedTCPPort(port)) self.assertTrue(self.IsUnusedUDPPort(port))
def test_silent_request(self): """Call log_message once on the silent request handler for coverage's sake.""" handler = AdderHandler() server = easyrpc.on(handler, adder_service, 'localhost', portpicker.pick_unused_port()) class _DumbHandler(server.httpd.RequestHandlerClass): def __init__(self): pass http_request_handler = _DumbHandler() http_request_handler.log_message('unused')
def setUp(self): super(ApiSslServerTestBase, self).setUp() key = rdf_crypto.RSAPrivateKey.GenerateKey() key_path = os.path.join(self.temp_dir, "key.pem") with open(key_path, "wb") as f: f.write(key.AsPEM()) subject = issuer = x509.Name([ x509.NameAttribute(oid.NameOID.COMMON_NAME, u"localhost"), ]) cert = x509.CertificateBuilder().subject_name(subject).issuer_name( issuer).public_key( key.GetPublicKey().GetRawPublicKey()).serial_number( x509.random_serial_number()).not_valid_before( datetime.datetime.utcnow()).not_valid_after( datetime.datetime.utcnow() + datetime.timedelta(days=1)).add_extension( x509.SubjectAlternativeName( [x509.DNSName(u"localhost")]), critical=False, ).sign(key.GetRawPrivateKey(), hashes.SHA256(), backends.default_backend()) self.cert_path = os.path.join(self.temp_dir, "certificate.pem") with open(self.cert_path, "wb") as f: f.write(cert.public_bytes(serialization.Encoding.PEM)) config_overrider = test_lib.ConfigOverrider({ "AdminUI.enable_ssl": True, "AdminUI.ssl_key_file": key_path, "AdminUI.ssl_cert_file": self.cert_path, }) config_overrider.Start() self.addCleanup(config_overrider.Stop) self.port = portpicker.pick_unused_port() thread = wsgiapp_testlib.ServerThread(self.port, name="ApiSslServerTest") thread.StartAndWaitUntilServing() self.addCleanup(thread.Stop) api_auth_manager.InitializeApiAuthManager( api_call_router_without_checks.ApiCallRouterWithoutChecks) self.token.username = "******" webauth.WEBAUTH_MANAGER.SetUserName(self.token.username) self.endpoint = "https://localhost:%s" % self.port
def checkIsPortFree(self): """This might be flaky unless this test is run with a portserver.""" # The port should be free initially. port = portpicker.pick_unused_port() self.assertTrue(portpicker.is_port_free(port)) cases = [ (socket.AF_INET, socket.SOCK_STREAM, None), (socket.AF_INET6, socket.SOCK_STREAM, 0), (socket.AF_INET6, socket.SOCK_STREAM, 1), (socket.AF_INET, socket.SOCK_DGRAM, None), (socket.AF_INET6, socket.SOCK_DGRAM, 0), (socket.AF_INET6, socket.SOCK_DGRAM, 1), ] for (sock_family, sock_type, v6only) in cases: # Occupy the port on a subset of possible protocols. try: sock = socket.socket(sock_family, sock_type, 0) except socket.error: print('Kernel does not support sock_family=%d' % sock_family, file=sys.stderr) # Skip this case, since we cannot occupy a port. continue if not hasattr(socket, 'IPPROTO_IPV6'): v6only = None if v6only is not None: try: sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, v6only) except socket.error: print('Kernel does not support IPV6_V6ONLY=%d' % v6only, file=sys.stderr) # Don't care; just proceed with the default. # Socket may have been taken in the mean time, so catch the # socket.error with errno set to EADDRINUSE and skip this # attempt. try: sock.bind(('', port)) except socket.error as e: if e.errno == errno.EADDRINUSE: raise portpicker.NoFreePortFoundError raise # The port should be busy. self.assertFalse(portpicker.is_port_free(port)) sock.close() # Now it's free again. self.assertTrue(portpicker.is_port_free(port))
def start_server_on_separate_thread(dump_to_filesystem=True, server_start_delay_sec=0.0, poll_server=False, blocking=True): """Create a test gRPC debug server and run on a separate thread. Args: dump_to_filesystem: (bool) whether the debug server will dump debug data to the filesystem. server_start_delay_sec: (float) amount of time (in sec) to delay the server start up for. poll_server: (bool) whether the server will be polled till success on startup. blocking: (bool) whether the server should be started in a blocking mode. Returns: server_port: (int) Port on which the server runs. debug_server_url: (str) grpc:// URL to the server. server_dump_dir: (str) The debug server's dump directory. server_thread: The server Thread object. server: The `EventListenerTestServicer` object. Raises: ValueError: If polling the server process for ready state is not successful within maximum polling count. """ server_port = portpicker.pick_unused_port() debug_server_url = "grpc://localhost:%d" % server_port server_dump_dir = tempfile.mkdtemp() if dump_to_filesystem else None server = EventListenerTestServicer(server_port=server_port, dump_dir=server_dump_dir) def delay_then_run_server(): time.sleep(server_start_delay_sec) server.run_server(blocking=blocking) server_thread = threading.Thread(target=delay_then_run_server) server_thread.start() if poll_server: if not _poll_server_till_success(50, 0.2, debug_server_url, server_dump_dir, server, gpu_memory_fraction=0.1): raise ValueError( "Failed to start test gRPC debug server at port %d" % server_port) server.clear_data() return server_port, debug_server_url, server_dump_dir, server_thread, server
def setUp(self): super(ExecutorServiceTest, self).setUp() port = portpicker.pick_unused_port() server_pool = logging_pool.pool(max_workers=1) self._server = grpc.server(server_pool) self._server.add_insecure_port('[::]:{}'.format(port)) self._service = executor_service.ExecutorService( eager_executor.EagerExecutor()) executor_pb2_grpc.add_ExecutorServicer_to_server( self._service, self._server) self._server.start() self._channel = grpc.insecure_channel('localhost:{}'.format(port)) self._stub = executor_pb2_grpc.ExecutorStub(self._channel)
def main(argv): logging.basicConfig( level=logging.INFO, format='%(levelname)s: %(threadName)s %(message)s') directory = os.getcwd() if argv[1:] == ['server']: port = gflags.FLAGS.port or portpicker.pick_unused_port() compilation_server_lib.RunServerAtPort(port) elif argv[1:] == ['local']: port = gflags.FLAGS.port or portpicker.pick_unused_port() server_proc = subprocess.Popen( ["python", argv[0], "--port", str(port), "server"]) server_address = '127.0.0.1:{}'.format(port) compilation_client_lib.WaitTillHealthy(server_address) try: RunEditingEnvironment( directory, server_address=server_address) finally: try: logging.info("Terminating latex compilation server.") server_proc.kill() except OSError: pass elif argv[1:] == []: if not gflags.FLAGS.using_server: logging.fatal( "Please specify the server address when running in the client mode " "via --using_server") RunEditingEnvironment(directory, server_address=gflags.FLAGS.using_server) else: print "Unable to recognize command %r" % argv print __doc__ sys.exit(1)
def local_port(self): """Selects a local port. Returns: int, local port ready for use. """ if not self._local_port: forwarded_port = self.get_forwarded_port() if forwarded_port: self._local_port = forwarded_port else: self._local_port = portpicker.pick_unused_port() return self._local_port
def _create_cluster_spec(has_chief=False, num_workers=1, num_ps=0, has_eval=False): if _portpicker_import_error: raise _portpicker_import_error # pylint: disable=raising-bad-type cluster_spec = {} if has_chief: cluster_spec[CHIEF] = ["localhost:%s" % portpicker.pick_unused_port()] if num_workers: cluster_spec[WORKER] = [ "localhost:%s" % portpicker.pick_unused_port() for _ in range(num_workers) ] if num_ps: cluster_spec[PS] = [ "localhost:%s" % portpicker.pick_unused_port() for _ in range(num_ps) ] if has_eval: cluster_spec[EVALUATOR] = ["localhost:%s" % portpicker.pick_unused_port()] return cluster_spec
def setUp(self): super(RpcOpTest, self).setUp() service_port = portpicker.pick_unused_port() server = grpc.server(logging_pool.pool(max_workers=25)) servicer = rpc_op_test_servicer.RpcOpTestServicer() test_example_pb2_grpc.add_TestCaseServiceServicer_to_server( servicer, server) self._address = 'localhost:%d' % service_port server.add_insecure_port(self._address) server.start() self._server = server
def test_server_context_shuts_down_uncaught_exception( self, mock_logging_info): ex = eager_tf_executor.EagerTFExecutor() ex_factory = executor_factory.ExecutorFactoryImpl(lambda _: ex) with self.assertRaises(TypeError): with server_utils.server_context( ex_factory, 1, portpicker.pick_unused_port()) as server: time.sleep(1) raise TypeError mock_logging_info.assert_called_once_with('Shutting down server.')
def testExecutionWatcher_LocalWithEmptyRequest(self): port = portpicker.pick_unused_port() sidecar = execution_watcher.ExecutionWatcher( port, creds=grpc.local_server_credentials()) sidecar.start() creds = grpc.local_channel_credentials() channel = grpc.secure_channel(sidecar.local_address, creds) stub = execution_watcher_pb2_grpc.ExecutionWatcherServiceStub(channel) req = execution_watcher_pb2.UpdateExecutionInfoRequest() res = stub.UpdateExecutionInfo(req) sidecar.stop() self.assertEqual(execution_watcher_pb2.UpdateExecutionInfoResponse(), res)
def __init__(self, host="127.0.0.1", port=None, fullscreen=False): assert isinstance(host, str) assert isinstance(port, int) or port is None self._fullscreen = fullscreen self._host = host if port is None: self._port = portpicker.pick_unused_port() else: self._port = port self._tmp_dir = tempfile.mkdtemp(prefix="SC2_") self._process = None self._ws = None
def _SetupServer(cls): for _ in range(5): try: port = portpicker.pick_unused_port() cls.server = shared_fake_data_store.SharedFakeDataStoreServer( port) config.CONFIG.Set("SharedFakeDataStore.port", port) return except socket.error as e: if e.errno != errno.EADDRINUSE: raise raise RuntimeError("Unable to find an unused port after 5 tries.")
def _create_in_process_tf_ps_cluster(num_workers, num_ps): """Create a cluster of TF workers and returns their addresses. Such cluster simulate the behavior of multiple TF parameter servers. Args: num_workers: Number of "worker" workers. num_ps: Number of "parameter server" workers. Returns: The ClusterResolver i.e. the ip addresses of the workers. """ worker_ports = [portpicker.pick_unused_port() for _ in range(num_workers)] ps_ports = [portpicker.pick_unused_port() for _ in range(num_ps)] cluster_dict = {"worker": ["localhost:%s" % port for port in worker_ports]} if num_ps > 0: cluster_dict["ps"] = ["localhost:%s" % port for port in ps_ports] cluster_spec = tf.train.ClusterSpec(cluster_dict) worker_config = tf.compat.v1.ConfigProto() for i in range(num_workers): tf.distribute.Server( cluster_spec, job_name="worker", task_index=i, config=worker_config, protocol="grpc") for i in range(num_ps): tf.distribute.Server( cluster_spec, job_name="ps", task_index=i, protocol="grpc") os.environ["GRPC_FAIL_FAST"] = "use_caller" return tf.distribute.cluster_resolver.SimpleClusterResolver( cluster_spec, rpc_layer="grpc")
def setUp(self): self._debugger_data_server_grpc_port = portpicker.pick_unused_port() self._debug_url = ( "grpc://localhost:%d" % self._debugger_data_server_grpc_port) self._logdir = tempfile.mkdtemp(prefix="tensorboard_dds_") self._debug_data_server = debugger_server_lib.DebuggerDataServer( self._debugger_data_server_grpc_port, self._logdir, always_flush=True) self._server_thread = threading.Thread( target=self._debug_data_server.start_the_debugger_data_receiving_server) self._server_thread.start() self.assertTrue(self._poll_server_till_success(50, 0.2))
def setUpClass(cls): def start_server(port): signal.signal(signal.SIGTERM, lambda signum, frame: os._exit(0)) logging.disable(logging.CRITICAL) handler = AdderHandler() server = easyrpc.on(handler, adder_service, 'localhost', port) server.serve() cls._server_port = portpicker.pick_unused_port() cls._server_p = multiprocessing.Process(target=start_server, args=(cls._server_port,)) cls._server_p.start()
def pick_unused_port(): """Returns an unused and unassigned local port.""" if _portpicker_import_error: raise _portpicker_import_error # pylint: disable=raising-bad-type global ASSIGNED_PORTS with lock: while True: port = portpicker.pick_unused_port() if port > 10000 and port not in ASSIGNED_PORTS: ASSIGNED_PORTS.add(port) logging.info('Using local port %r', port) return port
def create_local_cluster(num_workers, num_ps, protocol="grpc"): """Create local GRPC servers and return them.""" worker_ports = [portpicker.pick_unused_port() for _ in range(num_workers)] ps_ports = [portpicker.pick_unused_port() for _ in range(num_ps)] cluster_dict = { "worker": ["localhost:%s" % port for port in worker_ports], "ps": ["localhost:%s" % port for port in ps_ports] } cs = server_lib.ClusterSpec(cluster_dict) workers = [ server_lib.Server( cs, job_name="worker", protocol=protocol, task_index=ix, start=True) for ix in range(num_workers) ] ps_servers = [ server_lib.Server( cs, job_name="ps", protocol=protocol, task_index=ix, start=True) for ix in range(num_ps) ] return cluster_dict, workers, ps_servers
def load_from_disk(path, settings): """Load Hard Eight Tasks from disk. Args: path: Directory containing dm_hard_eight environment. settings: EnvironmentSettings required to start the environment. Returns: An implementation of dm_env.Environment. Raises: RuntimeError: If unable to start environment process. """ _validate_environment_settings(settings) executable_path = os.path.join(path, 'Linux64Player') libosmesa_path = os.path.join(path, 'external_libosmesa_llvmpipe.so') if not os.path.exists(executable_path) or not os.path.exists( libosmesa_path): raise RuntimeError( 'Cannot find dm_hard_eight executable or dependent files at path: {}' .format(path)) port = portpicker.pick_unused_port() process_flags = [ executable_path, # Unity command-line flags. '-logfile', '-batchmode', '-noaudio', # Other command-line flags. '--logtostderr', '--server_type=DM_ENV_RPC', '--uri_address=[::]:{}'.format(port), ] os.environ.update({ 'UNITY_RENDERER': 'software', 'UNITY_OSMESA_PATH': libosmesa_path, }) process = subprocess.Popen(process_flags, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) if process.poll() is not None: raise RuntimeError('Failed to start dm_hard_eight process correctly.') return _HardEightTasksProcessEnv(_connect_to_environment(port, settings), _HARD_EIGHT_TASK_OBSERVATIONS, settings.num_action_repeats, process)
async def run_match(controllers: List[Controller], match: GameMatch, close_ws=True): await _setup_host_game(controllers[0], **match.host_game_kwargs) # Setup portconfig beforehand, so all players use the same ports startport = None portconfig = None if match.needed_sc2_count > 1: if any(isinstance(player, BotProcess) for player in match.players): portconfig = Portconfig.contiguous_ports() # Most ladder bots generate their server and client ports as [s+2, s+3], [s+4, s+5] startport = portconfig.server[0] - 2 else: portconfig = Portconfig() proxies = [] coros = [] i = 0 for player in match.players: if player.needs_sc2: if isinstance(player, BotProcess): pport = portpicker.pick_unused_port() p = Proxy(controllers[i], player, pport, match.game_time_limit, match.realtime) proxies.append(p) coros.append(p.play_with_proxy(startport)) else: coros.append( play_from_websocket( controllers[i]._ws, player, match.realtime, portconfig, should_close=close_ws, game_time_limit=match.game_time_limit, )) i += 1 # async_results = await asyncio.wait_for(asyncio.gather(*coros, return_exceptions=True), timeout=None) async_results = await asyncio.gather(*coros, return_exceptions=True) if not isinstance(async_results, list): async_results = [async_results] for i, a in enumerate(async_results): if isinstance(a, Exception): logger.error( f"Exception[{a}] thrown by {[p for p in match.players if p.needs_sc2][i]}" ) return process_results(match.players, async_results)
def download(filename): """Downloads the file to the user's local disk via a browser download action. Args: filename: Name of the file on disk to be downloaded. Raises: OSError: if the file cannot be found. """ if not os.path.exists(filename): msg = 'Cannot find file: {}'.format(filename) if six.PY2: raise OSError(msg) else: raise FileNotFoundError(msg) # pylint: disable=undefined-variable started = threading.Event() port = portpicker.pick_unused_port() def server_entry(): httpd = _V6Server(('::', port), _FileHandler) started.set() # Handle a single request then exit the thread. httpd.handle_request() thread = threading.Thread(target=server_entry) thread.start() started.wait() output.eval_js( """ (async function() { const response = await fetch('https://localhost:%(port)d%(path)s'); if (!response.ok) { throw new Error('Failed to download: ' + response.statusText); } const blob = await response.blob(); const a = document.createElement('a'); a.href = window.URL.createObjectURL(blob); a.download = '%(name)s'; document.body.appendChild(a); a.click(); a.remove(); })(); """ % { 'port': port, 'path': os.path.abspath(filename), 'name': os.path.basename(filename), })
def getUnusedPort(): """Returns an unused TCP port on the local host inside the defined port range. To be used when starting peer SAS webserver or other database webserver. """ config = _GetSharedTestConfig() if config.min_port < 0: return portpicker.pick_unused_port() global _ports # Find the first available port in the defined range. for p in xrange(config.min_port, config.max_port): if p not in _ports and portpicker.is_port_free(p): _ports.add(p) return p raise AssertionError('No available new ports')
def setUp(self): super(ApiSslProxyTest, self).setUp() attempts_count = 0 self.proxy_server = None while self.proxy_server is None: try: self.proxy_port = portpicker.pick_unused_port() self.proxy_server = TCPServerV6(("::", self.proxy_port), Proxy) except socket.error: attempts_count += 1 if attempts_count == 10: self.fail("Can't initialize proxy server.") threading.Thread(target=self.proxy_server.serve_forever).start()
def test_server_context_shuts_down_under_keyboard_interrupt( self, mock_logging_info): ex = eager_tf_executor.EagerTFExecutor() with server_utils.server_context( ex, 1, portpicker.pick_unused_port()) as server: time.sleep(1) raise KeyboardInterrupt mock_logging_info.assert_has_calls([ mock.call('Server stopped by KeyboardInterrupt.'), mock.call('Shutting down server.') ])
def __init__(self, config_path, num_players=2, num_bots=0, mode='train', train_mode='frag', max_steps=2100, episode_timeout=2100, is_window_visible=False, is_window_cv_visible=False): self.port = pick_unused_port() self.mode = mode self.train_mode = train_mode # host cfg self.host_cfg = PlayerHostConfig(self.port) self.host_cfg.num_players = num_players # join cfg self.join_cfg = PlayerJoinConfig(self.port) # player cfg self.players_cfg = [] for i in range(self.host_cfg.num_players): cfg = PlayerConfig() cfg.config_path = config_path cfg.player_mode = vd.Mode.PLAYER cfg.screen_resolution = vd.ScreenResolution.RES_800X450 cfg.screen_format = vd.ScreenFormat.CBCGCR cfg.is_window_visible = is_window_visible # cfg.ticrate = vd.DEFAULT_TICRATE * 2 cfg.episode_timeout = episode_timeout if i == 0: # host cfg.host_cfg = self.host_cfg cfg.name = 'WhoAmI' cfg.num_bots = num_bots else: cfg.join_cfg = self.join_cfg cfg.name = 'P{}'.format(i) self.players_cfg.append(cfg) # the player-wise env and the vec-env # TODO(jackzbzheng): add different observation wrappers self.envs = [] for cfg in self.players_cfg: e = PlayerEnv(cfg, train_mode=self.train_mode) #e = RwdShapeWu2(e, dist_penalty_thres=3) self.envs.append(e) self.env = VecEnv(self.envs) self.observation_space = self.env.observation_space self.action_space = self.env.action_space
async def test_endpoint_task(): port = portpicker.pick_unused_port() cfg = EndpointConfig( interval=0.000001, timeout=1.0, description="test", url=f"http://*****:*****@routes.get("/") async def all(request: web.Request) -> web.Response: nonlocal request_count request_count += 1 return web.Response(text="OK") app = web.Application() app.add_routes(routes) runner = web.AppRunner(app) await runner.setup() tcp_site = web.TCPSite(runner, "localhost", port) call_count = 0 async def callback(msg): nonlocal call_count if call_count == 0: assert msg.reached == True call_count = False await runner.cleanup() # Stop the web server elif call_count == 1: assert msg.reached == False else: raise RuntimeError("Stop") call_count += 1 await tcp_site.start() try: await endpoint_task(cfg, callback) except RuntimeError as e: assert str(e) == "Stop" assert request_count == 1
def contiguous_ports(cls, guests=1, attempts=40): """Returns a Portconfig with adjacent ports""" for _ in range(attempts): start = portpicker.pick_unused_port() others = [start + j for j in range(1, 2 + guests * 2)] if all(portpicker.is_port_free(p) for p in others): server_ports = [start, others.pop(0)] player_ports = [] while others: player_ports.append([others.pop(0), others.pop(0)]) pc = cls(server_ports=server_ports, player_ports=player_ports) pc._picked_ports.append(start) return pc raise portpicker.NoFreePortFoundError()
def start_train_process(self, conf_path, run_id, options=None): unused_port = portpicker.pick_unused_port() command = [ 'python', 'learn.py', self.env_name, '--train', '--worker-id=' + str(unused_port), '--trainer-config-path=' + str(conf_path), '--run-id=' + run_id ] if options: command.append(options) proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) return proc
def _create_cluster_spec(self, has_chief=False, num_workers=1, num_ps=0, has_eval=False): if _portpicker_import_error: raise _portpicker_import_error # pylint: disable=raising-bad-type cluster_spec = {} if has_chief: cluster_spec[CHIEF] = ["localhost:%s" % portpicker.pick_unused_port()] if num_workers: cluster_spec[WORKER] = [ "localhost:%s" % portpicker.pick_unused_port() for _ in range(num_workers) ] if num_ps: cluster_spec[PS] = [ "localhost:%s" % portpicker.pick_unused_port() for _ in range(num_ps) ] if has_eval: cluster_spec[EVALUATOR] = ["localhost:%s" % portpicker.pick_unused_port()] return cluster_spec
def server_entry(): if ('HTTPD' in cfg) and (cfg['HTTPD']): stopLocalServer() cfg['HTTPD'] = None handler = SimpleHTTPServer.SimpleHTTPRequestHandler port = portpicker.pick_unused_port() with JBcd(cfg['REVEAL_DIR']): os.chdir(cfg['REVEAL_DIR']) httpd = V6Server(("::", port), handler) print("serving at port", port, 'cwd', os.getcwd(), 'reveal', cfg['REVEAL_DIR']) cfg['HTTPD'] = httpd cfg['HTTP_PORT'] = port httpd.serve_forever()
def setUpClass(cls): super(ApiSslServerTestBase, cls).setUpClass() if ApiSslServerTestBase._api_set_up_done: return key = rdf_crypto.RSAPrivateKey.GenerateKey() key_path = temp.TempFilePath("key.pem") with open(key_path, "wb") as f: f.write(key.AsPEM()) subject = issuer = x509.Name([ x509.NameAttribute(oid.NameOID.COMMON_NAME, u"localhost"), ]) cert = x509.CertificateBuilder().subject_name(subject).issuer_name( issuer).public_key( key.GetPublicKey().GetRawPublicKey()).serial_number( x509.random_serial_number()).not_valid_before( datetime.datetime.utcnow()).not_valid_after( datetime.datetime.utcnow() + datetime.timedelta(days=1)).add_extension( x509.SubjectAlternativeName( [x509.DNSName(u"localhost")]), critical=False, ).sign(key.GetRawPrivateKey(), hashes.SHA256(), backends.default_backend()) ApiSslServerTestBase.ssl_cert_path = temp.TempFilePath( "certificate.pem") with open(ApiSslServerTestBase.ssl_cert_path, "wb") as f: f.write(cert.public_bytes(serialization.Encoding.PEM)) ApiSslServerTestBase.ssl_port = portpicker.pick_unused_port() with test_lib.ConfigOverrider({ "AdminUI.enable_ssl": True, "AdminUI.ssl_key_file": key_path, "AdminUI.ssl_cert_file": ApiSslServerTestBase.ssl_cert_path, }): ApiSslServerTestBase._ssl_trd = wsgiapp_testlib.ServerThread( ApiSslServerTestBase.ssl_port, name="ApiSslServerTest") ApiSslServerTestBase._ssl_trd.StartAndWaitUntilServing() ApiSslServerTestBase.ssl_endpoint = ("https://localhost:%s" % ApiSslServerTestBase.ssl_port) ApiSslServerTestBase._api_set_up_done = True
def testTrace_ProfileIdleServerWithOptions(self): test_port = portpicker.pick_unused_port() profiler.start_server(test_port) # Test the profilers are successfully started and connected to profiler # service on the worker. Since there is no op running, it is expected to # return UnavailableError with no trace events collected string. with self.assertRaises(errors.UnavailableError) as error: options = profiler.ProfilerOptions(host_tracer_level=3, device_tracer_level=0) profiler_client.trace('localhost:' + str(test_port), self.get_temp_dir(), duration_ms=10, options=options) self.assertEqual('No trace event is collected', str(error.exception))
def get_available_host_port(): """Gets a host port number available for adb forward. Returns: An integer representing a port number on the host available for adb forward. Raises: Error: when no port is found after MAX_PORT_ALLOCATION_RETRY times. """ # Only import adb module if needed. from mobly.controllers.android_device_lib import adb port = portpicker.pick_unused_port() if not adb.is_adb_available(): return port for _ in range(MAX_PORT_ALLOCATION_RETRY): # Make sure adb is not using this port so we don't accidentally # interrupt ongoing runs by trying to bind to the port. if port not in adb.list_occupied_adb_ports(): return port port = portpicker.pick_unused_port() raise Error('Failed to find available port after {} retries'.format( MAX_PORT_ALLOCATION_RETRY))
def testSendFileTransmissionFailure(self): unused_port = portpicker.pick_unused_port() session = gcs.UploadSession(f"https://localhost:{unused_port}") opts = gcs.UploadSession.Opts() opts.retry_chunk_attempts = 1 opts.retry_chunk_init_delay = 0.0 with self.assertRaises(gcs.RequestError) as context: session.SendFile(io.BytesIO(b"foobar"), opts=opts) cause = context.exception.__cause__ self.assertIsInstance(cause, exceptions.ConnectionError)
def setUpClass(cls): super(GRRSeleniumTest, cls).setUpClass() with GRRSeleniumTest._selenium_set_up_lock: if not GRRSeleniumTest._selenium_set_up_done: port = portpicker.pick_unused_port() logging.info("Picked free AdminUI port %d.", port) # Start up a server in another thread GRRSeleniumTest._server_trd = wsgiapp_testlib.ServerThread( port, name="SeleniumServerThread") GRRSeleniumTest._server_trd.StartAndWaitUntilServing() GRRSeleniumTest._SetUpSelenium(port) GRRSeleniumTest._selenium_set_up_done = True
def setUpClass(cls): super(ApiE2ETest, cls).setUpClass() with ApiE2ETest._api_set_up_lock: if not ApiE2ETest._api_set_up_done: # Set up HTTP server port = portpicker.pick_unused_port() ApiE2ETest.server_port = port logging.info("Picked free AdminUI port for HTTP %d.", port) ApiE2ETest.trd = wsgiapp_testlib.ServerThread( port, name="api_e2e_server") ApiE2ETest.trd.StartAndWaitUntilServing() ApiE2ETest._api_set_up_done = True
def _create_local_cluster(self, **kargs): """Creates a local cluster.""" cluster_dict = {} for (k, v) in kargs.items(): cluster_dict[k] = [ 'localhost:%d' % portpicker.pick_unused_port() for _ in range(v)] # Launch servers: servers = {} for (k, v) in kargs.items(): servers[k] = [tf.train.Server(cluster_dict, job_name=k, task_index=idx, start=True) for idx in range(v)] return servers
def setUp(self): super(ApiSslProxyTest, self).setUp() attempts_count = 0 self.proxy_server = None while self.proxy_server is None: try: self.proxy_port = portpicker.pick_unused_port() self.proxy_server = TCPServerV6(("::", self.proxy_port), Proxy) except socket.error: attempts_count += 1 if attempts_count == 10: self.fail("Can't initialize proxy server.") threading.Thread(target=self.proxy_server.serve_forever).start() self.addCleanup(self.proxy_server.server_close) self.addCleanup(self.proxy_server.shutdown)
def setUp(self): super(ApiSslServerTestBase, self).setUp() key = rdf_crypto.RSAPrivateKey.GenerateKey() key_path = os.path.join(self.temp_dir, "key.pem") with open(key_path, "wb") as f: f.write(key.AsPEM()) subject = issuer = x509.Name([ x509.NameAttribute(oid.NameOID.COMMON_NAME, u"localhost"), ]) cert = x509.CertificateBuilder().subject_name(subject).issuer_name( issuer).public_key(key.GetPublicKey().GetRawPublicKey()).serial_number( x509.random_serial_number()).not_valid_before( datetime.datetime.utcnow()).not_valid_after( datetime.datetime.utcnow() + datetime.timedelta(days=1)).add_extension( x509.SubjectAlternativeName( [x509.DNSName(u"localhost")]), critical=False, ).sign(key.GetRawPrivateKey(), hashes.SHA256(), backends.default_backend()) self.cert_path = os.path.join(self.temp_dir, "certificate.pem") with open(self.cert_path, "wb") as f: f.write(cert.public_bytes(serialization.Encoding.PEM)) config_overrider = test_lib.ConfigOverrider({ "AdminUI.enable_ssl": True, "AdminUI.ssl_key_file": key_path, "AdminUI.ssl_cert_file": self.cert_path, }) config_overrider.Start() self.addCleanup(config_overrider.Stop) self.port = portpicker.pick_unused_port() thread = wsgiapp_testlib.ServerThread(self.port, name="ApiSslServerTest") thread.StartAndWaitUntilServing() self.addCleanup(thread.Stop) api_auth_manager.APIACLInit.InitApiAuthManager() self.token.username = "******" webauth.WEBAUTH_MANAGER.SetUserName(self.token.username) self.endpoint = "https://localhost:%s" % self.port
def launch_distributed_service(): port = portpicker.pick_unused_port() def launch_worker(worker_type): my_env = os.environ.copy() if not FLAGS.verbose: my_env["CUDA_VISIBLE_DEVICES"] = "" my_env["TF_CPP_MIN_LOG_LEVEL"] = "3" if FLAGS.profile: my_env["LD_PRELOAD"]="/usr/lib/libtcmalloc_and_profiler.so.4" my_env["CPUPROFILE"]="/tmp/profile.out.%s"%(worker_type) args = ["python"] + sys.argv + ["--port="+str(port), "--worker_type="+worker_type] proc = subprocess.Popen(args, stderr=subprocess.STDOUT, env=my_env) log("worker %s pid %s"%(worker_type, proc.pid)) launch_worker("worker") launch_worker("client")
def setUpClass(cls): super(GRRHTTPServerTest, cls).setUpClass() # Bring up a local server for testing. port = portpicker.pick_unused_port() ip = utils.ResolveHostnameToIP("localhost", port) cls.httpd = frontend.GRRHTTPServer((ip, port), frontend.GRRHTTPServerHandler) if ipaddress.ip_address(ip).version == 6: cls.address_family = socket.AF_INET6 cls.base_url = "http://[%s]:%d/" % (ip, port) else: cls.address_family = socket.AF_INET cls.base_url = "http://%s:%d/" % (ip, port) cls.httpd_thread = threading.Thread( name="GRRHTTPServerTestThread", target=cls.httpd.serve_forever) cls.httpd_thread.daemon = True cls.httpd_thread.start()
def setUp(self): super(InteractiveDebuggerPluginTest, self).setUp() self._dummy_logdir = tempfile.mkdtemp() self._dummy_multiplexer = event_multiplexer.EventMultiplexer({}) self._debugger_port = portpicker.pick_unused_port() self._debugger_url = 'grpc://localhost:%d' % self._debugger_port context = base_plugin.TBContext(logdir=self._dummy_logdir, multiplexer=self._dummy_multiplexer) self._debugger_plugin = ( interactive_debugger_plugin.InteractiveDebuggerPlugin(context)) self._debugger_plugin.listen(self._debugger_port) wsgi_app = application.TensorBoardWSGIApp( self._dummy_logdir, [self._debugger_plugin], self._dummy_multiplexer, reload_interval=0, path_prefix='') self._server = werkzeug_test.Client(wsgi_app, wrappers.BaseResponse)
def testIsPortFree(self): """This might be flaky unless this test is run with a portserver.""" # The port should be free initially. port = portpicker.pick_unused_port() self.assertTrue(portpicker.is_port_free(port)) cases = [ (socket.AF_INET, socket.SOCK_STREAM, None), (socket.AF_INET6, socket.SOCK_STREAM, 0), (socket.AF_INET6, socket.SOCK_STREAM, 1), (socket.AF_INET, socket.SOCK_DGRAM, None), (socket.AF_INET6, socket.SOCK_DGRAM, 0), (socket.AF_INET6, socket.SOCK_DGRAM, 1), ] for (sock_family, sock_type, v6only) in cases: # Occupy the port on a subset of possible protocols. try: sock = socket.socket(sock_family, sock_type, 0) except socket.error: print('Kernel does not support sock_family=%d' % sock_family, file=sys.stderr) # Skip this case, since we cannot occupy a port. continue if v6only is not None: try: sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, v6only) except socket.error: print('Kernel does not support IPV6_V6ONLY=%d' % v6only, file=sys.stderr) # Don't care; just proceed with the default. sock.bind(('', port)) # The port should be busy. self.assertFalse(portpicker.is_port_free(port)) sock.close() # Now it's free again. self.assertTrue(portpicker.is_port_free(port))