def testSerialize(self):
# pylint: disable=g-import-not-at-top
try:
import portpicker
except ImportError:
return
with context.graph_mode():
worker_port = portpicker.pick_unused_port()
ps_port = portpicker.pick_unused_port()
cluster_dict = {
"worker": ["localhost:%s" % worker_port],
"ps": ["localhost:%s" % ps_port]
}
cs = server_lib.ClusterSpec(cluster_dict)
worker = server_lib.Server(
cs, job_name="worker", protocol="grpc", task_index=0, start=True)
unused_ps = server_lib.Server(
cs, job_name="ps", protocol="grpc", task_index=0, start=True)
with ops.Graph().as_default(), session.Session(target=worker.target):
with ops.device("/job:worker"):
t = constant_op.constant([[1.0], [2.0]])
l = list_ops.tensor_list_from_tensor(t, element_shape=[1])
with ops.device("/job:ps"):
l_ps = array_ops.identity(l)
l_ps, e = list_ops.tensor_list_pop_back(
l_ps, element_dtype=dtypes.float32)
with ops.device("/job:worker"):
worker_e = array_ops.identity(e)
self.assertAllEqual(worker_e.eval(), [2.0])
def create_local_cluster(num_workers, num_ps, protocol="grpc"):
"""Create and start local servers and return the associated `Server` objects.
Example:
```python
workers, _ = tf.test.create_local_cluster(num_workers=2, num_ps=2)
worker_sessions = [tf.Session(w.target) for w in workers]
with tf.device("/job:ps/task:0"):
...
with tf.device("/job:ps/task:1"):
...
with tf.device("/job:worker/task:0"):
...
with tf.device("/job:worker/task:1"):
...
worker_sessions[0].run(...)
```
Args:
num_workers: Number of worker servers to start.
num_ps: Number of PS servers to start.
protocol: Communication protocol. Allowed values are documented in
the documentation of `tf.train.Server`.
Returns:
A tuple `(worker_servers, ps_servers)`. `worker_servers` is a list
of `num_workers` objects of type `tf.train.Server` (all running locally);
and `ps_servers` is a list of `num_ps` objects of similar type.
Raises:
ImportError: if portpicker module was not found at load time
"""
if _portpicker_import_error:
raise _portpicker_import_error # pylint: disable=raising-bad-type
worker_ports = [portpicker.pick_unused_port() for _ in range(num_workers)]
ps_ports = [portpicker.pick_unused_port() for _ in range(num_ps)]
cluster_dict = {
"worker": ["localhost:%s" % port for port in worker_ports],
"ps": ["localhost:%s" % port for port in ps_ports]
}
cs = server_lib.ClusterSpec(cluster_dict)
workers = [
server_lib.Server(
cs, job_name="worker", protocol=protocol, task_index=ix, start=True)
for ix in range(num_workers)
]
ps_servers = [
server_lib.Server(
cs, job_name="ps", protocol=protocol, task_index=ix, start=True)
for ix in range(num_ps)
]
return workers, ps_servers
def _create_local_cluster(num_workers,
num_ps,
has_eval=False,
protocol="grpc",
worker_config=None,
ps_config=None):
if _portpicker_import_error:
raise _portpicker_import_error # pylint: disable=raising-bad-type
worker_ports = [portpicker.pick_unused_port() for _ in range(num_workers)]
ps_ports = [portpicker.pick_unused_port() for _ in range(num_ps)]
cluster_dict = {
"worker": ["localhost:%s" % port for port in worker_ports],
"ps": ["localhost:%s" % port for port in ps_ports]
}
if has_eval:
cluster_dict["evaluator"] = ["localhost:%s" % portpicker.pick_unused_port()]
cs = server_lib.ClusterSpec(cluster_dict)
workers = [
server_lib.Server(
cs,
job_name="worker",
protocol=protocol,
task_index=ix,
config=worker_config,
start=True) for ix in range(num_workers)
]
ps_servers = [
server_lib.Server(
cs,
job_name="ps",
protocol=protocol,
task_index=ix,
config=ps_config,
start=True) for ix in range(num_ps)
]
if has_eval:
evals = [
server_lib.Server(
cs,
job_name="evaluator",
protocol=protocol,
task_index=0,
config=worker_config,
start=True)
]
else:
evals = []
return workers, ps_servers, evals
def main(argv):
del argv # Unused.
if not flags.FLAGS.dest_server_config_path:
raise ValueError("dest_server_config_path flag has to be provided.")
if not flags.FLAGS.dest_client_config_path:
raise ValueError("dest_client_config_path flag has to be provided.")
admin_ui_port = portpicker.pick_unused_port()
frontend_port = portpicker.pick_unused_port()
source_server_config_path = package.ResourcePath(
"grr-response-core", "install_data/etc/grr-server.yaml")
config_lib.LoadConfig(config.CONFIG, source_server_config_path)
config.CONFIG.SetWriteBack(flags.FLAGS.dest_server_config_path)
# TODO(user): remove when AFF4 is gone.
config.CONFIG.Set("Database.aff4_enabled", False)
config.CONFIG.Set("Database.enabled", True)
config.CONFIG.Set("Blobstore.implementation", "DbBlobStore")
config.CONFIG.Set("Database.implementation", "MysqlDB")
config.CONFIG.Set("Mysql.database", flags.FLAGS.config_mysql_database)
if flags.FLAGS.config_mysql_username is not None:
config.CONFIG.Set("Mysql.username", flags.FLAGS.config_mysql_username)
if flags.FLAGS.config_mysql_password is not None:
config.CONFIG.Set("Mysql.password", flags.FLAGS.config_mysql_password)
config.CONFIG.Set("AdminUI.port", admin_ui_port)
config.CONFIG.Set("AdminUI.headless", True)
config.CONFIG.Set("Frontend.bind_address", "127.0.0.1")
config.CONFIG.Set("Frontend.bind_port", frontend_port)
config.CONFIG.Set("Server.initialized", True)
config.CONFIG.Set("Client.poll_max", 1)
config.CONFIG.Set("Client.server_urls",
["http://localhost:%d/" % frontend_port])
if flags.FLAGS.config_logging_path is not None:
config.CONFIG.Set("Logging.path", flags.FLAGS.config_logging_path)
config_updater_keys_util.GenerateKeys(config.CONFIG)
config.CONFIG.Write()
config_lib.SetPlatformArchContext()
context = list(config.CONFIG.context)
context.append("Client Context")
deployer = build.ClientRepacker()
config_data = deployer.GetClientConfig(
context, validate=False, deploy_timestamp=False)
with io.open(flags.FLAGS.dest_client_config_path, "w") as fd:
fd.write(config_data)
def setUpClass(cls):
gpu_memory_fraction_opt = (
"--gpu_memory_fraction=%f" % cls.PER_PROC_GPU_MEMORY_FRACTION)
worker_port = portpicker.pick_unused_port()
cluster_spec = "worker|localhost:%d" % worker_port
tf_logging.info("cluster_spec: %s", cluster_spec)
server_bin = test.test_src_dir_path("python/debug/grpc_tensorflow_server")
cls.server_target = "grpc://localhost:%d" % worker_port
cls.server_procs = {}
cls.server_procs["worker"] = subprocess.Popen(
[
server_bin,
"--cluster_spec=%s" % cluster_spec,
"--job_name=worker",
"--task_id=0",
gpu_memory_fraction_opt,
],
stdout=sys.stdout,
stderr=sys.stderr)
# Start debug server in-process, on separate thread.
(cls.debug_server_port, cls.debug_server_url, _, cls.debug_server_thread,
cls.debug_server
) = grpc_debug_test_server.start_server_on_separate_thread(
dump_to_filesystem=False)
tf_logging.info("debug server url: %s", cls.debug_server_url)
cls.session_config = config_pb2.ConfigProto(
gpu_options=config_pb2.GPUOptions(
per_process_gpu_memory_fraction=cls.PER_PROC_GPU_MEMORY_FRACTION))
def __init__(self, run_config, full_screen=False, **kwargs):
self._proc = None
self._sock = None
self._controller = None
self._tmp_dir = tempfile.mkdtemp(prefix="sc-", dir=run_config.tmp_dir)
self._port = portpicker.pick_unused_port()
self._check_exists(run_config.exec_path)
args = [
run_config.exec_path,
"-listen", "127.0.0.1",
"-port", str(self._port),
"-dataDir", os.path.join(run_config.data_dir, ""),
"-tempDir", os.path.join(self._tmp_dir, ""),
"-displayMode", "1" if full_screen else "0",
]
try:
self._proc = self._launch(run_config, args, **kwargs)
self._sock = self._connect(self._port)
client = protocol.StarcraftProtocol(self._sock)
self._controller = remote_controller.RemoteController(client)
with sw("startup"):
self._controller.ping()
except:
self.close()
raise
def run_benchmark_distributed():
ops = create_graph("/job:worker/task:0", "/job:worker/task:1")
queues = [create_done_queue(0), create_done_queue(1)]
# launch distributed service
port0, port1 = [portpicker.pick_unused_port() for _ in range(2)]
flags = " ".join(sys.argv) # pass parent flags to children
def run_worker(w):
my_env = os.environ.copy()
if not FLAGS.verbose:
my_env["CUDA_VISIBLE_DEVICES"] = ""
my_env["TF_CPP_MIN_LOG_LEVEL"] = "2"
if FLAGS.profile:
my_env["LD_PRELOAD"]="/usr/lib/libtcmalloc_and_profiler.so.4"
my_env["CPUPROFILE"]="/tmp/profile.out.%s"%(w)
cmd = "python %s --task=%d --port0=%s --port1=%s"%(flags, w, port0, port1)
subprocess.Popen(cmd, shell=True, stderr=subprocess.STDOUT,
env=my_env)
run_worker(0)
run_worker(1)
sess = tf.Session("grpc://%s:%s"%(host, port0), config=session_config())
rate = run_benchmark(sess, *ops)
# bring down workers
if FLAGS.verbose:
print("Killing workers.")
sess.run(queues[1].enqueue(1))
sess.run(queues[0].enqueue(1)) # bring down master last
return rate
def _pick_unused_port():
"""For some reason portpicker returns the same port sometimes."""
while True:
p = portpicker.pick_unused_port()
if p not in _PORTS:
break
_PORTS.add(p)
return p
def start_server_on_separate_thread(dump_to_filesystem=True,
server_start_delay_sec=0.0,
poll_server=False,
blocking=True,
toggle_watch_on_core_metadata=None):
"""Create a test gRPC debug server and run on a separate thread.
Args:
dump_to_filesystem: (bool) whether the debug server will dump debug data
to the filesystem.
server_start_delay_sec: (float) amount of time (in sec) to delay the server
start up for.
poll_server: (bool) whether the server will be polled till success on
startup.
blocking: (bool) whether the server should be started in a blocking mode.
toggle_watch_on_core_metadata: A list of
(node_name, output_slot, debug_op) tuples to toggle the
watchpoint status during the on_core_metadata calls (optional).
Returns:
server_port: (int) Port on which the server runs.
debug_server_url: (str) grpc:// URL to the server.
server_dump_dir: (str) The debug server's dump directory.
server_thread: The server Thread object.
server: The `EventListenerTestServicer` object.
Raises:
ValueError: If polling the server process for ready state is not successful
within maximum polling count.
"""
server_port = portpicker.pick_unused_port()
debug_server_url = "grpc://localhost:%d" % server_port
server_dump_dir = tempfile.mkdtemp() if dump_to_filesystem else None
server = EventListenerTestServicer(
server_port=server_port,
dump_dir=server_dump_dir,
toggle_watch_on_core_metadata=toggle_watch_on_core_metadata)
def delay_then_run_server():
time.sleep(server_start_delay_sec)
server.run_server(blocking=blocking)
server_thread = threading.Thread(target=delay_then_run_server)
server_thread.start()
if poll_server:
if not _poll_server_till_success(
50,
0.2,
debug_server_url,
server_dump_dir,
server,
gpu_memory_fraction=0.1):
raise ValueError(
"Failed to start test gRPC debug server at port %d" % server_port)
server.clear_data()
return server_port, debug_server_url, server_dump_dir, server_thread, server
def __init__(self,
emulator_cmd=None,
deadline=10,
start_options=(),
silent=False):
"""Constructs a DatastoreEmulator.
Clients should use DatastoreEmulatorFactory to construct DatastoreEmulator
instances.
Args:
emulator_cmd: A string representing the path to an executable script that
invokes emulator binary.
deadline: A integer representing number of seconds to wait for the
datastore to start.
start_options: A list of additional command-line options to pass to the
emulator 'start' command.
silent: A bool indicates if emulator runs in silent mode.
Raises:
IOError: if the emulator failed to start within the deadline
"""
self._emulator_cmd = emulator_cmd
self._http = httplib2.Http()
self.__running = False
self._silent = silent
self._redirected_output = open(os.devnull, 'wb') if self._silent else None
# Start the emulator and wait for it to start responding to requests.
cmd = [self._emulator_cmd, 'start'] + _DEFAULT_EMULATOR_OPTIONS
if start_options:
cmd.extend(start_options)
port = ParsePortFromOption(start_options or [])
if not port:
port = portpicker.pick_unused_port()
cmd.append('--port=%d' % port)
self._host = 'http://localhost:%d' % port
cmd.append(tempfile.mkdtemp())
# On windows, cloud_datastore_emulator.bat always prompts up
# 'Terminate batch job (Y/N)'. Passing nul to this .bat avoids self.Stop()
# hang at this prompt up.
if sys.platform.startswith('win'):
cmd.append('<nul')
popen_kwargs = {}
if self._silent:
popen_kwargs.update(
stdout=self._redirected_output, stderr=self._redirected_output)
self.emulator_proc = subprocess.Popen(cmd, **popen_kwargs)
if not self._WaitForStartup(deadline):
raise IOError('emulator did not respond within %ds' % deadline)
self.__datastore = None
self.__running = True
def test_basic(self):
"""Basic functionality test of START_PROCESS_REVERSE flavor."""
portpicker.pick_unused_port().AndReturn(2345)
# As the lock is mocked out, this provides a mox expectation.
with self.proxy._process_lock:
safe_subprocess.start_process_file(
args=['/runtime'],
input_string=self.runtime_config.SerializeToString(),
env={'foo': 'bar',
'PORT': '2345'},
cwd=self.tmpdir,
stderr=subprocess.PIPE).AndReturn(self.process)
self.proxy._stderr_tee = FakeTee('')
self.mox.ReplayAll()
self.proxy.start()
self.assertEquals(2345, self.proxy._proxy._port)
self.mox.VerifyAll()
def create_local_cluster(num_workers, num_ps, protocol="grpc"):
"""Create local GRPC servers and return their servers."""
worker_ports = [portpicker.pick_unused_port() for _ in range(num_workers)]
ps_ports = [portpicker.pick_unused_port() for _ in range(num_ps)]
cluster_dict = {
"worker": ["localhost:%s" % port for port in worker_ports],
"ps": ["localhost:%s" % port for port in ps_ports]}
cs = tf.train.ClusterSpec(cluster_dict)
workers = [
tf.train.Server(
cs, job_name="worker", protocol=protocol, task_index=ix, start=True)
for ix in range(num_workers)]
ps_servers = [
tf.train.Server(
cs, job_name="ps", protocol=protocol, task_index=ix, start=True)
for ix in range(num_ps)]
return workers, ps_servers
def __init__(self, tmux_window, job, task_id):
self.tmux_window = tmux_window
self.job = job
self.ip = '127.0.0.1' # hostname/ip address
self.id = task_id
self.port = portpicker.pick_unused_port()
self.connect_instructions = 'tmux a -t '+self.tmux_window
self.last_stdout = '<unavailable>' # compatiblity with aws.py:Task
self.last_stderr = '<unavailable>'
def testPickUnusedCanSuccessfullyUsePortServer(self):
with mock.patch.object(portpicker, '_pick_unused_port_without_server'):
portpicker._pick_unused_port_without_server.side_effect = (
Exception('eek!')
)
# Since _PickUnusedPortWithoutServer() raises an exception, if we
# can successfully obtain a port, the portserver must be working.
port = portpicker.pick_unused_port()
self.assertTrue(self.IsUnusedTCPPort(port))
self.assertTrue(self.IsUnusedUDPPort(port))
def test_silent_request(self):
"""Call log_message once on the silent request handler for coverage's sake."""
handler = AdderHandler()
server = easyrpc.on(handler, adder_service, 'localhost', portpicker.pick_unused_port())
class _DumbHandler(server.httpd.RequestHandlerClass):
def __init__(self):
pass
http_request_handler = _DumbHandler()
http_request_handler.log_message('unused')
def setUp(self):
super(ApiSslServerTestBase, self).setUp()
key = rdf_crypto.RSAPrivateKey.GenerateKey()
key_path = os.path.join(self.temp_dir, "key.pem")
with open(key_path, "wb") as f:
f.write(key.AsPEM())
subject = issuer = x509.Name([
x509.NameAttribute(oid.NameOID.COMMON_NAME, u"localhost"),
])
cert = x509.CertificateBuilder().subject_name(subject).issuer_name(
issuer).public_key(
key.GetPublicKey().GetRawPublicKey()).serial_number(
x509.random_serial_number()).not_valid_before(
datetime.datetime.utcnow()).not_valid_after(
datetime.datetime.utcnow() +
datetime.timedelta(days=1)).add_extension(
x509.SubjectAlternativeName(
[x509.DNSName(u"localhost")]),
critical=False,
).sign(key.GetRawPrivateKey(), hashes.SHA256(),
backends.default_backend())
self.cert_path = os.path.join(self.temp_dir, "certificate.pem")
with open(self.cert_path, "wb") as f:
f.write(cert.public_bytes(serialization.Encoding.PEM))
config_overrider = test_lib.ConfigOverrider({
"AdminUI.enable_ssl":
True,
"AdminUI.ssl_key_file":
key_path,
"AdminUI.ssl_cert_file":
self.cert_path,
})
config_overrider.Start()
self.addCleanup(config_overrider.Stop)
self.port = portpicker.pick_unused_port()
thread = wsgiapp_testlib.ServerThread(self.port,
name="ApiSslServerTest")
thread.StartAndWaitUntilServing()
self.addCleanup(thread.Stop)
api_auth_manager.InitializeApiAuthManager(
api_call_router_without_checks.ApiCallRouterWithoutChecks)
self.token.username = "******"
webauth.WEBAUTH_MANAGER.SetUserName(self.token.username)
self.endpoint = "https://localhost:%s" % self.port
def checkIsPortFree(self):
"""This might be flaky unless this test is run with a portserver."""
# The port should be free initially.
port = portpicker.pick_unused_port()
self.assertTrue(portpicker.is_port_free(port))
cases = [
(socket.AF_INET, socket.SOCK_STREAM, None),
(socket.AF_INET6, socket.SOCK_STREAM, 0),
(socket.AF_INET6, socket.SOCK_STREAM, 1),
(socket.AF_INET, socket.SOCK_DGRAM, None),
(socket.AF_INET6, socket.SOCK_DGRAM, 0),
(socket.AF_INET6, socket.SOCK_DGRAM, 1),
]
for (sock_family, sock_type, v6only) in cases:
# Occupy the port on a subset of possible protocols.
try:
sock = socket.socket(sock_family, sock_type, 0)
except socket.error:
print('Kernel does not support sock_family=%d' % sock_family,
file=sys.stderr)
# Skip this case, since we cannot occupy a port.
continue
if not hasattr(socket, 'IPPROTO_IPV6'):
v6only = None
if v6only is not None:
try:
sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY,
v6only)
except socket.error:
print('Kernel does not support IPV6_V6ONLY=%d' % v6only,
file=sys.stderr)
# Don't care; just proceed with the default.
# Socket may have been taken in the mean time, so catch the
# socket.error with errno set to EADDRINUSE and skip this
# attempt.
try:
sock.bind(('', port))
except socket.error as e:
if e.errno == errno.EADDRINUSE:
raise portpicker.NoFreePortFoundError
raise
# The port should be busy.
self.assertFalse(portpicker.is_port_free(port))
sock.close()
# Now it's free again.
self.assertTrue(portpicker.is_port_free(port))
def start_server_on_separate_thread(dump_to_filesystem=True,
server_start_delay_sec=0.0,
poll_server=False,
blocking=True):
"""Create a test gRPC debug server and run on a separate thread.
Args:
dump_to_filesystem: (bool) whether the debug server will dump debug data
to the filesystem.
server_start_delay_sec: (float) amount of time (in sec) to delay the server
start up for.
poll_server: (bool) whether the server will be polled till success on
startup.
blocking: (bool) whether the server should be started in a blocking mode.
Returns:
server_port: (int) Port on which the server runs.
debug_server_url: (str) grpc:// URL to the server.
server_dump_dir: (str) The debug server's dump directory.
server_thread: The server Thread object.
server: The `EventListenerTestServicer` object.
Raises:
ValueError: If polling the server process for ready state is not successful
within maximum polling count.
"""
server_port = portpicker.pick_unused_port()
debug_server_url = "grpc://localhost:%d" % server_port
server_dump_dir = tempfile.mkdtemp() if dump_to_filesystem else None
server = EventListenerTestServicer(server_port=server_port,
dump_dir=server_dump_dir)
def delay_then_run_server():
time.sleep(server_start_delay_sec)
server.run_server(blocking=blocking)
server_thread = threading.Thread(target=delay_then_run_server)
server_thread.start()
if poll_server:
if not _poll_server_till_success(50,
0.2,
debug_server_url,
server_dump_dir,
server,
gpu_memory_fraction=0.1):
raise ValueError(
"Failed to start test gRPC debug server at port %d" %
server_port)
server.clear_data()
return server_port, debug_server_url, server_dump_dir, server_thread, server
def setUp(self):
super(ExecutorServiceTest, self).setUp()
port = portpicker.pick_unused_port()
server_pool = logging_pool.pool(max_workers=1)
self._server = grpc.server(server_pool)
self._server.add_insecure_port('[::]:{}'.format(port))
self._service = executor_service.ExecutorService(
eager_executor.EagerExecutor())
executor_pb2_grpc.add_ExecutorServicer_to_server(
self._service, self._server)
self._server.start()
self._channel = grpc.insecure_channel('localhost:{}'.format(port))
self._stub = executor_pb2_grpc.ExecutorStub(self._channel)
def main(argv):
logging.basicConfig(
level=logging.INFO,
format='%(levelname)s: %(threadName)s %(message)s')
directory = os.getcwd()
if argv[1:] == ['server']:
port = gflags.FLAGS.port or portpicker.pick_unused_port()
compilation_server_lib.RunServerAtPort(port)
elif argv[1:] == ['local']:
port = gflags.FLAGS.port or portpicker.pick_unused_port()
server_proc = subprocess.Popen(
["python", argv[0], "--port", str(port), "server"])
server_address = '127.0.0.1:{}'.format(port)
compilation_client_lib.WaitTillHealthy(server_address)
try:
RunEditingEnvironment(
directory,
server_address=server_address)
finally:
try:
logging.info("Terminating latex compilation server.")
server_proc.kill()
except OSError:
pass
elif argv[1:] == []:
if not gflags.FLAGS.using_server:
logging.fatal(
"Please specify the server address when running in the client mode "
"via --using_server")
RunEditingEnvironment(directory, server_address=gflags.FLAGS.using_server)
else:
print "Unable to recognize command %r" % argv
print __doc__
sys.exit(1)
def local_port(self):
"""Selects a local port.
Returns:
int, local port ready for use.
"""
if not self._local_port:
forwarded_port = self.get_forwarded_port()
if forwarded_port:
self._local_port = forwarded_port
else:
self._local_port = portpicker.pick_unused_port()
return self._local_port
def _create_cluster_spec(has_chief=False,
num_workers=1,
num_ps=0,
has_eval=False):
if _portpicker_import_error:
raise _portpicker_import_error # pylint: disable=raising-bad-type
cluster_spec = {}
if has_chief:
cluster_spec[CHIEF] = ["localhost:%s" % portpicker.pick_unused_port()]
if num_workers:
cluster_spec[WORKER] = [
"localhost:%s" % portpicker.pick_unused_port()
for _ in range(num_workers)
]
if num_ps:
cluster_spec[PS] = [
"localhost:%s" % portpicker.pick_unused_port() for _ in range(num_ps)
]
if has_eval:
cluster_spec[EVALUATOR] = ["localhost:%s" % portpicker.pick_unused_port()]
return cluster_spec
def setUp(self):
super(RpcOpTest, self).setUp()
service_port = portpicker.pick_unused_port()
server = grpc.server(logging_pool.pool(max_workers=25))
servicer = rpc_op_test_servicer.RpcOpTestServicer()
test_example_pb2_grpc.add_TestCaseServiceServicer_to_server(
servicer, server)
self._address = 'localhost:%d' % service_port
server.add_insecure_port(self._address)
server.start()
self._server = server
def test_server_context_shuts_down_uncaught_exception(
self, mock_logging_info):
ex = eager_tf_executor.EagerTFExecutor()
ex_factory = executor_factory.ExecutorFactoryImpl(lambda _: ex)
with self.assertRaises(TypeError):
with server_utils.server_context(
ex_factory, 1, portpicker.pick_unused_port()) as server:
time.sleep(1)
raise TypeError
mock_logging_info.assert_called_once_with('Shutting down server.')
def testExecutionWatcher_LocalWithEmptyRequest(self):
port = portpicker.pick_unused_port()
sidecar = execution_watcher.ExecutionWatcher(
port, creds=grpc.local_server_credentials())
sidecar.start()
creds = grpc.local_channel_credentials()
channel = grpc.secure_channel(sidecar.local_address, creds)
stub = execution_watcher_pb2_grpc.ExecutionWatcherServiceStub(channel)
req = execution_watcher_pb2.UpdateExecutionInfoRequest()
res = stub.UpdateExecutionInfo(req)
sidecar.stop()
self.assertEqual(execution_watcher_pb2.UpdateExecutionInfoResponse(),
res)
def __init__(self, host="127.0.0.1", port=None, fullscreen=False):
assert isinstance(host, str)
assert isinstance(port, int) or port is None
self._fullscreen = fullscreen
self._host = host
if port is None:
self._port = portpicker.pick_unused_port()
else:
self._port = port
self._tmp_dir = tempfile.mkdtemp(prefix="SC2_")
self._process = None
self._ws = None
def _SetupServer(cls):
for _ in range(5):
try:
port = portpicker.pick_unused_port()
cls.server = shared_fake_data_store.SharedFakeDataStoreServer(
port)
config.CONFIG.Set("SharedFakeDataStore.port", port)
return
except socket.error as e:
if e.errno != errno.EADDRINUSE:
raise
raise RuntimeError("Unable to find an unused port after 5 tries.")
def _create_in_process_tf_ps_cluster(num_workers, num_ps):
"""Create a cluster of TF workers and returns their addresses.
Such cluster simulate the behavior of multiple TF parameter servers.
Args:
num_workers: Number of "worker" workers.
num_ps: Number of "parameter server" workers.
Returns:
The ClusterResolver i.e. the ip addresses of the workers.
"""
worker_ports = [portpicker.pick_unused_port() for _ in range(num_workers)]
ps_ports = [portpicker.pick_unused_port() for _ in range(num_ps)]
cluster_dict = {"worker": ["localhost:%s" % port for port in worker_ports]}
if num_ps > 0:
cluster_dict["ps"] = ["localhost:%s" % port for port in ps_ports]
cluster_spec = tf.train.ClusterSpec(cluster_dict)
worker_config = tf.compat.v1.ConfigProto()
for i in range(num_workers):
tf.distribute.Server(
cluster_spec,
job_name="worker",
task_index=i,
config=worker_config,
protocol="grpc")
for i in range(num_ps):
tf.distribute.Server(
cluster_spec, job_name="ps", task_index=i, protocol="grpc")
os.environ["GRPC_FAIL_FAST"] = "use_caller"
return tf.distribute.cluster_resolver.SimpleClusterResolver(
cluster_spec, rpc_layer="grpc")
def setUp(self):
self._debugger_data_server_grpc_port = portpicker.pick_unused_port()
self._debug_url = (
"grpc://localhost:%d" % self._debugger_data_server_grpc_port)
self._logdir = tempfile.mkdtemp(prefix="tensorboard_dds_")
self._debug_data_server = debugger_server_lib.DebuggerDataServer(
self._debugger_data_server_grpc_port, self._logdir, always_flush=True)
self._server_thread = threading.Thread(
target=self._debug_data_server.start_the_debugger_data_receiving_server)
self._server_thread.start()
self.assertTrue(self._poll_server_till_success(50, 0.2))
def setUpClass(cls):
def start_server(port):
signal.signal(signal.SIGTERM, lambda signum, frame: os._exit(0))
logging.disable(logging.CRITICAL)
handler = AdderHandler()
server = easyrpc.on(handler, adder_service, 'localhost', port)
server.serve()
cls._server_port = portpicker.pick_unused_port()
cls._server_p = multiprocessing.Process(target=start_server, args=(cls._server_port,))
cls._server_p.start()
def pick_unused_port():
"""Returns an unused and unassigned local port."""
if _portpicker_import_error:
raise _portpicker_import_error # pylint: disable=raising-bad-type
global ASSIGNED_PORTS
with lock:
while True:
port = portpicker.pick_unused_port()
if port > 10000 and port not in ASSIGNED_PORTS:
ASSIGNED_PORTS.add(port)
logging.info('Using local port %r', port)
return port
def setUp(self):
super(RpcOpTest, self).setUp()
service_port = portpicker.pick_unused_port()
server = grpc.server(logging_pool.pool(max_workers=25))
servicer = rpc_op_test_servicer.RpcOpTestServicer()
test_example_pb2_grpc.add_TestCaseServiceServicer_to_server(
servicer, server)
self._address = 'localhost:%d' % service_port
server.add_insecure_port(self._address)
server.start()
self._server = server
def create_local_cluster(num_workers, num_ps, protocol="grpc"):
"""Create local GRPC servers and return them."""
worker_ports = [portpicker.pick_unused_port() for _ in range(num_workers)]
ps_ports = [portpicker.pick_unused_port() for _ in range(num_ps)]
cluster_dict = {
"worker": ["localhost:%s" % port for port in worker_ports],
"ps": ["localhost:%s" % port for port in ps_ports]
}
cs = server_lib.ClusterSpec(cluster_dict)
workers = [
server_lib.Server(
cs, job_name="worker", protocol=protocol, task_index=ix, start=True)
for ix in range(num_workers)
]
ps_servers = [
server_lib.Server(
cs, job_name="ps", protocol=protocol, task_index=ix, start=True)
for ix in range(num_ps)
]
return cluster_dict, workers, ps_servers
def pick_unused_port():
"""Returns an unused and unassigned local port."""
if _portpicker_import_error:
raise _portpicker_import_error # pylint: disable=raising-bad-type
global ASSIGNED_PORTS
with lock:
while True:
port = portpicker.pick_unused_port()
if port > 10000 and port not in ASSIGNED_PORTS:
ASSIGNED_PORTS.add(port)
logging.info('Using local port %r', port)
return port
def load_from_disk(path, settings):
"""Load Hard Eight Tasks from disk.
Args:
path: Directory containing dm_hard_eight environment.
settings: EnvironmentSettings required to start the environment.
Returns:
An implementation of dm_env.Environment.
Raises:
RuntimeError: If unable to start environment process.
"""
_validate_environment_settings(settings)
executable_path = os.path.join(path, 'Linux64Player')
libosmesa_path = os.path.join(path, 'external_libosmesa_llvmpipe.so')
if not os.path.exists(executable_path) or not os.path.exists(
libosmesa_path):
raise RuntimeError(
'Cannot find dm_hard_eight executable or dependent files at path: {}'
.format(path))
port = portpicker.pick_unused_port()
process_flags = [
executable_path,
# Unity command-line flags.
'-logfile',
'-batchmode',
'-noaudio',
# Other command-line flags.
'--logtostderr',
'--server_type=DM_ENV_RPC',
'--uri_address=[::]:{}'.format(port),
]
os.environ.update({
'UNITY_RENDERER': 'software',
'UNITY_OSMESA_PATH': libosmesa_path,
})
process = subprocess.Popen(process_flags,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
if process.poll() is not None:
raise RuntimeError('Failed to start dm_hard_eight process correctly.')
return _HardEightTasksProcessEnv(_connect_to_environment(port, settings),
_HARD_EIGHT_TASK_OBSERVATIONS,
settings.num_action_repeats, process)
async def run_match(controllers: List[Controller],
match: GameMatch,
close_ws=True):
await _setup_host_game(controllers[0], **match.host_game_kwargs)
# Setup portconfig beforehand, so all players use the same ports
startport = None
portconfig = None
if match.needed_sc2_count > 1:
if any(isinstance(player, BotProcess) for player in match.players):
portconfig = Portconfig.contiguous_ports()
# Most ladder bots generate their server and client ports as [s+2, s+3], [s+4, s+5]
startport = portconfig.server[0] - 2
else:
portconfig = Portconfig()
proxies = []
coros = []
i = 0
for player in match.players:
if player.needs_sc2:
if isinstance(player, BotProcess):
pport = portpicker.pick_unused_port()
p = Proxy(controllers[i], player, pport, match.game_time_limit,
match.realtime)
proxies.append(p)
coros.append(p.play_with_proxy(startport))
else:
coros.append(
play_from_websocket(
controllers[i]._ws,
player,
match.realtime,
portconfig,
should_close=close_ws,
game_time_limit=match.game_time_limit,
))
i += 1
# async_results = await asyncio.wait_for(asyncio.gather(*coros, return_exceptions=True), timeout=None)
async_results = await asyncio.gather(*coros, return_exceptions=True)
if not isinstance(async_results, list):
async_results = [async_results]
for i, a in enumerate(async_results):
if isinstance(a, Exception):
logger.error(
f"Exception[{a}] thrown by {[p for p in match.players if p.needs_sc2][i]}"
)
return process_results(match.players, async_results)
def download(filename):
"""Downloads the file to the user's local disk via a browser download action.
Args:
filename: Name of the file on disk to be downloaded.
Raises:
OSError: if the file cannot be found.
"""
if not os.path.exists(filename):
msg = 'Cannot find file: {}'.format(filename)
if six.PY2:
raise OSError(msg)
else:
raise FileNotFoundError(msg) # pylint: disable=undefined-variable
started = threading.Event()
port = portpicker.pick_unused_port()
def server_entry():
httpd = _V6Server(('::', port), _FileHandler)
started.set()
# Handle a single request then exit the thread.
httpd.handle_request()
thread = threading.Thread(target=server_entry)
thread.start()
started.wait()
output.eval_js(
"""
(async function() {
const response = await fetch('https://localhost:%(port)d%(path)s');
if (!response.ok) {
throw new Error('Failed to download: ' + response.statusText);
}
const blob = await response.blob();
const a = document.createElement('a');
a.href = window.URL.createObjectURL(blob);
a.download = '%(name)s';
document.body.appendChild(a);
a.click();
a.remove();
})();
""" % {
'port': port,
'path': os.path.abspath(filename),
'name': os.path.basename(filename),
})
def getUnusedPort():
"""Returns an unused TCP port on the local host inside the defined port range.
To be used when starting peer SAS webserver or other database webserver.
"""
config = _GetSharedTestConfig()
if config.min_port < 0:
return portpicker.pick_unused_port()
global _ports
# Find the first available port in the defined range.
for p in xrange(config.min_port, config.max_port):
if p not in _ports and portpicker.is_port_free(p):
_ports.add(p)
return p
raise AssertionError('No available new ports')
def setUp(self):
super(ApiSslProxyTest, self).setUp()
attempts_count = 0
self.proxy_server = None
while self.proxy_server is None:
try:
self.proxy_port = portpicker.pick_unused_port()
self.proxy_server = TCPServerV6(("::", self.proxy_port), Proxy)
except socket.error:
attempts_count += 1
if attempts_count == 10:
self.fail("Can't initialize proxy server.")
threading.Thread(target=self.proxy_server.serve_forever).start()
def test_server_context_shuts_down_under_keyboard_interrupt(
self, mock_logging_info):
ex = eager_tf_executor.EagerTFExecutor()
with server_utils.server_context(
ex, 1, portpicker.pick_unused_port()) as server:
time.sleep(1)
raise KeyboardInterrupt
mock_logging_info.assert_has_calls([
mock.call('Server stopped by KeyboardInterrupt.'),
mock.call('Shutting down server.')
])
def __init__(self,
config_path,
num_players=2,
num_bots=0,
mode='train',
train_mode='frag',
max_steps=2100,
episode_timeout=2100,
is_window_visible=False,
is_window_cv_visible=False):
self.port = pick_unused_port()
self.mode = mode
self.train_mode = train_mode
# host cfg
self.host_cfg = PlayerHostConfig(self.port)
self.host_cfg.num_players = num_players
# join cfg
self.join_cfg = PlayerJoinConfig(self.port)
# player cfg
self.players_cfg = []
for i in range(self.host_cfg.num_players):
cfg = PlayerConfig()
cfg.config_path = config_path
cfg.player_mode = vd.Mode.PLAYER
cfg.screen_resolution = vd.ScreenResolution.RES_800X450
cfg.screen_format = vd.ScreenFormat.CBCGCR
cfg.is_window_visible = is_window_visible
# cfg.ticrate = vd.DEFAULT_TICRATE * 2
cfg.episode_timeout = episode_timeout
if i == 0: # host
cfg.host_cfg = self.host_cfg
cfg.name = 'WhoAmI'
cfg.num_bots = num_bots
else:
cfg.join_cfg = self.join_cfg
cfg.name = 'P{}'.format(i)
self.players_cfg.append(cfg)
# the player-wise env and the vec-env
# TODO(jackzbzheng): add different observation wrappers
self.envs = []
for cfg in self.players_cfg:
e = PlayerEnv(cfg, train_mode=self.train_mode)
#e = RwdShapeWu2(e, dist_penalty_thres=3)
self.envs.append(e)
self.env = VecEnv(self.envs)
self.observation_space = self.env.observation_space
self.action_space = self.env.action_space
async def test_endpoint_task():
port = portpicker.pick_unused_port()
cfg = EndpointConfig(
interval=0.000001,
timeout=1.0,
description="test",
url=f"http://*****:*****@routes.get("/")
async def all(request: web.Request) -> web.Response:
nonlocal request_count
request_count += 1
return web.Response(text="OK")
app = web.Application()
app.add_routes(routes)
runner = web.AppRunner(app)
await runner.setup()
tcp_site = web.TCPSite(runner, "localhost", port)
call_count = 0
async def callback(msg):
nonlocal call_count
if call_count == 0:
assert msg.reached == True
call_count = False
await runner.cleanup() # Stop the web server
elif call_count == 1:
assert msg.reached == False
else:
raise RuntimeError("Stop")
call_count += 1
await tcp_site.start()
try:
await endpoint_task(cfg, callback)
except RuntimeError as e:
assert str(e) == "Stop"
assert request_count == 1
def contiguous_ports(cls, guests=1, attempts=40):
"""Returns a Portconfig with adjacent ports"""
for _ in range(attempts):
start = portpicker.pick_unused_port()
others = [start + j for j in range(1, 2 + guests * 2)]
if all(portpicker.is_port_free(p) for p in others):
server_ports = [start, others.pop(0)]
player_ports = []
while others:
player_ports.append([others.pop(0), others.pop(0)])
pc = cls(server_ports=server_ports, player_ports=player_ports)
pc._picked_ports.append(start)
return pc
raise portpicker.NoFreePortFoundError()
def start_train_process(self, conf_path, run_id, options=None):
unused_port = portpicker.pick_unused_port()
command = [
'python', 'learn.py', self.env_name, '--train',
'--worker-id=' + str(unused_port),
'--trainer-config-path=' + str(conf_path), '--run-id=' + run_id
]
if options:
command.append(options)
proc = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
return proc
def _create_cluster_spec(self,
has_chief=False,
num_workers=1,
num_ps=0,
has_eval=False):
if _portpicker_import_error:
raise _portpicker_import_error # pylint: disable=raising-bad-type
cluster_spec = {}
if has_chief:
cluster_spec[CHIEF] = ["localhost:%s" % portpicker.pick_unused_port()]
if num_workers:
cluster_spec[WORKER] = [
"localhost:%s" % portpicker.pick_unused_port()
for _ in range(num_workers)
]
if num_ps:
cluster_spec[PS] = [
"localhost:%s" % portpicker.pick_unused_port() for _ in range(num_ps)
]
if has_eval:
cluster_spec[EVALUATOR] = ["localhost:%s" % portpicker.pick_unused_port()]
return cluster_spec
def server_entry():
if ('HTTPD' in cfg) and (cfg['HTTPD']):
stopLocalServer()
cfg['HTTPD'] = None
handler = SimpleHTTPServer.SimpleHTTPRequestHandler
port = portpicker.pick_unused_port()
with JBcd(cfg['REVEAL_DIR']):
os.chdir(cfg['REVEAL_DIR'])
httpd = V6Server(("::", port), handler)
print("serving at port", port, 'cwd', os.getcwd(), 'reveal',
cfg['REVEAL_DIR'])
cfg['HTTPD'] = httpd
cfg['HTTP_PORT'] = port
httpd.serve_forever()
def setUpClass(cls):
super(ApiSslServerTestBase, cls).setUpClass()
if ApiSslServerTestBase._api_set_up_done:
return
key = rdf_crypto.RSAPrivateKey.GenerateKey()
key_path = temp.TempFilePath("key.pem")
with open(key_path, "wb") as f:
f.write(key.AsPEM())
subject = issuer = x509.Name([
x509.NameAttribute(oid.NameOID.COMMON_NAME, u"localhost"),
])
cert = x509.CertificateBuilder().subject_name(subject).issuer_name(
issuer).public_key(
key.GetPublicKey().GetRawPublicKey()).serial_number(
x509.random_serial_number()).not_valid_before(
datetime.datetime.utcnow()).not_valid_after(
datetime.datetime.utcnow() +
datetime.timedelta(days=1)).add_extension(
x509.SubjectAlternativeName(
[x509.DNSName(u"localhost")]),
critical=False,
).sign(key.GetRawPrivateKey(), hashes.SHA256(),
backends.default_backend())
ApiSslServerTestBase.ssl_cert_path = temp.TempFilePath(
"certificate.pem")
with open(ApiSslServerTestBase.ssl_cert_path, "wb") as f:
f.write(cert.public_bytes(serialization.Encoding.PEM))
ApiSslServerTestBase.ssl_port = portpicker.pick_unused_port()
with test_lib.ConfigOverrider({
"AdminUI.enable_ssl":
True,
"AdminUI.ssl_key_file":
key_path,
"AdminUI.ssl_cert_file":
ApiSslServerTestBase.ssl_cert_path,
}):
ApiSslServerTestBase._ssl_trd = wsgiapp_testlib.ServerThread(
ApiSslServerTestBase.ssl_port, name="ApiSslServerTest")
ApiSslServerTestBase._ssl_trd.StartAndWaitUntilServing()
ApiSslServerTestBase.ssl_endpoint = ("https://localhost:%s" %
ApiSslServerTestBase.ssl_port)
ApiSslServerTestBase._api_set_up_done = True
def testTrace_ProfileIdleServerWithOptions(self):
test_port = portpicker.pick_unused_port()
profiler.start_server(test_port)
# Test the profilers are successfully started and connected to profiler
# service on the worker. Since there is no op running, it is expected to
# return UnavailableError with no trace events collected string.
with self.assertRaises(errors.UnavailableError) as error:
options = profiler.ProfilerOptions(host_tracer_level=3,
device_tracer_level=0)
profiler_client.trace('localhost:' + str(test_port),
self.get_temp_dir(),
duration_ms=10,
options=options)
self.assertEqual('No trace event is collected', str(error.exception))
def get_available_host_port():
"""Gets a host port number available for adb forward.
Returns:
An integer representing a port number on the host available for adb
forward.
Raises:
Error: when no port is found after MAX_PORT_ALLOCATION_RETRY times.
"""
# Only import adb module if needed.
from mobly.controllers.android_device_lib import adb
port = portpicker.pick_unused_port()
if not adb.is_adb_available():
return port
for _ in range(MAX_PORT_ALLOCATION_RETRY):
# Make sure adb is not using this port so we don't accidentally
# interrupt ongoing runs by trying to bind to the port.
if port not in adb.list_occupied_adb_ports():
return port
port = portpicker.pick_unused_port()
raise Error('Failed to find available port after {} retries'.format(
MAX_PORT_ALLOCATION_RETRY))
def testSendFileTransmissionFailure(self):
unused_port = portpicker.pick_unused_port()
session = gcs.UploadSession(f"https://localhost:{unused_port}")
opts = gcs.UploadSession.Opts()
opts.retry_chunk_attempts = 1
opts.retry_chunk_init_delay = 0.0
with self.assertRaises(gcs.RequestError) as context:
session.SendFile(io.BytesIO(b"foobar"), opts=opts)
cause = context.exception.__cause__
self.assertIsInstance(cause, exceptions.ConnectionError)
def setUpClass(cls):
super(GRRSeleniumTest, cls).setUpClass()
with GRRSeleniumTest._selenium_set_up_lock:
if not GRRSeleniumTest._selenium_set_up_done:
port = portpicker.pick_unused_port()
logging.info("Picked free AdminUI port %d.", port)
# Start up a server in another thread
GRRSeleniumTest._server_trd = wsgiapp_testlib.ServerThread(
port, name="SeleniumServerThread")
GRRSeleniumTest._server_trd.StartAndWaitUntilServing()
GRRSeleniumTest._SetUpSelenium(port)
GRRSeleniumTest._selenium_set_up_done = True
def setUpClass(cls):
super(ApiE2ETest, cls).setUpClass()
with ApiE2ETest._api_set_up_lock:
if not ApiE2ETest._api_set_up_done:
# Set up HTTP server
port = portpicker.pick_unused_port()
ApiE2ETest.server_port = port
logging.info("Picked free AdminUI port for HTTP %d.", port)
ApiE2ETest.trd = wsgiapp_testlib.ServerThread(
port, name="api_e2e_server")
ApiE2ETest.trd.StartAndWaitUntilServing()
ApiE2ETest._api_set_up_done = True
def _create_local_cluster(self, **kargs):
"""Creates a local cluster."""
cluster_dict = {}
for (k, v) in kargs.items():
cluster_dict[k] = [
'localhost:%d' % portpicker.pick_unused_port() for _ in range(v)]
# Launch servers:
servers = {}
for (k, v) in kargs.items():
servers[k] = [tf.train.Server(cluster_dict,
job_name=k,
task_index=idx,
start=True) for idx in range(v)]
return servers
def setUp(self):
super(ApiSslProxyTest, self).setUp()
attempts_count = 0
self.proxy_server = None
while self.proxy_server is None:
try:
self.proxy_port = portpicker.pick_unused_port()
self.proxy_server = TCPServerV6(("::", self.proxy_port), Proxy)
except socket.error:
attempts_count += 1
if attempts_count == 10:
self.fail("Can't initialize proxy server.")
threading.Thread(target=self.proxy_server.serve_forever).start()
self.addCleanup(self.proxy_server.server_close)
self.addCleanup(self.proxy_server.shutdown)
def setUp(self):
super(ApiSslServerTestBase, self).setUp()
key = rdf_crypto.RSAPrivateKey.GenerateKey()
key_path = os.path.join(self.temp_dir, "key.pem")
with open(key_path, "wb") as f:
f.write(key.AsPEM())
subject = issuer = x509.Name([
x509.NameAttribute(oid.NameOID.COMMON_NAME, u"localhost"),
])
cert = x509.CertificateBuilder().subject_name(subject).issuer_name(
issuer).public_key(key.GetPublicKey().GetRawPublicKey()).serial_number(
x509.random_serial_number()).not_valid_before(
datetime.datetime.utcnow()).not_valid_after(
datetime.datetime.utcnow() +
datetime.timedelta(days=1)).add_extension(
x509.SubjectAlternativeName(
[x509.DNSName(u"localhost")]),
critical=False,
).sign(key.GetRawPrivateKey(), hashes.SHA256(),
backends.default_backend())
self.cert_path = os.path.join(self.temp_dir, "certificate.pem")
with open(self.cert_path, "wb") as f:
f.write(cert.public_bytes(serialization.Encoding.PEM))
config_overrider = test_lib.ConfigOverrider({
"AdminUI.enable_ssl": True,
"AdminUI.ssl_key_file": key_path,
"AdminUI.ssl_cert_file": self.cert_path,
})
config_overrider.Start()
self.addCleanup(config_overrider.Stop)
self.port = portpicker.pick_unused_port()
thread = wsgiapp_testlib.ServerThread(self.port, name="ApiSslServerTest")
thread.StartAndWaitUntilServing()
self.addCleanup(thread.Stop)
api_auth_manager.APIACLInit.InitApiAuthManager()
self.token.username = "******"
webauth.WEBAUTH_MANAGER.SetUserName(self.token.username)
self.endpoint = "https://localhost:%s" % self.port
def launch_distributed_service():
port = portpicker.pick_unused_port()
def launch_worker(worker_type):
my_env = os.environ.copy()
if not FLAGS.verbose:
my_env["CUDA_VISIBLE_DEVICES"] = ""
my_env["TF_CPP_MIN_LOG_LEVEL"] = "3"
if FLAGS.profile:
my_env["LD_PRELOAD"]="/usr/lib/libtcmalloc_and_profiler.so.4"
my_env["CPUPROFILE"]="/tmp/profile.out.%s"%(worker_type)
args = ["python"] + sys.argv + ["--port="+str(port),
"--worker_type="+worker_type]
proc = subprocess.Popen(args, stderr=subprocess.STDOUT, env=my_env)
log("worker %s pid %s"%(worker_type, proc.pid))
launch_worker("worker")
launch_worker("client")
def setUpClass(cls):
super(GRRHTTPServerTest, cls).setUpClass()
# Bring up a local server for testing.
port = portpicker.pick_unused_port()
ip = utils.ResolveHostnameToIP("localhost", port)
cls.httpd = frontend.GRRHTTPServer((ip, port),
frontend.GRRHTTPServerHandler)
if ipaddress.ip_address(ip).version == 6:
cls.address_family = socket.AF_INET6
cls.base_url = "http://[%s]:%d/" % (ip, port)
else:
cls.address_family = socket.AF_INET
cls.base_url = "http://%s:%d/" % (ip, port)
cls.httpd_thread = threading.Thread(
name="GRRHTTPServerTestThread", target=cls.httpd.serve_forever)
cls.httpd_thread.daemon = True
cls.httpd_thread.start()
def setUp(self):
super(InteractiveDebuggerPluginTest, self).setUp()
self._dummy_logdir = tempfile.mkdtemp()
self._dummy_multiplexer = event_multiplexer.EventMultiplexer({})
self._debugger_port = portpicker.pick_unused_port()
self._debugger_url = 'grpc://localhost:%d' % self._debugger_port
context = base_plugin.TBContext(logdir=self._dummy_logdir,
multiplexer=self._dummy_multiplexer)
self._debugger_plugin = (
interactive_debugger_plugin.InteractiveDebuggerPlugin(context))
self._debugger_plugin.listen(self._debugger_port)
wsgi_app = application.TensorBoardWSGIApp(
self._dummy_logdir,
[self._debugger_plugin],
self._dummy_multiplexer,
reload_interval=0,
path_prefix='')
self._server = werkzeug_test.Client(wsgi_app, wrappers.BaseResponse)
def testIsPortFree(self):
"""This might be flaky unless this test is run with a portserver."""
# The port should be free initially.
port = portpicker.pick_unused_port()
self.assertTrue(portpicker.is_port_free(port))
cases = [
(socket.AF_INET, socket.SOCK_STREAM, None),
(socket.AF_INET6, socket.SOCK_STREAM, 0),
(socket.AF_INET6, socket.SOCK_STREAM, 1),
(socket.AF_INET, socket.SOCK_DGRAM, None),
(socket.AF_INET6, socket.SOCK_DGRAM, 0),
(socket.AF_INET6, socket.SOCK_DGRAM, 1),
]
for (sock_family, sock_type, v6only) in cases:
# Occupy the port on a subset of possible protocols.
try:
sock = socket.socket(sock_family, sock_type, 0)
except socket.error:
print('Kernel does not support sock_family=%d' % sock_family,
file=sys.stderr)
# Skip this case, since we cannot occupy a port.
continue
if v6only is not None:
try:
sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY,
v6only)
except socket.error:
print('Kernel does not support IPV6_V6ONLY=%d' % v6only,
file=sys.stderr)
# Don't care; just proceed with the default.
sock.bind(('', port))
# The port should be busy.
self.assertFalse(portpicker.is_port_free(port))
sock.close()
# Now it's free again.
self.assertTrue(portpicker.is_port_free(port))
