def subprocess_cmd_and_endpoint(self):
jar_path = self.local_jar(self.path_to_jar())
artifacts_dir = self.local_temp_dir(prefix='artifacts')
job_port, = subprocess_server.pick_port(None)
return (['java', '-jar', jar_path] +
list(self.java_arguments(job_port, artifacts_dir)),
'localhost:%s' % job_port)
def subprocess_cmd_and_endpoint(self):
jar_path = self.local_jar(self.path_to_jar())
artifacts_dir = (self._artifacts_dir if self._artifacts_dir else
self.local_temp_dir(prefix='artifacts'))
job_port, = subprocess_server.pick_port(self._job_port)
return ([self._java_launcher, '-jar', jar_path] + list(
self.java_arguments(job_port, self._artifact_port,
self._expansion_port, artifacts_dir)),
'localhost:%s' % job_port)
def subprocess_cmd_and_endpoint(self):
# TODO This is hardcoded to Flink at the moment but should be changed
job_server_image_name = "apache/beam_flink1.9_job_server:latest"
docker_path = subprocess.check_output(['which',
'docker']).strip().decode('utf-8')
cmd = [
"docker",
"run",
# We mount the docker binary and socket to be able to spin up
# "sibling" containers for the SDK harness.
"-v",
':'.join([docker_path, "/bin/docker"]),
"-v",
"/var/run/docker.sock:/var/run/docker.sock"
]
self.job_port, self.artifact_port, self.expansion_port = (
subprocess_server.pick_port(
self.job_port, self.artifact_port, self.expansion_port))
args = [
'--job-host',
self.job_host,
'--job-port',
str(self.job_port),
'--artifact-port',
str(self.artifact_port),
'--expansion-port',
str(self.expansion_port)
]
if sys.platform == "darwin":
# Docker-for-Mac doesn't support host networking, so we need to explictly
# publish ports from the Docker container to be able to connect to it.
# Also, all other containers need to be aware that they run Docker-on-Mac
# to connect against the internal Docker-for-Mac address.
cmd += ["-e", "DOCKER_MAC_CONTAINER=1"]
cmd += ["-p", "{}:{}".format(self.job_port, self.job_port)]
cmd += ["-p", "{}:{}".format(self.artifact_port, self.artifact_port)]
cmd += ["-p", "{}:{}".format(self.expansion_port, self.expansion_port)]
cmd += [
"-p",
"{0}-{1}:{0}-{1}".format(
self.harness_port_range[0], self.harness_port_range[1])
]
else:
# This shouldn't be set for MacOS because it detroys port forwardings,
# even though host networking is not supported on MacOS.
cmd.append("--network=host")
cmd.append(job_server_image_name)
return cmd + args, '%s:%s' % (self.job_host, self.job_port)
def test_local_jar(self):
class Handler(socketserver.BaseRequestHandler):
timeout = 1
def handle(self):
self.request.recv(1024)
self.request.sendall(b'HTTP/1.1 200 OK\n\ndata')
port, = subprocess_server.pick_port(None)
server = socketserver.TCPServer(('localhost', port), Handler)
t = threading.Thread(target=server.handle_request)
t.daemon = True
t.start()
with TemporaryDirectory() as temp_dir:
subprocess_server.JavaJarServer.local_jar(
'http://localhost:%s/path/to/file.jar' % port, temp_dir)
with open(os.path.join(temp_dir, 'file.jar')) as fin:
self.assertEqual(fin.read(), 'data')
