def test_environment_encoding(self):
for environment in (DockerEnvironment(),
DockerEnvironment(container_image='img'),
ProcessEnvironment('run.sh'),
ProcessEnvironment('run.sh',
os='linux',
arch='amd64',
env={'k1': 'v1'}),
ExternalEnvironment('localhost:8080'),
ExternalEnvironment('localhost:8080',
params={'k1': 'v1'}),
EmbeddedPythonEnvironment(),
EmbeddedPythonGrpcEnvironment(),
EmbeddedPythonGrpcEnvironment(num_workers=2,
state_cache_size=0),
SubprocessSDKEnvironment(command_string=u'foö')):
context = pipeline_context.PipelineContext()
self.assertEqual(
environment,
Environment.from_runner_api(environment.to_runner_api(context),
context))
with self.assertRaises(ValueError) as ctx:
EmbeddedPythonGrpcEnvironment(num_workers=2).to_runner_api(
pipeline_context.PipelineContext())
self.assertIn('Must provide worker num and state cache size.',
ctx.exception.args)
def test_environment_encoding(self):
for environment in (DockerEnvironment(),
DockerEnvironment(container_image='img'),
ProcessEnvironment('run.sh'),
ProcessEnvironment('run.sh',
os='linux',
arch='amd64',
env={'k1': 'v1'}),
ExternalEnvironment('localhost:8080'),
ExternalEnvironment('localhost:8080',
params={'k1': 'v1'}),
EmbeddedPythonEnvironment(),
EmbeddedPythonGrpcEnvironment(),
EmbeddedPythonGrpcEnvironment(state_cache_size=0),
SubprocessSDKEnvironment(command_string=u'foö')):
context = pipeline_context.PipelineContext()
self.assertEqual(
environment,
Environment.from_runner_api(environment.to_runner_api(context),
context))
def test_environment_encoding(self):
for environment in (DockerEnvironment(),
DockerEnvironment(container_image='img'),
DockerEnvironment(capabilities=['x, y, z']),
ProcessEnvironment('run.sh'),
ProcessEnvironment('run.sh',
os='linux',
arch='amd64',
env={'k1': 'v1'}),
ExternalEnvironment('localhost:8080'),
ExternalEnvironment('localhost:8080',
params={'k1': 'v1'}),
EmbeddedPythonEnvironment(),
EmbeddedPythonGrpcEnvironment(),
EmbeddedPythonGrpcEnvironment(
state_cache_size=0, data_buffer_time_limit_ms=0),
SubprocessSDKEnvironment(command_string=u'foö')):
context = pipeline_context.PipelineContext()
proto = environment.to_runner_api(context)
reconstructed = Environment.from_runner_api(proto, context)
self.assertEqual(environment, reconstructed)
self.assertEqual(proto, reconstructed.to_runner_api(context))
def __init__(self,
payload, # type: bytes
state, # type: StateServicer
provision_info, # type: ExtendedProvisionInfo
grpc_server # type: GrpcServer
):
# type: (...) -> None
super().__init__(state, provision_info, grpc_server)
from apache_beam.transforms.environments import EmbeddedPythonGrpcEnvironment
config = EmbeddedPythonGrpcEnvironment.parse_config(payload.decode('utf-8'))
self._state_cache_size = config.get('state_cache_size') or STATE_CACHE_SIZE
self._data_buffer_time_limit_ms = \
config.get('data_buffer_time_limit_ms') or DATA_BUFFER_TIME_LIMIT_MS
