def test_environment_encoding(self): for environment in (DockerEnvironment(), DockerEnvironment(container_image='img'), ProcessEnvironment('run.sh'), ProcessEnvironment('run.sh', os='linux', arch='amd64', env={'k1': 'v1'}), ExternalEnvironment('localhost:8080'), ExternalEnvironment('localhost:8080', params={'k1': 'v1'}), EmbeddedPythonEnvironment(), EmbeddedPythonGrpcEnvironment(), EmbeddedPythonGrpcEnvironment(num_workers=2, state_cache_size=0), SubprocessSDKEnvironment(command_string=u'foö')): context = pipeline_context.PipelineContext() self.assertEqual( environment, Environment.from_runner_api(environment.to_runner_api(context), context)) with self.assertRaises(ValueError) as ctx: EmbeddedPythonGrpcEnvironment(num_workers=2).to_runner_api( pipeline_context.PipelineContext()) self.assertIn('Must provide worker num and state cache size.', ctx.exception.args)
def test_environment_encoding(self): for environment in (DockerEnvironment(), DockerEnvironment(container_image='img'), ProcessEnvironment('run.sh'), ProcessEnvironment('run.sh', os='linux', arch='amd64', env={'k1': 'v1'}), ExternalEnvironment('localhost:8080'), ExternalEnvironment('localhost:8080', params={'k1': 'v1'}), EmbeddedPythonEnvironment(), EmbeddedPythonGrpcEnvironment(), EmbeddedPythonGrpcEnvironment(state_cache_size=0), SubprocessSDKEnvironment(command_string=u'foö')): context = pipeline_context.PipelineContext() self.assertEqual( environment, Environment.from_runner_api(environment.to_runner_api(context), context))
def test_environment_encoding(self): for environment in (DockerEnvironment(), DockerEnvironment(container_image='img'), DockerEnvironment(capabilities=['x, y, z']), ProcessEnvironment('run.sh'), ProcessEnvironment('run.sh', os='linux', arch='amd64', env={'k1': 'v1'}), ExternalEnvironment('localhost:8080'), ExternalEnvironment('localhost:8080', params={'k1': 'v1'}), EmbeddedPythonEnvironment(), EmbeddedPythonGrpcEnvironment(), EmbeddedPythonGrpcEnvironment( state_cache_size=0, data_buffer_time_limit_ms=0), SubprocessSDKEnvironment(command_string=u'foö')): context = pipeline_context.PipelineContext() proto = environment.to_runner_api(context) reconstructed = Environment.from_runner_api(proto, context) self.assertEqual(environment, reconstructed) self.assertEqual(proto, reconstructed.to_runner_api(context))
def __init__(self, payload, # type: bytes state, # type: StateServicer provision_info, # type: ExtendedProvisionInfo grpc_server # type: GrpcServer ): # type: (...) -> None super().__init__(state, provision_info, grpc_server) from apache_beam.transforms.environments import EmbeddedPythonGrpcEnvironment config = EmbeddedPythonGrpcEnvironment.parse_config(payload.decode('utf-8')) self._state_cache_size = config.get('state_cache_size') or STATE_CACHE_SIZE self._data_buffer_time_limit_ms = \ config.get('data_buffer_time_limit_ms') or DATA_BUFFER_TIME_LIMIT_MS