def test_options_bool_flags(): # Most options are default False. # New options should follow the dash-case/snake_case convention. opt_str_to_attr_name = { name: re.sub(r"-", "_", name) for name in ["lazy-import", "old-naming", "add-iam-methods", "metadata", "warehouse-package-name", ]} for opt, attr in opt_str_to_attr_name.items(): options = Options.build("") assert not getattr(options, attr) options = Options.build(opt) assert getattr(options, attr) # Check autogen-snippets separately, as it is default True options = Options.build("") assert options.autogen_snippets options = Options.build("autogen-snippets=False") assert not options.autogen_snippets
def test_options_service_yaml_config(fs): opts = Options.build("") assert opts.service_yaml_config == {} service_yaml_fpath = "testapi_v1.yaml" fs.create_file(service_yaml_fpath, contents=("type: google.api.Service\n" "config_version: 3\n" "name: testapi.googleapis.com\n")) opt_string = f"service-yaml={service_yaml_fpath}" opts = Options.build(opt_string) expected_config = { "config_version": 3, "name": "testapi.googleapis.com" } assert opts.service_yaml_config == expected_config service_yaml_fpath = "testapi_v2.yaml" fs.create_file(service_yaml_fpath, contents=("config_version: 3\n" "name: testapi.googleapis.com\n")) opt_string = f"service-yaml={service_yaml_fpath}" opts = Options.build(opt_string) expected_config = { "config_version": 3, "name": "testapi.googleapis.com" } assert opts.service_yaml_config == expected_config
def test_options_service_config(fs): opts = Options.build("") assert opts.retry is None # Default of None is okay, verify build can read a config. service_config_fpath = "service_config.json" fs.create_file(service_config_fpath, contents="""{ "methodConfig": [ { "name": [ { "service": "animalia.mollusca.v1beta1.Cephalopod", "method": "IdentifySquid" } ], "retryPolicy": { "maxAttempts": 5, "maxBackoff": "3s", "initialBackoff": "0.2s", "backoffMultiplier": 2, "retryableStatusCodes": [ "UNAVAILABLE", "UNKNOWN" ] }, "timeout": "5s" } ] }""") opt_string = f"retry-config={service_config_fpath}" opts = Options.build(opt_string) # Verify the config was read in correctly. expected_cfg = { "methodConfig": [ { "name": [ { "service": "animalia.mollusca.v1beta1.Cephalopod", "method": "IdentifySquid", } ], "retryPolicy": { "maxAttempts": 5, "maxBackoff": "3s", "initialBackoff": "0.2s", "backoffMultiplier": 2, "retryableStatusCodes": [ "UNAVAILABLE", "UNKNOWN" ] }, "timeout":"5s" } ] } assert opts.retry == expected_cfg
def test_options_autogen_snippets_false_for_old_naming(): # NOTE: Snippets are not currently correct for the alternative (Ads) templates # so always disable snippetgen in that case # https://github.com/googleapis/gapic-generator-python/issues/1052 options = Options.build("old-naming") assert not options.autogen_snippets # Even if autogen-snippets is set to True, do not enable snippetgen options = Options.build("old-naming,autogen-snippets=True") assert not options.autogen_snippets
def test_parse_sample_paths(fs): fpath = "sampledir/sample.yaml" fs.create_file( fpath, contents=dedent(""" --- type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto schema_version: 1.2.0 samples: - service: google.cloud.language.v1.LanguageService """), ) with pytest.raises(types.InvalidConfig): Options.build("samples=sampledir/,")
def test_get_response_enumerates_services(): g = make_generator() with mock.patch.object(jinja2.FileSystemLoader, "list_templates") as lt: lt.return_value = [ "foo/%service/baz.py.j2", "molluscs/squid/sample.py.j2", ] with mock.patch.object(jinja2.Environment, "get_template") as gt: gt.return_value = jinja2.Template("Service: {{ service.name }}") cgr = g.get_response( api_schema=make_api( make_proto( descriptor_pb2.FileDescriptorProto(service=[ descriptor_pb2.ServiceDescriptorProto(name="Spam"), descriptor_pb2.ServiceDescriptorProto( name="EggsService"), ]), )), opts=Options.build(""), ) assert len(cgr.file) == 2 assert {i.name for i in cgr.file} == { "foo/spam/baz.py", "foo/eggs_service/baz.py", }
def test_custom_template_directory(): # Create a generator. opts = Options.build("python-gapic-templates=/templates/") g = generator.Generator(opts) # Assert that the Jinja loader will pull from the correct location. assert g._env.loader.searchpath == ["/templates"]
def generate(request: typing.BinaryIO, output: typing.BinaryIO) -> None: """Generate a full API client description.""" # Load the protobuf CodeGeneratorRequest. req = plugin_pb2.CodeGeneratorRequest.FromString(request.read()) # Pull apart arguments in the request. opts = Options.build(req.parameter) # Determine the appropriate package. # This generator uses a slightly different mechanism for determining # which files to generate; it tracks at package level rather than file # level. package = os.path.commonprefix([ p.package for p in req.proto_file if p.name in req.file_to_generate ]).rstrip('.') # Build the API model object. # This object is a frozen representation of the whole API, and is sent # to each template in the rendering step. api_schema = api.API.build(req.proto_file, opts=opts, package=package) # Translate into a protobuf CodeGeneratorResponse; this reads the # individual templates and renders them. # If there are issues, error out appropriately. res = generator.Generator(opts).get_response(api_schema, opts) # Output the serialized response. output.write(res.SerializeToString())
def test_generator_duplicate_samples(fs): config_fpath = "samples.yaml" fs.create_file( config_fpath, contents=dedent(""" # Note: the samples are duplicates. type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto schema_version: 1.2.0 samples: - id: squid_sample region_tag: humboldt_tag rpc: get_squid - id: squid_sample region_tag: humboldt_tag rpc: get_squid """), ) generator = make_generator("samples=samples.yaml") generator._env.loader = jinja2.DictLoader({"sample.py.j2": ""}) api_schema = make_api( naming=naming.NewNaming(name="Mollusc", version="v6")) with pytest.raises(types.DuplicateSample): generator.get_response(api_schema=api_schema, opts=Options.build(""))
def test_get_response_ignores_unwanted_transports_and_clients(): g = make_generator() with mock.patch.object(jinja2.FileSystemLoader, "list_templates") as lt: lt.return_value = [ "foo/%service/transports/river.py.j2", "foo/%service/transports/car.py.j2", "foo/%service/transports/grpc.py.j2", "foo/%service/transports/__init__.py.j2", "foo/%service/transports/base.py.j2", "foo/%service/async_client.py.j2", "foo/%service/client.py.j2", "mollusks/squid/sample.py.j2", ] with mock.patch.object(jinja2.Environment, "get_template") as gt: gt.return_value = jinja2.Template("Service: {{ service.name }}") api_schema = make_api( make_proto( descriptor_pb2.FileDescriptorProto(service=[ descriptor_pb2.ServiceDescriptorProto( name="SomeService"), ]), )) cgr = g.get_response(api_schema=api_schema, opts=Options.build("transport=river+car")) assert len(cgr.file) == 5 assert {i.name for i in cgr.file} == { "foo/some_service/transports/river.py", "foo/some_service/transports/car.py", "foo/some_service/transports/__init__.py", "foo/some_service/transports/base.py", # Only generate async client with grpc transport "foo/some_service/client.py", } cgr = g.get_response(api_schema=api_schema, opts=Options.build("transport=grpc")) assert len(cgr.file) == 5 assert {i.name for i in cgr.file} == { "foo/some_service/transports/grpc.py", "foo/some_service/transports/__init__.py", "foo/some_service/transports/base.py", "foo/some_service/client.py", "foo/some_service/async_client.py", }
def test_options_trim_whitespace(): # When writing shell scripts, users may construct options strings with # whitespace that needs to be trimmed after tokenizing. opts = Options.build(''' python-gapic-templates=/squid/clam/whelk , python-gapic-name=mollusca , ''') assert opts.templates[0] == '/squid/clam/whelk' assert opts.name == 'mollusca'
def test_get_response_fails_invalid_file_paths(): g = make_generator() with mock.patch.object(jinja2.FileSystemLoader, "list_templates") as lt: lt.return_value = [ "foo/bar/%service/%proto/baz.py.j2", ] with pytest.raises(ValueError) as ex: g.get_response(api_schema=make_api(), opts=Options.build("")) ex_str = str(ex.value) assert "%proto" in ex_str and "%service" in ex_str
def test_get_response_ignores_empty_files(): g = make_generator() with mock.patch.object(jinja2.FileSystemLoader, "list_templates") as lt: lt.return_value = ["foo/bar/baz.py.j2", "molluscs/squid/sample.py.j2"] with mock.patch.object(jinja2.Environment, "get_template") as gt: gt.return_value = jinja2.Template("# Meaningless comment") cgr = g.get_response(api_schema=make_api(), opts=Options.build("")) lt.assert_called_once() gt.assert_has_calls([ mock.call("foo/bar/baz.py.j2"), mock.call("molluscs/squid/sample.py.j2"), ]) assert len(cgr.file) == 0
def test_options_bool_flags(): # All these options are default False. # If new options violate this assumption, # this test may need to be tweaked. # New options should follow the dash-case/snake_case convention. opt_str_to_attr_name = { name: re.sub(r"-", "_", name) for name in [ "lazy-import", "old-naming", "add-iam-methods", "metadata", "warehouse-package-name", ] } for opt, attr in opt_str_to_attr_name.items(): options = Options.build("") assert not getattr(options, attr) options = Options.build(opt) assert getattr(options, attr)
def test_generate_autogen_samples(mock_generate_sample, mock_generate_specs): opts = Options.build("autogen-snippets") g = generator.Generator(opts) # Need to have the sample template visible to the generator. g._env.loader = jinja2.DictLoader({"sample.py.j2": ""}) api_schema = make_api( naming=naming.NewNaming(name="Mollusc", version="v6")) actual_response = g.get_response(api_schema, opts=opts) # Just check that generate_sample_specs was called # Correctness of the spec is tested in samplegen unit tests mock_generate_specs.assert_called_once_with(api_schema, opts=opts)
def test_get_response_ignore_gapic_metadata(): g = make_generator() with mock.patch.object(jinja2.FileSystemLoader, "list_templates") as lt: lt.return_value = ["gapic/gapic_metadata.json.j2"] with mock.patch.object(jinja2.Environment, "get_template") as gt: gt.return_value = jinja2.Template( "This is not something we want to see") res = g.get_response( api_schema=make_api(), opts=Options.build(""), ) # We don't expect any files because opts.metadata is not set. assert res.file == CodeGeneratorResponse().file
def test_get_response_divides_subpackages(): # NOTE: autogen-snippets is intentionally disabled for this test # The API schema below is incomplete and will result in errors when the # snippetgen logic tries to parse it. g = make_generator("autogen-snippets=false") api_schema = api.API.build( [ descriptor_pb2.FileDescriptorProto( name="top.proto", package="foo.v1", service=[descriptor_pb2.ServiceDescriptorProto(name="Top")], ), descriptor_pb2.FileDescriptorProto( name="a/spam/ham.proto", package="foo.v1.spam", service=[descriptor_pb2.ServiceDescriptorProto(name="Bacon")], ), descriptor_pb2.FileDescriptorProto( name="a/eggs/yolk.proto", package="foo.v1.eggs", service=[ descriptor_pb2.ServiceDescriptorProto(name="Scramble") ], ), ], package="foo.v1", ) with mock.patch.object(jinja2.FileSystemLoader, "list_templates") as lt: lt.return_value = [ "foo/%sub/types/%proto.py.j2", "foo/%sub/services/%service.py.j2", "molluscs/squid/sample.py.j2", ] with mock.patch.object(jinja2.Environment, "get_template") as gt: gt.return_value = jinja2.Template(""" {{- '' }}Subpackage: {{ '.'.join(api.subpackage_view) }} """.strip()) cgr = g.get_response(api_schema=api_schema, opts=Options.build("autogen-snippets=false")) assert len(cgr.file) == 6 assert {i.name for i in cgr.file} == { "foo/types/top.py", "foo/services/top.py", "foo/spam/types/ham.py", "foo/spam/services/bacon.py", "foo/eggs/types/yolk.py", "foo/eggs/services/scramble.py", }
def test_get_response(): g = make_generator() with mock.patch.object(jinja2.FileSystemLoader, "list_templates") as lt: lt.return_value = ["foo/bar/baz.py.j2", "molluscs/squid/sample.py.j2"] with mock.patch.object(jinja2.Environment, "get_template") as gt: gt.return_value = jinja2.Template("I am a template result.") cgr = g.get_response(api_schema=make_api(), opts=Options.build("")) lt.assert_called_once() gt.assert_has_calls([ mock.call("foo/bar/baz.py.j2"), mock.call("molluscs/squid/sample.py.j2"), ]) assert len(cgr.file) == 1 assert cgr.file[0].name == "foo/bar/baz.py" assert cgr.file[0].content == "I am a template result.\n"
def test_get_response_divides_subpackages(): g = make_generator() api_schema = api.API.build( [ descriptor_pb2.FileDescriptorProto( name="top.proto", package="foo.v1", service=[descriptor_pb2.ServiceDescriptorProto(name="Top")], ), descriptor_pb2.FileDescriptorProto( name="a/spam/ham.proto", package="foo.v1.spam", service=[descriptor_pb2.ServiceDescriptorProto(name="Bacon")], ), descriptor_pb2.FileDescriptorProto( name="a/eggs/yolk.proto", package="foo.v1.eggs", service=[descriptor_pb2.ServiceDescriptorProto( name="Scramble")], ), ], package="foo.v1", ) with mock.patch.object(jinja2.FileSystemLoader, "list_templates") as lt: lt.return_value = [ "foo/%sub/types/%proto.py.j2", "foo/%sub/services/%service.py.j2", "molluscs/squid/sample.py.j2", ] with mock.patch.object(jinja2.Environment, "get_template") as gt: gt.return_value = jinja2.Template( """ {{- '' }}Subpackage: {{ '.'.join(api.subpackage_view) }} """.strip() ) cgr = g.get_response(api_schema=api_schema, opts=Options.build("")) assert len(cgr.file) == 6 assert {i.name for i in cgr.file} == { "foo/types/top.py", "foo/services/top.py", "foo/spam/types/ham.py", "foo/spam/services/bacon.py", "foo/eggs/types/yolk.py", "foo/eggs/services/scramble.py", }
def test_get_response_enumerates_proto(): g = make_generator() with mock.patch.object(jinja2.FileSystemLoader, "list_templates") as lt: lt.return_value = [ "foo/%proto.py.j2", "molluscs/squid/sample.py.j2", ] with mock.patch.object(jinja2.Environment, "get_template") as gt: gt.return_value = jinja2.Template("Proto: {{ proto.module_name }}") cgr = g.get_response( api_schema=make_api( make_proto( descriptor_pb2.FileDescriptorProto(name="a.proto")), make_proto( descriptor_pb2.FileDescriptorProto(name="b.proto")), ), opts=Options.build(""), ) assert len(cgr.file) == 2 assert {i.name for i in cgr.file} == {"foo/a.py", "foo/b.py"}
def test_samplegen_id_disambiguation(mock_gmtime, mock_generate_sample, fs): # These time values are nothing special, # they just need to be deterministic. returner = mock.MagicMock() returner.tm_year = 2112 returner.tm_mon = 6 returner.tm_mday = 1 returner.tm_hour = 13 returner.tm_min = 13 returner.tm_sec = 13 mock_gmtime.return_value = returner # Note: The first two samples will have the same nominal ID, the first by # explicit naming and the second by falling back to the region_tag. # The third has no id of any kind, so the generator is required to make a # unique ID for it. fs.create_file( "samples.yaml", contents=dedent(""" --- type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto schema_version: 1.2.0 samples: - id: squid_sample region_tag: humboldt_tag rpc: get_squid_streaming # Note that this region tag collides with the id of the previous sample. - region_tag: squid_sample rpc: get_squid_streaming # No id or region tag. - rpc: get_squid_streaming """), ) g = generator.Generator(Options.build("samples=samples.yaml")) # Need to have the sample template visible to the generator. g._env.loader = jinja2.DictLoader({"sample.py.j2": ""}) api_schema = make_api( naming=naming.NewNaming(name="Mollusc", version="v6")) actual_response = g.get_response(api_schema, opts=Options.build("")) expected_response = CodeGeneratorResponse(file=[ CodeGeneratorResponse.File( name="samples/generated_samples/squid_sample_91a465c6.py", content="\n", ), CodeGeneratorResponse.File( name="samples/generated_samples/squid_sample_55051b38.py", content="\n", ), CodeGeneratorResponse.File( name="samples/generated_samples/157884ee.py", content="\n", ), # TODO(busunkim): Re-enable manifest generation once metadata # format has been formalized. # https://docs.google.com/document/d/1ghBam8vMj3xdoe4xfXhzVcOAIwrkbTpkMLgKc9RPD9k/edit#heading=h.sakzausv6hue # CodeGeneratorResponse.File( # name="samples/generated_samples/mollusc.v6.python.21120601.131313.manifest.yaml", # content=dedent( # """\ # --- # type: manifest/samples # schema_version: 3 # python: &python # environment: python # bin: python3 # base_path: samples # invocation: '{bin} {path} @args' # samples: # - <<: *python # sample: squid_sample_91a465c6 # path: '{base_path}/squid_sample_91a465c6.py' # region_tag: humboldt_tag # - <<: *python # sample: squid_sample_55051b38 # path: '{base_path}/squid_sample_55051b38.py' # region_tag: squid_sample # - <<: *python # sample: 157884ee # path: '{base_path}/157884ee.py' # """ # ), # ), ]) expected_response.supported_features |= ( CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL) assert actual_response == expected_response
def test_samplegen_config_to_output_files( mock_gmtime, mock_generate_sample, fs, ): # These time values are nothing special, # they just need to be deterministic. returner = mock.MagicMock() returner.tm_year = 2112 returner.tm_mon = 6 returner.tm_mday = 1 returner.tm_hour = 13 returner.tm_min = 13 returner.tm_sec = 13 mock_gmtime.return_value = returner fs.create_file( "samples.yaml", contents=dedent(""" --- type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto schema_version: 1.2.0 samples: - id: squid_sample region_tag: humboldt_tag rpc: get_squid_streaming - region_tag: clam_sample rpc: get_clam """), ) g = generator.Generator(Options.build("samples=samples.yaml", )) # Need to have the sample template visible to the generator. g._env.loader = jinja2.DictLoader({"sample.py.j2": ""}) api_schema = make_api( naming=naming.NewNaming(name="Mollusc", version="v6")) actual_response = g.get_response(api_schema, opts=Options.build("")) expected_response = CodeGeneratorResponse(file=[ CodeGeneratorResponse.File( name="samples/generated_samples/squid_sample.py", content="\n", ), CodeGeneratorResponse.File( name="samples/generated_samples/clam_sample.py", content="\n", ), # TODO(busunkim): Re-enable manifest generation once metadata # format has been formalized. # https://docs.google.com/document/d/1ghBam8vMj3xdoe4xfXhzVcOAIwrkbTpkMLgKc9RPD9k/edit#heading=h.sakzausv6hue # CodeGeneratorResponse.File( # name="samples/generated_samples/mollusc.v6.python.21120601.131313.manifest.yaml", # content=dedent( # """\ # --- # type: manifest/samples # schema_version: 3 # python: &python # environment: python # bin: python3 # base_path: samples # invocation: '{bin} {path} @args' # samples: # - <<: *python # sample: squid_sample # path: '{base_path}/squid_sample.py' # region_tag: humboldt_tag # - <<: *python # sample: clam_sample # path: '{base_path}/clam_sample.py' # region_tag: clam_sample # """ # ), # ), ]) expected_response.supported_features |= ( CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL) assert actual_response == expected_response
def test_options_empty(): opts = Options.build('') assert len(opts.templates) == 1 assert opts.templates[0].endswith('gapic/templates') assert not opts.lazy_import assert not opts.old_naming
def test_options_replace_templates(): opts = Options.build('python-gapic-templates=/foo/') assert len(opts.templates) == 1 assert opts.templates[0] == '/foo'
def test_options_relative_templates(): opts = Options.build('python-gapic-templates=../../squid/clam') expected = (os.path.abspath('../squid/clam'), ) assert opts.templates == expected
def test_dont_generate_in_code_samples(mock_gmtime, mock_generate_sample, fs): # These time values are nothing special, # they just need to be deterministic. returner = mock.MagicMock() returner.tm_year = 2112 returner.tm_mon = 6 returner.tm_mday = 1 returner.tm_hour = 13 returner.tm_min = 13 returner.tm_sec = 13 mock_gmtime.return_value = returner config_fpath = "samples.yaml" fs.create_file( config_fpath, contents=dedent(""" type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto schema_version: 1.2.0 samples: - id: squid_sample rpc: IdentifyMollusc service: Mollusc.v1.Mollusc sample_type: - standalone - incode/SQUID - id: clam_sample rpc: IdentifyMollusc service: Mollusc.v1.Mollusc sample_type: - incode/CLAM - id: whelk_sample rpc: IdentifyMollusc service: Mollusc.v1.Mollusc sample_type: - standalone - id: octopus_sample rpc: IdentifyMollusc service: Mollusc.v1.Mollusc """), ) generator = make_generator(f"samples={config_fpath}") generator._env.loader = jinja2.DictLoader({"sample.py.j2": ""}) api_schema = make_api( make_proto( descriptor_pb2.FileDescriptorProto( name="mollusc.proto", package="Mollusc.v1", service=[ descriptor_pb2.ServiceDescriptorProto(name="Mollusc") ], ), ), naming=naming.NewNaming(name="Mollusc", version="v6"), ) # Note that we do NOT expect a clam sample. # There are four tests going on: # 1) Just an explicit standalone sample type. # 2) Multiple sample types, one of which is standalone. # 3) Explicit sample types but NO standalone sample type. # 4) Implicit standalone sample type. expected = CodeGeneratorResponse(file=[ CodeGeneratorResponse.File( name="samples/generated_samples/squid_sample.py", content="\n", ), CodeGeneratorResponse.File( name="samples/generated_samples/whelk_sample.py", content="\n", ), CodeGeneratorResponse.File( name="samples/generated_samples/octopus_sample.py", content="\n", ), # TODO(busunkim): Re-enable manifest generation once metadata # format has been formalized. # https://docs.google.com/document/d/1ghBam8vMj3xdoe4xfXhzVcOAIwrkbTpkMLgKc9RPD9k/edit#heading=h.sakzausv6hue # CodeGeneratorResponse.File( # name="samples/generated_samples/mollusc.v6.python.21120601.131313.manifest.yaml", # content=dedent( # """ --- # type: manifest/samples # schema_version: 3 # python: &python # environment: python # bin: python3 # base_path: samples # invocation: \'{bin} {path} @args\' # samples: # - <<: *python # sample: squid_sample # path: \'{base_path}/squid_sample.py\' # - <<: *python # sample: whelk_sample # path: \'{base_path}/whelk_sample.py\' # - <<: *python # sample: octopus_sample # path: \'{base_path}/octopus_sample.py\' # """ # ), # ), ]) expected.supported_features |= CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL actual = generator.get_response(api_schema=api_schema, opts=Options.build("")) assert actual == expected
def test_options_unrecognized_likely_typo(): with mock.patch.object(warnings, 'warn') as warn: Options.build('go-gapic-abc=xyz') assert len(warn.mock_calls) == 0
def test_flags_unrecognized(): with mock.patch.object(warnings, 'warn') as warn: Options.build('python-gapic-abc') warn.assert_called_once_with('Unrecognized option: `python-gapic-abc`.')
def make_generator(opts_str: str = "") -> generator.Generator: return generator.Generator(Options.build(opts_str))
def test_options_no_valid_sample_config(fs): fs.create_file("sampledir/not_a_config.yaml") with pytest.raises(types.InvalidConfig): Options.build("samples=sampledir/")