def main(input_file=sys.stdin, output_file=sys.stdout):
"""Parse a CodeGeneratorRequest and return a CodeGeneratorResponse."""
# Ensure we are getting a bytestream, and writing to a bytestream.
if hasattr(input_file, 'buffer'):
input_file = input_file.buffer
if hasattr(output_file, 'buffer'):
output_file = output_file.buffer
# Instantiate a parser.
parser = CodeGeneratorParser.from_input_file(input_file)
# Find all the docs and amalgamate them together.
comment_data = {}
for filename, message_structure in parser.find_docs():
comment_data.setdefault(filename, set())
comment_data[filename].add(message_structure)
# Iterate over the data that came back and parse it into a single,
# coherent CodeGeneratorResponse.
answer = []
_BATCH_TOKEN = "CD985272F78311"
meta_docstrings = []
meta_structs = []
for fn, structs in comment_data.items():
for struct in structs:
if meta_docstrings:
meta_docstrings.append("\n%s" % _BATCH_TOKEN)
meta_docstrings.append(struct.get_meta_docstring())
meta_structs.append((fn, struct))
meta_docstring = convert_text("".join(meta_docstrings), 'rst', format='md')
meta_docstrings = meta_docstring.split("%s" % _BATCH_TOKEN)
index = 0
while index < len(meta_structs) and index < len(meta_docstrings):
fn = meta_structs[index][0]
struct = meta_structs[index][1]
answer.append(
CodeGeneratorResponse.File(
name=fn.replace('.proto', '_pb2.py'),
insertion_point='class_scope:%s' % struct.name,
content=',\n__doc__ = """{docstring}""",'.format(
docstring=struct.get_python_docstring(
meta_docstrings[index]), ),
))
index += 1
for fn in _init_files(comment_data.keys()):
answer.append(CodeGeneratorResponse.File(
name=fn,
content='',
))
cgr = CodeGeneratorResponse(file=answer)
output_file.write(cgr.SerializeToString())
def _get_file(
self,
template_name: str,
*,
opts: Options,
api_schema: api.API,
**context,
):
"""Render a template to a protobuf plugin File object."""
# Determine the target filename.
fn = self._get_filename(
template_name, api_schema=api_schema, context=context,)
# Render the file contents.
cgr_file = CodeGeneratorResponse.File(
content=formatter.fix_whitespace(
self._env.get_template(template_name).render(
api=api_schema, opts=opts, **context
),
),
name=fn,
)
# Quick check: Do not render empty files.
if utils.empty(cgr_file.content) and not fn.endswith(
("py.typed", "__init__.py")
):
return {}
# Return the filename and content in a length-1 dictionary
# (because we track output files overall in a dictionary).
return {fn: cgr_file}
def main(input_file=sys.stdin, output_file=sys.stdout):
"""Parse a CodeGeneratorRequest and return a CodeGeneratorResponse."""
logging.basicConfig(filename='logging.log', level=logging.DEBUG)
# Ensure we are getting a bytestream, and writing to a bytestream.
if hasattr(input_file, 'buffer'):
input_file = input_file.buffer
if hasattr(output_file, 'buffer'):
output_file = output_file.buffer
try:
# Instantiate a parser.
parser = CodeGeneratorParser.from_input_file(input_file)
docs = parser.generate_docs()
except:
logging.exception("Error when generating docs: ")
raise
answer = []
for k, item in docs.items():
answer.append(
CodeGeneratorResponse.File(
name=item.filename,
content=item.content,
))
cgr = CodeGeneratorResponse(file=answer)
output_file.write(cgr.SerializeToString())
def main(input_file=sys.stdin, output_file=sys.stdout):
request = CodeGeneratorRequest.FromString(input_file.buffer.read())
answer = []
for fname in request.file_to_generate:
answer.append(
CodeGeneratorResponse.File(
name=fname.replace('.proto', '_pb2.py'),
insertion_point='module_scope',
content="# Hello {}, I'm a dummy plugin!".format(fname),
))
cgr = CodeGeneratorResponse(file=answer)
output_file.buffer.write(cgr.SerializeToString())
def _get_file(self, template_name: str, *,
api_schema=api.API,
**context: Mapping):
"""Render a template to a protobuf plugin File object."""
fn = self._get_filename(template_name,
api_schema=api_schema,
context=context,
)
return {fn: CodeGeneratorResponse.File(
content=formatter.fix_whitespace(
self._env.get_template(template_name).render(
api=api_schema,
**context
),
),
name=fn,
)}
def _render_templates(
self,
templates: Iterable[str],
*,
additional_context: Mapping[str, Any] = None,
) -> Sequence[CodeGeneratorResponse.File]:
"""Render the requested templates.
Args:
templates (Iterable[str]): The set of templates to be rendered.
It is expected that these come from the methods on
:class:`~.loader.TemplateLoader`, and they should be
able to be set to the :meth:`jinja2.Environment.get_template`
method.
additional_context (Mapping[str, Any]): Additional variables
to be sent to the templates. The ``api`` variable
is always available.
Returns:
Sequence[~.CodeGeneratorResponse.File]: A sequence of File
objects for inclusion in the final response.
"""
answer = []
additional_context = additional_context or {}
# Iterate over the provided templates and generate a File object
# for each.
for template_name in templates:
# Generate the File object.
answer.append(
CodeGeneratorResponse.File(
content=formatter.fix_whitespace(
self._env.get_template(template_name).render(
api=self._api, **additional_context), ),
name=self._get_output_filename(
template_name,
context=additional_context,
),
))
# Done; return the File objects based on these templates.
return answer
def test_samplegen_id_disambiguation(mock_gmtime, mock_generate_sample, fs):
# These time values are nothing special,
# they just need to be deterministic.
returner = mock.MagicMock()
returner.tm_year = 2112
returner.tm_mon = 6
returner.tm_mday = 1
returner.tm_hour = 13
returner.tm_min = 13
returner.tm_sec = 13
mock_gmtime.return_value = returner
# Note: The first two samples will have the same nominal ID, the first by
# explicit naming and the second by falling back to the region_tag.
# The third has no id of any kind, so the generator is required to make a
# unique ID for it.
fs.create_file(
"samples.yaml",
contents=dedent("""
---
type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto
schema_version: 1.2.0
samples:
- id: squid_sample
region_tag: humboldt_tag
rpc: get_squid_streaming
# Note that this region tag collides with the id of the previous sample.
- region_tag: squid_sample
rpc: get_squid_streaming
# No id or region tag.
- rpc: get_squid_streaming
"""),
)
g = generator.Generator(Options.build("samples=samples.yaml"))
# Need to have the sample template visible to the generator.
g._env.loader = jinja2.DictLoader({"sample.py.j2": ""})
api_schema = make_api(
naming=naming.NewNaming(name="Mollusc", version="v6"))
actual_response = g.get_response(api_schema, opts=Options.build(""))
expected_response = CodeGeneratorResponse(file=[
CodeGeneratorResponse.File(
name="samples/squid_sample_91a465c6.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/squid_sample_55051b38.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/157884ee.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/mollusc.v6.python.21120601.131313.manifest.yaml",
content=dedent("""\
---
type: manifest/samples
schema_version: 3
python: &python
environment: python
bin: python3
base_path: samples
invocation: '{bin} {path} @args'
samples:
- <<: *python
sample: squid_sample_91a465c6
path: '{base_path}/squid_sample_91a465c6.py'
region_tag: humboldt_tag
- <<: *python
sample: squid_sample_55051b38
path: '{base_path}/squid_sample_55051b38.py'
region_tag: squid_sample
- <<: *python
sample: 157884ee
path: '{base_path}/157884ee.py'
"""),
),
])
expected_response.supported_features |= (
CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL)
assert actual_response == expected_response
def _generate_samples_and_manifest(
self,
api_schema: api.API,
sample_template: jinja2.Template,
) -> Dict[str, CodeGeneratorResponse.File]:
"""Generate samples and samplegen manifest for the API.
Arguments:
api_schema (api.API): The schema for the API to which the samples belong.
Returns:
Dict[str, CodeGeneratorResponse.File]: A dict mapping filepath to rendered file.
"""
# The two-layer data structure lets us do two things:
# * detect duplicate samples, which is an error
# * detect distinct samples with the same ID, which are disambiguated
id_to_hash_to_spec: DefaultDict[str, Dict[str,
Any]] = defaultdict(dict)
STANDALONE_TYPE = "standalone"
for config_fpath in self._sample_configs:
with open(config_fpath) as f:
configs = yaml.safe_load_all(f.read())
spec_generator = (
spec for cfg in configs if is_valid_sample_cfg(cfg)
for spec in cfg.get("samples", [])
# If unspecified, assume a sample config describes a standalone.
# If sample_types are specified, standalone samples must be
# explicitly enabled.
if STANDALONE_TYPE in spec.get("sample_type",
[STANDALONE_TYPE]))
for spec in spec_generator:
# Every sample requires an ID, preferably provided by the
# samplegen config author.
# If no ID is provided, fall back to the region tag.
# If there's no region tag, generate a unique ID.
#
# Ideally the sample author should pick a descriptive, unique ID,
# but this may be impractical and can be error-prone.
spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8]
sample_id = spec.get("id") or spec.get(
"region_tag") or spec_hash
spec["id"] = sample_id
hash_to_spec = id_to_hash_to_spec[sample_id]
if spec_hash in hash_to_spec:
raise DuplicateSample(
f"Duplicate samplegen spec found: {spec}")
hash_to_spec[spec_hash] = spec
out_dir = "samples"
fpath_to_spec_and_rendered = {}
for hash_to_spec in id_to_hash_to_spec.values():
for spec_hash, spec in hash_to_spec.items():
id_is_unique = len(hash_to_spec) == 1
# The ID is used to generate the file name and by sample tester
# to link filenames to invoked samples. It must be globally unique.
if not id_is_unique:
spec["id"] += f"_{spec_hash}"
sample = samplegen.generate_sample(
spec,
api_schema,
sample_template,
)
fpath = spec["id"] + ".py"
fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = (
spec,
sample,
)
output_files = {
fname: CodeGeneratorResponse.File(
content=formatter.fix_whitespace(sample), name=fname)
for fname, (_, sample) in fpath_to_spec_and_rendered.items()
}
# Only generate a manifest if we generated samples.
if output_files:
manifest_fname, manifest_doc = manifest.generate(
((fname, spec)
for fname, (spec, _) in fpath_to_spec_and_rendered.items()),
api_schema,
)
manifest_fname = os.path.join(out_dir, manifest_fname)
output_files[manifest_fname] = CodeGeneratorResponse.File(
content=manifest_doc.render(), name=manifest_fname)
return output_files
def _generate_samples_and_manifest(
self, api_schema: api.API, index: snippet_index.SnippetIndex, sample_template: jinja2.Template, *, opts: Options) -> Tuple[Dict, snippet_index.SnippetIndex]:
"""Generate samples and samplegen manifest for the API.
Arguments:
api_schema (api.API): The schema for the API to which the samples belong.
sample_template (jinja2.Template): The template to use to generate samples.
opts (Options): Additional generator options.
Returns:
Tuple[Dict[str, CodeGeneratorResponse.File], snippet_index.SnippetIndex] : A dict mapping filepath to rendered file.
"""
# The two-layer data structure lets us do two things:
# * detect duplicate samples, which is an error
# * detect distinct samples with the same ID, which are disambiguated
id_to_hash_to_spec: DefaultDict[str,
Dict[str, Any]] = defaultdict(dict)
# Autogenerated sample specs
autogen_specs: typing.List[typing.Dict[str, Any]] = []
if opts.autogen_snippets:
autogen_specs = list(
samplegen.generate_sample_specs(api_schema, opts=opts))
# Also process any handwritten sample specs
handwritten_specs = samplegen.parse_handwritten_specs(
self._sample_configs)
sample_specs = autogen_specs + list(handwritten_specs)
for spec in sample_specs:
# Every sample requires an ID. This may be provided
# by a samplegen config author.
# If no ID is provided, fall back to the region tag.
#
# Ideally the sample author should pick a descriptive, unique ID,
# but this may be impractical and can be error-prone.
spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8]
sample_id = spec.get("id") or spec.get("region_tag") or spec_hash
spec["id"] = sample_id
hash_to_spec = id_to_hash_to_spec[sample_id]
if spec_hash in hash_to_spec:
raise DuplicateSample(
f"Duplicate samplegen spec found: {spec}")
hash_to_spec[spec_hash] = spec
out_dir = "samples/generated_samples"
fpath_to_spec_and_rendered = {}
for hash_to_spec in id_to_hash_to_spec.values():
for spec_hash, spec in hash_to_spec.items():
id_is_unique = len(hash_to_spec) == 1
# The ID is used to generate the file name. It must be globally unique.
if not id_is_unique:
spec["id"] += f"_{spec_hash}"
sample, snippet_metadata = samplegen.generate_sample(
spec, api_schema, sample_template,)
fpath = utils.to_snake_case(spec["id"]) + ".py"
fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = (
spec,
sample,
)
snippet_metadata.file = fpath
snippet_metadata.title = fpath
index.add_snippet(
snippet_index.Snippet(sample, snippet_metadata))
output_files = {
fname: CodeGeneratorResponse.File(
content=formatter.fix_whitespace(sample), name=fname
)
for fname, (_, sample) in fpath_to_spec_and_rendered.items()
}
if index.metadata_index.snippets:
# NOTE(busunkim): Not all fields are yet populated in the snippet metadata.
# Expected filename: snippet_metadata_{apishortname}_{apiversion}.json
snippet_metadata_path = str(pathlib.Path(
out_dir) / f"snippet_metadata_{api_schema.naming.name}_{api_schema.naming.version}.json").lower()
output_files[snippet_metadata_path] = CodeGeneratorResponse.File(
content=formatter.fix_whitespace(index.get_metadata_json()), name=snippet_metadata_path)
return output_files, index
def test_samplegen_id_disambiguation(mock_gmtime, mock_generate_sample, fs):
# These time values are nothing special,
# they just need to be deterministic.
returner = mock.MagicMock()
returner.tm_year = 2112
returner.tm_mon = 6
returner.tm_mday = 1
returner.tm_hour = 13
returner.tm_min = 13
returner.tm_sec = 13
mock_gmtime.return_value = returner
# Note: The first two samples will have the same nominal ID, the first by
# explicit naming and the second by falling back to the region_tag.
# The third has no id of any kind, so the generator is required to make a
# unique ID for it.
fs.create_file(
"samples.yaml",
contents=dedent("""
---
type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto
schema_version: 1.2.0
samples:
- id: squid_sample
region_tag: humboldt_tag
rpc: get_squid_streaming
# Note that this region tag collides with the id of the previous sample.
- region_tag: squid_sample
rpc: get_squid_streaming
# No id or region tag.
- rpc: get_squid_streaming
"""),
)
g = generator.Generator(Options.build("samples=samples.yaml"))
# Need to have the sample template visible to the generator.
g._env.loader = jinja2.DictLoader({"sample.py.j2": ""})
api_schema = make_api(
naming=naming.NewNaming(name="Mollusc", version="v6"))
actual_response = g.get_response(api_schema, opts=Options.build(""))
expected_response = CodeGeneratorResponse(file=[
CodeGeneratorResponse.File(
name="samples/generated_samples/squid_sample_91a465c6.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/generated_samples/squid_sample_55051b38.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/generated_samples/157884ee.py",
content="\n",
),
# TODO(busunkim): Re-enable manifest generation once metadata
# format has been formalized.
# https://docs.google.com/document/d/1ghBam8vMj3xdoe4xfXhzVcOAIwrkbTpkMLgKc9RPD9k/edit#heading=h.sakzausv6hue
# CodeGeneratorResponse.File(
# name="samples/generated_samples/mollusc.v6.python.21120601.131313.manifest.yaml",
# content=dedent(
# """\
# ---
# type: manifest/samples
# schema_version: 3
# python: &python
# environment: python
# bin: python3
# base_path: samples
# invocation: '{bin} {path} @args'
# samples:
# - <<: *python
# sample: squid_sample_91a465c6
# path: '{base_path}/squid_sample_91a465c6.py'
# region_tag: humboldt_tag
# - <<: *python
# sample: squid_sample_55051b38
# path: '{base_path}/squid_sample_55051b38.py'
# region_tag: squid_sample
# - <<: *python
# sample: 157884ee
# path: '{base_path}/157884ee.py'
# """
# ),
# ),
])
expected_response.supported_features |= (
CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL)
assert actual_response == expected_response
def test_dont_generate_in_code_samples(mock_gmtime, mock_generate_sample, fs):
# These time values are nothing special,
# they just need to be deterministic.
returner = mock.MagicMock()
returner.tm_year = 2112
returner.tm_mon = 6
returner.tm_mday = 1
returner.tm_hour = 13
returner.tm_min = 13
returner.tm_sec = 13
mock_gmtime.return_value = returner
config_fpath = "samples.yaml"
fs.create_file(
config_fpath,
contents=dedent("""
type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto
schema_version: 1.2.0
samples:
- id: squid_sample
rpc: IdentifyMollusc
service: Mollusc.v1.Mollusc
sample_type:
- standalone
- incode/SQUID
- id: clam_sample
rpc: IdentifyMollusc
service: Mollusc.v1.Mollusc
sample_type:
- incode/CLAM
- id: whelk_sample
rpc: IdentifyMollusc
service: Mollusc.v1.Mollusc
sample_type:
- standalone
- id: octopus_sample
rpc: IdentifyMollusc
service: Mollusc.v1.Mollusc
"""),
)
generator = make_generator(f"samples={config_fpath}")
generator._env.loader = jinja2.DictLoader({"sample.py.j2": ""})
api_schema = make_api(
make_proto(
descriptor_pb2.FileDescriptorProto(
name="mollusc.proto",
package="Mollusc.v1",
service=[
descriptor_pb2.ServiceDescriptorProto(name="Mollusc")
],
), ),
naming=naming.NewNaming(name="Mollusc", version="v6"),
)
# Note that we do NOT expect a clam sample.
# There are four tests going on:
# 1) Just an explicit standalone sample type.
# 2) Multiple sample types, one of which is standalone.
# 3) Explicit sample types but NO standalone sample type.
# 4) Implicit standalone sample type.
expected = CodeGeneratorResponse(file=[
CodeGeneratorResponse.File(
name="samples/generated_samples/squid_sample.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/generated_samples/whelk_sample.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/generated_samples/octopus_sample.py",
content="\n",
),
# TODO(busunkim): Re-enable manifest generation once metadata
# format has been formalized.
# https://docs.google.com/document/d/1ghBam8vMj3xdoe4xfXhzVcOAIwrkbTpkMLgKc9RPD9k/edit#heading=h.sakzausv6hue
# CodeGeneratorResponse.File(
# name="samples/generated_samples/mollusc.v6.python.21120601.131313.manifest.yaml",
# content=dedent(
# """ ---
# type: manifest/samples
# schema_version: 3
# python: &python
# environment: python
# bin: python3
# base_path: samples
# invocation: \'{bin} {path} @args\'
# samples:
# - <<: *python
# sample: squid_sample
# path: \'{base_path}/squid_sample.py\'
# - <<: *python
# sample: whelk_sample
# path: \'{base_path}/whelk_sample.py\'
# - <<: *python
# sample: octopus_sample
# path: \'{base_path}/octopus_sample.py\'
# """
# ),
# ),
])
expected.supported_features |= CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL
actual = generator.get_response(api_schema=api_schema,
opts=Options.build(""))
assert actual == expected
def _generate_samples_and_manifest(self, api_schema: api.API,
sample_template: jinja2.Template, *,
opts: Options) -> Dict:
"""Generate samples and samplegen manifest for the API.
Arguments:
api_schema (api.API): The schema for the API to which the samples belong.
sample_template (jinja2.Template): The template to use to generate samples.
opts (Options): Additional generator options.
Returns:
Dict[str, CodeGeneratorResponse.File]: A dict mapping filepath to rendered file.
"""
# The two-layer data structure lets us do two things:
# * detect duplicate samples, which is an error
# * detect distinct samples with the same ID, which are disambiguated
id_to_hash_to_spec: DefaultDict[str, Dict[str,
Any]] = defaultdict(dict)
# Autogenerated sample specs
autogen_specs: typing.List[typing.Dict[str, Any]] = []
if opts.autogen_snippets:
autogen_specs = list(
samplegen.generate_sample_specs(api_schema, opts=opts))
# Also process any handwritten sample specs
handwritten_specs = samplegen.parse_handwritten_specs(
self._sample_configs)
sample_specs = autogen_specs + list(handwritten_specs)
for spec in sample_specs:
# Every sample requires an ID. This may be provided
# by a samplegen config author.
# If no ID is provided, fall back to the region tag.
#
# Ideally the sample author should pick a descriptive, unique ID,
# but this may be impractical and can be error-prone.
spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8]
sample_id = spec.get("id") or spec.get("region_tag") or spec_hash
spec["id"] = sample_id
hash_to_spec = id_to_hash_to_spec[sample_id]
if spec_hash in hash_to_spec:
raise DuplicateSample(
f"Duplicate samplegen spec found: {spec}")
hash_to_spec[spec_hash] = spec
out_dir = "samples/generated_samples"
fpath_to_spec_and_rendered = {}
for hash_to_spec in id_to_hash_to_spec.values():
for spec_hash, spec in hash_to_spec.items():
id_is_unique = len(hash_to_spec) == 1
# The ID is used to generate the file name. It must be globally unique.
if not id_is_unique:
spec["id"] += f"_{spec_hash}"
sample = samplegen.generate_sample(
spec,
api_schema,
sample_template,
)
fpath = utils.to_snake_case(spec["id"]) + ".py"
fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = (
spec,
sample,
)
output_files = {
fname: CodeGeneratorResponse.File(
content=formatter.fix_whitespace(sample), name=fname)
for fname, (_, sample) in fpath_to_spec_and_rendered.items()
}
# TODO(busunkim): Re-enable manifest generation once metadata
# format has been formalized.
# https://docs.google.com/document/d/1ghBam8vMj3xdoe4xfXhzVcOAIwrkbTpkMLgKc9RPD9k/edit#heading=h.sakzausv6hue
#
# if output_files:
# manifest_fname, manifest_doc = manifest.generate(
# (
# (fname, spec)
# for fname, (spec, _) in fpath_to_spec_and_rendered.items()
# ),
# api_schema,
# )
# manifest_fname = os.path.join(out_dir, manifest_fname)
# output_files[manifest_fname] = CodeGeneratorResponse.File(
# content=manifest_doc.render(), name=manifest_fname
# )
return output_files
def test_samplegen_config_to_output_files(
mock_gmtime,
mock_generate_sample,
fs,
):
# These time values are nothing special,
# they just need to be deterministic.
returner = mock.MagicMock()
returner.tm_year = 2112
returner.tm_mon = 6
returner.tm_mday = 1
returner.tm_hour = 13
returner.tm_min = 13
returner.tm_sec = 13
mock_gmtime.return_value = returner
fs.create_file(
"samples.yaml",
contents=dedent("""
---
type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto
schema_version: 1.2.0
samples:
- id: squid_sample
region_tag: humboldt_tag
rpc: get_squid_streaming
- region_tag: clam_sample
rpc: get_clam
"""),
)
g = generator.Generator(Options.build("samples=samples.yaml", ))
# Need to have the sample template visible to the generator.
g._env.loader = jinja2.DictLoader({"sample.py.j2": ""})
api_schema = make_api(
naming=naming.NewNaming(name="Mollusc", version="v6"))
actual_response = g.get_response(api_schema, opts=Options.build(""))
expected_response = CodeGeneratorResponse(file=[
CodeGeneratorResponse.File(
name="samples/generated_samples/squid_sample.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/generated_samples/clam_sample.py",
content="\n",
),
# TODO(busunkim): Re-enable manifest generation once metadata
# format has been formalized.
# https://docs.google.com/document/d/1ghBam8vMj3xdoe4xfXhzVcOAIwrkbTpkMLgKc9RPD9k/edit#heading=h.sakzausv6hue
# CodeGeneratorResponse.File(
# name="samples/generated_samples/mollusc.v6.python.21120601.131313.manifest.yaml",
# content=dedent(
# """\
# ---
# type: manifest/samples
# schema_version: 3
# python: &python
# environment: python
# bin: python3
# base_path: samples
# invocation: '{bin} {path} @args'
# samples:
# - <<: *python
# sample: squid_sample
# path: '{base_path}/squid_sample.py'
# region_tag: humboldt_tag
# - <<: *python
# sample: clam_sample
# path: '{base_path}/clam_sample.py'
# region_tag: clam_sample
# """
# ),
# ),
])
expected_response.supported_features |= (
CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL)
assert actual_response == expected_response
def test_samplegen_config_to_output_files(mock_gmtime, fs):
# These time values are nothing special,
# they just need to be deterministic.
returner = mock.MagicMock()
returner.tm_year = 2112
returner.tm_mon = 6
returner.tm_mday = 1
returner.tm_hour = 13
returner.tm_min = 13
returner.tm_sec = 13
mock_gmtime.return_value = returner
fs.create_file(
"samples.yaml",
contents=dedent("""
---
type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto
schema_version: 1.2.0
samples:
- id: squid_sample
region_tag: humboldt_tag
service: Mollusc.v1.Mollusc
rpc: GetSquidStreaming
- region_tag: clam_sample
service: Mollusc.v1.Mollusc
rpc: GetClam
"""),
)
g = generator.Generator(Options.build("samples=samples.yaml", ))
# Need to have the sample template visible to the generator.
g._env.loader = jinja2.DictLoader({"sample.py.j2": ""})
api_schema = DummyApiSchema(
services={
"Mollusc":
DummyService(
name="Mollusc",
methods={
# For this test the generator only cares about the dictionary keys
"GetSquidStreaming": DummyMethod(),
"GetClam": DummyMethod(),
},
)
},
naming=DummyNaming(
name="mollusc",
version="v1",
warehouse_package_name="mollusc-cephalopod-teuthida-",
versioned_module_name="teuthida_v1",
module_namespace="mollusc.cephalopod",
proto_package="google.mollusca"),
)
with mock.patch("gapic.samplegen.samplegen.generate_sample",
side_effect=mock_generate_sample):
actual_response = g.get_response(
api_schema, opts=Options.build("autogen-snippets=False"))
expected_snippet_index_json = {
"clientLibrary": {
"apis": [{
"id": "google.mollusca",
"version": "v1"
}],
"language": "PYTHON",
"name": "mollusc-cephalopod-teuthida-"
},
"snippets": [{
"clientMethod": {
"method": {
"service": {
"shortName": "Mollusc"
},
"shortName": "GetSquidStreaming"
}
},
"file":
"squid_sample.py",
"segments": [{
"type": "FULL"
}, {
"type": "SHORT"
}, {
"type": "CLIENT_INITIALIZATION"
}, {
"type": "REQUEST_INITIALIZATION"
}, {
"type": "REQUEST_EXECUTION"
}, {
"type": "RESPONSE_HANDLING"
}],
"title":
"squid_sample.py"
}, {
"clientMethod": {
"method": {
"service": {
"shortName": "Mollusc"
},
"shortName": "GetClam"
}
},
"file":
"clam_sample.py",
"segments": [{
"type": "FULL"
}, {
"type": "SHORT"
}, {
"type": "CLIENT_INITIALIZATION"
}, {
"type": "REQUEST_INITIALIZATION"
}, {
"type": "REQUEST_EXECUTION"
}, {
"type": "RESPONSE_HANDLING"
}],
"title":
"clam_sample.py"
}]
}
assert actual_response.supported_features == CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL
assert len(actual_response.file) == 3
assert actual_response.file[0] == CodeGeneratorResponse.File(
name="samples/generated_samples/squid_sample.py",
content="\n",
)
assert actual_response.file[1] == CodeGeneratorResponse.File(
name="samples/generated_samples/clam_sample.py",
content="\n",
)
assert actual_response.file[
2].name == "samples/generated_samples/snippet_metadata_mollusc_v1.json"
assert json.loads(
actual_response.file[2].content) == expected_snippet_index_json
def test_samplegen_id_disambiguation(mock_gmtime, fs):
# These time values are nothing special,
# they just need to be deterministic.
returner = mock.MagicMock()
returner.tm_year = 2112
returner.tm_mon = 6
returner.tm_mday = 1
returner.tm_hour = 13
returner.tm_min = 13
returner.tm_sec = 13
mock_gmtime.return_value = returner
# Note: The first two samples will have the same nominal ID, the first by
# explicit naming and the second by falling back to the region_tag.
# The third has no id of any kind, so the generator is required to make a
# unique ID for it.
fs.create_file(
"samples.yaml",
contents=dedent("""
---
type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto
schema_version: 1.2.0
samples:
- id: squid_sample
region_tag: humboldt_tag
rpc: GetSquidStreaming
service: Mollusc.v1.Mollusc
# Note that this region tag collides with the id of the previous sample.
- region_tag: squid_sample
rpc: GetSquidStreaming
service: Mollusc.v1.Mollusc
# No id or region tag.
- rpc: GetSquidStreaming
service: Mollusc.v1.Mollusc
"""),
)
g = generator.Generator(Options.build("samples=samples.yaml"))
# Need to have the sample template visible to the generator.
g._env.loader = jinja2.DictLoader({"sample.py.j2": ""})
api_schema = DummyApiSchema(
services={
"Mollusc":
DummyService(
name="Mollusc",
methods={
# The generator only cares about the dictionary keys
"GetSquidStreaming": DummyMethod(),
"GetClam": DummyMethod(),
},
)
},
naming=DummyNaming(
name="mollusc",
version="v1",
warehouse_package_name="mollusc-cephalopod-teuthida-",
versioned_module_name="teuthida_v1",
module_namespace="mollusc.cephalopod",
proto_package="google.mollusca"),
)
with mock.patch("gapic.samplegen.samplegen.generate_sample",
side_effect=mock_generate_sample):
actual_response = g.get_response(
api_schema, opts=Options.build("autogen-snippets=False"))
expected_snippet_metadata_json = {
"clientLibrary": {
"apis": [{
"id": "google.mollusca",
"version": "v1"
}],
"language": "PYTHON",
"name": "mollusc-cephalopod-teuthida-"
},
"snippets": [{
"clientMethod": {
"method": {
"service": {
"shortName": "Mollusc"
},
"shortName": "GetSquidStreaming"
}
},
"file":
"squid_sample_1cfd0b3d.py",
"segments": [{
"type": "FULL"
}, {
"type": "SHORT"
}, {
"type": "CLIENT_INITIALIZATION"
}, {
"type": "REQUEST_INITIALIZATION"
}, {
"type": "REQUEST_EXECUTION"
}, {
"type": "RESPONSE_HANDLING"
}],
"title":
"squid_sample_1cfd0b3d.py"
}, {
"clientMethod": {
"method": {
"service": {
"shortName": "Mollusc"
},
"shortName": "GetSquidStreaming"
}
},
"file":
"squid_sample_cf4d4fa4.py",
"segments": [{
"type": "FULL"
}, {
"type": "SHORT"
}, {
"type": "CLIENT_INITIALIZATION"
}, {
"type": "REQUEST_INITIALIZATION"
}, {
"type": "REQUEST_EXECUTION"
}, {
"type": "RESPONSE_HANDLING"
}],
"title":
"squid_sample_cf4d4fa4.py"
}, {
"clientMethod": {
"method": {
"service": {
"shortName": "Mollusc"
},
"shortName": "GetSquidStreaming"
}
},
"file":
"7384949e.py",
"segments": [{
"type": "FULL"
}, {
"type": "SHORT"
}, {
"type": "CLIENT_INITIALIZATION"
}, {
"type": "REQUEST_INITIALIZATION"
}, {
"type": "REQUEST_EXECUTION"
}, {
"type": "RESPONSE_HANDLING"
}],
"title":
"7384949e.py"
}]
}
assert actual_response.supported_features == CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL
assert len(actual_response.file) == 4
assert actual_response.file[0] == CodeGeneratorResponse.File(
name="samples/generated_samples/squid_sample_1cfd0b3d.py",
content="\n",
)
assert actual_response.file[1] == CodeGeneratorResponse.File(
name="samples/generated_samples/squid_sample_cf4d4fa4.py",
content="\n",
)
assert actual_response.file[2] == CodeGeneratorResponse.File(
name="samples/generated_samples/7384949e.py",
content="\n",
)
print(actual_response.file[3].content)
assert actual_response.file[
3].name == "samples/generated_samples/snippet_metadata_mollusc_v1.json"
assert json.loads(
actual_response.file[3].content) == expected_snippet_metadata_json
def test_samplegen_config_to_output_files(mock_gmtime, mock_generate_sample,
fs):
# These time values are nothing special,
# they just need to be deterministic.
returner = mock.MagicMock()
returner.tm_year = 2112
returner.tm_mon = 6
returner.tm_mday = 1
returner.tm_hour = 13
returner.tm_min = 13
returner.tm_sec = 13
mock_gmtime.return_value = returner
fs.create_file('samples.yaml',
contents=dedent('''
---
type: com.google.api.codegen.SampleConfigProto
schema_version: 1.2.0
samples:
- id: squid_sample
region_tag: humboldt_tag
rpc: get_squid_streaming
- region_tag: clam_sample
rpc: get_clam
'''))
mock_generate_sample
g = generator.Generator(options.Options.build('samples=samples.yaml', ))
api_schema = make_api(naming=naming.Naming(name='Mollusc', version='v6'))
actual_response = g.get_response(api_schema)
expected_response = CodeGeneratorResponse(file=[
CodeGeneratorResponse.File(
name="samples/squid_sample.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/clam_sample.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/Mollusc.v6.python.21120601.131313.manifest.yaml",
content=dedent("""\
---
type: manifest/samples
schema_version: 3
python: &python
environment: python
bin: python3
base_path: samples
invocation: '{bin} {path} @args'
samples:
- <<: *python
sample: squid_sample
path: '{base_path}/squid_sample.py'
region_tag: humboldt_tag
- <<: *python
sample: clam_sample
path: '{base_path}/clam_sample.py'
region_tag: clam_sample
"""),
)
])
assert actual_response == expected_response
def test_samplegen_config_to_output_files(
mock_gmtime,
mock_generate_sample,
fs,
):
# These time values are nothing special,
# they just need to be deterministic.
returner = mock.MagicMock()
returner.tm_year = 2112
returner.tm_mon = 6
returner.tm_mday = 1
returner.tm_hour = 13
returner.tm_min = 13
returner.tm_sec = 13
mock_gmtime.return_value = returner
fs.create_file(
"samples.yaml",
contents=dedent("""
---
type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto
schema_version: 1.2.0
samples:
- id: squid_sample
region_tag: humboldt_tag
rpc: get_squid_streaming
- region_tag: clam_sample
rpc: get_clam
"""),
)
g = generator.Generator(Options.build("samples=samples.yaml", ))
# Need to have the sample template visible to the generator.
g._env.loader = jinja2.DictLoader({"sample.py.j2": ""})
api_schema = make_api(
naming=naming.NewNaming(name="Mollusc", version="v6"))
actual_response = g.get_response(api_schema, opts=Options.build(""))
expected_response = CodeGeneratorResponse(file=[
CodeGeneratorResponse.File(
name="samples/squid_sample.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/clam_sample.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/mollusc.v6.python.21120601.131313.manifest.yaml",
content=dedent("""\
---
type: manifest/samples
schema_version: 3
python: &python
environment: python
bin: python3
base_path: samples
invocation: '{bin} {path} @args'
samples:
- <<: *python
sample: squid_sample
path: '{base_path}/squid_sample.py'
region_tag: humboldt_tag
- <<: *python
sample: clam_sample
path: '{base_path}/clam_sample.py'
region_tag: clam_sample
"""),
),
])
expected_response.supported_features |= (
CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL)
assert actual_response == expected_response
def _generate_samples_and_manifest(
self,
api_schema: api.API
) -> Dict[str, CodeGeneratorResponse.File]:
"""Generate samples and samplegen manifest for the API.
Arguments:
api_schema (api.API): The schema for the API to which the samples belong.
Returns:
Dict[str, CodeGeneratorResponse.File]: A dict mapping filepath to rendered file.
"""
id_to_samples: DefaultDict[str, List[Any]] = defaultdict(list)
for config_fpath in self._sample_configs:
with open(config_fpath) as f:
configs = yaml.safe_load_all(f.read())
spec_generator = (
spec
for cfg in configs if is_valid_sample_cfg(cfg)
for spec in cfg.get("samples", [])
)
for spec in spec_generator:
# Every sample requires an ID, preferably provided by the
# samplegen config author.
# If no ID is provided, fall back to the region tag.
# If there's no region tag, generate a unique ID.
#
# Ideally the sample author should pick a descriptive, unique ID,
# but this may be impractical and can be error-prone.
sample_id = (spec.get("id")
or spec.get("region_tag")
or sha256(str(spec).encode('utf8')).hexdigest()[:8])
spec["id"] = sample_id
id_to_samples[sample_id].append(spec)
out_dir = "samples"
fpath_to_spec_and_rendered = {}
for samples in id_to_samples.values():
for spec in samples:
id_is_unique = len(samples) == 1
# The ID is used to generate the file name and by sample tester
# to link filenames to invoked samples. It must be globally unique.
if not id_is_unique:
spec_hash = sha256(
str(spec).encode('utf8')).hexdigest()[:8]
spec["id"] += f"_{spec_hash}"
sample = samplegen.generate_sample(spec, self._env, api_schema)
fpath = spec["id"] + ".py"
fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = (spec,
sample)
output_files = {
fname: CodeGeneratorResponse.File(
content=formatter.fix_whitespace(sample),
name=fname
)
for fname, (_, sample) in fpath_to_spec_and_rendered.items()
}
# Only generate a manifest if we generated samples.
if output_files:
manifest_fname, manifest_doc = manifest.generate(
((fname, spec)
for fname, (spec, _) in fpath_to_spec_and_rendered.items()),
api_schema
)
manifest_fname = os.path.join(out_dir, manifest_fname)
output_files[manifest_fname] = CodeGeneratorResponse.File(
content=manifest_doc.render(),
name=manifest_fname
)
return output_files
def test_dont_generate_in_code_samples(mock_gmtime, mock_generate_sample, fs):
# These time values are nothing special,
# they just need to be deterministic.
returner = mock.MagicMock()
returner.tm_year = 2112
returner.tm_mon = 6
returner.tm_mday = 1
returner.tm_hour = 13
returner.tm_min = 13
returner.tm_sec = 13
mock_gmtime.return_value = returner
config_fpath = "samples.yaml"
fs.create_file(config_fpath,
contents=dedent('''
type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto
schema_version: 1.2.0
samples:
- id: squid_sample
rpc: IdentifyMollusc
service: Mollusc.v1.Mollusc
sample_type:
- standalone
- incode/SQUID
- id: clam_sample
rpc: IdentifyMollusc
service: Mollusc.v1.Mollusc
sample_type:
- incode/CLAM
- id: whelk_sample
rpc: IdentifyMollusc
service: Mollusc.v1.Mollusc
sample_type:
- standalone
- id: octopus_sample
rpc: IdentifyMollusc
service: Mollusc.v1.Mollusc
'''))
generator = make_generator(f'samples={config_fpath}')
generator._env.loader = jinja2.DictLoader({'sample.py.j2': ''})
api_schema = make_api(
make_proto(
descriptor_pb2.FileDescriptorProto(
name='mollusc.proto',
package='Mollusc.v1',
service=[
descriptor_pb2.ServiceDescriptorProto(name='Mollusc')
],
), ),
naming=naming.Naming(name='Mollusc', version='v6'),
)
# Note that we do NOT expect a clam sample.
# There are four tests going on:
# 1) Just an explicit standalone sample type.
# 2) Multiple sample types, one of which is standalone.
# 3) Explicit sample types but NO standalone sample type.
# 4) Implicit standalone sample type.
expected = CodeGeneratorResponse(file=[
CodeGeneratorResponse.File(
name="samples/squid_sample.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/whelk_sample.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/octopus_sample.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/mollusc.v6.python.21120601.131313.manifest.yaml",
content=dedent(""" ---
type: manifest/samples
schema_version: 3
python: &python
environment: python
bin: python3
base_path: samples
invocation: \'{bin} {path} @args\'
samples:
- <<: *python
sample: squid_sample
path: \'{base_path}/squid_sample.py\'
- <<: *python
sample: whelk_sample
path: \'{base_path}/whelk_sample.py\'
- <<: *python
sample: octopus_sample
path: \'{base_path}/octopus_sample.py\'
"""))
])
actual = generator.get_response(api_schema=api_schema,
opts=options.Options.build(''))
assert actual == expected
def test_samplegen_id_disambiguation(mock_gmtime, mock_generate_sample, fs):
# These time values are nothing special,
# they just need to be deterministic.
returner = mock.MagicMock()
returner.tm_year = 2112
returner.tm_mon = 6
returner.tm_mday = 1
returner.tm_hour = 13
returner.tm_min = 13
returner.tm_sec = 13
mock_gmtime.return_value = returner
# Note: The first two samples will have the same nominal ID, the first by
# explicit naming and the second by falling back to the region_tag.
# The third has no id of any kind, so the generator is required to make a
# unique ID for it.
fs.create_file('samples.yaml',
contents=dedent('''
---
type: com.google.api.codegen.SampleConfigProto
schema_version: 1.2.0
samples:
- id: squid_sample
region_tag: humboldt_tag
rpc: get_squid_streaming
# Note that this region tag collides with the id of the previous sample.
- region_tag: squid_sample
rpc: get_squid_streaming
# No id or region tag.
- rpc: get_squid_streaming
'''))
g = generator.Generator(options.Options.build('samples=samples.yaml'))
api_schema = make_api(naming=naming.Naming(name='Mollusc', version='v6'))
actual_response = g.get_response(api_schema)
expected_response = CodeGeneratorResponse(file=[
CodeGeneratorResponse.File(
name="samples/squid_sample_91a465c6.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/squid_sample_c8014108.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/157884ee.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/Mollusc.v6.python.21120601.131313.manifest.yaml",
content=dedent("""\
---
type: manifest/samples
schema_version: 3
python: &python
environment: python
bin: python3
base_path: samples
invocation: '{bin} {path} @args'
samples:
- <<: *python
sample: squid_sample_91a465c6
path: '{base_path}/squid_sample_91a465c6.py'
region_tag: humboldt_tag
- <<: *python
sample: squid_sample_c8014108
path: '{base_path}/squid_sample_c8014108.py'
region_tag: squid_sample
- <<: *python
sample: 157884ee
path: '{base_path}/157884ee.py'
""")),
])
assert actual_response == expected_response