def main():
request = CodeGeneratorRequest.FromString(sys.stdin.buffer.read())
files_to_generate = set(request.file_to_generate)
response = CodeGeneratorResponse()
proto_for_entity = dict()
for proto_file in request.proto_file:
package_name = proto_file.package
for named_entity in itertools.chain(proto_file.message_type,
proto_file.enum_type,
proto_file.service,
proto_file.extension):
if package_name:
fully_qualified_name = ".".join(
["", package_name, named_entity.name])
else:
fully_qualified_name = "." + named_entity.name
proto_for_entity[fully_qualified_name] = proto_file.name
for proto_file in request.proto_file:
if proto_file.name in files_to_generate:
out = response.file.add()
out.name = proto_file.name.replace('.proto', "_grpc.py")
out.content = generate_single_proto(proto_file, proto_for_entity)
sys.stdout.buffer.write(response.SerializeToString())
def main(input_file=sys.stdin, output_file=sys.stdout):
"""Parse a CodeGeneratorRequest and return a CodeGeneratorResponse."""
logging.basicConfig(filename='logging.log', level=logging.DEBUG)
# Ensure we are getting a bytestream, and writing to a bytestream.
if hasattr(input_file, 'buffer'):
input_file = input_file.buffer
if hasattr(output_file, 'buffer'):
output_file = output_file.buffer
try:
# Instantiate a parser.
parser = CodeGeneratorParser.from_input_file(input_file)
docs = parser.generate_docs()
except:
logging.exception("Error when generating docs: ")
raise
answer = []
for k, item in docs.items():
answer.append(
CodeGeneratorResponse.File(
name=item.filename,
content=item.content,
))
cgr = CodeGeneratorResponse(file=answer)
output_file.write(cgr.SerializeToString())
def main(input_file=sys.stdin, output_file=sys.stdout):
"""Parse a CodeGeneratorRequest and return a CodeGeneratorResponse."""
# Ensure we are getting a bytestream, and writing to a bytestream.
if hasattr(input_file, 'buffer'):
input_file = input_file.buffer
if hasattr(output_file, 'buffer'):
output_file = output_file.buffer
# Instantiate a parser.
parser = CodeGeneratorParser.from_input_file(input_file)
# Find all the docs and amalgamate them together.
comment_data = {}
for filename, message_structure in parser.find_docs():
comment_data.setdefault(filename, set())
comment_data[filename].add(message_structure)
# Iterate over the data that came back and parse it into a single,
# coherent CodeGeneratorResponse.
answer = []
_BATCH_TOKEN = "CD985272F78311"
meta_docstrings = []
meta_structs = []
for fn, structs in comment_data.items():
for struct in structs:
if meta_docstrings:
meta_docstrings.append("\n%s" % _BATCH_TOKEN)
meta_docstrings.append(struct.get_meta_docstring())
meta_structs.append((fn, struct))
meta_docstring = convert_text("".join(meta_docstrings), 'rst', format='md')
meta_docstrings = meta_docstring.split("%s" % _BATCH_TOKEN)
index = 0
while index < len(meta_structs) and index < len(meta_docstrings):
fn = meta_structs[index][0]
struct = meta_structs[index][1]
answer.append(
CodeGeneratorResponse.File(
name=fn.replace('.proto', '_pb2.py'),
insertion_point='class_scope:%s' % struct.name,
content=',\n__doc__ = """{docstring}""",'.format(
docstring=struct.get_python_docstring(
meta_docstrings[index]), ),
))
index += 1
for fn in _init_files(comment_data.keys()):
answer.append(CodeGeneratorResponse.File(
name=fn,
content='',
))
cgr = CodeGeneratorResponse(file=answer)
output_file.write(cgr.SerializeToString())
def main() -> None:
with os.fdopen(sys.stdin.fileno(), 'rb') as inp:
request = CodeGeneratorRequest.FromString(inp.read())
types_map: Dict[str, str] = {}
for pf in request.proto_file:
for mt in pf.message_type:
types_map.update(_type_names(pf, mt))
response = CodeGeneratorResponse()
# See https://github.com/protocolbuffers/protobuf/blob/v3.12.0/docs/implementing_proto3_presence.md # noqa
if hasattr(CodeGeneratorResponse, 'Feature'):
response.supported_features = ( # type: ignore
CodeGeneratorResponse.FEATURE_PROTO3_OPTIONAL # type: ignore
)
for file_to_generate in request.file_to_generate:
proto_file = _get_proto(request, file_to_generate)
imports = [
_proto2pb2_module_name(dep)
for dep in list(proto_file.dependency) + [file_to_generate]
]
services = []
for service in proto_file.service:
methods = []
for method in service.method:
cardinality = _CARDINALITY[(method.client_streaming,
method.server_streaming)]
methods.append(
Method(
name=method.name,
cardinality=cardinality,
request_type=types_map[method.input_type],
reply_type=types_map[method.output_type],
))
services.append(Service(name=service.name, methods=methods))
file = response.file.add()
module_name = _proto2grpc_module_name(file_to_generate)
file.name = module_name.replace(".", "/") + ".py"
file.content = render(
proto_file=proto_file.name,
package=proto_file.package,
imports=imports,
services=services,
)
with os.fdopen(sys.stdout.fileno(), 'wb') as out:
out.write(response.SerializeToString())
def main(input_file=sys.stdin, output_file=sys.stdout):
request = CodeGeneratorRequest.FromString(input_file.buffer.read())
answer = []
for fname in request.file_to_generate:
answer.append(
CodeGeneratorResponse.File(
name=fname.replace('.proto', '_pb2.py'),
insertion_point='module_scope',
content="# Hello {}, I'm a dummy plugin!".format(fname),
))
cgr = CodeGeneratorResponse(file=answer)
output_file.buffer.write(cgr.SerializeToString())
def get_response(self) -> CodeGeneratorResponse:
"""Return a :class:`~.CodeGeneratorResponse` for this library.
This is a complete response to be written to (usually) stdout, and
thus read by ``protoc``.
Returns:
~.CodeGeneratorResponse: A response describing appropriate
files and contents. See ``plugin.proto``.
"""
output_files = []
# Some templates are rendered once per API client library.
# These are generally boilerplate packaging and metadata files.
output_files += self._render_templates(self._env.loader.api_templates)
# Some templates are rendered once per service (an API may have
# one or more services).
for service in self._api.services.values():
output_files += self._render_templates(
self._env.loader.service_templates,
additional_context={'service': service},
)
# Return the CodeGeneratorResponse output.
return CodeGeneratorResponse(file=output_files)
def _get_file(
self,
template_name: str,
*,
opts: Options,
api_schema: api.API,
**context,
):
"""Render a template to a protobuf plugin File object."""
# Determine the target filename.
fn = self._get_filename(
template_name, api_schema=api_schema, context=context,)
# Render the file contents.
cgr_file = CodeGeneratorResponse.File(
content=formatter.fix_whitespace(
self._env.get_template(template_name).render(
api=api_schema, opts=opts, **context
),
),
name=fn,
)
# Quick check: Do not render empty files.
if utils.empty(cgr_file.content) and not fn.endswith(
("py.typed", "__init__.py")
):
return {}
# Return the filename and content in a length-1 dictionary
# (because we track output files overall in a dictionary).
return {fn: cgr_file}
def get_response(self, api_schema: api.API) -> CodeGeneratorResponse:
"""Return a :class:`~.CodeGeneratorResponse` for this library.
This is a complete response to be written to (usually) stdout, and
thus read by ``protoc``.
Args:
api_schema (~api.API): An API schema object.
Returns:
~.CodeGeneratorResponse: A response describing appropriate
files and contents. See ``plugin.proto``.
"""
output_files = collections.OrderedDict()
# Iterate over each template and add the appropriate output files
# based on that template.
for template_name in self._env.loader.list_templates():
# Sanity check: Skip "private" templates.
filename = template_name.split('/')[-1]
if filename.startswith('_') and filename != '__init__.py.j2':
continue
# Append to the output files dictionary.
output_files.update(self._render_template(template_name,
api_schema=api_schema,
))
# Return the CodeGeneratorResponse output.
return CodeGeneratorResponse(file=[i for i in output_files.values()])
def process(request: plugin.CodeGeneratorRequest,
response: plugin.CodeGeneratorResponse) -> None:
response.supported_features = plugin.CodeGeneratorResponse.FEATURE_PROTO3_OPTIONAL
for proto_file in request.proto_file:
has_service = False
file = plugin.CodeGeneratorResponse.File()
proto_file_name = proto_file.name.split('.')[0]
file.name = "{}.client.ts".format(proto_file_name)
file_content = ""
file_content += f"import * as {proto_file_name} from './{proto_file_name}';\n"
file_content += "import init from '../wasm/pkg/wasm';\n"
file_content += "import * as wasm from '../wasm/pkg/wasm';\n\n"
file_content += "export async function WasmInit() {\n"
file_content += " await init();\n"
file_content += "}\n\n"
for service in proto_file.service:
has_service = True
service_name = service.name
for method in service.method:
# type names pave . as prefix
input_type = method.input_type[1:]
output_type = method.output_type[1:]
snake_method_name = camel_to_snake(method.name)
file_content += f"export function {service_name}{method.name}(input: {input_type}): {output_type} {{\n"
file_content += f" let input_bytes = {input_type}.toBinary(input);\n"
file_content += f" let output_bytes = wasm.{snake_method_name}(input_bytes);\n"
file_content += f" return {output_type}.fromBinary(output_bytes);\n"
file_content += "}\n\n"
if has_service:
file.content = file_content
response.file.append(file)
def main() -> None:
with os.fdopen(sys.stdin.fileno(), 'rb') as inp:
request = CodeGeneratorRequest.FromString(inp.read())
types_map: Dict[str, str] = {}
for pf in request.proto_file:
for mt in pf.message_type:
types_map.update(_type_names(pf, mt))
response = CodeGeneratorResponse()
for file_to_generate in request.file_to_generate:
proto_file = _get_proto(request, file_to_generate)
imports = [
_proto2pb2_module_name(dep)
for dep in list(proto_file.dependency) + [file_to_generate]
]
services = []
for service in proto_file.service:
methods = []
for method in service.method:
cardinality = _CARDINALITY[(method.client_streaming,
method.server_streaming)]
methods.append(
Method(
name=method.name,
cardinality=cardinality,
request_type=types_map[method.input_type],
reply_type=types_map[method.output_type],
))
services.append(Service(name=service.name, methods=methods))
file = response.file.add()
module_name = _proto2grpc_module_name(file_to_generate)
file.name = module_name.replace(".", "/") + ".py"
file.content = render(
proto_file=proto_file.name,
package=proto_file.package,
imports=imports,
services=services,
)
with os.fdopen(sys.stdout.fileno(), 'wb') as out:
out.write(response.SerializeToString())
def get_response(self, api_schema: api.API,
opts: Options) -> CodeGeneratorResponse:
"""Return a :class:`~.CodeGeneratorResponse` for this library.
This is a complete response to be written to (usually) stdout, and
thus read by ``protoc``.
Args:
api_schema (~api.API): An API schema object.
opts (~.options.Options): An options instance.
Returns:
~.CodeGeneratorResponse: A response describing appropriate
files and contents. See ``plugin.proto``.
"""
output_files: Dict[str, CodeGeneratorResponse.File] = OrderedDict()
sample_templates, client_templates = utils.partition(
lambda fname: os.path.basename(fname) == samplegen.
DEFAULT_TEMPLATE_NAME,
self._env.loader.list_templates(), # type: ignore
)
# We generate code snippets *before* the library code so snippets
# can be inserted into method docstrings.
snippet_idx = snippet_index.SnippetIndex(api_schema)
if sample_templates:
sample_output, snippet_idx = self._generate_samples_and_manifest(
api_schema,
snippet_idx,
self._env.get_template(sample_templates[0]),
opts=opts,
)
output_files.update(sample_output)
# Iterate over each template and add the appropriate output files
# based on that template.
# Sample templates work differently: there's (usually) only one,
# and instead of iterating over it/them, we iterate over samples
# and plug those into the template.
for template_name in client_templates:
# Quick check: Skip "private" templates.
filename = template_name.split("/")[-1]
if filename.startswith("_") and filename != "__init__.py.j2":
continue
# Append to the output files dictionary.
output_files.update(
self._render_template(template_name,
api_schema=api_schema,
opts=opts,
snippet_index=snippet_idx))
# Return the CodeGeneratorResponse output.
res = CodeGeneratorResponse(file=[i for i in output_files.values()
]) # type: ignore
res.supported_features |= CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL # type: ignore
return res
def main() -> None:
with os.fdopen(sys.stdin.fileno(), 'rb') as inp:
request = CodeGeneratorRequest.FromString(inp.read())
types_map = {
_type_name(pf, mt): '.'.join((_proto2py(pf.name), mt.name))
for pf in request.proto_file for mt in pf.message_type
}
response = CodeGeneratorResponse()
for file_to_generate in request.file_to_generate:
proto_file = _get_proto(request, file_to_generate)
imports = [
_proto2py(dep)
for dep in list(proto_file.dependency) + [file_to_generate]
]
services = []
for service in proto_file.service:
methods = []
for method in service.method:
cardinality = _CARDINALITY[(method.client_streaming,
method.server_streaming)]
methods.append(
Method(
name=method.name,
cardinality=cardinality,
request_type=types_map[method.input_type],
reply_type=types_map[method.output_type],
))
services.append(Service(name=service.name, methods=methods))
file = response.file.add()
file.name = file_to_generate.replace('.proto', SUFFIX)
file.content = render(
proto_file=proto_file.name,
package=proto_file.package,
imports=imports,
services=services,
)
with os.fdopen(sys.stdout.fileno(), 'wb') as out:
out.write(response.SerializeToString())
def get_response(
self,
api_schema: api.API,
opts: options.Options
) -> CodeGeneratorResponse:
"""Return a :class:`~.CodeGeneratorResponse` for this library.
This is a complete response to be written to (usually) stdout, and
thus read by ``protoc``.
Args:
api_schema (~api.API): An API schema object.
opts (~.options.Options): An options instance.
Returns:
~.CodeGeneratorResponse: A response describing appropriate
files and contents. See ``plugin.proto``.
"""
output_files: Dict[str, CodeGeneratorResponse.File] = OrderedDict()
sample_templates, client_templates = utils.partition(
lambda fname: os.path.basename(
fname) == samplegen.DEFAULT_TEMPLATE_NAME,
self._env.loader.list_templates())
# Iterate over each template and add the appropriate output files
# based on that template.
# Sample templates work differently: there's (usually) only one,
# and instead of iterating over it/them, we iterate over samples
# and plug those into the template.
for template_name in client_templates:
# Sanity check: Skip "private" templates.
filename = template_name.split('/')[-1]
if filename.startswith('_') and filename != '__init__.py.j2':
continue
# Append to the output files dictionary.
output_files.update(
self._render_template(
template_name,
api_schema=api_schema,
opts=opts
)
)
output_files.update(self._generate_samples_and_manifest(
api_schema,
self._env.get_template(sample_templates[0]),
))
# Return the CodeGeneratorResponse output.
return CodeGeneratorResponse(file=[i for i in output_files.values()])
def test_get_response_ignore_gapic_metadata():
g = make_generator()
with mock.patch.object(jinja2.FileSystemLoader, "list_templates") as lt:
lt.return_value = ["gapic/gapic_metadata.json.j2"]
with mock.patch.object(jinja2.Environment, "get_template") as gt:
gt.return_value = jinja2.Template(
"This is not something we want to see")
res = g.get_response(
api_schema=make_api(),
opts=Options.build(""),
)
# We don't expect any files because opts.metadata is not set.
assert res.file == CodeGeneratorResponse().file
def _get_file(self, template_name: str, *,
api_schema=api.API,
**context: Mapping):
"""Render a template to a protobuf plugin File object."""
fn = self._get_filename(template_name,
api_schema=api_schema,
context=context,
)
return {fn: CodeGeneratorResponse.File(
content=formatter.fix_whitespace(
self._env.get_template(template_name).render(
api=api_schema,
**context
),
),
name=fn,
)}
def _render_templates(
self,
templates: Iterable[str],
*,
additional_context: Mapping[str, Any] = None,
) -> Sequence[CodeGeneratorResponse.File]:
"""Render the requested templates.
Args:
templates (Iterable[str]): The set of templates to be rendered.
It is expected that these come from the methods on
:class:`~.loader.TemplateLoader`, and they should be
able to be set to the :meth:`jinja2.Environment.get_template`
method.
additional_context (Mapping[str, Any]): Additional variables
to be sent to the templates. The ``api`` variable
is always available.
Returns:
Sequence[~.CodeGeneratorResponse.File]: A sequence of File
objects for inclusion in the final response.
"""
answer = []
additional_context = additional_context or {}
# Iterate over the provided templates and generate a File object
# for each.
for template_name in templates:
# Generate the File object.
answer.append(
CodeGeneratorResponse.File(
content=formatter.fix_whitespace(
self._env.get_template(template_name).render(
api=self._api, **additional_context), ),
name=self._get_output_filename(
template_name,
context=additional_context,
),
))
# Done; return the File objects based on these templates.
return answer
def process(request: CodeGeneratorRequest) -> CodeGeneratorResponse:
params = {}
for pair in request.parameter.split(","):
key, _, value = pair.partition("=")
params[key] = value
if "runtime" not in params:
raise ConfigurationError("Runtime parameter is not specified")
runtime = params["runtime"]
if runtime not in RUNTIMES:
raise ConfigurationError(f"Unknown runtime: {runtime}")
renderer = RUNTIMES[runtime]
files_to_generate = set(request.file_to_generate)
response = CodeGeneratorResponse()
for pf in request.proto_file:
if pf.name in files_to_generate:
process_file(renderer, pf, response)
return response
def test_dont_generate_in_code_samples(mock_gmtime, mock_generate_sample, fs):
# These time values are nothing special,
# they just need to be deterministic.
returner = mock.MagicMock()
returner.tm_year = 2112
returner.tm_mon = 6
returner.tm_mday = 1
returner.tm_hour = 13
returner.tm_min = 13
returner.tm_sec = 13
mock_gmtime.return_value = returner
config_fpath = "samples.yaml"
fs.create_file(config_fpath,
contents=dedent('''
type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto
schema_version: 1.2.0
samples:
- id: squid_sample
rpc: IdentifyMollusc
service: Mollusc.v1.Mollusc
sample_type:
- standalone
- incode/SQUID
- id: clam_sample
rpc: IdentifyMollusc
service: Mollusc.v1.Mollusc
sample_type:
- incode/CLAM
- id: whelk_sample
rpc: IdentifyMollusc
service: Mollusc.v1.Mollusc
sample_type:
- standalone
- id: octopus_sample
rpc: IdentifyMollusc
service: Mollusc.v1.Mollusc
'''))
generator = make_generator(f'samples={config_fpath}')
generator._env.loader = jinja2.DictLoader({'sample.py.j2': ''})
api_schema = make_api(
make_proto(
descriptor_pb2.FileDescriptorProto(
name='mollusc.proto',
package='Mollusc.v1',
service=[
descriptor_pb2.ServiceDescriptorProto(name='Mollusc')
],
), ),
naming=naming.Naming(name='Mollusc', version='v6'),
)
# Note that we do NOT expect a clam sample.
# There are four tests going on:
# 1) Just an explicit standalone sample type.
# 2) Multiple sample types, one of which is standalone.
# 3) Explicit sample types but NO standalone sample type.
# 4) Implicit standalone sample type.
expected = CodeGeneratorResponse(file=[
CodeGeneratorResponse.File(
name="samples/squid_sample.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/whelk_sample.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/octopus_sample.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/mollusc.v6.python.21120601.131313.manifest.yaml",
content=dedent(""" ---
type: manifest/samples
schema_version: 3
python: &python
environment: python
bin: python3
base_path: samples
invocation: \'{bin} {path} @args\'
samples:
- <<: *python
sample: squid_sample
path: \'{base_path}/squid_sample.py\'
- <<: *python
sample: whelk_sample
path: \'{base_path}/whelk_sample.py\'
- <<: *python
sample: octopus_sample
path: \'{base_path}/octopus_sample.py\'
"""))
])
actual = generator.get_response(api_schema=api_schema,
opts=options.Options.build(''))
assert actual == expected
def test_samplegen_config_to_output_files(
mock_gmtime,
mock_generate_sample,
fs,
):
# These time values are nothing special,
# they just need to be deterministic.
returner = mock.MagicMock()
returner.tm_year = 2112
returner.tm_mon = 6
returner.tm_mday = 1
returner.tm_hour = 13
returner.tm_min = 13
returner.tm_sec = 13
mock_gmtime.return_value = returner
fs.create_file(
"samples.yaml",
contents=dedent("""
---
type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto
schema_version: 1.2.0
samples:
- id: squid_sample
region_tag: humboldt_tag
rpc: get_squid_streaming
- region_tag: clam_sample
rpc: get_clam
"""),
)
g = generator.Generator(Options.build("samples=samples.yaml", ))
# Need to have the sample template visible to the generator.
g._env.loader = jinja2.DictLoader({"sample.py.j2": ""})
api_schema = make_api(
naming=naming.NewNaming(name="Mollusc", version="v6"))
actual_response = g.get_response(api_schema, opts=Options.build(""))
expected_response = CodeGeneratorResponse(file=[
CodeGeneratorResponse.File(
name="samples/generated_samples/squid_sample.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/generated_samples/clam_sample.py",
content="\n",
),
# TODO(busunkim): Re-enable manifest generation once metadata
# format has been formalized.
# https://docs.google.com/document/d/1ghBam8vMj3xdoe4xfXhzVcOAIwrkbTpkMLgKc9RPD9k/edit#heading=h.sakzausv6hue
# CodeGeneratorResponse.File(
# name="samples/generated_samples/mollusc.v6.python.21120601.131313.manifest.yaml",
# content=dedent(
# """\
# ---
# type: manifest/samples
# schema_version: 3
# python: &python
# environment: python
# bin: python3
# base_path: samples
# invocation: '{bin} {path} @args'
# samples:
# - <<: *python
# sample: squid_sample
# path: '{base_path}/squid_sample.py'
# region_tag: humboldt_tag
# - <<: *python
# sample: clam_sample
# path: '{base_path}/clam_sample.py'
# region_tag: clam_sample
# """
# ),
# ),
])
expected_response.supported_features |= (
CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL)
assert actual_response == expected_response
def _generate_samples_and_manifest(self, api_schema: api.API,
sample_template: jinja2.Template, *,
opts: Options) -> Dict:
"""Generate samples and samplegen manifest for the API.
Arguments:
api_schema (api.API): The schema for the API to which the samples belong.
sample_template (jinja2.Template): The template to use to generate samples.
opts (Options): Additional generator options.
Returns:
Dict[str, CodeGeneratorResponse.File]: A dict mapping filepath to rendered file.
"""
# The two-layer data structure lets us do two things:
# * detect duplicate samples, which is an error
# * detect distinct samples with the same ID, which are disambiguated
id_to_hash_to_spec: DefaultDict[str, Dict[str,
Any]] = defaultdict(dict)
# Autogenerated sample specs
autogen_specs: typing.List[typing.Dict[str, Any]] = []
if opts.autogen_snippets:
autogen_specs = list(
samplegen.generate_sample_specs(api_schema, opts=opts))
# Also process any handwritten sample specs
handwritten_specs = samplegen.parse_handwritten_specs(
self._sample_configs)
sample_specs = autogen_specs + list(handwritten_specs)
for spec in sample_specs:
# Every sample requires an ID. This may be provided
# by a samplegen config author.
# If no ID is provided, fall back to the region tag.
#
# Ideally the sample author should pick a descriptive, unique ID,
# but this may be impractical and can be error-prone.
spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8]
sample_id = spec.get("id") or spec.get("region_tag") or spec_hash
spec["id"] = sample_id
hash_to_spec = id_to_hash_to_spec[sample_id]
if spec_hash in hash_to_spec:
raise DuplicateSample(
f"Duplicate samplegen spec found: {spec}")
hash_to_spec[spec_hash] = spec
out_dir = "samples/generated_samples"
fpath_to_spec_and_rendered = {}
for hash_to_spec in id_to_hash_to_spec.values():
for spec_hash, spec in hash_to_spec.items():
id_is_unique = len(hash_to_spec) == 1
# The ID is used to generate the file name. It must be globally unique.
if not id_is_unique:
spec["id"] += f"_{spec_hash}"
sample = samplegen.generate_sample(
spec,
api_schema,
sample_template,
)
fpath = utils.to_snake_case(spec["id"]) + ".py"
fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = (
spec,
sample,
)
output_files = {
fname: CodeGeneratorResponse.File(
content=formatter.fix_whitespace(sample), name=fname)
for fname, (_, sample) in fpath_to_spec_and_rendered.items()
}
# TODO(busunkim): Re-enable manifest generation once metadata
# format has been formalized.
# https://docs.google.com/document/d/1ghBam8vMj3xdoe4xfXhzVcOAIwrkbTpkMLgKc9RPD9k/edit#heading=h.sakzausv6hue
#
# if output_files:
# manifest_fname, manifest_doc = manifest.generate(
# (
# (fname, spec)
# for fname, (spec, _) in fpath_to_spec_and_rendered.items()
# ),
# api_schema,
# )
# manifest_fname = os.path.join(out_dir, manifest_fname)
# output_files[manifest_fname] = CodeGeneratorResponse.File(
# content=manifest_doc.render(), name=manifest_fname
# )
return output_files
#!/usr/bin/env python
# encoding: utf-8
import sys
from google.protobuf.compiler.plugin_pb2 import CodeGeneratorRequest, CodeGeneratorResponse
import google.protobuf.descriptor_pb2 as descriptor
FieldD = descriptor.FieldDescriptorProto
request = CodeGeneratorRequest()
request.ParseFromString(sys.stdin.read())
response = CodeGeneratorResponse()
for file in request.proto_file:
out = ''
out += str(FieldD)
out += """package io.devision.gen;\n\n"""
out += """import java.util.Date;\n"""
out += """import javax.persistence.Column;\n"""
out += """import javax.persistence.Entity;\n"""
out += """import javax.persistence.Id;\n"""
out += """import javax.persistence.Table;\n"""
# for enum in file.enum_type:
# out += 'enum ' + enum.name + '\n'
# for value in enum.value:
# out += '\t' + value.name + '\n'
type_name = None
for message in file.message_type:
type_name = message.name
if type_name == 'Column':
continue
def test_samplegen_id_disambiguation(mock_gmtime, mock_generate_sample, fs):
# These time values are nothing special,
# they just need to be deterministic.
returner = mock.MagicMock()
returner.tm_year = 2112
returner.tm_mon = 6
returner.tm_mday = 1
returner.tm_hour = 13
returner.tm_min = 13
returner.tm_sec = 13
mock_gmtime.return_value = returner
# Note: The first two samples will have the same nominal ID, the first by
# explicit naming and the second by falling back to the region_tag.
# The third has no id of any kind, so the generator is required to make a
# unique ID for it.
fs.create_file(
"samples.yaml",
contents=dedent("""
---
type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto
schema_version: 1.2.0
samples:
- id: squid_sample
region_tag: humboldt_tag
rpc: get_squid_streaming
# Note that this region tag collides with the id of the previous sample.
- region_tag: squid_sample
rpc: get_squid_streaming
# No id or region tag.
- rpc: get_squid_streaming
"""),
)
g = generator.Generator(Options.build("samples=samples.yaml"))
# Need to have the sample template visible to the generator.
g._env.loader = jinja2.DictLoader({"sample.py.j2": ""})
api_schema = make_api(
naming=naming.NewNaming(name="Mollusc", version="v6"))
actual_response = g.get_response(api_schema, opts=Options.build(""))
expected_response = CodeGeneratorResponse(file=[
CodeGeneratorResponse.File(
name="samples/generated_samples/squid_sample_91a465c6.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/generated_samples/squid_sample_55051b38.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/generated_samples/157884ee.py",
content="\n",
),
# TODO(busunkim): Re-enable manifest generation once metadata
# format has been formalized.
# https://docs.google.com/document/d/1ghBam8vMj3xdoe4xfXhzVcOAIwrkbTpkMLgKc9RPD9k/edit#heading=h.sakzausv6hue
# CodeGeneratorResponse.File(
# name="samples/generated_samples/mollusc.v6.python.21120601.131313.manifest.yaml",
# content=dedent(
# """\
# ---
# type: manifest/samples
# schema_version: 3
# python: &python
# environment: python
# bin: python3
# base_path: samples
# invocation: '{bin} {path} @args'
# samples:
# - <<: *python
# sample: squid_sample_91a465c6
# path: '{base_path}/squid_sample_91a465c6.py'
# region_tag: humboldt_tag
# - <<: *python
# sample: squid_sample_55051b38
# path: '{base_path}/squid_sample_55051b38.py'
# region_tag: squid_sample
# - <<: *python
# sample: 157884ee
# path: '{base_path}/157884ee.py'
# """
# ),
# ),
])
expected_response.supported_features |= (
CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL)
assert actual_response == expected_response
def test_samplegen_id_disambiguation(mock_gmtime, fs):
# These time values are nothing special,
# they just need to be deterministic.
returner = mock.MagicMock()
returner.tm_year = 2112
returner.tm_mon = 6
returner.tm_mday = 1
returner.tm_hour = 13
returner.tm_min = 13
returner.tm_sec = 13
mock_gmtime.return_value = returner
# Note: The first two samples will have the same nominal ID, the first by
# explicit naming and the second by falling back to the region_tag.
# The third has no id of any kind, so the generator is required to make a
# unique ID for it.
fs.create_file(
"samples.yaml",
contents=dedent("""
---
type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto
schema_version: 1.2.0
samples:
- id: squid_sample
region_tag: humboldt_tag
rpc: GetSquidStreaming
service: Mollusc.v1.Mollusc
# Note that this region tag collides with the id of the previous sample.
- region_tag: squid_sample
rpc: GetSquidStreaming
service: Mollusc.v1.Mollusc
# No id or region tag.
- rpc: GetSquidStreaming
service: Mollusc.v1.Mollusc
"""),
)
g = generator.Generator(Options.build("samples=samples.yaml"))
# Need to have the sample template visible to the generator.
g._env.loader = jinja2.DictLoader({"sample.py.j2": ""})
api_schema = DummyApiSchema(
services={
"Mollusc":
DummyService(
name="Mollusc",
methods={
# The generator only cares about the dictionary keys
"GetSquidStreaming": DummyMethod(),
"GetClam": DummyMethod(),
},
)
},
naming=DummyNaming(
name="mollusc",
version="v1",
warehouse_package_name="mollusc-cephalopod-teuthida-",
versioned_module_name="teuthida_v1",
module_namespace="mollusc.cephalopod",
proto_package="google.mollusca"),
)
with mock.patch("gapic.samplegen.samplegen.generate_sample",
side_effect=mock_generate_sample):
actual_response = g.get_response(
api_schema, opts=Options.build("autogen-snippets=False"))
expected_snippet_metadata_json = {
"clientLibrary": {
"apis": [{
"id": "google.mollusca",
"version": "v1"
}],
"language": "PYTHON",
"name": "mollusc-cephalopod-teuthida-"
},
"snippets": [{
"clientMethod": {
"method": {
"service": {
"shortName": "Mollusc"
},
"shortName": "GetSquidStreaming"
}
},
"file":
"squid_sample_1cfd0b3d.py",
"segments": [{
"type": "FULL"
}, {
"type": "SHORT"
}, {
"type": "CLIENT_INITIALIZATION"
}, {
"type": "REQUEST_INITIALIZATION"
}, {
"type": "REQUEST_EXECUTION"
}, {
"type": "RESPONSE_HANDLING"
}],
"title":
"squid_sample_1cfd0b3d.py"
}, {
"clientMethod": {
"method": {
"service": {
"shortName": "Mollusc"
},
"shortName": "GetSquidStreaming"
}
},
"file":
"squid_sample_cf4d4fa4.py",
"segments": [{
"type": "FULL"
}, {
"type": "SHORT"
}, {
"type": "CLIENT_INITIALIZATION"
}, {
"type": "REQUEST_INITIALIZATION"
}, {
"type": "REQUEST_EXECUTION"
}, {
"type": "RESPONSE_HANDLING"
}],
"title":
"squid_sample_cf4d4fa4.py"
}, {
"clientMethod": {
"method": {
"service": {
"shortName": "Mollusc"
},
"shortName": "GetSquidStreaming"
}
},
"file":
"7384949e.py",
"segments": [{
"type": "FULL"
}, {
"type": "SHORT"
}, {
"type": "CLIENT_INITIALIZATION"
}, {
"type": "REQUEST_INITIALIZATION"
}, {
"type": "REQUEST_EXECUTION"
}, {
"type": "RESPONSE_HANDLING"
}],
"title":
"7384949e.py"
}]
}
assert actual_response.supported_features == CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL
assert len(actual_response.file) == 4
assert actual_response.file[0] == CodeGeneratorResponse.File(
name="samples/generated_samples/squid_sample_1cfd0b3d.py",
content="\n",
)
assert actual_response.file[1] == CodeGeneratorResponse.File(
name="samples/generated_samples/squid_sample_cf4d4fa4.py",
content="\n",
)
assert actual_response.file[2] == CodeGeneratorResponse.File(
name="samples/generated_samples/7384949e.py",
content="\n",
)
print(actual_response.file[3].content)
assert actual_response.file[
3].name == "samples/generated_samples/snippet_metadata_mollusc_v1.json"
assert json.loads(
actual_response.file[3].content) == expected_snippet_metadata_json
def _generate_samples_and_manifest(
self, api_schema: api.API, index: snippet_index.SnippetIndex, sample_template: jinja2.Template, *, opts: Options) -> Tuple[Dict, snippet_index.SnippetIndex]:
"""Generate samples and samplegen manifest for the API.
Arguments:
api_schema (api.API): The schema for the API to which the samples belong.
sample_template (jinja2.Template): The template to use to generate samples.
opts (Options): Additional generator options.
Returns:
Tuple[Dict[str, CodeGeneratorResponse.File], snippet_index.SnippetIndex] : A dict mapping filepath to rendered file.
"""
# The two-layer data structure lets us do two things:
# * detect duplicate samples, which is an error
# * detect distinct samples with the same ID, which are disambiguated
id_to_hash_to_spec: DefaultDict[str,
Dict[str, Any]] = defaultdict(dict)
# Autogenerated sample specs
autogen_specs: typing.List[typing.Dict[str, Any]] = []
if opts.autogen_snippets:
autogen_specs = list(
samplegen.generate_sample_specs(api_schema, opts=opts))
# Also process any handwritten sample specs
handwritten_specs = samplegen.parse_handwritten_specs(
self._sample_configs)
sample_specs = autogen_specs + list(handwritten_specs)
for spec in sample_specs:
# Every sample requires an ID. This may be provided
# by a samplegen config author.
# If no ID is provided, fall back to the region tag.
#
# Ideally the sample author should pick a descriptive, unique ID,
# but this may be impractical and can be error-prone.
spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8]
sample_id = spec.get("id") or spec.get("region_tag") or spec_hash
spec["id"] = sample_id
hash_to_spec = id_to_hash_to_spec[sample_id]
if spec_hash in hash_to_spec:
raise DuplicateSample(
f"Duplicate samplegen spec found: {spec}")
hash_to_spec[spec_hash] = spec
out_dir = "samples/generated_samples"
fpath_to_spec_and_rendered = {}
for hash_to_spec in id_to_hash_to_spec.values():
for spec_hash, spec in hash_to_spec.items():
id_is_unique = len(hash_to_spec) == 1
# The ID is used to generate the file name. It must be globally unique.
if not id_is_unique:
spec["id"] += f"_{spec_hash}"
sample, snippet_metadata = samplegen.generate_sample(
spec, api_schema, sample_template,)
fpath = utils.to_snake_case(spec["id"]) + ".py"
fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = (
spec,
sample,
)
snippet_metadata.file = fpath
snippet_metadata.title = fpath
index.add_snippet(
snippet_index.Snippet(sample, snippet_metadata))
output_files = {
fname: CodeGeneratorResponse.File(
content=formatter.fix_whitespace(sample), name=fname
)
for fname, (_, sample) in fpath_to_spec_and_rendered.items()
}
if index.metadata_index.snippets:
# NOTE(busunkim): Not all fields are yet populated in the snippet metadata.
# Expected filename: snippet_metadata_{apishortname}_{apiversion}.json
snippet_metadata_path = str(pathlib.Path(
out_dir) / f"snippet_metadata_{api_schema.naming.name}_{api_schema.naming.version}.json").lower()
output_files[snippet_metadata_path] = CodeGeneratorResponse.File(
content=formatter.fix_whitespace(index.get_metadata_json()), name=snippet_metadata_path)
return output_files, index
def test_dont_generate_in_code_samples(mock_gmtime, mock_generate_sample, fs):
# These time values are nothing special,
# they just need to be deterministic.
returner = mock.MagicMock()
returner.tm_year = 2112
returner.tm_mon = 6
returner.tm_mday = 1
returner.tm_hour = 13
returner.tm_min = 13
returner.tm_sec = 13
mock_gmtime.return_value = returner
config_fpath = "samples.yaml"
fs.create_file(
config_fpath,
contents=dedent("""
type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto
schema_version: 1.2.0
samples:
- id: squid_sample
rpc: IdentifyMollusc
service: Mollusc.v1.Mollusc
sample_type:
- standalone
- incode/SQUID
- id: clam_sample
rpc: IdentifyMollusc
service: Mollusc.v1.Mollusc
sample_type:
- incode/CLAM
- id: whelk_sample
rpc: IdentifyMollusc
service: Mollusc.v1.Mollusc
sample_type:
- standalone
- id: octopus_sample
rpc: IdentifyMollusc
service: Mollusc.v1.Mollusc
"""),
)
generator = make_generator(f"samples={config_fpath}")
generator._env.loader = jinja2.DictLoader({"sample.py.j2": ""})
api_schema = make_api(
make_proto(
descriptor_pb2.FileDescriptorProto(
name="mollusc.proto",
package="Mollusc.v1",
service=[
descriptor_pb2.ServiceDescriptorProto(name="Mollusc")
],
), ),
naming=naming.NewNaming(name="Mollusc", version="v6"),
)
# Note that we do NOT expect a clam sample.
# There are four tests going on:
# 1) Just an explicit standalone sample type.
# 2) Multiple sample types, one of which is standalone.
# 3) Explicit sample types but NO standalone sample type.
# 4) Implicit standalone sample type.
expected = CodeGeneratorResponse(file=[
CodeGeneratorResponse.File(
name="samples/generated_samples/squid_sample.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/generated_samples/whelk_sample.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/generated_samples/octopus_sample.py",
content="\n",
),
# TODO(busunkim): Re-enable manifest generation once metadata
# format has been formalized.
# https://docs.google.com/document/d/1ghBam8vMj3xdoe4xfXhzVcOAIwrkbTpkMLgKc9RPD9k/edit#heading=h.sakzausv6hue
# CodeGeneratorResponse.File(
# name="samples/generated_samples/mollusc.v6.python.21120601.131313.manifest.yaml",
# content=dedent(
# """ ---
# type: manifest/samples
# schema_version: 3
# python: &python
# environment: python
# bin: python3
# base_path: samples
# invocation: \'{bin} {path} @args\'
# samples:
# - <<: *python
# sample: squid_sample
# path: \'{base_path}/squid_sample.py\'
# - <<: *python
# sample: whelk_sample
# path: \'{base_path}/whelk_sample.py\'
# - <<: *python
# sample: octopus_sample
# path: \'{base_path}/octopus_sample.py\'
# """
# ),
# ),
])
expected.supported_features |= CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL
actual = generator.get_response(api_schema=api_schema,
opts=Options.build(""))
assert actual == expected
def test_samplegen_config_to_output_files(
mock_gmtime,
mock_generate_sample,
fs,
):
# These time values are nothing special,
# they just need to be deterministic.
returner = mock.MagicMock()
returner.tm_year = 2112
returner.tm_mon = 6
returner.tm_mday = 1
returner.tm_hour = 13
returner.tm_min = 13
returner.tm_sec = 13
mock_gmtime.return_value = returner
fs.create_file(
"samples.yaml",
contents=dedent("""
---
type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto
schema_version: 1.2.0
samples:
- id: squid_sample
region_tag: humboldt_tag
rpc: get_squid_streaming
- region_tag: clam_sample
rpc: get_clam
"""),
)
g = generator.Generator(Options.build("samples=samples.yaml", ))
# Need to have the sample template visible to the generator.
g._env.loader = jinja2.DictLoader({"sample.py.j2": ""})
api_schema = make_api(
naming=naming.NewNaming(name="Mollusc", version="v6"))
actual_response = g.get_response(api_schema, opts=Options.build(""))
expected_response = CodeGeneratorResponse(file=[
CodeGeneratorResponse.File(
name="samples/squid_sample.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/clam_sample.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/mollusc.v6.python.21120601.131313.manifest.yaml",
content=dedent("""\
---
type: manifest/samples
schema_version: 3
python: &python
environment: python
bin: python3
base_path: samples
invocation: '{bin} {path} @args'
samples:
- <<: *python
sample: squid_sample
path: '{base_path}/squid_sample.py'
region_tag: humboldt_tag
- <<: *python
sample: clam_sample
path: '{base_path}/clam_sample.py'
region_tag: clam_sample
"""),
),
])
expected_response.supported_features |= (
CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL)
assert actual_response == expected_response
def _generate_samples_and_manifest(
self,
api_schema: api.API,
sample_template: jinja2.Template,
) -> Dict[str, CodeGeneratorResponse.File]:
"""Generate samples and samplegen manifest for the API.
Arguments:
api_schema (api.API): The schema for the API to which the samples belong.
Returns:
Dict[str, CodeGeneratorResponse.File]: A dict mapping filepath to rendered file.
"""
# The two-layer data structure lets us do two things:
# * detect duplicate samples, which is an error
# * detect distinct samples with the same ID, which are disambiguated
id_to_hash_to_spec: DefaultDict[str, Dict[str,
Any]] = defaultdict(dict)
STANDALONE_TYPE = "standalone"
for config_fpath in self._sample_configs:
with open(config_fpath) as f:
configs = yaml.safe_load_all(f.read())
spec_generator = (
spec for cfg in configs if is_valid_sample_cfg(cfg)
for spec in cfg.get("samples", [])
# If unspecified, assume a sample config describes a standalone.
# If sample_types are specified, standalone samples must be
# explicitly enabled.
if STANDALONE_TYPE in spec.get("sample_type",
[STANDALONE_TYPE]))
for spec in spec_generator:
# Every sample requires an ID, preferably provided by the
# samplegen config author.
# If no ID is provided, fall back to the region tag.
# If there's no region tag, generate a unique ID.
#
# Ideally the sample author should pick a descriptive, unique ID,
# but this may be impractical and can be error-prone.
spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8]
sample_id = spec.get("id") or spec.get(
"region_tag") or spec_hash
spec["id"] = sample_id
hash_to_spec = id_to_hash_to_spec[sample_id]
if spec_hash in hash_to_spec:
raise DuplicateSample(
f"Duplicate samplegen spec found: {spec}")
hash_to_spec[spec_hash] = spec
out_dir = "samples"
fpath_to_spec_and_rendered = {}
for hash_to_spec in id_to_hash_to_spec.values():
for spec_hash, spec in hash_to_spec.items():
id_is_unique = len(hash_to_spec) == 1
# The ID is used to generate the file name and by sample tester
# to link filenames to invoked samples. It must be globally unique.
if not id_is_unique:
spec["id"] += f"_{spec_hash}"
sample = samplegen.generate_sample(
spec,
api_schema,
sample_template,
)
fpath = spec["id"] + ".py"
fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = (
spec,
sample,
)
output_files = {
fname: CodeGeneratorResponse.File(
content=formatter.fix_whitespace(sample), name=fname)
for fname, (_, sample) in fpath_to_spec_and_rendered.items()
}
# Only generate a manifest if we generated samples.
if output_files:
manifest_fname, manifest_doc = manifest.generate(
((fname, spec)
for fname, (spec, _) in fpath_to_spec_and_rendered.items()),
api_schema,
)
manifest_fname = os.path.join(out_dir, manifest_fname)
output_files[manifest_fname] = CodeGeneratorResponse.File(
content=manifest_doc.render(), name=manifest_fname)
return output_files
def test_samplegen_id_disambiguation(mock_gmtime, mock_generate_sample, fs):
# These time values are nothing special,
# they just need to be deterministic.
returner = mock.MagicMock()
returner.tm_year = 2112
returner.tm_mon = 6
returner.tm_mday = 1
returner.tm_hour = 13
returner.tm_min = 13
returner.tm_sec = 13
mock_gmtime.return_value = returner
# Note: The first two samples will have the same nominal ID, the first by
# explicit naming and the second by falling back to the region_tag.
# The third has no id of any kind, so the generator is required to make a
# unique ID for it.
fs.create_file(
"samples.yaml",
contents=dedent("""
---
type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto
schema_version: 1.2.0
samples:
- id: squid_sample
region_tag: humboldt_tag
rpc: get_squid_streaming
# Note that this region tag collides with the id of the previous sample.
- region_tag: squid_sample
rpc: get_squid_streaming
# No id or region tag.
- rpc: get_squid_streaming
"""),
)
g = generator.Generator(Options.build("samples=samples.yaml"))
# Need to have the sample template visible to the generator.
g._env.loader = jinja2.DictLoader({"sample.py.j2": ""})
api_schema = make_api(
naming=naming.NewNaming(name="Mollusc", version="v6"))
actual_response = g.get_response(api_schema, opts=Options.build(""))
expected_response = CodeGeneratorResponse(file=[
CodeGeneratorResponse.File(
name="samples/squid_sample_91a465c6.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/squid_sample_55051b38.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/157884ee.py",
content="\n",
),
CodeGeneratorResponse.File(
name="samples/mollusc.v6.python.21120601.131313.manifest.yaml",
content=dedent("""\
---
type: manifest/samples
schema_version: 3
python: &python
environment: python
bin: python3
base_path: samples
invocation: '{bin} {path} @args'
samples:
- <<: *python
sample: squid_sample_91a465c6
path: '{base_path}/squid_sample_91a465c6.py'
region_tag: humboldt_tag
- <<: *python
sample: squid_sample_55051b38
path: '{base_path}/squid_sample_55051b38.py'
region_tag: squid_sample
- <<: *python
sample: 157884ee
path: '{base_path}/157884ee.py'
"""),
),
])
expected_response.supported_features |= (
CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL)
assert actual_response == expected_response
def _generate_samples_and_manifest(
self,
api_schema: api.API
) -> Dict[str, CodeGeneratorResponse.File]:
"""Generate samples and samplegen manifest for the API.
Arguments:
api_schema (api.API): The schema for the API to which the samples belong.
Returns:
Dict[str, CodeGeneratorResponse.File]: A dict mapping filepath to rendered file.
"""
id_to_samples: DefaultDict[str, List[Any]] = defaultdict(list)
for config_fpath in self._sample_configs:
with open(config_fpath) as f:
configs = yaml.safe_load_all(f.read())
spec_generator = (
spec
for cfg in configs if is_valid_sample_cfg(cfg)
for spec in cfg.get("samples", [])
)
for spec in spec_generator:
# Every sample requires an ID, preferably provided by the
# samplegen config author.
# If no ID is provided, fall back to the region tag.
# If there's no region tag, generate a unique ID.
#
# Ideally the sample author should pick a descriptive, unique ID,
# but this may be impractical and can be error-prone.
sample_id = (spec.get("id")
or spec.get("region_tag")
or sha256(str(spec).encode('utf8')).hexdigest()[:8])
spec["id"] = sample_id
id_to_samples[sample_id].append(spec)
out_dir = "samples"
fpath_to_spec_and_rendered = {}
for samples in id_to_samples.values():
for spec in samples:
id_is_unique = len(samples) == 1
# The ID is used to generate the file name and by sample tester
# to link filenames to invoked samples. It must be globally unique.
if not id_is_unique:
spec_hash = sha256(
str(spec).encode('utf8')).hexdigest()[:8]
spec["id"] += f"_{spec_hash}"
sample = samplegen.generate_sample(spec, self._env, api_schema)
fpath = spec["id"] + ".py"
fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = (spec,
sample)
output_files = {
fname: CodeGeneratorResponse.File(
content=formatter.fix_whitespace(sample),
name=fname
)
for fname, (_, sample) in fpath_to_spec_and_rendered.items()
}
# Only generate a manifest if we generated samples.
if output_files:
manifest_fname, manifest_doc = manifest.generate(
((fname, spec)
for fname, (spec, _) in fpath_to_spec_and_rendered.items()),
api_schema
)
manifest_fname = os.path.join(out_dir, manifest_fname)
output_files[manifest_fname] = CodeGeneratorResponse.File(
content=manifest_doc.render(),
name=manifest_fname
)
return output_files
def test_samplegen_config_to_output_files(mock_gmtime, fs):
# These time values are nothing special,
# they just need to be deterministic.
returner = mock.MagicMock()
returner.tm_year = 2112
returner.tm_mon = 6
returner.tm_mday = 1
returner.tm_hour = 13
returner.tm_min = 13
returner.tm_sec = 13
mock_gmtime.return_value = returner
fs.create_file(
"samples.yaml",
contents=dedent("""
---
type: com.google.api.codegen.samplegen.v1p2.SampleConfigProto
schema_version: 1.2.0
samples:
- id: squid_sample
region_tag: humboldt_tag
service: Mollusc.v1.Mollusc
rpc: GetSquidStreaming
- region_tag: clam_sample
service: Mollusc.v1.Mollusc
rpc: GetClam
"""),
)
g = generator.Generator(Options.build("samples=samples.yaml", ))
# Need to have the sample template visible to the generator.
g._env.loader = jinja2.DictLoader({"sample.py.j2": ""})
api_schema = DummyApiSchema(
services={
"Mollusc":
DummyService(
name="Mollusc",
methods={
# For this test the generator only cares about the dictionary keys
"GetSquidStreaming": DummyMethod(),
"GetClam": DummyMethod(),
},
)
},
naming=DummyNaming(
name="mollusc",
version="v1",
warehouse_package_name="mollusc-cephalopod-teuthida-",
versioned_module_name="teuthida_v1",
module_namespace="mollusc.cephalopod",
proto_package="google.mollusca"),
)
with mock.patch("gapic.samplegen.samplegen.generate_sample",
side_effect=mock_generate_sample):
actual_response = g.get_response(
api_schema, opts=Options.build("autogen-snippets=False"))
expected_snippet_index_json = {
"clientLibrary": {
"apis": [{
"id": "google.mollusca",
"version": "v1"
}],
"language": "PYTHON",
"name": "mollusc-cephalopod-teuthida-"
},
"snippets": [{
"clientMethod": {
"method": {
"service": {
"shortName": "Mollusc"
},
"shortName": "GetSquidStreaming"
}
},
"file":
"squid_sample.py",
"segments": [{
"type": "FULL"
}, {
"type": "SHORT"
}, {
"type": "CLIENT_INITIALIZATION"
}, {
"type": "REQUEST_INITIALIZATION"
}, {
"type": "REQUEST_EXECUTION"
}, {
"type": "RESPONSE_HANDLING"
}],
"title":
"squid_sample.py"
}, {
"clientMethod": {
"method": {
"service": {
"shortName": "Mollusc"
},
"shortName": "GetClam"
}
},
"file":
"clam_sample.py",
"segments": [{
"type": "FULL"
}, {
"type": "SHORT"
}, {
"type": "CLIENT_INITIALIZATION"
}, {
"type": "REQUEST_INITIALIZATION"
}, {
"type": "REQUEST_EXECUTION"
}, {
"type": "RESPONSE_HANDLING"
}],
"title":
"clam_sample.py"
}]
}
assert actual_response.supported_features == CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL
assert len(actual_response.file) == 3
assert actual_response.file[0] == CodeGeneratorResponse.File(
name="samples/generated_samples/squid_sample.py",
content="\n",
)
assert actual_response.file[1] == CodeGeneratorResponse.File(
name="samples/generated_samples/clam_sample.py",
content="\n",
)
assert actual_response.file[
2].name == "samples/generated_samples/snippet_metadata_mollusc_v1.json"
assert json.loads(
actual_response.file[2].content) == expected_snippet_index_json