def test_generate_manifest_relative_path_sanity():
with pytest.raises(types.InvalidSampleFpath):
manifest.generate(
{"molluscs/squid.py": {
"id": "squid_sample"
}}.items(),
DummyApiSchema(naming=DummyNaming(name="Mollusc", version="v1")))
def _generate_samples_and_manifest(
self,
api_schema: api.API,
sample_template: jinja2.Template,
) -> Dict[str, CodeGeneratorResponse.File]:
"""Generate samples and samplegen manifest for the API.
Arguments:
api_schema (api.API): The schema for the API to which the samples belong.
Returns:
Dict[str, CodeGeneratorResponse.File]: A dict mapping filepath to rendered file.
"""
# The two-layer data structure lets us do two things:
# * detect duplicate samples, which is an error
# * detect distinct samples with the same ID, which are disambiguated
id_to_hash_to_spec: DefaultDict[str, Dict[str,
Any]] = defaultdict(dict)
STANDALONE_TYPE = "standalone"
for config_fpath in self._sample_configs:
with open(config_fpath) as f:
configs = yaml.safe_load_all(f.read())
spec_generator = (
spec for cfg in configs if is_valid_sample_cfg(cfg)
for spec in cfg.get("samples", [])
# If unspecified, assume a sample config describes a standalone.
# If sample_types are specified, standalone samples must be
# explicitly enabled.
if STANDALONE_TYPE in spec.get("sample_type",
[STANDALONE_TYPE]))
for spec in spec_generator:
# Every sample requires an ID, preferably provided by the
# samplegen config author.
# If no ID is provided, fall back to the region tag.
# If there's no region tag, generate a unique ID.
#
# Ideally the sample author should pick a descriptive, unique ID,
# but this may be impractical and can be error-prone.
spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8]
sample_id = spec.get("id") or spec.get(
"region_tag") or spec_hash
spec["id"] = sample_id
hash_to_spec = id_to_hash_to_spec[sample_id]
if spec_hash in hash_to_spec:
raise DuplicateSample(
f"Duplicate samplegen spec found: {spec}")
hash_to_spec[spec_hash] = spec
out_dir = "samples"
fpath_to_spec_and_rendered = {}
for hash_to_spec in id_to_hash_to_spec.values():
for spec_hash, spec in hash_to_spec.items():
id_is_unique = len(hash_to_spec) == 1
# The ID is used to generate the file name and by sample tester
# to link filenames to invoked samples. It must be globally unique.
if not id_is_unique:
spec["id"] += f"_{spec_hash}"
sample = samplegen.generate_sample(
spec,
api_schema,
sample_template,
)
fpath = spec["id"] + ".py"
fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = (
spec,
sample,
)
output_files = {
fname: CodeGeneratorResponse.File(
content=formatter.fix_whitespace(sample), name=fname)
for fname, (_, sample) in fpath_to_spec_and_rendered.items()
}
# Only generate a manifest if we generated samples.
if output_files:
manifest_fname, manifest_doc = manifest.generate(
((fname, spec)
for fname, (spec, _) in fpath_to_spec_and_rendered.items()),
api_schema,
)
manifest_fname = os.path.join(out_dir, manifest_fname)
output_files[manifest_fname] = CodeGeneratorResponse.File(
content=manifest_doc.render(), name=manifest_fname)
return output_files
def _generate_samples_and_manifest(
self,
api_schema: api.API
) -> Dict[str, CodeGeneratorResponse.File]:
"""Generate samples and samplegen manifest for the API.
Arguments:
api_schema (api.API): The schema for the API to which the samples belong.
Returns:
Dict[str, CodeGeneratorResponse.File]: A dict mapping filepath to rendered file.
"""
id_to_samples: DefaultDict[str, List[Any]] = defaultdict(list)
for config_fpath in self._sample_configs:
with open(config_fpath) as f:
configs = yaml.safe_load_all(f.read())
spec_generator = (
spec
for cfg in configs if is_valid_sample_cfg(cfg)
for spec in cfg.get("samples", [])
)
for spec in spec_generator:
# Every sample requires an ID, preferably provided by the
# samplegen config author.
# If no ID is provided, fall back to the region tag.
# If there's no region tag, generate a unique ID.
#
# Ideally the sample author should pick a descriptive, unique ID,
# but this may be impractical and can be error-prone.
sample_id = (spec.get("id")
or spec.get("region_tag")
or sha256(str(spec).encode('utf8')).hexdigest()[:8])
spec["id"] = sample_id
id_to_samples[sample_id].append(spec)
out_dir = "samples"
fpath_to_spec_and_rendered = {}
for samples in id_to_samples.values():
for spec in samples:
id_is_unique = len(samples) == 1
# The ID is used to generate the file name and by sample tester
# to link filenames to invoked samples. It must be globally unique.
if not id_is_unique:
spec_hash = sha256(
str(spec).encode('utf8')).hexdigest()[:8]
spec["id"] += f"_{spec_hash}"
sample = samplegen.generate_sample(spec, self._env, api_schema)
fpath = spec["id"] + ".py"
fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = (spec,
sample)
output_files = {
fname: CodeGeneratorResponse.File(
content=formatter.fix_whitespace(sample),
name=fname
)
for fname, (_, sample) in fpath_to_spec_and_rendered.items()
}
# Only generate a manifest if we generated samples.
if output_files:
manifest_fname, manifest_doc = manifest.generate(
((fname, spec)
for fname, (spec, _) in fpath_to_spec_and_rendered.items()),
api_schema
)
manifest_fname = os.path.join(out_dir, manifest_fname)
output_files[manifest_fname] = CodeGeneratorResponse.File(
content=manifest_doc.render(),
name=manifest_fname
)
return output_files
def test_generate_manifest():
fpath_to_dummy_sample = {
"samples/squid_fpath.py": {"id": "squid_sample"},
"samples/clam_fpath.py": {"id": "clam_sample",
"region_tag": "giant_clam_sample"},
}
fname, info = manifest.generate(
fpath_to_dummy_sample.items(),
DummyApiSchema(naming=DummyNaming(name="Mollusc", version="v1")),
# Empirically derived number such that the
# corresponding time_struct tests the zero
# padding in the returned filename.
manifest_time=4486525628
)
assert fname == "Mollusc.v1.python.21120304.090708.manifest.yaml"
doc = gapic_yaml.Doc([
gapic_yaml.KeyVal("type", "manifest/samples"),
gapic_yaml.KeyVal("schema_version", "3"),
gapic_yaml.Map(name="python",
anchor_name="python",
elements=[
gapic_yaml.KeyVal(
"environment", "python"),
gapic_yaml.KeyVal(
"bin", "python3"),
gapic_yaml.KeyVal(
"base_path", "samples"),
gapic_yaml.KeyVal(
"invocation", "'{bin} {path} @args'"),
]),
gapic_yaml.Collection(name="samples",
elements=[
[
gapic_yaml.Alias(
"python"),
gapic_yaml.KeyVal(
"sample", "squid_sample"),
gapic_yaml.KeyVal(
"path", "'{base_path}/squid_fpath.py'"),
gapic_yaml.Null,
],
[
gapic_yaml.Alias("python"),
gapic_yaml.KeyVal(
"sample", "clam_sample"),
gapic_yaml.KeyVal(
"path", "'{base_path}/clam_fpath.py'"),
gapic_yaml.KeyVal(
"region_tag", "giant_clam_sample")
],
])
])
assert info == doc
expected_rendering = dedent(
"""\
---
type: manifest/samples
schema_version: 3
python: &python
environment: python
bin: python3
base_path: samples
invocation: '{bin} {path} @args'
samples:
- <<: *python
sample: squid_sample
path: '{base_path}/squid_fpath.py'
- <<: *python
sample: clam_sample
path: '{base_path}/clam_fpath.py'
region_tag: giant_clam_sample
""")
rendered_yaml = doc.render()
assert rendered_yaml == expected_rendering
expected_parsed_manifest = {
"type": "manifest/samples",
"schema_version": 3,
"python": {
"environment": "python",
"bin": "python3",
"base_path": "samples",
"invocation": "{bin} {path} @args",
},
"samples": [
{
"environment": "python",
"bin": "python3",
"base_path": "samples",
"invocation": "{bin} {path} @args",
"sample": "squid_sample",
"path": "{base_path}/squid_fpath.py",
},
{
"environment": "python",
"bin": "python3",
"base_path": "samples",
"invocation": "{bin} {path} @args",
"sample": "clam_sample",
"path": "{base_path}/clam_fpath.py",
"region_tag": "giant_clam_sample",
},
],
}
parsed_manifest = yaml.safe_load(rendered_yaml)
assert parsed_manifest == expected_parsed_manifest
