def test_generate_manifest_relative_path_sanity(): with pytest.raises(types.InvalidSampleFpath): manifest.generate( {"molluscs/squid.py": { "id": "squid_sample" }}.items(), DummyApiSchema(naming=DummyNaming(name="Mollusc", version="v1")))
def _generate_samples_and_manifest( self, api_schema: api.API, sample_template: jinja2.Template, ) -> Dict[str, CodeGeneratorResponse.File]: """Generate samples and samplegen manifest for the API. Arguments: api_schema (api.API): The schema for the API to which the samples belong. Returns: Dict[str, CodeGeneratorResponse.File]: A dict mapping filepath to rendered file. """ # The two-layer data structure lets us do two things: # * detect duplicate samples, which is an error # * detect distinct samples with the same ID, which are disambiguated id_to_hash_to_spec: DefaultDict[str, Dict[str, Any]] = defaultdict(dict) STANDALONE_TYPE = "standalone" for config_fpath in self._sample_configs: with open(config_fpath) as f: configs = yaml.safe_load_all(f.read()) spec_generator = ( spec for cfg in configs if is_valid_sample_cfg(cfg) for spec in cfg.get("samples", []) # If unspecified, assume a sample config describes a standalone. # If sample_types are specified, standalone samples must be # explicitly enabled. if STANDALONE_TYPE in spec.get("sample_type", [STANDALONE_TYPE])) for spec in spec_generator: # Every sample requires an ID, preferably provided by the # samplegen config author. # If no ID is provided, fall back to the region tag. # If there's no region tag, generate a unique ID. # # Ideally the sample author should pick a descriptive, unique ID, # but this may be impractical and can be error-prone. spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8] sample_id = spec.get("id") or spec.get( "region_tag") or spec_hash spec["id"] = sample_id hash_to_spec = id_to_hash_to_spec[sample_id] if spec_hash in hash_to_spec: raise DuplicateSample( f"Duplicate samplegen spec found: {spec}") hash_to_spec[spec_hash] = spec out_dir = "samples" fpath_to_spec_and_rendered = {} for hash_to_spec in id_to_hash_to_spec.values(): for spec_hash, spec in hash_to_spec.items(): id_is_unique = len(hash_to_spec) == 1 # The ID is used to generate the file name and by sample tester # to link filenames to invoked samples. It must be globally unique. if not id_is_unique: spec["id"] += f"_{spec_hash}" sample = samplegen.generate_sample( spec, api_schema, sample_template, ) fpath = spec["id"] + ".py" fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = ( spec, sample, ) output_files = { fname: CodeGeneratorResponse.File( content=formatter.fix_whitespace(sample), name=fname) for fname, (_, sample) in fpath_to_spec_and_rendered.items() } # Only generate a manifest if we generated samples. if output_files: manifest_fname, manifest_doc = manifest.generate( ((fname, spec) for fname, (spec, _) in fpath_to_spec_and_rendered.items()), api_schema, ) manifest_fname = os.path.join(out_dir, manifest_fname) output_files[manifest_fname] = CodeGeneratorResponse.File( content=manifest_doc.render(), name=manifest_fname) return output_files
def _generate_samples_and_manifest( self, api_schema: api.API ) -> Dict[str, CodeGeneratorResponse.File]: """Generate samples and samplegen manifest for the API. Arguments: api_schema (api.API): The schema for the API to which the samples belong. Returns: Dict[str, CodeGeneratorResponse.File]: A dict mapping filepath to rendered file. """ id_to_samples: DefaultDict[str, List[Any]] = defaultdict(list) for config_fpath in self._sample_configs: with open(config_fpath) as f: configs = yaml.safe_load_all(f.read()) spec_generator = ( spec for cfg in configs if is_valid_sample_cfg(cfg) for spec in cfg.get("samples", []) ) for spec in spec_generator: # Every sample requires an ID, preferably provided by the # samplegen config author. # If no ID is provided, fall back to the region tag. # If there's no region tag, generate a unique ID. # # Ideally the sample author should pick a descriptive, unique ID, # but this may be impractical and can be error-prone. sample_id = (spec.get("id") or spec.get("region_tag") or sha256(str(spec).encode('utf8')).hexdigest()[:8]) spec["id"] = sample_id id_to_samples[sample_id].append(spec) out_dir = "samples" fpath_to_spec_and_rendered = {} for samples in id_to_samples.values(): for spec in samples: id_is_unique = len(samples) == 1 # The ID is used to generate the file name and by sample tester # to link filenames to invoked samples. It must be globally unique. if not id_is_unique: spec_hash = sha256( str(spec).encode('utf8')).hexdigest()[:8] spec["id"] += f"_{spec_hash}" sample = samplegen.generate_sample(spec, self._env, api_schema) fpath = spec["id"] + ".py" fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = (spec, sample) output_files = { fname: CodeGeneratorResponse.File( content=formatter.fix_whitespace(sample), name=fname ) for fname, (_, sample) in fpath_to_spec_and_rendered.items() } # Only generate a manifest if we generated samples. if output_files: manifest_fname, manifest_doc = manifest.generate( ((fname, spec) for fname, (spec, _) in fpath_to_spec_and_rendered.items()), api_schema ) manifest_fname = os.path.join(out_dir, manifest_fname) output_files[manifest_fname] = CodeGeneratorResponse.File( content=manifest_doc.render(), name=manifest_fname ) return output_files
def test_generate_manifest(): fpath_to_dummy_sample = { "samples/squid_fpath.py": {"id": "squid_sample"}, "samples/clam_fpath.py": {"id": "clam_sample", "region_tag": "giant_clam_sample"}, } fname, info = manifest.generate( fpath_to_dummy_sample.items(), DummyApiSchema(naming=DummyNaming(name="Mollusc", version="v1")), # Empirically derived number such that the # corresponding time_struct tests the zero # padding in the returned filename. manifest_time=4486525628 ) assert fname == "Mollusc.v1.python.21120304.090708.manifest.yaml" doc = gapic_yaml.Doc([ gapic_yaml.KeyVal("type", "manifest/samples"), gapic_yaml.KeyVal("schema_version", "3"), gapic_yaml.Map(name="python", anchor_name="python", elements=[ gapic_yaml.KeyVal( "environment", "python"), gapic_yaml.KeyVal( "bin", "python3"), gapic_yaml.KeyVal( "base_path", "samples"), gapic_yaml.KeyVal( "invocation", "'{bin} {path} @args'"), ]), gapic_yaml.Collection(name="samples", elements=[ [ gapic_yaml.Alias( "python"), gapic_yaml.KeyVal( "sample", "squid_sample"), gapic_yaml.KeyVal( "path", "'{base_path}/squid_fpath.py'"), gapic_yaml.Null, ], [ gapic_yaml.Alias("python"), gapic_yaml.KeyVal( "sample", "clam_sample"), gapic_yaml.KeyVal( "path", "'{base_path}/clam_fpath.py'"), gapic_yaml.KeyVal( "region_tag", "giant_clam_sample") ], ]) ]) assert info == doc expected_rendering = dedent( """\ --- type: manifest/samples schema_version: 3 python: &python environment: python bin: python3 base_path: samples invocation: '{bin} {path} @args' samples: - <<: *python sample: squid_sample path: '{base_path}/squid_fpath.py' - <<: *python sample: clam_sample path: '{base_path}/clam_fpath.py' region_tag: giant_clam_sample """) rendered_yaml = doc.render() assert rendered_yaml == expected_rendering expected_parsed_manifest = { "type": "manifest/samples", "schema_version": 3, "python": { "environment": "python", "bin": "python3", "base_path": "samples", "invocation": "{bin} {path} @args", }, "samples": [ { "environment": "python", "bin": "python3", "base_path": "samples", "invocation": "{bin} {path} @args", "sample": "squid_sample", "path": "{base_path}/squid_fpath.py", }, { "environment": "python", "bin": "python3", "base_path": "samples", "invocation": "{bin} {path} @args", "sample": "clam_sample", "path": "{base_path}/clam_fpath.py", "region_tag": "giant_clam_sample", }, ], } parsed_manifest = yaml.safe_load(rendered_yaml) assert parsed_manifest == expected_parsed_manifest