def _generate_samples_and_manifest( self, api_schema: api.API, sample_template: jinja2.Template, ) -> Dict[str, CodeGeneratorResponse.File]: """Generate samples and samplegen manifest for the API. Arguments: api_schema (api.API): The schema for the API to which the samples belong. Returns: Dict[str, CodeGeneratorResponse.File]: A dict mapping filepath to rendered file. """ # The two-layer data structure lets us do two things: # * detect duplicate samples, which is an error # * detect distinct samples with the same ID, which are disambiguated id_to_hash_to_spec: DefaultDict[str, Dict[str, Any]] = defaultdict(dict) STANDALONE_TYPE = "standalone" for config_fpath in self._sample_configs: with open(config_fpath) as f: configs = yaml.safe_load_all(f.read()) spec_generator = ( spec for cfg in configs if is_valid_sample_cfg(cfg) for spec in cfg.get("samples", []) # If unspecified, assume a sample config describes a standalone. # If sample_types are specified, standalone samples must be # explicitly enabled. if STANDALONE_TYPE in spec.get("sample_type", [STANDALONE_TYPE])) for spec in spec_generator: # Every sample requires an ID, preferably provided by the # samplegen config author. # If no ID is provided, fall back to the region tag. # If there's no region tag, generate a unique ID. # # Ideally the sample author should pick a descriptive, unique ID, # but this may be impractical and can be error-prone. spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8] sample_id = spec.get("id") or spec.get( "region_tag") or spec_hash spec["id"] = sample_id hash_to_spec = id_to_hash_to_spec[sample_id] if spec_hash in hash_to_spec: raise DuplicateSample( f"Duplicate samplegen spec found: {spec}") hash_to_spec[spec_hash] = spec out_dir = "samples" fpath_to_spec_and_rendered = {} for hash_to_spec in id_to_hash_to_spec.values(): for spec_hash, spec in hash_to_spec.items(): id_is_unique = len(hash_to_spec) == 1 # The ID is used to generate the file name and by sample tester # to link filenames to invoked samples. It must be globally unique. if not id_is_unique: spec["id"] += f"_{spec_hash}" sample = samplegen.generate_sample( spec, api_schema, sample_template, ) fpath = spec["id"] + ".py" fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = ( spec, sample, ) output_files = { fname: CodeGeneratorResponse.File( content=formatter.fix_whitespace(sample), name=fname) for fname, (_, sample) in fpath_to_spec_and_rendered.items() } # Only generate a manifest if we generated samples. if output_files: manifest_fname, manifest_doc = manifest.generate( ((fname, spec) for fname, (spec, _) in fpath_to_spec_and_rendered.items()), api_schema, ) manifest_fname = os.path.join(out_dir, manifest_fname) output_files[manifest_fname] = CodeGeneratorResponse.File( content=manifest_doc.render(), name=manifest_fname) return output_files
def _generate_samples_and_manifest( self, api_schema: api.API, index: snippet_index.SnippetIndex, sample_template: jinja2.Template, *, opts: Options) -> Tuple[Dict, snippet_index.SnippetIndex]: """Generate samples and samplegen manifest for the API. Arguments: api_schema (api.API): The schema for the API to which the samples belong. sample_template (jinja2.Template): The template to use to generate samples. opts (Options): Additional generator options. Returns: Tuple[Dict[str, CodeGeneratorResponse.File], snippet_index.SnippetIndex] : A dict mapping filepath to rendered file. """ # The two-layer data structure lets us do two things: # * detect duplicate samples, which is an error # * detect distinct samples with the same ID, which are disambiguated id_to_hash_to_spec: DefaultDict[str, Dict[str, Any]] = defaultdict(dict) # Autogenerated sample specs autogen_specs: typing.List[typing.Dict[str, Any]] = [] if opts.autogen_snippets: autogen_specs = list( samplegen.generate_sample_specs(api_schema, opts=opts)) # Also process any handwritten sample specs handwritten_specs = samplegen.parse_handwritten_specs( self._sample_configs) sample_specs = autogen_specs + list(handwritten_specs) for spec in sample_specs: # Every sample requires an ID. This may be provided # by a samplegen config author. # If no ID is provided, fall back to the region tag. # # Ideally the sample author should pick a descriptive, unique ID, # but this may be impractical and can be error-prone. spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8] sample_id = spec.get("id") or spec.get("region_tag") or spec_hash spec["id"] = sample_id hash_to_spec = id_to_hash_to_spec[sample_id] if spec_hash in hash_to_spec: raise DuplicateSample( f"Duplicate samplegen spec found: {spec}") hash_to_spec[spec_hash] = spec out_dir = "samples/generated_samples" fpath_to_spec_and_rendered = {} for hash_to_spec in id_to_hash_to_spec.values(): for spec_hash, spec in hash_to_spec.items(): id_is_unique = len(hash_to_spec) == 1 # The ID is used to generate the file name. It must be globally unique. if not id_is_unique: spec["id"] += f"_{spec_hash}" sample, snippet_metadata = samplegen.generate_sample( spec, api_schema, sample_template,) fpath = utils.to_snake_case(spec["id"]) + ".py" fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = ( spec, sample, ) snippet_metadata.file = fpath snippet_metadata.title = fpath index.add_snippet( snippet_index.Snippet(sample, snippet_metadata)) output_files = { fname: CodeGeneratorResponse.File( content=formatter.fix_whitespace(sample), name=fname ) for fname, (_, sample) in fpath_to_spec_and_rendered.items() } if index.metadata_index.snippets: # NOTE(busunkim): Not all fields are yet populated in the snippet metadata. # Expected filename: snippet_metadata_{apishortname}_{apiversion}.json snippet_metadata_path = str(pathlib.Path( out_dir) / f"snippet_metadata_{api_schema.naming.name}_{api_schema.naming.version}.json").lower() output_files[snippet_metadata_path] = CodeGeneratorResponse.File( content=formatter.fix_whitespace(index.get_metadata_json()), name=snippet_metadata_path) return output_files, index
def _generate_samples_and_manifest(self, api_schema: api.API, sample_template: jinja2.Template, *, opts: Options) -> Dict: """Generate samples and samplegen manifest for the API. Arguments: api_schema (api.API): The schema for the API to which the samples belong. sample_template (jinja2.Template): The template to use to generate samples. opts (Options): Additional generator options. Returns: Dict[str, CodeGeneratorResponse.File]: A dict mapping filepath to rendered file. """ # The two-layer data structure lets us do two things: # * detect duplicate samples, which is an error # * detect distinct samples with the same ID, which are disambiguated id_to_hash_to_spec: DefaultDict[str, Dict[str, Any]] = defaultdict(dict) # Autogenerated sample specs autogen_specs: typing.List[typing.Dict[str, Any]] = [] if opts.autogen_snippets: autogen_specs = list( samplegen.generate_sample_specs(api_schema, opts=opts)) # Also process any handwritten sample specs handwritten_specs = samplegen.parse_handwritten_specs( self._sample_configs) sample_specs = autogen_specs + list(handwritten_specs) for spec in sample_specs: # Every sample requires an ID. This may be provided # by a samplegen config author. # If no ID is provided, fall back to the region tag. # # Ideally the sample author should pick a descriptive, unique ID, # but this may be impractical and can be error-prone. spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8] sample_id = spec.get("id") or spec.get("region_tag") or spec_hash spec["id"] = sample_id hash_to_spec = id_to_hash_to_spec[sample_id] if spec_hash in hash_to_spec: raise DuplicateSample( f"Duplicate samplegen spec found: {spec}") hash_to_spec[spec_hash] = spec out_dir = "samples/generated_samples" fpath_to_spec_and_rendered = {} for hash_to_spec in id_to_hash_to_spec.values(): for spec_hash, spec in hash_to_spec.items(): id_is_unique = len(hash_to_spec) == 1 # The ID is used to generate the file name. It must be globally unique. if not id_is_unique: spec["id"] += f"_{spec_hash}" sample = samplegen.generate_sample( spec, api_schema, sample_template, ) fpath = utils.to_snake_case(spec["id"]) + ".py" fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = ( spec, sample, ) output_files = { fname: CodeGeneratorResponse.File( content=formatter.fix_whitespace(sample), name=fname) for fname, (_, sample) in fpath_to_spec_and_rendered.items() } # TODO(busunkim): Re-enable manifest generation once metadata # format has been formalized. # https://docs.google.com/document/d/1ghBam8vMj3xdoe4xfXhzVcOAIwrkbTpkMLgKc9RPD9k/edit#heading=h.sakzausv6hue # # if output_files: # manifest_fname, manifest_doc = manifest.generate( # ( # (fname, spec) # for fname, (spec, _) in fpath_to_spec_and_rendered.items() # ), # api_schema, # ) # manifest_fname = os.path.join(out_dir, manifest_fname) # output_files[manifest_fname] = CodeGeneratorResponse.File( # content=manifest_doc.render(), name=manifest_fname # ) return output_files