def _get_file( self, template_name: str, *, opts: Options, api_schema: api.API, **context, ): """Render a template to a protobuf plugin File object.""" # Determine the target filename. fn = self._get_filename( template_name, api_schema=api_schema, context=context,) # Render the file contents. cgr_file = CodeGeneratorResponse.File( content=formatter.fix_whitespace( self._env.get_template(template_name).render( api=api_schema, opts=opts, **context ), ), name=fn, ) # Quick check: Do not render empty files. if utils.empty(cgr_file.content) and not fn.endswith( ("py.typed", "__init__.py") ): return {} # Return the filename and content in a length-1 dictionary # (because we track output files overall in a dictionary). return {fn: cgr_file}
def test_fix_whitespace_top_level(): assert formatter.fix_whitespace( textwrap.dedent("""\ import something class Correct: pass class TooFarDown: pass class TooClose: # remains too close pass """)) == textwrap.dedent("""\ import something class Correct: pass class TooFarDown: pass class TooClose: # remains too close pass """)
def test_fix_whitespace_comment(): assert formatter.fix_whitespace( textwrap.dedent("""\ def do_something(): do_first_thing() # Something something something. do_second_thing() """)) == textwrap.dedent("""\ def do_something(): do_first_thing() # Something something something. do_second_thing() """)
def _get_file(self, template_name: str, *, api_schema=api.API, **context: Mapping): """Render a template to a protobuf plugin File object.""" fn = self._get_filename(template_name, api_schema=api_schema, context=context, ) return {fn: CodeGeneratorResponse.File( content=formatter.fix_whitespace( self._env.get_template(template_name).render( api=api_schema, **context ), ), name=fn, )}
def test_fix_whitespace_nested(): assert formatter.fix_whitespace( textwrap.dedent("""\ class JustAClass: def foo(self): pass def too_far_down(self): pass """)) == textwrap.dedent("""\ class JustAClass: def foo(self): pass def too_far_down(self): pass """)
def _render_templates( self, templates: Iterable[str], *, additional_context: Mapping[str, Any] = None, ) -> Sequence[CodeGeneratorResponse.File]: """Render the requested templates. Args: templates (Iterable[str]): The set of templates to be rendered. It is expected that these come from the methods on :class:`~.loader.TemplateLoader`, and they should be able to be set to the :meth:`jinja2.Environment.get_template` method. additional_context (Mapping[str, Any]): Additional variables to be sent to the templates. The ``api`` variable is always available. Returns: Sequence[~.CodeGeneratorResponse.File]: A sequence of File objects for inclusion in the final response. """ answer = [] additional_context = additional_context or {} # Iterate over the provided templates and generate a File object # for each. for template_name in templates: # Generate the File object. answer.append( CodeGeneratorResponse.File( content=formatter.fix_whitespace( self._env.get_template(template_name).render( api=self._api, **additional_context), ), name=self._get_output_filename( template_name, context=additional_context, ), )) # Done; return the File objects based on these templates. return answer
def test_fix_whitespace_decorators(): assert formatter.fix_whitespace( textwrap.dedent("""\ class JustAClass: def foo(self): pass @property def too_far_down(self): return 42 """)) == textwrap.dedent("""\ class JustAClass: def foo(self): pass @property def too_far_down(self): return 42 """)
def _generate_samples_and_manifest( self, api_schema: api.API, index: snippet_index.SnippetIndex, sample_template: jinja2.Template, *, opts: Options) -> Tuple[Dict, snippet_index.SnippetIndex]: """Generate samples and samplegen manifest for the API. Arguments: api_schema (api.API): The schema for the API to which the samples belong. sample_template (jinja2.Template): The template to use to generate samples. opts (Options): Additional generator options. Returns: Tuple[Dict[str, CodeGeneratorResponse.File], snippet_index.SnippetIndex] : A dict mapping filepath to rendered file. """ # The two-layer data structure lets us do two things: # * detect duplicate samples, which is an error # * detect distinct samples with the same ID, which are disambiguated id_to_hash_to_spec: DefaultDict[str, Dict[str, Any]] = defaultdict(dict) # Autogenerated sample specs autogen_specs: typing.List[typing.Dict[str, Any]] = [] if opts.autogen_snippets: autogen_specs = list( samplegen.generate_sample_specs(api_schema, opts=opts)) # Also process any handwritten sample specs handwritten_specs = samplegen.parse_handwritten_specs( self._sample_configs) sample_specs = autogen_specs + list(handwritten_specs) for spec in sample_specs: # Every sample requires an ID. This may be provided # by a samplegen config author. # If no ID is provided, fall back to the region tag. # # Ideally the sample author should pick a descriptive, unique ID, # but this may be impractical and can be error-prone. spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8] sample_id = spec.get("id") or spec.get("region_tag") or spec_hash spec["id"] = sample_id hash_to_spec = id_to_hash_to_spec[sample_id] if spec_hash in hash_to_spec: raise DuplicateSample( f"Duplicate samplegen spec found: {spec}") hash_to_spec[spec_hash] = spec out_dir = "samples/generated_samples" fpath_to_spec_and_rendered = {} for hash_to_spec in id_to_hash_to_spec.values(): for spec_hash, spec in hash_to_spec.items(): id_is_unique = len(hash_to_spec) == 1 # The ID is used to generate the file name. It must be globally unique. if not id_is_unique: spec["id"] += f"_{spec_hash}" sample, snippet_metadata = samplegen.generate_sample( spec, api_schema, sample_template,) fpath = utils.to_snake_case(spec["id"]) + ".py" fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = ( spec, sample, ) snippet_metadata.file = fpath snippet_metadata.title = fpath index.add_snippet( snippet_index.Snippet(sample, snippet_metadata)) output_files = { fname: CodeGeneratorResponse.File( content=formatter.fix_whitespace(sample), name=fname ) for fname, (_, sample) in fpath_to_spec_and_rendered.items() } if index.metadata_index.snippets: # NOTE(busunkim): Not all fields are yet populated in the snippet metadata. # Expected filename: snippet_metadata_{apishortname}_{apiversion}.json snippet_metadata_path = str(pathlib.Path( out_dir) / f"snippet_metadata_{api_schema.naming.name}_{api_schema.naming.version}.json").lower() output_files[snippet_metadata_path] = CodeGeneratorResponse.File( content=formatter.fix_whitespace(index.get_metadata_json()), name=snippet_metadata_path) return output_files, index
def _generate_samples_and_manifest( self, api_schema: api.API, sample_template: jinja2.Template, ) -> Dict[str, CodeGeneratorResponse.File]: """Generate samples and samplegen manifest for the API. Arguments: api_schema (api.API): The schema for the API to which the samples belong. Returns: Dict[str, CodeGeneratorResponse.File]: A dict mapping filepath to rendered file. """ # The two-layer data structure lets us do two things: # * detect duplicate samples, which is an error # * detect distinct samples with the same ID, which are disambiguated id_to_hash_to_spec: DefaultDict[str, Dict[str, Any]] = defaultdict(dict) STANDALONE_TYPE = "standalone" for config_fpath in self._sample_configs: with open(config_fpath) as f: configs = yaml.safe_load_all(f.read()) spec_generator = ( spec for cfg in configs if is_valid_sample_cfg(cfg) for spec in cfg.get("samples", []) # If unspecified, assume a sample config describes a standalone. # If sample_types are specified, standalone samples must be # explicitly enabled. if STANDALONE_TYPE in spec.get("sample_type", [STANDALONE_TYPE])) for spec in spec_generator: # Every sample requires an ID, preferably provided by the # samplegen config author. # If no ID is provided, fall back to the region tag. # If there's no region tag, generate a unique ID. # # Ideally the sample author should pick a descriptive, unique ID, # but this may be impractical and can be error-prone. spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8] sample_id = spec.get("id") or spec.get( "region_tag") or spec_hash spec["id"] = sample_id hash_to_spec = id_to_hash_to_spec[sample_id] if spec_hash in hash_to_spec: raise DuplicateSample( f"Duplicate samplegen spec found: {spec}") hash_to_spec[spec_hash] = spec out_dir = "samples" fpath_to_spec_and_rendered = {} for hash_to_spec in id_to_hash_to_spec.values(): for spec_hash, spec in hash_to_spec.items(): id_is_unique = len(hash_to_spec) == 1 # The ID is used to generate the file name and by sample tester # to link filenames to invoked samples. It must be globally unique. if not id_is_unique: spec["id"] += f"_{spec_hash}" sample = samplegen.generate_sample( spec, api_schema, sample_template, ) fpath = spec["id"] + ".py" fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = ( spec, sample, ) output_files = { fname: CodeGeneratorResponse.File( content=formatter.fix_whitespace(sample), name=fname) for fname, (_, sample) in fpath_to_spec_and_rendered.items() } # Only generate a manifest if we generated samples. if output_files: manifest_fname, manifest_doc = manifest.generate( ((fname, spec) for fname, (spec, _) in fpath_to_spec_and_rendered.items()), api_schema, ) manifest_fname = os.path.join(out_dir, manifest_fname) output_files[manifest_fname] = CodeGeneratorResponse.File( content=manifest_doc.render(), name=manifest_fname) return output_files
def _generate_samples_and_manifest(self, api_schema: api.API, sample_template: jinja2.Template, *, opts: Options) -> Dict: """Generate samples and samplegen manifest for the API. Arguments: api_schema (api.API): The schema for the API to which the samples belong. sample_template (jinja2.Template): The template to use to generate samples. opts (Options): Additional generator options. Returns: Dict[str, CodeGeneratorResponse.File]: A dict mapping filepath to rendered file. """ # The two-layer data structure lets us do two things: # * detect duplicate samples, which is an error # * detect distinct samples with the same ID, which are disambiguated id_to_hash_to_spec: DefaultDict[str, Dict[str, Any]] = defaultdict(dict) # Autogenerated sample specs autogen_specs: typing.List[typing.Dict[str, Any]] = [] if opts.autogen_snippets: autogen_specs = list( samplegen.generate_sample_specs(api_schema, opts=opts)) # Also process any handwritten sample specs handwritten_specs = samplegen.parse_handwritten_specs( self._sample_configs) sample_specs = autogen_specs + list(handwritten_specs) for spec in sample_specs: # Every sample requires an ID. This may be provided # by a samplegen config author. # If no ID is provided, fall back to the region tag. # # Ideally the sample author should pick a descriptive, unique ID, # but this may be impractical and can be error-prone. spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8] sample_id = spec.get("id") or spec.get("region_tag") or spec_hash spec["id"] = sample_id hash_to_spec = id_to_hash_to_spec[sample_id] if spec_hash in hash_to_spec: raise DuplicateSample( f"Duplicate samplegen spec found: {spec}") hash_to_spec[spec_hash] = spec out_dir = "samples/generated_samples" fpath_to_spec_and_rendered = {} for hash_to_spec in id_to_hash_to_spec.values(): for spec_hash, spec in hash_to_spec.items(): id_is_unique = len(hash_to_spec) == 1 # The ID is used to generate the file name. It must be globally unique. if not id_is_unique: spec["id"] += f"_{spec_hash}" sample = samplegen.generate_sample( spec, api_schema, sample_template, ) fpath = utils.to_snake_case(spec["id"]) + ".py" fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = ( spec, sample, ) output_files = { fname: CodeGeneratorResponse.File( content=formatter.fix_whitespace(sample), name=fname) for fname, (_, sample) in fpath_to_spec_and_rendered.items() } # TODO(busunkim): Re-enable manifest generation once metadata # format has been formalized. # https://docs.google.com/document/d/1ghBam8vMj3xdoe4xfXhzVcOAIwrkbTpkMLgKc9RPD9k/edit#heading=h.sakzausv6hue # # if output_files: # manifest_fname, manifest_doc = manifest.generate( # ( # (fname, spec) # for fname, (spec, _) in fpath_to_spec_and_rendered.items() # ), # api_schema, # ) # manifest_fname = os.path.join(out_dir, manifest_fname) # output_files[manifest_fname] = CodeGeneratorResponse.File( # content=manifest_doc.render(), name=manifest_fname # ) return output_files
def test_file_newline_ending(): assert formatter.fix_whitespace('') == '\n'
def _generate_samples_and_manifest( self, api_schema: api.API ) -> Dict[str, CodeGeneratorResponse.File]: """Generate samples and samplegen manifest for the API. Arguments: api_schema (api.API): The schema for the API to which the samples belong. Returns: Dict[str, CodeGeneratorResponse.File]: A dict mapping filepath to rendered file. """ id_to_samples: DefaultDict[str, List[Any]] = defaultdict(list) for config_fpath in self._sample_configs: with open(config_fpath) as f: configs = yaml.safe_load_all(f.read()) spec_generator = ( spec for cfg in configs if is_valid_sample_cfg(cfg) for spec in cfg.get("samples", []) ) for spec in spec_generator: # Every sample requires an ID, preferably provided by the # samplegen config author. # If no ID is provided, fall back to the region tag. # If there's no region tag, generate a unique ID. # # Ideally the sample author should pick a descriptive, unique ID, # but this may be impractical and can be error-prone. sample_id = (spec.get("id") or spec.get("region_tag") or sha256(str(spec).encode('utf8')).hexdigest()[:8]) spec["id"] = sample_id id_to_samples[sample_id].append(spec) out_dir = "samples" fpath_to_spec_and_rendered = {} for samples in id_to_samples.values(): for spec in samples: id_is_unique = len(samples) == 1 # The ID is used to generate the file name and by sample tester # to link filenames to invoked samples. It must be globally unique. if not id_is_unique: spec_hash = sha256( str(spec).encode('utf8')).hexdigest()[:8] spec["id"] += f"_{spec_hash}" sample = samplegen.generate_sample(spec, self._env, api_schema) fpath = spec["id"] + ".py" fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = (spec, sample) output_files = { fname: CodeGeneratorResponse.File( content=formatter.fix_whitespace(sample), name=fname ) for fname, (_, sample) in fpath_to_spec_and_rendered.items() } # Only generate a manifest if we generated samples. if output_files: manifest_fname, manifest_doc = manifest.generate( ((fname, spec) for fname, (spec, _) in fpath_to_spec_and_rendered.items()), api_schema ) manifest_fname = os.path.join(out_dir, manifest_fname) output_files[manifest_fname] = CodeGeneratorResponse.File( content=manifest_doc.render(), name=manifest_fname ) return output_files