def test_add_snippet_no_matching_service(sample_str): snippet_metadata = snippet_metadata_pb2.Snippet() snippet_metadata.client_method.method.service.short_name = "Clam" snippet = snippet_index.Snippet(sample_str, snippet_metadata) # No 'Clam' service in API Schema index = snippet_index.SnippetIndex(api_schema=DummyApiSchema( services={"Squid": DummyService(name="Squid", methods={})})) with pytest.raises(types.UnknownService): index.add_snippet(snippet)
def test_add_snippet_no_matching_rpc(sample_str): snippet_metadata = snippet_metadata_pb2.Snippet() snippet_metadata.client_method.method.service.short_name = "Squid" snippet_metadata.client_method.short_name = "classify" snippet = snippet_index.Snippet(sample_str, snippet_metadata) # No 'classify' method in 'Squid' service index = snippet_index.SnippetIndex(api_schema=DummyApiSchema( services={"Squid": DummyService(name="Squid", methods={"list": None}) })) with pytest.raises(types.RpcMethodNotFound): index.add_snippet(snippet)
def test_get_metadata_json(sample_str): snippet_metadata = snippet_metadata_pb2.Snippet() snippet_metadata.client_method.method.service.short_name = "Squid" snippet_metadata.client_method.method.short_name = "classify" snippet = snippet_index.Snippet(sample_str, snippet_metadata) index = snippet_index.SnippetIndex(api_schema=DummyApiSchema( services={ "Squid": DummyService(name="Squid", methods={"classify": DummyMethod()}) })) index.add_snippet(snippet) assert json.loads(index.get_metadata_json()) == { 'snippets': [{ 'clientMethod': { 'method': { 'shortName': 'classify', 'service': { 'shortName': 'Squid' } } }, 'segments': [{ 'end': 28, 'start': 2, 'type': 'FULL' }, { 'end': 28, 'start': 2, 'type': 'SHORT' }, { 'end': 8, 'start': 6, 'type': 'CLIENT_INITIALIZATION' }, { 'end': 22, 'start': 9, 'type': 'REQUEST_INITIALIZATION' }, { 'end': 25, 'start': 23, 'type': 'REQUEST_EXECUTION' }, { 'end': 29, 'start': 26, 'type': 'RESPONSE_HANDLING' }] }] }
def test_add_snippet_no_matching_service(sample_str): snippet_metadata = snippet_metadata_pb2.Snippet() snippet_metadata.client_method.method.service.short_name = "Clam" snippet = snippet_index.Snippet(sample_str, snippet_metadata) # No 'Clam' service in API Schema index = snippet_index.SnippetIndex(api_schema=DummyApiSchema( services={"Squid": DummyService(name="Squid", methods={})}, naming=DummyNaming(proto_package="google.mollusca", warehouse_package_name="google-mollusca", version="v1"), )) with pytest.raises(types.UnknownService): index.add_snippet(snippet)
def test_add_snippet_no_matching_rpc(sample_str): snippet_metadata = snippet_metadata_pb2.Snippet() snippet_metadata.client_method.method.service.short_name = "Squid" snippet_metadata.client_method.short_name = "classify" snippet = snippet_index.Snippet(sample_str, snippet_metadata) # No 'classify' method in 'Squid' service index = snippet_index.SnippetIndex(api_schema=DummyApiSchema( services={"Squid": DummyService(name="Squid", methods={"list": None})}, naming=DummyNaming(proto_package="google.mollusca", warehouse_package_name="google-mollusca", version="v1"), )) with pytest.raises(types.RpcMethodNotFound): index.add_snippet(snippet)
def test_add_and_get_snippet_sync(sample_str): snippet_metadata = snippet_metadata_pb2.Snippet() snippet_metadata.client_method.method.service.short_name = "Squid" snippet_metadata.client_method.method.short_name = "classify" snippet = snippet_index.Snippet(sample_str, snippet_metadata) index = snippet_index.SnippetIndex(api_schema=DummyApiSchema( services={ "Squid": DummyService(name="Squid", methods={"classify": DummyMethod()}) })) index.add_snippet(snippet) index.get_snippet(service_name="Squid", rpc_name="classify")
def test_add_and_get_snippet_sync(sample_str): snippet_metadata = snippet_metadata_pb2.Snippet() snippet_metadata.client_method.method.service.short_name = "Squid" snippet_metadata.client_method.method.short_name = "classify" snippet = snippet_index.Snippet(sample_str, snippet_metadata) index = snippet_index.SnippetIndex(api_schema=DummyApiSchema( services={ "Squid": DummyService(name="Squid", methods={"classify": DummyMethod()}) }, naming=DummyNaming(proto_package="google.mollusca", warehouse_package_name="google-mollusca", version="v1"), )) index.add_snippet(snippet) index.get_snippet(service_name="Squid", rpc_name="classify")
def _generate_samples_and_manifest( self, api_schema: api.API, index: snippet_index.SnippetIndex, sample_template: jinja2.Template, *, opts: Options) -> Tuple[Dict, snippet_index.SnippetIndex]: """Generate samples and samplegen manifest for the API. Arguments: api_schema (api.API): The schema for the API to which the samples belong. sample_template (jinja2.Template): The template to use to generate samples. opts (Options): Additional generator options. Returns: Tuple[Dict[str, CodeGeneratorResponse.File], snippet_index.SnippetIndex] : A dict mapping filepath to rendered file. """ # The two-layer data structure lets us do two things: # * detect duplicate samples, which is an error # * detect distinct samples with the same ID, which are disambiguated id_to_hash_to_spec: DefaultDict[str, Dict[str, Any]] = defaultdict(dict) # Autogenerated sample specs autogen_specs: typing.List[typing.Dict[str, Any]] = [] if opts.autogen_snippets: autogen_specs = list( samplegen.generate_sample_specs(api_schema, opts=opts)) # Also process any handwritten sample specs handwritten_specs = samplegen.parse_handwritten_specs( self._sample_configs) sample_specs = autogen_specs + list(handwritten_specs) for spec in sample_specs: # Every sample requires an ID. This may be provided # by a samplegen config author. # If no ID is provided, fall back to the region tag. # # Ideally the sample author should pick a descriptive, unique ID, # but this may be impractical and can be error-prone. spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8] sample_id = spec.get("id") or spec.get("region_tag") or spec_hash spec["id"] = sample_id hash_to_spec = id_to_hash_to_spec[sample_id] if spec_hash in hash_to_spec: raise DuplicateSample( f"Duplicate samplegen spec found: {spec}") hash_to_spec[spec_hash] = spec out_dir = "samples/generated_samples" fpath_to_spec_and_rendered = {} for hash_to_spec in id_to_hash_to_spec.values(): for spec_hash, spec in hash_to_spec.items(): id_is_unique = len(hash_to_spec) == 1 # The ID is used to generate the file name. It must be globally unique. if not id_is_unique: spec["id"] += f"_{spec_hash}" sample, snippet_metadata = samplegen.generate_sample( spec, api_schema, sample_template,) fpath = utils.to_snake_case(spec["id"]) + ".py" fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = ( spec, sample, ) snippet_metadata.file = fpath snippet_metadata.title = fpath index.add_snippet( snippet_index.Snippet(sample, snippet_metadata)) output_files = { fname: CodeGeneratorResponse.File( content=formatter.fix_whitespace(sample), name=fname ) for fname, (_, sample) in fpath_to_spec_and_rendered.items() } if index.metadata_index.snippets: # NOTE(busunkim): Not all fields are yet populated in the snippet metadata. # Expected filename: snippet_metadata_{apishortname}_{apiversion}.json snippet_metadata_path = str(pathlib.Path( out_dir) / f"snippet_metadata_{api_schema.naming.name}_{api_schema.naming.version}.json").lower() output_files[snippet_metadata_path] = CodeGeneratorResponse.File( content=formatter.fix_whitespace(index.get_metadata_json()), name=snippet_metadata_path) return output_files, index
def test_snippet_init(sample_str): # We are not trying to exhaustively test the snippet metadata protobuf, # just checking that fields are not unset sample_metadata = snippet_metadata_pb2.Snippet(title="classify_squid.py") sample_metadata.language = snippet_metadata_pb2.Language.PYTHON snippet = snippet_index.Snippet(sample_str, sample_metadata) assert snippet.sample_str == sample_str # It's easier to eyeball diffs on the dictionary representation assert json_format.MessageToDict(snippet.metadata) == { "language": "PYTHON", "title": "classify_squid.py", "segments": [ { "end": 28, "start": 2, "type": "FULL" }, { "end": 28, "start": 2, "type": "SHORT" }, { "end": 8, "start": 6, "type": "CLIENT_INITIALIZATION" }, { "end": 22, "start": 9, "type": "REQUEST_INITIALIZATION" }, { "end": 25, "start": 23, "type": "REQUEST_EXECUTION" }, { "end": 29, "start": 26, "type": "RESPONSE_HANDLING" }, ] } # This is the same as the sample_str above, minus the # [START ...] # and # [END ...] lines expected_full_snipppet = """from molluscs.v1 import molluscclient def sample_classify(video, location): # Create a client client = molluscclient.MolluscServiceClient() # Initialize request argument(s) classify_target = molluscclient.ClassifyTarget() # video = "path/to/mollusc/video.mkv" with open(video, "rb") as f: classify_target.video = f.read() # location = "New Zealand" classify_target.location_annotation = location request = molluscclient.molluscs.v1.ClassifyRequest( classify_target=classify_target, ) # Make the request response = client.classify(request=request) # Handle the response print(f"Mollusc is a \"{response.taxonomy}\"") """ assert snippet.full_snippet == expected_full_snipppet
def test_get_metadata_json(sample_str): snippet_metadata = snippet_metadata_pb2.Snippet() snippet_metadata.client_method.method.service.short_name = "Squid" snippet_metadata.client_method.method.short_name = "classify" snippet = snippet_index.Snippet(sample_str, snippet_metadata) index = snippet_index.SnippetIndex(api_schema=DummyApiSchema( services={ "Squid": DummyService(name="Squid", methods={"classify": DummyMethod()}) }, naming=DummyNaming(proto_package="google.mollusca", warehouse_package_name="google-mollusca", version="v1"), )) index.add_snippet(snippet) print(index.get_metadata_json()) assert json.loads(index.get_metadata_json()) == { "clientLibrary": { "apis": [{ "id": "google.mollusca", "version": "v1" }], "language": "PYTHON", "name": "google-mollusca" }, "snippets": [{ "clientMethod": { "method": { "service": { "shortName": "Squid" }, "shortName": "classify" } }, "segments": [{ "end": 28, "start": 2, "type": "FULL" }, { "end": 28, "start": 2, "type": "SHORT" }, { "end": 8, "start": 6, "type": "CLIENT_INITIALIZATION" }, { "end": 22, "start": 9, "type": "REQUEST_INITIALIZATION" }, { "end": 25, "start": 23, "type": "REQUEST_EXECUTION" }, { "end": 29, "start": 26, "type": "RESPONSE_HANDLING" }] }] }