def test_add_snippet_no_matching_service(sample_str):
snippet_metadata = snippet_metadata_pb2.Snippet()
snippet_metadata.client_method.method.service.short_name = "Clam"
snippet = snippet_index.Snippet(sample_str, snippet_metadata)
# No 'Clam' service in API Schema
index = snippet_index.SnippetIndex(api_schema=DummyApiSchema(
services={"Squid": DummyService(name="Squid", methods={})}))
with pytest.raises(types.UnknownService):
index.add_snippet(snippet)
def test_add_snippet_no_matching_rpc(sample_str):
snippet_metadata = snippet_metadata_pb2.Snippet()
snippet_metadata.client_method.method.service.short_name = "Squid"
snippet_metadata.client_method.short_name = "classify"
snippet = snippet_index.Snippet(sample_str, snippet_metadata)
# No 'classify' method in 'Squid' service
index = snippet_index.SnippetIndex(api_schema=DummyApiSchema(
services={"Squid": DummyService(name="Squid", methods={"list": None})
}))
with pytest.raises(types.RpcMethodNotFound):
index.add_snippet(snippet)
def test_get_metadata_json(sample_str):
snippet_metadata = snippet_metadata_pb2.Snippet()
snippet_metadata.client_method.method.service.short_name = "Squid"
snippet_metadata.client_method.method.short_name = "classify"
snippet = snippet_index.Snippet(sample_str, snippet_metadata)
index = snippet_index.SnippetIndex(api_schema=DummyApiSchema(
services={
"Squid":
DummyService(name="Squid", methods={"classify": DummyMethod()})
}))
index.add_snippet(snippet)
assert json.loads(index.get_metadata_json()) == {
'snippets': [{
'clientMethod': {
'method': {
'shortName': 'classify',
'service': {
'shortName': 'Squid'
}
}
},
'segments': [{
'end': 28,
'start': 2,
'type': 'FULL'
}, {
'end': 28,
'start': 2,
'type': 'SHORT'
}, {
'end': 8,
'start': 6,
'type': 'CLIENT_INITIALIZATION'
}, {
'end': 22,
'start': 9,
'type': 'REQUEST_INITIALIZATION'
}, {
'end': 25,
'start': 23,
'type': 'REQUEST_EXECUTION'
}, {
'end': 29,
'start': 26,
'type': 'RESPONSE_HANDLING'
}]
}]
}
def test_add_snippet_no_matching_service(sample_str):
snippet_metadata = snippet_metadata_pb2.Snippet()
snippet_metadata.client_method.method.service.short_name = "Clam"
snippet = snippet_index.Snippet(sample_str, snippet_metadata)
# No 'Clam' service in API Schema
index = snippet_index.SnippetIndex(api_schema=DummyApiSchema(
services={"Squid": DummyService(name="Squid", methods={})},
naming=DummyNaming(proto_package="google.mollusca",
warehouse_package_name="google-mollusca",
version="v1"),
))
with pytest.raises(types.UnknownService):
index.add_snippet(snippet)
def test_add_snippet_no_matching_rpc(sample_str):
snippet_metadata = snippet_metadata_pb2.Snippet()
snippet_metadata.client_method.method.service.short_name = "Squid"
snippet_metadata.client_method.short_name = "classify"
snippet = snippet_index.Snippet(sample_str, snippet_metadata)
# No 'classify' method in 'Squid' service
index = snippet_index.SnippetIndex(api_schema=DummyApiSchema(
services={"Squid": DummyService(name="Squid", methods={"list": None})},
naming=DummyNaming(proto_package="google.mollusca",
warehouse_package_name="google-mollusca",
version="v1"),
))
with pytest.raises(types.RpcMethodNotFound):
index.add_snippet(snippet)
def test_add_and_get_snippet_sync(sample_str):
snippet_metadata = snippet_metadata_pb2.Snippet()
snippet_metadata.client_method.method.service.short_name = "Squid"
snippet_metadata.client_method.method.short_name = "classify"
snippet = snippet_index.Snippet(sample_str, snippet_metadata)
index = snippet_index.SnippetIndex(api_schema=DummyApiSchema(
services={
"Squid":
DummyService(name="Squid", methods={"classify": DummyMethod()})
}))
index.add_snippet(snippet)
index.get_snippet(service_name="Squid", rpc_name="classify")
def test_add_and_get_snippet_sync(sample_str):
snippet_metadata = snippet_metadata_pb2.Snippet()
snippet_metadata.client_method.method.service.short_name = "Squid"
snippet_metadata.client_method.method.short_name = "classify"
snippet = snippet_index.Snippet(sample_str, snippet_metadata)
index = snippet_index.SnippetIndex(api_schema=DummyApiSchema(
services={
"Squid":
DummyService(name="Squid", methods={"classify": DummyMethod()})
},
naming=DummyNaming(proto_package="google.mollusca",
warehouse_package_name="google-mollusca",
version="v1"),
))
index.add_snippet(snippet)
index.get_snippet(service_name="Squid", rpc_name="classify")
def _generate_samples_and_manifest(
self, api_schema: api.API, index: snippet_index.SnippetIndex, sample_template: jinja2.Template, *, opts: Options) -> Tuple[Dict, snippet_index.SnippetIndex]:
"""Generate samples and samplegen manifest for the API.
Arguments:
api_schema (api.API): The schema for the API to which the samples belong.
sample_template (jinja2.Template): The template to use to generate samples.
opts (Options): Additional generator options.
Returns:
Tuple[Dict[str, CodeGeneratorResponse.File], snippet_index.SnippetIndex] : A dict mapping filepath to rendered file.
"""
# The two-layer data structure lets us do two things:
# * detect duplicate samples, which is an error
# * detect distinct samples with the same ID, which are disambiguated
id_to_hash_to_spec: DefaultDict[str,
Dict[str, Any]] = defaultdict(dict)
# Autogenerated sample specs
autogen_specs: typing.List[typing.Dict[str, Any]] = []
if opts.autogen_snippets:
autogen_specs = list(
samplegen.generate_sample_specs(api_schema, opts=opts))
# Also process any handwritten sample specs
handwritten_specs = samplegen.parse_handwritten_specs(
self._sample_configs)
sample_specs = autogen_specs + list(handwritten_specs)
for spec in sample_specs:
# Every sample requires an ID. This may be provided
# by a samplegen config author.
# If no ID is provided, fall back to the region tag.
#
# Ideally the sample author should pick a descriptive, unique ID,
# but this may be impractical and can be error-prone.
spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8]
sample_id = spec.get("id") or spec.get("region_tag") or spec_hash
spec["id"] = sample_id
hash_to_spec = id_to_hash_to_spec[sample_id]
if spec_hash in hash_to_spec:
raise DuplicateSample(
f"Duplicate samplegen spec found: {spec}")
hash_to_spec[spec_hash] = spec
out_dir = "samples/generated_samples"
fpath_to_spec_and_rendered = {}
for hash_to_spec in id_to_hash_to_spec.values():
for spec_hash, spec in hash_to_spec.items():
id_is_unique = len(hash_to_spec) == 1
# The ID is used to generate the file name. It must be globally unique.
if not id_is_unique:
spec["id"] += f"_{spec_hash}"
sample, snippet_metadata = samplegen.generate_sample(
spec, api_schema, sample_template,)
fpath = utils.to_snake_case(spec["id"]) + ".py"
fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = (
spec,
sample,
)
snippet_metadata.file = fpath
snippet_metadata.title = fpath
index.add_snippet(
snippet_index.Snippet(sample, snippet_metadata))
output_files = {
fname: CodeGeneratorResponse.File(
content=formatter.fix_whitespace(sample), name=fname
)
for fname, (_, sample) in fpath_to_spec_and_rendered.items()
}
if index.metadata_index.snippets:
# NOTE(busunkim): Not all fields are yet populated in the snippet metadata.
# Expected filename: snippet_metadata_{apishortname}_{apiversion}.json
snippet_metadata_path = str(pathlib.Path(
out_dir) / f"snippet_metadata_{api_schema.naming.name}_{api_schema.naming.version}.json").lower()
output_files[snippet_metadata_path] = CodeGeneratorResponse.File(
content=formatter.fix_whitespace(index.get_metadata_json()), name=snippet_metadata_path)
return output_files, index
def test_snippet_init(sample_str):
# We are not trying to exhaustively test the snippet metadata protobuf,
# just checking that fields are not unset
sample_metadata = snippet_metadata_pb2.Snippet(title="classify_squid.py")
sample_metadata.language = snippet_metadata_pb2.Language.PYTHON
snippet = snippet_index.Snippet(sample_str, sample_metadata)
assert snippet.sample_str == sample_str
# It's easier to eyeball diffs on the dictionary representation
assert json_format.MessageToDict(snippet.metadata) == {
"language":
"PYTHON",
"title":
"classify_squid.py",
"segments": [
{
"end": 28,
"start": 2,
"type": "FULL"
},
{
"end": 28,
"start": 2,
"type": "SHORT"
},
{
"end": 8,
"start": 6,
"type": "CLIENT_INITIALIZATION"
},
{
"end": 22,
"start": 9,
"type": "REQUEST_INITIALIZATION"
},
{
"end": 25,
"start": 23,
"type": "REQUEST_EXECUTION"
},
{
"end": 29,
"start": 26,
"type": "RESPONSE_HANDLING"
},
]
}
# This is the same as the sample_str above, minus the # [START ...]
# and # [END ...] lines
expected_full_snipppet = """from molluscs.v1 import molluscclient
def sample_classify(video, location):
# Create a client
client = molluscclient.MolluscServiceClient()
# Initialize request argument(s)
classify_target = molluscclient.ClassifyTarget()
# video = "path/to/mollusc/video.mkv"
with open(video, "rb") as f:
classify_target.video = f.read()
# location = "New Zealand"
classify_target.location_annotation = location
request = molluscclient.molluscs.v1.ClassifyRequest(
classify_target=classify_target,
)
# Make the request
response = client.classify(request=request)
# Handle the response
print(f"Mollusc is a \"{response.taxonomy}\"")
"""
assert snippet.full_snippet == expected_full_snipppet
def test_get_metadata_json(sample_str):
snippet_metadata = snippet_metadata_pb2.Snippet()
snippet_metadata.client_method.method.service.short_name = "Squid"
snippet_metadata.client_method.method.short_name = "classify"
snippet = snippet_index.Snippet(sample_str, snippet_metadata)
index = snippet_index.SnippetIndex(api_schema=DummyApiSchema(
services={
"Squid":
DummyService(name="Squid", methods={"classify": DummyMethod()})
},
naming=DummyNaming(proto_package="google.mollusca",
warehouse_package_name="google-mollusca",
version="v1"),
))
index.add_snippet(snippet)
print(index.get_metadata_json())
assert json.loads(index.get_metadata_json()) == {
"clientLibrary": {
"apis": [{
"id": "google.mollusca",
"version": "v1"
}],
"language": "PYTHON",
"name": "google-mollusca"
},
"snippets": [{
"clientMethod": {
"method": {
"service": {
"shortName": "Squid"
},
"shortName": "classify"
}
},
"segments": [{
"end": 28,
"start": 2,
"type": "FULL"
}, {
"end": 28,
"start": 2,
"type": "SHORT"
}, {
"end": 8,
"start": 6,
"type": "CLIENT_INITIALIZATION"
}, {
"end": 22,
"start": 9,
"type": "REQUEST_INITIALIZATION"
}, {
"end": 25,
"start": 23,
"type": "REQUEST_EXECUTION"
}, {
"end": 29,
"start": 26,
"type": "RESPONSE_HANDLING"
}]
}]
}
