v1_library = gapic.ruby_library(
'datastore',
'v1',
config_path='/google/datastore/artman_datastore.yaml',
artman_output_name='google-cloud-ruby/google-cloud-datastore')
s.copy(v1_library / 'lib/google/cloud/datastore/v1')
s.copy(v1_library / 'lib/google/datastore/v1')
# Omitting lib/google/cloud/datastore/v1.rb for now because we are not exposing
# the low-level API.
# Support for service_address
s.replace(
'lib/google/cloud/datastore/v*/*_client.rb',
'\n(\\s+)#(\\s+)@param exception_transformer',
'\n\\1#\\2@param service_address [String]\n' +
'\\1#\\2 Override for the service hostname, or `nil` to leave as the default.\n'
+ '\\1#\\2@param service_port [Integer]\n' +
'\\1#\\2 Override for the service port, or `nil` to leave as the default.\n'
+ '\\1#\\2@param exception_transformer')
s.replace(
'lib/google/cloud/datastore/v*/*_client.rb',
'\n(\\s+)metadata: nil,\n\\s+exception_transformer: nil,\n',
'\n\\1metadata: nil,\n\\1service_address: nil,\n\\1service_port: nil,\n\\1exception_transformer: nil,\n'
)
s.replace('lib/google/cloud/datastore/v*/*_client.rb',
'service_path = self\\.class::SERVICE_ADDRESS',
'service_path = service_address || self.class::SERVICE_ADDRESS')
s.replace('lib/google/cloud/datastore/v*/*_client.rb',
'port = self\\.class::DEFAULT_SERVICE_PORT',
'port = service_port || self.class::DEFAULT_SERVICE_PORT')
s.move(
library,
excludes=[
'docs/conf.py',
'docs/index.rst',
'google/cloud/bigquery_storage_v1beta1/__init__.py',
'README.rst',
'nox*.py',
'setup.py',
'setup.cfg',
],
)
s.replace(
['google/cloud/bigquery_storage_v1beta1/proto/storage_pb2.py',
'google/cloud/bigquery_storage_v1beta1/proto/storage_pb2_grpc.py'],
'from google.cloud.bigquery.storage_v1beta1.proto',
'from google.cloud.bigquery_storage_v1beta1.proto',
)
s.replace(
'google/cloud/bigquery_storage_v1beta1/gapic/'
'big_query_storage_client.py',
'google-cloud-bigquerystorage',
'google-cloud-bigquery-storage')
s.replace(
'google/cloud/bigquery_storage_v1beta1/gapic/'
'big_query_storage_client.py',
'import google.api_core.gapic_v1.method\n',
'\g<0>import google.api_core.path_template\n'
)
# Generate admin client
library = gapic.py_library(
"bigtable_admin",
"v2",
config_path="/google/bigtable/admin/artman_bigtableadmin.yaml",
artman_output_name="bigtable-admin-v2",
include_protos=True,
)
s.move(library / "google/cloud/bigtable_admin_v2")
s.move(library / "tests")
s.replace(
[
"google/cloud/bigtable_admin_v2/gapic/bigtable_instance_admin_client.py",
"google/cloud/bigtable_admin_v2/gapic/bigtable_table_admin_client.py",
],
"'google-cloud-bigtable-admin'",
"'google-cloud-bigtable'",
)
s.replace(
"google/**/*.py",
"from google\.cloud\.bigtable\.admin_v2.proto",
"from google.cloud.bigtable_admin_v2.proto",
)
s.replace(
[
"google/cloud/bigtable_admin_v2/gapic/transports/"
"bigtable_table_admin_grpc_transport.py",
"google/cloud/bigtable_v2/gapic/transports/bigtable_grpc_transport.py",
artman_output_name='google-cloud-ruby/google-cloud-cloudscheduler',
)
s.copy(v1_library / 'lib/google/cloud/scheduler/v1')
s.copy(v1_library / 'lib/google/cloud/scheduler/v1.rb')
s.copy(v1_library / 'lib/google/cloud/scheduler.rb')
s.copy(v1_library / 'test/google/cloud/scheduler/v1')
s.copy(v1_library / 'README.md')
s.copy(v1_library / 'LICENSE')
s.copy(v1_library / '.gitignore')
s.copy(v1_library / '.yardopts')
s.copy(v1_library / 'google-cloud-scheduler.gemspec', merge=ruby.merge_gemspec)
s.replace(
'google-cloud-scheduler.gemspec',
'gem.add_development_dependency "rubocop".*$',
'gem.add_development_dependency "rubocop", "~> 0.64.0"'
)
# https://github.com/googleapis/gapic-generator/issues/2279
s.replace(
'lib/**/*.rb',
'\\A(((#[^\n]*)?\n)*# (Copyright \\d+|Generated by the protocol buffer compiler)[^\n]+\n(#[^\n]*\n)*\n)([^\n])',
'\\1\n\\6')
# https://github.com/googleapis/gapic-generator/issues/2323
s.replace(
['lib/**/*.rb', 'README.md'],
'https://github\\.com/GoogleCloudPlatform/google-cloud-ruby',
'https://github.com/googleapis/google-cloud-ruby'
)
v1beta1_library = gapic.ruby_library(
'asset', 'v1beta1', artman_output_name='google-cloud-ruby/google-cloud-asset',
config_path='artman_cloudasset_v1beta1.yaml'
)
s.copy(v1beta1_library / 'lib/google/cloud/asset/v1beta1.rb')
s.copy(v1beta1_library / 'lib/google/cloud/asset/v1beta1')
s.copy(v1beta1_library / 'test/google/cloud/asset/v1beta1')
# Copy common templates
templates = gcp.CommonTemplates().ruby_library()
s.copy(templates)
# https://github.com/googleapis/gapic-generator/issues/2180
s.replace(
'google-cloud-asset.gemspec',
'\n gem\\.add_dependency "google-gax", "~> ([\\d\\.]+)"\n\n',
'\n gem.add_dependency "google-gax", "~> \\1"\n gem.add_dependency "grpc-google-iam-v1", "~> 0.6.9"\n\n')
# https://github.com/googleapis/gapic-generator/issues/2232
s.replace(
'lib/google/cloud/asset/**/asset_service_client.rb',
'\n\n(\\s+)class OperationsClient < Google::Longrunning::OperationsClient',
'\n\n\\1# @private\n\\1class OperationsClient < Google::Longrunning::OperationsClient')
# https://github.com/googleapis/gapic-generator/issues/2242
def escape_braces(match):
expr = re.compile('^([^`]*(`[^`]*`[^`]*)*)([^`#\\$\\\\])\\{([\\w,]+)\\}')
content = match.group(0)
while True:
gapic = gcp.GAPICGenerator()
version = 'v1'
library = gapic.py_library(
'pubsub', version, config_path='/google/pubsub/artman_pubsub.yaml')
s.move(
library,
excludes=[
'docs/**/*', 'nox.py', 'README.rst', 'setup.py',
'google/cloud/pubsub_v1/__init__.py', 'google/cloud/pubsub_v1/types.py'])
# Adjust tests to import the clients directly.
s.replace(
'tests/unit/gapic/v1/test_publisher_client_v1.py',
'from google.cloud import pubsub_v1',
'from google.cloud.pubsub_v1.gapic import publisher_client')
s.replace(
'tests/unit/gapic/v1/test_publisher_client_v1.py',
' pubsub_v1',
' publisher_client')
s.replace(
'tests/unit/gapic/v1/test_subscriber_client_v1.py',
'from google.cloud import pubsub_v1',
'from google.cloud.pubsub_v1.gapic import subscriber_client')
s.replace(
'tests/unit/gapic/v1/test_subscriber_client_v1.py',
' pubsub_v1',
for version in versions:
library = gapic.py_library(
"translate",
version,
include_protos=True,
)
#s.move(library / f'google/cloud/translation_{version}', f'google/cloud/translate_{version}', excludes=excludes)
s.move(library / f'google/cloud/translate_{version}', excludes=excludes)
s.move(library / 'tests')
s.move(library / f"docs/gapic/{version}")
# translation -> translate
s.replace(
"google/**/translation_service_pb2_grpc.py",
"google.cloud.translation_v3beta1.proto",
"google.cloud.translate_v3beta1.proto",
)
s.replace(
"google/cloud/**/translation_service_pb2.py",
r"""record delimiters are ':raw-latex:`\\n`' instead of
':raw-latex:`\\r`:raw-latex:`\\n`'.""",
r"""record delimiters are ``\\\\\\\\n`` instead of
``\\\\\\\\r\\\\\\\\n``.""",)
# ----------------------------------------------------------------------------
# Add templated files
# ----------------------------------------------------------------------------
templated_files = gcp.CommonTemplates().py_library(unit_cov_level=100, cov_level=100)
s.move(templated_files)
v1beta1_library = gapic.ruby_library(
'firestore',
'v1beta1',
config_path='/google/firestore/artman_firestore.yaml',
artman_output_name='google-cloud-ruby/google-cloud-firestore')
s.copy(v1beta1_library / 'lib/google/cloud/firestore/v1beta1')
s.copy(v1beta1_library / 'lib/google/cloud/firestore/v1beta1.rb')
s.copy(v1beta1_library / 'lib/google/firestore/v1beta1')
s.copy(v1beta1_library / 'test/google/cloud/firestore/v1beta1')
# PERMANENT: Handwritten layer owns Firestore.new so low-level clients need to
# use Firestore::V1beta1.new instead of Firestore.new(version: :v1beta1).
# Update the examples and tests.
s.replace([
'lib/google/cloud/firestore/v1beta1/firestore_client.rb',
'test/google/cloud/firestore/v1beta1/firestore_client_test.rb'
], 'require "google/cloud/firestore"',
'require "google/cloud/firestore/v1beta1"')
s.replace([
'lib/google/cloud/firestore/v1beta1/firestore_client.rb',
'test/google/cloud/firestore/v1beta1/firestore_client_test.rb'
], 'Google::Cloud::Firestore\\.new\\(version: :v1beta1\\)',
'Google::Cloud::Firestore::V1beta1.new')
s.replace([
'lib/google/cloud/firestore/v1/firestore_client.rb',
'test/google/cloud/firestore/v1/firestore_client_test.rb'
], 'require "google/cloud/firestore"', 'require "google/cloud/firestore/v1"')
s.replace([
'lib/google/cloud/firestore/v1/firestore_client.rb',
'test/google/cloud/firestore/v1/firestore_client_test.rb'
], 'Google::Cloud::Firestore\\.new\\(version: :v1\\)',
# ----------------------------------------------------------------------------
# Generate asset GAPIC layer
# ----------------------------------------------------------------------------
for version in versions:
library = gapic.py_library("translate", version, include_protos=True)
# s.move(library / f'google/cloud/translation_{version}', f'google/cloud/translate_{version}', excludes=excludes)
s.move(library / f"google/cloud/translate_{version}", excludes=excludes)
s.move(library / "tests")
s.move(library / f"docs/gapic/{version}")
# translation -> translate
s.replace(
"google/**/translation_service_pb2_grpc.py",
f"google.cloud.translation_{version}.proto",
f"google.cloud.translate_{version}.proto",
)
# Use the highest version library to generate documentation import alias.
s.move(library / "google/cloud/translate.py")
s.replace(
"google/cloud/**/translation_service_pb2.py",
r"""record delimiters are ':raw-latex:`\\n`' instead of
':raw-latex:`\\r`:raw-latex:`\\n`'.""",
r"""record delimiters are ``\\\\\\\\n`` instead of
``\\\\\\\\r\\\\\\\\n``.""",
)
# Fix docstring issue for classes with no summary line
artman_output_name='google-cloud-ruby/google-cloud-dialogflow')
s.copy(v2_library / 'lib')
s.copy(v2_library / 'test')
s.copy(v2_library / 'Rakefile')
s.copy(v2_library / 'README.md')
s.copy(v2_library / 'LICENSE')
s.copy(v2_library / '.gitignore')
s.copy(v2_library / '.yardopts')
s.copy(v2_library / 'google-cloud-dialogflow.gemspec', merge=merge_gemspec)
# https://github.com/googleapis/gapic-generator/issues/2232
s.replace(
[
'lib/google/cloud/dialogflow/v2/agents_client.rb',
'lib/google/cloud/dialogflow/v2/entity_types_client.rb',
'lib/google/cloud/dialogflow/v2/intents_client.rb'
],
'\n\n(\\s+)class OperationsClient < Google::Longrunning::OperationsClient',
'\n\n\\1# @private\n\\1class OperationsClient < Google::Longrunning::OperationsClient'
)
# https://github.com/googleapis/gapic-generator/issues/2243
s.replace('lib/google/cloud/dialogflow/*/*_client.rb',
'(\n\\s+class \\w+Client\n)(\\s+)(attr_reader :\\w+_stub)',
'\\1\\2# @private\n\\2\\3')
# https://github.com/googleapis/gapic-generator/issues/2278
s.replace('Rakefile',
'\ndesc[^\n]+\ntask :jsondoc [^\n]+\n+( [^\n]+\n+)*end\n', '')
s.replace(
'Rakefile',
logging.basicConfig(level=logging.DEBUG)
gapic = gcp.GAPICGenerator()
common = gcp.CommonTemplates()
# ----------------------------------------------------------------------------
# Generate dlp GAPIC layer
# ----------------------------------------------------------------------------
library = gapic.py_library(
"dlp", "v2", config_path="/google/privacy/dlp/artman_dlp_v2.yaml")
excludes = ["README.rst", "nox.py", "setup.py", "docs/index.rst"]
s.move(library, excludes=excludes)
# Fix namespace
s.replace("google/**/*.py", "google\.cloud\.privacy\.dlp_v2",
"google.cloud.dlp_v2")
# Add missing utf-8 marker
s.replace(
"google/cloud/dlp_v2/proto/dlp_pb2.py",
"# Generated by the protocol buffer compiler. DO NOT EDIT!",
"# -*- coding: utf-8 -*-\n\g<0>",
)
# Fix unindentation of bullet list second line
s.replace(
"google/cloud/dlp_v2/gapic/dlp_service_client.py",
"( \* .*\n )([^\s*])",
"\g<1> \g<2>",
)
# ----------------------------------------------------------------------------
library = gapic.py_library("billing", "v1")
excludes = ["setup.py", "docs/index.rst"]
s.move(library, excludes=excludes)
# correct license headers
python.fix_pb2_headers()
python.fix_pb2_grpc_headers()
# ----------------------------------------------------------------------------
# Add templated files
# ----------------------------------------------------------------------------
templated_files = common.py_library(cov_level=100)
s.move(templated_files,
excludes=[".coveragerc"
]) # the microgenerator has a good coveragerc file
s.replace(".gitignore", "bigquery/docs/generated",
"htmlcov") # temporary hack to ignore htmlcov
# Remove 2.7 and 3.5 tests from noxfile.py
s.replace("noxfile.py", '''\["2\.7", ''', '[')
s.replace("noxfile.py", '''"3.5", ''', '')
# Expand flake errors permitted to accomodate the Microgenerator
# TODO: remove extra error codes once issues below are resolved
# https://github.com/googleapis/gapic-generator-python/issues/425
s.replace(".flake8", "(ignore = .*)", "\g<1>, F401, F841")
s.shell.run(["nox", "-s", "blacken"], hide_output=False)
s.move(library / f"google/cloud/speech_{version}/gapic")
s.move(library / f"google/cloud/speech_{version}/proto")
s.move(library / f"tests/unit/gapic/{version}")
s.move(library / f"docs/gapic/{version}")
s.move(library / f"samples")
# Use the highest version library to generate documentation import alias.
s.move(library / "google/cloud/speech.py")
# Fix tests to use the direct gapic client instead of the wrapped helper
# client.
s.replace(
"tests/unit/**/test*client*.py",
r"from google\.cloud import speech_(.+?)$",
r"from google.cloud.speech_\1.gapic import speech_client as speech_\1",
)
# Fix bad docstring
s.replace(
"google/**/resource_pb2.py",
"""``\\\\ ``e\.g\.``\\\\
\$MONTH\\\\ ``\.""",
"""``\ ``e.g.``\$MONTH\ ``."""
)
s.replace(
"google/**/resource_pb2.py",
"""\(e\.g\. ``\\\\ \{my-
s.copy(
spanner_admin_database,
excludes=["src/v1/index.js", "src/index.js", "README.md", "package.json"])
s.copy(
spanner_admin_instance,
excludes=["src/v1/index.js", "src/index.js", "README.md", "package.json"])
common_templates = gcp.CommonTemplates()
templates = common_templates.node_library(source_location='build/src')
s.copy(templates)
# nodejs-spanner is composed of 3 APIs: SpannerClient, SpannerAdminDatabase and
# SpannerAdminInstance, export all 3 in src/v1/index.js
s.replace(
"src/v1/index.js",
"(const SpannerClient = require\('\./spanner_client\'\);)",
"""const DatabaseAdminClient = require('./database_admin_client');
const InstanceAdminClient = require('./instance_admin_client');
\g<1>""")
s.replace(
"src/v1/index.js", "(module\.exports\.SpannerClient = SpannerClient;)",
"""module.exports.DatabaseAdminClient = DatabaseAdminClient;
module.exports.InstanceAdminClient = InstanceAdminClient;
\g<1>""")
# Update path discovery due to build/ dir and TypeScript conversion.
s.replace("src/v1/database_admin_client.js", "../../package.json",
"../../../package.json")
s.replace("src/v1/instance_admin_client.js", "../../package.json",
"../../../package.json")
s.replace("src/v1/spanner_client.js", "../../package.json",
# ----------------------------------------------------------------------------
for version in ["v1", "v2"]:
library = gapic.py_library(
"trace",
version,
config_path=f"/google/devtools/cloudtrace" f"/artman_cloudtrace_{version}.yaml",
artman_output_name=f"trace-{version}",
)
s.move(library / f"google/cloud/trace_{version}")
s.move(library / f"tests/unit/gapic/{version}")
# Fix up imports
s.replace(
"google/**/*.py",
f"from google.devtools.cloudtrace_{version}.proto import ",
f"from google.cloud.trace_{version}.proto import ",
)
# Issues exist where python files should define the source encoding
# https://github.com/googleapis/gapic-generator/issues/2097
s.replace("google/**/proto/*_pb2.py", r"(^.*$\n)*", r"# -*- coding: utf-8 -*-\n\g<0>")
# ----------------------------------------------------------------------------
# Add templated files
# ----------------------------------------------------------------------------
templated_files = common.py_library(unit_cov_level=97, cov_level=100)
s.move(templated_files)
s.shell.run(["nox", "-s", "blacken"], hide_output=False)
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This script is used to synthesize generated parts of this library."""
import synthtool as s
import synthtool.gcp as gcp
import synthtool.languages.ruby as ruby
import logging
logging.basicConfig(level=logging.DEBUG)
gapic = gcp.GAPICBazel()
library = gapic.ruby_library(
"automl", "v1beta1",
proto_path="google/cloud/automl/v1beta1",
bazel_target="//google/cloud/automl/v1beta1:google-cloud-automl-v1beta1-ruby",
)
s.copy(library, merge=ruby.global_merge)
# Fixes for some misformatted markdown links.
# See internal issue b/153077040.
s.replace(
"proto_docs/google/cloud/automl/v1beta1/io.rb",
"https:\n\\s+# //",
"https://")
# ----------------------------------------------------------------------------
for version in versions:
library = gapic.py_library("automl", version, include_protos=True)
s.move(library / f"google/cloud/automl_{version}")
s.move(library / f"tests/unit/gapic/{version}")
s.move(library / f"docs/gapic/{version}")
s.move(library / f"docs/conf.py")
# Use the highest version library to generate import alias.
s.move(library / "google/cloud/automl.py")
# Fixup issues in generated code
s.replace(
"**/gapic/*_client.py",
r"metadata_type=operations_pb2.OperationMetadata",
r"metadata_type=proto_operations_pb2.OperationMetadata",
)
# Fix spacing/'::' issues in docstrings
s.replace(
"google/cloud/automl_v1beta1/gapic/prediction_service_client.py", "^\s+::", ""
)
s.replace(
"google/cloud/automl_v1beta1/gapic/auto_ml_client.py",
"^(\s+)(::)\n\n\s+?([^\s])",
" \g<1>\g<2>\n \g<1>\g<3>",
)
# Remove 'raw-latex' sections with sample JSON Lines files
# limitations under the License.
"""This script is used to synthesize generated parts of this library."""
import re
import synthtool as s
from synthtool import gcp
from synthtool.languages import python
common = gcp.CommonTemplates()
default_version = "v1"
for library in s.get_staging_dirs(default_version):
# Fix docstring with regex pattern that breaks docgen
s.replace(library / "google/**/*client.py", "(/\^.*\$/)", "``\g<1>``")
# Fix more regex in docstrings
s.replace(library / "google/**/types/*.py",
"(regex\s+)(/.*?/)\.",
"\g<1>``\g<2>``.",
flags=re.MULTILINE | re.DOTALL,
)
# Fix docstring with JSON example by wrapping with backticks
s.replace(library / "google/**/types/recommendation.py",
"( - Example: )(\{.*?\})",
"\g<1>``\g<2>``",
flags=re.MULTILINE | re.DOTALL,
)
from synthtool import gcp
gapic = gcp.GAPICGenerator()
common = gcp.CommonTemplates()
versions = ["v1beta2", "v1"]
# ----------------------------------------------------------------------------
# Generate dataproc GAPIC layer
# ----------------------------------------------------------------------------
for version in versions:
library = gapic.py_library("dataproc", version)
s.move(library, excludes=["docs/index.rst", "nox.py", "README.rst", "setup.py"])
s.replace(
f"google/cloud/dataproc_{version}/gapic/cluster_controller_client.py",
"metadata_type=operations_pb2.ClusterOperationMetadata,",
"metadata_type=proto_operations_pb2.ClusterOperationMetadata,",
)
s.replace(
f"google/cloud/dataproc_{version}/gapic/cluster_controller_client.py",
"\s+<strong>Note:</strong>.*\n(.*\n)+?.*types.FieldMask.",
f"""
.. note::
Currently, only the following fields can be updated:
* ``labels``: Update labels
* ``config.worker_config.num_instances``: Resize primary
version,
config_path="/google/pubsub/artman_pubsub.yaml")
# skip index, protos, package.json, and README.md
s.copy(library, excludes=['package.json', 'README.md', 'src/index.js'])
templates = common_templates.node_library(source_location='build/src')
s.copy(templates)
# https://github.com/googleapis/gapic-generator/issues/2127
s.replace(
"src/v1/subscriber_client.js",
" }\n\s*/\*\*\n\s+\* The DNS address for this API service.",
"\n // note: editing generated code\n"
" this.waitForReady = function(deadline, callback) {\n"
" return subscriberStub.then(\n"
" stub => stub.waitForReady(deadline, callback),\n"
" callback\n"
" );\n"
" };\n"
"\g<0>")
# Update path discovery due to build/ dir and TypeScript conversion.
s.replace("src/v1/publisher_client.js", "../../package.json",
"../../../package.json")
s.replace("src/v1/subscriber_client.js", "../../package.json",
"../../../package.json")
# Node.js specific cleanup
subprocess.run(['npm', 'install'])
subprocess.run(['npm', 'run', 'fix'])
s.copy(v1_library / 'acceptance')
s.copy(v1_library / 'lib')
s.copy(v1_library / 'test')
s.copy(v1_library / 'README.md')
s.copy(v1_library / 'LICENSE')
s.copy(v1_library / '.gitignore')
s.copy(v1_library / '.yardopts')
s.copy(v1_library / 'google-cloud-bigquery-data_transfer.gemspec', merge=ruby.merge_gemspec)
# Copy common templates
templates = gcp.CommonTemplates().ruby_library()
s.copy(templates)
# PERMANENT: Use custom credentials env variable names
s.replace(
'lib/google/cloud/bigquery/data_transfer/v1/credentials.rb',
'BIGQUERYDATATRANSFER_KEYFILE', 'DATA_TRANSFER_KEYFILE')
s.replace(
'lib/google/cloud/bigquery/data_transfer/v1/credentials.rb',
'BIGQUERYDATATRANSFER_CREDENTIALS', 'DATA_TRANSFER_CREDENTIALS')
# https://github.com/googleapis/gapic-generator/issues/2179
# https://github.com/googleapis/gapic-generator/issues/2196
s.replace(
[
'README.md',
'lib/google/cloud/bigquery/data_transfer.rb',
'lib/google/cloud/bigquery/data_transfer/v1.rb'
],
'\\[Product Documentation\\]: https://cloud\\.google\\.com/bigquerydatatransfer\n',
'[Product Documentation]: https://cloud.google.com/bigquery/transfer/\n')
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This script is used to synthesize generated parts of this library."""
import os
import synthtool as s
import synthtool.gcp as gcp
from synthtool.languages import python
common = gcp.CommonTemplates()
default_version = "v1"
for library in s.get_staging_dirs(default_version):
excludes = ["setup.py", "README.rst", "docs/index.rst", "scripts/fixup*"]
s.move(library, excludes=excludes)
s.remove_staging_dirs()
# ----------------------------------------------------------------------------
# Add templated files
# ----------------------------------------------------------------------------
templated_files = common.py_library(cov_level=98, microgenerator=True)
python.py_samples(skip_readmes=True)
s.move(templated_files, excludes=[".coveragerc"]) # the microgenerator has a good coveragerc file
s.replace(".gitignore", "bigquery/docs/generated", "htmlcov") # temporary hack to ignore htmlcov
s.shell.run(["nox", "-s", "blacken"], hide_output=False)
v1p3beta1 = gapic.ruby_library(
'vision', 'v1p3beta1',
artman_output_name='google-cloud-ruby/google-cloud-vision'
)
s.copy(v1p3beta1 / 'lib/google/cloud/vision/v1p3beta1')
s.copy(v1p3beta1 / 'lib/google/cloud/vision/v1p3beta1.rb')
s.copy(v1p3beta1 / 'acceptance/google/cloud/vision/v1p3beta1')
s.copy(v1p3beta1 / 'test/google/cloud/vision/v1p3beta1')
# PERMANENT: Add migration guide to docs
s.replace(
'lib/google/cloud/vision.rb',
'# ### Preview',
dedent("""\
# ### Migration Guide
#
# The 0.32.0 release introduced breaking changes relative to the previous
# release, 0.31.0. For more details and instructions to migrate your code,
# please visit the [migration
# guide](https://cloud.google.com/vision/docs/ruby-client-migration).
#
# ### Preview"""))
# PERMANENT: Add migration guide to readme
s.replace(
'README.md',
'### Preview\n',
dedent("""\
### Migration Guide
The 0.32.0 release introduced breaking changes relative to the previous release,
0.31.0. For more details and instructions to migrate your code, please visit the
library = gapic.php_library(service="servicedirectory", version="v1beta1")
# copy all src
s.move(library / f"src/V1beta1")
# copy proto files to src also
s.move(library / f"proto/src/Google/Cloud/ServiceDirectory", f"src/")
s.move(library / f"tests/")
# copy GPBMetadata file to metadata
s.move(library / f'proto/src/GPBMetadata/Google/Cloud/Servicedirectory',
f"metadata/")
# document and utilize apiEndpoint instead of serviceAddress
s.replace("**/Gapic/*GapicClient.php", r"serviceAddress =>", r"apiEndpoint =>")
s.replace(
"**/Gapic/*GapicClient.php",
r"@type string \$serviceAddress\n\s+\*\s+The address",
r"""@type string $serviceAddress
* **Deprecated**. This option will be removed in a future major release. Please
* utilize the `$apiEndpoint` option instead.
* @type string $apiEndpoint
* The address""")
s.replace("**/Gapic/*GapicClient.php",
r"\$transportConfig, and any \$serviceAddress",
r"$transportConfig, and any `$apiEndpoint`")
# fix year
s.replace("src/**/**/*.php", r"Copyright d{4}", r"Copyright 2020)")
s.replace("tests/**/**/*Test.php", r"Copyright d{4}", r"Copyright 2020")
v1beta2 = gapic.ruby_library(
'dataproc', 'v1beta2', config_path='/google/cloud/dataproc/artman_dataproc_v1beta2.yaml',
artman_output_name='google-cloud-ruby/google-cloud-dataproc'
)
s.copy(v1beta2 / 'lib/google/cloud/dataproc/v1beta2')
s.copy(v1beta2 / 'lib/google/cloud/dataproc/v1beta2.rb')
s.copy(v1beta2 / 'acceptance/google/cloud/dataproc/v1beta2')
s.copy(v1beta2 / 'test/google/cloud/dataproc/v1beta2')
# Use v1beta2 version of dataproc.rb because it includes services not found in
# v1. Need to change the default version back to v1.
s.copy(v1beta2 / 'lib/google/cloud/dataproc.rb')
s.replace(
'lib/google/cloud/dataproc.rb',
':v1beta2',
':v1'
)
# Copy common templates
templates = gcp.CommonTemplates().ruby_library()
s.copy(templates)
# https://github.com/googleapis/gapic-generator/issues/2242
def escape_braces(match):
expr = re.compile('^([^`]*(`[^`]*`[^`]*)*)([^`#\\$\\\\])\\{([\\w,]+)\\}')
content = match.group(0)
while True:
content, count = expr.subn('\\1\\3\\\\\\\\{\\4}', content)
if count == 0:
return content
'logging',
'v2',
config_path='/google/logging/artman_logging.yaml',
artman_output_name='google-cloud-ruby/google-cloud-logging')
s.copy(v2_library / 'lib/google/cloud/logging/v2')
s.copy(v2_library / 'lib/google/logging/v2')
# Omitting lib/google/cloud/logging/v2.rb for now because we are not exposing
# the low-level API.
# PERMANENT: Handwritten layer owns Logging.new so low-level clients need to
# use Logging::V2.new instead of Logging.new(version: :v2). Update the
# examples and tests.
s.replace([
'lib/google/cloud/logging/v2/config_service_v2_client.rb',
'lib/google/cloud/logging/v2/logging_service_v2_client.rb',
'lib/google/cloud/logging/v2/metrics_service_v2_client.rb'
], 'require "google/cloud/logging"', 'require "google/cloud/logging/v2"')
s.replace('lib/google/cloud/logging/v2/config_service_v2_client.rb',
'Google::Cloud::Logging::Config\\.new\\(version: :v2\\)',
'Google::Cloud::Logging::V2::ConfigServiceV2Client.new')
s.replace('lib/google/cloud/logging/v2/logging_service_v2_client.rb',
'Google::Cloud::Logging::Logging\\.new\\(version: :v2\\)',
'Google::Cloud::Logging::V2::LoggingServiceV2Client.new')
s.replace('lib/google/cloud/logging/v2/metrics_service_v2_client.rb',
'Google::Cloud::Logging::Metrics\\.new\\(version: :v2\\)',
'Google::Cloud::Logging::V2::MetricsServiceV2Client.new')
# Support for service_address
s.replace(
'lib/google/cloud/logging/v*/*_client.rb',
)
s.copy(v1beta2_library / 'lib/google/cloud/language/v1beta2')
s.copy(v1beta2_library / 'lib/google/cloud/language/v1beta2.rb')
s.copy(v1beta2_library / 'lib/google/cloud/language/v1beta2')
s.copy(v1beta2_library / 'test/google/cloud/language/v1beta2')
# Copy common templates
templates = gcp.CommonTemplates().ruby_library()
s.copy(templates)
# https://github.com/googleapis/gapic-generator/issues/2196
s.replace(
[
'README.md',
'lib/google/cloud/language.rb',
'lib/google/cloud/language/v1.rb',
'lib/google/cloud/language/v1beta2.rb'
],
'\\[Product Documentation\\]: https://cloud\\.google\\.com/language\n',
'[Product Documentation]: https://cloud.google.com/natural-language\n')
# https://github.com/googleapis/gapic-generator/issues/2243
s.replace(
'lib/google/cloud/language/*/*_client.rb',
'(\n\\s+class \\w+Client\n)(\\s+)(attr_reader :\\w+_stub)',
'\\1\\2# @private\n\\2\\3')
# https://github.com/googleapis/gapic-generator/issues/2279
s.replace(
'lib/**/*.rb',
'\\A(((#[^\n]*)?\n)*# (Copyright \\d+|Generated by the protocol buffer compiler)[^\n]+\n(#[^\n]*\n)*\n)([^\n])',
config_path="/google/cloud/iot/"
f"artman_cloudiot.yaml")
# skip index, protos, package.json, and README.md
s.copy(
library,
excludes=['package.json', 'README.md', 'src/index.js'],
)
# Streaming is broken and missing grpc handling
# https://github.com/googleapis/gapic-generator/issues/2152
s.replace(
'src/v1/device_manager_client.js',
f'(listDeviceRegistriesStream\(.*\n\s+options = .*\n)(\n\s+return)',
'''\g<1>options.otherArgs = options.otherArgs || {};
options.otherArgs.headers = options.otherArgs.headers || {};
options.otherArgs.headers[
'x-goog-request-params'
] = gax.routingHeader.fromParams({
parent: request.parent,
});\n\g<2>''')
# Copy common templated files
common_templates = gcp.CommonTemplates()
templates = common_templates.node_library()
s.copy(templates)
# Node.js specific cleanup
subprocess.run(['npm', 'install'])
subprocess.run(['npm', 'run', 'fix'])
# Don't move over __init__.py, as we modify it to make the generated client
# use helpers.py.
s.move(library / f"google/cloud/speech_{version}/types.py")
s.move(library / f"google/cloud/speech_{version}/gapic")
s.move(library / f"google/cloud/speech_{version}/proto")
s.move(library / f"tests/unit/gapic/{version}")
s.move(library / f"docs/gapic/{version}")
# Use the highest version library to generate documentation import alias.
s.move(library / "google/cloud/speech.py")
# Issues exist where python files should define the source encoding
# https://github.com/googleapis/gapic-generator/issues/2097
s.replace("**/proto/*_pb2.py", r"(^.*$\n)*", r"# -*- coding: utf-8 -*-\n\g<0>")
# Fix tests to use the direct gapic client instead of the wrapped helper
# client.
s.replace(
"tests/unit/**/test*client*.py",
r"from google\.cloud import speech_(.+?)$",
r"from google.cloud.speech_\1.gapic import speech_client as speech_\1",
)
# ----------------------------------------------------------------------------
# Add templated files
# ----------------------------------------------------------------------------
templated_files = common.py_library(unit_cov_level=97, cov_level=100)
s.move(templated_files)
version='v2',
config_path='/google/logging/artman_logging.yaml',
artman_output_name='google-cloud-logging-v2')
# copy all src including partial veneer classes
s.move(library / 'src')
# copy proto files to src also
s.move(library / 'proto/src/Google/Cloud/Logging', 'src/')
s.move(library / 'tests/')
# copy GPBMetadata file to metadata
s.move(library / 'proto/src/GPBMetadata/Google/Logging', 'metadata/')
# document and utilize apiEndpoint instead of serviceAddress
s.replace("**/Gapic/*GapicClient.php", r"'serviceAddress' =>",
r"'apiEndpoint' =>")
s.replace(
"**/Gapic/*GapicClient.php",
r"@type string \$serviceAddress\n\s+\*\s+The address",
r"""@type string $serviceAddress
* **Deprecated**. This option will be removed in a future major release. Please
* utilize the `$apiEndpoint` option instead.
* @type string $apiEndpoint
* The address""")
s.replace("**/Gapic/*GapicClient.php",
r"\$transportConfig, and any \$serviceAddress",
r"$transportConfig, and any `$apiEndpoint`")
# fix year
s.replace('**/Gapic/*GapicClient.php', r'Copyright \d{4}', r'Copyright 2016')
for client in ['ConfigServiceV2', 'LoggingServiceV2', 'MetricsServiceV2']:
config_path='/google/devtools/artman_clouddebugger.yaml',
artman_output_name='google-cloud-ruby/google-cloud-debugger'
)
s.copy(v2_library / 'lib/google/cloud/debugger/v2')
s.copy(v2_library / 'lib/google/cloud/debugger/v2.rb')
s.copy(v2_library / 'lib/google/devtools')
s.copy(v2_library / 'test/google/cloud/debugger/v2')
# PERMANENT: Handwritten layer owns Debugger.new so low-level clients need to
# use Debugger::V2.new instead of Debugger.new(version: :v2). Update the
# examples and tests.
s.replace(
[
'lib/google/cloud/debugger/v2/controller2_client.rb',
'lib/google/cloud/debugger/v2/debugger2_client.rb',
'test/google/cloud/debugger/v2/controller2_client_test.rb',
'test/google/cloud/debugger/v2/debugger2_client_test.rb'
],
'require "google/cloud/debugger"',
'require "google/cloud/debugger/v2"')
s.replace(
[
'lib/google/cloud/debugger/v2/controller2_client.rb',
'test/google/cloud/debugger/v2/controller2_client_test.rb'
],
'Google::Cloud::Debugger::Controller2\\.new\\(version: :v2\\)',
'Google::Cloud::Debugger::V2::Controller2.new')
s.replace(
[
'lib/google/cloud/debugger/v2/debugger2_client.rb',
'test/google/cloud/debugger/v2/debugger2_client_test.rb'
common = gcp.CommonTemplates()
library = gapic.php_library(service='pubsub',
version='v1',
config_path='/google/pubsub/artman_pubsub.yaml',
artman_output_name='google-cloud-pubsub-v1')
# copy all src including partial veneer classes
s.move(library / 'src')
# copy proto files to src also
s.move(library / 'proto/src/Google/Cloud/PubSub', 'src/')
s.move(library / 'tests/')
# copy GPBMetadata file to metadata
s.move(library / 'proto/src/GPBMetadata/Google/Pubsub', 'metadata/')
# fix year
s.replace('**/Gapic/*GapicClient.php', r'Copyright \d{4}', 'Copyright 2016')
for client in ['Publisher', 'Subscriber']:
s.replace(f'**/V1/{client}Client.php', r'Copyright \d{4}',
'Copyright 2016')
s.replace(f'**/V1/{client}GrpcClient.php', r'Copyright \d{4}',
'Copyright 2017')
s.replace('tests/**/V1/*Test.php', r'Copyright \d{4}', 'Copyright 2018')
# fix the link to the official doc
s.replace('src/**/*.php', r'<a href="/pubsub/docs/',
'<a href="https://cloud.google.com/pubsub/docs/')
s.copy(v1beta1_library / 'lib/google/cloud/error_reporting/v1beta1')
s.copy(v1beta1_library / 'lib/google/devtools/clouderrorreporting/v1beta1')
# Omitting lib/google/cloud/error_reporting/v1beta1.rb for now because we are
# not exposing the low-level API.
# https://github.com/googleapis/gapic-generator/issues/2242
def escape_braces(match):
expr = re.compile('^([^`]*(`[^`]*`[^`]*)*)([^`#\\$\\\\])\\{([\\w,]+)\\}')
content = match.group(0)
while True:
content, count = expr.subn('\\1\\3\\\\\\\\{\\4}', content)
if count == 0:
return content
s.replace(
'lib/google/cloud/error_reporting/v1beta1/**/*.rb',
'\n(\\s+)#[^\n]*[^\n#\\$\\\\]\\{[\\w,]+\\}',
escape_braces)
# https://github.com/googleapis/gapic-generator/issues/2243
s.replace(
'lib/google/cloud/error_reporting/v1beta1/*_client.rb',
'(\n\\s+class \\w+Client\n)(\\s+)(attr_reader :\\w+_stub)',
'\\1\\2# @private\n\\2\\3')
# https://github.com/googleapis/gapic-generator/issues/2279
s.replace(
'lib/**/*.rb',
'\\A(((#[^\n]*)?\n)*# (Copyright \\d+|Generated by the protocol buffer compiler)[^\n]+\n(#[^\n]*\n)*\n)([^\n])',
'\\1\n\\6')
version='v3',
config_path='/google/monitoring/artman_monitoring.yaml',
artman_output_name='google-cloud-monitoring-v3')
# copy all src including partial veneer classes
s.move(library / 'src')
# copy proto files to src also
s.move(library / 'proto/src/Google/Cloud/Monitoring', 'src/')
s.move(library / 'tests/')
# copy GPBMetadata file to metadata
s.move(library / 'proto/src/GPBMetadata/Google/Monitoring', 'metadata/')
# document and utilize apiEndpoint instead of serviceAddress
s.replace("**/Gapic/*GapicClient.php", r"'serviceAddress' =>",
r"'apiEndpoint' =>")
s.replace(
"**/Gapic/*GapicClient.php",
r"@type string \$serviceAddress\n\s+\*\s+The address",
r"""@type string $serviceAddress
* **Deprecated**. This option will be removed in a future major release. Please
* utilize the `$apiEndpoint` option instead.
* @type string $apiEndpoint
* The address""")
s.replace("**/Gapic/*GapicClient.php",
r"\$transportConfig, and any \$serviceAddress",
r"$transportConfig, and any `$apiEndpoint`")
# prevent proto messages from being marked final
s.replace("src/V*/**/*.php", r"final class", r"class")
import synthtool as s
from synthtool import gcp
gapic = gcp.GAPICGenerator()
common = gcp.CommonTemplates()
versions = ["v1beta1", "v1"]
# ----------------------------------------------------------------------------
# Generate securitycenter GAPIC layer
# ----------------------------------------------------------------------------
for version in versions:
library = gapic.py_library("securitycenter", version, include_protos=True)
s.move(library / f"google/cloud/securitycenter_{version}")
s.move(library / f"tests/unit/gapic/{version}")
s.move(library / f"docs/gapic/{version}")
# Use the highest version library to generate import alias.
s.move(library / "google/cloud/securitycenter.py")
# Add encoding header to protoc-generated files.
# See: https://github.com/googleapis/gapic-generator/issues/2097
s.replace("**/proto/*_pb2.py", r"(^.*$\n)*", r"# -*- coding: utf-8 -*-\n\g<0>")
# ----------------------------------------------------------------------------
# Add templated files
# ----------------------------------------------------------------------------
templated_files = common.py_library(unit_cov_level=97, cov_level=100)
s.move(templated_files, excludes=['noxfile.py'])
s.shell.run(["nox", "-s", "blacken"], hide_output=False)
'/google/cloud/bigquery/datatransfer/artman_bigquerydatatransfer.yaml',
artman_output_name='google-cloud-bigquerydatatransfer-v1')
# copy all src including partial veneer classes
s.move(library / 'src')
# copy proto files to src also
s.move(library / 'proto/src/Google/Cloud/BigQuery/DataTransfer', 'src/')
s.move(library / 'tests/')
# copy GPBMetadata file to metadata
s.move(library / 'proto/src/GPBMetadata/Google/Cloud/Bigquery/Datatransfer',
'metadata/')
# document and utilize apiEndpoint instead of serviceAddress
s.replace("**/Gapic/*GapicClient.php", r"'serviceAddress' =>",
r"'apiEndpoint' =>")
s.replace(
"**/Gapic/*GapicClient.php",
r"@type string \$serviceAddress\n\s+\*\s+The address",
r"""@type string $serviceAddress
* **Deprecated**. This option will be removed in a future major release. Please
* utilize the `$apiEndpoint` option instead.
* @type string $apiEndpoint
* The address""")
s.replace("**/Gapic/*GapicClient.php",
r"\$transportConfig, and any \$serviceAddress",
r"$transportConfig, and any `$apiEndpoint`")
# fix year
s.replace('**/Gapic/*GapicClient.php', r'Copyright \d{4}', 'Copyright 2017')
s.replace('**/V1/DataTransferServiceClient.php', r'Copyright \d{4}',
config_path=f'artman_cloudasset_{lower_version}.yaml',
artman_output_name=f'google-cloud-asset-{lower_version}')
# copy all src including partial veneer classes
s.move(library / 'src')
# copy proto files to src also
s.move(library / f'proto/src/Google/Cloud/Asset', f'src/')
s.move(library / f'tests/')
# copy GPBMetadata file to metadata
s.move(library / f'proto/src/GPBMetadata/Google/Cloud/Asset', f'metadata/')
# fix year
s.replace(
'src/V1beta1/**/*.php',
r'Copyright \d{4}',
r'Copyright 2018')
s.replace(
'tests/*/V1beta1/*Test.php',
r'Copyright \d{4}',
r'Copyright 2018')
s.replace(
'src/V1/**/*.php',
r'Copyright \d{4}',
r'Copyright 2019')
s.replace(
'tests/*/V1/*Test.php',
r'Copyright \d{4}',
r'Copyright 2019')
"""This script is used to synthesize generated parts of this library."""
import synthtool as s
import synthtool.gcp as gcp
import logging
import os
logging.basicConfig(level=logging.DEBUG)
gapic = gcp.GAPICGenerator()
v1_library = gapic.ruby_library(
'pubsub',
'v1',
config_path='/google/pubsub/artman_pubsub.yaml',
artman_output_name='google-cloud-ruby/google-cloud-pubsub')
s.copy(v1_library / 'lib/google/cloud/pubsub/v1')
s.copy(v1_library / 'lib/google/pubsub/v1')
# Omitting lib/google/cloud/pusbusb/v1.rb for now because we are not exposing
# the low-level API.
# PERMANENT: We don't want the generated overview.rb file because we have our
# own toplevel docs for the handwritten layer.
os.remove('lib/google/cloud/pubsub/v1/doc/overview.rb')
# https://github.com/googleapis/gapic-generator/issues/2124
s.replace('lib/google/cloud/pubsub/v1/credentials.rb',
'SCOPE = \[[^\]]+\]\.freeze',
'SCOPE = ["https://www.googleapis.com/auth/pubsub"].freeze')
version='v2',
artman_output_name='google-cloud-dialogflow-v2')
# copy all src including partial veneer classes
s.move(library / 'src')
# copy proto files to src also
s.move(library / 'proto/src/Google/Cloud/Dialogflow', 'src/')
s.move(library / 'tests/')
# copy GPBMetadata file to metadata
s.move(library / 'proto/src/GPBMetadata/Google/Cloud/Dialogflow', 'metadata/')
# fix year
s.replace(
'**/Gapic/*GapicClient.php',
r'Copyright \d{4}',
'Copyright 2018')
for client in ['Agents', 'Contexts', 'EntityTypes', 'Intents', 'SessionEntityTypes', 'Sessions']:
s.replace(
f'**/V1/{client}Client.php',
r'Copyright \d{4}',
'Copyright 2018')
s.replace(
'tests/**/V2/*Test.php',
r'Copyright \d{4}',
'Copyright 2018')
# Change the wording for the deprecation warning.
s.replace(
'src/*/*_*.php',
r'will be removed in the next major release',
import synthtool as s
import synthtool.gcp as gcp
import logging
logging.basicConfig(level=logging.DEBUG)
gapic = gcp.GAPICGenerator()
common = gcp.CommonTemplates()
v1_library = gapic.php_library(
service='container',
version='v1',
config_path='/google/container/artman_container.yaml',
artman_output_name='google-cloud-container-v1')
# copy all src including partial veneer classes
s.move(v1_library / 'src')
# copy proto files to src also
s.move(v1_library / 'proto/src/Google/Cloud/Container', 'src/')
s.move(v1_library / 'tests/')
# copy GPBMetadata file to metadata
s.move(v1_library / 'proto/src/GPBMetadata/Google/Container', 'metadata/')
# fix year
s.replace('**/Gapic/*GapicClient.php', r'Copyright \d{4}', 'Copyright 2017')
s.replace('**/V1/ClusterManagerClient.php', r'Copyright \d{4}',
'Copyright 2017')
s.replace('tests/**/V1/*Test.php', r'Copyright \d{4}', 'Copyright 2018')
# ----------------------------------------------------------------------------
# Generate dlp GAPIC layer
# ----------------------------------------------------------------------------
library = gapic.py_library(
"irm",
"v1alpha2",
config_path="/google/cloud/irm/artman_irm_v1alpha2.yaml",
include_protos=True,
)
excludes = ["README.rst", "nox*.py", "setup.py", "docs/index.rst"]
s.move(library, excludes=excludes)
# Fix docstrings
s.replace("google/**/*.py", r"\\_", "_")
s.replace("google/**/incidents_service_pb2.py", r"""\\\*""", r"""*""")
s.replace("google/**/incident_service_client.py", r"""\\\*""", r"""*""")
s.replace(
"google/**/incident_service_client.py",
r""" This will fail if:
a\. there are too many \(50\) subscriptions in the incident already
b\. a subscription using the given channel already exists""",
r""" This will fail if:
a. there are too many (50) subscriptions in the incident already
b. a subscription using the given channel already exists""",
)
# ----------------------------------------------------------------------------
# Add templated files
# ----------------------------------------------------------------------------
v1beta_library = gapic.ruby_library(
'oslogin',
'v1beta',
config_path='/google/cloud/oslogin/artman_oslogin_v1beta.yaml',
artman_output_name='google-cloud-ruby/google-cloud-os_login')
s.copy(v1beta_library / 'lib/google/cloud/os_login/v1beta')
s.copy(v1beta_library / 'lib/google/cloud/os_login/v1beta.rb')
s.copy(v1beta_library / 'lib/google/cloud/oslogin/v1beta')
s.copy(v1beta_library / 'test/google/cloud/os_login/v1beta')
# Update gemspec to reflect Ruby 2.4
ruby.update_gemspec('google-cloud-os_login.gemspec')
# Update README to reflect Ruby 2.4
s.replace('README.md', 'Ruby 2.3', 'Ruby 2.4')
# PERMANENT: API name for oslogin
s.replace([
'README.md', 'lib/google/cloud/os_login.rb',
'lib/google/cloud/os_login/v1.rb', 'lib/google/cloud/os_login/v1beta.rb'
], '/os-login\\.googleapis\\.com', '/oslogin.googleapis.com')
# Support for service_address
s.replace(
[
'lib/google/cloud/os_login.rb', 'lib/google/cloud/os_login/v*.rb',
'lib/google/cloud/os_login/v*/*_client.rb'
], '\n(\\s+)#(\\s+)@param exception_transformer',
'\n\\1#\\2@param service_address [String]\n' +
'\\1#\\2 Override for the service hostname, or `nil` to leave as the default.\n'
library = gapic.py_library("vision", version, include_protos=True)
s.move(library / f"google/cloud/vision_{version}/gapic")
s.move(library / f"google/cloud/vision_{version}/__init__.py")
s.move(library / f"google/cloud/vision_{version}/types.py")
s.move(library / f"google/cloud/vision_{version}/proto")
s.move(library / f"tests/unit/gapic/{version}")
# don't publish docs for these versions
if version not in ["v1p1beta1"]:
s.move(library / f"docs/gapic/{version}")
# Add vision helpers to each version
s.replace(
f"google/cloud/vision_{version}/__init__.py",
f"from __future__ import absolute_import",
f"\g<0>\n\n"
f"from google.cloud.vision_helpers.decorators import "
f"add_single_feature_methods\n"
f"from google.cloud.vision_helpers import VisionHelpers",
)
s.replace(
f"google/cloud/vision_{version}/__init__.py", f"image_annotator_client", f"iac"
)
s.replace(
f"google/cloud/vision_{version}/__init__.py",
f"from google.cloud.vision_{version}.gapic import iac",
f"from google.cloud.vision_{version}.gapic import "
f"image_annotator_client as iac",
)
# ----------------------------------------------------------------------------
# Generate kms GAPIC layer
# ----------------------------------------------------------------------------
library = gapic.py_library(
service="kms",
version=version,
bazel_target="//google/cloud/kms/v1:kms-v1-py",
include_protos=True,
)
s.move(library, excludes=["README.rst", "setup.py", "nox*.py", "docs/**/*"])
# Temporary fixup for 'grpc-google-iam-vi 0.12.4' (before generation).
s.replace(
"google/cloud/kms_v1/gapic/transports/key_management_service_grpc_transport.py",
"from google.iam.v1 import iam_policy_pb2",
"from google.iam.v1 import iam_policy_pb2_grpc as iam_policy_pb2",
)
# re-insert `crypto_key_path_path` method as this was used in the published samples
# TODO: remove when this library is moved to the microgenerator and mention it in the relase
# notes
count = s.replace(
"google/cloud/kms_v1/gapic/key_management_service_client.py",
"""(@classmethod
\s+def crypto_key_version_path\(.*)""", """
@classmethod
def crypto_key_path_path(cls, project, location, key_ring, crypto_key_path):
\"\"\"Return a fully-qualified crypto_key_path string.\"\"\"
return google.api_core.path_template.expand(
"projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{crypto_key_path=**}",
project=project,
'kms', 'v1', artman_output_name='google-cloud-ruby/google-cloud-kms',
config_path='artman_cloudkms.yaml'
)
s.copy(v1_library / 'lib')
s.copy(v1_library / 'test')
s.copy(v1_library / 'README.md')
s.copy(v1_library / 'LICENSE')
s.copy(v1_library / '.gitignore')
s.copy(v1_library / '.yardopts')
s.copy(v1_library / 'google-cloud-kms.gemspec', merge=ruby.merge_gemspec)
# PERMANENT: API name for cloudkms
s.replace(
[
'README.md',
'lib/google/cloud/kms.rb',
'lib/google/cloud/kms/v1.rb'
],
'/kms\\.googleapis\\.com', '/cloudkms.googleapis.com')
# https://github.com/googleapis/gapic-generator/issues/2242
def escape_braces(match):
expr = re.compile('^([^`]*(`[^`]*`[^`]*)*)([^`#\\$\\\\])\\{([\\w,]+)\\}')
content = match.group(0)
while True:
content, count = expr.subn('\\1\\3\\\\\\\\{\\4}', content)
if count == 0:
return content
s.replace(
'lib/google/cloud/**/*.rb',
'\n(\\s+)#[^\n]*[^\n#\\$\\\\]\\{[\\w,]+\\}',
from synthtool.languages import php
from synthtool import _tracked_paths
logging.basicConfig(level=logging.DEBUG)
src = Path(f"../{php.STAGING_DIR}/AnalyticsData").resolve()
dest = Path().resolve()
# Added so that we can pass copy_excludes in the owlbot_main() call
_tracked_paths.add(src)
php.owlbot_main(src=src, dest=dest, copy_excludes=[src / "**/[A-Z]*_*.php"])
# remove class_alias code
s.replace(
"src/V*/**/*.php",
r"^// Adding a class alias for backwards compatibility with the previous class name.$"
+ "\n" + r"^class_alias\(.*\);$" + "\n", '')
# document and utilize apiEndpoint instead of serviceAddress
s.replace("**/Gapic/*GapicClient.php", r"'serviceAddress' =>",
r"'apiEndpoint' =>")
s.replace(
"**/Gapic/*GapicClient.php",
r"@type string \$serviceAddress\n\s+\*\s+The address",
r"""@type string $serviceAddress
* **Deprecated**. This option will be removed in a future major release. Please
* utilize the `$apiEndpoint` option instead.
* @type string $apiEndpoint
* The address""")
s.replace("**/Gapic/*GapicClient.php",
r"\$transportConfig, and any \$serviceAddress",
common = gcp.CommonTemplates()
v1_library = gapic.php_library(
service='kms',
version='v1',
config_path='artman_cloudkms.yaml',
artman_output_name='google-cloud-kms-v1')
s.copy(v1_library / f'src/')
s.copy(v1_library / f'proto/src/GPBMetadata/Google/Cloud/Kms', f'metadata')
s.copy(v1_library / f'proto/src/Google/Cloud/Kms', f'src')
s.copy(v1_library / f'tests')
# fix copyright year
s.replace(
'src/V1/**/*Client.php',
r'Copyright \d{4}',
r'Copyright 2018')
s.replace(
'tests/**/V1/*Test.php',
r'Copyright \d{4}',
r'Copyright 2018')
# Use new namespace in the doc sample. See
# https://github.com/googleapis/gapic-generator/issues/2141
s.replace(
'src/V1/Gapic/KeyManagementServiceGapicClient.php',
r'CryptoKey_CryptoKeyPurpose',
'CryptoKeyPurpose')
# Change the wording for the deprecation warning.
s.replace(
'src/V1/CryptoKey*_*.php',
def fix_grpc_headers(grpc_root: Path, package_name: str) -> None:
s.replace(
[grpc_root / "src/**/*.java"],
"^package (.*);",
f"{GOOD_LICENSE}package \\1;",
)
for version, artman_config in versions:
library = gapic.py_library(
"firestore",
version,
config_path=f"/google/firestore/{artman_config}",
artman_output_name=f"firestore-{version}",
include_protos=True,
)
s.move(library / f"google/cloud/firestore_{version}/proto")
s.move(library / f"google/cloud/firestore_{version}/gapic")
s.move(library / f"tests/unit/gapic/{version}")
s.replace(
f"tests/unit/gapic/{version}/test_firestore_client_{version}.py",
f"from google.cloud import firestore_{version}",
f"from google.cloud.firestore_{version}.gapic import firestore_client",
)
s.replace(
f"tests/unit/gapic/{version}/test_firestore_client_{version}.py",
f"client = firestore_{version}.FirestoreClient",
"client = firestore_client.FirestoreClient",
)
# ----------------------------------------------------------------------------
# Add templated files
# ----------------------------------------------------------------------------
templated_files = common.py_library(unit_cov_level=97, cov_level=100)
s.move(templated_files)
library,
excludes=[
'docs/conf.py',
'docs/index.rst',
'google/cloud/datalabeling_v1beta1/__init__.py',
'README.rst',
'nox*.py',
'setup.py',
'setup.cfg',
],
)
# Fixup issues in generated code
s.replace(
"./**/gapic/**/*client.py",
r"operations_pb2.ImportDataOperationResponse",
"proto_operations_pb2.ImportDataOperationResponse",
)
s.replace(
"./**/gapic/**/*client.py",
r"operations_pb2.ImportDataOperationMetadata",
"proto_operations_pb2.ImportDataOperationMetadata",
)
s.replace(
"./tests/unit/gapic/**/test*_client*.py",
r"operations_pb2.Operation\(",
"longrunning_operations_pb2.Operation(",
)
library,
excludes=[
"docs/conf.py",
"docs/index.rst",
"google/cloud/bigquery_storage_v1beta1/__init__.py",
"README.rst",
"nox*.py",
"setup.py",
"setup.cfg",
],
)
s.replace(
[
"google/cloud/bigquery_storage_v1beta1/proto/storage_pb2.py",
"google/cloud/bigquery_storage_v1beta1/proto/storage_pb2_grpc.py",
],
"from google.cloud.bigquery.storage_v1beta1.proto",
"from google.cloud.bigquery_storage_v1beta1.proto",
)
s.replace(
"google/cloud/bigquery_storage_v1beta1/gapic/" "big_query_storage_client.py",
"google-cloud-bigquerystorage",
"google-cloud-bigquery-storage",
)
s.replace(
"google/cloud/bigquery_storage_v1beta1/gapic/" "big_query_storage_client.py",
"import google.api_core.gapic_v1.method\n",
"\g<0>import google.api_core.path_template\n",
)
import synthtool as s
from synthtool import gcp
from synthtool.languages import python
common = gcp.CommonTemplates()
# ----------------------------------------------------------------------------
# Generate bigquery_datatransfer GAPIC layer
# ----------------------------------------------------------------------------
for library in s.get_staging_dirs("v1"):
# Comment out broken assertion in unit test
# https://github.com/googleapis/gapic-generator-python/issues/897
s.replace(
library / "tests/**/*.py",
"assert args\[0\]\.start_time == timestamp_pb2\.Timestamp\(seconds=751\)",
"# assert args[0].start_time == timestamp_pb2.Timestamp(seconds=751)")
s.replace(
library / "tests/**/*.py",
"assert args\[0\]\.end_time == timestamp_pb2\.Timestamp\(seconds=751\)",
"# assert args[0].end_time == timestamp_pb2.Timestamp(seconds=751)")
s.move(library,
excludes=["*.tar.gz", "docs/index.rst", "README.rst", "setup.py"])
s.remove_staging_dirs()
# ----------------------------------------------------------------------------
# Add templated files
# ----------------------------------------------------------------------------
templated_files = common.py_library(microgenerator=True,
v1beta_library = gapic.ruby_library(
'oslogin', 'v1beta',
config_path='/google/cloud/oslogin/artman_oslogin_v1beta.yaml',
artman_output_name='google-cloud-ruby/google-cloud-os_login'
)
s.copy(v1beta_library / 'lib/google/cloud/os_login/v1beta')
s.copy(v1beta_library / 'lib/google/cloud/os_login/v1beta.rb')
s.copy(v1beta_library / 'lib/google/cloud/oslogin/v1beta')
s.copy(v1beta_library / 'test/google/cloud/os_login/v1beta')
# PERMANENT: API name for oslogin
s.replace(
[
'README.md',
'lib/google/cloud/os_login.rb',
'lib/google/cloud/os_login/v1.rb',
'lib/google/cloud/os_login/v1beta.rb'
],
'/os-login\\.googleapis\\.com', '/oslogin.googleapis.com')
# https://github.com/googleapis/gapic-generator/issues/2196
s.replace(
[
'README.md',
'lib/google/cloud/os_login.rb',
'lib/google/cloud/os_login/v1.rb',
'lib/google/cloud/os_login/v1beta.rb'
],
'\\[Product Documentation\\]: https://cloud\\.google\\.com/os-login\n',
'[Product Documentation]: https://cloud.google.com/compute/docs/oslogin/rest/\n')
logging.basicConfig(level=logging.DEBUG)
src = Path(f"../{php.STAGING_DIR}/DatastoreAdmin").resolve()
dest = Path().resolve()
# Added so that we can pass copy_excludes in the owlbot_main() call
_tracked_paths.add(src)
php.owlbot_main(src=src, dest=dest)
# document and utilize apiEndpoint instead of serviceAddress
s.replace(
"**/Gapic/*GapicClient.php",
r"'serviceAddress' =>",
r"'apiEndpoint' =>")
s.replace(
"**/Gapic/*GapicClient.php",
r"@type string \$serviceAddress\n\s+\*\s+The address",
r"""@type string $serviceAddress
* **Deprecated**. This option will be removed in a future major release. Please
* utilize the `$apiEndpoint` option instead.
* @type string $apiEndpoint
* The address""")
s.replace(
"**/Gapic/*GapicClient.php",
r"\$transportConfig, and any \$serviceAddress",
r"$transportConfig, and any `$apiEndpoint`")
s.replace(
config_path='/google/cloud/bigquery/datatransfer/artman_bigquerydatatransfer.yaml',
artman_output_name='google-cloud-bigquerydatatransfer-v1')
# copy all src including partial veneer classes
s.move(library / 'src')
# copy proto files to src also
s.move(library / 'proto/src/Google/Cloud/BigQuery/DataTransfer', 'src/')
s.move(library / 'tests/')
# copy GPBMetadata file to metadata
s.move(library / 'proto/src/GPBMetadata/Google/Cloud/Bigquery/Datatransfer', 'metadata/')
# fix year
s.replace(
'**/Gapic/*GapicClient.php',
r'Copyright \d{4}',
'Copyright 2017')
s.replace(
'**/V1/DataTransferServiceClient.php',
r'Copyright \d{4}',
'Copyright 2017')
s.replace(
'tests/**/V1/*Test.php',
r'Copyright \d{4}',
'Copyright 2018')
# Change the wording for the deprecation warning.
s.replace(
'src/*/*_*.php',
r'will be removed in the next major release',
'will be removed in a future release')
logging.basicConfig(level=logging.DEBUG)
common_templates = gcp.CommonTemplates()
templates = common_templates.node_library()
# Don't use .nycrc for code coverage (see "Fix Code Coverage")
s.copy(templates, excludes=['.nycrc'])
### SUPPORT DATABASE PLUGINS ###
# Database plugins require that an instance of that database is running at a
# known address. On Unix we spin up Docker containers to do so, while on Windows
# we disable plugin integration tests, as there is no known procedure for doing
# something equivalent.
s.replace('.kokoro/test.bat', r'(call npm install \|\| goto :error)',
r"""
@rem Plugin integration tests on Windows are skipped, because there is no known
@rem procedure to start up database docker containers for the Kokoro Windows CI.
set TRACE_TEST_EXCLUDE_INTEGRATION=1
\1""")
s.replace('.kokoro/trampoline.sh', r'(python3 "\$\{KOKORO_GFILE_DIR\}/trampoline_v1\.py")',
r"""# From bin/docker-trace.sh
# Start up database docker containers for plugin integration tests.
docker run --name trace-test-mongo -p 127.0.0.1:27017:27017 -d mongo
docker run --name trace-test-redis -p 127.0.0.1:6379:6379 -d redis
docker run --name trace-test-mysql -p 127.0.0.1:3306:3306\
-e MYSQL_ROOT_PASSWORD='******'\
-e MYSQL_DATABASE=test\
-d mysql:5
docker run --name trace-test-postgres -p 127.0.0.1:5432:5432\
-e POSTGRES_USER=postgres\
from synthtool import gcp
gapic = gcp.GAPICGenerator()
common = gcp.CommonTemplates()
versions = ["v1beta1"]
# ----------------------------------------------------------------------------
# Generate webrisk GAPIC layer
# ----------------------------------------------------------------------------
for version in versions:
library = gapic.py_library("webrisk", version, include_protos=True)
s.copy(library, excludes=["docs/index.rst", "nox.py", "README.rst", "setup.py"])
# Fix docstring issue for classes with no summary line
s.replace(
"google/cloud/**/proto/webrisk_pb2.py",
'''__doc__ = """Attributes:''',
'''__doc__ = """
Attributes:''',
)
# ----------------------------------------------------------------------------
# Add templated files
# ----------------------------------------------------------------------------
templated_files = common.py_library(unit_cov_level=97, cov_level=100)
s.move(templated_files, excludes=["noxfile.py"])
s.shell.run(["nox", "-s", "blacken"], hide_output=False)
v1beta2_library = gapic.ruby_library(
'language', 'v1beta2',
artman_output_name='google-cloud-ruby/google-cloud-language'
)
s.copy(v1beta2_library / 'lib/google/cloud/language/v1beta2')
s.copy(v1beta2_library / 'lib/google/cloud/language/v1beta2.rb')
s.copy(v1beta2_library / 'lib/google/cloud/language/v1beta2')
s.copy(v1beta2_library / 'test/google/cloud/language/v1beta2')
# https://github.com/googleapis/gapic-generator/issues/2196
s.replace(
[
'README.md',
'lib/google/cloud/language.rb',
'lib/google/cloud/language/v1.rb',
'lib/google/cloud/language/v1beta2.rb'
],
'\\[Product Documentation\\]: https://cloud\\.google\\.com/language\n',
'[Product Documentation]: https://cloud.google.com/natural-language\n')
# https://github.com/googleapis/gapic-generator/issues/2243
s.replace(
'lib/google/cloud/language/*/*_client.rb',
'(\n\\s+class \\w+Client\n)(\\s+)(attr_reader :\\w+_stub)',
'\\1\\2# @private\n\\2\\3')
# https://github.com/googleapis/gapic-generator/issues/2279
s.replace(
'lib/**/*.rb',
'\\A(((#[^\n]*)?\n)*# (Copyright \\d+|Generated by the protocol buffer compiler)[^\n]+\n(#[^\n]*\n)*\n)([^\n])',
common = gcp.CommonTemplates()
# ----------------------------------------------------------------------------
# Generate oslogin GAPIC layer
# ----------------------------------------------------------------------------
library = gapic.py_library(
"oslogin",
"v1",
config_path="/google/cloud/oslogin/artman_oslogin_v1.yaml",
artman_output_name="os-login-v1",
include_protos=True,
)
s.move(library / "google/cloud/oslogin_v1")
s.move(library / "tests/unit/gapic/v1")
# Fix up imports
s.replace(
"google/**/proto/*.py",
"from google.cloud.oslogin.common import common_pb2",
"from google.cloud.oslogin_v1.proto import common_pb2",
)
# ----------------------------------------------------------------------------
# Add templated files
# ----------------------------------------------------------------------------
templated_files = common.py_library(unit_cov_level=97, cov_level=100)
s.move(templated_files)
s.shell.run(["nox", "-s", "blacken"], hide_output=False)
s.move(library / f"google/cloud/automl_{version}")
s.move(library / f"tests/unit/gapic/{version}")
s.move(library / f"docs/gapic/{version}")
s.move(library / f"docs/conf.py")
# Use the highest version library to generate import alias.
s.move(library / "google/cloud/automl.py")
# Add TablesClient and GcsClient to v1beta1
s.replace(
f"google/cloud/automl_v1beta1/__init__.py",
f"from google.cloud.automl_v1beta1.gapic import prediction_service_client",
f"from google.cloud.automl_v1beta1.gapic import prediction_service_client\n"
f"from google.cloud.automl_v1beta1.tables import tables_client\n"
f"from google.cloud.automl_v1beta1.tables import gcs_client"
f"\n\n"
f"class TablesClient(tables_client.TablesClient):"
f" __doc__ = tables_client.TablesClient.__doc__"
f"\n\nclass GcsClient(gcs_client.GcsClient):"
f" __doc__ = gcs_client.GcsClient.__doc__",
)
s.replace(
f"google/cloud/automl_v1beta1/__init__.py",
f"""__all__ = \(
'enums',
'types',
'AutoMlClient',
'PredictionServiceClient',
\)""",
f'__all__ = ("enums", "types", "AutoMlClient", "PredictionServiceClient", "TablesClient", "GcsClient")',
