def new_stats_exporter(options=None, interval=None):
"""Get a stats exporter and running transport thread.
Create a new `StackdriverStatsExporter` with the given options and start
periodically exporting stats to stackdriver in the background.
Fall back to default auth if `options` is null. This will raise
`google.auth.exceptions.DefaultCredentialsError` if default credentials
aren't configured.
See `opencensus.metrics.transport.get_exporter_thread` for details on the
transport thread.
:type options: :class:`Options`
:param exporter: Options to pass to the exporter
:type interval: int or float
:param interval: Seconds between export calls.
:rtype: :class:`StackdriverStatsExporter`
:return: The newly-created exporter.
"""
if options is None:
_, project_id = google.auth.default()
options = Options(project_id=project_id)
if str(options.project_id).strip() == "":
raise ValueError(ERROR_BLANK_PROJECT_ID)
ci = client_info.ClientInfo(client_library_version=get_user_agent_slug())
client = monitoring_v3.MetricServiceClient(client_info=ci)
exporter = StackdriverStatsExporter(client=client, options=options)
transport.get_exporter_thread([stats.stats], exporter, interval=interval)
return exporter
def _make_bqstorage_client(use_bqstorage_api, credentials):
if not use_bqstorage_api:
return None
try:
from google.cloud import bigquery_storage_v1
except ImportError as err:
customized_error = ImportError(
"The default BigQuery Storage API client cannot be used, install "
"the missing google-cloud-bigquery-storage and pyarrow packages "
"to use it. Alternatively, use the classic REST API by specifying "
"the --use_rest_api magic option.")
six.raise_from(customized_error, err)
try:
from google.api_core.gapic_v1 import client_info as gapic_client_info
except ImportError as err:
customized_error = ImportError(
"Install the grpcio package to use the BigQuery Storage API.")
six.raise_from(customized_error, err)
return bigquery_storage_v1.BigQueryReadClient(
credentials=credentials,
client_info=gapic_client_info.ClientInfo(
user_agent=IPYTHON_USER_AGENT),
)
def _make_bqstorage_client(use_bqstorage_api, credentials):
if not use_bqstorage_api:
return None
try:
from google.cloud import bigquery_storage_v1beta1
except ImportError as err:
customized_error = ImportError(
"Install the google-cloud-bigquery-storage and pyarrow packages "
"to use the BigQuery Storage API."
)
six.raise_from(customized_error, err)
try:
from google.api_core.gapic_v1 import client_info as gapic_client_info
except ImportError as err:
customized_error = ImportError(
"Install the grpcio package to use the BigQuery Storage API."
)
six.raise_from(customized_error, err)
return bigquery_storage_v1beta1.BigQueryStorageClient(
credentials=credentials,
client_info=gapic_client_info.ClientInfo(user_agent=IPYTHON_USER_AGENT),
)
def test_constructor_defaults():
info = client_info.ClientInfo()
assert info.python_version is not None
assert info.grpc_version is not None
assert info.api_core_version is not None
assert info.gapic_version is None
assert info.client_library_version is None
def test_to_user_agent_minimal():
info = client_info.ClientInfo(python_version='1',
grpc_version='2',
api_core_version='3')
user_agent = info.to_user_agent()
assert user_agent == 'gl-python/1 gax/3 grpc/2'
def test_to_user_agent_minimal():
info = client_info.ClientInfo(
python_version="1", api_core_version="2", grpc_version=None
)
user_agent = info.to_user_agent()
assert user_agent == "gl-python/1 gax/2"
def test_to_user_agent_full():
info = client_info.ClientInfo(python_version='1',
grpc_version='2',
api_core_version='3',
gapic_version='4',
client_library_version='5')
user_agent = info.to_user_agent()
assert user_agent == 'gl-python/1 gccl/5 gapic/4 gax/3 grpc/2'
def get_client(project, namespace):
"""Returns a Cloud Datastore client."""
_client_info = client_info.ClientInfo(
client_library_version=__version__,
gapic_version=__version__,
user_agent=f'beam-python-sdk/{beam_version}')
_client = client.Client(
project=project, namespace=namespace, client_info=_client_info)
# Avoid overwriting user setting. BEAM-7608
if not os.environ.get(environment_vars.GCD_HOST, None):
_client.base_url = 'https://batch-datastore.googleapis.com' # BEAM-1387
return _client
def test_constructor_options():
info = client_info.ClientInfo(python_version='1',
grpc_version='2',
api_core_version='3',
gapic_version='4',
client_library_version='5')
assert info.python_version == '1'
assert info.grpc_version == '2'
assert info.api_core_version == '3'
assert info.gapic_version == '4'
assert info.client_library_version == '5'
def test_to_user_agent_full():
info = client_info.ClientInfo(
python_version="1",
grpc_version="2",
api_core_version="3",
gapic_version="4",
client_library_version="5",
)
user_agent = info.to_user_agent()
assert user_agent == "gl-python/1 grpc/2 gax/3 gapic/4 gccl/5"
def output(self):
if 'bucket' not in self.output_config:
raise NotConfiguredException(
'No destination bucket defined in GCS output.')
if 'object' not in self.output_config:
raise NotConfiguredException(
'No destination object defined in GCS output.')
bucket_template = self.jinja_environment.from_string(
self.output_config['bucket'])
bucket_template.name = 'bucket'
destination_bucket = bucket_template.render()
object_template = self.jinja_environment.from_string(
self.output_config['object'])
object_template.name = 'object'
destination_object = object_template.render()
self.logger.debug('Creating destination file in bucket.',
extra={
'url':
'gs://%s/%s' %
(destination_bucket, destination_object)
})
contents_template = self.jinja_environment.from_string(
self.output_config['contents'])
contents_template.name = 'contents'
contents = contents_template.render()
if 'base64decode' in self.output_config and self.output_config[
'base64decode']:
contents = base64.decodebytes(contents.encode('ascii'))
client_info = grpc_client_info.ClientInfo(
user_agent='google-pso-tool/pubsub2inbox/1.1.0')
project = self.output_config[
'project'] if 'project' in self.output_config else None
storage_client = storage.Client(client_info=client_info,
project=project)
bucket = storage_client.bucket(destination_bucket)
blob = bucket.blob(destination_object)
blob.upload_from_string(contents)
self.logger.info('Object created in Cloud Storage bucket.',
extra={
'url':
'gs://%s/%s' %
(destination_bucket, destination_object),
'size':
len(contents)
})
def test_constructor_w_client_info(mock_transport):
from google.cloud.bigquery_storage_v1beta1 import client
def transport_callable(credentials=None, default_class=None):
return mock_transport
client_under_test = client.BigQueryStorageClient(
transport=transport_callable,
client_info=client_info.ClientInfo(
client_library_version="test-client-version"),
)
user_agent = client_under_test._client_info.to_user_agent()
assert "test-client-version" in user_agent
def test_constructor_options():
info = client_info.ClientInfo(
python_version="1",
grpc_version="2",
api_core_version="3",
gapic_version="4",
client_library_version="5",
)
assert info.python_version == "1"
assert info.grpc_version == "2"
assert info.api_core_version == "3"
assert info.gapic_version == "4"
assert info.client_library_version == "5"
def process(self):
if 'bigquery' not in self.config:
raise NotConfiguredException(
'No BigQuery configuration specified in config!')
bigquery_config = self.config['bigquery']
if 'query' not in bigquery_config:
raise NotConfiguredException(
'No BigQuery query specified in configuration!')
data = json.loads(self.data)
self.jinja_environment.globals = {
**self.jinja_environment.globals,
**data
}
query_template = self.jinja_environment.from_string(
bigquery_config['query'])
query_template.name = 'query'
query = query_template.render()
dialect = 'legacy' if 'dialect' in bigquery_config and bigquery_config[
'dialect'].lower() == 'legacy' else 'standard'
self.logger.debug('Running BigQuery query.', extra={'query': query})
client_info = grpc_client_info.ClientInfo(
user_agent='google-pso-tool/pubsub2inbox/1.1.0')
project = bigquery_config[
'project'] if 'project' in bigquery_config else None
client = bigquery.Client(client_info=client_info, project=project)
labels = {}
if 'labels' in bigquery_config:
labels = bigquery_config['labels']
job_options = bigquery.job.QueryJobConfig(
use_legacy_sql=True if dialect == 'legacy' else False,
labels=labels)
query_job = client.query(query, job_config=job_options)
results = query_job.result()
records = []
for row in results:
record = {}
for k in row.keys():
record[k] = row.get(k)
records.append(record)
self.logger.debug('BigQuery execution finished.',
extra={'count': len(records)})
return {
'records': records,
}
def new_stats_exporter(options):
""" new_stats_exporter returns an exporter that
uploads stats data to Stackdriver Monitoring.
"""
if str(options.project_id).strip() == "":
raise Exception(ERROR_BLANK_PROJECT_ID)
ci = client_info.ClientInfo(client_library_version=get_user_agent_slug())
client = monitoring_v3.MetricServiceClient(client_info=ci)
exporter = StackdriverStatsExporter(client=client, options=options)
if options.default_monitoring_labels is not None:
exporter.set_default_labels(options.default_monitoring_labels)
return exporter
def __init__(
self,
project=None,
credentials=None,
read_only=False,
admin=False,
client_info=None,
client_options=None,
admin_client_options=None,
channel=None,
):
if client_info is None:
client_info = client_info_lib.ClientInfo(
client_library_version=bigtable.__version__, )
if read_only and admin:
raise ValueError("A read-only client cannot also perform"
"administrative actions.")
# NOTE: We set the scopes **before** calling the parent constructor.
# It **may** use those scopes in ``with_scopes_if_required``.
self._read_only = bool(read_only)
self._admin = bool(admin)
self._client_info = client_info
self._emulator_host = os.getenv(BIGTABLE_EMULATOR)
if self._emulator_host is not None:
if credentials is None:
credentials = AnonymousCredentials()
if project is None:
project = _DEFAULT_BIGTABLE_EMULATOR_CLIENT
if channel is not None:
warnings.warn(
"'channel' is deprecated and no longer used.",
DeprecationWarning,
stacklevel=2,
)
self._client_options = client_options
self._admin_client_options = admin_client_options
self._channel = channel
self.SCOPE = self._get_scopes()
super(Client, self).__init__(
project=project,
credentials=credentials,
client_options=client_options,
)
def __init__(self, project=None, instance=None, database=None, credentials=None, with_pager=False,
inp=None, output=None):
# setup environment variables
# less option for pager
if not os.environ.get(config.EnvironmentVariables.LESS):
os.environ[config.EnvironmentVariables.LESS] = config.Constants.LESS_FLAG
self.with_pager = with_pager
self.logger = logging.getLogger('spanner-cli')
self.logger.debug("Staring spanner-cli project=%s, instance=%s, database=%s", project, instance, database)
self.project = project
with warnings.catch_warnings(record=True) as warns:
warnings.simplefilter("always")
self.client = spanner.Client(
project=self.project,
credentials=credentials,
client_info=client_info.ClientInfo(user_agent=__name__),
)
if len(warns) > 0:
for w in warns:
self.logger.debug(w)
click.echo(message=w.message, err=True, nl=True)
self.instance = self.client.instance(instance)
self.database = self.instance.database(database)
self.prompt_message = self.get_prompt_message()
self.completer = SQLCompleter()
self.open_history_file()
self.rehash()
self.session = PromptSession(
message=self.prompt_message,
lexer=PygmentsLexer(lexer.SpannerLexer),
completer=DynamicCompleter(lambda: self.completer),
style=style_from_pygments_cls(get_style_by_name(config.get_pygment_style())),
history=self.history,
auto_suggest=AutoSuggestFromHistory(),
input_processors=[ConditionalProcessor(
processor=HighlightMatchingBracketProcessor(
chars='[](){}'),
filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() # pylint: disable=invalid-unary-operand-type
)],
input=inp,
output=output,
)
self.formatter = tabular_output.TabularOutputFormatter('ascii')
def load_configuration(file_name):
if os.getenv('CONFIG'):
logger = logging.getLogger('pubsub2inbox')
secret_manager_url = os.getenv('CONFIG')
logger.debug('Loading configuration from Secret Manager: %s' %
(secret_manager_url))
client_info = grpc_client_info.ClientInfo(
user_agent='google-pso-tool/pubsub2inbox/1.0.0')
client = secretmanager.SecretManagerServiceClient(
client_info=client_info)
response = client.access_secret_version(name=secret_manager_url)
configuration = response.payload.data.decode('UTF-8')
else:
with open(file_name) as config_file:
configuration = config_file.read()
cfg = yaml.load(configuration, Loader=yaml.SafeLoader)
return cfg
def new_stats_exporter(options):
""" new_stats_exporter returns an exporter that
uploads stats data to Stackdriver Monitoring.
"""
if str(options.project_id).strip() == "":
raise Exception(ERROR_BLANK_PROJECT_ID)
ci = client_info.ClientInfo(client_library_version=get_user_agent_slug())
client = monitoring_v3.MetricServiceClient(client_info=ci)
exporter = StackdriverStatsExporter(client=client, options=options)
if options.default_monitoring_labels is not None:
exporter.set_default_labels(options.default_monitoring_labels)
else:
label = {}
key = remove_non_alphanumeric(get_task_value())
label[key] = OPENCENSUS_TASK_DESCRIPTION
exporter.set_default_labels(label)
return exporter
def test_constructor_w_client_info():
from google.cloud import bigquery_storage
class MyTransport:
def __init__(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
transport_class_patcher = mock.patch.object(
bigquery_storage.BigQueryReadClient,
"get_transport_class",
return_value=MyTransport,
)
with transport_class_patcher:
client_under_test = bigquery_storage.BigQueryReadClient(
client_info=client_info.ClientInfo(
client_library_version="test-client-version"), )
transport_client_info = client_under_test._transport.kwargs["client_info"]
user_agent = transport_client_info.to_user_agent()
assert "test-client-version" in user_agent
def _read_charging_codes(self):
if len(self.charging_codes) == 0:
client_info = grpc_client_info.ClientInfo(
user_agent='google-pso-tool/turbo-project-factory/1.0.0')
storage_client = storage.Client(client_info=client_info)
bucket = storage_client.bucket(self.bucket)
blob = bucket.blob(self.object_path)
if blob.exists():
blob_contents = io.BytesIO()
storage_client.download_blob_to_file(blob, blob_contents)
charging_codes = json.loads(
blob_contents.getvalue().decode('utf-8'))
if isinstance(charging_codes, list):
for code in charging_codes:
self.charging_codes.append({
'id': code,
'title': code,
'description': None,
'group': ''
})
else:
self.charging_codes = charging_codes
def make_datastore_api(client):
"""Create an instance of the GAPIC Datastore API.
:type client: :class:`~google.cloud.datastore.client.Client`
:param client: The client that holds configuration details.
:rtype: :class:`.datastore.v1.datastore_client.DatastoreClient`
:returns: A datastore API instance with the proper credentials.
"""
parse_result = six.moves.urllib_parse.urlparse(client._base_url)
host = parse_result.netloc
if parse_result.scheme == "https":
channel = make_secure_channel(client._credentials, DEFAULT_USER_AGENT,
host)
else:
channel = insecure_channel(host)
return datastore_client.DatastoreClient(
channel=channel,
client_info=client_info.ClientInfo(client_library_version=__version__,
gapic_version=__version__),
)
def test_to_grpc_metadata():
info = client_info.ClientInfo()
metadata = info.to_grpc_metadata()
assert metadata == (client_info.METRICS_METADATA_KEY, info.to_user_agent())
def process_message(config, data, event, context):
logger = logging.getLogger('pubsub2inbox')
# Ignore messages submitted before our retry period
retry_period = '2 days ago'
if 'retryPeriod' in config:
retry_period = config['retryPeriod']
retry_period_parsed = parsedatetime.Calendar().parse(retry_period)
if len(retry_period_parsed) > 1:
retry_earliest = datetime.fromtimestamp(mktime(retry_period_parsed[0]),
timezone.utc)
else:
retry_earliest = datetime.fromtimestamp(mktime(retry_period_parsed),
timezone.utc)
message_time = parser.parse(context.timestamp)
if (message_time - retry_earliest) < timedelta(0, 0):
logger.warning('Ignoring message because it\'s past the retry period.',
extra={
'event_id': context.event_id,
'retry_period': retry_period,
'retry_earliest': retry_earliest.strftime('%c'),
'event_timestamp': message_time
})
raise MessageTooOldException(
'Ignoring message because it\'s past the retry period.')
template_variables = {
'data': data,
'event': event,
'context': context,
}
jinja_environment = get_jinja_environment()
if 'processors' in config:
for processor in config['processors']:
logger.debug('Processing message using input processor: %s' %
processor)
mod = __import__('processors.%s' % processor)
processor_module = getattr(mod, processor)
processor_class = getattr(processor_module,
'%sProcessor' % processor.capitalize())
processor_instance = processor_class(config, jinja_environment,
data, event, context)
processor_variables = processor_instance.process()
template_variables.update(processor_variables)
jinja_environment.globals = template_variables
if 'processIf' in config:
processif_template = jinja_environment.from_string(config['processIf'])
processif_template.name = 'processif'
processif_contents = processif_template.render()
if processif_contents.strip() == '':
logger.info(
'Will not send message because processIf evaluated to empty.')
return
if 'resendBucket' in config:
if 'resendPeriod' not in config:
raise NoResendConfigException(
'No resendPeriod configured, even though resendBucket is set!')
resend_key_hash = hashlib.sha256()
if 'resendKey' not in config:
default_resend_key = template_variables.copy()
default_resend_key.pop('context')
resend_key_hash.update(
json.dumps(default_resend_key).encode('utf-8'))
else:
key_template = jinja_environment.from_string(config['resendKey'])
key_template.name = 'resend'
key_contents = key_template.render()
resend_key_hash.update(key_contents.encode('utf-8'))
resend_file = resend_key_hash.hexdigest()
logger.debug('Checking for resend object in bucket...',
extra={
'bucket': config['resendBucket'],
'blob': resend_file
})
client_info = grpc_client_info.ClientInfo(
user_agent='google-pso-tool/pubsub2inbox/1.0.0')
storage_client = storage.Client(client_info=client_info)
bucket = storage_client.bucket(config['resendBucket'])
resend_blob = bucket.blob(resend_file)
if resend_blob.exists():
resend_blob.reload()
resend_period = config['resendPeriod']
resend_period_parsed = parsedatetime.Calendar().parse(
resend_period, sourceTime=resend_blob.time_created)
if len(resend_period_parsed) > 1:
resend_earliest = datetime.fromtimestamp(
mktime(resend_period_parsed[0]))
else:
resend_earliest = datetime.fromtimestamp(
mktime(resend_period_parsed))
if datetime.now() >= resend_earliest:
logger.debug('Resending the message now.',
extra={
'resend_earliest': resend_earliest,
'blob_time_created': resend_blob.time_created
})
resend_blob.upload_from_string('')
else:
logger.info(
'Can\'t resend the message now, resend period not elapsed.',
extra={
'resend_earliest': resend_earliest,
'blob_time_created': resend_blob.time_created
})
return
else:
try:
resend_blob.upload_from_string('', if_generation_match=0)
except Exception as exc:
# Handle TOCTOU condition
if 'conditionNotMet' in str(exc):
logger.warning(
'Message (re)sending already in progress (resend key already exist).',
extra={'exception': exc})
return
else:
raise exc
return
if 'outputs' in config:
for output_config in config['outputs']:
if 'type' not in output_config:
raise NoTypeConfiguredException(
'No type configured for output!')
logger.debug('Processing message using output processor: %s' %
output_config['type'])
output_type = output_config['type']
mod = __import__('output.%s' % output_type)
output_module = getattr(mod, output_type)
output_class = getattr(output_module,
'%sOutput' % output_type.capitalize())
output_instance = output_class(config, output_config,
jinja_environment, data, event,
context)
output_instance.output()
else:
raise NoOutputsConfiguredException('No outputs configured!')
def process(self):
if 'recommendations' not in self.config:
raise NotConfiguredException(
'No Recommender configuration specified in config!')
recommender_config = self.config['recommendations']
for recommender in recommender_config['recommender_types']:
if recommender not in self.recommenders:
raise UnknownRecommenderException(
'Unknown recommender %s specified in config!' %
(recommender))
data = json.loads(self.data)
self.jinja_environment.globals = {
**self.jinja_environment.globals,
**data
}
projects = []
if 'projects' in recommender_config:
projects = self._jinja_var_to_list(recommender_config['projects'],
'projects')
folders = []
if 'folders' in recommender_config:
folders = self._jinja_var_to_list(recommender_config['folders'],
'folders')
organizations = []
if 'organizations' in recommender_config:
organizations = self._jinja_var_to_list(
recommender_config['organizations'], 'organizations')
billing_accounts = []
if 'billingAccounts' in recommender_config:
billing_accounts = self._jinja_var_to_list(
recommender_config['billingAccounts'], 'billing_accounts')
if len(projects) == 0 and len(folders) == 0 and len(
organizations) == 0 and len(billing_accounts) == 0:
raise NotConfiguredException(
'No projects, organizations, folders or billing accounts specified in config!'
)
location_filters = self._jinja_var_to_list(
recommender_config['locations'], 'locations')
if len(location_filters) == 0:
raise NotConfiguredException(
'No location filters specified in config!')
client_info = grpc_client_info.ClientInfo(
user_agent='google-pso-tool/pubsub2inbox/1.1.0')
client = RecommenderClient(client_info=client_info)
credentials, project_id = google.auth.default(
['https://www.googleapis.com/auth/cloud-platform'])
branded_http = google_auth_httplib2.AuthorizedHttp(credentials)
branded_http = http.set_user_agent(
branded_http, 'google-pso-tool/pubsub2inbox/1.1.0')
compute_service = discovery.build('compute', 'v1', http=branded_http)
if len(projects) == 0:
raise NotConfiguredException(
'Please specify at least one project to fetch regions and zones.'
)
all_zones = self.get_zones(compute_service, projects[0],
location_filters)
all_regions = self.get_regions(compute_service, projects[0],
location_filters)
all_locations = all_zones + all_regions
self.logger.debug('Fetched all available locations.',
extra={'locations': all_locations})
parents = []
for project in self.expand_projects(projects):
parents.append(('projects/%s' % project[1], project))
for organization in organizations:
parents.append(('organizations/%s' % organization, [organization]))
for folder in folders:
parents.append(('folder/%s' % folder, [folder]))
for billing_account in billing_accounts:
parents.append(
('billingAccounts/%s' % billing_account, [billing_account]))
self.logger.debug('Determined all parents.',
extra={'parents': parents})
recommendations = {}
recommendations_rollup = {}
if 'fetch_recommendations' in recommender_config:
fetch_recommendations = self._jinja_expand_bool(
recommender_config['fetch_recommendations'])
if fetch_recommendations:
recommendations = self.get_recommendations(
client, recommender_config['recommender_types'], parents,
all_locations, recommender_config['recommendation_filter']
if 'recommendation_filter' in recommender_config else None)
recommendations_rollup = self.rollup_recommendations(
recommendations)
insights = {}
insights_rollup = {}
if 'fetch_insights' in recommender_config:
fetch_insights = self._jinja_expand_bool(
recommender_config['fetch_insights'])
if fetch_insights:
insights = self.get_insights(
client, recommender_config['insight_types'], parents,
all_locations, recommender_config['insight_filter']
if 'insight_filter' in recommender_config else None)
insights_rollup = self.rollup_insights(insights)
self.logger.debug('Fetching recommendations and/or insights finished.')
_ret = {
'recommendations': recommendations,
'recommendations_rollup': recommendations_rollup,
'insights': insights,
'insights_rollup': insights_rollup,
}
if 'vars' in recommender_config:
return {**recommender_config['vars'], **_ret}
return _ret
description='Group members'),
)),
bigquery.SchemaField("chargingCode",
"STRING",
mode="REQUIRED",
description="Chargeback code"),
bigquery.SchemaField("budget",
"INT64",
mode="NULLABLE",
description="Project budget"),
bigquery.SchemaField("environment",
"STRING",
mode="REQUIRED",
description="Environment"),
bigquery.SchemaField("status",
"STRING",
mode="REQUIRED",
description="Project status"),
],
write_disposition=bigquery.job.WriteDisposition.WRITE_TRUNCATE)
client_info = grpc_client_info.ClientInfo(
user_agent='google-pso-tool/turbo-project-factory/1.0.0')
client = bigquery.Client(client_info=client_info)
print('Writing projects to Bigquery (%s)...' % config['bigqueryDestination'],
file=sys.stderr)
client.load_table_from_json(rows,
bigquery.table.Table(config['bigqueryDestination']),
job_config=job_config)
print('All done.', file=sys.stderr)
from google.cloud.firestore_v1.base_collection import BaseCollectionReference
from google.cloud.firestore_v1.base_document import BaseDocumentReference
from google.cloud.firestore_v1.base_transaction import BaseTransaction
from google.cloud.firestore_v1.base_batch import BaseWriteBatch
from google.cloud.firestore_v1.base_query import BaseQuery
DEFAULT_DATABASE = "(default)"
"""str: The default database used in a :class:`~google.cloud.firestore_v1.client.Client`."""
_DEFAULT_EMULATOR_PROJECT = "google-cloud-firestore-emulator"
_BAD_OPTION_ERR = ("Exactly one of ``last_update_time`` or ``exists`` "
"must be provided.")
_BAD_DOC_TEMPLATE: str = (
"Document {!r} appeared in response but was not present among references")
_ACTIVE_TXN: str = "There is already an active transaction."
_INACTIVE_TXN: str = "There is no active transaction."
_CLIENT_INFO: Any = client_info.ClientInfo(client_library_version=__version__)
_FIRESTORE_EMULATOR_HOST: str = "FIRESTORE_EMULATOR_HOST"
class BaseClient(ClientWithProject):
"""Client for interacting with Google Cloud Firestore API.
.. note::
Since the Cloud Firestore API requires the gRPC transport, no
``_http`` argument is accepted by this class.
Args:
project (Optional[str]): The project which the client acts on behalf
of. If not passed, falls back to the default inferred
from the environment.
def output(self):
if 'sourceBucket' not in self.output_config:
raise NotConfiguredException(
'No source bucket defined in GCS output.')
if 'sourceObject' not in self.output_config:
raise NotConfiguredException(
'No source object defined in GCS output.')
if 'destinationBucket' not in self.output_config:
raise NotConfiguredException(
'No destination bucket defined in GCS output.')
if 'destinationObject' not in self.output_config:
raise NotConfiguredException(
'No destination object defined in GCS output.')
bucket_template = self.jinja_environment.from_string(
self.output_config['sourceBucket'])
bucket_template.name = 'bucket'
source_bucket = bucket_template.render()
object_template = self.jinja_environment.from_string(
self.output_config['sourceObject'])
object_template.name = 'object'
source_object = object_template.render()
bucket_template = self.jinja_environment.from_string(
self.output_config['destinationBucket'])
bucket_template.name = 'bucket'
destination_bucket = bucket_template.render()
object_template = self.jinja_environment.from_string(
self.output_config['destinationObject'])
object_template.name = 'object'
destination_object = object_template.render()
self.logger.debug('Starting to copy source to destination.',
extra={
'source_url':
'gs://%s/%s' % (source_bucket, source_object),
'destination_url':
'gs://%s/%s' %
(destination_bucket, destination_object)
})
client_info = grpc_client_info.ClientInfo(
user_agent='google-pso-tool/pubsub2inbox/1.1.0')
project = self.output_config[
'project'] if 'project' in self.output_config else None
storage_client = storage.Client(client_info=client_info,
project=project)
bucket = storage_client.bucket(source_bucket)
source_blob = bucket.blob(source_object)
bucket = storage_client.bucket(destination_bucket)
destination_blob = bucket.blob(destination_object)
token = None
while True:
self.logger.debug(
'Copying file...',
extra={
'token':
token,
'source_url':
'gs://%s/%s' % (source_bucket, source_object),
'destination_url':
'gs://%s/%s' % (destination_bucket, destination_object)
})
ret = destination_blob.rewrite(source_blob, token=token)
token = ret[0]
if token is None:
break
self.logger.info('Object copied from source to destination.',
extra={
'bytes_rewritten':
ret[1],
'total_bytes':
ret[2],
'source_url':
'gs://%s/%s' % (source_bucket, source_object),
'destination_url':
'gs://%s/%s' %
(destination_bucket, destination_object)
})