def ReportMetrics(metrics_file_path, log_level):
"""Sends the specified anonymous usage event to the given analytics endpoint.
Args:
metrics_file_path: str, File with pickled metrics (list of tuples).
log_level: int, The logging level of gsutil's root logger.
"""
logger = logging.getLogger()
handler = logging.FileHandler(LOG_FILE_PATH, mode='w')
logger.addHandler(handler)
logger.setLevel(log_level)
with open(metrics_file_path, 'rb') as metrics_file:
metrics = pickle.load(metrics_file)
os.remove(metrics_file_path)
http = GetNewHttp()
for metric in metrics:
try:
headers = {'User-Agent': metric.user_agent}
response = http.request(metric.endpoint,
method=metric.method,
body=metric.body,
headers=headers)
logger.debug(metric)
logger.debug('RESPONSE: %s', response[0]['status'])
except Exception as e: # pylint: disable=broad-except
logger.debug(e)
def ReportMetrics(metrics_file_path, log_level):
"""Sends the specified anonymous usage event to the given analytics endpoint.
Args:
metrics_file_path: str, File with pickled metrics (list of tuples).
log_level: int, The logging level of gsutil's root logger.
"""
logger = logging.getLogger()
handler = logging.FileHandler(LOG_FILE_PATH, mode='w')
logger.addHandler(handler)
logger.setLevel(log_level)
with open(metrics_file_path, 'rb') as metrics_file:
metrics = pickle.load(metrics_file)
os.remove(metrics_file_path)
http = GetNewHttp()
for metric in metrics:
try:
headers = {'User-Agent': metric.user_agent}
response = http.request(metric.endpoint,
method=metric.method,
body=metric.body,
headers=headers)
logger.debug(metric)
logger.debug('RESPONSE: %s', response[0]['status'])
except Exception as e: # pylint: disable=broad-except
logger.debug(e)
def _CheckClientCanRead(self, key, client_id, gcs_path):
"""Performs a head request against a signed url to check for read access."""
signed_url = _GenSignedUrl(key, client_id, 'HEAD', '', '',
int(time.time()) + 10, gcs_path)
h = GetNewHttp()
try:
response, _ = h.request(signed_url, 'HEAD')
return response.status == 200
except httplib2.HttpLib2Error as e:
raise CommandException('Unexpected error while querying'
'object readability ({0})'.format(
e.message))
def _ProbeObjectAccessWithClient(self, key, client_email, gcs_path,
logger):
"""Performs a head request against a signed url to check for read access."""
# Choose a reasonable time in the future; if the user's system clock is
# 60 or more seconds behind the server's this will generate an error.
signed_url = _GenSignedUrl(key, client_email, 'HEAD', '', '',
int(time.time()) + 60, gcs_path, logger)
try:
h = GetNewHttp()
req = Request(signed_url, 'HEAD')
response = MakeRequest(h, req)
if response.status_code not in [200, 403, 404]:
raise HttpError.FromResponse(response)
return response.status_code
except HttpError:
error_string = (
'Unexpected HTTP response code %s while querying '
'object readability. Is your system clock accurate?' %
response.status_code)
if response.content:
error_string += ' Content: %s' % response.content
raise CommandException(error_string)
def _CheckClientCanRead(self, key, client_id, gcs_path):
"""Performs a head request against a signed url to check for read access."""
signed_url = _GenSignedUrl(key, client_id,
'HEAD', '', '',
int(time.time()) + 10,
gcs_path)
h = GetNewHttp()
try:
response, _ = h.request(signed_url, 'HEAD')
return response.status == 200
except httplib2.HttpLib2Error as e:
raise CommandException('Unexpected error while querying'
'object readability ({0})'
.format(e.message))
def _ProbeObjectAccessWithClient(self, key, client_email, gcs_path, logger,
region):
"""Performs a head request against a signed url to check for read access."""
# Choose a reasonable time in the future; if the user's system clock is
# 60 or more seconds behind the server's this will generate an error.
signed_url = _GenSignedUrl(key, client_email, 'HEAD',
timedelta(seconds=60), gcs_path, logger,
region)
try:
h = GetNewHttp()
req = Request(signed_url, 'HEAD')
response = MakeRequest(h, req)
if response.status_code not in [200, 403, 404]:
raise HttpError.FromResponse(response)
return response.status_code
except HttpError as http_error:
if http_error.has_attr('response'):
error_response = http_error.response
error_string = (
'Unexpected HTTP response code %s while querying '
'object readability. Is your system clock accurate?' %
error_response.status_code)
if error_response.content:
error_string += ' Content: %s' % error_response.content
else:
error_string = (
'Expected an HTTP response code of '
'200 while querying object readability, but received '
'an error: %s' % http_error)
raise CommandException(error_string)
def __init__(self, logger=None, credentials=None, debug=0):
"""Performs necessary setup for interacting with Google Cloud Pub/Sub.
Args:
logger: logging.logger for outputting log messages.
credentials: Credentials to be used for interacting with Google Cloud
Pub/Sub
debug: Debug level for the API implementation (0..3).
"""
super(PubsubApi, self).__init__()
self.logger = logger
no_op_credentials = False
if not credentials:
loaded_credentials = CheckAndGetCredentials(logger)
if not loaded_credentials:
loaded_credentials = NoOpCredentials()
no_op_credentials = True
else:
if isinstance(credentials, NoOpCredentials):
no_op_credentials = True
self.credentials = credentials or loaded_credentials
self.certs_file = GetCertsFile()
self.http = GetNewHttp()
self.http_base = 'https://'
self.host_base = config.get('Credentials', 'gs_pubsub_host',
'pubsub.googleapis.com')
gs_pubsub_port = config.get('Credentials', 'gs_pubsub_port', None)
if not gs_pubsub_port:
self.host_port = ''
else:
self.host_port = ':' + gs_pubsub_port
self.url_base = (self.http_base + self.host_base + self.host_port)
self.num_retries = GetNumRetries()
self.max_retry_wait = GetMaxRetryDelay()
log_request = (debug >= 3)
log_response = (debug >= 3)
self.api_client = apitools_client.PubsubV1(
url=self.url_base,
http=self.http,
log_request=log_request,
log_response=log_response,
credentials=self.credentials)
self.api_client.max_retry_wait = self.max_retry_wait
self.api_client.num_retries = self.num_retries
if no_op_credentials:
# This API key is not secret and is used to identify gsutil during
# anonymous requests.
self.api_client.AddGlobalParam(
'key', u'AIzaSyDnacJHrKma0048b13sh8cgxNUwulubmJM')
def ReportMetrics(metrics_file_path, log_level, log_file_path=None):
"""Sends the specified anonymous usage event to the given analytics endpoint.
Args:
metrics_file_path: str, File with pickled metrics (list of tuples).
log_level: int, The logging level of gsutil's root logger.
log_file_path: str, The file that this module should write its logs to.
This parameter is intended for use by tests that need to evaluate the
contents of the file at this path.
"""
logger = logging.getLogger()
if log_file_path is not None:
# Use a separate logger so that we don't add another handler to the default
# module-level logger. This is intended to prevent multiple calls from tests
# running in parallel from writing output to the same file.
new_name = '%s.%s' % (
logger.name,
''.join(random.choice(string.ascii_lowercase) for _ in range(8)))
logger = logging.getLogger(new_name)
handler = logging.FileHandler(log_file_path or LOG_FILE_PATH, mode='w')
logger.addHandler(handler)
logger.setLevel(log_level)
with open(metrics_file_path, 'rb') as metrics_file:
metrics = pickle.load(metrics_file)
os.remove(metrics_file_path)
ConfigureCertsFile()
http = GetNewHttp()
for metric in metrics:
try:
headers = {'User-Agent': metric.user_agent}
response = http.request(metric.endpoint,
method=metric.method,
body=metric.body,
headers=headers)
logger.debug(metric)
logger.debug('RESPONSE: %s', response[0]['status'])
except Exception as e: # pylint: disable=broad-except
logger.debug(e)
def _ProbeObjectAccessWithClient(self, key, client_email, gcs_path):
"""Performs a head request against a signed url to check for read access."""
signed_url = _GenSignedUrl(key, client_email, 'HEAD', '', '',
int(time.time()) + 10, gcs_path)
try:
h = GetNewHttp()
req = Request(signed_url, 'HEAD')
response = MakeRequest(h, req)
if response.status_code not in [200, 403, 404]:
raise HttpError(response)
return response.status_code
except HttpError as e:
raise CommandException('Unexpected response code while querying'
'object readability ({0})'.format(e.message))
def _UploadObject(self, upload_stream, object_metadata, canned_acl=None,
size=None, preconditions=None, provider=None, fields=None,
serialization_data=None, tracker_callback=None,
progress_callback=None, apitools_strategy='simple'):
"""Upload implementation, apitools_strategy plus gsutil Cloud API args."""
ValidateDstObjectMetadata(object_metadata)
assert not canned_acl, 'Canned ACLs not supported by JSON API.'
bytes_uploaded_container = BytesUploadedContainer()
callback_per_bytes = CALLBACK_PER_X_BYTES
total_size = 0
if progress_callback and size:
total_size = size
progress_callback(0, size)
callback_class_factory = UploadCallbackConnectionClassFactory(
bytes_uploaded_container, total_size=total_size,
callback_per_bytes=callback_per_bytes,
progress_callback=progress_callback)
upload_http = GetNewHttp()
upload_http_class = callback_class_factory.GetConnectionClass()
upload_http.connections = {'http': upload_http_class,
'https': upload_http_class}
# Disable apitools' default print callbacks.
def _NoopCallback(unused_response, unused_upload_object):
pass
authorized_upload_http = self.credentials.authorize(upload_http)
WrapUploadHttpRequest(authorized_upload_http)
# Since bytes_http is created in this function, we don't get the
# user-agent header from api_client's http automatically.
additional_headers = {
'user-agent': self.api_client.user_agent
}
try:
if not serialization_data:
# This is a new upload, set up initial upload state.
content_type = object_metadata.contentType
if not content_type:
content_type = DEFAULT_CONTENT_TYPE
if not preconditions:
preconditions = Preconditions()
apitools_request = apitools_messages.StorageObjectsInsertRequest(
bucket=object_metadata.bucket, object=object_metadata,
ifGenerationMatch=preconditions.gen_match,
ifMetagenerationMatch=preconditions.meta_gen_match)
global_params = apitools_messages.StandardQueryParameters()
if fields:
global_params.fields = ','.join(set(fields))
if apitools_strategy == 'simple': # One-shot upload.
apitools_upload = apitools_transfer.Upload(
upload_stream, content_type, total_size=size, auto_transfer=True)
apitools_upload.strategy = apitools_strategy
apitools_upload.bytes_http = authorized_upload_http
return self.api_client.objects.Insert(
apitools_request,
upload=apitools_upload,
global_params=global_params)
else: # Resumable upload.
try:
if serialization_data:
# Resuming an existing upload.
apitools_upload = apitools_transfer.Upload.FromData(
upload_stream, serialization_data, self.api_client.http)
apitools_upload.chunksize = _ResumableChunkSize()
apitools_upload.bytes_http = authorized_upload_http
else:
# New resumable upload.
apitools_upload = apitools_transfer.Upload(
upload_stream, content_type, total_size=size,
chunksize=_ResumableChunkSize(), auto_transfer=False)
apitools_upload.strategy = apitools_strategy
apitools_upload.bytes_http = authorized_upload_http
self.api_client.objects.Insert(
apitools_request,
upload=apitools_upload,
global_params=global_params)
# If we're resuming an upload, apitools has at this point received
# from the server how many bytes it already has. Update our
# callback class with this information.
bytes_uploaded_container.bytes_uploaded = apitools_upload.progress
if tracker_callback:
tracker_callback(json.dumps(apitools_upload.serialization_data))
http_response = apitools_upload.StreamInChunks(
callback=_NoopCallback, finish_callback=_NoopCallback,
additional_headers=additional_headers)
return self.api_client.objects.ProcessHttpResponse(
self.api_client.objects.GetMethodConfig('Insert'), http_response)
except TRANSLATABLE_APITOOLS_EXCEPTIONS, e:
resumable_ex = self._TranslateApitoolsResumableUploadException(e)
if resumable_ex:
raise resumable_ex
else:
raise
except TRANSLATABLE_APITOOLS_EXCEPTIONS, e:
self._TranslateExceptionAndRaise(e, bucket_name=object_metadata.bucket,
object_name=object_metadata.name)
def _UploadObject(self, upload_stream, object_metadata, canned_acl=None,
size=None, preconditions=None, provider=None, fields=None,
serialization_data=None, tracker_callback=None,
progress_callback=None, apitools_strategy='simple'):
"""Upload implementation, apitools_strategy plus gsutil Cloud API args."""
ValidateDstObjectMetadata(object_metadata)
assert not canned_acl, 'Canned ACLs not supported by JSON API.'
bytes_uploaded_container = BytesTransferredContainer()
total_size = 0
if progress_callback and size:
total_size = size
progress_callback(0, size)
callback_class_factory = UploadCallbackConnectionClassFactory(
bytes_uploaded_container, total_size=total_size,
progress_callback=progress_callback)
upload_http = GetNewHttp()
upload_http_class = callback_class_factory.GetConnectionClass()
upload_http.connections = {'http': upload_http_class,
'https': upload_http_class}
# Disable apitools' default print callbacks.
def _NoopCallback(unused_response, unused_upload_object):
pass
authorized_upload_http = self.credentials.authorize(upload_http)
WrapUploadHttpRequest(authorized_upload_http)
# Since bytes_http is created in this function, we don't get the
# user-agent header from api_client's http automatically.
additional_headers = {
'user-agent': self.api_client.user_agent
}
try:
content_type = None
apitools_request = None
global_params = None
if not serialization_data:
# This is a new upload, set up initial upload state.
content_type = object_metadata.contentType
if not content_type:
content_type = DEFAULT_CONTENT_TYPE
if not preconditions:
preconditions = Preconditions()
apitools_request = apitools_messages.StorageObjectsInsertRequest(
bucket=object_metadata.bucket, object=object_metadata,
ifGenerationMatch=preconditions.gen_match,
ifMetagenerationMatch=preconditions.meta_gen_match)
global_params = apitools_messages.StandardQueryParameters()
if fields:
global_params.fields = ','.join(set(fields))
if apitools_strategy == 'simple': # One-shot upload.
apitools_upload = apitools_transfer.Upload(
upload_stream, content_type, total_size=size, auto_transfer=True)
apitools_upload.strategy = apitools_strategy
apitools_upload.bytes_http = authorized_upload_http
return self.api_client.objects.Insert(
apitools_request,
upload=apitools_upload,
global_params=global_params)
else: # Resumable upload.
return self._PerformResumableUpload(
upload_stream, authorized_upload_http, content_type, size,
serialization_data, apitools_strategy, apitools_request,
global_params, bytes_uploaded_container, tracker_callback,
_NoopCallback, additional_headers)
except TRANSLATABLE_APITOOLS_EXCEPTIONS, e:
self._TranslateExceptionAndRaise(e, bucket_name=object_metadata.bucket,
object_name=object_metadata.name)
