def ReportMetrics(metrics_file_path, log_level): """Sends the specified anonymous usage event to the given analytics endpoint. Args: metrics_file_path: str, File with pickled metrics (list of tuples). log_level: int, The logging level of gsutil's root logger. """ logger = logging.getLogger() handler = logging.FileHandler(LOG_FILE_PATH, mode='w') logger.addHandler(handler) logger.setLevel(log_level) with open(metrics_file_path, 'rb') as metrics_file: metrics = pickle.load(metrics_file) os.remove(metrics_file_path) http = GetNewHttp() for metric in metrics: try: headers = {'User-Agent': metric.user_agent} response = http.request(metric.endpoint, method=metric.method, body=metric.body, headers=headers) logger.debug(metric) logger.debug('RESPONSE: %s', response[0]['status']) except Exception as e: # pylint: disable=broad-except logger.debug(e)
def _CheckClientCanRead(self, key, client_id, gcs_path): """Performs a head request against a signed url to check for read access.""" signed_url = _GenSignedUrl(key, client_id, 'HEAD', '', '', int(time.time()) + 10, gcs_path) h = GetNewHttp() try: response, _ = h.request(signed_url, 'HEAD') return response.status == 200 except httplib2.HttpLib2Error as e: raise CommandException('Unexpected error while querying' 'object readability ({0})'.format( e.message))
def _ProbeObjectAccessWithClient(self, key, client_email, gcs_path, logger): """Performs a head request against a signed url to check for read access.""" # Choose a reasonable time in the future; if the user's system clock is # 60 or more seconds behind the server's this will generate an error. signed_url = _GenSignedUrl(key, client_email, 'HEAD', '', '', int(time.time()) + 60, gcs_path, logger) try: h = GetNewHttp() req = Request(signed_url, 'HEAD') response = MakeRequest(h, req) if response.status_code not in [200, 403, 404]: raise HttpError.FromResponse(response) return response.status_code except HttpError: error_string = ( 'Unexpected HTTP response code %s while querying ' 'object readability. Is your system clock accurate?' % response.status_code) if response.content: error_string += ' Content: %s' % response.content raise CommandException(error_string)
def _CheckClientCanRead(self, key, client_id, gcs_path): """Performs a head request against a signed url to check for read access.""" signed_url = _GenSignedUrl(key, client_id, 'HEAD', '', '', int(time.time()) + 10, gcs_path) h = GetNewHttp() try: response, _ = h.request(signed_url, 'HEAD') return response.status == 200 except httplib2.HttpLib2Error as e: raise CommandException('Unexpected error while querying' 'object readability ({0})' .format(e.message))
def _ProbeObjectAccessWithClient(self, key, client_email, gcs_path, logger, region): """Performs a head request against a signed url to check for read access.""" # Choose a reasonable time in the future; if the user's system clock is # 60 or more seconds behind the server's this will generate an error. signed_url = _GenSignedUrl(key, client_email, 'HEAD', timedelta(seconds=60), gcs_path, logger, region) try: h = GetNewHttp() req = Request(signed_url, 'HEAD') response = MakeRequest(h, req) if response.status_code not in [200, 403, 404]: raise HttpError.FromResponse(response) return response.status_code except HttpError as http_error: if http_error.has_attr('response'): error_response = http_error.response error_string = ( 'Unexpected HTTP response code %s while querying ' 'object readability. Is your system clock accurate?' % error_response.status_code) if error_response.content: error_string += ' Content: %s' % error_response.content else: error_string = ( 'Expected an HTTP response code of ' '200 while querying object readability, but received ' 'an error: %s' % http_error) raise CommandException(error_string)
def __init__(self, logger=None, credentials=None, debug=0): """Performs necessary setup for interacting with Google Cloud Pub/Sub. Args: logger: logging.logger for outputting log messages. credentials: Credentials to be used for interacting with Google Cloud Pub/Sub debug: Debug level for the API implementation (0..3). """ super(PubsubApi, self).__init__() self.logger = logger no_op_credentials = False if not credentials: loaded_credentials = CheckAndGetCredentials(logger) if not loaded_credentials: loaded_credentials = NoOpCredentials() no_op_credentials = True else: if isinstance(credentials, NoOpCredentials): no_op_credentials = True self.credentials = credentials or loaded_credentials self.certs_file = GetCertsFile() self.http = GetNewHttp() self.http_base = 'https://' self.host_base = config.get('Credentials', 'gs_pubsub_host', 'pubsub.googleapis.com') gs_pubsub_port = config.get('Credentials', 'gs_pubsub_port', None) if not gs_pubsub_port: self.host_port = '' else: self.host_port = ':' + gs_pubsub_port self.url_base = (self.http_base + self.host_base + self.host_port) self.num_retries = GetNumRetries() self.max_retry_wait = GetMaxRetryDelay() log_request = (debug >= 3) log_response = (debug >= 3) self.api_client = apitools_client.PubsubV1( url=self.url_base, http=self.http, log_request=log_request, log_response=log_response, credentials=self.credentials) self.api_client.max_retry_wait = self.max_retry_wait self.api_client.num_retries = self.num_retries if no_op_credentials: # This API key is not secret and is used to identify gsutil during # anonymous requests. self.api_client.AddGlobalParam( 'key', u'AIzaSyDnacJHrKma0048b13sh8cgxNUwulubmJM')
def ReportMetrics(metrics_file_path, log_level, log_file_path=None): """Sends the specified anonymous usage event to the given analytics endpoint. Args: metrics_file_path: str, File with pickled metrics (list of tuples). log_level: int, The logging level of gsutil's root logger. log_file_path: str, The file that this module should write its logs to. This parameter is intended for use by tests that need to evaluate the contents of the file at this path. """ logger = logging.getLogger() if log_file_path is not None: # Use a separate logger so that we don't add another handler to the default # module-level logger. This is intended to prevent multiple calls from tests # running in parallel from writing output to the same file. new_name = '%s.%s' % ( logger.name, ''.join(random.choice(string.ascii_lowercase) for _ in range(8))) logger = logging.getLogger(new_name) handler = logging.FileHandler(log_file_path or LOG_FILE_PATH, mode='w') logger.addHandler(handler) logger.setLevel(log_level) with open(metrics_file_path, 'rb') as metrics_file: metrics = pickle.load(metrics_file) os.remove(metrics_file_path) ConfigureCertsFile() http = GetNewHttp() for metric in metrics: try: headers = {'User-Agent': metric.user_agent} response = http.request(metric.endpoint, method=metric.method, body=metric.body, headers=headers) logger.debug(metric) logger.debug('RESPONSE: %s', response[0]['status']) except Exception as e: # pylint: disable=broad-except logger.debug(e)
def _ProbeObjectAccessWithClient(self, key, client_email, gcs_path): """Performs a head request against a signed url to check for read access.""" signed_url = _GenSignedUrl(key, client_email, 'HEAD', '', '', int(time.time()) + 10, gcs_path) try: h = GetNewHttp() req = Request(signed_url, 'HEAD') response = MakeRequest(h, req) if response.status_code not in [200, 403, 404]: raise HttpError(response) return response.status_code except HttpError as e: raise CommandException('Unexpected response code while querying' 'object readability ({0})'.format(e.message))
def _UploadObject(self, upload_stream, object_metadata, canned_acl=None, size=None, preconditions=None, provider=None, fields=None, serialization_data=None, tracker_callback=None, progress_callback=None, apitools_strategy='simple'): """Upload implementation, apitools_strategy plus gsutil Cloud API args.""" ValidateDstObjectMetadata(object_metadata) assert not canned_acl, 'Canned ACLs not supported by JSON API.' bytes_uploaded_container = BytesUploadedContainer() callback_per_bytes = CALLBACK_PER_X_BYTES total_size = 0 if progress_callback and size: total_size = size progress_callback(0, size) callback_class_factory = UploadCallbackConnectionClassFactory( bytes_uploaded_container, total_size=total_size, callback_per_bytes=callback_per_bytes, progress_callback=progress_callback) upload_http = GetNewHttp() upload_http_class = callback_class_factory.GetConnectionClass() upload_http.connections = {'http': upload_http_class, 'https': upload_http_class} # Disable apitools' default print callbacks. def _NoopCallback(unused_response, unused_upload_object): pass authorized_upload_http = self.credentials.authorize(upload_http) WrapUploadHttpRequest(authorized_upload_http) # Since bytes_http is created in this function, we don't get the # user-agent header from api_client's http automatically. additional_headers = { 'user-agent': self.api_client.user_agent } try: if not serialization_data: # This is a new upload, set up initial upload state. content_type = object_metadata.contentType if not content_type: content_type = DEFAULT_CONTENT_TYPE if not preconditions: preconditions = Preconditions() apitools_request = apitools_messages.StorageObjectsInsertRequest( bucket=object_metadata.bucket, object=object_metadata, ifGenerationMatch=preconditions.gen_match, ifMetagenerationMatch=preconditions.meta_gen_match) global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) if apitools_strategy == 'simple': # One-shot upload. apitools_upload = apitools_transfer.Upload( upload_stream, content_type, total_size=size, auto_transfer=True) apitools_upload.strategy = apitools_strategy apitools_upload.bytes_http = authorized_upload_http return self.api_client.objects.Insert( apitools_request, upload=apitools_upload, global_params=global_params) else: # Resumable upload. try: if serialization_data: # Resuming an existing upload. apitools_upload = apitools_transfer.Upload.FromData( upload_stream, serialization_data, self.api_client.http) apitools_upload.chunksize = _ResumableChunkSize() apitools_upload.bytes_http = authorized_upload_http else: # New resumable upload. apitools_upload = apitools_transfer.Upload( upload_stream, content_type, total_size=size, chunksize=_ResumableChunkSize(), auto_transfer=False) apitools_upload.strategy = apitools_strategy apitools_upload.bytes_http = authorized_upload_http self.api_client.objects.Insert( apitools_request, upload=apitools_upload, global_params=global_params) # If we're resuming an upload, apitools has at this point received # from the server how many bytes it already has. Update our # callback class with this information. bytes_uploaded_container.bytes_uploaded = apitools_upload.progress if tracker_callback: tracker_callback(json.dumps(apitools_upload.serialization_data)) http_response = apitools_upload.StreamInChunks( callback=_NoopCallback, finish_callback=_NoopCallback, additional_headers=additional_headers) return self.api_client.objects.ProcessHttpResponse( self.api_client.objects.GetMethodConfig('Insert'), http_response) except TRANSLATABLE_APITOOLS_EXCEPTIONS, e: resumable_ex = self._TranslateApitoolsResumableUploadException(e) if resumable_ex: raise resumable_ex else: raise except TRANSLATABLE_APITOOLS_EXCEPTIONS, e: self._TranslateExceptionAndRaise(e, bucket_name=object_metadata.bucket, object_name=object_metadata.name)
def _UploadObject(self, upload_stream, object_metadata, canned_acl=None, size=None, preconditions=None, provider=None, fields=None, serialization_data=None, tracker_callback=None, progress_callback=None, apitools_strategy='simple'): """Upload implementation, apitools_strategy plus gsutil Cloud API args.""" ValidateDstObjectMetadata(object_metadata) assert not canned_acl, 'Canned ACLs not supported by JSON API.' bytes_uploaded_container = BytesTransferredContainer() total_size = 0 if progress_callback and size: total_size = size progress_callback(0, size) callback_class_factory = UploadCallbackConnectionClassFactory( bytes_uploaded_container, total_size=total_size, progress_callback=progress_callback) upload_http = GetNewHttp() upload_http_class = callback_class_factory.GetConnectionClass() upload_http.connections = {'http': upload_http_class, 'https': upload_http_class} # Disable apitools' default print callbacks. def _NoopCallback(unused_response, unused_upload_object): pass authorized_upload_http = self.credentials.authorize(upload_http) WrapUploadHttpRequest(authorized_upload_http) # Since bytes_http is created in this function, we don't get the # user-agent header from api_client's http automatically. additional_headers = { 'user-agent': self.api_client.user_agent } try: content_type = None apitools_request = None global_params = None if not serialization_data: # This is a new upload, set up initial upload state. content_type = object_metadata.contentType if not content_type: content_type = DEFAULT_CONTENT_TYPE if not preconditions: preconditions = Preconditions() apitools_request = apitools_messages.StorageObjectsInsertRequest( bucket=object_metadata.bucket, object=object_metadata, ifGenerationMatch=preconditions.gen_match, ifMetagenerationMatch=preconditions.meta_gen_match) global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) if apitools_strategy == 'simple': # One-shot upload. apitools_upload = apitools_transfer.Upload( upload_stream, content_type, total_size=size, auto_transfer=True) apitools_upload.strategy = apitools_strategy apitools_upload.bytes_http = authorized_upload_http return self.api_client.objects.Insert( apitools_request, upload=apitools_upload, global_params=global_params) else: # Resumable upload. return self._PerformResumableUpload( upload_stream, authorized_upload_http, content_type, size, serialization_data, apitools_strategy, apitools_request, global_params, bytes_uploaded_container, tracker_callback, _NoopCallback, additional_headers) except TRANSLATABLE_APITOOLS_EXCEPTIONS, e: self._TranslateExceptionAndRaise(e, bucket_name=object_metadata.bucket, object_name=object_metadata.name)