def _ExecuteApiRequest(self, request, retry_count=3): """ Executes a Compute API request and returns True on success. Returns: (True, Response) in case of success, or (False, error_content) otherwise. """ self._logger.info('Compute API request:\n' + request.to_json()) try: response = request.execute() self._logger.info('Compute API response:\n' + str(response)) return (True, response) except errors.HttpError as err: error_content = google_error_helper.GetErrorContent(err) error_reason = google_error_helper.GetErrorReason(error_content) if error_reason == 'resourceNotReady' and retry_count > 0: # Retry after a delay delay_seconds = 1 self._logger.info( 'Resource not ready, retrying in %i seconds.' % delay_seconds) time.sleep(delay_seconds) return self._ExecuteApiRequest(request, retry_count - 1) else: self._logger.error('Compute API error (reason: "%s"):\n%s' % (error_reason, err)) if error_content: self._logger.error('Error details:\n%s' % error_content) return (False, error_content)
def DoesBigQueryTableExist(bigquery_service, project_name, table_id, logger): """Returns wether the BigQuery table identified by table_id exists. Raises a HttpError exception if the call to BigQuery API fails. Args: bigquery_service: The BigQuery service. project_name: (str) Name of the Google Cloud project. table_id: (str) table_id as returned by GetBigQueryTableID(). Returns: bool: True if the table exists. """ table_name = BIGQUERY_TABLE_TEMPLATE + '_' + table_id logger.info('Getting table information for %s.' % table_name) try: table = bigquery_service.tables().get(projectId=project_name, datasetId=BIGQUERY_DATASET, tableId=table_name).execute() return bool(table) except errors.HttpError as http_error: error_content = google_error_helper.GetErrorContent(http_error) error_reason = google_error_helper.GetErrorReason(error_content) if error_reason == google_error_helper.REASON_NOT_FOUND: return False else: logger.error('BigQuery API error (reason: "%s"):\n%s' % ( error_reason, http_error)) if error_content: logger.error('Error details:\n%s' % error_content) raise # Re-raise the exception. return False
def _StreamRowsToBigQuery(self, rows, table_id): """Uploads a list of rows to the BigQuery table associated with the given table_id. Args: rows: (list of dict) Each dictionary is a row to add to the table. table_id: (str) Identifier of the BigQuery table to update. """ try: response = common.google_bigquery_helper.InsertInTemplatedBigQueryTable( self._bigquery_service, self._project_name, table_id, rows, self._logger) except errors.HttpError as http_error: # Handles HTTP error response codes (such as 404), typically indicating a # problem in parameters other than 'body'. error_content = google_error_helper.GetErrorContent(http_error) error_reason = google_error_helper.GetErrorReason(error_content) self._logger.error('BigQuery API error (reason: "%s"):\n%s' % (error_reason, http_error)) self._failure_database.AddFailure('big_query_error', error_reason) if error_content: self._logger.error('Error details:\n%s' % error_content) return # Handles other errors, typically when the body is ill-formatted. insert_errors = response.get('insertErrors') if insert_errors: self._logger.error('BigQuery API error:\n' + str(insert_errors)) for insert_error in insert_errors: self._failure_database.AddFailure( 'big_query_insert_error', str(insert_error.get('errors')))
def _StreamRowsToBigQuery(self, rows, table_id): """Uploads a list of rows to the BigQuery table associated with the given table_id. Args: rows: (list of dict) Each dictionary is a row to add to the table. table_id: (str) Identifier of the BigQuery table to update. """ # Assumes that the dataset and the table template already exist. dataset = 'clovis_dataset' template = 'report' rows_data = [{ 'json': row, 'insertId': str(uuid.uuid4()) } for row in rows] body = {'rows': rows_data, 'templateSuffix': '_' + table_id} self._logger.info('BigQuery API request:\n' + str(body)) try: response = self._bigquery_service.tabledata().insertAll( projectId=self._project_name, datasetId=dataset, tableId=template, body=body).execute() self._logger.info('BigQuery API response:\n' + str(response)) except errors.HttpError as http_error: # Handles HTTP error response codes (such as 404), typically indicating a # problem in parameters other than 'body'. error_content = google_error_helper.GetErrorContent(http_error) error_reason = google_error_helper.GetErrorReason(error_content) self._logger.error('BigQuery API error (reason: "%s"):\n%s' % (error_reason, http_error)) self._failure_database.AddFailure('big_query_error', error_reason) if error_content: self._logger.error('Error details:\n%s' % error_content) return # Handles other errors, typically when the body is ill-formatted. insert_errors = response.get('insertErrors') if insert_errors: self._logger.error('BigQuery API error:\n' + str(insert_errors)) for insert_error in insert_errors: self._failure_database.AddFailure( 'big_query_insert_error', str(insert_error.get('errors')))