def _ExecuteApiRequest(self, request, retry_count=3):
""" Executes a Compute API request and returns True on success.
Returns:
(True, Response) in case of success, or (False, error_content) otherwise.
"""
self._logger.info('Compute API request:\n' + request.to_json())
try:
response = request.execute()
self._logger.info('Compute API response:\n' + str(response))
return (True, response)
except errors.HttpError as err:
error_content = google_error_helper.GetErrorContent(err)
error_reason = google_error_helper.GetErrorReason(error_content)
if error_reason == 'resourceNotReady' and retry_count > 0:
# Retry after a delay
delay_seconds = 1
self._logger.info(
'Resource not ready, retrying in %i seconds.' %
delay_seconds)
time.sleep(delay_seconds)
return self._ExecuteApiRequest(request, retry_count - 1)
else:
self._logger.error('Compute API error (reason: "%s"):\n%s' %
(error_reason, err))
if error_content:
self._logger.error('Error details:\n%s' % error_content)
return (False, error_content)
def DoesBigQueryTableExist(bigquery_service, project_name, table_id, logger):
"""Returns wether the BigQuery table identified by table_id exists.
Raises a HttpError exception if the call to BigQuery API fails.
Args:
bigquery_service: The BigQuery service.
project_name: (str) Name of the Google Cloud project.
table_id: (str) table_id as returned by GetBigQueryTableID().
Returns:
bool: True if the table exists.
"""
table_name = BIGQUERY_TABLE_TEMPLATE + '_' + table_id
logger.info('Getting table information for %s.' % table_name)
try:
table = bigquery_service.tables().get(projectId=project_name,
datasetId=BIGQUERY_DATASET,
tableId=table_name).execute()
return bool(table)
except errors.HttpError as http_error:
error_content = google_error_helper.GetErrorContent(http_error)
error_reason = google_error_helper.GetErrorReason(error_content)
if error_reason == google_error_helper.REASON_NOT_FOUND:
return False
else:
logger.error('BigQuery API error (reason: "%s"):\n%s' % (
error_reason, http_error))
if error_content:
logger.error('Error details:\n%s' % error_content)
raise # Re-raise the exception.
return False
def _StreamRowsToBigQuery(self, rows, table_id):
"""Uploads a list of rows to the BigQuery table associated with the given
table_id.
Args:
rows: (list of dict) Each dictionary is a row to add to the table.
table_id: (str) Identifier of the BigQuery table to update.
"""
try:
response = common.google_bigquery_helper.InsertInTemplatedBigQueryTable(
self._bigquery_service, self._project_name, table_id, rows,
self._logger)
except errors.HttpError as http_error:
# Handles HTTP error response codes (such as 404), typically indicating a
# problem in parameters other than 'body'.
error_content = google_error_helper.GetErrorContent(http_error)
error_reason = google_error_helper.GetErrorReason(error_content)
self._logger.error('BigQuery API error (reason: "%s"):\n%s' %
(error_reason, http_error))
self._failure_database.AddFailure('big_query_error', error_reason)
if error_content:
self._logger.error('Error details:\n%s' % error_content)
return
# Handles other errors, typically when the body is ill-formatted.
insert_errors = response.get('insertErrors')
if insert_errors:
self._logger.error('BigQuery API error:\n' + str(insert_errors))
for insert_error in insert_errors:
self._failure_database.AddFailure(
'big_query_insert_error', str(insert_error.get('errors')))
def DeleteTemplate(self, tag):
"""Deletes the instance template associated with tag. Returns True if
successful.
"""
template_name = self._GetTemplateName(tag)
request = self._compute_api.instanceTemplates().delete(
project=self._project, instanceTemplate=template_name)
(success, result) = self._ExecuteApiRequest(request)
if success:
return True
if google_error_helper.GetErrorReason(result) == \
google_error_helper.REASON_NOT_FOUND:
# The template does not exist, nothing to do.
self._logger.warning('Template not found: ' + template_name)
return True
return False
def DeleteInstanceGroup(self, tag):
"""Deletes the instance group identified by tag. If instances are still
running in this group, they are deleted as well.
"""
group_name = self._GetInstanceGroupName(tag)
request = self._compute_api.instanceGroupManagers().delete(
project=self._project, zone=self._zone,
instanceGroupManager=group_name)
(success, result) = self._ExecuteApiRequest(request)
if success:
return True
if google_error_helper.GetErrorReason(result) == \
google_error_helper.REASON_NOT_FOUND:
# The group does not exist, nothing to do.
self._logger.warning('Instance group not found: ' + group_name)
return True
return False
def _StreamRowsToBigQuery(self, rows, table_id):
"""Uploads a list of rows to the BigQuery table associated with the given
table_id.
Args:
rows: (list of dict) Each dictionary is a row to add to the table.
table_id: (str) Identifier of the BigQuery table to update.
"""
# Assumes that the dataset and the table template already exist.
dataset = 'clovis_dataset'
template = 'report'
rows_data = [{
'json': row,
'insertId': str(uuid.uuid4())
} for row in rows]
body = {'rows': rows_data, 'templateSuffix': '_' + table_id}
self._logger.info('BigQuery API request:\n' + str(body))
try:
response = self._bigquery_service.tabledata().insertAll(
projectId=self._project_name,
datasetId=dataset,
tableId=template,
body=body).execute()
self._logger.info('BigQuery API response:\n' + str(response))
except errors.HttpError as http_error:
# Handles HTTP error response codes (such as 404), typically indicating a
# problem in parameters other than 'body'.
error_content = google_error_helper.GetErrorContent(http_error)
error_reason = google_error_helper.GetErrorReason(error_content)
self._logger.error('BigQuery API error (reason: "%s"):\n%s' %
(error_reason, http_error))
self._failure_database.AddFailure('big_query_error', error_reason)
if error_content:
self._logger.error('Error details:\n%s' % error_content)
return
# Handles other errors, typically when the body is ill-formatted.
insert_errors = response.get('insertErrors')
if insert_errors:
self._logger.error('BigQuery API error:\n' + str(insert_errors))
for insert_error in insert_errors:
self._failure_database.AddFailure(
'big_query_insert_error', str(insert_error.get('errors')))
