def check_bulk_quota_usage(self):
endpoint = "limits"
url = self.sf.data_url.format(self.sf.instance_url, endpoint)
with metrics.http_request_timer(endpoint):
resp = self.sf._make_request(
'GET', url, headers=self.sf._get_standard_headers()).json()
quota_max = resp['DailyBulkApiRequests']['Max']
max_requests_for_run = int(
(self.sf.quota_percent_per_run * quota_max) / 100)
quota_remaining = resp['DailyBulkApiRequests']['Remaining']
percent_used = (1 - (quota_remaining / quota_max)) * 100
if percent_used > self.sf.quota_percent_total:
total_message = (
"Salesforce has reported {}/{} ({:3.2f}%) total Bulk API quota "
+ "used across all Salesforce Applications. Terminating " +
"replication to not continue past configured percentage " +
"of {}% total quota.").format(quota_max - quota_remaining,
quota_max, percent_used,
self.sf.quota_percent_total)
raise TapSalesforceQuotaExceededException(total_message)
elif self.sf.jobs_completed > max_requests_for_run:
partial_message = (
"This replication job has completed {} Bulk API jobs ({:3.2f}% of "
+ "total quota). Terminating replication due to allotted " +
"quota of {}% per replication.").format(
self.sf.jobs_completed,
(self.sf.jobs_completed / quota_max) * 100,
self.sf.quota_percent_per_run)
raise TapSalesforceQuotaExceededException(partial_message)
def check_rest_quota_usage(self, headers):
match = re.search("^api-usage=(\d+)/(\d+)$",
headers.get("Sforce-Limit-Info"))
if match is None:
return
remaining, allotted = map(int, match.groups())
LOGGER.info("Used %s of %s daily REST API quota", remaining, allotted)
percent_used_from_total = (remaining / allotted) * 100
max_requests_for_run = int(
(self.quota_percent_per_run * allotted) / 100)
if percent_used_from_total > self.quota_percent_total:
total_message = (
"Salesforce has reported {}/{} ({:3.2f}%) total REST quota " +
"used across all Salesforce Applications. Terminating " +
"replication to not continue past configured percentage " +
"of {}% total quota.").format(remaining, allotted,
percent_used_from_total,
self.quota_percent_total)
raise TapSalesforceQuotaExceededException(total_message)
elif self.rest_requests_attempted > max_requests_for_run:
partial_message = (
"This replication job has made {} REST requests ({:3.2f}% of "
+ "total quota). Terminating replication due to allotted " +
"quota of {}% per replication.").format(
self.rest_requests_attempted,
(self.rest_requests_attempted / allotted) * 100,
self.quota_percent_per_run,
)
raise TapSalesforceQuotaExceededException(partial_message)