def revoke_token(self, current_token):
# Perform a GET request, because that's by far the easiest way
url = "{site}{revoke_url}".format(site=self.auth_site,
revoke_url=self.revoke_url)
data = {'token': quote(current_token)}
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
return requests.post(url, data=data, headers=headers, verify=False)
def close_job(self, job_id):
doc = self.create_close_job_doc()
url = self.bulk_api_url + "/job/%s" % job_id
# resp, content = http.request(url, "POST", headers=self.bulk_headers(),
# body=doc)
# resp = self._soap_post(url=url, request_body=doc, headers=self.bulk_headers())
resp = requests.post(url, doc, headers=self.bulk_headers(), proxies=self.proxies, verify=False)
content = resp.text
self.check_status(resp, content)
def post_bulk_batch(self, job_id, csv_generator):
uri = self.bulk_api_url + "/job/%s/batch" % job_id
headers = self.bulk_headers({"Content-Type": "text/csv"})
resp = requests.post(uri, data=csv_generator, headers=headers, verify=False)
content = resp.content
if resp.status_code >= 400:
self.raise_error(content, resp.status_code)
tree = ET.fromstring(content)
batch_id = tree.findtext("{%s}id" % self.jobNS)
return batch_id
def abort_job(self, job_id):
"""Abort a given bulk job"""
doc = self.create_abort_job_doc()
url = self.bulk_api_url + "/job/%s" % job_id
# resp, content = http.request(
# url,
# "POST",
# headers=self.bulk_headers(),
# body=doc
# )
# resp = self._soap_post(url=url, request_body=doc, headers=self.bulk_headers())
resp = requests.post(url, doc, headers=self.bulk_headers(), proxies=self.proxies, verify=False)
content = resp.text
self.check_status(resp, content)
def _request_token(self, data):
url = "{site}{token_url}".format(site=self.auth_site,
token_url=self.token_url)
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
post_data = {
'client_id': self.client_id,
'client_secret': self.client_secret
}
post_data.update(data)
result = requests.post(url,
data=post_data,
headers=headers,
verify=False)
# print('--------->')
# print(result)
# print(result.json())
return result, result.json()
def _soap_post(self, url, request_body, headers, **kwargs):
self.myconsole.debug('>>>start soap post : ' + url)
result = requests.post(url,
request_body,
headers=headers,
proxies=self.proxies,
verify=False)
self.myconsole.debug("request headers:\n%s" % headers)
self.myconsole.debug("request body:\n%s" % request_body)
self.myconsole.debug('response result:\n%s' % result)
self.myconsole.debug('response status_code: %s' % result.status_code)
if result.status_code >= 300:
self.myconsole.debug(result.status_code)
self.myconsole.debug(result.text)
raise SoapException(result, result.status_code)
return result
def _soap_post(self, url, request_body, headers, **kwargs):
self.myconsole.debug('>>>start soap post : ' + url)
result = requests.post(url,
request_body,
headers=headers,
proxies=self.proxies,
verify=False)
self.myconsole.debug(request_body)
self.myconsole.debug(headers)
self.myconsole.debug('>>>response:')
self.myconsole.debug(result)
self.myconsole.debug(result.status_code)
# self.myconsole.debug(result.text)
# self.myconsole.debug(result.content)
if result.status_code >= 300:
print(result.status_code)
print(result.text)
raise SoapException(result, result.status_code)
return result
def create_job(self, object_name=None, operation=None, contentType='CSV',
concurrency=None, external_id_name=None):
assert(object_name is not None)
assert(operation is not None)
job_id = ''
doc = self.create_job_doc(object_name=object_name,
operation=operation,
contentType=contentType,
concurrency=concurrency,
external_id_name=external_id_name)
url = self.bulk_api_url + "/job"
# print('------->doc')
# print(doc)
# print('------->url')
# print(url)
# print('------->self.bulk_headers')
# print(self.bulk_headers)
# resp = self._soap_post(url=url, request_body=doc, headers=self.bulk_headers())
resp = requests.post(url, doc, headers=self.bulk_headers(), proxies=self.proxies, verify=False)
content = resp.text
# from . import requests
# resp = requests.post(url, doc, headers=self.bulk_headers, proxies=self.proxies, verify=False)
# print(resp)
# print('resp.text------->')
# print(resp.text)
# print('resp.content----->')
# print(resp.content)
# print('resp.status_code----->')
# print(resp.status_code)
tree = ET.fromstring(content)
job_id = tree.findtext("{%s}id" % self.jobNS)
self.jobs[job_id] = job_id
print("----->jobid " + job_id)
return job_id
def bulk_csv_upload(self, job_id, csv, batch_size=2500):
# Split a large CSV into manageable batches
batches = self.split_csv(csv, batch_size)
batch_ids = []
uri = self.bulk_api_url + "/job/%s/batch" % job_id
headers = self.bulk_headers({"Content-Type": "text/csv"})
for batch in batches:
resp = requests.post(uri, data=batch, headers=headers, verify=False)
content = resp.content
if resp.status_code >= 400:
self.raise_error(content, resp.status)
tree = ET.fromstring(content)
batch_id = tree.findtext("{%s}id" % self.jobNS)
self.batches[batch_id] = job_id
batch_ids.append(batch_id)
return batch_ids
def query(self, job_id, soql):
if job_id is None:
job_id = self.create_job(
re.search(re.compile("from (\w+)", re.I), soql).group(1),
"query")
uri = self.bulk_api_url + "/job/%s/batch" % job_id
print('uri-->')
print(uri)
headers = self.bulk_headers({"Content-Type": "text/csv"})
resp = requests.post(uri, soql, headers=headers, proxies=self.proxies, verify=False)
content = resp.text
self.check_status(resp, content)
tree = ET.fromstring(content)
batch_id = tree.findtext("{%s}id" % self.jobNS)
self.batches[batch_id] = job_id
return batch_id
def bulk_delete(self, job_id, object_type, where, batch_size=2500):
query_job_id = self.create_query_job(object_type)
soql = "Select Id from %s where %s Limit 10000" % (object_type, where)
query_batch_id = self.query(query_job_id, soql)
self.wait_for_batch(query_job_id, query_batch_id, timeout=120)
results = []
def save_results(tf, **kwargs):
results.append(tf.read())
flag = self.get_batch_results(
query_job_id, query_batch_id)
if job_id is None:
job_id = self.create_job(object_type, "delete")
# Split a large CSV into manageable batches
batches = self.split_csv(csv, batch_size)
batch_ids = []
uri = self.bulk_api_url + "/job/%s/batch" % job_id
headers = self.bulk_headers({"Content-Type": "text/csv"})
for batch in results:
resp = requests.post(uri, data=batch, headers=headers, verify=False)
content = resp.content
if resp.status_code >= 400:
self.raise_error(content, resp.status)
tree = ET.fromstring(content)
batch_id = tree.findtext("{%s}id" % self.jobNS)
self.batches[batch_id] = job_id
batch_ids.append(batch_id)
return batch_ids