class SalesforceSOAP(object):
def __init__(self, token):
self.token = token
profile = requests.post(token.id_url, data={'access_token': self.token.access_token})
self.partner_url = profile.json()['urls']['partner'].replace('{version}', '35.0')
self.client = SforcePartnerClient('integrations/salesforce/partner.wsdl')
self.force_oauth_login(self.token.access_token, self.partner_url)
self.merge_batches = []
self.merge_queue = []
self.log = open_s3(token.user.email + '-merge_log', 'wb')
self.log_writer = csv.writer(self.log)
self.log_writer.writerow(['Record group', 'Record ID', 'Master ID', 'Canonical'])
self.merge_count = 0
def refresh_token(self):
self.token.refresh()
self.force_oauth_login(self.token.access_token, self.partner_url)
def force_oauth_login(self, access_token, soap_endpoint):
self.client._setHeaders('login')
header = self.client.generateHeader('SessionHeader')
header.sessionId = access_token
self.client.setSessionHeader(header)
self.client._sessionId = access_token
self.client._setEndpoint(soap_endpoint)
def append_merge(self, merge_request):
self.merge_queue.append(merge_request)
if len(self.merge_queue) > 6:
self.merge_batches.append(self.merge_queue)
self.merge_queue = []
def commit_merges(self):
if self.merge_queue:
self.merge_batches.append(self.merge_queue)
self.merge_queue = []
for batch in self.merge_batches:
self.refresh_token()
try:
result = self.client.merge(batch)
except Exception as e:
print 'SF SOAP merge batch failed with message: {0}'.format(e.message) # TODO: log this somewhere
yield []
else:
yield result
def merge_group(self, group, canonical, updateable):
self.merge_count += 1
master = self.client.generateObject('Lead')
master.Id = group[0]['Id'].decode('utf-8')
for key, value in canonical.viewitems():
if value and key in updateable:
value = value.decode('utf-8')
setattr(master, key, value)
self.log_writer.writerow([self.merge_count, group[0]['Id'], '', json.dumps(canonical)])
for i in range(1, len(group), 2):
merge_request = self.client.generateObject('MergeRequest')
merge_request.masterRecord = master
merge_request.recordToMergeIds = [group[i]['Id'].decode('utf-8')]
self.log_writer.writerow([self.merge_count, group[i]['Id'], master.Id, json.dumps(canonical)])
try:
merge_request.recordToMergeIds.append(group[i + 1]['Id'].decode('utf-8'))
self.log_writer.writerow([self.merge_count, group[i + 1]['Id'], master.Id, json.dumps(canonical)])
except IndexError:
pass
self.append_merge(merge_request)
class SalesforceBatch(object):
def __init__(self, tenant = None, sessionId=None, endpoint=None,
username=None, password=None, token=None, partner_wsdl=None):
self.tenant = tenant
self.sfclient = None
self.db = None
if partner_wsdl is None:
partner_wsdl = os.path.join(os.path.dirname(__file__), "partnerwsdl.xml")
if username and password:
login = salesforce_oauth_request.login(username=username, password=password,
token=token, cache_session=True)
if 'access_token' in login:
sessionId = login['access_token']
endpoint = login['endpoint']
else:
raise RuntimeError("SF Login error: {0}".format(login))
if sessionId:
"""This is the trick to use an OAuth access_token as the session id in the SOAP client."""
self.h = SforcePartnerClient(partner_wsdl)
self.h._sessionId = sessionId
self.h._setEndpoint(endpoint)
header = self.h.generateHeader('SessionHeader')
header.sessionId = sessionId
self.h.setSessionHeader(header)
self.sfclient = self.h
@property
def sessionId(self):
return self.h._sessionId
@property
def endpoint(self):
return self.h._location
@property
def host(self):
o = urlparse.urlparse(self.h._location)
return o.hostname.replace("-api","")
def sf_client(self):
"""
@return: SforcePartnerClient
"""
# if not self.sfclient:
# system = self.tenant.default_salesforce
# sfadapter = SFAdapter.init_from_system(system)
# sfadapter.login()
# self.sfclient = sfadapter.h
return self.sfclient
def query_salesforce(self, object_name, fields=["Id"], limit = 200, where=None):
client = self.sf_client()
clauses = ["IsDeleted = False"] if object_name != "User" else []
if where:
clauses.append(where)
if len(clauses) > 0:
where = "WHERE %s" % " AND ".join(clauses)
else:
where = ""
return client.queryAll("SELECT %s from %s %s LIMIT %d" % (",".join(fields), object_name, where, limit))
def query_salesforce_id_set(self, object_name, count, where=None):
client = self.sf_client()
qOpts = client.generateHeader('QueryOptions')
qOpts.batchSize = 200
client.setQueryOptions(qOpts)
clauses = [] if object_name == 'User' else ["IsDeleted = False"]
if where:
clauses.append(where)
if len(clauses) > 0:
where = "WHERE %s" % " AND ".join(clauses)
else:
where = ""
results = client.queryAll("SELECT Id from %s %s LIMIT %d" % (object_name, where, count))
ids = [r.Id for r in results.records if hasattr(r, 'Id')]
while not results.done:
results = client.queryMore(results.queryLocator)
ids += [r.Id for r in results.records if hasattr(r, 'Id')]
return ids
def batches(self, items):
for x in range(0, len(items), 200):
yield items[x:x+200]
def batch_client(self, service, items):
client = self.sf_client()
results = []
for tranch in self.batches(items):
results += getattr(client, service)(tranch)
return results
def update_salesforce(self, object_name, **kwargs):
client = self.sf_client()
limit = kwargs.pop('limit', 10)
where = kwargs.pop('where', None)
updates = []
for id in self.query_salesforce_id_set(object_name, limit, where=where):
record = client.generateObject(object_name)
record.Id = id
for field, val in kwargs.iteritems():
setattr(record, field, val)
updates.append(record)
uresult = self.batch_client('update', updates)
self.show_results(uresult)
def show_results(self, sf_results):
try:
if not isinstance(sf_results, list):
sf_results = [sf_results]
updated_ids = [r.id for r in sf_results if hasattr(r, 'id')]
print "Updated: %s" % updated_ids
err_results = [r for r in sf_results if not r.success]
msgs = [er.errors[0].message for er in err_results]
if len(msgs) > 0:
print "WARNING, SF UPDATE RETURNED ERRORS:"
print msgs
except:
print sf_results
def insert_salesforce(self, object_name, values_list):
if isinstance(values_list, basestring):
values_list = yaml.load(values_list)
client = self.sf_client()
inserts = []
for values in values_list:
record = client.generateObject(object_name)
for field, val in values.iteritems():
setattr(record, field, val)
inserts.append(record)
results = self.batch_client('create', inserts)
self.show_results(results)
def delete_salesforce(self, object_name, ids_or_count, where=None):
client = self.sf_client()
if isinstance(ids_or_count, (int,long)):
ids_or_count = self.query_salesforce_id_set(object_name, ids_or_count, where=where)
if isinstance(ids_or_count, list):
results = self.batch_client('delete', ids_or_count)
self.show_results(results)
else:
raise ValueError("Invalid arg ids_or_count")
def copy_object(self, source, target):
fields = subprocess.check_output(("force field list %s" % source).split())
fields = fields.split("\n")
bads = ['reference','email','date','id']
good = []
for f in fields:
if len([t for t in bads if f.startswith(t) or f.endswith(t) or (t+" ") in f]) > 0:
print "Skipping field: %s" % f
else:
good.append(f)
fields = good
fields = ["%s:%s" % (m.group(1), m.group(3)) if m else "" for m in (re.match(r"(\w+?)(__c)?:\s*(\w+)", f) for f in fields)]
cmd = "force sobject create %s %s" % (target, fields[0])
print cmd
try:
print subprocess.check_output(cmd, shell=True)
except subprocess.CalledProcessError, e:
print e
for f in fields[1:]:
cmd = "force field create %s__c %s" % (target, f)
print cmd
try:
print subprocess.check_output(cmd, shell=True)
except subprocess.CalledProcessError, e:
print e