class SalesforceSOAP(object): def __init__(self, token): self.token = token profile = requests.post(token.id_url, data={'access_token': self.token.access_token}) self.partner_url = profile.json()['urls']['partner'].replace('{version}', '35.0') self.client = SforcePartnerClient('integrations/salesforce/partner.wsdl') self.force_oauth_login(self.token.access_token, self.partner_url) self.merge_batches = [] self.merge_queue = [] self.log = open_s3(token.user.email + '-merge_log', 'wb') self.log_writer = csv.writer(self.log) self.log_writer.writerow(['Record group', 'Record ID', 'Master ID', 'Canonical']) self.merge_count = 0 def refresh_token(self): self.token.refresh() self.force_oauth_login(self.token.access_token, self.partner_url) def force_oauth_login(self, access_token, soap_endpoint): self.client._setHeaders('login') header = self.client.generateHeader('SessionHeader') header.sessionId = access_token self.client.setSessionHeader(header) self.client._sessionId = access_token self.client._setEndpoint(soap_endpoint) def append_merge(self, merge_request): self.merge_queue.append(merge_request) if len(self.merge_queue) > 6: self.merge_batches.append(self.merge_queue) self.merge_queue = [] def commit_merges(self): if self.merge_queue: self.merge_batches.append(self.merge_queue) self.merge_queue = [] for batch in self.merge_batches: self.refresh_token() try: result = self.client.merge(batch) except Exception as e: print 'SF SOAP merge batch failed with message: {0}'.format(e.message) # TODO: log this somewhere yield [] else: yield result def merge_group(self, group, canonical, updateable): self.merge_count += 1 master = self.client.generateObject('Lead') master.Id = group[0]['Id'].decode('utf-8') for key, value in canonical.viewitems(): if value and key in updateable: value = value.decode('utf-8') setattr(master, key, value) self.log_writer.writerow([self.merge_count, group[0]['Id'], '', json.dumps(canonical)]) for i in range(1, len(group), 2): merge_request = self.client.generateObject('MergeRequest') merge_request.masterRecord = master merge_request.recordToMergeIds = [group[i]['Id'].decode('utf-8')] self.log_writer.writerow([self.merge_count, group[i]['Id'], master.Id, json.dumps(canonical)]) try: merge_request.recordToMergeIds.append(group[i + 1]['Id'].decode('utf-8')) self.log_writer.writerow([self.merge_count, group[i + 1]['Id'], master.Id, json.dumps(canonical)]) except IndexError: pass self.append_merge(merge_request)
class SalesforceBatch(object): def __init__(self, tenant = None, sessionId=None, endpoint=None, username=None, password=None, token=None, partner_wsdl=None): self.tenant = tenant self.sfclient = None self.db = None if partner_wsdl is None: partner_wsdl = os.path.join(os.path.dirname(__file__), "partnerwsdl.xml") if username and password: login = salesforce_oauth_request.login(username=username, password=password, token=token, cache_session=True) if 'access_token' in login: sessionId = login['access_token'] endpoint = login['endpoint'] else: raise RuntimeError("SF Login error: {0}".format(login)) if sessionId: """This is the trick to use an OAuth access_token as the session id in the SOAP client.""" self.h = SforcePartnerClient(partner_wsdl) self.h._sessionId = sessionId self.h._setEndpoint(endpoint) header = self.h.generateHeader('SessionHeader') header.sessionId = sessionId self.h.setSessionHeader(header) self.sfclient = self.h @property def sessionId(self): return self.h._sessionId @property def endpoint(self): return self.h._location @property def host(self): o = urlparse.urlparse(self.h._location) return o.hostname.replace("-api","") def sf_client(self): """ @return: SforcePartnerClient """ # if not self.sfclient: # system = self.tenant.default_salesforce # sfadapter = SFAdapter.init_from_system(system) # sfadapter.login() # self.sfclient = sfadapter.h return self.sfclient def query_salesforce(self, object_name, fields=["Id"], limit = 200, where=None): client = self.sf_client() clauses = ["IsDeleted = False"] if object_name != "User" else [] if where: clauses.append(where) if len(clauses) > 0: where = "WHERE %s" % " AND ".join(clauses) else: where = "" return client.queryAll("SELECT %s from %s %s LIMIT %d" % (",".join(fields), object_name, where, limit)) def query_salesforce_id_set(self, object_name, count, where=None): client = self.sf_client() qOpts = client.generateHeader('QueryOptions') qOpts.batchSize = 200 client.setQueryOptions(qOpts) clauses = [] if object_name == 'User' else ["IsDeleted = False"] if where: clauses.append(where) if len(clauses) > 0: where = "WHERE %s" % " AND ".join(clauses) else: where = "" results = client.queryAll("SELECT Id from %s %s LIMIT %d" % (object_name, where, count)) ids = [r.Id for r in results.records if hasattr(r, 'Id')] while not results.done: results = client.queryMore(results.queryLocator) ids += [r.Id for r in results.records if hasattr(r, 'Id')] return ids def batches(self, items): for x in range(0, len(items), 200): yield items[x:x+200] def batch_client(self, service, items): client = self.sf_client() results = [] for tranch in self.batches(items): results += getattr(client, service)(tranch) return results def update_salesforce(self, object_name, **kwargs): client = self.sf_client() limit = kwargs.pop('limit', 10) where = kwargs.pop('where', None) updates = [] for id in self.query_salesforce_id_set(object_name, limit, where=where): record = client.generateObject(object_name) record.Id = id for field, val in kwargs.iteritems(): setattr(record, field, val) updates.append(record) uresult = self.batch_client('update', updates) self.show_results(uresult) def show_results(self, sf_results): try: if not isinstance(sf_results, list): sf_results = [sf_results] updated_ids = [r.id for r in sf_results if hasattr(r, 'id')] print "Updated: %s" % updated_ids err_results = [r for r in sf_results if not r.success] msgs = [er.errors[0].message for er in err_results] if len(msgs) > 0: print "WARNING, SF UPDATE RETURNED ERRORS:" print msgs except: print sf_results def insert_salesforce(self, object_name, values_list): if isinstance(values_list, basestring): values_list = yaml.load(values_list) client = self.sf_client() inserts = [] for values in values_list: record = client.generateObject(object_name) for field, val in values.iteritems(): setattr(record, field, val) inserts.append(record) results = self.batch_client('create', inserts) self.show_results(results) def delete_salesforce(self, object_name, ids_or_count, where=None): client = self.sf_client() if isinstance(ids_or_count, (int,long)): ids_or_count = self.query_salesforce_id_set(object_name, ids_or_count, where=where) if isinstance(ids_or_count, list): results = self.batch_client('delete', ids_or_count) self.show_results(results) else: raise ValueError("Invalid arg ids_or_count") def copy_object(self, source, target): fields = subprocess.check_output(("force field list %s" % source).split()) fields = fields.split("\n") bads = ['reference','email','date','id'] good = [] for f in fields: if len([t for t in bads if f.startswith(t) or f.endswith(t) or (t+" ") in f]) > 0: print "Skipping field: %s" % f else: good.append(f) fields = good fields = ["%s:%s" % (m.group(1), m.group(3)) if m else "" for m in (re.match(r"(\w+?)(__c)?:\s*(\w+)", f) for f in fields)] cmd = "force sobject create %s %s" % (target, fields[0]) print cmd try: print subprocess.check_output(cmd, shell=True) except subprocess.CalledProcessError, e: print e for f in fields[1:]: cmd = "force field create %s__c %s" % (target, f) print cmd try: print subprocess.check_output(cmd, shell=True) except subprocess.CalledProcessError, e: print e