class AnalyticsCloudUploader(object): MAX_FILE_SIZE = 10 * 1024 * 1024 def __init__(self, metadata, data, client=None): # XXX serve via public github? ok to publish? self.metadata = metadata self.data = data self.client = client self.parts = None def login(self, wsdl, username, password, token): self.client = SforcePartnerClient(wsdl) self.client.login(username, password, token) def upload(self, edgemart): output = StringIO() writer = AnalyticsWriter(output, encoding='utf-8') self.start(edgemart, self.metadata) biggest_record = 0 for record in self.data: before_write = output.tell() writer.writerow(record) after_write = output.tell() record_size = after_write - before_write biggest_record = max(biggest_record, record_size * 2) if after_write + biggest_record > self.MAX_FILE_SIZE: output.seek(0) self.add_data(output) output.truncate(0) if output.tell(): output.seek(0) self.add_data(output) self.complete() def start(self, edgemart, metadata): log.info('starting upload %s', edgemart) self.parts = [] obj = self.client.generateObject('InsightsExternalData') obj.EdgemartAlias = edgemart obj.EdgemartContainer = '' obj.MetadataJson = b64encode(json.dumps(metadata)) obj.Format = 'CSV' obj.Operation = 'Overwrite' obj.Action = None result = self.client.create(obj) if not result.success: raise ConnectionError(result) self.data_id = result.id def add_data(self, data): log.info('uploading chunk') obj = self.client.generateObject('InsightsExternalDataPart') obj.PartNumber = len(self.parts) + 1 obj.InsightsExternalDataId = self.data_id obj.DataFile = b64encode(data.read()) result = self.client.create(obj) if not result.success: raise ConnectionError(result) self.parts.append(result) def complete(self): log.info('upload complete') obj = self.client.generateObject('InsightsExternalData') obj.Id = self.data_id obj.Action = 'Process' result = self.client.update(obj) if not result.success: raise ConnectionError(result)
class Ticket2SForce(Component): implements(ITicketManipulator, ITicketChangeListener) # map Trac ticket field names to Salesforce custom object Ticket__c field names fieldMap = \ {'id': 'Trac_Ticket_Id__c', 'status': 'Status__c', 'summary': 'Summary__c', 'reporter': 'Reporter__c', 'cc': 'Cc__c', 'changetime': 'Last_Update_Time__c', 'time': 'Time__c', 'description': 'Description__c', 'component': 'Component__c', 'priority': 'Priority__c', 'owner': 'Owner_cc', 'version': 'Version__c', 'milestone': 'Milestone__c', 'keywords': 'Keywords__c', 'type': 'Type__c'} def connect2SForce(self): self.env.log.debug('--- Called connect2SForce: %s, %s, %s, %s' % \ (self.username, self.password, self.sectoken, self.wsdlPath)) self.sf = SforcePartnerClient(self.wsdlPath) self.sf.login(self.username, self.password, self.sectoken) #self.env.log.debug('**** SessionId: ' + self.sf._sessionHeader.sessionId) def __init__(self): self.env.log.debug('--------------- Ticket2SForce init') self.wsdlPath = 'file://' + self.env.config.getpath( 'ticket2sforce', 'wsdl') self.username = self.config.get('ticket2sforce', 'username', '') self.password = self.config.get('ticket2sforce', 'password', '') self.sectoken = self.config.get('ticket2sforce', 'sectoken', '') self.delete_closed_ticket = self.config.get('ticket2sforce', 'delete_closed_ticket', 'false') self.connect2SForce() def prepare_ticket(self, req, ticket, fields, actions): """Not currently called, but should be provided for future compatibility. """ def validate_ticket(self, req, ticket): """Validate a ticket after it's been populated from user input. Must return a list of `(field, message)` tuples, one for each problem """ self.env.log.debug("******** Called validate_ticket ***") caseNumber = ticket['case_number'] if caseNumber == None or len(caseNumber) < 1: return [('case_number', 'case_number is required')] qstr = u"select Id, CaseNumber from Case where CaseNumber = '%s'" \ % (caseNumber) result = self.sf.query(qstr) if result.size < 1: return [('case_number', 'case_number is not in the configured Org') ] self.caseId = result.records[0].Id return [] def createCaseTicketLink(self, caseId, ticketId): """ Create M2M link Case <==> Ticket """ link = self.sf.generateObject('CaseTicketLink__c') link.Case__c = caseId link.Ticket__c = ticketId result = self.sf.create(link) if result.success != True: msg = "Error: Can't create CaseTicketLink record for %s, result was %s" \ % (str(link), result) raise Exception(msg) return result def createComment(self, author, comment_text): """ Create Comment_cc and link object, to be linked to Ticket__cc """ comment = self.sf.generateObject('Comment__c') comment.Author__c = author comment.Comment__c = comment_text result = self.sf.create(comment) if result.success != True: msg = "Error: Can't create Comment record for %s, result was %s" \ % (str(link), result) raise Exception(msg) return result def createTicketCommentLink(self, ticketId, commentId): """ Create M2M link Ticket <==> Comment """ link = self.sf.generateObject('TicketCommentLink__c') link.Ticket__c = ticketId link.Comment__c = commentId result = self.sf.create(link) if result.success != True: msg = "Error: Can't create TicketCommentLink record for %s, result was %s" \ % (str(link), result) raise Exception(msg) return result def createTicket(self, record): """ Create a Ticket__c record, associated with the ticket in Trac """ ticket = self.sf.generateObject('Ticket__c') ticket.Trac_Ticket_Id__c = record.id ticket.Name = record.id ticket.Status__c = record['status'] ticket.Summary__c = record['summary'] ticket.Reporter__c = record['reporter'] ticket.Cc__c = record['cc'] ticket.Last_Update_Time__c = record['changetime'].isoformat() ticket.Time__c = record['time'].isoformat() ticket.Description__c = record['description'] ticket.Component__c = record['component'] ticket.Priority__c = record['priority'] ticket.Owner__c = record['owner'] ticket.Version__c = record['version'] ticket.Milestone__c = record['milestone'] ticket.Keywords__c = record['keywords'] ticket.Type__c = record['type'] result = self.sf.create(ticket) if result.success != True: msg = "Error: Can't create Ticket record for %s, result was %s" \ % (str(record), result) raise Exception(msg) return result def updateTicket(self, ticketId, ticket, old_values): """ Update existing Ticket - only fields that changed """ fieldList = ','.join( [Ticket2SForce.fieldMap[fld] for fld in old_values.iterkeys()]) sfTicket = self.sf.retrieve(fieldList, 'Ticket__c', ticketId) for fld in old_values.iterkeys(): sfTicket[Ticket2SForce.fieldMap[fld]] = ticket.values[fld] result = self.sf.update(sfTicket) if result.success != True: msg = "Error: Can't update Ticket record for %s, result was %s" \ % (str(sfTicket), result) raise Exception(msg) return result def create_sfticket(self, ticket): self.env.log.debug('--------------- create_sfticket') self.env.log.debug(ticket['priority']) result = self.createTicket(ticket) ticketId = result.id result = self.createCaseTicketLink(self.caseId, ticketId) def change_sfticket(self, ticket, comment, author, old_values): self.env.log.debug('--------------- change_sfticket') qstr = u"select Id, Trac_Ticket_Id__c from Ticket__c where Trac_Ticket_Id__c = '%s'" \ % (ticket.id) result = self.sf.query(qstr) if result.size < 1: msg = "Error: Can't locate Ticket record for %s, result was %s" \ % (str(ticket), result) raise Exception(msg) ticketId = result.records[0].Id ticketNumber = result.records[0].Trac_Ticket_Id__c if len(comment) > 0: result = self.createComment(author, comment) commentId = result.id result = self.createTicketCommentLink(ticketId, commentId) if len(old_values) > 0: self.updateTicket(ticketId, ticket, old_values) def complete_sfticket(self, ticket): self.env.log.debug('--------------- complete_sfticket') #self.rtm_instance.complete_task(ticket) # ITicketChangeListener def ticket_created(self, ticket): #self.env.log.debug("*** ticket: %d: %s %s\n" % (ticket.id, type(ticket.values['changetime']), str(ticket.values))) #for key, value in ticket.values.iteritems(): # self.env.log.debug(" %s: %s\n" % (key, value)) self.create_sfticket(ticket) def ticket_changed(self, ticket, comment, author, old_values): self.env.log.debug('################ ticket_changed') self.env.log.debug('>>>>>>>>>> ' + comment) self.env.log.debug(ticket['status']) self.env.log.debug(old_values) #for key, value in old_values.iteritems(): # self.env.log.debug(" %s: %s\n" % (key, value)) for key, value in ticket.values.iteritems(): self.env.log.debug(" %s: %s" % (key, value)) if ticket['status'] == 'closed' and old_values['status'] != 'closed': self.complete_sfticket(ticket) else: self.change_sfticket(ticket, comment, author, old_values) def ticket_deleted(self, ticket): self.env.log.debug('################ ticket_deleted') self.complete_sfticket(ticket)
class AnalyticsCloudUploader(object): MAX_FILE_SIZE = 10 * 1024 * 1024 def __init__(self, metadata, data, client=None): # XXX serve via public github? ok to publish? self.metadata = metadata self.data = data self.client = client self.parts = None def login(self, wsdl, username, password, token): self.client = SforcePartnerClient(wsdl) self.client.login(username, password, token) def upload(self, edgemart): output = StringIO() writer = AnalyticsWriter(output, encoding='utf-8') self.start(edgemart, self.metadata) biggest_record = 0 for record in self.data: before_write = output.tell() writer.writerow(record) after_write = output.tell() record_size = after_write - before_write biggest_record = max(biggest_record, record_size * 2) if after_write + biggest_record > self.MAX_FILE_SIZE: output.seek(0) self.add_data(output) output.truncate(0) if output.tell(): output.seek(0) self.add_data(output) self.complete() def start(self, edgemart, metadata): log.info('starting upload %s', edgemart) self.parts = [] obj = self.client.generateObject('InsightsExternalData') obj.EdgemartAlias = edgemart obj.EdgemartContainer = '' obj.MetadataJson = b64encode(json.dumps(metadata)) obj.Format = 'CSV' obj.Operation = 'Overwrite' obj.Action = None result = self.client.create(obj) if not result.success: raise ConnectionError(result) self.data_id = result.id def add_data(self, data): log.info('uploading chunk') obj = self.client.generateObject('InsightsExternalDataPart') obj.PartNumber = len(self.parts) + 1 obj.InsightsExternalDataId = self.data_id obj.DataFile = b64encode(data.read()) result = self.client.create(obj) if not result.success: raise ConnectionError(result) self.parts.append(result) def complete(self): log.info('upload complete') obj = self.client.generateObject('InsightsExternalData') obj.Id = self.data_id obj.Action = 'Process' result = self.client.update(obj) if not result.success: raise ConnectionError(result)
class SalesforceSOAP(object): def __init__(self, token): self.token = token profile = requests.post(token.id_url, data={'access_token': self.token.access_token}) self.partner_url = profile.json()['urls']['partner'].replace('{version}', '35.0') self.client = SforcePartnerClient('integrations/salesforce/partner.wsdl') self.force_oauth_login(self.token.access_token, self.partner_url) self.merge_batches = [] self.merge_queue = [] self.log = open_s3(token.user.email + '-merge_log', 'wb') self.log_writer = csv.writer(self.log) self.log_writer.writerow(['Record group', 'Record ID', 'Master ID', 'Canonical']) self.merge_count = 0 def refresh_token(self): self.token.refresh() self.force_oauth_login(self.token.access_token, self.partner_url) def force_oauth_login(self, access_token, soap_endpoint): self.client._setHeaders('login') header = self.client.generateHeader('SessionHeader') header.sessionId = access_token self.client.setSessionHeader(header) self.client._sessionId = access_token self.client._setEndpoint(soap_endpoint) def append_merge(self, merge_request): self.merge_queue.append(merge_request) if len(self.merge_queue) > 6: self.merge_batches.append(self.merge_queue) self.merge_queue = [] def commit_merges(self): if self.merge_queue: self.merge_batches.append(self.merge_queue) self.merge_queue = [] for batch in self.merge_batches: self.refresh_token() try: result = self.client.merge(batch) except Exception as e: print 'SF SOAP merge batch failed with message: {0}'.format(e.message) # TODO: log this somewhere yield [] else: yield result def merge_group(self, group, canonical, updateable): self.merge_count += 1 master = self.client.generateObject('Lead') master.Id = group[0]['Id'].decode('utf-8') for key, value in canonical.viewitems(): if value and key in updateable: value = value.decode('utf-8') setattr(master, key, value) self.log_writer.writerow([self.merge_count, group[0]['Id'], '', json.dumps(canonical)]) for i in range(1, len(group), 2): merge_request = self.client.generateObject('MergeRequest') merge_request.masterRecord = master merge_request.recordToMergeIds = [group[i]['Id'].decode('utf-8')] self.log_writer.writerow([self.merge_count, group[i]['Id'], master.Id, json.dumps(canonical)]) try: merge_request.recordToMergeIds.append(group[i + 1]['Id'].decode('utf-8')) self.log_writer.writerow([self.merge_count, group[i + 1]['Id'], master.Id, json.dumps(canonical)]) except IndexError: pass self.append_merge(merge_request)
class Ticket2SForce(Component): implements(ITicketManipulator, ITicketChangeListener) # map Trac ticket field names to Salesforce custom object Ticket__c field names fieldMap = \ {'id': 'Trac_Ticket_Id__c', 'status': 'Status__c', 'summary': 'Summary__c', 'reporter': 'Reporter__c', 'cc': 'Cc__c', 'changetime': 'Last_Update_Time__c', 'time': 'Time__c', 'description': 'Description__c', 'component': 'Component__c', 'priority': 'Priority__c', 'owner': 'Owner_cc', 'version': 'Version__c', 'milestone': 'Milestone__c', 'keywords': 'Keywords__c', 'type': 'Type__c'} def connect2SForce(self): self.env.log.debug('--- Called connect2SForce: %s, %s, %s, %s' % \ (self.username, self.password, self.sectoken, self.wsdlPath)) self.sf = SforcePartnerClient(self.wsdlPath) self.sf.login(self.username, self.password, self.sectoken) #self.env.log.debug('**** SessionId: ' + self.sf._sessionHeader.sessionId) def __init__(self): self.env.log.debug('--------------- Ticket2SForce init') self.wsdlPath = 'file://' + self.env.config.getpath('ticket2sforce', 'wsdl') self.username = self.config.get('ticket2sforce', 'username', '') self.password = self.config.get('ticket2sforce', 'password', '') self.sectoken = self.config.get('ticket2sforce', 'sectoken', '') self.delete_closed_ticket = self.config.get('ticket2sforce', 'delete_closed_ticket', 'false') self.connect2SForce() def prepare_ticket(self, req, ticket, fields, actions): """Not currently called, but should be provided for future compatibility. """ def validate_ticket(self, req, ticket): """Validate a ticket after it's been populated from user input. Must return a list of `(field, message)` tuples, one for each problem """ self.env.log.debug("******** Called validate_ticket ***") caseNumber = ticket['case_number'] if caseNumber == None or len(caseNumber) < 1: return [('case_number', 'case_number is required')] qstr = u"select Id, CaseNumber from Case where CaseNumber = '%s'" \ % (caseNumber) result = self.sf.query(qstr) if result.size < 1: return [('case_number', 'case_number is not in the configured Org')] self.caseId = result.records[0].Id return [] def createCaseTicketLink(self, caseId, ticketId): """ Create M2M link Case <==> Ticket """ link = self.sf.generateObject('CaseTicketLink__c') link.Case__c = caseId link.Ticket__c = ticketId result = self.sf.create(link) if result.success != True: msg = "Error: Can't create CaseTicketLink record for %s, result was %s" \ % (str(link), result) raise Exception(msg) return result def createComment(self, author, comment_text): """ Create Comment_cc and link object, to be linked to Ticket__cc """ comment = self.sf.generateObject('Comment__c') comment.Author__c = author comment.Comment__c = comment_text result = self.sf.create(comment) if result.success != True: msg = "Error: Can't create Comment record for %s, result was %s" \ % (str(link), result) raise Exception(msg) return result def createTicketCommentLink(self, ticketId, commentId): """ Create M2M link Ticket <==> Comment """ link = self.sf.generateObject('TicketCommentLink__c') link.Ticket__c = ticketId link.Comment__c = commentId result = self.sf.create(link) if result.success != True: msg = "Error: Can't create TicketCommentLink record for %s, result was %s" \ % (str(link), result) raise Exception(msg) return result def createTicket(self, record): """ Create a Ticket__c record, associated with the ticket in Trac """ ticket = self.sf.generateObject('Ticket__c') ticket.Trac_Ticket_Id__c = record.id ticket.Name = record.id ticket.Status__c = record['status'] ticket.Summary__c = record['summary'] ticket.Reporter__c = record['reporter'] ticket.Cc__c = record['cc'] ticket.Last_Update_Time__c = record['changetime'].isoformat() ticket.Time__c = record['time'].isoformat() ticket.Description__c = record['description'] ticket.Component__c = record['component'] ticket.Priority__c = record['priority'] ticket.Owner__c = record['owner'] ticket.Version__c = record['version'] ticket.Milestone__c = record['milestone'] ticket.Keywords__c = record['keywords'] ticket.Type__c = record['type'] result = self.sf.create(ticket) if result.success != True: msg = "Error: Can't create Ticket record for %s, result was %s" \ % (str(record), result) raise Exception(msg) return result def updateTicket(self, ticketId, ticket, old_values): """ Update existing Ticket - only fields that changed """ fieldList = ','.join([Ticket2SForce.fieldMap[fld] for fld in old_values.iterkeys()]) sfTicket = self.sf.retrieve(fieldList, 'Ticket__c', ticketId) for fld in old_values.iterkeys(): sfTicket[Ticket2SForce.fieldMap[fld]] = ticket.values[fld] result = self.sf.update(sfTicket) if result.success != True: msg = "Error: Can't update Ticket record for %s, result was %s" \ % (str(sfTicket), result) raise Exception(msg) return result def create_sfticket(self, ticket): self.env.log.debug('--------------- create_sfticket') self.env.log.debug(ticket['priority']) result = self.createTicket(ticket) ticketId = result.id result = self.createCaseTicketLink(self.caseId, ticketId) def change_sfticket(self, ticket, comment, author, old_values): self.env.log.debug('--------------- change_sfticket') qstr = u"select Id, Trac_Ticket_Id__c from Ticket__c where Trac_Ticket_Id__c = '%s'" \ % (ticket.id) result = self.sf.query(qstr) if result.size < 1: msg = "Error: Can't locate Ticket record for %s, result was %s" \ % (str(ticket), result) raise Exception(msg) ticketId = result.records[0].Id ticketNumber = result.records[0].Trac_Ticket_Id__c if len(comment) > 0: result = self.createComment(author, comment) commentId = result.id result = self.createTicketCommentLink(ticketId, commentId) if len(old_values) > 0: self.updateTicket(ticketId, ticket, old_values) def complete_sfticket(self, ticket): self.env.log.debug('--------------- complete_sfticket') #self.rtm_instance.complete_task(ticket) # ITicketChangeListener def ticket_created(self, ticket): #self.env.log.debug("*** ticket: %d: %s %s\n" % (ticket.id, type(ticket.values['changetime']), str(ticket.values))) #for key, value in ticket.values.iteritems(): # self.env.log.debug(" %s: %s\n" % (key, value)) self.create_sfticket(ticket) def ticket_changed(self, ticket, comment, author, old_values): self.env.log.debug('################ ticket_changed') self.env.log.debug('>>>>>>>>>> ' + comment) self.env.log.debug(ticket['status']) self.env.log.debug(old_values) #for key, value in old_values.iteritems(): # self.env.log.debug(" %s: %s\n" % (key, value)) for key, value in ticket.values.iteritems(): self.env.log.debug(" %s: %s" % (key, value)) if ticket['status'] == 'closed' and old_values['status'] != 'closed': self.complete_sfticket(ticket) else: self.change_sfticket(ticket, comment, author, old_values) def ticket_deleted(self, ticket): self.env.log.debug('################ ticket_deleted') self.complete_sfticket(ticket)