class AnalyticsCloudUploader(object):
MAX_FILE_SIZE = 10 * 1024 * 1024
def __init__(self, metadata, data, client=None):
# XXX serve via public github? ok to publish?
self.metadata = metadata
self.data = data
self.client = client
self.parts = None
def login(self, wsdl, username, password, token):
self.client = SforcePartnerClient(wsdl)
self.client.login(username, password, token)
def upload(self, edgemart):
output = StringIO()
writer = AnalyticsWriter(output, encoding='utf-8')
self.start(edgemart, self.metadata)
biggest_record = 0
for record in self.data:
before_write = output.tell()
writer.writerow(record)
after_write = output.tell()
record_size = after_write - before_write
biggest_record = max(biggest_record, record_size * 2)
if after_write + biggest_record > self.MAX_FILE_SIZE:
output.seek(0)
self.add_data(output)
output.truncate(0)
if output.tell():
output.seek(0)
self.add_data(output)
self.complete()
def start(self, edgemart, metadata):
log.info('starting upload %s', edgemart)
self.parts = []
obj = self.client.generateObject('InsightsExternalData')
obj.EdgemartAlias = edgemart
obj.EdgemartContainer = ''
obj.MetadataJson = b64encode(json.dumps(metadata))
obj.Format = 'CSV'
obj.Operation = 'Overwrite'
obj.Action = None
result = self.client.create(obj)
if not result.success:
raise ConnectionError(result)
self.data_id = result.id
def add_data(self, data):
log.info('uploading chunk')
obj = self.client.generateObject('InsightsExternalDataPart')
obj.PartNumber = len(self.parts) + 1
obj.InsightsExternalDataId = self.data_id
obj.DataFile = b64encode(data.read())
result = self.client.create(obj)
if not result.success:
raise ConnectionError(result)
self.parts.append(result)
def complete(self):
log.info('upload complete')
obj = self.client.generateObject('InsightsExternalData')
obj.Id = self.data_id
obj.Action = 'Process'
result = self.client.update(obj)
if not result.success:
raise ConnectionError(result)
class Ticket2SForce(Component):
implements(ITicketManipulator, ITicketChangeListener)
# map Trac ticket field names to Salesforce custom object Ticket__c field names
fieldMap = \
{'id': 'Trac_Ticket_Id__c',
'status': 'Status__c',
'summary': 'Summary__c',
'reporter': 'Reporter__c',
'cc': 'Cc__c',
'changetime': 'Last_Update_Time__c',
'time': 'Time__c',
'description': 'Description__c',
'component': 'Component__c',
'priority': 'Priority__c',
'owner': 'Owner_cc',
'version': 'Version__c',
'milestone': 'Milestone__c',
'keywords': 'Keywords__c',
'type': 'Type__c'}
def connect2SForce(self):
self.env.log.debug('--- Called connect2SForce: %s, %s, %s, %s' % \
(self.username, self.password, self.sectoken, self.wsdlPath))
self.sf = SforcePartnerClient(self.wsdlPath)
self.sf.login(self.username, self.password, self.sectoken)
#self.env.log.debug('**** SessionId: ' + self.sf._sessionHeader.sessionId)
def __init__(self):
self.env.log.debug('--------------- Ticket2SForce init')
self.wsdlPath = 'file://' + self.env.config.getpath(
'ticket2sforce', 'wsdl')
self.username = self.config.get('ticket2sforce', 'username', '')
self.password = self.config.get('ticket2sforce', 'password', '')
self.sectoken = self.config.get('ticket2sforce', 'sectoken', '')
self.delete_closed_ticket = self.config.get('ticket2sforce',
'delete_closed_ticket',
'false')
self.connect2SForce()
def prepare_ticket(self, req, ticket, fields, actions):
"""Not currently called, but should be provided for future
compatibility.
"""
def validate_ticket(self, req, ticket):
"""Validate a ticket after it's been populated from user input.
Must return a list of `(field, message)` tuples, one for each problem
"""
self.env.log.debug("******** Called validate_ticket ***")
caseNumber = ticket['case_number']
if caseNumber == None or len(caseNumber) < 1:
return [('case_number', 'case_number is required')]
qstr = u"select Id, CaseNumber from Case where CaseNumber = '%s'" \
% (caseNumber)
result = self.sf.query(qstr)
if result.size < 1:
return [('case_number', 'case_number is not in the configured Org')
]
self.caseId = result.records[0].Id
return []
def createCaseTicketLink(self, caseId, ticketId):
"""
Create M2M link Case <==> Ticket
"""
link = self.sf.generateObject('CaseTicketLink__c')
link.Case__c = caseId
link.Ticket__c = ticketId
result = self.sf.create(link)
if result.success != True:
msg = "Error: Can't create CaseTicketLink record for %s, result was %s" \
% (str(link), result)
raise Exception(msg)
return result
def createComment(self, author, comment_text):
"""
Create Comment_cc and link object, to be linked to Ticket__cc
"""
comment = self.sf.generateObject('Comment__c')
comment.Author__c = author
comment.Comment__c = comment_text
result = self.sf.create(comment)
if result.success != True:
msg = "Error: Can't create Comment record for %s, result was %s" \
% (str(link), result)
raise Exception(msg)
return result
def createTicketCommentLink(self, ticketId, commentId):
"""
Create M2M link Ticket <==> Comment
"""
link = self.sf.generateObject('TicketCommentLink__c')
link.Ticket__c = ticketId
link.Comment__c = commentId
result = self.sf.create(link)
if result.success != True:
msg = "Error: Can't create TicketCommentLink record for %s, result was %s" \
% (str(link), result)
raise Exception(msg)
return result
def createTicket(self, record):
"""
Create a Ticket__c record, associated with the ticket in Trac
"""
ticket = self.sf.generateObject('Ticket__c')
ticket.Trac_Ticket_Id__c = record.id
ticket.Name = record.id
ticket.Status__c = record['status']
ticket.Summary__c = record['summary']
ticket.Reporter__c = record['reporter']
ticket.Cc__c = record['cc']
ticket.Last_Update_Time__c = record['changetime'].isoformat()
ticket.Time__c = record['time'].isoformat()
ticket.Description__c = record['description']
ticket.Component__c = record['component']
ticket.Priority__c = record['priority']
ticket.Owner__c = record['owner']
ticket.Version__c = record['version']
ticket.Milestone__c = record['milestone']
ticket.Keywords__c = record['keywords']
ticket.Type__c = record['type']
result = self.sf.create(ticket)
if result.success != True:
msg = "Error: Can't create Ticket record for %s, result was %s" \
% (str(record), result)
raise Exception(msg)
return result
def updateTicket(self, ticketId, ticket, old_values):
"""
Update existing Ticket - only fields that changed
"""
fieldList = ','.join(
[Ticket2SForce.fieldMap[fld] for fld in old_values.iterkeys()])
sfTicket = self.sf.retrieve(fieldList, 'Ticket__c', ticketId)
for fld in old_values.iterkeys():
sfTicket[Ticket2SForce.fieldMap[fld]] = ticket.values[fld]
result = self.sf.update(sfTicket)
if result.success != True:
msg = "Error: Can't update Ticket record for %s, result was %s" \
% (str(sfTicket), result)
raise Exception(msg)
return result
def create_sfticket(self, ticket):
self.env.log.debug('--------------- create_sfticket')
self.env.log.debug(ticket['priority'])
result = self.createTicket(ticket)
ticketId = result.id
result = self.createCaseTicketLink(self.caseId, ticketId)
def change_sfticket(self, ticket, comment, author, old_values):
self.env.log.debug('--------------- change_sfticket')
qstr = u"select Id, Trac_Ticket_Id__c from Ticket__c where Trac_Ticket_Id__c = '%s'" \
% (ticket.id)
result = self.sf.query(qstr)
if result.size < 1:
msg = "Error: Can't locate Ticket record for %s, result was %s" \
% (str(ticket), result)
raise Exception(msg)
ticketId = result.records[0].Id
ticketNumber = result.records[0].Trac_Ticket_Id__c
if len(comment) > 0:
result = self.createComment(author, comment)
commentId = result.id
result = self.createTicketCommentLink(ticketId, commentId)
if len(old_values) > 0:
self.updateTicket(ticketId, ticket, old_values)
def complete_sfticket(self, ticket):
self.env.log.debug('--------------- complete_sfticket')
#self.rtm_instance.complete_task(ticket)
# ITicketChangeListener
def ticket_created(self, ticket):
#self.env.log.debug("*** ticket: %d: %s %s\n" % (ticket.id, type(ticket.values['changetime']), str(ticket.values)))
#for key, value in ticket.values.iteritems():
# self.env.log.debug(" %s: %s\n" % (key, value))
self.create_sfticket(ticket)
def ticket_changed(self, ticket, comment, author, old_values):
self.env.log.debug('################ ticket_changed')
self.env.log.debug('>>>>>>>>>> ' + comment)
self.env.log.debug(ticket['status'])
self.env.log.debug(old_values)
#for key, value in old_values.iteritems():
# self.env.log.debug(" %s: %s\n" % (key, value))
for key, value in ticket.values.iteritems():
self.env.log.debug(" %s: %s" % (key, value))
if ticket['status'] == 'closed' and old_values['status'] != 'closed':
self.complete_sfticket(ticket)
else:
self.change_sfticket(ticket, comment, author, old_values)
def ticket_deleted(self, ticket):
self.env.log.debug('################ ticket_deleted')
self.complete_sfticket(ticket)
class AnalyticsCloudUploader(object):
MAX_FILE_SIZE = 10 * 1024 * 1024
def __init__(self, metadata, data, client=None):
# XXX serve via public github? ok to publish?
self.metadata = metadata
self.data = data
self.client = client
self.parts = None
def login(self, wsdl, username, password, token):
self.client = SforcePartnerClient(wsdl)
self.client.login(username, password, token)
def upload(self, edgemart):
output = StringIO()
writer = AnalyticsWriter(output, encoding='utf-8')
self.start(edgemart, self.metadata)
biggest_record = 0
for record in self.data:
before_write = output.tell()
writer.writerow(record)
after_write = output.tell()
record_size = after_write - before_write
biggest_record = max(biggest_record, record_size * 2)
if after_write + biggest_record > self.MAX_FILE_SIZE:
output.seek(0)
self.add_data(output)
output.truncate(0)
if output.tell():
output.seek(0)
self.add_data(output)
self.complete()
def start(self, edgemart, metadata):
log.info('starting upload %s', edgemart)
self.parts = []
obj = self.client.generateObject('InsightsExternalData')
obj.EdgemartAlias = edgemart
obj.EdgemartContainer = ''
obj.MetadataJson = b64encode(json.dumps(metadata))
obj.Format = 'CSV'
obj.Operation = 'Overwrite'
obj.Action = None
result = self.client.create(obj)
if not result.success:
raise ConnectionError(result)
self.data_id = result.id
def add_data(self, data):
log.info('uploading chunk')
obj = self.client.generateObject('InsightsExternalDataPart')
obj.PartNumber = len(self.parts) + 1
obj.InsightsExternalDataId = self.data_id
obj.DataFile = b64encode(data.read())
result = self.client.create(obj)
if not result.success:
raise ConnectionError(result)
self.parts.append(result)
def complete(self):
log.info('upload complete')
obj = self.client.generateObject('InsightsExternalData')
obj.Id = self.data_id
obj.Action = 'Process'
result = self.client.update(obj)
if not result.success:
raise ConnectionError(result)
class SalesforceSOAP(object):
def __init__(self, token):
self.token = token
profile = requests.post(token.id_url, data={'access_token': self.token.access_token})
self.partner_url = profile.json()['urls']['partner'].replace('{version}', '35.0')
self.client = SforcePartnerClient('integrations/salesforce/partner.wsdl')
self.force_oauth_login(self.token.access_token, self.partner_url)
self.merge_batches = []
self.merge_queue = []
self.log = open_s3(token.user.email + '-merge_log', 'wb')
self.log_writer = csv.writer(self.log)
self.log_writer.writerow(['Record group', 'Record ID', 'Master ID', 'Canonical'])
self.merge_count = 0
def refresh_token(self):
self.token.refresh()
self.force_oauth_login(self.token.access_token, self.partner_url)
def force_oauth_login(self, access_token, soap_endpoint):
self.client._setHeaders('login')
header = self.client.generateHeader('SessionHeader')
header.sessionId = access_token
self.client.setSessionHeader(header)
self.client._sessionId = access_token
self.client._setEndpoint(soap_endpoint)
def append_merge(self, merge_request):
self.merge_queue.append(merge_request)
if len(self.merge_queue) > 6:
self.merge_batches.append(self.merge_queue)
self.merge_queue = []
def commit_merges(self):
if self.merge_queue:
self.merge_batches.append(self.merge_queue)
self.merge_queue = []
for batch in self.merge_batches:
self.refresh_token()
try:
result = self.client.merge(batch)
except Exception as e:
print 'SF SOAP merge batch failed with message: {0}'.format(e.message) # TODO: log this somewhere
yield []
else:
yield result
def merge_group(self, group, canonical, updateable):
self.merge_count += 1
master = self.client.generateObject('Lead')
master.Id = group[0]['Id'].decode('utf-8')
for key, value in canonical.viewitems():
if value and key in updateable:
value = value.decode('utf-8')
setattr(master, key, value)
self.log_writer.writerow([self.merge_count, group[0]['Id'], '', json.dumps(canonical)])
for i in range(1, len(group), 2):
merge_request = self.client.generateObject('MergeRequest')
merge_request.masterRecord = master
merge_request.recordToMergeIds = [group[i]['Id'].decode('utf-8')]
self.log_writer.writerow([self.merge_count, group[i]['Id'], master.Id, json.dumps(canonical)])
try:
merge_request.recordToMergeIds.append(group[i + 1]['Id'].decode('utf-8'))
self.log_writer.writerow([self.merge_count, group[i + 1]['Id'], master.Id, json.dumps(canonical)])
except IndexError:
pass
self.append_merge(merge_request)
class Ticket2SForce(Component):
implements(ITicketManipulator, ITicketChangeListener)
# map Trac ticket field names to Salesforce custom object Ticket__c field names
fieldMap = \
{'id': 'Trac_Ticket_Id__c',
'status': 'Status__c',
'summary': 'Summary__c',
'reporter': 'Reporter__c',
'cc': 'Cc__c',
'changetime': 'Last_Update_Time__c',
'time': 'Time__c',
'description': 'Description__c',
'component': 'Component__c',
'priority': 'Priority__c',
'owner': 'Owner_cc',
'version': 'Version__c',
'milestone': 'Milestone__c',
'keywords': 'Keywords__c',
'type': 'Type__c'}
def connect2SForce(self):
self.env.log.debug('--- Called connect2SForce: %s, %s, %s, %s' % \
(self.username, self.password, self.sectoken, self.wsdlPath))
self.sf = SforcePartnerClient(self.wsdlPath)
self.sf.login(self.username, self.password, self.sectoken)
#self.env.log.debug('**** SessionId: ' + self.sf._sessionHeader.sessionId)
def __init__(self):
self.env.log.debug('--------------- Ticket2SForce init')
self.wsdlPath = 'file://' + self.env.config.getpath('ticket2sforce', 'wsdl')
self.username = self.config.get('ticket2sforce', 'username', '')
self.password = self.config.get('ticket2sforce', 'password', '')
self.sectoken = self.config.get('ticket2sforce', 'sectoken', '')
self.delete_closed_ticket = self.config.get('ticket2sforce', 'delete_closed_ticket', 'false')
self.connect2SForce()
def prepare_ticket(self, req, ticket, fields, actions):
"""Not currently called, but should be provided for future
compatibility.
"""
def validate_ticket(self, req, ticket):
"""Validate a ticket after it's been populated from user input.
Must return a list of `(field, message)` tuples, one for each problem
"""
self.env.log.debug("******** Called validate_ticket ***")
caseNumber = ticket['case_number']
if caseNumber == None or len(caseNumber) < 1:
return [('case_number', 'case_number is required')]
qstr = u"select Id, CaseNumber from Case where CaseNumber = '%s'" \
% (caseNumber)
result = self.sf.query(qstr)
if result.size < 1:
return [('case_number', 'case_number is not in the configured Org')]
self.caseId = result.records[0].Id
return []
def createCaseTicketLink(self, caseId, ticketId):
"""
Create M2M link Case <==> Ticket
"""
link = self.sf.generateObject('CaseTicketLink__c')
link.Case__c = caseId
link.Ticket__c = ticketId
result = self.sf.create(link)
if result.success != True:
msg = "Error: Can't create CaseTicketLink record for %s, result was %s" \
% (str(link), result)
raise Exception(msg)
return result
def createComment(self, author, comment_text):
"""
Create Comment_cc and link object, to be linked to Ticket__cc
"""
comment = self.sf.generateObject('Comment__c')
comment.Author__c = author
comment.Comment__c = comment_text
result = self.sf.create(comment)
if result.success != True:
msg = "Error: Can't create Comment record for %s, result was %s" \
% (str(link), result)
raise Exception(msg)
return result
def createTicketCommentLink(self, ticketId, commentId):
"""
Create M2M link Ticket <==> Comment
"""
link = self.sf.generateObject('TicketCommentLink__c')
link.Ticket__c = ticketId
link.Comment__c = commentId
result = self.sf.create(link)
if result.success != True:
msg = "Error: Can't create TicketCommentLink record for %s, result was %s" \
% (str(link), result)
raise Exception(msg)
return result
def createTicket(self, record):
"""
Create a Ticket__c record, associated with the ticket in Trac
"""
ticket = self.sf.generateObject('Ticket__c')
ticket.Trac_Ticket_Id__c = record.id
ticket.Name = record.id
ticket.Status__c = record['status']
ticket.Summary__c = record['summary']
ticket.Reporter__c = record['reporter']
ticket.Cc__c = record['cc']
ticket.Last_Update_Time__c = record['changetime'].isoformat()
ticket.Time__c = record['time'].isoformat()
ticket.Description__c = record['description']
ticket.Component__c = record['component']
ticket.Priority__c = record['priority']
ticket.Owner__c = record['owner']
ticket.Version__c = record['version']
ticket.Milestone__c = record['milestone']
ticket.Keywords__c = record['keywords']
ticket.Type__c = record['type']
result = self.sf.create(ticket)
if result.success != True:
msg = "Error: Can't create Ticket record for %s, result was %s" \
% (str(record), result)
raise Exception(msg)
return result
def updateTicket(self, ticketId, ticket, old_values):
"""
Update existing Ticket - only fields that changed
"""
fieldList = ','.join([Ticket2SForce.fieldMap[fld] for fld in old_values.iterkeys()])
sfTicket = self.sf.retrieve(fieldList, 'Ticket__c', ticketId)
for fld in old_values.iterkeys():
sfTicket[Ticket2SForce.fieldMap[fld]] = ticket.values[fld]
result = self.sf.update(sfTicket)
if result.success != True:
msg = "Error: Can't update Ticket record for %s, result was %s" \
% (str(sfTicket), result)
raise Exception(msg)
return result
def create_sfticket(self, ticket):
self.env.log.debug('--------------- create_sfticket')
self.env.log.debug(ticket['priority'])
result = self.createTicket(ticket)
ticketId = result.id
result = self.createCaseTicketLink(self.caseId, ticketId)
def change_sfticket(self, ticket, comment, author, old_values):
self.env.log.debug('--------------- change_sfticket')
qstr = u"select Id, Trac_Ticket_Id__c from Ticket__c where Trac_Ticket_Id__c = '%s'" \
% (ticket.id)
result = self.sf.query(qstr)
if result.size < 1:
msg = "Error: Can't locate Ticket record for %s, result was %s" \
% (str(ticket), result)
raise Exception(msg)
ticketId = result.records[0].Id
ticketNumber = result.records[0].Trac_Ticket_Id__c
if len(comment) > 0:
result = self.createComment(author, comment)
commentId = result.id
result = self.createTicketCommentLink(ticketId, commentId)
if len(old_values) > 0:
self.updateTicket(ticketId, ticket, old_values)
def complete_sfticket(self, ticket):
self.env.log.debug('--------------- complete_sfticket')
#self.rtm_instance.complete_task(ticket)
# ITicketChangeListener
def ticket_created(self, ticket):
#self.env.log.debug("*** ticket: %d: %s %s\n" % (ticket.id, type(ticket.values['changetime']), str(ticket.values)))
#for key, value in ticket.values.iteritems():
# self.env.log.debug(" %s: %s\n" % (key, value))
self.create_sfticket(ticket)
def ticket_changed(self, ticket, comment, author, old_values):
self.env.log.debug('################ ticket_changed')
self.env.log.debug('>>>>>>>>>> ' + comment)
self.env.log.debug(ticket['status'])
self.env.log.debug(old_values)
#for key, value in old_values.iteritems():
# self.env.log.debug(" %s: %s\n" % (key, value))
for key, value in ticket.values.iteritems():
self.env.log.debug(" %s: %s" % (key, value))
if ticket['status'] == 'closed' and old_values['status'] != 'closed':
self.complete_sfticket(ticket)
else:
self.change_sfticket(ticket, comment, author, old_values)
def ticket_deleted(self, ticket):
self.env.log.debug('################ ticket_deleted')
self.complete_sfticket(ticket)