def retrieveSfdcCampaigns(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
sdfc = Salesforce()
campaignList = sdfc.get_campaigns_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type)
saveSfdcCampaigns(user_id=user_id, company_id=company_id, campaignList=campaignList, job_id=job_id, run_type=run_type)
try:
message = 'Campaigns retrieved from Salesforce'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Campaigns'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return campaignList
except Exception as e:
print 'error while retrieving sfdc campaigns: ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def retrieveSfdcAccounts(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
if sinceDateTime is None:
sinceDateTime = (datetime.now() - timedelta(days=1)).date()
sfdc = Salesforce()
accountList = sfdc.get_accounts_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type)
print 'got back accounts ' + str(len(accountList['records']))
saveSfdcAccounts(user_id=user_id, company_id=company_id, accountList=accountList, job_id=job_id, run_type=run_type)
try:
message = 'Accounts retrieved from Salesforce'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Accounts'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return accountList
except Exception as e:
send_notification(dict(type='error', success=False, message=str(e)))
def retrieveSfdcContacts(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
sdfc = Salesforce()
if sinceDateTime is None:
sinceDateTime = (datetime.now() - timedelta(days=30)).date()
contactList = sdfc.get_contacts_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type)
print "got back contacts " + str(len(contactList["records"]))
saveSfdcContacts(
user_id=user_id, company_id=company_id, contactList=contactList, job_id=job_id, run_type=run_type
)
try:
message = "Contacts retrieved from Salesforce"
notification = Notification()
# notification.company_id = company_id
notification.owner = user_id
notification.module = "Contacts"
notification.type = "Background task"
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(type="error", success=False, message=str(e)))
return contactList
except Exception as e:
send_notification(dict(type="error", success=False, message=str(e)))
def retrieveSfdcOppStageHistory(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): #needs to be changed - satya
try:
#delete later
#job_id_new = job_id
#job_id = ObjectId("56a690e98afb006883048e7e")
#set variables
sfdc = Salesforce()
#for leads
if run_type == 'initial':
opps = TempData.objects(Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
else:
opps = TempDataDelta.objects(Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
oppListTemp = list(opps)
if not oppListTemp:
print 'no opps found'
return
oppList = [i['source_record'] for i in oppListTemp]
batch_size = 500 #10 Activity Types at a time
activitiesList = []
for i in range(0, len(oppList), batch_size):
opp_list = '('
for opp in oppList[i:i+batch_size]:
opp_list += '\'' + opp['Id'] + '\'' + ', '
opp_list = opp_list[:-2]
opp_list += ')'
activitiesList.extend(sfdc.get_stage_history_for_opportunity(user_id, company_id, opp_list, _str_from_date(sinceDateTime)))
print 'got back stage history for SFDC opportunities ' + str(len(activitiesList))
#delete later
#job_id = job_id_new
saveSfdcOppStageHistory(user_id=user_id, company_id=company_id, activityList=activitiesList, job_id=job_id, run_type=run_type)
try:
message = 'Opportunity stage history retrieved from Salesforce'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Opportunities'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return opp_list
except Exception as e:
print 'exception while retrieving SFDC opportunity stage history: ' + str(e)
send_notification(dict(type='error', success=False, message=str(e)))
def retrieveSfdcOpportunities(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
#company_id = request.user.company_id
existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
# code commented out since we are no longer getting only Mkto related opportunities into Cx
# if existingIntegration is not None and 'mkto' in existingIntegration['integrations']: # if this system is connected to Marketo
# company_qry = 'company_id'
# type_field_qry = 'leads__mkto__exists'
# sfdc_account_field_qry = 'leads__mkto__sfdcAccountId__ne'
# querydict = {company_qry: company_id, type_field_qry: True, sfdc_account_field_qry: None}
# leads_with_sfdc_opps = Lead.objects(**querydict).only('mkto_id').only('leads__mkto__sfdcAccountId')
#
sfdc = Salesforce()
# code commented out since we are no longer getting only Mkto related opportunities into Cx
# account_list = '('
# for lead in leads_with_sfdc_opps:
# account_list += '\'' + lead['leads']['mkto']['sfdcAccountId'] + '\'' + ', '
# account_list = account_list[:-2]
# account_list += ')'
if sinceDateTime is None:
sinceDateTime = (datetime.now() - timedelta(days=30)).date()
oppList = sfdc.get_opportunities_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type)
print 'got opps ' + str(len(oppList['records']))
#create list of Opp IDs to send for get_contacts call
oppid_list = '('
for opp in oppList['records']:
oppid_list += '\'' + opp['Id'] + '\'' + ', '
oppid_list = oppid_list[:-2]
oppid_list += ')'
contactList = sfdc.get_contacts_for_opportunities(user_id, company_id, oppid_list) # needed because SFDC does not have the Contact ID within the Opp record
print 'got contacts for opps ' + str(len(contactList['records']))
saveSfdcOpportunities(user_id=user_id, company_id=company_id, oppList=oppList, contactList=contactList, job_id=job_id, run_type=run_type)
try:
message = 'Opportunities retrieved from Salesforce'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Opportunities'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return oppList
except Exception as e:
send_notification(dict(type='error', success=False, message=str(e)))
def retrieveSfdcLeadHistory(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): #needs to be changed - satya
try:
#delete later
#job_id_new = job_id
#job_id = ObjectId("56a690e98afb006883048e7e")
#set variables
sfdc = Salesforce()
#for leads
if run_type == 'initial':
leads = TempData.objects(Q(record_type='lead') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
else:
leads = TempDataDelta.objects(Q(record_type='lead') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
leadListTemp = list(leads)
if not leadListTemp:
print 'no leads found'
return
leadList = [i['source_record'] for i in leadListTemp]
batch_size = 500 #10 Activity Types at a time
activitiesList = []
for i in range(0, len(leadList), batch_size):
lead_list = '('
for lead in leadList[i:i+batch_size]:
lead_list += '\'' + lead['Id'] + '\'' + ', '
lead_list = lead_list[:-2]
lead_list += ')'
activitiesList.extend(sfdc.get_history_for_lead(user_id, company_id, lead_list, _str_from_date(sinceDateTime)))
print 'got back history for SFDC leads ' + str(len(activitiesList))
#delete later
#job_id = job_id_new
saveSfdcLeadHistory(user_id=user_id, company_id=company_id, activityList=activitiesList, job_id=job_id, run_type=run_type)
# #for contacts
# if run_type == 'initial':
# contacts = TempData.objects(Q(record_type='contact') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
# else:
# contacts = TempDataDelta.objects(Q(record_type='contact') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
#
# contactListTemp = list(contacts)
# contactList = [i['source_record'] for i in contactListTemp]
#
# sfdc = Salesforce()
#
# batch_size = 500 #10 Activity Types at a time
# activitiesList = []
#
# for i in range(0, len(contactList), batch_size):
# contact_list = '('
# for contact in contactList[i:i+batch_size]:
# contact_list += '\'' + contact['Id'] + '\'' + ', '
# contact_list = contact_list[:-2]
# contact_list += ')'
# activitiesList.extend(sfdc.get_history_for_contact(user_id, company_id, contact_list, _str_from_date(sinceDateTime)))
#
# print 'got back history for SFDC contacts ' + str(len(activitiesList))
# saveSfdcHistory(user_id=user_id, company_id=company_id, activityList=activitiesList, job_id=job_id, run_type=run_type)
#
try:
message = 'Lead history retrieved from Salesforce'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Leads'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return leadList
except Exception as e:
print 'exception while retrieving SFDC lead history: ' + str(e)
send_notification(dict(type='error', success=False, message=str(e)))
def saveSfdcOpportunitiesToMaster(user_id=None, company_id=None, job_id=None, run_type=None):
#job_id = ObjectId("56a3f89f8afb003c13a59e26")
if run_type == 'initial':
opps = TempData.objects(Q(company_id=company_id) & Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
contacts = TempData.objects(Q(company_id=company_id) & Q(record_type='contact') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
else:
opps = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
contacts = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='contact') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
oppListTemp = list(opps)
oppList = [i['source_record'] for i in oppListTemp]
contactListTemp = list(contacts)
contactList = [i['source_record'] for i in contactListTemp]
try:
allOpps = oppList #['records']
# below code copied from contacts.tasks
for newContact in contactList: #['records']:
if 'OpportunityContactRoles' not in newContact or newContact['OpportunityContactRoles'] is None: # if this contact has no opportunities
continue # move to next contact
# company_id = request.user.company_id
sfdc_account_id = None
thisLeadsOppsIds = newContact['OpportunityContactRoles']['records']
thisLeadsOpps = []
for opp in thisLeadsOppsIds: #loop through all the Opp records in the Contact record
print 'trying for opp with id ' + str(opp['OpportunityId'])
thisOpp = next((x for x in allOpps if x['Id'] == opp['OpportunityId']), None) # if this opp is found in the list of opps retrieved separately
if thisOpp is not None: # if found
print 'found this opp'
sfdc_account_id = thisOpp['AccountId']
thisLeadsOpps.append(thisOpp) #add it
sfdc_contact_Id = str(newContact['Id'])
print 'contact id is ' + sfdc_contact_Id
# sfdc_mkto_id = str(newLead['sfdcLeadId']) #check if there is a corresponding lead from MKTO
existingLeadMkto = None
existingLeadSfdc = None
existingLeadHspt = None
existingContact = Lead.objects(Q(company_id=company_id) & Q(sfdc_contact_id=sfdc_contact_Id)).first()
if existingContact is not None: # we found this contact already in the DB
print ' eC is not none'
if 'sfdc' not in existingContact.opportunities:
opportunities = {}
opportunities['sfdc'] = []
opportunities['sfdc'].extend(thisLeadsOpps)
existingContact.update(opportunities__sfdc = opportunities['sfdc'])
existingContact.update(sfdc_account_id = sfdc_account_id)
print 'just updated acct id 1'
else:
for newOpp in thisLeadsOpps:
print ' nefre get'
if not any (e.get('Id', None) == newOpp['Id'] for e in existingContact.opportunities['sfdc']): # does an opportunity with this Id already exist
opportunities = existingContact.opportunities['sfdc']
opportunities.append(newOpp)
existingContact.sfdc_account_id = sfdc_account_id
# save this opportunity
existingContact.update(opportunities__sfdc = opportunities)
existingContact.update(sfdc_account_id = sfdc_account_id)
print 'just updated acct id 2'
else: #this opp already exists
for i in range(len(existingContact.opportunities['sfdc'])):
if existingContact.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
existingContact.opportunities['sfdc'][i] = newOpp
existingContact.sfdc_account_id = sfdc_account_id
existingContact.save()
print 'just updated acct id 3'
elif existingContact is None: # this lead does not exist
print ' eC is much none'
existingLeadSfdc = Lead.objects(Q(company_id=company_id) & Q(leads__sfdc__ConvertedContactId=sfdc_contact_Id)).first()
if existingLeadSfdc is not None:
if 'sfdc' not in existingLeadSfdc.opportunities:
opportunities = {}
opportunities['sfdc'] = []
opportunities['sfdc'].extend(thisLeadsOpps)
existingLeadSfdc.update(opportunities__sfdc = opportunities['sfdc'])
existingLeadSfdc.update(sfdc_account_id = sfdc_account_id)
print 'just updated acct id 4'
else:
for newOpp in thisLeadsOpps:
if not any (e.get('Id', None) == newOpp['Id'] for e in existingLeadSfdc.opportunities['sfdc']): # does an opportunity with this Id already exist
opportunities = existingLeadSfdc.opportunities['sfdc']
opportunities.append(newOpp)
# save this opportunity
existingLeadSfdc.update(opportunities__sfdc = opportunities)
existingLeadSfdc.update(sfdc_account_id = sfdc_account_id)
print 'just updated acct id 5'
else: #this opp already exists
for i in range(len(existingLeadSfdc.opportunities['sfdc'])):
if existingLeadSfdc.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
existingLeadSfdc.opportunities['sfdc'][i] = newOpp
existingLeadSfdc.sfdc_account_id = sfdc_account_id
existingLeadSfdc.save()
print 'just updated acct id 6'
else:
existingLeadMkto = Lead.objects(Q(company_id=company_id) & Q(leads__mkto__sfdcContactId=sfdc_contact_Id)).first()
if existingLeadMkto is not None: # we found a MKto lead record which is matched to this new Sfdc lead
if 'sfdc' not in existingLeadMkto.opportunities:
opportunities = {}
opportunities['sfdc'] = []
opportunities['sfdc'].extend(thisLeadsOpps)
existingLeadMkto.update(opportunities__sfdc = opportunities['sfdc'])
existingLeadMkto.update(sfdc_account_id = sfdc_account_id)
else:
for newOpp in thisLeadsOpps:
if not any (e.get('Id', None) == newOpp['Id'] for e in existingLeadMkto.opportunities['sfdc']): # does an opportunity with this Id already exist
opportunities = existingLeadMkto.opportunities['sfdc']
opportunities.append(newOpp)
# save this opportunity
existingLeadMkto.update(opportunities__sfdc = opportunities)
existingLeadMkto.update(sfdc_account_id = sfdc_account_id)
else: #this opp already exists
for i in range(len(existingLeadMkto.opportunities['sfdc'])):
if existingLeadMkto.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
existingLeadMkto.opportunities['sfdc'][i] = newOpp
existingLeadMkto.sfdc_account_id = sfdc_account_id
existingLeadMkto.save()
existingLeadHspt = Lead.objects(Q(company_id=company_id) & Q(leads__hspt__properties__salesforcecontactid=sfdc_contact_Id)).first()
if existingLeadHspt is not None: # we found a Hspt lead record which is matched to this new Sfdc lead
if 'sfdc' not in existingLeadHspt.opportunities:
opportunities = {}
opportunities['sfdc'] = []
opportunities['sfdc'].extend(thisLeadsOpps)
existingLeadHspt.update(opportunities__sfdc = opportunities['sfdc'])
existingLeadHspt.update(sfdc_account_id = sfdc_account_id)
else:
for newOpp in thisLeadsOpps:
if not any (e.get('Id', None) == newOpp['Id'] for e in existingLeadHspt.opportunities['sfdc']): # does an opportunity with this Id already exist
opportunities = existingLeadHspt.opportunities['sfdc']
opportunities.append(newOpp)
# save this opportunity
existingLeadHspt.update(opportunities__sfdc = opportunities)
existingLeadHspt.update(sfdc_account_id = sfdc_account_id)
else: #this opp already exists
for i in range(len(existingLeadHspt.opportunities['sfdc'])):
if existingLeadHspt.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
existingLeadHspt.opportunities['sfdc'][i] = newOpp
existingLeadHspt.sfdc_account_id = sfdc_account_id
existingLeadHspt.save()
if existingLeadSfdc is None and existingLeadMkto is None and existingLeadHspt is None and existingContact is None: # no matches found so save new record
lead = Lead()
lead.sfdc_contact_id = sfdc_contact_Id
lead.company_id = company_id
lead.source_first_name = newContact['FirstName']
lead.source_last_name = newContact['LastName']
lead.source_email = newContact['Email']
lead.source_created_date = str(newContact['CreatedDate'])
lead.source_source = newContact['LeadSource']
lead.sfdc_account_id = sfdc_account_id
lead.save()
contacts = {}
contacts['sfdc'] = newContact
#lead.contacts["sfdc"] = newContact
lead.update(contacts__sfdc = contacts['sfdc'])
print 'just updated acct id 7'
opportunities = {}
opportunities['sfdc'] = []
opportunities['sfdc'].extend(thisLeadsOpps)
lead.update(opportunities__sfdc = opportunities['sfdc'])
#new code added on 1/24/2016 - add each opportunity to related account (to capture opps with no contacts)
for opp in allOpps:
account = Account.objects(Q(company_id=company_id) & Q(sfdc_id=opp['AccountId'])).first()
if account is None:
print 'no account for opp with ID ' + str(opp['Id'])
sfdc = Salesforce()
accountList = sfdc.get_single_account(user_id, company_id, opp['AccountId'])
for newAccount in accountList['records']:
account = _saveSfdcNewAccount(newAccount['Id'], newAccount, None, company_id)
if 'opportunities' not in account:
account.update(opportunities = {})
if 'sfdc' not in account.opportunities: #no opps exist for this account, so add this opp
opportunities = {}
opportunities['sfdc'] = []
opportunities['sfdc'].append(opp)
account.update(opportunities__sfdc = opportunities['sfdc'])
else:
if not any (e.get('Id', None) == opp['Id'] for e in account.opportunities['sfdc']): # there is no opportunity with this Id already exist
opportunities = account.opportunities['sfdc']
opportunities.append(opp)
# save this opportunity
account.update(opportunities__sfdc = opportunities)
else: # this opp already exists
for i in range(len(account.opportunities['sfdc'])):
if account.opportunities['sfdc'][i]['Id'] == opp['Id']:
account.opportunities['sfdc'][i] = opp
account.save()
# code commented out since we are no longer getting only Mkto related opportunities into Cx
# for newOpp in oppList['records']:
#
# # company_id = request.user.company_id
# sfdc_account_id = str(newOpp['AccountId']) # find the account ID
# print 'account id is ' + sfdc_account_id
# # sfdc_mkto_id = str(newLead['sfdcLeadId']) #check if there is a corresponding lead from MKTO
# existingLeadMkto = None
# existingLeadSfdc = None
# existingLeadHspt = None
# #existingContact = Lead.objects(Q(company_id=company_id) & Q(sfdc_contact_id=sfdc_contact_Id)).first()
# existingLeadSfdc = Lead.objects(Q(company_id=company_id) & Q(leads__sfdc__ConvertedAccountId=sfdc_account_id)).first()
# if existingLeadSfdc is not None:
# if 'sfdc' not in existingLeadSfdc.opportunities:
# opportunities = {}
# opportunities['sfdc'] = []
# opportunities['sfdc'].append(newOpp) # add this opp to the new array
# existingLeadSfdc.update(opportunities__sfdc = opportunities)
# else:
# if not any (e.get('Id', None) == newOpp['Id'] for e in existingLeadSfdc.opportunities['sfdc']): # does an opportunity with this Id already exist
# opportunities = existingLeadSfdc.opportunities['sfdc']
# opportunities.append(newOpp)
# # save this opportunity
# existingLeadSfdc.update(opportunities__sfdc = opportunities)
# else: #this opp already exists
# for i in range(len(existingLeadSfdc.opportunities['sfdc'])):
# if existingLeadSfdc.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
# existingLeadSfdc.opportunities['sfdc'][i] = newOpp
# existingLeadSfdc.save()
#
# # and move to the next opportunity in the loop
# continue
# else: #this opp does not exist within an SFDC lead
# # check if it is a Marketo lead
# existingLeadMkto = Lead.objects(Q(company_id=company_id) & Q(leads__mkto__sfdcAccountId=sfdc_account_id)).first()
# if existingLeadMkto is not None: # we found a MKto lead record which is matched to this opp
# print 'found mkto lead' + existingLeadMkto.mkto_id
# if 'sfdc' not in existingLeadMkto.opportunities:
# opportunities = []
# opportunities.append(newOpp) # add this opp to the new array
# existingLeadMkto.update(opportunities__sfdc = opportunities)
# print 'saved opps'
# else: # if ['opportunities']['sfdc'] already exists
# print 'opp exist'
# if not any (e.get('Id', None) == newOpp['Id'] for e in existingLeadMkto.opportunities['sfdc']): # does an opportunity with this Id already exist
# opportunities = existingLeadMkto.opportunities['sfdc']
# opportunities.append(newOpp)
# # save this opportunity
# existingLeadMkto.update(opportunities__sfdc = opportunities)
# print 'saved sfdc'
# else: #this opp already exists
# print 'opp does not exist'
# for i in range(len(existingLeadMkto.opportunities['sfdc'])):
# if existingLeadMkto.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
# existingLeadMkto.opportunities['sfdc'][i] = newOpp
# existingLeadMkto.save()
# print 'saved update'
# #move on to the next opp
# continue
# existingLeadHspt = Lead.objects(Q(company_id=company_id) & Q(leads__hspt__properties__salesforceaccountid=sfdc_account_id)).first()
# if existingLeadHspt is not None: # we found a MKto lead record which is matched to this new Sfdc lead
# if 'sfdc' not in existingLeadHspt.opportunities:
# opportunities = {}
# opportunities['sfdc'] = []
# opportunities['sfdc'].append(newOpp) # add this opp to the new array
# existingLeadHspt.update(opportunities__sfdc = opportunities)
# else:
# if not any (e.get('Id', None) == newOpp['Id'] for e in existingLeadHspt.opportunities['sfdc']): # does an opportunity with this Id already exist
# opportunities = existingLeadHspt.opportunities['sfdc']
# opportunities.append(newOpp)
# # save this opportunity
# existingLeadHspt.update(opportunities__sfdc = opportunities)
# else: #this opp already exists
# for i in range(len(existingLeadHspt.opportunities['sfdc'])):
# if existingLeadHspt.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
# existingLeadHspt.opportunities['sfdc'][i] = newOpp
# existingLeadHspt.save()
# # move onto the next opp
# continue
#
# if existingLeadSfdc is None and existingLeadMkto is None and existingLeadHspt is None: # no matches found so throw error i,e, not possible
# #raise ValueError('Opportunity found without lead or contact')
# lead = Lead()
# lead.sfdc_id = sfdc_Id
# lead.company_id = company_id
# lead.source_first_name = newLead['FirstName']
# lead.source_last_name = newLead['LastName']
# lead.source_email = newLead['Email']
# lead.source_created_date = str(newLead['CreatedDate'])
# lead.source_source = newLead['LeadSource']
# lead.source_status = newLead['Status']
# lead.leads["sfdc"] = newLead
# print '5th save'
# lead.save()
except Exception as e:
print 'Error while saving SFDC opportunities ' + str(e)
send_notification(dict(type='error', success=False, message=str(e)))
def companyDataExtract(user_id=None, company_id=None, run_type=None, sinceDateTime=None):
superJobMonitor = None
existingIntegration = None
if run_type is None or company_id is None or sinceDateTime is None:
return
try:
sinceDateTime = datetime.fromtimestamp(float(sinceDateTime) / 1000)
#sinceDateTime = datetime.now() - timedelta(days=30)
print 'start date is ' + str(sinceDateTime)
print 'run type is ' + run_type
print 'company id is ' + str(company_id)
#create an entry in the Job Monitor
superJobMonitor = SuperJobMonitor(company_id=company_id, type=run_type, started_date=datetime.utcnow())
superJobMonitor.save()
#do pre-check
_superJobMonitorAddTask(superJobMonitor, "Claritix", "Pre-check started")
# get the integration record
existingIntegration = CompanyIntegration.objects(company_id=company_id).first()
if existingIntegration is None:
_superJobMonitorEnd(superJobMonitor, None, 'Failed', 'No integration record found')
mail_admins('Could not find integration record for company ' + company_id , 'Check settings')
return False
if run_type == 'initial':
existingIntegration['initial_run_in_process'] = True #set the flag
else: #delta
existingIntegration['delta_run_in_process'] = True #set the flag
existingIntegration.save() # save the flag
# set up the Request and Cookie
user = _get_superadmin()
if user is None:
_superJobMonitorEnd(superJobMonitor, existingIntegration, 'Failed', 'No super admin found')
mail_admins('Could not find super admin!', 'Check settings')
return False
# remotely login the user
host = settings.BASE_URL
url = host + '/api/v1/auth/login/'
creds = {'email': '*****@*****.**', 'password':'******'}
s = requests.Session()
resp = s.post(url, data=json.dumps(creds))
if not resp.status_code == 200:
_superJobMonitorEnd(superJobMonitor, existingIntegration, 'Failed', 'Could not login super admin!')
mail_admins('Could not login super admin!', 'Check credentials')
return False
#do cookie thing - refer to SuperAdmin Cron Job Task for details
cookies = dict(sessionid = resp.cookies['sessionid'])
url = host + '/api/v1/users/'
resp = s.get(url, cookies=cookies)
_superJobMonitorAddTask(superJobMonitor, "Claritix", "Pre-check completed")
# #delete data in Lead Master
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of Leads, Contacts, Opportunities and Activities started")
# count = Lead.objects(company_id=company_id).count()
# Lead.objects(company_id=company_id).delete()
# _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " Leads, Contacts, Opportunities and Activities deleted")
#
# #delete data in Campaign Master
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of Campaigns started")
# count = Campaign.objects(company_id=company_id).count()
# Campaign.objects(company_id=company_id).delete()
# _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " Campaigns deleted")
#
# #delete data in Account Master
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of Accounts started")
# count = Account.objects(company_id=company_id).count()
# Account.objects(company_id=company_id).delete()
# _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " Accounts deleted")
#
# find out which systems are integrated and therefore which tasks should be run
task_map = {
#"mkto" : [retrieveMktoCampaigns, retrieveMktoLeadsByProgram], #retrieveMktoLeadCreatedActivities, retrieveMktoLeads, retrieveMktoActivities, retrieveMktoCampaigns, retrieveMktoLeadsByProgram], #IMPORTANT - Lead Created Activities has to be before Leads
#"hspt" : [retrieveHsptCampaigns], # , ], # retrieveHsptLeads, retrieveHsptOpportunities, retrieveHsptWebsiteTraffic, ,
"prdt" : [retrievePrdtLeads],
# "sfdc" : [retrieveSfdcCampaigns] #retrieveSfdcLeads, retrieveSfdcContacts, retrieveSfdcCampaigns, retrieveSfdcAccounts, retrieveSfdcOpportunities, retrieveSfdcLeadHistory, retrieveSfdcContactHistory, retrieveSfdcOppHistory, retrieveSfdcOppStageHistory],
# "sugr" : [retrieveSugrLeads],
# "bufr" : [retrieveBufrTwInteractions],
# "goog" : [retrieveGoogWebsiteTraffic], \
# "fbok" : [retrieveFbokPageStats, retrieveFbokAdStats]#, retrieveFbokPostStats] # , ]
}
# # for future use - retrieveMktoContacts, retrieveMktoOpportunities, retrieveSfdcActivities,
final_task_map = {
# "mkto" : [saveMktoCampaignsToMaster, saveMktoLeadsByProgramToMaster], #saveMktoLeadsToMaster, saveMktoActivitiesToMaster, saveMktoCampaignsToMaster, saveMktoLeadsByProgramToMaster],#mergeMktoSfdcLeads, deleteLeads, deleteDuplicateMktoIdLeads #
#"hspt" : [saveHsptCampaignsToMaster ], #saveHsptLeadsToMaster, saveHsptOpportunitiesToMaster, saveHsptWebsiteTrafficToMaster, , ], # ,
# "sfdc" : [saveSfdcCampaignsToMaster], #saveSfdcLeadsToMaster, saveSfdcContactsToMaster, saveSfdcCampaignsToMaster, saveSfdcAccountsToMaster, saveSfdcOpportunitiesToMaster, saveSfdcLeadHistoryToMaster, saveSfdcContactHistoryToMaster, saveSfdcOppHistoryToMaster, saveSfdcOppStageHistoryToMaster], #
# "sugr" : [saveSugrLeadsToMaster],
# "bufr" : [saveBufrTwInteractionsToMaster],
# "goog": [saveGoogleWebsiteTrafficToMaster],
# "fbok": [saveFbokPageStatsToMaster, saveFbokAdStatsToMaster]#, saveFbokPostStatsToMaster] #
}
# #
# # #saveSfdcLeadsToMaster, saveSfdcContactsToMaster, saveSfdcOpportunitiesToMaster, saveSfdcCampaignsToMaster,
# # # saveSfdcLeadsToMaster, saveSfdcContactsToMaster, saveSfdcOpportunitiesToMaster, saveSfdcCampaignsToMaster, saveSfdcAccountsToMaster
# # #collect all relevant tasks in one list and retrieve metadata as well
for source in existingIntegration.integrations.keys():
#change metadata depending on source system
if source == 'sfdc':
metadata_objects = ['user', 'lead', 'contact', 'campaign', 'opportunity', 'task', 'account'] #[] #objects for which metadata should be collected
elif source == 'mkto':
metadata_objects = ['lead', 'activity', 'campaign'] #[] #objects for which metadata should be collected
elif source == 'hspt':
metadata_objects = ['lead'] #objects for which metadata should be collected
else:
metadata_objects = [] #objects for which metadata should be collected
# if sfdc, explicitly refresh the access token
if source == 'sfdc':
sfdc = Salesforce()
sfdc.refresh_token(company_id)
#collect meta data
url = host + '/api/v1/company/' + str(company_id) + '/integrations/metadata/'
# for object in metadata_objects:
# _superJobMonitorAddTask(superJobMonitor, source, "Retrieval of metadata for " + object + " started")
# params = {'code': source, 'object': object}
# resp = s.get(url, params=params) # get metadata about activities
# if not resp.status_code == 200:
# _superJobMonitorAddTask(superJobMonitor, source, "Retrieval of metadata for " + object + " failed")
# continue
# else:
# _superJobMonitorAddTask(superJobMonitor, source, "Retrieval of metadata for " + object + " completed")
#collect retrieval tasks
print 'starting retrieval tasks'
tasks = []
if source in task_map:
tasks.extend(task_map[source])
#now run the tasks
for task in tasks:
_superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " started to store temp data")
print 'starting task ' + str(task)
task(user_id=user_id, company_id=company_id, job_id=superJobMonitor.id, run_type=run_type, sinceDateTime=sinceDateTime)
_superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " completed")
print 'starting save tasks'
#collect save tasks
tasks = []
if source in final_task_map:
tasks.extend(final_task_map[source])
#now run the tasks
for task in tasks:
_superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " started to save master data")
print 'starting task ' + str(task)
task(user_id=user_id, company_id=company_id, job_id=superJobMonitor.id, run_type=run_type)
_superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " completed")
#return #REMOVE THIS IN PRODUCTION
#if initial run, delete all analytics data
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, 'Claritix', "Deletion of analytics data started")
# count1 = AnalyticsData.objects(company_id=company_id).count() #ensure that website_traffic chart data is not deleted
# AnalyticsData.objects(company_id=company_id).delete() #ensure that website_traffic chart data is not deleted
# count2 = AnalyticsIds.objects(company_id=company_id).count()
# AnalyticsIds.objects(company_id=company_id).delete()
# _superJobMonitorAddTask(superJobMonitor, 'Claritix', str(count1 + count2) + " records deleted from analytics tables")
#
# call chart calculate tasks
charts = [\
# {'chart_name': 'sources_bar', 'system_type': 'MA', 'chart_title':'Timeline', 'mode': run_type, 'start_date': sinceDateTime}, \
# #{'chart_name': 'pipeline_duration', 'system_type': 'MA', 'chart_title':'Pipeline Duration', 'mode': run_type, 'start_date': sinceDateTime}, \
# #{'chart_name': 'contacts_distr', 'system_type': 'MA', 'chart_title':'Contacts Distribution', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'source_pie', 'system_type': 'MA', 'chart_title':'Source Distribution', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'revenue_source_pie', 'system_type': 'MA', 'chart_title':'Revenue by Source', 'mode': run_type, 'start_date': sinceDateTime}, \
# #{'chart_name': 'multichannel_leads', 'system_type': 'MA', 'chart_title':'Multichannel Leads', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'tw_performance', 'system_type': 'SO', 'chart_title':'Twitter Performance', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'fb_performance', 'system_type': 'SO', 'chart_title':'Facebook_Performance', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'google_analytics', 'system_type': 'AD', 'chart_title':'Google Analytics', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'campaign_email_performance', 'system_type': 'MA', 'chart_title':'Campaign Performance by Email', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'email_cta_performance', 'system_type': 'MA', 'chart_title':'Email Performance by CTA', 'mode': run_type, 'start_date': sinceDateTime}, \
#
]
url = host + '/api/v1/company/' + str(company_id) + '/analytics/calculate/'
for chart in charts:
print 'starting chart ' + str(chart['chart_title'])
_superJobMonitorAddTask(superJobMonitor, 'Claritix', "Started calculating " + str(chart['chart_title']))
resp = s.get(url, params=chart)
if not resp.status_code == 200:
print 'incorrect status code was ' + str(resp.status_code)
_superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Could not retrieve data for ' + chart['chart_title'])
continue
else:
_superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Retrieved data for ' + chart['chart_title'])
# call dashboard calculate tasks
dashboards = [\
# {'chart_name': 'social_roi', 'system_type': 'MA', 'chart_title':'Social Performance', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'funnel', 'system_type': 'MA', 'chart_title':'Funnel', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'opp_funnel', 'system_type': 'CRM', 'chart_title':'Opportunity Funnel', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'waterfall_chart', 'system_type': 'MA', 'chart_title':'Waterfall Chart', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'form_fills', 'system_type': 'MA', 'chart_title':'Form Fills', 'mode': run_type, 'start_date': sinceDateTime}, \
]
url = host + '/api/v1/company/' + str(company_id) + '/dashboards/calculate/'
for dashboard in dashboards:
print 'starting dashboard ' + str(dashboard['chart_title'])
_superJobMonitorAddTask(superJobMonitor, 'Claritix', "Started calculating " + str(dashboard['chart_title']))
resp = s.get(url, params=dashboard)
if not resp.status_code == 200:
print 'incorrect status code was ' + str(resp.status_code)
_superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Could not retrieve data for ' + dashboard['chart_title'])
continue
else:
_superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Retrieved data for ' + dashboard['chart_title'])
#delete all data from past successful initial runs in Temp Table
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of temp data table started")
# successfulJobs = SuperJobMonitor.objects(Q(company_id=company_id) & Q(type='initial') & Q(status='Completed'))
# successfulJobsListTemp = list(successfulJobs)
# print 'found job ids ' + str(len(successfulJobsListTemp))
# successfulJobsList = [i.id for i in successfulJobsListTemp]
# count = TempData.objects(job_id__in=successfulJobsList).count()
# TempData.objects(job_id__in=successfulJobsList).delete()
# _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " records deleted from temp data table")
#delete all data from past successful delta runs in Temp Table
if run_type == 'delta':
_superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of temp data delta table started")
successfulJobs = SuperJobMonitor.objects(Q(company_id=company_id) & Q(type='delta') & Q(status='Completed'))
successfulJobsListTemp = list(successfulJobs)
print 'found delta job ids ' + str(len(successfulJobsListTemp))
successfulJobsList = [i.id for i in successfulJobsListTemp]
count = TempDataDelta.objects(job_id__in=successfulJobsList).count()
TempDataDelta.objects(job_id__in=successfulJobsList).delete()
_superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " records deleted from temp data delta table")
#update last run date for initial run in Company Integration record
_superJobMonitorEnd(superJobMonitor, existingIntegration, run_type, 'Completed', 'All tasks completed successfully')
return True
except Exception as e:
if superJobMonitor is not None and existingIntegration is not None:
_superJobMonitorEnd(superJobMonitor, existingIntegration, run_type, 'Failed', str(e))
print str(e)
return False
