def saveHsptCampaignEmailEventRecords(company_id=None, run_type=None, job_id=None, guid=None):
'''called from saveHsptCampaignsToMaster to save all events for a single Hspt campaign email record'''
try:
#print 'saving single event'
if run_type == 'initial':
events = TempData.objects(Q(company_id=company_id) & Q(record_type='campaign_email_event') & Q(source_system='hspt') & Q(job_id=job_id) & Q(source_record__campaign_guid=guid) ).only('source_record') #& Q(job_id=job_id)
else:
events = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='campaign_email_event') & Q(source_system='hspt') & Q(job_id=job_id) & Q(source_record__campaign_guid=guid) ).only('source_record') #& Q(job_id=job_id)
eventsListTemp = list(events)
eventsList = [j['source_record'] for j in eventsListTemp]
#print 'found events ' + str(eventsList)
for event in eventsList:
#print 'about to check for event with ' + guid + ' email id ' + str(email['id']) + ' event id ' + str(event['event_id']) + ' recipient ' + str(event['recipient']) + ' created ' + str(event['created']) + ' details ' + str(event)
existingEvent = EmailEvent.objects(Q(company_id=company_id) & Q(campaign_guid=guid) & Q(email_id=event['email_id']) & Q(event_id=event['event_id'])).first()
if existingEvent is None:
newEvent = EmailEvent(company_id=company_id, source_system='hspt', campaign_guid=guid, email_id=event['email_id'], event_id=event['event_id'], details=event['details'], recipient=event['recipient'], created=event['created'], event_type=event['event_type'])
#print 'about to save new event with ' + guid + ' email id ' + str(email['id']) + ' event id ' + str(event['event_id']) + ' recipient ' + str(event['recipient']) + ' created ' + str(event['created']) + ' details ' + str(event)
newEvent.save()
except Exception as e:
print 'error while saving single Hspt campaign event record ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def saveFbokAdCampaignStatsToMaster(user_id=None, company_id=None, job_id=None, run_type=None): #behaves differently because it directly saves the data to the AnalyticsData collection
if run_type == 'initial':
fb_stats = TempData.objects(Q(company_id=company_id) & Q(record_type='fb_ad_campaign_stat') & Q(source_system='fbok') & Q(job_id=job_id) ).only('source_record')
else:
fb_stats = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='fb_ad_campaign_stat') & Q(source_system='fbok') & Q(job_id=job_id) ).only('source_record')
fbList = list(fb_stats)
fbList = [i['source_record'] for i in fbList]
try:
for i in range(len(fbList)):
fb_record = {}
source_campaign_id = fbList[i].get('id')
source_campaign_name = fbList[i].get('name')
source_account_id = fbList[i].get('account_id')
insights = fbList[i]['insights']
for insight in insights:
fb_record = insight['data']
source_created_date = fb_record['date_stop']
source_source = 'fbok'
FbAdCampaignInsight.objects(Q(company_id=company_id) & Q(source_created_date=source_created_date)& Q(source_campaign_id=source_campaign_id)).delete()
fbAdCampaignInsight = FbAdCampaignInsight(data=fb_record, company_id=company_id, source_campaign_id=source_campaign_id, source_campaign_name=source_campaign_name, source_account_id=source_account_id, source_created_date=source_created_date)
fbAdCampaignInsight.save()
except Exception as e:
print 'exception is ' + str(e)
send_notification(dict(type='error', success=False, message=str(e)))
def _get_hspt_email_details(email_id, existingCampaign, company_id, job_id, run_type):
try:
if run_type == 'initial':
campaigns = TempData.objects(Q(company_id=company_id) & Q(record_type='campaign') & Q(source_system='hspt') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
else:
campaigns = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='campaign') & Q(source_system='hspt') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
campaignListTemp = list(campaigns)
campaignList = [i['source_record'] for i in campaignListTemp]
#print 'temp recs found ' + str(len(campaignList))
#now get the appId for this email Id
app_id = None
int_email_id = 0
for campaign in campaignList:
#print 'campaign in temp ' + str(campaign)
if str(campaign['id']) == email_id:
print 'campaign id is ' + str(campaign['id']) + ' and email id is ' + email_id
app_id = campaign['appId']
int_email_id = campaign['id']
break
#if appId found, get details
emailDetails = None
if app_id is not None:
hspt = Hubspot(company_id)
emailDetails = hspt.get_campaign_details(campaignId=int_email_id, appId=app_id)
return emailDetails
except Exception as e:
print 'error while getting email details with Hspt campaign from traffic ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def saveSfdcCampaignsToMaster(user_id=None, company_id=None, job_id=None, run_type=None):
#job_id = ObjectId("56d25c6af6fd2a15df46cd60")
if run_type == 'initial':
campaigns = TempData.objects(Q(company_id=company_id) & Q(record_type='campaign') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
else:
campaigns = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='campaign') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
campaignListTemp = list(campaigns)
campaignList = [i['source_record'] for i in campaignListTemp]
try:
for newCampaign in campaignList: #['records']:
#company_id = request.user.company_id
guid = 'sfdc_' + str(newCampaign['Id'])
Campaign.objects(Q(guid = guid) & Q(company_id=company_id)).modify(upsert=True, new=True, set__campaigns__sfdc = newCampaign, set__updated_date = datetime.utcnow, set_on_insert__source_system = 'sfdc', set_on_insert__guid = guid, set_on_insert__company_id = company_id)
# for oldCampaign in campaignList['records']:
# # mktoCampaigns = []
# # mktoCampaigns.append(campaign)
# campaign = Campaign()
# campaign.company_id = request.user.company_id
# campaign.derived_id = 'sfdc_' + str(oldCampaign['Id'])
# campaign.campaigns["sfdc"] = oldCampaign
# campaign.save()
except Exception as e:
print 'error while saving sfdc campaign: ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def saveSfdcAccountsToMaster(user_id=None, company_id=None, job_id=None, run_type=None):
#delete later
#job_id = ObjectId("569fd4078afb002426ef2fd3")
if run_type == 'initial':
accounts = TempData.objects(Q(company_id=company_id) & Q(record_type='account') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
else:
accounts = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='account') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
accountListTemp = list(accounts)
accountList = [i['source_record'] for i in accountListTemp]
try:
for newAccount in accountList:
sfdc_id = str(newAccount['Id'])
#find all leads that have this account ID
relatedLeadList = []
relatedLeadListTemp = None
relatedLeads = Lead.objects(Q(company_id=company_id) & Q(sfdc_account_id=sfdc_id)).only('sfdc_contact_id') #if SFDC Account, then matching lead must have a SFDC Contact ID
if relatedLeads is not None:
relatedLeadListTemp = [lead.to_mongo().to_dict() for lead in relatedLeads]
#print 'rll is ' + str(relatedLeadListTemp)
for i in relatedLeadListTemp:
if 'sfdc_contact_id' in i:
relatedLeadList.append({'sfdc_contact_id': i['sfdc_contact_id']})
#print 'related leads are ' + str(relatedLeadList)
# if relatedLeads is not None:
# #leadListTemp = list(relatedLeads)
# #relatedLeadList = [i.id for i in leadListTemp]
# for lead in relatedLeads:
# relatedLeadList.append(lead)
print 'account id is ' + sfdc_id
# sfdc_mkto_id = str(newLead['sfdcLeadId']) #check if there is a corresponding lead from MKTO
existingAccount = None
existingAccount = Account.objects(Q(company_id=company_id) & Q(sfdc_id=sfdc_id)).first()
if existingAccount is not None: # we found this contact already in the DB
print 'found existing account for id ' + str(sfdc_id)
if 'sfdc' in existingAccount.accounts:
existingAccount.source_name = newAccount['Name']
existingAccount.source_source = newAccount['AccountSource']
existingAccount.source_industry = newAccount['Industry']
existingAccount.source_created_date = newAccount['CreatedDate']
existingAccount.accounts["sfdc"] = newAccount
if relatedLeadList is not None:
existingAccount.leads = relatedLeadList
existingAccount.save()
#Lead.objects(Q(company_id=company_id) & Q(sfdc_contact_id=sfdc_contact_Id)).update(contacts__sfdc=newContact)
else:
existingAccount.accounts['sfdc'] = {}
existingAccount.accounts['sfdc'] = newAccount
if relatedLeadList is not None:
existingAccount.leads = relatedLeadList
existingAccount.save()
elif existingAccount is None: # this account does not exist
account = _saveSfdcNewAccount(sfdc_id, newAccount, relatedLeadList, company_id)
except Exception as e:
print 'exception while saving accounts ' + str(e)
send_notification(dict(type='error', success=False, message=str(e)))
def saveHsptCampaignsToMaster(user_id=None, company_id=None, job_id=None, run_type=None):
if run_type == 'initial':
campaigns = TempData.objects(Q(company_id=company_id) & Q(record_type='campaign') & Q(source_system='hspt') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
else:
campaigns = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='campaign') & Q(source_system='hspt') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
campaignListTemp = list(campaigns)
campaignList = [i['source_record'] for i in campaignListTemp]
try:
for newCampaign in campaignList:
#company_id = request.user.company_id
#derived_id = 'hspt_' + str(newCampaign['id'])
guid = newCampaign['guid']
name = newCampaign['name']
source_system = 'hspt'
channel = 'email'
emails = newCampaign['emails']
#save all email events for this campaign first
saveHsptCampaignEmailEventRecords(company_id=company_id, run_type=run_type, job_id=job_id, guid=guid)
#now update or create campaign record
existingCampaign = Campaign.objects(Q(guid = guid) & Q(company_id=company_id) & Q(source_system=source_system)).first() #.modify(upsert=True, new=True, set__emails = emails, set_on_insert__name = name, set_on_insert__guid = guid, set_on_insert__company_id = company_id, set_on_insert__updated_date=datetime.utcnow)
if existingCampaign is None: #new campaign so add it
newCampaign = Campaign(emails = emails, name = name, guid = guid, company_id = company_id, updated_date=datetime.utcnow, source_system=source_system)
newCampaign.save()
else: #campaign already exists so update relevant fields
#print 'existing campaign is ' + str(existingCampaign)
if 'emails' not in existingCampaign: #should never happen
continue
#print 'before emails ' + str(emails)
for email in emails: #loop through each email found in API call for this campaign
#save events separately
#print 'entering emails'
#now create/update the campaign record
email_found = False
for existingEmail in existingCampaign['emails']: #check if the API email already exists
if email['id'] == existingEmail['id']: #we found this email already existing so update specific fields
email_found = True
existingEmail['stats'] = email['stats']
existingEmail['details'] = email['details']
if email_found:
existingCampaign.save()
break #move to the next API email since we already found a match
if email_found == False: #email was not found so add it to existing emails
existingCampaign['emails'].append(email)
existingCampaign.save()
except Exception as e:
print 'exception ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def retrieveSfdcOppStageHistory(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): #needs to be changed - satya
try:
#delete later
#job_id_new = job_id
#job_id = ObjectId("56a690e98afb006883048e7e")
#set variables
sfdc = Salesforce()
#for leads
if run_type == 'initial':
opps = TempData.objects(Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
else:
opps = TempDataDelta.objects(Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
oppListTemp = list(opps)
if not oppListTemp:
print 'no opps found'
return
oppList = [i['source_record'] for i in oppListTemp]
batch_size = 500 #10 Activity Types at a time
activitiesList = []
for i in range(0, len(oppList), batch_size):
opp_list = '('
for opp in oppList[i:i+batch_size]:
opp_list += '\'' + opp['Id'] + '\'' + ', '
opp_list = opp_list[:-2]
opp_list += ')'
activitiesList.extend(sfdc.get_stage_history_for_opportunity(user_id, company_id, opp_list, _str_from_date(sinceDateTime)))
print 'got back stage history for SFDC opportunities ' + str(len(activitiesList))
#delete later
#job_id = job_id_new
saveSfdcOppStageHistory(user_id=user_id, company_id=company_id, activityList=activitiesList, job_id=job_id, run_type=run_type)
try:
message = 'Opportunity stage history retrieved from Salesforce'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Opportunities'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return opp_list
except Exception as e:
print 'exception while retrieving SFDC opportunity stage history: ' + str(e)
send_notification(dict(type='error', success=False, message=str(e)))
def saveSfdcOppStageHistoryToMaster(user_id=None, company_id=None, job_id=None, run_type=None):
#job_id = ObjectId("56b2b92c8afb0070a795c4b2")
if run_type == 'initial':
#activities = TempData.objects(Q(company_id=company_id) & Q(record_type='activity') & Q(source_system='mkto') & Q(job_id=job_id) ).only('source_record')
collection = TempData._get_collection()
activities = collection.find({'company_id': int(company_id), 'record_type': 'opp_stage_history', 'source_system': 'sfdc', 'job_id': job_id}, projection={'source_record': True}, batch_size=1000)
else:
collection = TempDataDelta._get_collection()
activities = collection.find({'company_id': int(company_id), 'record_type': 'opp_stage_history', 'source_system': 'sfdc', 'job_id': job_id}, projection={'source_record': True}, batch_size=1000)
try:
print 'got history ' + str(activities.count())
for activity in activities:
skipThis = False
newActivity = activity['source_record']
addThisActivity = True
print 'act is ' + str(newActivity)
newOppId = newActivity["OpportunityId"]
if newOppId is not None:
print 'trying to get opp'
existingAccount = Account.objects(Q(company_id = company_id) & Q(opportunities__sfdc__Id = newOppId)).first()
print 'account is ' + str(existingAccount)
else:
continue
if existingAccount is None:
continue
for existingOpp in existingAccount['opportunities']['sfdc']: #loop through each opp to find the right one
if existingOpp['Id'] == newOppId:
if 'stage_activities' not in existingOpp: #if no prior activities
existingOpp['stage_activities'] = [] #create list
existingOpp['stage_activities'].append(newActivity) #save activity
print 'saved virgin stage activity for opp'
else:
for existingActivity in existingOpp['stage_activities']: #check if this activity already exists
if existingActivity['Id'] == newActivity['Id']: #it exists, so skip the activity
skipThis = True
print 'skipping opp stage activity'
break
if skipThis:
break # get out of this for loop
else:
existingOpp['stage_activities'].append(newActivity)
print 'saved stage activity for opp'
existingAccount.save()
except Exception as e:
print 'exception while saving SFDC Opp stage history to master' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def saveSfdcContactHistoryToMaster(user_id=None, company_id=None, job_id=None, run_type=None):
#job_id = ObjectId("56a6faa28afb00042171cd89")
if run_type == 'initial':
#activities = TempData.objects(Q(company_id=company_id) & Q(record_type='activity') & Q(source_system='mkto') & Q(job_id=job_id) ).only('source_record')
collection = TempData._get_collection()
activities = collection.find({'company_id': int(company_id), 'record_type': 'contact_history', 'source_system': 'sfdc', 'job_id': job_id}, projection={'source_record': True}, batch_size=1000)
else:
collection = TempDataDelta._get_collection()
activities = collection.find({'company_id': int(company_id), 'record_type': 'contact_history', 'source_system': 'sfdc', 'job_id': job_id}, projection={'source_record': True}, batch_size=1000)
try:
print 'got history ' + str(activities.count())
for activity in activities:
newActivity = activity['source_record']
addThisActivity = True
print 'act is ' + str(newActivity)
leadId = newActivity["ContactId"]
if leadId is not None:
print 'trying to get lead'
existingLead = Lead.objects(Q(company_id = company_id) & Q(sfdc_contact_id = leadId)).first()
print 'contact is ' + str(existingLead)
else:
continue
if existingLead is None:
continue
if 'sfdc' in existingLead['activities']: # there are activities from SFDC for this leadId
for existingActivity in existingLead['activities']['sfdc']:
if existingActivity['Id'] == newActivity['Id']: # this activity already exists so exit the loop
addThisActivity = False
break
if addThisActivity:
existingLead['activities']['sfdc'].append(newActivity)
existingLead.save()
print 'saved new activity 1'
else:
print 'no sfdc acts'
addThisActivity = True
sfdc = []
print 'appending'
sfdc.append(newActivity)
print 'appended'
existingLead['activities']['sfdc'] = sfdc
existingLead.save()
print 'saved new activity 2'
except Exception as e:
print 'exception while saving SFDC Contact history to master' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def saveHsptOpportunitiesToMaster(user_id=None, company_id=None, job_id=None, run_type=None):
#job_id = ObjectId("55e6b0198afb002ef6a8c292")
print 'saving hspt opps to master'
if run_type == 'initial':
opps = TempData.objects(Q(company_id=company_id) & Q(record_type='opportunity') & Q(source_system='hspt') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
else:
opps = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='opportunity') & Q(source_system='hspt') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
oppListTemp = list(opps)
oppList = [i['source_record'] for i in oppListTemp]
try:
for opp in oppList:
associations = opp.get('associations', None)
if associations is not None:
#print 'found assoc'
related_leads_list = associations.get('associatedVids', None)
#print 'releated leads list is ' + str(len(related_leads_list))
for i in range(len(related_leads_list)):
lead_id = related_leads_list[i]
#print 'lead id is ' + str(lead_id)
existingLead = Lead.objects(Q(company_id=company_id) & Q(hspt_id=str(lead_id))).first()
#we found an existing lead with the same VID (hspt_id) as the deal
if existingLead is not None:
#print 'found existing lead'
if 'hspt' not in existingLead.opportunities:
#print 'hspt not in opps'
opportunities = {}
opportunities['hspt'] = []
opportunities['hspt'].append(opp)
existingLead.update(opportunities__hspt = opportunities['hspt'])
else:
if not any (e.get('dealId', None) == opp['dealId'] for e in existingLead.opportunities['hspt']): # does an opportunity with this Id already exist
opportunities = existingLead.opportunities['hspt']
opportunities.append(opp)
# save this opportunity
existingLead.update(opportunities__hspt = opportunities)
else: #this opp already exists
for i in range(len(existingLead.opportunities['hspt'])):
if existingLead.opportunities['hspt'][i]['dealId'] == opp['dealId']:
existingLead.opportunities['hspt'][i] = opp
existingLead.save()
#if no matching lead found, continue to next opportunity
except Exception as e:
send_notification(dict(type='error', success=False, message=str(e)))
def saveFbokPostStatsToMaster(user_id=None, company_id=None, job_id=None, run_type=None): #behaves differently because it directly saves the data to the AnalyticsData collection
if run_type == 'initial':
fb_stats = TempData.objects(Q(company_id=company_id) & Q(record_type='fb_post_stat') & Q(source_system='fbok') & Q(job_id=job_id) ).only('source_record')
else:
fb_stats = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='fb_post_stat') & Q(source_system='fbok') & Q(job_id=job_id) ).only('source_record')
fbList = list(fb_stats)
fbList = [i['source_record'] for i in fbList]
try:
for i in range(len(fbList)):
fb_record = {}
source_page_id = fbList[i].get('page_id', None)
source_post_id = fbList[i].get('post_id', None)
source_created_date = fbList[i].get('created_time', None)
insights = fbList[i].get('insights', None)
for insight in insights:
fb_record = insight
source_metric_id = fb_record['id']
source_metric_name = fb_record['name']
fbPostInsight = FbPostInsight.objects(Q(company_id=company_id) & Q(source_metric_id=source_metric_id)& Q(source_page_id=source_page_id) & Q(source_post_id=source_post_id)).first()
if fbPostInsight is None:
fbPostInsight = FbPostInsight(data=fb_record, company_id=company_id, source_created_date=source_created_date, source_metric_id=source_metric_id, source_metric_name=source_metric_name, source_page_id=source_page_id, source_post_id=source_post_id)
else:
fbPostInsight['source_metric_name'] = source_metric_name
fbPostInsight['source_created_date'] = source_created_date
fbValuesList = fbPostInsight['data']['values']
for entry in fb_record['values']: #iterate through each new date value of metric to see if it already exists
if fb_record['period'] == 'lifetime':
fbPostInsight['data']['values'][0]['value'] = entry['value']
else:
if not any(d['end_time'] == entry['end_time'] for d in fbValuesList): # does this already exist?
fbValuesList.extend(entry) #if not, add the date entry to the existing record
fbPostInsight.save()
except Exception as e:
print 'exception is ' + str(e)
send_notification(dict(type='error', success=False, message=str(e)))
def saveBufrTwInteractionsToMaster(user_id=None, company_id=None, job_id=None, run_type=None): #behaves differently because it directly saves the data to the AnalyticsData collection
if run_type == 'initial':
tw_interactions = TempData.objects(Q(company_id=company_id) & Q(record_type='tw_interaction') & Q(source_system='bufr') & Q(job_id=job_id) ).only('source_record')
else:
tw_interactions = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='tw_interaction') & Q(source_system='bufr') & Q(job_id=job_id) ).only('source_record')
company_query = 'company_id'
interaction_id_query = 'interaction_id'
tw_interactionsList = list(tw_interactions)
tw_interactionsList = [i['source_record'] for i in tw_interactionsList]
try:
for interaction in tw_interactionsList:
profile_id = interaction['profile_id']
published_date = datetime.fromtimestamp(float(interaction['sent_at']))
local_published_date = get_current_timezone().localize(published_date, is_dst=None)
date = local_published_date.strftime('%Y-%m-%d')
queryDict = {company_query : company_id, interaction_id_query: interaction['id']}
publishedTweet = PublishedTweet.objects(**queryDict).first()
if publishedTweet is None: #this tweet's record not found
publishedTweet = PublishedTweet()
publishedTweet.company_id = company_id
publishedTweet.interaction_id = interaction['id']
publishedTweet.published_date = date
publishedTweet.published_timestamp = interaction['sent_at']
publishedTweet.data = interaction
publishedTweet.save()
#
except Exception as e:
send_notification(dict(type='error', success=False, message=str(e)))
def saveSfdcContactsToMaster(user_id=None, company_id=None, job_id=None, run_type=None):
# delete later
# job_id = ObjectId("569adcfc8afb00205c799f28")
if run_type == "initial":
contacts = TempData.objects(
Q(company_id=company_id) & Q(record_type="contact") & Q(source_system="sfdc") & Q(job_id=job_id)
).only(
"source_record"
) # & Q(job_id=job_id)
else:
contacts = TempDataDelta.objects(
Q(company_id=company_id) & Q(record_type="contact") & Q(source_system="sfdc") & Q(job_id=job_id)
).only(
"source_record"
) # & Q(job_id=job_id)
print "co id is " + str(company_id)
contactListTemp = list(contacts)
contactList = [i["source_record"] for i in contactListTemp]
# print 'saving sfdc contacts'
try:
# get the custom field for Contact Status, if it exists
existingIntegration = CompanyIntegration.objects(company_id=company_id).first()
contact_status = None
if "sfdc" in existingIntegration["integrations"]:
contact_status = existingIntegration["mapping"].get("sfdc_contact_status", None)
for newContact in contactList: # ['records']:
# company_id = request.user.company_id
sfdc_contact_Id = str(newContact["Id"])
print "contact id is " + sfdc_contact_Id
# sfdc_mkto_id = str(newLead['sfdcLeadId']) #check if there is a corresponding lead from MKTO
existingLeadMkto = None
existingLeadSfdc = None
existingLeadHspt = None
existingContact = Lead.objects(Q(company_id=company_id) & Q(sfdc_contact_id=sfdc_contact_Id)).first()
if existingContact is not None: # we found this contact already in the DB
print "found contact match for " + str(sfdc_contact_Id)
if "sfdc" in existingContact.contacts:
existingContact.source_first_name = newContact["FirstName"]
existingContact.source_last_name = newContact["LastName"]
existingContact.source_email = newContact["Email"]
# existingContact.source_created_date = str(newContact['CreatedDate'])
existingContact.source_source = newContact["LeadSource"]
if contact_status is not None and contact_status in newContact:
existingContact.source_status = newContact[contact_status]
existingContact.contacts["sfdc"] = newContact
existingContact.save()
# Lead.objects(Q(company_id=company_id) & Q(sfdc_contact_id=sfdc_contact_Id)).update(contacts__sfdc=newContact)
else:
existingContact.contacts["sfdc"] = {}
existingContact.contacts["sfdc"] = newContact
existingContact.save()
# elif existingContact is None: # this lead does not exist
else:
existingLeadSfdc = (
Lead.objects(Q(company_id=company_id) & Q(leads__sfdc__ConvertedContactId=sfdc_contact_Id))
.hint("company_id_1_leads.sfdc.convertedContactId_1")
.first()
)
if existingLeadSfdc is not None:
print "found match for sfdc lead for contact " + str(sfdc_contact_Id)
# existingLeadSfdcList = list(existingLeadSfdc)
existingLeadSfdc.sfdc_contact_id = sfdc_contact_Id
if contact_status is not None and contact_status in newContact:
existingLeadSfdc.source_status = newContact[contact_status]
existingLeadSfdc.contacts = {}
existingLeadSfdc.contacts["sfdc"] = newContact
existingLeadSfdc.save()
# remove below comments after figuring out how Mkto stored SFDC contact ID
else:
existingLeadMkto = Lead.objects(
Q(company_id=company_id) & Q(leads__mkto__sfdcContactId=sfdc_contact_Id)
).first()
if (
existingLeadMkto is not None
): # we found a MKto lead record which is matched to this new Sfdc lead
print "found mkto lead" + existingLeadMkto.mkto_id
existingLeadMkto.sfdc_contact_id = sfdc_contact_Id
# existingLeadMkto.contacts = {}
existingLeadMkto.contacts["sfdc"] = newContact
existingLeadMkto.save()
else:
existingLeadHspt = Lead.objects(
Q(company_id=company_id) & Q(leads__hspt__properties__salesforcecontactid=sfdc_contact_Id)
).first()
if (
existingLeadHspt is not None
): # we found a MKto lead record which is matched to this new Sfdc lead
existingLeadHspt.sfdc_contact_id = sfdc_contact_Id
existingLeadHspt.contacts = {}
existingLeadHspt.contacts["sfdc"] = newContact
existingLeadHspt.save()
if (
existingLeadSfdc is None
and existingLeadMkto is None
and existingLeadHspt is None
and existingContact is None
): # no matches found so save new record
lead = Lead()
lead.sfdc_contact_id = sfdc_contact_Id
lead.company_id = company_id
lead.source_first_name = newContact["FirstName"]
lead.source_last_name = newContact["LastName"]
lead.source_email = newContact["Email"]
lead.source_created_date = str(newContact["CreatedDate"])
lead.source_source = newContact["LeadSource"]
if contact_status is not None and contact_status in newContact:
lead.source_status = newContact[contact_status]
lead.contacts = {}
lead.contacts["sfdc"] = newContact
lead.save()
# lead = Lead()
# company_id = request.user.company_id
# derived_id = 'sfdc_' + str(newLead['Id'])
# Lead.objects(derived_id = derived_id).modify(upsert=True, new=True, set__leads__sfdc = newLead, set_on_insert__derived_id = derived_id, set_on_insert__company_id = company_id)
#
# oldLead = Lead.objects(derived_id = lead.derived_id)
# if oldLead.count() == 0:
# lead.leads["sfdc"] = newLead
# lead.save()
# else:
# oldLead.leads["sfdc"] = newLead
# Lead.objects(derived_id = lead.derived_id).update(oldLead)
except Exception as e:
print "exception while saving SFDC contact " + str(e)
send_notification(dict(type="error", success=False, message=str(e)))
def companyDataExtract(user_id=None, company_id=None, run_type=None, sinceDateTime=None):
superJobMonitor = None
existingIntegration = None
if run_type is None or company_id is None or sinceDateTime is None:
return
try:
sinceDateTime = datetime.fromtimestamp(float(sinceDateTime) / 1000)
#sinceDateTime = datetime.now() - timedelta(days=30)
print 'start date is ' + str(sinceDateTime)
print 'run type is ' + run_type
print 'company id is ' + str(company_id)
#create an entry in the Job Monitor
superJobMonitor = SuperJobMonitor(company_id=company_id, type=run_type, started_date=datetime.utcnow())
superJobMonitor.save()
#do pre-check
_superJobMonitorAddTask(superJobMonitor, "Claritix", "Pre-check started")
# get the integration record
existingIntegration = CompanyIntegration.objects(company_id=company_id).first()
if existingIntegration is None:
_superJobMonitorEnd(superJobMonitor, None, 'Failed', 'No integration record found')
mail_admins('Could not find integration record for company ' + company_id , 'Check settings')
return False
if run_type == 'initial':
existingIntegration['initial_run_in_process'] = True #set the flag
else: #delta
existingIntegration['delta_run_in_process'] = True #set the flag
existingIntegration.save() # save the flag
# set up the Request and Cookie
user = _get_superadmin()
if user is None:
_superJobMonitorEnd(superJobMonitor, existingIntegration, 'Failed', 'No super admin found')
mail_admins('Could not find super admin!', 'Check settings')
return False
# remotely login the user
host = settings.BASE_URL
url = host + '/api/v1/auth/login/'
creds = {'email': '*****@*****.**', 'password':'******'}
s = requests.Session()
resp = s.post(url, data=json.dumps(creds))
if not resp.status_code == 200:
_superJobMonitorEnd(superJobMonitor, existingIntegration, 'Failed', 'Could not login super admin!')
mail_admins('Could not login super admin!', 'Check credentials')
return False
#do cookie thing - refer to SuperAdmin Cron Job Task for details
cookies = dict(sessionid = resp.cookies['sessionid'])
url = host + '/api/v1/users/'
resp = s.get(url, cookies=cookies)
_superJobMonitorAddTask(superJobMonitor, "Claritix", "Pre-check completed")
# #delete data in Lead Master
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of Leads, Contacts, Opportunities and Activities started")
# count = Lead.objects(company_id=company_id).count()
# Lead.objects(company_id=company_id).delete()
# _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " Leads, Contacts, Opportunities and Activities deleted")
#
# #delete data in Campaign Master
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of Campaigns started")
# count = Campaign.objects(company_id=company_id).count()
# Campaign.objects(company_id=company_id).delete()
# _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " Campaigns deleted")
#
# #delete data in Account Master
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of Accounts started")
# count = Account.objects(company_id=company_id).count()
# Account.objects(company_id=company_id).delete()
# _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " Accounts deleted")
#
# find out which systems are integrated and therefore which tasks should be run
task_map = {
#"mkto" : [retrieveMktoCampaigns, retrieveMktoLeadsByProgram], #retrieveMktoLeadCreatedActivities, retrieveMktoLeads, retrieveMktoActivities, retrieveMktoCampaigns, retrieveMktoLeadsByProgram], #IMPORTANT - Lead Created Activities has to be before Leads
#"hspt" : [retrieveHsptCampaigns], # , ], # retrieveHsptLeads, retrieveHsptOpportunities, retrieveHsptWebsiteTraffic, ,
"prdt" : [retrievePrdtLeads],
# "sfdc" : [retrieveSfdcCampaigns] #retrieveSfdcLeads, retrieveSfdcContacts, retrieveSfdcCampaigns, retrieveSfdcAccounts, retrieveSfdcOpportunities, retrieveSfdcLeadHistory, retrieveSfdcContactHistory, retrieveSfdcOppHistory, retrieveSfdcOppStageHistory],
# "sugr" : [retrieveSugrLeads],
# "bufr" : [retrieveBufrTwInteractions],
# "goog" : [retrieveGoogWebsiteTraffic], \
# "fbok" : [retrieveFbokPageStats, retrieveFbokAdStats]#, retrieveFbokPostStats] # , ]
}
# # for future use - retrieveMktoContacts, retrieveMktoOpportunities, retrieveSfdcActivities,
final_task_map = {
# "mkto" : [saveMktoCampaignsToMaster, saveMktoLeadsByProgramToMaster], #saveMktoLeadsToMaster, saveMktoActivitiesToMaster, saveMktoCampaignsToMaster, saveMktoLeadsByProgramToMaster],#mergeMktoSfdcLeads, deleteLeads, deleteDuplicateMktoIdLeads #
#"hspt" : [saveHsptCampaignsToMaster ], #saveHsptLeadsToMaster, saveHsptOpportunitiesToMaster, saveHsptWebsiteTrafficToMaster, , ], # ,
# "sfdc" : [saveSfdcCampaignsToMaster], #saveSfdcLeadsToMaster, saveSfdcContactsToMaster, saveSfdcCampaignsToMaster, saveSfdcAccountsToMaster, saveSfdcOpportunitiesToMaster, saveSfdcLeadHistoryToMaster, saveSfdcContactHistoryToMaster, saveSfdcOppHistoryToMaster, saveSfdcOppStageHistoryToMaster], #
# "sugr" : [saveSugrLeadsToMaster],
# "bufr" : [saveBufrTwInteractionsToMaster],
# "goog": [saveGoogleWebsiteTrafficToMaster],
# "fbok": [saveFbokPageStatsToMaster, saveFbokAdStatsToMaster]#, saveFbokPostStatsToMaster] #
}
# #
# # #saveSfdcLeadsToMaster, saveSfdcContactsToMaster, saveSfdcOpportunitiesToMaster, saveSfdcCampaignsToMaster,
# # # saveSfdcLeadsToMaster, saveSfdcContactsToMaster, saveSfdcOpportunitiesToMaster, saveSfdcCampaignsToMaster, saveSfdcAccountsToMaster
# # #collect all relevant tasks in one list and retrieve metadata as well
for source in existingIntegration.integrations.keys():
#change metadata depending on source system
if source == 'sfdc':
metadata_objects = ['user', 'lead', 'contact', 'campaign', 'opportunity', 'task', 'account'] #[] #objects for which metadata should be collected
elif source == 'mkto':
metadata_objects = ['lead', 'activity', 'campaign'] #[] #objects for which metadata should be collected
elif source == 'hspt':
metadata_objects = ['lead'] #objects for which metadata should be collected
else:
metadata_objects = [] #objects for which metadata should be collected
# if sfdc, explicitly refresh the access token
if source == 'sfdc':
sfdc = Salesforce()
sfdc.refresh_token(company_id)
#collect meta data
url = host + '/api/v1/company/' + str(company_id) + '/integrations/metadata/'
# for object in metadata_objects:
# _superJobMonitorAddTask(superJobMonitor, source, "Retrieval of metadata for " + object + " started")
# params = {'code': source, 'object': object}
# resp = s.get(url, params=params) # get metadata about activities
# if not resp.status_code == 200:
# _superJobMonitorAddTask(superJobMonitor, source, "Retrieval of metadata for " + object + " failed")
# continue
# else:
# _superJobMonitorAddTask(superJobMonitor, source, "Retrieval of metadata for " + object + " completed")
#collect retrieval tasks
print 'starting retrieval tasks'
tasks = []
if source in task_map:
tasks.extend(task_map[source])
#now run the tasks
for task in tasks:
_superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " started to store temp data")
print 'starting task ' + str(task)
task(user_id=user_id, company_id=company_id, job_id=superJobMonitor.id, run_type=run_type, sinceDateTime=sinceDateTime)
_superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " completed")
print 'starting save tasks'
#collect save tasks
tasks = []
if source in final_task_map:
tasks.extend(final_task_map[source])
#now run the tasks
for task in tasks:
_superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " started to save master data")
print 'starting task ' + str(task)
task(user_id=user_id, company_id=company_id, job_id=superJobMonitor.id, run_type=run_type)
_superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " completed")
#return #REMOVE THIS IN PRODUCTION
#if initial run, delete all analytics data
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, 'Claritix', "Deletion of analytics data started")
# count1 = AnalyticsData.objects(company_id=company_id).count() #ensure that website_traffic chart data is not deleted
# AnalyticsData.objects(company_id=company_id).delete() #ensure that website_traffic chart data is not deleted
# count2 = AnalyticsIds.objects(company_id=company_id).count()
# AnalyticsIds.objects(company_id=company_id).delete()
# _superJobMonitorAddTask(superJobMonitor, 'Claritix', str(count1 + count2) + " records deleted from analytics tables")
#
# call chart calculate tasks
charts = [\
# {'chart_name': 'sources_bar', 'system_type': 'MA', 'chart_title':'Timeline', 'mode': run_type, 'start_date': sinceDateTime}, \
# #{'chart_name': 'pipeline_duration', 'system_type': 'MA', 'chart_title':'Pipeline Duration', 'mode': run_type, 'start_date': sinceDateTime}, \
# #{'chart_name': 'contacts_distr', 'system_type': 'MA', 'chart_title':'Contacts Distribution', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'source_pie', 'system_type': 'MA', 'chart_title':'Source Distribution', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'revenue_source_pie', 'system_type': 'MA', 'chart_title':'Revenue by Source', 'mode': run_type, 'start_date': sinceDateTime}, \
# #{'chart_name': 'multichannel_leads', 'system_type': 'MA', 'chart_title':'Multichannel Leads', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'tw_performance', 'system_type': 'SO', 'chart_title':'Twitter Performance', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'fb_performance', 'system_type': 'SO', 'chart_title':'Facebook_Performance', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'google_analytics', 'system_type': 'AD', 'chart_title':'Google Analytics', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'campaign_email_performance', 'system_type': 'MA', 'chart_title':'Campaign Performance by Email', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'email_cta_performance', 'system_type': 'MA', 'chart_title':'Email Performance by CTA', 'mode': run_type, 'start_date': sinceDateTime}, \
#
]
url = host + '/api/v1/company/' + str(company_id) + '/analytics/calculate/'
for chart in charts:
print 'starting chart ' + str(chart['chart_title'])
_superJobMonitorAddTask(superJobMonitor, 'Claritix', "Started calculating " + str(chart['chart_title']))
resp = s.get(url, params=chart)
if not resp.status_code == 200:
print 'incorrect status code was ' + str(resp.status_code)
_superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Could not retrieve data for ' + chart['chart_title'])
continue
else:
_superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Retrieved data for ' + chart['chart_title'])
# call dashboard calculate tasks
dashboards = [\
# {'chart_name': 'social_roi', 'system_type': 'MA', 'chart_title':'Social Performance', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'funnel', 'system_type': 'MA', 'chart_title':'Funnel', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'opp_funnel', 'system_type': 'CRM', 'chart_title':'Opportunity Funnel', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'waterfall_chart', 'system_type': 'MA', 'chart_title':'Waterfall Chart', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'form_fills', 'system_type': 'MA', 'chart_title':'Form Fills', 'mode': run_type, 'start_date': sinceDateTime}, \
]
url = host + '/api/v1/company/' + str(company_id) + '/dashboards/calculate/'
for dashboard in dashboards:
print 'starting dashboard ' + str(dashboard['chart_title'])
_superJobMonitorAddTask(superJobMonitor, 'Claritix', "Started calculating " + str(dashboard['chart_title']))
resp = s.get(url, params=dashboard)
if not resp.status_code == 200:
print 'incorrect status code was ' + str(resp.status_code)
_superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Could not retrieve data for ' + dashboard['chart_title'])
continue
else:
_superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Retrieved data for ' + dashboard['chart_title'])
#delete all data from past successful initial runs in Temp Table
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of temp data table started")
# successfulJobs = SuperJobMonitor.objects(Q(company_id=company_id) & Q(type='initial') & Q(status='Completed'))
# successfulJobsListTemp = list(successfulJobs)
# print 'found job ids ' + str(len(successfulJobsListTemp))
# successfulJobsList = [i.id for i in successfulJobsListTemp]
# count = TempData.objects(job_id__in=successfulJobsList).count()
# TempData.objects(job_id__in=successfulJobsList).delete()
# _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " records deleted from temp data table")
#delete all data from past successful delta runs in Temp Table
if run_type == 'delta':
_superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of temp data delta table started")
successfulJobs = SuperJobMonitor.objects(Q(company_id=company_id) & Q(type='delta') & Q(status='Completed'))
successfulJobsListTemp = list(successfulJobs)
print 'found delta job ids ' + str(len(successfulJobsListTemp))
successfulJobsList = [i.id for i in successfulJobsListTemp]
count = TempDataDelta.objects(job_id__in=successfulJobsList).count()
TempDataDelta.objects(job_id__in=successfulJobsList).delete()
_superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " records deleted from temp data delta table")
#update last run date for initial run in Company Integration record
_superJobMonitorEnd(superJobMonitor, existingIntegration, run_type, 'Completed', 'All tasks completed successfully')
return True
except Exception as e:
if superJobMonitor is not None and existingIntegration is not None:
_superJobMonitorEnd(superJobMonitor, existingIntegration, run_type, 'Failed', str(e))
print str(e)
return False
def saveMktoActivitiesToMaster(user_id=None, company_id=None, job_id=None, run_type=None):
#job_id = ObjectId("56a2dd408afb006f9e7cb851")
if run_type == 'initial':
#activities = TempData.objects(Q(company_id=company_id) & Q(record_type='activity') & Q(source_system='mkto') & Q(job_id=job_id) ).only('source_record')
collection = TempData._get_collection()
activities = collection.find({'company_id': int(company_id), 'record_type': 'activity', 'source_system': 'mkto', 'job_id': job_id}, projection={'source_record': True}, batch_size=1000)
else:
collection = TempDataDelta._get_collection()
activities = collection.find({'company_id': int(company_id), 'record_type': 'activity', 'source_system': 'mkto', 'job_id': job_id}, projection={'source_record': True}, batch_size=1000)
#activities = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='activity') & Q(source_system='mkto') & Q(job_id=job_id) ).only('source_record')
# activityListTemp = list(activities)
# activityList = [i['source_record'] for i in activityListTemp]
existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
if existingIntegration is not None:
activityTypeArray = existingIntegration.integrations['mkto']['metadata']['activity']
else:
print 'No activity type metadata found for Marketo'
raise ValueError('No activity type metadata found for Marketo')
try:
for i in range(len(activityTypeArray)):
if activityTypeArray[i]['name'] == 'Change Data Value':
changeActivityId = activityTypeArray[i]['id']
print 'change id is ' + str(changeActivityId)
for activity in activities:
newActivity = activity['source_record']
addThisActivity = True
#company_id = request.user.company_id
mkto_id = str(newActivity['leadId'])
print 'doing lead ' + mkto_id
existingLead = Lead.objects(Q(mkto_id = str(mkto_id)) & Q(company_id = company_id)).first()
if existingLead is not None: # we found this lead to attach the activities
if 'mkto' in existingLead.activities:
currentActivities = existingLead.activities['mkto']
for i in range(len(currentActivities)):
if currentActivities[i]['id'] == newActivity['id']: #check if this activity already exists in the lead dict
addThisActivity = False
break
if addThisActivity == True:
for i in range(len(activityTypeArray)):
if activityTypeArray[i]['id'] == newActivity['activityTypeId']:
newActivity['activityTypeName'] = activityTypeArray[i]['name']
break
currentActivities.append(newActivity)
existingLead.update(activities__mkto = currentActivities)
else:
currentActivities = []
for i in range(len(activityTypeArray)):
if activityTypeArray[i]['id'] == newActivity['activityTypeId']:
newActivity['activityTypeName'] = activityTypeArray[i]['name']
break
currentActivities.append(newActivity)
existingLead.update(activities__mkto = currentActivities)
#addThisActivity == True and
if addThisActivity == True and newActivity['activityTypeId'] == changeActivityId and newActivity['primaryAttributeValue'] == 'Lead Status':
print 'processing status activity for id ' + mkto_id
#statusRecord = [];
newStatus = ''
oldStatus = ''
for attribute in newActivity['attributes']:
if attribute['name'] == 'New Value':
newStatus = attribute['value']
elif attribute['name'] == 'Old Value':
oldStatus = attribute['value']
# elif attribute['name'] == 'Reason':
# reason = attribute['value']
#break
#statusRecord.append({'status': newStatus, 'date': newActivity['activityDate']})
newActivity['newStatus'] = newStatus
newActivity['oldStatus'] = oldStatus
newActivity['date'] = newActivity['activityDate']
if 'mkto' in existingLead.statuses:
currentStatuses = existingLead.statuses['mkto']
currentStatuses.append(newActivity) # changed on 1/22/2016 {'status': newStatus, 'date': newActivity['activityDate']})
existingLead.update(statuses__mkto = currentStatuses)
else:
currentStatuses = []
currentStatuses.append(newActivity) # changed on 1/22/2016{'status': newStatus, 'date': newActivity['activityDate']})
existingLead.update(statuses__mkto = currentStatuses)
# if addThisActivity == True: # this activity was not foudn in the lead, so add it
# existingLead.activities['mkto'].append(newActivity)
# existingLead.save() # no concept of saving the activity if the lead does not exist
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def saveTempDataDelta(company_id=None, record_type=None, source_system=None, source_record=None, job_id=None):
try:
tempDataDelta = TempDataDelta(company_id=company_id, record_type=record_type, source_system=source_system, source_record=source_record, updated_date=datetime.utcnow(), job_id=job_id)
tempDataDelta.save()
except Exception as e:
print 'exception while saving: ' + str(e)
def retrieveSfdcLeadHistory(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): #needs to be changed - satya
try:
#delete later
#job_id_new = job_id
#job_id = ObjectId("56a690e98afb006883048e7e")
#set variables
sfdc = Salesforce()
#for leads
if run_type == 'initial':
leads = TempData.objects(Q(record_type='lead') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
else:
leads = TempDataDelta.objects(Q(record_type='lead') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
leadListTemp = list(leads)
if not leadListTemp:
print 'no leads found'
return
leadList = [i['source_record'] for i in leadListTemp]
batch_size = 500 #10 Activity Types at a time
activitiesList = []
for i in range(0, len(leadList), batch_size):
lead_list = '('
for lead in leadList[i:i+batch_size]:
lead_list += '\'' + lead['Id'] + '\'' + ', '
lead_list = lead_list[:-2]
lead_list += ')'
activitiesList.extend(sfdc.get_history_for_lead(user_id, company_id, lead_list, _str_from_date(sinceDateTime)))
print 'got back history for SFDC leads ' + str(len(activitiesList))
#delete later
#job_id = job_id_new
saveSfdcLeadHistory(user_id=user_id, company_id=company_id, activityList=activitiesList, job_id=job_id, run_type=run_type)
# #for contacts
# if run_type == 'initial':
# contacts = TempData.objects(Q(record_type='contact') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
# else:
# contacts = TempDataDelta.objects(Q(record_type='contact') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
#
# contactListTemp = list(contacts)
# contactList = [i['source_record'] for i in contactListTemp]
#
# sfdc = Salesforce()
#
# batch_size = 500 #10 Activity Types at a time
# activitiesList = []
#
# for i in range(0, len(contactList), batch_size):
# contact_list = '('
# for contact in contactList[i:i+batch_size]:
# contact_list += '\'' + contact['Id'] + '\'' + ', '
# contact_list = contact_list[:-2]
# contact_list += ')'
# activitiesList.extend(sfdc.get_history_for_contact(user_id, company_id, contact_list, _str_from_date(sinceDateTime)))
#
# print 'got back history for SFDC contacts ' + str(len(activitiesList))
# saveSfdcHistory(user_id=user_id, company_id=company_id, activityList=activitiesList, job_id=job_id, run_type=run_type)
#
try:
message = 'Lead history retrieved from Salesforce'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Leads'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return leadList
except Exception as e:
print 'exception while retrieving SFDC lead history: ' + str(e)
send_notification(dict(type='error', success=False, message=str(e)))
def saveSfdcOpportunitiesToMaster(user_id=None, company_id=None, job_id=None, run_type=None):
#job_id = ObjectId("56a3f89f8afb003c13a59e26")
if run_type == 'initial':
opps = TempData.objects(Q(company_id=company_id) & Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
contacts = TempData.objects(Q(company_id=company_id) & Q(record_type='contact') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
else:
opps = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
contacts = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='contact') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
oppListTemp = list(opps)
oppList = [i['source_record'] for i in oppListTemp]
contactListTemp = list(contacts)
contactList = [i['source_record'] for i in contactListTemp]
try:
allOpps = oppList #['records']
# below code copied from contacts.tasks
for newContact in contactList: #['records']:
if 'OpportunityContactRoles' not in newContact or newContact['OpportunityContactRoles'] is None: # if this contact has no opportunities
continue # move to next contact
# company_id = request.user.company_id
sfdc_account_id = None
thisLeadsOppsIds = newContact['OpportunityContactRoles']['records']
thisLeadsOpps = []
for opp in thisLeadsOppsIds: #loop through all the Opp records in the Contact record
print 'trying for opp with id ' + str(opp['OpportunityId'])
thisOpp = next((x for x in allOpps if x['Id'] == opp['OpportunityId']), None) # if this opp is found in the list of opps retrieved separately
if thisOpp is not None: # if found
print 'found this opp'
sfdc_account_id = thisOpp['AccountId']
thisLeadsOpps.append(thisOpp) #add it
sfdc_contact_Id = str(newContact['Id'])
print 'contact id is ' + sfdc_contact_Id
# sfdc_mkto_id = str(newLead['sfdcLeadId']) #check if there is a corresponding lead from MKTO
existingLeadMkto = None
existingLeadSfdc = None
existingLeadHspt = None
existingContact = Lead.objects(Q(company_id=company_id) & Q(sfdc_contact_id=sfdc_contact_Id)).first()
if existingContact is not None: # we found this contact already in the DB
print ' eC is not none'
if 'sfdc' not in existingContact.opportunities:
opportunities = {}
opportunities['sfdc'] = []
opportunities['sfdc'].extend(thisLeadsOpps)
existingContact.update(opportunities__sfdc = opportunities['sfdc'])
existingContact.update(sfdc_account_id = sfdc_account_id)
print 'just updated acct id 1'
else:
for newOpp in thisLeadsOpps:
print ' nefre get'
if not any (e.get('Id', None) == newOpp['Id'] for e in existingContact.opportunities['sfdc']): # does an opportunity with this Id already exist
opportunities = existingContact.opportunities['sfdc']
opportunities.append(newOpp)
existingContact.sfdc_account_id = sfdc_account_id
# save this opportunity
existingContact.update(opportunities__sfdc = opportunities)
existingContact.update(sfdc_account_id = sfdc_account_id)
print 'just updated acct id 2'
else: #this opp already exists
for i in range(len(existingContact.opportunities['sfdc'])):
if existingContact.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
existingContact.opportunities['sfdc'][i] = newOpp
existingContact.sfdc_account_id = sfdc_account_id
existingContact.save()
print 'just updated acct id 3'
elif existingContact is None: # this lead does not exist
print ' eC is much none'
existingLeadSfdc = Lead.objects(Q(company_id=company_id) & Q(leads__sfdc__ConvertedContactId=sfdc_contact_Id)).first()
if existingLeadSfdc is not None:
if 'sfdc' not in existingLeadSfdc.opportunities:
opportunities = {}
opportunities['sfdc'] = []
opportunities['sfdc'].extend(thisLeadsOpps)
existingLeadSfdc.update(opportunities__sfdc = opportunities['sfdc'])
existingLeadSfdc.update(sfdc_account_id = sfdc_account_id)
print 'just updated acct id 4'
else:
for newOpp in thisLeadsOpps:
if not any (e.get('Id', None) == newOpp['Id'] for e in existingLeadSfdc.opportunities['sfdc']): # does an opportunity with this Id already exist
opportunities = existingLeadSfdc.opportunities['sfdc']
opportunities.append(newOpp)
# save this opportunity
existingLeadSfdc.update(opportunities__sfdc = opportunities)
existingLeadSfdc.update(sfdc_account_id = sfdc_account_id)
print 'just updated acct id 5'
else: #this opp already exists
for i in range(len(existingLeadSfdc.opportunities['sfdc'])):
if existingLeadSfdc.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
existingLeadSfdc.opportunities['sfdc'][i] = newOpp
existingLeadSfdc.sfdc_account_id = sfdc_account_id
existingLeadSfdc.save()
print 'just updated acct id 6'
else:
existingLeadMkto = Lead.objects(Q(company_id=company_id) & Q(leads__mkto__sfdcContactId=sfdc_contact_Id)).first()
if existingLeadMkto is not None: # we found a MKto lead record which is matched to this new Sfdc lead
if 'sfdc' not in existingLeadMkto.opportunities:
opportunities = {}
opportunities['sfdc'] = []
opportunities['sfdc'].extend(thisLeadsOpps)
existingLeadMkto.update(opportunities__sfdc = opportunities['sfdc'])
existingLeadMkto.update(sfdc_account_id = sfdc_account_id)
else:
for newOpp in thisLeadsOpps:
if not any (e.get('Id', None) == newOpp['Id'] for e in existingLeadMkto.opportunities['sfdc']): # does an opportunity with this Id already exist
opportunities = existingLeadMkto.opportunities['sfdc']
opportunities.append(newOpp)
# save this opportunity
existingLeadMkto.update(opportunities__sfdc = opportunities)
existingLeadMkto.update(sfdc_account_id = sfdc_account_id)
else: #this opp already exists
for i in range(len(existingLeadMkto.opportunities['sfdc'])):
if existingLeadMkto.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
existingLeadMkto.opportunities['sfdc'][i] = newOpp
existingLeadMkto.sfdc_account_id = sfdc_account_id
existingLeadMkto.save()
existingLeadHspt = Lead.objects(Q(company_id=company_id) & Q(leads__hspt__properties__salesforcecontactid=sfdc_contact_Id)).first()
if existingLeadHspt is not None: # we found a Hspt lead record which is matched to this new Sfdc lead
if 'sfdc' not in existingLeadHspt.opportunities:
opportunities = {}
opportunities['sfdc'] = []
opportunities['sfdc'].extend(thisLeadsOpps)
existingLeadHspt.update(opportunities__sfdc = opportunities['sfdc'])
existingLeadHspt.update(sfdc_account_id = sfdc_account_id)
else:
for newOpp in thisLeadsOpps:
if not any (e.get('Id', None) == newOpp['Id'] for e in existingLeadHspt.opportunities['sfdc']): # does an opportunity with this Id already exist
opportunities = existingLeadHspt.opportunities['sfdc']
opportunities.append(newOpp)
# save this opportunity
existingLeadHspt.update(opportunities__sfdc = opportunities)
existingLeadHspt.update(sfdc_account_id = sfdc_account_id)
else: #this opp already exists
for i in range(len(existingLeadHspt.opportunities['sfdc'])):
if existingLeadHspt.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
existingLeadHspt.opportunities['sfdc'][i] = newOpp
existingLeadHspt.sfdc_account_id = sfdc_account_id
existingLeadHspt.save()
if existingLeadSfdc is None and existingLeadMkto is None and existingLeadHspt is None and existingContact is None: # no matches found so save new record
lead = Lead()
lead.sfdc_contact_id = sfdc_contact_Id
lead.company_id = company_id
lead.source_first_name = newContact['FirstName']
lead.source_last_name = newContact['LastName']
lead.source_email = newContact['Email']
lead.source_created_date = str(newContact['CreatedDate'])
lead.source_source = newContact['LeadSource']
lead.sfdc_account_id = sfdc_account_id
lead.save()
contacts = {}
contacts['sfdc'] = newContact
#lead.contacts["sfdc"] = newContact
lead.update(contacts__sfdc = contacts['sfdc'])
print 'just updated acct id 7'
opportunities = {}
opportunities['sfdc'] = []
opportunities['sfdc'].extend(thisLeadsOpps)
lead.update(opportunities__sfdc = opportunities['sfdc'])
#new code added on 1/24/2016 - add each opportunity to related account (to capture opps with no contacts)
for opp in allOpps:
account = Account.objects(Q(company_id=company_id) & Q(sfdc_id=opp['AccountId'])).first()
if account is None:
print 'no account for opp with ID ' + str(opp['Id'])
sfdc = Salesforce()
accountList = sfdc.get_single_account(user_id, company_id, opp['AccountId'])
for newAccount in accountList['records']:
account = _saveSfdcNewAccount(newAccount['Id'], newAccount, None, company_id)
if 'opportunities' not in account:
account.update(opportunities = {})
if 'sfdc' not in account.opportunities: #no opps exist for this account, so add this opp
opportunities = {}
opportunities['sfdc'] = []
opportunities['sfdc'].append(opp)
account.update(opportunities__sfdc = opportunities['sfdc'])
else:
if not any (e.get('Id', None) == opp['Id'] for e in account.opportunities['sfdc']): # there is no opportunity with this Id already exist
opportunities = account.opportunities['sfdc']
opportunities.append(opp)
# save this opportunity
account.update(opportunities__sfdc = opportunities)
else: # this opp already exists
for i in range(len(account.opportunities['sfdc'])):
if account.opportunities['sfdc'][i]['Id'] == opp['Id']:
account.opportunities['sfdc'][i] = opp
account.save()
# code commented out since we are no longer getting only Mkto related opportunities into Cx
# for newOpp in oppList['records']:
#
# # company_id = request.user.company_id
# sfdc_account_id = str(newOpp['AccountId']) # find the account ID
# print 'account id is ' + sfdc_account_id
# # sfdc_mkto_id = str(newLead['sfdcLeadId']) #check if there is a corresponding lead from MKTO
# existingLeadMkto = None
# existingLeadSfdc = None
# existingLeadHspt = None
# #existingContact = Lead.objects(Q(company_id=company_id) & Q(sfdc_contact_id=sfdc_contact_Id)).first()
# existingLeadSfdc = Lead.objects(Q(company_id=company_id) & Q(leads__sfdc__ConvertedAccountId=sfdc_account_id)).first()
# if existingLeadSfdc is not None:
# if 'sfdc' not in existingLeadSfdc.opportunities:
# opportunities = {}
# opportunities['sfdc'] = []
# opportunities['sfdc'].append(newOpp) # add this opp to the new array
# existingLeadSfdc.update(opportunities__sfdc = opportunities)
# else:
# if not any (e.get('Id', None) == newOpp['Id'] for e in existingLeadSfdc.opportunities['sfdc']): # does an opportunity with this Id already exist
# opportunities = existingLeadSfdc.opportunities['sfdc']
# opportunities.append(newOpp)
# # save this opportunity
# existingLeadSfdc.update(opportunities__sfdc = opportunities)
# else: #this opp already exists
# for i in range(len(existingLeadSfdc.opportunities['sfdc'])):
# if existingLeadSfdc.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
# existingLeadSfdc.opportunities['sfdc'][i] = newOpp
# existingLeadSfdc.save()
#
# # and move to the next opportunity in the loop
# continue
# else: #this opp does not exist within an SFDC lead
# # check if it is a Marketo lead
# existingLeadMkto = Lead.objects(Q(company_id=company_id) & Q(leads__mkto__sfdcAccountId=sfdc_account_id)).first()
# if existingLeadMkto is not None: # we found a MKto lead record which is matched to this opp
# print 'found mkto lead' + existingLeadMkto.mkto_id
# if 'sfdc' not in existingLeadMkto.opportunities:
# opportunities = []
# opportunities.append(newOpp) # add this opp to the new array
# existingLeadMkto.update(opportunities__sfdc = opportunities)
# print 'saved opps'
# else: # if ['opportunities']['sfdc'] already exists
# print 'opp exist'
# if not any (e.get('Id', None) == newOpp['Id'] for e in existingLeadMkto.opportunities['sfdc']): # does an opportunity with this Id already exist
# opportunities = existingLeadMkto.opportunities['sfdc']
# opportunities.append(newOpp)
# # save this opportunity
# existingLeadMkto.update(opportunities__sfdc = opportunities)
# print 'saved sfdc'
# else: #this opp already exists
# print 'opp does not exist'
# for i in range(len(existingLeadMkto.opportunities['sfdc'])):
# if existingLeadMkto.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
# existingLeadMkto.opportunities['sfdc'][i] = newOpp
# existingLeadMkto.save()
# print 'saved update'
# #move on to the next opp
# continue
# existingLeadHspt = Lead.objects(Q(company_id=company_id) & Q(leads__hspt__properties__salesforceaccountid=sfdc_account_id)).first()
# if existingLeadHspt is not None: # we found a MKto lead record which is matched to this new Sfdc lead
# if 'sfdc' not in existingLeadHspt.opportunities:
# opportunities = {}
# opportunities['sfdc'] = []
# opportunities['sfdc'].append(newOpp) # add this opp to the new array
# existingLeadHspt.update(opportunities__sfdc = opportunities)
# else:
# if not any (e.get('Id', None) == newOpp['Id'] for e in existingLeadHspt.opportunities['sfdc']): # does an opportunity with this Id already exist
# opportunities = existingLeadHspt.opportunities['sfdc']
# opportunities.append(newOpp)
# # save this opportunity
# existingLeadHspt.update(opportunities__sfdc = opportunities)
# else: #this opp already exists
# for i in range(len(existingLeadHspt.opportunities['sfdc'])):
# if existingLeadHspt.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
# existingLeadHspt.opportunities['sfdc'][i] = newOpp
# existingLeadHspt.save()
# # move onto the next opp
# continue
#
# if existingLeadSfdc is None and existingLeadMkto is None and existingLeadHspt is None: # no matches found so throw error i,e, not possible
# #raise ValueError('Opportunity found without lead or contact')
# lead = Lead()
# lead.sfdc_id = sfdc_Id
# lead.company_id = company_id
# lead.source_first_name = newLead['FirstName']
# lead.source_last_name = newLead['LastName']
# lead.source_email = newLead['Email']
# lead.source_created_date = str(newLead['CreatedDate'])
# lead.source_source = newLead['LeadSource']
# lead.source_status = newLead['Status']
# lead.leads["sfdc"] = newLead
# print '5th save'
# lead.save()
except Exception as e:
print 'Error while saving SFDC opportunities ' + str(e)
send_notification(dict(type='error', success=False, message=str(e)))