def retrieveSfdcCampaigns(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
sdfc = Salesforce()
campaignList = sdfc.get_campaigns_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type)
saveSfdcCampaigns(user_id=user_id, company_id=company_id, campaignList=campaignList, job_id=job_id, run_type=run_type)
try:
message = 'Campaigns retrieved from Salesforce'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Campaigns'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return campaignList
except Exception as e:
print 'error while retrieving sfdc campaigns: ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def retrieveFbokAdCampaignStats(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
print 'starting retrieveFbokAdCampaignStats for company ' + str(company_id)
existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
if 'fbok' not in existingIntegration['integrations']: # if Buffer is present and configured
print 'did not find fbok'
raise Exception('Facebook integration not found')
integration = existingIntegration.integrations['fbok']
if integration['access_token'] is None:
raise Exception('Facebook access token not found')
fbok = Facebook(integration['host'], integration['client_id'], integration['client_secret'], integration['redirect_uri'])
if fbok is None:
raise Exception('Facebook object could not be created')
print 'calling campaigns'
campaigns = fbok.get_campaign_stats(company_id, run_type)
print 'found FB campaigns: ' + str(campaigns)
saveFbokAdCampaignStats(user_id=user_id, company_id=company_id, results=campaigns, job_id=job_id, run_type=run_type)
except Exception as e:
print 'exception was ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def _get_hspt_email_details(email_id, existingCampaign, company_id, job_id, run_type):
try:
if run_type == 'initial':
campaigns = TempData.objects(Q(company_id=company_id) & Q(record_type='campaign') & Q(source_system='hspt') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
else:
campaigns = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='campaign') & Q(source_system='hspt') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
campaignListTemp = list(campaigns)
campaignList = [i['source_record'] for i in campaignListTemp]
#print 'temp recs found ' + str(len(campaignList))
#now get the appId for this email Id
app_id = None
int_email_id = 0
for campaign in campaignList:
#print 'campaign in temp ' + str(campaign)
if str(campaign['id']) == email_id:
print 'campaign id is ' + str(campaign['id']) + ' and email id is ' + email_id
app_id = campaign['appId']
int_email_id = campaign['id']
break
#if appId found, get details
emailDetails = None
if app_id is not None:
hspt = Hubspot(company_id)
emailDetails = hspt.get_campaign_details(campaignId=int_email_id, appId=app_id)
return emailDetails
except Exception as e:
print 'error while getting email details with Hspt campaign from traffic ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def saveFbokAdCampaignStatsToMaster(user_id=None, company_id=None, job_id=None, run_type=None): #behaves differently because it directly saves the data to the AnalyticsData collection
if run_type == 'initial':
fb_stats = TempData.objects(Q(company_id=company_id) & Q(record_type='fb_ad_campaign_stat') & Q(source_system='fbok') & Q(job_id=job_id) ).only('source_record')
else:
fb_stats = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='fb_ad_campaign_stat') & Q(source_system='fbok') & Q(job_id=job_id) ).only('source_record')
fbList = list(fb_stats)
fbList = [i['source_record'] for i in fbList]
try:
for i in range(len(fbList)):
fb_record = {}
source_campaign_id = fbList[i].get('id')
source_campaign_name = fbList[i].get('name')
source_account_id = fbList[i].get('account_id')
insights = fbList[i]['insights']
for insight in insights:
fb_record = insight['data']
source_created_date = fb_record['date_stop']
source_source = 'fbok'
FbAdCampaignInsight.objects(Q(company_id=company_id) & Q(source_created_date=source_created_date)& Q(source_campaign_id=source_campaign_id)).delete()
fbAdCampaignInsight = FbAdCampaignInsight(data=fb_record, company_id=company_id, source_campaign_id=source_campaign_id, source_campaign_name=source_campaign_name, source_account_id=source_account_id, source_created_date=source_created_date)
fbAdCampaignInsight.save()
except Exception as e:
print 'exception is ' + str(e)
send_notification(dict(type='error', success=False, message=str(e)))
def associateEmailWithCampaign(name, channel, company_id, email_id, job_id, run_type):
'''Associate an email template with a Hubspot campaign - from website traffic because there is no direct API method to extract campaigns '''
try:
existingCampaign = Campaign.objects(Q(name=name) & Q(channel=channel) & Q(company_id=company_id) & Q(source_system='hspt')).first()
if existingCampaign is None:
return #this should not happen
else:
emails = existingCampaign['emails']
if emails is None: #create a new email for the campaign
_doAssociationEmailWithCampaign(email_id, existingCampaign, company_id, job_id, run_type)
else:
email_exists = False
for email in emails:
emailDetails = None
if email['id'] == email_id: #if email exists, only update the details
email_exists = True
emailDetails = _get_hspt_email_details(email_id, existingCampaign, company_id, job_id, run_type)
email['details'] = emailDetails
existingCampaign.save()
print 'existing email updated'
if email_exists == False: #else, create a new email for the campaign
_doAssociationEmailWithCampaign(email_id, existingCampaign, company_id, job_id, run_type)
except Exception as e:
print 'error while associating email with Hspt campaign from traffic ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def retrieveMktoCampaigns(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
mkto = Marketo(company_id)
campaignList = mkto.get_programs()
saveMktoCampaigns(user_id=user_id, company_id=company_id, campaignList=campaignList, job_id=job_id, run_type=run_type)
try:
message = 'Campaigns retrieved from Marketo'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Campaigns'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return campaignList
except Exception as e:
print 'error while retrieving marketo campaigns: ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def saveHsptCampaignEmailEventRecords(company_id=None, run_type=None, job_id=None, guid=None):
'''called from saveHsptCampaignsToMaster to save all events for a single Hspt campaign email record'''
try:
#print 'saving single event'
if run_type == 'initial':
events = TempData.objects(Q(company_id=company_id) & Q(record_type='campaign_email_event') & Q(source_system='hspt') & Q(job_id=job_id) & Q(source_record__campaign_guid=guid) ).only('source_record') #& Q(job_id=job_id)
else:
events = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='campaign_email_event') & Q(source_system='hspt') & Q(job_id=job_id) & Q(source_record__campaign_guid=guid) ).only('source_record') #& Q(job_id=job_id)
eventsListTemp = list(events)
eventsList = [j['source_record'] for j in eventsListTemp]
#print 'found events ' + str(eventsList)
for event in eventsList:
#print 'about to check for event with ' + guid + ' email id ' + str(email['id']) + ' event id ' + str(event['event_id']) + ' recipient ' + str(event['recipient']) + ' created ' + str(event['created']) + ' details ' + str(event)
existingEvent = EmailEvent.objects(Q(company_id=company_id) & Q(campaign_guid=guid) & Q(email_id=event['email_id']) & Q(event_id=event['event_id'])).first()
if existingEvent is None:
newEvent = EmailEvent(company_id=company_id, source_system='hspt', campaign_guid=guid, email_id=event['email_id'], event_id=event['event_id'], details=event['details'], recipient=event['recipient'], created=event['created'], event_type=event['event_type'])
#print 'about to save new event with ' + guid + ' email id ' + str(email['id']) + ' event id ' + str(event['event_id']) + ' recipient ' + str(event['recipient']) + ' created ' + str(event['created']) + ' details ' + str(event)
newEvent.save()
except Exception as e:
print 'error while saving single Hspt campaign event record ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def saveSfdcCampaignsToMaster(user_id=None, company_id=None, job_id=None, run_type=None):
#job_id = ObjectId("56d25c6af6fd2a15df46cd60")
if run_type == 'initial':
campaigns = TempData.objects(Q(company_id=company_id) & Q(record_type='campaign') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
else:
campaigns = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='campaign') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
campaignListTemp = list(campaigns)
campaignList = [i['source_record'] for i in campaignListTemp]
try:
for newCampaign in campaignList: #['records']:
#company_id = request.user.company_id
guid = 'sfdc_' + str(newCampaign['Id'])
Campaign.objects(Q(guid = guid) & Q(company_id=company_id)).modify(upsert=True, new=True, set__campaigns__sfdc = newCampaign, set__updated_date = datetime.utcnow, set_on_insert__source_system = 'sfdc', set_on_insert__guid = guid, set_on_insert__company_id = company_id)
# for oldCampaign in campaignList['records']:
# # mktoCampaigns = []
# # mktoCampaigns.append(campaign)
# campaign = Campaign()
# campaign.company_id = request.user.company_id
# campaign.derived_id = 'sfdc_' + str(oldCampaign['Id'])
# campaign.campaigns["sfdc"] = oldCampaign
# campaign.save()
except Exception as e:
print 'error while saving sfdc campaign: ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def retrieveSfdcAccounts(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
if sinceDateTime is None:
sinceDateTime = (datetime.now() - timedelta(days=1)).date()
sfdc = Salesforce()
accountList = sfdc.get_accounts_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type)
print 'got back accounts ' + str(len(accountList['records']))
saveSfdcAccounts(user_id=user_id, company_id=company_id, accountList=accountList, job_id=job_id, run_type=run_type)
try:
message = 'Accounts retrieved from Salesforce'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Accounts'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return accountList
except Exception as e:
send_notification(dict(type='error', success=False, message=str(e)))
def saveSfdcAccountsToMaster(user_id=None, company_id=None, job_id=None, run_type=None):
#delete later
#job_id = ObjectId("569fd4078afb002426ef2fd3")
if run_type == 'initial':
accounts = TempData.objects(Q(company_id=company_id) & Q(record_type='account') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
else:
accounts = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='account') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
accountListTemp = list(accounts)
accountList = [i['source_record'] for i in accountListTemp]
try:
for newAccount in accountList:
sfdc_id = str(newAccount['Id'])
#find all leads that have this account ID
relatedLeadList = []
relatedLeadListTemp = None
relatedLeads = Lead.objects(Q(company_id=company_id) & Q(sfdc_account_id=sfdc_id)).only('sfdc_contact_id') #if SFDC Account, then matching lead must have a SFDC Contact ID
if relatedLeads is not None:
relatedLeadListTemp = [lead.to_mongo().to_dict() for lead in relatedLeads]
#print 'rll is ' + str(relatedLeadListTemp)
for i in relatedLeadListTemp:
if 'sfdc_contact_id' in i:
relatedLeadList.append({'sfdc_contact_id': i['sfdc_contact_id']})
#print 'related leads are ' + str(relatedLeadList)
# if relatedLeads is not None:
# #leadListTemp = list(relatedLeads)
# #relatedLeadList = [i.id for i in leadListTemp]
# for lead in relatedLeads:
# relatedLeadList.append(lead)
print 'account id is ' + sfdc_id
# sfdc_mkto_id = str(newLead['sfdcLeadId']) #check if there is a corresponding lead from MKTO
existingAccount = None
existingAccount = Account.objects(Q(company_id=company_id) & Q(sfdc_id=sfdc_id)).first()
if existingAccount is not None: # we found this contact already in the DB
print 'found existing account for id ' + str(sfdc_id)
if 'sfdc' in existingAccount.accounts:
existingAccount.source_name = newAccount['Name']
existingAccount.source_source = newAccount['AccountSource']
existingAccount.source_industry = newAccount['Industry']
existingAccount.source_created_date = newAccount['CreatedDate']
existingAccount.accounts["sfdc"] = newAccount
if relatedLeadList is not None:
existingAccount.leads = relatedLeadList
existingAccount.save()
#Lead.objects(Q(company_id=company_id) & Q(sfdc_contact_id=sfdc_contact_Id)).update(contacts__sfdc=newContact)
else:
existingAccount.accounts['sfdc'] = {}
existingAccount.accounts['sfdc'] = newAccount
if relatedLeadList is not None:
existingAccount.leads = relatedLeadList
existingAccount.save()
elif existingAccount is None: # this account does not exist
account = _saveSfdcNewAccount(sfdc_id, newAccount, relatedLeadList, company_id)
except Exception as e:
print 'exception while saving accounts ' + str(e)
send_notification(dict(type='error', success=False, message=str(e)))
def saveMktoContacts(user_id=None, company_id=None, leadList=None, newList=None):
try:
pass
except Exception as e:
send_notification(dict(type='error', success=False, message=str(e)))
def retrieveSfdcContacts(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
sdfc = Salesforce()
if sinceDateTime is None:
sinceDateTime = (datetime.now() - timedelta(days=30)).date()
contactList = sdfc.get_contacts_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type)
print "got back contacts " + str(len(contactList["records"]))
saveSfdcContacts(
user_id=user_id, company_id=company_id, contactList=contactList, job_id=job_id, run_type=run_type
)
try:
message = "Contacts retrieved from Salesforce"
notification = Notification()
# notification.company_id = company_id
notification.owner = user_id
notification.module = "Contacts"
notification.type = "Background task"
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(type="error", success=False, message=str(e)))
return contactList
except Exception as e:
send_notification(dict(type="error", success=False, message=str(e)))
def saveHsptCampaignsToMaster(user_id=None, company_id=None, job_id=None, run_type=None):
if run_type == 'initial':
campaigns = TempData.objects(Q(company_id=company_id) & Q(record_type='campaign') & Q(source_system='hspt') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
else:
campaigns = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='campaign') & Q(source_system='hspt') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
campaignListTemp = list(campaigns)
campaignList = [i['source_record'] for i in campaignListTemp]
try:
for newCampaign in campaignList:
#company_id = request.user.company_id
#derived_id = 'hspt_' + str(newCampaign['id'])
guid = newCampaign['guid']
name = newCampaign['name']
source_system = 'hspt'
channel = 'email'
emails = newCampaign['emails']
#save all email events for this campaign first
saveHsptCampaignEmailEventRecords(company_id=company_id, run_type=run_type, job_id=job_id, guid=guid)
#now update or create campaign record
existingCampaign = Campaign.objects(Q(guid = guid) & Q(company_id=company_id) & Q(source_system=source_system)).first() #.modify(upsert=True, new=True, set__emails = emails, set_on_insert__name = name, set_on_insert__guid = guid, set_on_insert__company_id = company_id, set_on_insert__updated_date=datetime.utcnow)
if existingCampaign is None: #new campaign so add it
newCampaign = Campaign(emails = emails, name = name, guid = guid, company_id = company_id, updated_date=datetime.utcnow, source_system=source_system)
newCampaign.save()
else: #campaign already exists so update relevant fields
#print 'existing campaign is ' + str(existingCampaign)
if 'emails' not in existingCampaign: #should never happen
continue
#print 'before emails ' + str(emails)
for email in emails: #loop through each email found in API call for this campaign
#save events separately
#print 'entering emails'
#now create/update the campaign record
email_found = False
for existingEmail in existingCampaign['emails']: #check if the API email already exists
if email['id'] == existingEmail['id']: #we found this email already existing so update specific fields
email_found = True
existingEmail['stats'] = email['stats']
existingEmail['details'] = email['details']
if email_found:
existingCampaign.save()
break #move to the next API email since we already found a match
if email_found == False: #email was not found so add it to existing emails
existingCampaign['emails'].append(email)
existingCampaign.save()
except Exception as e:
print 'exception ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def retrieveSfdcOppStageHistory(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): #needs to be changed - satya
try:
#delete later
#job_id_new = job_id
#job_id = ObjectId("56a690e98afb006883048e7e")
#set variables
sfdc = Salesforce()
#for leads
if run_type == 'initial':
opps = TempData.objects(Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
else:
opps = TempDataDelta.objects(Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
oppListTemp = list(opps)
if not oppListTemp:
print 'no opps found'
return
oppList = [i['source_record'] for i in oppListTemp]
batch_size = 500 #10 Activity Types at a time
activitiesList = []
for i in range(0, len(oppList), batch_size):
opp_list = '('
for opp in oppList[i:i+batch_size]:
opp_list += '\'' + opp['Id'] + '\'' + ', '
opp_list = opp_list[:-2]
opp_list += ')'
activitiesList.extend(sfdc.get_stage_history_for_opportunity(user_id, company_id, opp_list, _str_from_date(sinceDateTime)))
print 'got back stage history for SFDC opportunities ' + str(len(activitiesList))
#delete later
#job_id = job_id_new
saveSfdcOppStageHistory(user_id=user_id, company_id=company_id, activityList=activitiesList, job_id=job_id, run_type=run_type)
try:
message = 'Opportunity stage history retrieved from Salesforce'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Opportunities'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return opp_list
except Exception as e:
print 'exception while retrieving SFDC opportunity stage history: ' + str(e)
send_notification(dict(type='error', success=False, message=str(e)))
def saveSfdcOppStageHistoryToMaster(user_id=None, company_id=None, job_id=None, run_type=None):
#job_id = ObjectId("56b2b92c8afb0070a795c4b2")
if run_type == 'initial':
#activities = TempData.objects(Q(company_id=company_id) & Q(record_type='activity') & Q(source_system='mkto') & Q(job_id=job_id) ).only('source_record')
collection = TempData._get_collection()
activities = collection.find({'company_id': int(company_id), 'record_type': 'opp_stage_history', 'source_system': 'sfdc', 'job_id': job_id}, projection={'source_record': True}, batch_size=1000)
else:
collection = TempDataDelta._get_collection()
activities = collection.find({'company_id': int(company_id), 'record_type': 'opp_stage_history', 'source_system': 'sfdc', 'job_id': job_id}, projection={'source_record': True}, batch_size=1000)
try:
print 'got history ' + str(activities.count())
for activity in activities:
skipThis = False
newActivity = activity['source_record']
addThisActivity = True
print 'act is ' + str(newActivity)
newOppId = newActivity["OpportunityId"]
if newOppId is not None:
print 'trying to get opp'
existingAccount = Account.objects(Q(company_id = company_id) & Q(opportunities__sfdc__Id = newOppId)).first()
print 'account is ' + str(existingAccount)
else:
continue
if existingAccount is None:
continue
for existingOpp in existingAccount['opportunities']['sfdc']: #loop through each opp to find the right one
if existingOpp['Id'] == newOppId:
if 'stage_activities' not in existingOpp: #if no prior activities
existingOpp['stage_activities'] = [] #create list
existingOpp['stage_activities'].append(newActivity) #save activity
print 'saved virgin stage activity for opp'
else:
for existingActivity in existingOpp['stage_activities']: #check if this activity already exists
if existingActivity['Id'] == newActivity['Id']: #it exists, so skip the activity
skipThis = True
print 'skipping opp stage activity'
break
if skipThis:
break # get out of this for loop
else:
existingOpp['stage_activities'].append(newActivity)
print 'saved stage activity for opp'
existingAccount.save()
except Exception as e:
print 'exception while saving SFDC Opp stage history to master' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def saveSfdcContactHistoryToMaster(user_id=None, company_id=None, job_id=None, run_type=None):
#job_id = ObjectId("56a6faa28afb00042171cd89")
if run_type == 'initial':
#activities = TempData.objects(Q(company_id=company_id) & Q(record_type='activity') & Q(source_system='mkto') & Q(job_id=job_id) ).only('source_record')
collection = TempData._get_collection()
activities = collection.find({'company_id': int(company_id), 'record_type': 'contact_history', 'source_system': 'sfdc', 'job_id': job_id}, projection={'source_record': True}, batch_size=1000)
else:
collection = TempDataDelta._get_collection()
activities = collection.find({'company_id': int(company_id), 'record_type': 'contact_history', 'source_system': 'sfdc', 'job_id': job_id}, projection={'source_record': True}, batch_size=1000)
try:
print 'got history ' + str(activities.count())
for activity in activities:
newActivity = activity['source_record']
addThisActivity = True
print 'act is ' + str(newActivity)
leadId = newActivity["ContactId"]
if leadId is not None:
print 'trying to get lead'
existingLead = Lead.objects(Q(company_id = company_id) & Q(sfdc_contact_id = leadId)).first()
print 'contact is ' + str(existingLead)
else:
continue
if existingLead is None:
continue
if 'sfdc' in existingLead['activities']: # there are activities from SFDC for this leadId
for existingActivity in existingLead['activities']['sfdc']:
if existingActivity['Id'] == newActivity['Id']: # this activity already exists so exit the loop
addThisActivity = False
break
if addThisActivity:
existingLead['activities']['sfdc'].append(newActivity)
existingLead.save()
print 'saved new activity 1'
else:
print 'no sfdc acts'
addThisActivity = True
sfdc = []
print 'appending'
sfdc.append(newActivity)
print 'appended'
existingLead['activities']['sfdc'] = sfdc
existingLead.save()
print 'saved new activity 2'
except Exception as e:
print 'exception while saving SFDC Contact history to master' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def saveHsptContacts(user_id=None, company_id=None, leadList=None):
try:
for newLead in leadList:
newLead = vars(newLead)['_field_values']
hspt_id = str(newLead['vid'])
#print 'gs id is ' + str(hspt_id)
#hspt_sfdc_id = str(newLead['sfdcLeadId']) # check if there is a corresponding lead from SFDC
hspt_sfdc_id = None
if 'salesforceleadid' in newLead['properties']:
hspt_sfdc_id = str(newLead['properties']['salesforceleadid']) # temp fix by satya till SFDC ID field in Hubspot is discovered
#addThisList = True
existingLeadSfdc = None
existingLead = None
existingLead = Lead.objects(Q(company_id=company_id) & Q(hspt_id=hspt_id)).first()
if existingLead is not None and 'hspt' in existingLead.leads: # we found this lead already in the DB
Lead.objects(Q(company_id=company_id) & Q(hspt_id=hspt_id)).update(leads__hspt=newLead)
# if 'hspt' in existingLead.lists:
# currentLists = existingLead.lists['mkto']
# for i in range(len(currentLists)):
# if currentLists[i]['id'] == newList['id']: # check if this activity already exists in the lead dict
# addThisList = False
# if addThisList == True:
# currentLists.append(newList)
# existingLead.update(lists__mkto=currentLists)
# else:
# currentLists = []
# currentLists.append(newList)
# existingLead.update(lists__mkto=currentLists)
elif existingLead is None: # this lead does not exist
if hspt_sfdc_id is not None: # but has a SFDC lead id
print 'found lead with SFDC ID ' + str(hspt_sfdc_id)
existingLeadSfdc = Lead.objects(Q(company_id=company_id) & Q(leads__sfdc__Id=hspt_sfdc_id)).first()
if existingLeadSfdc is not None: # we found a SFDC lead record which is matched to this new Mkto lead
existingLeadSfdc.hspt_id = hspt_id
existingLeadSfdc.leads['hspt'] = newLead
existingLeadSfdc.save()
# currentLists = []
# currentLists.append(newList)
# existingLeadSfdc.update(lists__mkto=currentLists)
if existingLeadSfdc is None and existingLead is None: # no matches found so save new record
lead = Lead()
lead.hspt_id = hspt_id
lead.company_id = company_id
lead.leads["hspt"] = newLead
lead.save()
# currentLists = []
# currentLists.append(newList)
# lead.update(lists__mkto=currentLists)
except Exception as e:
send_notification(dict(type='error', success=False, message=str(e)))
def retrieveHsptOpportunities(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
print 'retrieving Hspt deals'
hspt = Hubspot(company_id)
oppList = []
# if run_type == 'initial':
# oppList = hspt.get_deals(count=10000)
# else:
# oppList = hspt.get_deals(count=1000)
# print 'got opps ' + str(len(oppList['results']))
# saveHsptOpportunities(user_id=user_id, company_id=company_id, oppList=oppList, job_id=job_id, run_type=run_type)
#print 'job id is ' + str(job_id)
if run_type == 'initial':
print 'initial run for opps'
try:
for opp in hspt.get_deals(count=10000)['results']:
oppList.append(opp)
if len(oppList) == 100:
#print 'going to save'
saveHsptOpportunities(user_id=user_id, company_id=company_id, oppList=oppList, job_id=job_id, run_type=run_type)
oppList = []
except Exception as e:
print 'exception: ' + str(e)
else:
try:
for opp in hspt.get_deals(count=1000)['results']:
oppList.append(opp)
if len(oppList) == 100:
saveHsptOpportunities(user_id=user_id, company_id=company_id, oppList=oppList, job_id=job_id, run_type=run_type)
oppList = []
except Exception as e:
print 'exception: ' + str(e)
try:
message = 'Opportunities retrieved from Hubspot'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Opportunities'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return oppList
except Exception as e:
send_notification(dict(type='error', success=False, message=str(e)))
def exportMktoLeadsToCsv(data, chart_name, user_id, company_id):
ids = data.get('results', None)
leads = Lead.objects().filter(company_id=company_id, mkto_id__in=ids).order_by('mkto_id').hint('co_mkto_id')
leads = list(leads)
leads = [lead.to_mongo().to_dict() for lead in leads]
if leads is None or len(leads) == 0:
print 'input is none'
return {'file_name': '', 'content_type' : 'text/csv'}
try:
print 'input not none ' + str(leads)
#open a temp file for writing
end_date_string = time.strftime('%Y-%m-%d_%H:%M:%S', time.localtime())
base_file_name = end_date_string + '_' + chart_name + '_leads' + '_cx.csv'
file_name = '/tmp/' + base_file_name
csv_out = open(file_name, 'wb')
fieldnames = ['Marketo ID', 'First Name', 'Last Name', 'Email', 'Company', 'Status', 'Source', 'Original Source Type', 'Created Date', 'SFDC Lead ID', 'SFDC Contact ID', 'SFDC Account ID', ]
#create writer
csv_writer = csv.DictWriter(csv_out, fieldnames=fieldnames, restval='', extrasaction='ignore')
csv_writer.writeheader()
for lead in leads:
created_date = ''
print 'lead is ' + str(lead['mkto_id'])
mkto_id_url = "http://app-sj09.marketo.com/leadDatabase/loadLeadDetail?leadId=" + str(lead['mkto_id'])
mkto_id = '=HYPERLINK("' + mkto_id_url + '", "' + str(lead['mkto_id']) + '")'
#print '1'
if 'createdAt' in lead['leads']['mkto'] and lead['leads']['mkto']['createdAt'] is not None:
created_date = lead['leads']['mkto']['createdAt']
#print '2'
if not 'leads' in lead or not 'mkto' in lead.get('leads', ''):
lead['leads'] = {}
lead['leads']['mkto'] = {}
for key, value in lead['leads']['mkto'].items():
if lead['leads']['mkto'][key] is None:
lead['leads']['mkto'][key] = ''
csv_writer.writerow({'Marketo ID' : mkto_id, 'First Name': lead.get('source_first_name', '').encode('utf-8'), 'Last Name': lead.get('source_last_name', '').encode('utf-8'), 'Email': lead.get('source_email', '').encode('utf-8'),
'Company': lead['leads']['mkto'].get('company', '').encode('utf-8'), 'Status': lead['leads']['mkto'].get('leadStatus', '').encode('utf-8'), 'Source': lead['leads']['mkto'].get('leadSource', '').encode('utf-8'),
'Original Source Type': lead['leads']['mkto'].get('originalSourceType', '').encode('utf-8'), 'Created Date': created_date,
'SFDC Lead ID': lead['leads']['mkto'].get('sfdcLeadId', '').encode('utf-8'), 'SFDC Contact ID': lead['leads']['mkto'].get('sfdcContactId', '').encode('utf-8'), 'SFDC Account ID': lead['leads']['mkto'].get('sfdcAccountId', '').encode('utf-8'),
})
#print '4'
csv_out.close()
return {'file_name': file_name, 'content_type' : 'text/csv'}
except Exception as e:
print 'exception while trying to create CSV file: ' + str(e)
send_notification(dict(type='error', success=False, message=str(e)))
def retrieveSfdcOpportunities(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
#company_id = request.user.company_id
existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
# code commented out since we are no longer getting only Mkto related opportunities into Cx
# if existingIntegration is not None and 'mkto' in existingIntegration['integrations']: # if this system is connected to Marketo
# company_qry = 'company_id'
# type_field_qry = 'leads__mkto__exists'
# sfdc_account_field_qry = 'leads__mkto__sfdcAccountId__ne'
# querydict = {company_qry: company_id, type_field_qry: True, sfdc_account_field_qry: None}
# leads_with_sfdc_opps = Lead.objects(**querydict).only('mkto_id').only('leads__mkto__sfdcAccountId')
#
sfdc = Salesforce()
# code commented out since we are no longer getting only Mkto related opportunities into Cx
# account_list = '('
# for lead in leads_with_sfdc_opps:
# account_list += '\'' + lead['leads']['mkto']['sfdcAccountId'] + '\'' + ', '
# account_list = account_list[:-2]
# account_list += ')'
if sinceDateTime is None:
sinceDateTime = (datetime.now() - timedelta(days=30)).date()
oppList = sfdc.get_opportunities_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type)
print 'got opps ' + str(len(oppList['records']))
#create list of Opp IDs to send for get_contacts call
oppid_list = '('
for opp in oppList['records']:
oppid_list += '\'' + opp['Id'] + '\'' + ', '
oppid_list = oppid_list[:-2]
oppid_list += ')'
contactList = sfdc.get_contacts_for_opportunities(user_id, company_id, oppid_list) # needed because SFDC does not have the Contact ID within the Opp record
print 'got contacts for opps ' + str(len(contactList['records']))
saveSfdcOpportunities(user_id=user_id, company_id=company_id, oppList=oppList, contactList=contactList, job_id=job_id, run_type=run_type)
try:
message = 'Opportunities retrieved from Salesforce'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Opportunities'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return oppList
except Exception as e:
send_notification(dict(type='error', success=False, message=str(e)))
def createHsptCampaignFromTraffic(name, channel, company_id):
'''Create campaigns for Hubspot from website traffic because there is no direct API method to extract campaigns '''
try:
existingCampaign = Campaign.objects(Q(name=name) & Q(channel=channel) & Q(company_id=company_id) & Q(source_system='hspt')).first()
if existingCampaign is None:
newCampaign = Campaign(name=name, channel=channel, company_id=company_id, source_system='hspt')
newCampaign.save()
except Exception as e:
print 'error while saving Hspt campaign from traffic ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def retrieveFbokPageStats(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
print 'starting retrieveFbokPageStats for company ' + str(company_id)
#print 'timestamp ' + str(sinceDateTime)
if (datetime.now() - sinceDateTime).days > 89:
print 'FB does not allow for more than 89 days of data'
sinceDateTime = datetime.now() - timedelta(days=89)
sinceTimestamp = calendar.timegm(sinceDateTime.timetuple())
sinceTimestamp = str('{0:f}'.format(sinceTimestamp).rstrip('0').rstrip('.'))
untilTimestamp = time.mktime(datetime.now().timetuple())
untilTimestamp = str('{0:f}'.format(untilTimestamp).rstrip('0').rstrip('.'))
#print 'since ' + str(sinceTimestamp) + ' until ' + str(untilTimestamp)
existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
if 'fbok' not in existingIntegration['integrations']: # if Buffer is present and configured
print 'did not find fbok'
raise Exception('Facebook integration not found')
integration = existingIntegration.integrations['fbok']
if integration['access_token'] is None:
raise Exception('Facebook access token not found')
fbok = FacebookPage(integration['access_token'])
if fbok is None:
raise Exception('Facebook Page object could not be created')
definedPages = integration.get('pages', None)
print 'calling pages'
pages = fbok.get_pages()['data']
print 'found FB pages: ' + str(pages)
for page in pages:
page_token = page['access_token']
page_id = page['id']
if not any(d['id'] == page_id for d in definedPages): #if page not defined in company instance, skip it
continue
print 'page id is ' + str(page_id) + ' and token is ' + str(page_token)
page_insights = fbok.get_page_insights(page_id, page_token, sinceTimestamp, untilTimestamp)
#print 'page insights for ' + page['name'] + ': ' + str(page_insights)
page_insights_cleaned = json.loads(json.dumps(page_insights['data']), object_hook = replace_dots)
results = {'page_id': page_id, 'insights': page_insights_cleaned}
saveFbokPageStats(user_id=user_id, company_id=company_id, results=results, job_id=job_id, run_type=run_type)
#saveFbokAdStats(user_id=user_id, company_id=company_id, results=campaigns, job_id=job_id, run_type=run_type)
except Exception as e:
print 'exception was ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def post_save(cls, sender, document, **kwargs): #(self, *args, **kwargs): #
try:
send_notification(dict(
updated_date=json.dumps(document.updated_date, default=json_util.default),
message=document.message,
owner=json.dumps(document.owner, default=json_util.default),
id=json.dumps(document.id, default=json_util.default),
read = document.read,
success = document.success
))
except Exception as e:
send_notification(dict(
type='error',
message=str(e)
))
def saveHsptOpportunitiesToMaster(user_id=None, company_id=None, job_id=None, run_type=None):
#job_id = ObjectId("55e6b0198afb002ef6a8c292")
print 'saving hspt opps to master'
if run_type == 'initial':
opps = TempData.objects(Q(company_id=company_id) & Q(record_type='opportunity') & Q(source_system='hspt') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
else:
opps = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='opportunity') & Q(source_system='hspt') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id)
oppListTemp = list(opps)
oppList = [i['source_record'] for i in oppListTemp]
try:
for opp in oppList:
associations = opp.get('associations', None)
if associations is not None:
#print 'found assoc'
related_leads_list = associations.get('associatedVids', None)
#print 'releated leads list is ' + str(len(related_leads_list))
for i in range(len(related_leads_list)):
lead_id = related_leads_list[i]
#print 'lead id is ' + str(lead_id)
existingLead = Lead.objects(Q(company_id=company_id) & Q(hspt_id=str(lead_id))).first()
#we found an existing lead with the same VID (hspt_id) as the deal
if existingLead is not None:
#print 'found existing lead'
if 'hspt' not in existingLead.opportunities:
#print 'hspt not in opps'
opportunities = {}
opportunities['hspt'] = []
opportunities['hspt'].append(opp)
existingLead.update(opportunities__hspt = opportunities['hspt'])
else:
if not any (e.get('dealId', None) == opp['dealId'] for e in existingLead.opportunities['hspt']): # does an opportunity with this Id already exist
opportunities = existingLead.opportunities['hspt']
opportunities.append(opp)
# save this opportunity
existingLead.update(opportunities__hspt = opportunities)
else: #this opp already exists
for i in range(len(existingLead.opportunities['hspt'])):
if existingLead.opportunities['hspt'][i]['dealId'] == opp['dealId']:
existingLead.opportunities['hspt'][i] = opp
existingLead.save()
#if no matching lead found, continue to next opportunity
except Exception as e:
send_notification(dict(type='error', success=False, message=str(e)))
def retrieveBufrTwInteractions(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
print 'starting retrieveBufrTwInteractions for company ' + str(company_id)
existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
if 'bufr' in existingIntegration['integrations']: # if Buffer is present and configured
print 'found buffer'
client_id = existingIntegration['integrations']['bufr']['client_id']
client_secret = existingIntegration['integrations']['bufr']['client_secret']
access_token = existingIntegration['integrations']['bufr']['access_token']
buffer = Buffer()
api = Buffer.get_api(buffer, client_id=client_id, client_secret=client_secret, access_token=access_token)
profiles = Buffer.get_twitter_profiles(buffer, api)
for profile in profiles:
results = buffer.get_twitter_updates(profile)
saveBufrTwInteractions(user_id=user_id, company_id=company_id, results=results, job_id=job_id, run_type=run_type)
#print 'Tw results are ' + str(results)
else:
print 'No integration found with Buffer'
return JsonResponse({'error' : 'No integration found with Buffer'})
try:
message = 'Twitter interactions retrieved from Buffer'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Social'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def _doAssociationEmailWithCampaign(email_id, existingCampaign, company_id, job_id, run_type):
try:
#first get email templates stored in temp table
emailDetails = _get_hspt_email_details(email_id, existingCampaign, company_id, job_id, run_type)
#now populate details in the record and save back into campaign
emails = existingCampaign['emails']
new_email = {}
new_email['id'] = email_id
new_email['details'] = emailDetails
emails.append(new_email)
existingCampaign['emails'] = emails
existingCampaign.save()
except Exception as e:
print 'error while doing association of email with Hspt campaign from traffic ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def saveFbokPostStatsToMaster(user_id=None, company_id=None, job_id=None, run_type=None): #behaves differently because it directly saves the data to the AnalyticsData collection
if run_type == 'initial':
fb_stats = TempData.objects(Q(company_id=company_id) & Q(record_type='fb_post_stat') & Q(source_system='fbok') & Q(job_id=job_id) ).only('source_record')
else:
fb_stats = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='fb_post_stat') & Q(source_system='fbok') & Q(job_id=job_id) ).only('source_record')
fbList = list(fb_stats)
fbList = [i['source_record'] for i in fbList]
try:
for i in range(len(fbList)):
fb_record = {}
source_page_id = fbList[i].get('page_id', None)
source_post_id = fbList[i].get('post_id', None)
source_created_date = fbList[i].get('created_time', None)
insights = fbList[i].get('insights', None)
for insight in insights:
fb_record = insight
source_metric_id = fb_record['id']
source_metric_name = fb_record['name']
fbPostInsight = FbPostInsight.objects(Q(company_id=company_id) & Q(source_metric_id=source_metric_id)& Q(source_page_id=source_page_id) & Q(source_post_id=source_post_id)).first()
if fbPostInsight is None:
fbPostInsight = FbPostInsight(data=fb_record, company_id=company_id, source_created_date=source_created_date, source_metric_id=source_metric_id, source_metric_name=source_metric_name, source_page_id=source_page_id, source_post_id=source_post_id)
else:
fbPostInsight['source_metric_name'] = source_metric_name
fbPostInsight['source_created_date'] = source_created_date
fbValuesList = fbPostInsight['data']['values']
for entry in fb_record['values']: #iterate through each new date value of metric to see if it already exists
if fb_record['period'] == 'lifetime':
fbPostInsight['data']['values'][0]['value'] = entry['value']
else:
if not any(d['end_time'] == entry['end_time'] for d in fbValuesList): # does this already exist?
fbValuesList.extend(entry) #if not, add the date entry to the existing record
fbPostInsight.save()
except Exception as e:
print 'exception is ' + str(e)
send_notification(dict(type='error', success=False, message=str(e)))
def exportFileToPdf(request, type=None):
try:
#set the renderer class
#renderer_classes = (PDFRenderer, )
#get URL parameters
object = request.GET.get('object', None)
id = request.GET.get('id', None)
user_id = str(request.user.id) # need to convert to string in order to pass it to celery task else it complains.. so reconvert back to objectid in celery task
company_id = request.GET.get('company', None)
template_name = request.GET.get('template_name', None)
print 'template is ' + str(template_name)
source_type = request.GET.get('source_type', None)
if object is None or id is None or company_id is None or template_name is None:
return JsonResponse({'Error' : 'Input parameter was empty'})
#get cookie
cookies = request.COOKIES
token = cookies['csrftoken']
sessionid = cookies['sessionid']
authenticatedAccount = cookies['authenticatedAccount']
#set up variables
end_date_string = time.strftime('%Y-%m-%d_%H:%M:%S', time.localtime())
base_file_name = end_date_string + '_binder' + '_cx.pdf'
file_name = '/tmp/' + base_file_name
phantomjs_script = '/var/www/webapps/3m/mmm/staticfiles/javascripts/common/services/generate_pdf.js'
#phantomjs_script = '/home/satya/workspace/3m/mmm/static/javascripts/common/services/generate_pdf.js'
url = ''
if object == 'binder':
url = 'http://app.claritix.io/pdf/binder/' + str(id)
#url = 'http://localhost:8000/integrations'
#output = NamedTemporaryFile(delete=False)
#error = NamedTemporaryFile(delete=False)
content_type = 'application/pdf'
exportToPdf.delay(company_id, user_id, template_name, source_type, content_type, phantomjs_script, url, token, sessionid, authenticatedAccount, file_name) #, output, error
return JsonResponse({'Success' : 'File export started'})
except Exception as e:
print 'exception while trying to create PDF file: ' + str(e)
send_notification(dict(type='error', success=False, message=str(e)))
def retrieveFbokPostStats(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
print 'starting retrieveFbokPostStats for company ' + str(company_id)
existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
if 'fbok' not in existingIntegration['integrations']: # if Buffer is present and configured
print 'did not find fbok'
raise Exception('Facebook integration not found')
integration = existingIntegration.integrations['fbok']
if integration['access_token'] is None:
raise Exception('Facebook access token not found')
fbok = FacebookPage(integration['access_token'])
if fbok is None:
raise Exception('Facebook Page object could not be created')
print 'calling pages'
pages = fbok.get_pages()['data']
print 'found FB pages: ' + str(pages)
for page in pages:
page_token = page['access_token']
page_id = page['id']
posts = fbok.get_posts(page_id, page_token)['data']
print 'got back #posts ' + str(len(posts))
for post in posts:
post_insights = fbok.get_post_insights(post['id'], page_token)
#print 'post insights for ' + post['id'] + ': ' + str(post_insights)
post_insights_cleaned = json.loads(json.dumps(post_insights['data']), object_hook = replace_dots)
results = {'page_id': page_id, 'post_id' : post['id'], 'created_time': post['created_time'], 'insights': post_insights_cleaned}
saveFbokPostStats(user_id=user_id, company_id=company_id, results=results, job_id=job_id, run_type=run_type)
#saveFbokAdStats(user_id=user_id, company_id=company_id, results=campaigns, job_id=job_id, run_type=run_type)
except Exception as e:
print 'exception was ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def exportToPdf(company_id, user_id, template_name, source_type, content_type, phantomjs_script, url, token, sessionid, authenticatedAccount, file_name): #, output, error
try:
user_id = ObjectId(user_id)
output = NamedTemporaryFile(delete=False)
error = NamedTemporaryFile(delete=False)
external_process = Popen(["phantomjs", phantomjs_script, url, token, sessionid, authenticatedAccount, file_name], stdout=output, stderr=error) #
external_process.communicate(30)
export_file = open(file_name, 'rb')
exportFile = ExportFile(company_id=company_id, owner_id=user_id, source=template_name, source_type=source_type, type=content_type, file_name=os.path.basename(file_name))
exportFile.file.put(export_file, content_type=content_type)
exportFile.save()
try:
message = 'PDF file successfully exported'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Exports'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
user = CustomUser.objects(id=user_id).first()
if user is not None:
html_msg = '<p>Hola ' + user['first_name'] + '</p><p>Your export of data from ' + template_name + ' is ready. It is available in My Exports with the file name ' + os.path.basename(file_name) + '.</p><p>Cheers</p><p>The Claritix crew<p>'
send_mail('[Claritix] Your PDF export is baked and ready', '', '*****@*****.**', [user['email']], html_message=html_msg)
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
except Exception as e:
print 'exception was ' + str(e)
return str(e)
