def retrieveSfdcCampaigns(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): try: sdfc = Salesforce() campaignList = sdfc.get_campaigns_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type) saveSfdcCampaigns(user_id=user_id, company_id=company_id, campaignList=campaignList, job_id=job_id, run_type=run_type) try: message = 'Campaigns retrieved from Salesforce' notification = Notification() #notification.company_id = company_id notification.owner = user_id notification.module = 'Campaigns' notification.type = 'Background task' notification.method = os.path.basename(__file__) notification.message = message notification.success = True notification.read = False notification.save() except Exception as e: send_notification(dict( type='error', success=False, message=str(e) )) return campaignList except Exception as e: print 'error while retrieving sfdc campaigns: ' + str(e) send_notification(dict( type='error', success=False, message=str(e) ))
def retrieveFbokAdCampaignStats(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): try: print 'starting retrieveFbokAdCampaignStats for company ' + str(company_id) existingIntegration = CompanyIntegration.objects(company_id = company_id).first() if 'fbok' not in existingIntegration['integrations']: # if Buffer is present and configured print 'did not find fbok' raise Exception('Facebook integration not found') integration = existingIntegration.integrations['fbok'] if integration['access_token'] is None: raise Exception('Facebook access token not found') fbok = Facebook(integration['host'], integration['client_id'], integration['client_secret'], integration['redirect_uri']) if fbok is None: raise Exception('Facebook object could not be created') print 'calling campaigns' campaigns = fbok.get_campaign_stats(company_id, run_type) print 'found FB campaigns: ' + str(campaigns) saveFbokAdCampaignStats(user_id=user_id, company_id=company_id, results=campaigns, job_id=job_id, run_type=run_type) except Exception as e: print 'exception was ' + str(e) send_notification(dict( type='error', success=False, message=str(e) ))
def _get_hspt_email_details(email_id, existingCampaign, company_id, job_id, run_type): try: if run_type == 'initial': campaigns = TempData.objects(Q(company_id=company_id) & Q(record_type='campaign') & Q(source_system='hspt') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id) else: campaigns = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='campaign') & Q(source_system='hspt') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id) campaignListTemp = list(campaigns) campaignList = [i['source_record'] for i in campaignListTemp] #print 'temp recs found ' + str(len(campaignList)) #now get the appId for this email Id app_id = None int_email_id = 0 for campaign in campaignList: #print 'campaign in temp ' + str(campaign) if str(campaign['id']) == email_id: print 'campaign id is ' + str(campaign['id']) + ' and email id is ' + email_id app_id = campaign['appId'] int_email_id = campaign['id'] break #if appId found, get details emailDetails = None if app_id is not None: hspt = Hubspot(company_id) emailDetails = hspt.get_campaign_details(campaignId=int_email_id, appId=app_id) return emailDetails except Exception as e: print 'error while getting email details with Hspt campaign from traffic ' + str(e) send_notification(dict( type='error', success=False, message=str(e) ))
def saveFbokAdCampaignStatsToMaster(user_id=None, company_id=None, job_id=None, run_type=None): #behaves differently because it directly saves the data to the AnalyticsData collection if run_type == 'initial': fb_stats = TempData.objects(Q(company_id=company_id) & Q(record_type='fb_ad_campaign_stat') & Q(source_system='fbok') & Q(job_id=job_id) ).only('source_record') else: fb_stats = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='fb_ad_campaign_stat') & Q(source_system='fbok') & Q(job_id=job_id) ).only('source_record') fbList = list(fb_stats) fbList = [i['source_record'] for i in fbList] try: for i in range(len(fbList)): fb_record = {} source_campaign_id = fbList[i].get('id') source_campaign_name = fbList[i].get('name') source_account_id = fbList[i].get('account_id') insights = fbList[i]['insights'] for insight in insights: fb_record = insight['data'] source_created_date = fb_record['date_stop'] source_source = 'fbok' FbAdCampaignInsight.objects(Q(company_id=company_id) & Q(source_created_date=source_created_date)& Q(source_campaign_id=source_campaign_id)).delete() fbAdCampaignInsight = FbAdCampaignInsight(data=fb_record, company_id=company_id, source_campaign_id=source_campaign_id, source_campaign_name=source_campaign_name, source_account_id=source_account_id, source_created_date=source_created_date) fbAdCampaignInsight.save() except Exception as e: print 'exception is ' + str(e) send_notification(dict(type='error', success=False, message=str(e)))
def associateEmailWithCampaign(name, channel, company_id, email_id, job_id, run_type): '''Associate an email template with a Hubspot campaign - from website traffic because there is no direct API method to extract campaigns ''' try: existingCampaign = Campaign.objects(Q(name=name) & Q(channel=channel) & Q(company_id=company_id) & Q(source_system='hspt')).first() if existingCampaign is None: return #this should not happen else: emails = existingCampaign['emails'] if emails is None: #create a new email for the campaign _doAssociationEmailWithCampaign(email_id, existingCampaign, company_id, job_id, run_type) else: email_exists = False for email in emails: emailDetails = None if email['id'] == email_id: #if email exists, only update the details email_exists = True emailDetails = _get_hspt_email_details(email_id, existingCampaign, company_id, job_id, run_type) email['details'] = emailDetails existingCampaign.save() print 'existing email updated' if email_exists == False: #else, create a new email for the campaign _doAssociationEmailWithCampaign(email_id, existingCampaign, company_id, job_id, run_type) except Exception as e: print 'error while associating email with Hspt campaign from traffic ' + str(e) send_notification(dict( type='error', success=False, message=str(e) ))
def retrieveMktoCampaigns(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): try: mkto = Marketo(company_id) campaignList = mkto.get_programs() saveMktoCampaigns(user_id=user_id, company_id=company_id, campaignList=campaignList, job_id=job_id, run_type=run_type) try: message = 'Campaigns retrieved from Marketo' notification = Notification() #notification.company_id = company_id notification.owner = user_id notification.module = 'Campaigns' notification.type = 'Background task' notification.method = os.path.basename(__file__) notification.message = message notification.success = True notification.read = False notification.save() except Exception as e: send_notification(dict( type='error', success=False, message=str(e) )) return campaignList except Exception as e: print 'error while retrieving marketo campaigns: ' + str(e) send_notification(dict( type='error', success=False, message=str(e) ))
def saveHsptCampaignEmailEventRecords(company_id=None, run_type=None, job_id=None, guid=None): '''called from saveHsptCampaignsToMaster to save all events for a single Hspt campaign email record''' try: #print 'saving single event' if run_type == 'initial': events = TempData.objects(Q(company_id=company_id) & Q(record_type='campaign_email_event') & Q(source_system='hspt') & Q(job_id=job_id) & Q(source_record__campaign_guid=guid) ).only('source_record') #& Q(job_id=job_id) else: events = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='campaign_email_event') & Q(source_system='hspt') & Q(job_id=job_id) & Q(source_record__campaign_guid=guid) ).only('source_record') #& Q(job_id=job_id) eventsListTemp = list(events) eventsList = [j['source_record'] for j in eventsListTemp] #print 'found events ' + str(eventsList) for event in eventsList: #print 'about to check for event with ' + guid + ' email id ' + str(email['id']) + ' event id ' + str(event['event_id']) + ' recipient ' + str(event['recipient']) + ' created ' + str(event['created']) + ' details ' + str(event) existingEvent = EmailEvent.objects(Q(company_id=company_id) & Q(campaign_guid=guid) & Q(email_id=event['email_id']) & Q(event_id=event['event_id'])).first() if existingEvent is None: newEvent = EmailEvent(company_id=company_id, source_system='hspt', campaign_guid=guid, email_id=event['email_id'], event_id=event['event_id'], details=event['details'], recipient=event['recipient'], created=event['created'], event_type=event['event_type']) #print 'about to save new event with ' + guid + ' email id ' + str(email['id']) + ' event id ' + str(event['event_id']) + ' recipient ' + str(event['recipient']) + ' created ' + str(event['created']) + ' details ' + str(event) newEvent.save() except Exception as e: print 'error while saving single Hspt campaign event record ' + str(e) send_notification(dict( type='error', success=False, message=str(e) ))
def saveSfdcCampaignsToMaster(user_id=None, company_id=None, job_id=None, run_type=None): #job_id = ObjectId("56d25c6af6fd2a15df46cd60") if run_type == 'initial': campaigns = TempData.objects(Q(company_id=company_id) & Q(record_type='campaign') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id) else: campaigns = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='campaign') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id) campaignListTemp = list(campaigns) campaignList = [i['source_record'] for i in campaignListTemp] try: for newCampaign in campaignList: #['records']: #company_id = request.user.company_id guid = 'sfdc_' + str(newCampaign['Id']) Campaign.objects(Q(guid = guid) & Q(company_id=company_id)).modify(upsert=True, new=True, set__campaigns__sfdc = newCampaign, set__updated_date = datetime.utcnow, set_on_insert__source_system = 'sfdc', set_on_insert__guid = guid, set_on_insert__company_id = company_id) # for oldCampaign in campaignList['records']: # # mktoCampaigns = [] # # mktoCampaigns.append(campaign) # campaign = Campaign() # campaign.company_id = request.user.company_id # campaign.derived_id = 'sfdc_' + str(oldCampaign['Id']) # campaign.campaigns["sfdc"] = oldCampaign # campaign.save() except Exception as e: print 'error while saving sfdc campaign: ' + str(e) send_notification(dict( type='error', success=False, message=str(e) ))
def retrieveSfdcAccounts(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): try: if sinceDateTime is None: sinceDateTime = (datetime.now() - timedelta(days=1)).date() sfdc = Salesforce() accountList = sfdc.get_accounts_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type) print 'got back accounts ' + str(len(accountList['records'])) saveSfdcAccounts(user_id=user_id, company_id=company_id, accountList=accountList, job_id=job_id, run_type=run_type) try: message = 'Accounts retrieved from Salesforce' notification = Notification() #notification.company_id = company_id notification.owner = user_id notification.module = 'Accounts' notification.type = 'Background task' notification.method = os.path.basename(__file__) notification.message = message notification.success = True notification.read = False notification.save() except Exception as e: send_notification(dict( type='error', success=False, message=str(e) )) return accountList except Exception as e: send_notification(dict(type='error', success=False, message=str(e)))
def saveSfdcAccountsToMaster(user_id=None, company_id=None, job_id=None, run_type=None): #delete later #job_id = ObjectId("569fd4078afb002426ef2fd3") if run_type == 'initial': accounts = TempData.objects(Q(company_id=company_id) & Q(record_type='account') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id) else: accounts = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='account') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id) accountListTemp = list(accounts) accountList = [i['source_record'] for i in accountListTemp] try: for newAccount in accountList: sfdc_id = str(newAccount['Id']) #find all leads that have this account ID relatedLeadList = [] relatedLeadListTemp = None relatedLeads = Lead.objects(Q(company_id=company_id) & Q(sfdc_account_id=sfdc_id)).only('sfdc_contact_id') #if SFDC Account, then matching lead must have a SFDC Contact ID if relatedLeads is not None: relatedLeadListTemp = [lead.to_mongo().to_dict() for lead in relatedLeads] #print 'rll is ' + str(relatedLeadListTemp) for i in relatedLeadListTemp: if 'sfdc_contact_id' in i: relatedLeadList.append({'sfdc_contact_id': i['sfdc_contact_id']}) #print 'related leads are ' + str(relatedLeadList) # if relatedLeads is not None: # #leadListTemp = list(relatedLeads) # #relatedLeadList = [i.id for i in leadListTemp] # for lead in relatedLeads: # relatedLeadList.append(lead) print 'account id is ' + sfdc_id # sfdc_mkto_id = str(newLead['sfdcLeadId']) #check if there is a corresponding lead from MKTO existingAccount = None existingAccount = Account.objects(Q(company_id=company_id) & Q(sfdc_id=sfdc_id)).first() if existingAccount is not None: # we found this contact already in the DB print 'found existing account for id ' + str(sfdc_id) if 'sfdc' in existingAccount.accounts: existingAccount.source_name = newAccount['Name'] existingAccount.source_source = newAccount['AccountSource'] existingAccount.source_industry = newAccount['Industry'] existingAccount.source_created_date = newAccount['CreatedDate'] existingAccount.accounts["sfdc"] = newAccount if relatedLeadList is not None: existingAccount.leads = relatedLeadList existingAccount.save() #Lead.objects(Q(company_id=company_id) & Q(sfdc_contact_id=sfdc_contact_Id)).update(contacts__sfdc=newContact) else: existingAccount.accounts['sfdc'] = {} existingAccount.accounts['sfdc'] = newAccount if relatedLeadList is not None: existingAccount.leads = relatedLeadList existingAccount.save() elif existingAccount is None: # this account does not exist account = _saveSfdcNewAccount(sfdc_id, newAccount, relatedLeadList, company_id) except Exception as e: print 'exception while saving accounts ' + str(e) send_notification(dict(type='error', success=False, message=str(e)))
def saveMktoContacts(user_id=None, company_id=None, leadList=None, newList=None): try: pass except Exception as e: send_notification(dict(type='error', success=False, message=str(e)))
def retrieveSfdcContacts(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): try: sdfc = Salesforce() if sinceDateTime is None: sinceDateTime = (datetime.now() - timedelta(days=30)).date() contactList = sdfc.get_contacts_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type) print "got back contacts " + str(len(contactList["records"])) saveSfdcContacts( user_id=user_id, company_id=company_id, contactList=contactList, job_id=job_id, run_type=run_type ) try: message = "Contacts retrieved from Salesforce" notification = Notification() # notification.company_id = company_id notification.owner = user_id notification.module = "Contacts" notification.type = "Background task" notification.method = os.path.basename(__file__) notification.message = message notification.success = True notification.read = False notification.save() except Exception as e: send_notification(dict(type="error", success=False, message=str(e))) return contactList except Exception as e: send_notification(dict(type="error", success=False, message=str(e)))
def saveHsptCampaignsToMaster(user_id=None, company_id=None, job_id=None, run_type=None): if run_type == 'initial': campaigns = TempData.objects(Q(company_id=company_id) & Q(record_type='campaign') & Q(source_system='hspt') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id) else: campaigns = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='campaign') & Q(source_system='hspt') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id) campaignListTemp = list(campaigns) campaignList = [i['source_record'] for i in campaignListTemp] try: for newCampaign in campaignList: #company_id = request.user.company_id #derived_id = 'hspt_' + str(newCampaign['id']) guid = newCampaign['guid'] name = newCampaign['name'] source_system = 'hspt' channel = 'email' emails = newCampaign['emails'] #save all email events for this campaign first saveHsptCampaignEmailEventRecords(company_id=company_id, run_type=run_type, job_id=job_id, guid=guid) #now update or create campaign record existingCampaign = Campaign.objects(Q(guid = guid) & Q(company_id=company_id) & Q(source_system=source_system)).first() #.modify(upsert=True, new=True, set__emails = emails, set_on_insert__name = name, set_on_insert__guid = guid, set_on_insert__company_id = company_id, set_on_insert__updated_date=datetime.utcnow) if existingCampaign is None: #new campaign so add it newCampaign = Campaign(emails = emails, name = name, guid = guid, company_id = company_id, updated_date=datetime.utcnow, source_system=source_system) newCampaign.save() else: #campaign already exists so update relevant fields #print 'existing campaign is ' + str(existingCampaign) if 'emails' not in existingCampaign: #should never happen continue #print 'before emails ' + str(emails) for email in emails: #loop through each email found in API call for this campaign #save events separately #print 'entering emails' #now create/update the campaign record email_found = False for existingEmail in existingCampaign['emails']: #check if the API email already exists if email['id'] == existingEmail['id']: #we found this email already existing so update specific fields email_found = True existingEmail['stats'] = email['stats'] existingEmail['details'] = email['details'] if email_found: existingCampaign.save() break #move to the next API email since we already found a match if email_found == False: #email was not found so add it to existing emails existingCampaign['emails'].append(email) existingCampaign.save() except Exception as e: print 'exception ' + str(e) send_notification(dict( type='error', success=False, message=str(e) ))
def retrieveSfdcOppStageHistory(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): #needs to be changed - satya try: #delete later #job_id_new = job_id #job_id = ObjectId("56a690e98afb006883048e7e") #set variables sfdc = Salesforce() #for leads if run_type == 'initial': opps = TempData.objects(Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) & else: opps = TempDataDelta.objects(Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) & oppListTemp = list(opps) if not oppListTemp: print 'no opps found' return oppList = [i['source_record'] for i in oppListTemp] batch_size = 500 #10 Activity Types at a time activitiesList = [] for i in range(0, len(oppList), batch_size): opp_list = '(' for opp in oppList[i:i+batch_size]: opp_list += '\'' + opp['Id'] + '\'' + ', ' opp_list = opp_list[:-2] opp_list += ')' activitiesList.extend(sfdc.get_stage_history_for_opportunity(user_id, company_id, opp_list, _str_from_date(sinceDateTime))) print 'got back stage history for SFDC opportunities ' + str(len(activitiesList)) #delete later #job_id = job_id_new saveSfdcOppStageHistory(user_id=user_id, company_id=company_id, activityList=activitiesList, job_id=job_id, run_type=run_type) try: message = 'Opportunity stage history retrieved from Salesforce' notification = Notification() #notification.company_id = company_id notification.owner = user_id notification.module = 'Opportunities' notification.type = 'Background task' notification.method = os.path.basename(__file__) notification.message = message notification.success = True notification.read = False notification.save() except Exception as e: send_notification(dict( type='error', success=False, message=str(e) )) return opp_list except Exception as e: print 'exception while retrieving SFDC opportunity stage history: ' + str(e) send_notification(dict(type='error', success=False, message=str(e)))
def saveSfdcOppStageHistoryToMaster(user_id=None, company_id=None, job_id=None, run_type=None): #job_id = ObjectId("56b2b92c8afb0070a795c4b2") if run_type == 'initial': #activities = TempData.objects(Q(company_id=company_id) & Q(record_type='activity') & Q(source_system='mkto') & Q(job_id=job_id) ).only('source_record') collection = TempData._get_collection() activities = collection.find({'company_id': int(company_id), 'record_type': 'opp_stage_history', 'source_system': 'sfdc', 'job_id': job_id}, projection={'source_record': True}, batch_size=1000) else: collection = TempDataDelta._get_collection() activities = collection.find({'company_id': int(company_id), 'record_type': 'opp_stage_history', 'source_system': 'sfdc', 'job_id': job_id}, projection={'source_record': True}, batch_size=1000) try: print 'got history ' + str(activities.count()) for activity in activities: skipThis = False newActivity = activity['source_record'] addThisActivity = True print 'act is ' + str(newActivity) newOppId = newActivity["OpportunityId"] if newOppId is not None: print 'trying to get opp' existingAccount = Account.objects(Q(company_id = company_id) & Q(opportunities__sfdc__Id = newOppId)).first() print 'account is ' + str(existingAccount) else: continue if existingAccount is None: continue for existingOpp in existingAccount['opportunities']['sfdc']: #loop through each opp to find the right one if existingOpp['Id'] == newOppId: if 'stage_activities' not in existingOpp: #if no prior activities existingOpp['stage_activities'] = [] #create list existingOpp['stage_activities'].append(newActivity) #save activity print 'saved virgin stage activity for opp' else: for existingActivity in existingOpp['stage_activities']: #check if this activity already exists if existingActivity['Id'] == newActivity['Id']: #it exists, so skip the activity skipThis = True print 'skipping opp stage activity' break if skipThis: break # get out of this for loop else: existingOpp['stage_activities'].append(newActivity) print 'saved stage activity for opp' existingAccount.save() except Exception as e: print 'exception while saving SFDC Opp stage history to master' + str(e) send_notification(dict( type='error', success=False, message=str(e) ))
def saveSfdcContactHistoryToMaster(user_id=None, company_id=None, job_id=None, run_type=None): #job_id = ObjectId("56a6faa28afb00042171cd89") if run_type == 'initial': #activities = TempData.objects(Q(company_id=company_id) & Q(record_type='activity') & Q(source_system='mkto') & Q(job_id=job_id) ).only('source_record') collection = TempData._get_collection() activities = collection.find({'company_id': int(company_id), 'record_type': 'contact_history', 'source_system': 'sfdc', 'job_id': job_id}, projection={'source_record': True}, batch_size=1000) else: collection = TempDataDelta._get_collection() activities = collection.find({'company_id': int(company_id), 'record_type': 'contact_history', 'source_system': 'sfdc', 'job_id': job_id}, projection={'source_record': True}, batch_size=1000) try: print 'got history ' + str(activities.count()) for activity in activities: newActivity = activity['source_record'] addThisActivity = True print 'act is ' + str(newActivity) leadId = newActivity["ContactId"] if leadId is not None: print 'trying to get lead' existingLead = Lead.objects(Q(company_id = company_id) & Q(sfdc_contact_id = leadId)).first() print 'contact is ' + str(existingLead) else: continue if existingLead is None: continue if 'sfdc' in existingLead['activities']: # there are activities from SFDC for this leadId for existingActivity in existingLead['activities']['sfdc']: if existingActivity['Id'] == newActivity['Id']: # this activity already exists so exit the loop addThisActivity = False break if addThisActivity: existingLead['activities']['sfdc'].append(newActivity) existingLead.save() print 'saved new activity 1' else: print 'no sfdc acts' addThisActivity = True sfdc = [] print 'appending' sfdc.append(newActivity) print 'appended' existingLead['activities']['sfdc'] = sfdc existingLead.save() print 'saved new activity 2' except Exception as e: print 'exception while saving SFDC Contact history to master' + str(e) send_notification(dict( type='error', success=False, message=str(e) ))
def saveHsptContacts(user_id=None, company_id=None, leadList=None): try: for newLead in leadList: newLead = vars(newLead)['_field_values'] hspt_id = str(newLead['vid']) #print 'gs id is ' + str(hspt_id) #hspt_sfdc_id = str(newLead['sfdcLeadId']) # check if there is a corresponding lead from SFDC hspt_sfdc_id = None if 'salesforceleadid' in newLead['properties']: hspt_sfdc_id = str(newLead['properties']['salesforceleadid']) # temp fix by satya till SFDC ID field in Hubspot is discovered #addThisList = True existingLeadSfdc = None existingLead = None existingLead = Lead.objects(Q(company_id=company_id) & Q(hspt_id=hspt_id)).first() if existingLead is not None and 'hspt' in existingLead.leads: # we found this lead already in the DB Lead.objects(Q(company_id=company_id) & Q(hspt_id=hspt_id)).update(leads__hspt=newLead) # if 'hspt' in existingLead.lists: # currentLists = existingLead.lists['mkto'] # for i in range(len(currentLists)): # if currentLists[i]['id'] == newList['id']: # check if this activity already exists in the lead dict # addThisList = False # if addThisList == True: # currentLists.append(newList) # existingLead.update(lists__mkto=currentLists) # else: # currentLists = [] # currentLists.append(newList) # existingLead.update(lists__mkto=currentLists) elif existingLead is None: # this lead does not exist if hspt_sfdc_id is not None: # but has a SFDC lead id print 'found lead with SFDC ID ' + str(hspt_sfdc_id) existingLeadSfdc = Lead.objects(Q(company_id=company_id) & Q(leads__sfdc__Id=hspt_sfdc_id)).first() if existingLeadSfdc is not None: # we found a SFDC lead record which is matched to this new Mkto lead existingLeadSfdc.hspt_id = hspt_id existingLeadSfdc.leads['hspt'] = newLead existingLeadSfdc.save() # currentLists = [] # currentLists.append(newList) # existingLeadSfdc.update(lists__mkto=currentLists) if existingLeadSfdc is None and existingLead is None: # no matches found so save new record lead = Lead() lead.hspt_id = hspt_id lead.company_id = company_id lead.leads["hspt"] = newLead lead.save() # currentLists = [] # currentLists.append(newList) # lead.update(lists__mkto=currentLists) except Exception as e: send_notification(dict(type='error', success=False, message=str(e)))
def retrieveHsptOpportunities(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): try: print 'retrieving Hspt deals' hspt = Hubspot(company_id) oppList = [] # if run_type == 'initial': # oppList = hspt.get_deals(count=10000) # else: # oppList = hspt.get_deals(count=1000) # print 'got opps ' + str(len(oppList['results'])) # saveHsptOpportunities(user_id=user_id, company_id=company_id, oppList=oppList, job_id=job_id, run_type=run_type) #print 'job id is ' + str(job_id) if run_type == 'initial': print 'initial run for opps' try: for opp in hspt.get_deals(count=10000)['results']: oppList.append(opp) if len(oppList) == 100: #print 'going to save' saveHsptOpportunities(user_id=user_id, company_id=company_id, oppList=oppList, job_id=job_id, run_type=run_type) oppList = [] except Exception as e: print 'exception: ' + str(e) else: try: for opp in hspt.get_deals(count=1000)['results']: oppList.append(opp) if len(oppList) == 100: saveHsptOpportunities(user_id=user_id, company_id=company_id, oppList=oppList, job_id=job_id, run_type=run_type) oppList = [] except Exception as e: print 'exception: ' + str(e) try: message = 'Opportunities retrieved from Hubspot' notification = Notification() #notification.company_id = company_id notification.owner = user_id notification.module = 'Opportunities' notification.type = 'Background task' notification.method = os.path.basename(__file__) notification.message = message notification.success = True notification.read = False notification.save() except Exception as e: send_notification(dict( type='error', success=False, message=str(e) )) return oppList except Exception as e: send_notification(dict(type='error', success=False, message=str(e)))
def exportMktoLeadsToCsv(data, chart_name, user_id, company_id): ids = data.get('results', None) leads = Lead.objects().filter(company_id=company_id, mkto_id__in=ids).order_by('mkto_id').hint('co_mkto_id') leads = list(leads) leads = [lead.to_mongo().to_dict() for lead in leads] if leads is None or len(leads) == 0: print 'input is none' return {'file_name': '', 'content_type' : 'text/csv'} try: print 'input not none ' + str(leads) #open a temp file for writing end_date_string = time.strftime('%Y-%m-%d_%H:%M:%S', time.localtime()) base_file_name = end_date_string + '_' + chart_name + '_leads' + '_cx.csv' file_name = '/tmp/' + base_file_name csv_out = open(file_name, 'wb') fieldnames = ['Marketo ID', 'First Name', 'Last Name', 'Email', 'Company', 'Status', 'Source', 'Original Source Type', 'Created Date', 'SFDC Lead ID', 'SFDC Contact ID', 'SFDC Account ID', ] #create writer csv_writer = csv.DictWriter(csv_out, fieldnames=fieldnames, restval='', extrasaction='ignore') csv_writer.writeheader() for lead in leads: created_date = '' print 'lead is ' + str(lead['mkto_id']) mkto_id_url = "http://app-sj09.marketo.com/leadDatabase/loadLeadDetail?leadId=" + str(lead['mkto_id']) mkto_id = '=HYPERLINK("' + mkto_id_url + '", "' + str(lead['mkto_id']) + '")' #print '1' if 'createdAt' in lead['leads']['mkto'] and lead['leads']['mkto']['createdAt'] is not None: created_date = lead['leads']['mkto']['createdAt'] #print '2' if not 'leads' in lead or not 'mkto' in lead.get('leads', ''): lead['leads'] = {} lead['leads']['mkto'] = {} for key, value in lead['leads']['mkto'].items(): if lead['leads']['mkto'][key] is None: lead['leads']['mkto'][key] = '' csv_writer.writerow({'Marketo ID' : mkto_id, 'First Name': lead.get('source_first_name', '').encode('utf-8'), 'Last Name': lead.get('source_last_name', '').encode('utf-8'), 'Email': lead.get('source_email', '').encode('utf-8'), 'Company': lead['leads']['mkto'].get('company', '').encode('utf-8'), 'Status': lead['leads']['mkto'].get('leadStatus', '').encode('utf-8'), 'Source': lead['leads']['mkto'].get('leadSource', '').encode('utf-8'), 'Original Source Type': lead['leads']['mkto'].get('originalSourceType', '').encode('utf-8'), 'Created Date': created_date, 'SFDC Lead ID': lead['leads']['mkto'].get('sfdcLeadId', '').encode('utf-8'), 'SFDC Contact ID': lead['leads']['mkto'].get('sfdcContactId', '').encode('utf-8'), 'SFDC Account ID': lead['leads']['mkto'].get('sfdcAccountId', '').encode('utf-8'), }) #print '4' csv_out.close() return {'file_name': file_name, 'content_type' : 'text/csv'} except Exception as e: print 'exception while trying to create CSV file: ' + str(e) send_notification(dict(type='error', success=False, message=str(e)))
def retrieveSfdcOpportunities(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): try: #company_id = request.user.company_id existingIntegration = CompanyIntegration.objects(company_id = company_id).first() # code commented out since we are no longer getting only Mkto related opportunities into Cx # if existingIntegration is not None and 'mkto' in existingIntegration['integrations']: # if this system is connected to Marketo # company_qry = 'company_id' # type_field_qry = 'leads__mkto__exists' # sfdc_account_field_qry = 'leads__mkto__sfdcAccountId__ne' # querydict = {company_qry: company_id, type_field_qry: True, sfdc_account_field_qry: None} # leads_with_sfdc_opps = Lead.objects(**querydict).only('mkto_id').only('leads__mkto__sfdcAccountId') # sfdc = Salesforce() # code commented out since we are no longer getting only Mkto related opportunities into Cx # account_list = '(' # for lead in leads_with_sfdc_opps: # account_list += '\'' + lead['leads']['mkto']['sfdcAccountId'] + '\'' + ', ' # account_list = account_list[:-2] # account_list += ')' if sinceDateTime is None: sinceDateTime = (datetime.now() - timedelta(days=30)).date() oppList = sfdc.get_opportunities_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type) print 'got opps ' + str(len(oppList['records'])) #create list of Opp IDs to send for get_contacts call oppid_list = '(' for opp in oppList['records']: oppid_list += '\'' + opp['Id'] + '\'' + ', ' oppid_list = oppid_list[:-2] oppid_list += ')' contactList = sfdc.get_contacts_for_opportunities(user_id, company_id, oppid_list) # needed because SFDC does not have the Contact ID within the Opp record print 'got contacts for opps ' + str(len(contactList['records'])) saveSfdcOpportunities(user_id=user_id, company_id=company_id, oppList=oppList, contactList=contactList, job_id=job_id, run_type=run_type) try: message = 'Opportunities retrieved from Salesforce' notification = Notification() #notification.company_id = company_id notification.owner = user_id notification.module = 'Opportunities' notification.type = 'Background task' notification.method = os.path.basename(__file__) notification.message = message notification.success = True notification.read = False notification.save() except Exception as e: send_notification(dict( type='error', success=False, message=str(e) )) return oppList except Exception as e: send_notification(dict(type='error', success=False, message=str(e)))
def createHsptCampaignFromTraffic(name, channel, company_id): '''Create campaigns for Hubspot from website traffic because there is no direct API method to extract campaigns ''' try: existingCampaign = Campaign.objects(Q(name=name) & Q(channel=channel) & Q(company_id=company_id) & Q(source_system='hspt')).first() if existingCampaign is None: newCampaign = Campaign(name=name, channel=channel, company_id=company_id, source_system='hspt') newCampaign.save() except Exception as e: print 'error while saving Hspt campaign from traffic ' + str(e) send_notification(dict( type='error', success=False, message=str(e) ))
def retrieveFbokPageStats(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): try: print 'starting retrieveFbokPageStats for company ' + str(company_id) #print 'timestamp ' + str(sinceDateTime) if (datetime.now() - sinceDateTime).days > 89: print 'FB does not allow for more than 89 days of data' sinceDateTime = datetime.now() - timedelta(days=89) sinceTimestamp = calendar.timegm(sinceDateTime.timetuple()) sinceTimestamp = str('{0:f}'.format(sinceTimestamp).rstrip('0').rstrip('.')) untilTimestamp = time.mktime(datetime.now().timetuple()) untilTimestamp = str('{0:f}'.format(untilTimestamp).rstrip('0').rstrip('.')) #print 'since ' + str(sinceTimestamp) + ' until ' + str(untilTimestamp) existingIntegration = CompanyIntegration.objects(company_id = company_id).first() if 'fbok' not in existingIntegration['integrations']: # if Buffer is present and configured print 'did not find fbok' raise Exception('Facebook integration not found') integration = existingIntegration.integrations['fbok'] if integration['access_token'] is None: raise Exception('Facebook access token not found') fbok = FacebookPage(integration['access_token']) if fbok is None: raise Exception('Facebook Page object could not be created') definedPages = integration.get('pages', None) print 'calling pages' pages = fbok.get_pages()['data'] print 'found FB pages: ' + str(pages) for page in pages: page_token = page['access_token'] page_id = page['id'] if not any(d['id'] == page_id for d in definedPages): #if page not defined in company instance, skip it continue print 'page id is ' + str(page_id) + ' and token is ' + str(page_token) page_insights = fbok.get_page_insights(page_id, page_token, sinceTimestamp, untilTimestamp) #print 'page insights for ' + page['name'] + ': ' + str(page_insights) page_insights_cleaned = json.loads(json.dumps(page_insights['data']), object_hook = replace_dots) results = {'page_id': page_id, 'insights': page_insights_cleaned} saveFbokPageStats(user_id=user_id, company_id=company_id, results=results, job_id=job_id, run_type=run_type) #saveFbokAdStats(user_id=user_id, company_id=company_id, results=campaigns, job_id=job_id, run_type=run_type) except Exception as e: print 'exception was ' + str(e) send_notification(dict( type='error', success=False, message=str(e) ))
def post_save(cls, sender, document, **kwargs): #(self, *args, **kwargs): # try: send_notification(dict( updated_date=json.dumps(document.updated_date, default=json_util.default), message=document.message, owner=json.dumps(document.owner, default=json_util.default), id=json.dumps(document.id, default=json_util.default), read = document.read, success = document.success )) except Exception as e: send_notification(dict( type='error', message=str(e) ))
def saveHsptOpportunitiesToMaster(user_id=None, company_id=None, job_id=None, run_type=None): #job_id = ObjectId("55e6b0198afb002ef6a8c292") print 'saving hspt opps to master' if run_type == 'initial': opps = TempData.objects(Q(company_id=company_id) & Q(record_type='opportunity') & Q(source_system='hspt') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id) else: opps = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='opportunity') & Q(source_system='hspt') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id) oppListTemp = list(opps) oppList = [i['source_record'] for i in oppListTemp] try: for opp in oppList: associations = opp.get('associations', None) if associations is not None: #print 'found assoc' related_leads_list = associations.get('associatedVids', None) #print 'releated leads list is ' + str(len(related_leads_list)) for i in range(len(related_leads_list)): lead_id = related_leads_list[i] #print 'lead id is ' + str(lead_id) existingLead = Lead.objects(Q(company_id=company_id) & Q(hspt_id=str(lead_id))).first() #we found an existing lead with the same VID (hspt_id) as the deal if existingLead is not None: #print 'found existing lead' if 'hspt' not in existingLead.opportunities: #print 'hspt not in opps' opportunities = {} opportunities['hspt'] = [] opportunities['hspt'].append(opp) existingLead.update(opportunities__hspt = opportunities['hspt']) else: if not any (e.get('dealId', None) == opp['dealId'] for e in existingLead.opportunities['hspt']): # does an opportunity with this Id already exist opportunities = existingLead.opportunities['hspt'] opportunities.append(opp) # save this opportunity existingLead.update(opportunities__hspt = opportunities) else: #this opp already exists for i in range(len(existingLead.opportunities['hspt'])): if existingLead.opportunities['hspt'][i]['dealId'] == opp['dealId']: existingLead.opportunities['hspt'][i] = opp existingLead.save() #if no matching lead found, continue to next opportunity except Exception as e: send_notification(dict(type='error', success=False, message=str(e)))
def retrieveBufrTwInteractions(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): try: print 'starting retrieveBufrTwInteractions for company ' + str(company_id) existingIntegration = CompanyIntegration.objects(company_id = company_id).first() if 'bufr' in existingIntegration['integrations']: # if Buffer is present and configured print 'found buffer' client_id = existingIntegration['integrations']['bufr']['client_id'] client_secret = existingIntegration['integrations']['bufr']['client_secret'] access_token = existingIntegration['integrations']['bufr']['access_token'] buffer = Buffer() api = Buffer.get_api(buffer, client_id=client_id, client_secret=client_secret, access_token=access_token) profiles = Buffer.get_twitter_profiles(buffer, api) for profile in profiles: results = buffer.get_twitter_updates(profile) saveBufrTwInteractions(user_id=user_id, company_id=company_id, results=results, job_id=job_id, run_type=run_type) #print 'Tw results are ' + str(results) else: print 'No integration found with Buffer' return JsonResponse({'error' : 'No integration found with Buffer'}) try: message = 'Twitter interactions retrieved from Buffer' notification = Notification() #notification.company_id = company_id notification.owner = user_id notification.module = 'Social' notification.type = 'Background task' notification.method = os.path.basename(__file__) notification.message = message notification.success = True notification.read = False notification.save() except Exception as e: send_notification(dict( type='error', success=False, message=str(e) )) except Exception as e: send_notification(dict( type='error', success=False, message=str(e) ))
def _doAssociationEmailWithCampaign(email_id, existingCampaign, company_id, job_id, run_type): try: #first get email templates stored in temp table emailDetails = _get_hspt_email_details(email_id, existingCampaign, company_id, job_id, run_type) #now populate details in the record and save back into campaign emails = existingCampaign['emails'] new_email = {} new_email['id'] = email_id new_email['details'] = emailDetails emails.append(new_email) existingCampaign['emails'] = emails existingCampaign.save() except Exception as e: print 'error while doing association of email with Hspt campaign from traffic ' + str(e) send_notification(dict( type='error', success=False, message=str(e) ))
def saveFbokPostStatsToMaster(user_id=None, company_id=None, job_id=None, run_type=None): #behaves differently because it directly saves the data to the AnalyticsData collection if run_type == 'initial': fb_stats = TempData.objects(Q(company_id=company_id) & Q(record_type='fb_post_stat') & Q(source_system='fbok') & Q(job_id=job_id) ).only('source_record') else: fb_stats = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='fb_post_stat') & Q(source_system='fbok') & Q(job_id=job_id) ).only('source_record') fbList = list(fb_stats) fbList = [i['source_record'] for i in fbList] try: for i in range(len(fbList)): fb_record = {} source_page_id = fbList[i].get('page_id', None) source_post_id = fbList[i].get('post_id', None) source_created_date = fbList[i].get('created_time', None) insights = fbList[i].get('insights', None) for insight in insights: fb_record = insight source_metric_id = fb_record['id'] source_metric_name = fb_record['name'] fbPostInsight = FbPostInsight.objects(Q(company_id=company_id) & Q(source_metric_id=source_metric_id)& Q(source_page_id=source_page_id) & Q(source_post_id=source_post_id)).first() if fbPostInsight is None: fbPostInsight = FbPostInsight(data=fb_record, company_id=company_id, source_created_date=source_created_date, source_metric_id=source_metric_id, source_metric_name=source_metric_name, source_page_id=source_page_id, source_post_id=source_post_id) else: fbPostInsight['source_metric_name'] = source_metric_name fbPostInsight['source_created_date'] = source_created_date fbValuesList = fbPostInsight['data']['values'] for entry in fb_record['values']: #iterate through each new date value of metric to see if it already exists if fb_record['period'] == 'lifetime': fbPostInsight['data']['values'][0]['value'] = entry['value'] else: if not any(d['end_time'] == entry['end_time'] for d in fbValuesList): # does this already exist? fbValuesList.extend(entry) #if not, add the date entry to the existing record fbPostInsight.save() except Exception as e: print 'exception is ' + str(e) send_notification(dict(type='error', success=False, message=str(e)))
def exportFileToPdf(request, type=None): try: #set the renderer class #renderer_classes = (PDFRenderer, ) #get URL parameters object = request.GET.get('object', None) id = request.GET.get('id', None) user_id = str(request.user.id) # need to convert to string in order to pass it to celery task else it complains.. so reconvert back to objectid in celery task company_id = request.GET.get('company', None) template_name = request.GET.get('template_name', None) print 'template is ' + str(template_name) source_type = request.GET.get('source_type', None) if object is None or id is None or company_id is None or template_name is None: return JsonResponse({'Error' : 'Input parameter was empty'}) #get cookie cookies = request.COOKIES token = cookies['csrftoken'] sessionid = cookies['sessionid'] authenticatedAccount = cookies['authenticatedAccount'] #set up variables end_date_string = time.strftime('%Y-%m-%d_%H:%M:%S', time.localtime()) base_file_name = end_date_string + '_binder' + '_cx.pdf' file_name = '/tmp/' + base_file_name phantomjs_script = '/var/www/webapps/3m/mmm/staticfiles/javascripts/common/services/generate_pdf.js' #phantomjs_script = '/home/satya/workspace/3m/mmm/static/javascripts/common/services/generate_pdf.js' url = '' if object == 'binder': url = 'http://app.claritix.io/pdf/binder/' + str(id) #url = 'http://localhost:8000/integrations' #output = NamedTemporaryFile(delete=False) #error = NamedTemporaryFile(delete=False) content_type = 'application/pdf' exportToPdf.delay(company_id, user_id, template_name, source_type, content_type, phantomjs_script, url, token, sessionid, authenticatedAccount, file_name) #, output, error return JsonResponse({'Success' : 'File export started'}) except Exception as e: print 'exception while trying to create PDF file: ' + str(e) send_notification(dict(type='error', success=False, message=str(e)))
def retrieveFbokPostStats(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): try: print 'starting retrieveFbokPostStats for company ' + str(company_id) existingIntegration = CompanyIntegration.objects(company_id = company_id).first() if 'fbok' not in existingIntegration['integrations']: # if Buffer is present and configured print 'did not find fbok' raise Exception('Facebook integration not found') integration = existingIntegration.integrations['fbok'] if integration['access_token'] is None: raise Exception('Facebook access token not found') fbok = FacebookPage(integration['access_token']) if fbok is None: raise Exception('Facebook Page object could not be created') print 'calling pages' pages = fbok.get_pages()['data'] print 'found FB pages: ' + str(pages) for page in pages: page_token = page['access_token'] page_id = page['id'] posts = fbok.get_posts(page_id, page_token)['data'] print 'got back #posts ' + str(len(posts)) for post in posts: post_insights = fbok.get_post_insights(post['id'], page_token) #print 'post insights for ' + post['id'] + ': ' + str(post_insights) post_insights_cleaned = json.loads(json.dumps(post_insights['data']), object_hook = replace_dots) results = {'page_id': page_id, 'post_id' : post['id'], 'created_time': post['created_time'], 'insights': post_insights_cleaned} saveFbokPostStats(user_id=user_id, company_id=company_id, results=results, job_id=job_id, run_type=run_type) #saveFbokAdStats(user_id=user_id, company_id=company_id, results=campaigns, job_id=job_id, run_type=run_type) except Exception as e: print 'exception was ' + str(e) send_notification(dict( type='error', success=False, message=str(e) ))
def exportToPdf(company_id, user_id, template_name, source_type, content_type, phantomjs_script, url, token, sessionid, authenticatedAccount, file_name): #, output, error try: user_id = ObjectId(user_id) output = NamedTemporaryFile(delete=False) error = NamedTemporaryFile(delete=False) external_process = Popen(["phantomjs", phantomjs_script, url, token, sessionid, authenticatedAccount, file_name], stdout=output, stderr=error) # external_process.communicate(30) export_file = open(file_name, 'rb') exportFile = ExportFile(company_id=company_id, owner_id=user_id, source=template_name, source_type=source_type, type=content_type, file_name=os.path.basename(file_name)) exportFile.file.put(export_file, content_type=content_type) exportFile.save() try: message = 'PDF file successfully exported' notification = Notification() #notification.company_id = company_id notification.owner = user_id notification.module = 'Exports' notification.type = 'Background task' notification.method = os.path.basename(__file__) notification.message = message notification.success = True notification.read = False notification.save() user = CustomUser.objects(id=user_id).first() if user is not None: html_msg = '<p>Hola ' + user['first_name'] + '</p><p>Your export of data from ' + template_name + ' is ready. It is available in My Exports with the file name ' + os.path.basename(file_name) + '.</p><p>Cheers</p><p>The Claritix crew<p>' send_mail('[Claritix] Your PDF export is baked and ready', '', '*****@*****.**', [user['email']], html_message=html_msg) except Exception as e: send_notification(dict( type='error', success=False, message=str(e) )) except Exception as e: print 'exception was ' + str(e) return str(e)