Example #1
0
def retrieveSfdcCampaigns(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
    try:
        sdfc = Salesforce()
        campaignList = sdfc.get_campaigns_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type)
        saveSfdcCampaigns(user_id=user_id, company_id=company_id, campaignList=campaignList, job_id=job_id, run_type=run_type)
        try:
            message = 'Campaigns retrieved from Salesforce'
            notification = Notification()
            #notification.company_id = company_id
            notification.owner = user_id
            notification.module = 'Campaigns'
            notification.type = 'Background task' 
            notification.method = os.path.basename(__file__)
            notification.message = message
            notification.success = True
            notification.read = False
            notification.save()
        except Exception as e:
            send_notification(dict(
                 type='error',
                 success=False,
                 message=str(e)
                ))    
        return campaignList
    except Exception as e:
        print 'error while retrieving sfdc campaigns: ' + str(e)
        send_notification(dict(
             type='error',
             success=False,
             message=str(e)
            ))    
Example #2
0
def retrieveSfdcAccounts(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
    try:
        if sinceDateTime is None:
            sinceDateTime = (datetime.now() - timedelta(days=1)).date()
        sfdc = Salesforce()
        accountList = sfdc.get_accounts_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type)
        print 'got back accounts ' + str(len(accountList['records']))
        saveSfdcAccounts(user_id=user_id, company_id=company_id, accountList=accountList, job_id=job_id, run_type=run_type)
        try:
            message = 'Accounts retrieved from Salesforce'
            notification = Notification()
            #notification.company_id = company_id
            notification.owner = user_id
            notification.module = 'Accounts'
            notification.type = 'Background task' 
            notification.method = os.path.basename(__file__)
            notification.message = message
            notification.success = True
            notification.read = False
            notification.save()
        except Exception as e:
            send_notification(dict(
                 type='error',
                 success=False,
                 message=str(e)
                ))    
        return accountList
    except Exception as e:
        send_notification(dict(type='error', success=False, message=str(e)))  
Example #3
0
def retrieveSfdcContacts(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
    try:
        sdfc = Salesforce()

        if sinceDateTime is None:
            sinceDateTime = (datetime.now() - timedelta(days=30)).date()

        contactList = sdfc.get_contacts_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type)
        print "got back contacts " + str(len(contactList["records"]))
        saveSfdcContacts(
            user_id=user_id, company_id=company_id, contactList=contactList, job_id=job_id, run_type=run_type
        )
        try:
            message = "Contacts retrieved from Salesforce"
            notification = Notification()
            # notification.company_id = company_id
            notification.owner = user_id
            notification.module = "Contacts"
            notification.type = "Background task"
            notification.method = os.path.basename(__file__)
            notification.message = message
            notification.success = True
            notification.read = False
            notification.save()
        except Exception as e:
            send_notification(dict(type="error", success=False, message=str(e)))
        return contactList
    except Exception as e:
        send_notification(dict(type="error", success=False, message=str(e)))
Example #4
0
def retrieveSfdcOppStageHistory(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): #needs to be changed - satya
    try:
        #delete later
        #job_id_new = job_id
        #job_id = ObjectId("56a690e98afb006883048e7e")

        #set variables
        sfdc = Salesforce()
        #for leads
        if run_type == 'initial':
            opps = TempData.objects(Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) & 
        else:
            opps = TempDataDelta.objects(Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) & 
        
        oppListTemp = list(opps)
        if not oppListTemp:
            print 'no opps found'
            return
        oppList = [i['source_record'] for i in oppListTemp]
 
        batch_size = 500  #10 Activity Types at a time
        activitiesList = []
        
        for i in range(0, len(oppList), batch_size):
            opp_list = '('
            for opp in oppList[i:i+batch_size]:
                opp_list += '\'' + opp['Id'] + '\'' + ', '
            opp_list = opp_list[:-2]
            opp_list += ')'
            activitiesList.extend(sfdc.get_stage_history_for_opportunity(user_id, company_id, opp_list, _str_from_date(sinceDateTime)))
        
        print 'got back stage history for SFDC opportunities ' + str(len(activitiesList))
        #delete later
        #job_id = job_id_new
        saveSfdcOppStageHistory(user_id=user_id, company_id=company_id, activityList=activitiesList, job_id=job_id, run_type=run_type)
        
        try:
            message = 'Opportunity stage history retrieved from Salesforce'
            notification = Notification()
            #notification.company_id = company_id
            notification.owner = user_id
            notification.module = 'Opportunities'
            notification.type = 'Background task' 
            notification.method = os.path.basename(__file__)
            notification.message = message
            notification.success = True
            notification.read = False
            notification.save()
        except Exception as e:
            send_notification(dict(
                 type='error',
                 success=False,
                 message=str(e)
                ))    
        return opp_list
    except Exception as e:
        print 'exception while retrieving SFDC opportunity stage history: ' + str(e)
        send_notification(dict(type='error', success=False, message=str(e)))   
Example #5
0
def retrieveSfdcOpportunities(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):  
    try:
        #company_id = request.user.company_id
        existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
# code commented out since we are no longer getting only Mkto related opportunities into Cx
#         if existingIntegration is not None and 'mkto' in existingIntegration['integrations']: # if this system is connected to Marketo
#             company_qry = 'company_id'
#             type_field_qry = 'leads__mkto__exists'
#             sfdc_account_field_qry = 'leads__mkto__sfdcAccountId__ne'
#             querydict = {company_qry: company_id, type_field_qry: True, sfdc_account_field_qry: None}
#             leads_with_sfdc_opps = Lead.objects(**querydict).only('mkto_id').only('leads__mkto__sfdcAccountId')
#             
        sfdc = Salesforce()
# code commented out since we are no longer getting only Mkto related opportunities into Cx
#         account_list = '('
#         for lead in leads_with_sfdc_opps:
#             account_list += '\'' + lead['leads']['mkto']['sfdcAccountId'] + '\'' + ', '
#         account_list = account_list[:-2]
#         account_list += ')'
    
        if sinceDateTime is None:
            sinceDateTime = (datetime.now() - timedelta(days=30)).date()
        oppList = sfdc.get_opportunities_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type)
        print 'got opps ' + str(len(oppList['records']))
        #create list of Opp IDs to send for get_contacts call
        oppid_list = '('
        for opp in oppList['records']:
            oppid_list += '\'' + opp['Id'] + '\'' + ', '
        oppid_list = oppid_list[:-2]
        oppid_list += ')'
        contactList = sfdc.get_contacts_for_opportunities(user_id, company_id, oppid_list) # needed because SFDC does not have the Contact ID within the Opp record
        print 'got contacts for opps ' + str(len(contactList['records']))
        saveSfdcOpportunities(user_id=user_id, company_id=company_id, oppList=oppList, contactList=contactList, job_id=job_id, run_type=run_type)
        try:
            message = 'Opportunities retrieved from Salesforce'
            notification = Notification()
            #notification.company_id = company_id
            notification.owner = user_id
            notification.module = 'Opportunities'
            notification.type = 'Background task' 
            notification.method = os.path.basename(__file__)
            notification.message = message
            notification.success = True
            notification.read = False
            notification.save()
        except Exception as e:
            send_notification(dict(
                 type='error',
                 success=False,
                 message=str(e)
                ))    
        return oppList
    except Exception as e:
        send_notification(dict(type='error', success=False, message=str(e)))      
Example #6
0
def retrieveSfdcLeadHistory(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): #needs to be changed - satya
    try:
        #delete later
        #job_id_new = job_id
        #job_id = ObjectId("56a690e98afb006883048e7e")

        #set variables
        sfdc = Salesforce()
        
        #for leads
        if run_type == 'initial':
            leads = TempData.objects(Q(record_type='lead') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) & 
        else:
            leads = TempDataDelta.objects(Q(record_type='lead') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) & 
        
        leadListTemp = list(leads)
        if not leadListTemp:
            print 'no leads found'
            return
        leadList = [i['source_record'] for i in leadListTemp]
        
        
        
        batch_size = 500  #10 Activity Types at a time
        activitiesList = []
        
        for i in range(0, len(leadList), batch_size):
            lead_list = '('
            for lead in leadList[i:i+batch_size]:
                lead_list += '\'' + lead['Id'] + '\'' + ', '
            lead_list = lead_list[:-2]
            lead_list += ')'
            activitiesList.extend(sfdc.get_history_for_lead(user_id, company_id, lead_list, _str_from_date(sinceDateTime)))
        
        print 'got back history for SFDC leads ' + str(len(activitiesList))
        #delete later
        #job_id = job_id_new
        saveSfdcLeadHistory(user_id=user_id, company_id=company_id, activityList=activitiesList, job_id=job_id, run_type=run_type)
        
#         #for contacts
#         if run_type == 'initial':
#             contacts = TempData.objects(Q(record_type='contact') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) & 
#         else:
#             contacts = TempDataDelta.objects(Q(record_type='contact') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) & 
#             
#         contactListTemp = list(contacts)
#         contactList = [i['source_record'] for i in contactListTemp]
#         
#         sfdc = Salesforce()
#         
#         batch_size = 500  #10 Activity Types at a time
#         activitiesList = []
#         
#         for i in range(0, len(contactList), batch_size):
#             contact_list = '('
#             for contact in contactList[i:i+batch_size]:
#                 contact_list += '\'' + contact['Id'] + '\'' + ', '
#             contact_list = contact_list[:-2]
#             contact_list += ')'
#             activitiesList.extend(sfdc.get_history_for_contact(user_id, company_id, contact_list, _str_from_date(sinceDateTime)))
# 
#         print 'got back history for SFDC contacts ' + str(len(activitiesList))
#         saveSfdcHistory(user_id=user_id, company_id=company_id, activityList=activitiesList, job_id=job_id, run_type=run_type)
#         
        
        try:
            message = 'Lead history retrieved from Salesforce'
            notification = Notification()
            #notification.company_id = company_id
            notification.owner = user_id
            notification.module = 'Leads'
            notification.type = 'Background task' 
            notification.method = os.path.basename(__file__)
            notification.message = message
            notification.success = True
            notification.read = False
            notification.save()
        except Exception as e:
            send_notification(dict(
                 type='error',
                 success=False,
                 message=str(e)
                ))    
        return leadList
    except Exception as e:
        print 'exception while retrieving SFDC lead history: ' + str(e)
        send_notification(dict(type='error', success=False, message=str(e)))   
Example #7
0
def saveSfdcOpportunitiesToMaster(user_id=None, company_id=None, job_id=None, run_type=None):  
    #job_id = ObjectId("56a3f89f8afb003c13a59e26")
    
    if run_type == 'initial':
        opps = TempData.objects(Q(company_id=company_id) & Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id) 
        contacts = TempData.objects(Q(company_id=company_id) & Q(record_type='contact') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id) 
    else:
        opps = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id) 
        contacts = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='contact') & Q(source_system='sfdc') & Q(job_id=job_id) ).only('source_record') #& Q(job_id=job_id) 
  
    oppListTemp = list(opps)
    oppList = [i['source_record'] for i in oppListTemp]
    
    contactListTemp = list(contacts)
    contactList = [i['source_record'] for i in contactListTemp]
     
    try:
        allOpps = oppList #['records']
        # below code copied from contacts.tasks
        for newContact in contactList: #['records']:
            if 'OpportunityContactRoles' not in newContact or newContact['OpportunityContactRoles'] is None: # if this contact has no opportunities
                continue # move to next contact
            # company_id = request.user.company_id
            sfdc_account_id = None
            thisLeadsOppsIds = newContact['OpportunityContactRoles']['records']
            thisLeadsOpps = []
            for opp in thisLeadsOppsIds: #loop through all the Opp records in the Contact record
                print 'trying for opp with id ' + str(opp['OpportunityId'])
                thisOpp = next((x for x in allOpps if x['Id'] == opp['OpportunityId']), None) # if this opp is found in the list of opps retrieved separately
                if thisOpp is not None: # if found
                    print 'found this opp'
                    sfdc_account_id = thisOpp['AccountId']
                    thisLeadsOpps.append(thisOpp) #add it
            
            sfdc_contact_Id = str(newContact['Id']) 
            print 'contact id is ' + sfdc_contact_Id
            # sfdc_mkto_id = str(newLead['sfdcLeadId']) #check if there is a corresponding lead from MKTO
            existingLeadMkto = None
            existingLeadSfdc = None
            existingLeadHspt = None
            existingContact = Lead.objects(Q(company_id=company_id) & Q(sfdc_contact_id=sfdc_contact_Id)).first()
            
            if existingContact is not None:  # we found this contact already in the DB
                print ' eC is not none'        
                if 'sfdc' not in existingContact.opportunities:
                    opportunities = {}
                    opportunities['sfdc'] = []
                    opportunities['sfdc'].extend(thisLeadsOpps)  
                    existingContact.update(opportunities__sfdc = opportunities['sfdc'])
                    existingContact.update(sfdc_account_id = sfdc_account_id)
                    print 'just updated acct id 1'
                else:
                    for newOpp in thisLeadsOpps:
                        print ' nefre get' 
                        if not any (e.get('Id', None) == newOpp['Id'] for e in existingContact.opportunities['sfdc']): # does an opportunity with this Id already exist
                            opportunities = existingContact.opportunities['sfdc']
                            opportunities.append(newOpp)
                            existingContact.sfdc_account_id = sfdc_account_id
                            # save this opportunity        
                            existingContact.update(opportunities__sfdc = opportunities)
                            existingContact.update(sfdc_account_id = sfdc_account_id)
                            print 'just updated acct id 2'
                        else: #this opp already exists
                            for i in range(len(existingContact.opportunities['sfdc'])):
                                if existingContact.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
                                    existingContact.opportunities['sfdc'][i] = newOpp
                                    existingContact.sfdc_account_id = sfdc_account_id
                                    existingContact.save()
                                    print 'just updated acct id 3'
            elif existingContact is None:  # this lead does not exist 
                print ' eC is much none' 
                existingLeadSfdc = Lead.objects(Q(company_id=company_id) & Q(leads__sfdc__ConvertedContactId=sfdc_contact_Id)).first()
                if existingLeadSfdc is not None:
                    
                    if 'sfdc' not in existingLeadSfdc.opportunities:
                        opportunities = {}
                        opportunities['sfdc'] = []
                        opportunities['sfdc'].extend(thisLeadsOpps)  
                        existingLeadSfdc.update(opportunities__sfdc = opportunities['sfdc'])
                        existingLeadSfdc.update(sfdc_account_id = sfdc_account_id)
                        print 'just updated acct id 4'
                    else:
                        for newOpp in thisLeadsOpps:
                            if not any (e.get('Id', None) == newOpp['Id'] for e in existingLeadSfdc.opportunities['sfdc']): # does an opportunity with this Id already exist
                                opportunities = existingLeadSfdc.opportunities['sfdc']
                                opportunities.append(newOpp)
                                # save this opportunity        
                                existingLeadSfdc.update(opportunities__sfdc = opportunities)
                                existingLeadSfdc.update(sfdc_account_id = sfdc_account_id)
                                print 'just updated acct id 5'
                            else: #this opp already exists
                                for i in range(len(existingLeadSfdc.opportunities['sfdc'])):
                                    if existingLeadSfdc.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
                                        existingLeadSfdc.opportunities['sfdc'][i] = newOpp
                                        existingLeadSfdc.sfdc_account_id = sfdc_account_id
                                        existingLeadSfdc.save()
                                        print 'just updated acct id 6'
                else:
                    existingLeadMkto = Lead.objects(Q(company_id=company_id) & Q(leads__mkto__sfdcContactId=sfdc_contact_Id)).first()
                    if existingLeadMkto is not None:  # we found a MKto lead record which is matched to this new Sfdc lead
                        
                        if 'sfdc' not in existingLeadMkto.opportunities:
                            opportunities = {}
                            opportunities['sfdc'] = []
                            opportunities['sfdc'].extend(thisLeadsOpps)  
                            existingLeadMkto.update(opportunities__sfdc = opportunities['sfdc'])
                            existingLeadMkto.update(sfdc_account_id = sfdc_account_id)
                        else:
                            for newOpp in thisLeadsOpps:
                                if not any (e.get('Id', None) == newOpp['Id'] for e in existingLeadMkto.opportunities['sfdc']): # does an opportunity with this Id already exist
                                    opportunities = existingLeadMkto.opportunities['sfdc']
                                    opportunities.append(newOpp)
                                    # save this opportunity        
                                    existingLeadMkto.update(opportunities__sfdc = opportunities)
                                    existingLeadMkto.update(sfdc_account_id = sfdc_account_id)
                                else: #this opp already exists
                                    for i in range(len(existingLeadMkto.opportunities['sfdc'])):
                                        if existingLeadMkto.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
                                            existingLeadMkto.opportunities['sfdc'][i] = newOpp
                                            existingLeadMkto.sfdc_account_id = sfdc_account_id
                                            existingLeadMkto.save()
                        
                    existingLeadHspt = Lead.objects(Q(company_id=company_id) & Q(leads__hspt__properties__salesforcecontactid=sfdc_contact_Id)).first()
                    if existingLeadHspt is not None:  # we found a Hspt lead record which is matched to this new Sfdc lead
                        
                        if 'sfdc' not in existingLeadHspt.opportunities:
                            opportunities = {}
                            opportunities['sfdc'] = []
                            opportunities['sfdc'].extend(thisLeadsOpps)  
                            existingLeadHspt.update(opportunities__sfdc = opportunities['sfdc'])
                            existingLeadHspt.update(sfdc_account_id = sfdc_account_id)
                        else:
                            for newOpp in thisLeadsOpps:
                                if not any (e.get('Id', None) == newOpp['Id'] for e in existingLeadHspt.opportunities['sfdc']): # does an opportunity with this Id already exist
                                    opportunities = existingLeadHspt.opportunities['sfdc']
                                    opportunities.append(newOpp)
                                    # save this opportunity        
                                    existingLeadHspt.update(opportunities__sfdc = opportunities)
                                    existingLeadHspt.update(sfdc_account_id = sfdc_account_id)
                                else: #this opp already exists
                                    for i in range(len(existingLeadHspt.opportunities['sfdc'])):
                                        if existingLeadHspt.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
                                            existingLeadHspt.opportunities['sfdc'][i] = newOpp
                                            existingLeadHspt.sfdc_account_id = sfdc_account_id
                                            existingLeadHspt.save()
            
            if existingLeadSfdc is None and existingLeadMkto is None and existingLeadHspt is None and existingContact is None:  # no matches found so save new record
                lead = Lead()
                lead.sfdc_contact_id = sfdc_contact_Id
                lead.company_id = company_id
                lead.source_first_name = newContact['FirstName']
                lead.source_last_name = newContact['LastName']
                lead.source_email = newContact['Email']
                lead.source_created_date = str(newContact['CreatedDate'])
                lead.source_source = newContact['LeadSource']
                lead.sfdc_account_id = sfdc_account_id
                lead.save()
                contacts = {}
                contacts['sfdc'] = newContact
                #lead.contacts["sfdc"] = newContact
                lead.update(contacts__sfdc = contacts['sfdc'])
                print 'just updated acct id 7'
                opportunities = {}
                opportunities['sfdc'] = []
                opportunities['sfdc'].extend(thisLeadsOpps)  
                lead.update(opportunities__sfdc = opportunities['sfdc'])
        
        #new code added on 1/24/2016 - add each opportunity to related account (to capture opps with no contacts)
        for opp in allOpps:
            account = Account.objects(Q(company_id=company_id) & Q(sfdc_id=opp['AccountId'])).first()
            if account is None:
                print 'no account for opp with ID ' + str(opp['Id'])
                sfdc = Salesforce()
                accountList = sfdc.get_single_account(user_id, company_id, opp['AccountId'])
                for newAccount in accountList['records']:
                    account = _saveSfdcNewAccount(newAccount['Id'], newAccount, None, company_id)
            if 'opportunities' not in account:
                account.update(opportunities = {})
            if 'sfdc' not in account.opportunities: #no opps exist for this account, so add this opp
                opportunities = {}
                opportunities['sfdc'] = []
                opportunities['sfdc'].append(opp)  
                account.update(opportunities__sfdc = opportunities['sfdc'])
            else:
                if not any (e.get('Id', None) == opp['Id'] for e in account.opportunities['sfdc']): # there is no opportunity with this Id already exist
                    opportunities = account.opportunities['sfdc']
                    opportunities.append(opp)
                    # save this opportunity        
                    account.update(opportunities__sfdc = opportunities)
                else: # this opp already exists
                    for i in range(len(account.opportunities['sfdc'])):
                        if account.opportunities['sfdc'][i]['Id'] == opp['Id']:
                            account.opportunities['sfdc'][i] = opp
                            account.save()
                
            
        
# code commented out since we are no longer getting only Mkto related opportunities into Cx       
#         for newOpp in oppList['records']:
# 
#             # company_id = request.user.company_id
#             sfdc_account_id = str(newOpp['AccountId']) # find the account ID
#             print 'account id is ' + sfdc_account_id
#             # sfdc_mkto_id = str(newLead['sfdcLeadId']) #check if there is a corresponding lead from MKTO
#             existingLeadMkto = None
#             existingLeadSfdc = None
#             existingLeadHspt = None
#             #existingContact = Lead.objects(Q(company_id=company_id) & Q(sfdc_contact_id=sfdc_contact_Id)).first()
#             existingLeadSfdc = Lead.objects(Q(company_id=company_id) & Q(leads__sfdc__ConvertedAccountId=sfdc_account_id)).first()
#             if existingLeadSfdc is not None:
#                 if 'sfdc' not in existingLeadSfdc.opportunities:
#                     opportunities = {}
#                     opportunities['sfdc'] = []
#                     opportunities['sfdc'].append(newOpp) # add this opp to the new array
#                     existingLeadSfdc.update(opportunities__sfdc = opportunities)
#                 else:
#                     if not any (e.get('Id', None) == newOpp['Id'] for e in existingLeadSfdc.opportunities['sfdc']): # does an opportunity with this Id already exist
#                         opportunities = existingLeadSfdc.opportunities['sfdc']
#                         opportunities.append(newOpp)
#                         # save this opportunity        
#                         existingLeadSfdc.update(opportunities__sfdc = opportunities)
#                     else: #this opp already exists
#                         for i in range(len(existingLeadSfdc.opportunities['sfdc'])):
#                             if existingLeadSfdc.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
#                                 existingLeadSfdc.opportunities['sfdc'][i] = newOpp
#                                 existingLeadSfdc.save()
#                 
#                 # and move to the next opportunity in the loop
#                 continue
#             else: #this opp does not exist within an SFDC lead
#                 # check if it is a Marketo lead   
#                 existingLeadMkto = Lead.objects(Q(company_id=company_id) & Q(leads__mkto__sfdcAccountId=sfdc_account_id)).first()
#                 if existingLeadMkto is not None:  # we found a MKto lead record which is matched to this opp
#                     print 'found mkto lead' + existingLeadMkto.mkto_id
#                     if 'sfdc' not in existingLeadMkto.opportunities:
#                         opportunities = []
#                         opportunities.append(newOpp) # add this opp to the new array
#                         existingLeadMkto.update(opportunities__sfdc = opportunities)
#                         print 'saved opps'
#                     else: # if ['opportunities']['sfdc'] already exists
#                         print 'opp exist'
#                         if not any (e.get('Id', None) == newOpp['Id'] for e in existingLeadMkto.opportunities['sfdc']): # does an opportunity with this Id already exist
#                             opportunities = existingLeadMkto.opportunities['sfdc']
#                             opportunities.append(newOpp)
#                             # save this opportunity        
#                             existingLeadMkto.update(opportunities__sfdc = opportunities)
#                             print 'saved sfdc'
#                         else: #this opp already exists
#                             print 'opp does not exist'
#                             for i in range(len(existingLeadMkto.opportunities['sfdc'])):
#                                 if existingLeadMkto.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
#                                     existingLeadMkto.opportunities['sfdc'][i] = newOpp
#                                     existingLeadMkto.save()
#                                     print 'saved update'
#                     #move on to the next opp
#                     continue
#                 existingLeadHspt = Lead.objects(Q(company_id=company_id) & Q(leads__hspt__properties__salesforceaccountid=sfdc_account_id)).first()
#                 if existingLeadHspt is not None:  # we found a MKto lead record which is matched to this new Sfdc lead
#                     if 'sfdc' not in existingLeadHspt.opportunities:
#                         opportunities = {}
#                         opportunities['sfdc'] = []
#                         opportunities['sfdc'].append(newOpp) # add this opp to the new array
#                         existingLeadHspt.update(opportunities__sfdc = opportunities)
#                     else:
#                         if not any (e.get('Id', None) == newOpp['Id'] for e in existingLeadHspt.opportunities['sfdc']): # does an opportunity with this Id already exist
#                             opportunities = existingLeadHspt.opportunities['sfdc']
#                             opportunities.append(newOpp)
#                             # save this opportunity        
#                             existingLeadHspt.update(opportunities__sfdc = opportunities)
#                         else: #this opp already exists
#                             for i in range(len(existingLeadHspt.opportunities['sfdc'])):
#                                 if existingLeadHspt.opportunities['sfdc'][i]['Id'] == newOpp['Id']:
#                                     existingLeadHspt.opportunities['sfdc'][i] = newOpp
#                                     existingLeadHspt.save()
#                     # move onto the next opp
#                     continue
#                 
#             if existingLeadSfdc is None and existingLeadMkto is None and existingLeadHspt is None:  # no matches found so throw error i,e, not possible
#                 #raise ValueError('Opportunity found without lead or contact')
#                 lead = Lead()
#                 lead.sfdc_id = sfdc_Id
#                 lead.company_id = company_id
#                 lead.source_first_name = newLead['FirstName']
#                 lead.source_last_name = newLead['LastName']
#                 lead.source_email = newLead['Email']
#                 lead.source_created_date = str(newLead['CreatedDate'])
#                 lead.source_source = newLead['LeadSource']
#                 lead.source_status = newLead['Status']
#                 lead.leads["sfdc"] = newLead
#                 print '5th save'
#                 lead.save()
    except Exception as e:
        print 'Error while saving SFDC opportunities ' + str(e)
        send_notification(dict(type='error', success=False, message=str(e))) 
Example #8
0
def companyDataExtract(user_id=None, company_id=None, run_type=None, sinceDateTime=None):
    superJobMonitor = None
    existingIntegration = None
    
    if run_type is None or company_id is None or sinceDateTime is None:
        return
    try:
        sinceDateTime = datetime.fromtimestamp(float(sinceDateTime) / 1000)
        #sinceDateTime = datetime.now() - timedelta(days=30)
        print 'start date is ' + str(sinceDateTime)
        print 'run type is ' + run_type
        print 'company id is ' + str(company_id)

        #create an entry in the Job Monitor
        superJobMonitor = SuperJobMonitor(company_id=company_id, type=run_type, started_date=datetime.utcnow())
        superJobMonitor.save()
        
        #do pre-check
        _superJobMonitorAddTask(superJobMonitor, "Claritix", "Pre-check started") 
        # get the integration record
        existingIntegration = CompanyIntegration.objects(company_id=company_id).first() 
        if existingIntegration is None:
            _superJobMonitorEnd(superJobMonitor, None, 'Failed', 'No integration record found') 
            mail_admins('Could not find integration record for company ' + company_id , 'Check settings')
            return False 
        if run_type == 'initial':   
            existingIntegration['initial_run_in_process'] = True #set the flag
        else: #delta
            existingIntegration['delta_run_in_process'] = True #set the flag
        existingIntegration.save() # save the flag   
        
        # set up the Request and Cookie
        user = _get_superadmin()
        if user is None:
            _superJobMonitorEnd(superJobMonitor, existingIntegration, 'Failed', 'No super admin found') 
            mail_admins('Could not find super admin!', 'Check settings')
            return False
            
        # remotely login the user
        host = settings.BASE_URL
        url = host + '/api/v1/auth/login/'
        creds = {'email': '*****@*****.**', 'password':'******'}
        s = requests.Session()
        resp = s.post(url, data=json.dumps(creds))
        if not resp.status_code == 200:
            _superJobMonitorEnd(superJobMonitor, existingIntegration, 'Failed', 'Could not login super admin!') 
            mail_admins('Could not login super admin!', 'Check credentials')
            return False
            
        #do cookie thing - refer to SuperAdmin Cron Job Task for details
        cookies = dict(sessionid = resp.cookies['sessionid'])
        url = host + '/api/v1/users/'
        resp = s.get(url, cookies=cookies)
        _superJobMonitorAddTask(superJobMonitor, "Claritix", "Pre-check completed") 

#         #delete data in Lead Master
#         if run_type == 'initial':
#             _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of Leads, Contacts, Opportunities and Activities started") 
#             count = Lead.objects(company_id=company_id).count()
#             Lead.objects(company_id=company_id).delete()
#             _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " Leads, Contacts, Opportunities and Activities deleted")
#                       
#         #delete data in Campaign Master
#         if run_type == 'initial':
#             _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of Campaigns started") 
#             count = Campaign.objects(company_id=company_id).count()
#             Campaign.objects(company_id=company_id).delete()
#             _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " Campaigns deleted")
#                
#         #delete data in Account Master
#         if run_type == 'initial':
#             _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of Accounts started") 
#             count = Account.objects(company_id=company_id).count()
#             Account.objects(company_id=company_id).delete()
#             _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " Accounts deleted")
#                             
        # find out which systems are integrated and therefore which  tasks should be run
        task_map = {
                    #"mkto" : [retrieveMktoCampaigns, retrieveMktoLeadsByProgram], #retrieveMktoLeadCreatedActivities, retrieveMktoLeads, retrieveMktoActivities, retrieveMktoCampaigns, retrieveMktoLeadsByProgram], #IMPORTANT - Lead Created Activities has to be before Leads
                    #"hspt" : [retrieveHsptCampaigns], # , ], # retrieveHsptLeads, retrieveHsptOpportunities, retrieveHsptWebsiteTraffic, ,   
                     "prdt" : [retrievePrdtLeads],
#                    "sfdc" : [retrieveSfdcCampaigns] #retrieveSfdcLeads, retrieveSfdcContacts, retrieveSfdcCampaigns, retrieveSfdcAccounts, retrieveSfdcOpportunities, retrieveSfdcLeadHistory, retrieveSfdcContactHistory, retrieveSfdcOppHistory, retrieveSfdcOppStageHistory],
                    #  "sugr" : [retrieveSugrLeads],                    
#                    "bufr" : [retrieveBufrTwInteractions], 
#                    "goog" : [retrieveGoogWebsiteTraffic], \
#                    "fbok" : [retrieveFbokPageStats, retrieveFbokAdStats]#, retrieveFbokPostStats] # , ]
                    }
#         # for future use - retrieveMktoContacts, retrieveMktoOpportunities, retrieveSfdcActivities, 
        final_task_map = {
#                     "mkto" : [saveMktoCampaignsToMaster, saveMktoLeadsByProgramToMaster], #saveMktoLeadsToMaster, saveMktoActivitiesToMaster, saveMktoCampaignsToMaster, saveMktoLeadsByProgramToMaster],#mergeMktoSfdcLeads, deleteLeads, deleteDuplicateMktoIdLeads #
                    #"hspt" : [saveHsptCampaignsToMaster ], #saveHsptLeadsToMaster, saveHsptOpportunitiesToMaster, saveHsptWebsiteTrafficToMaster, , ], # ,
#                     "sfdc" : [saveSfdcCampaignsToMaster], #saveSfdcLeadsToMaster, saveSfdcContactsToMaster, saveSfdcCampaignsToMaster, saveSfdcAccountsToMaster, saveSfdcOpportunitiesToMaster, saveSfdcLeadHistoryToMaster, saveSfdcContactHistoryToMaster, saveSfdcOppHistoryToMaster, saveSfdcOppStageHistoryToMaster],  # 
                    # "sugr" : [saveSugrLeadsToMaster], 
#                     "bufr" : [saveBufrTwInteractionsToMaster], 
#                     "goog": [saveGoogleWebsiteTrafficToMaster], 
#                     "fbok": [saveFbokPageStatsToMaster, saveFbokAdStatsToMaster]#, saveFbokPostStatsToMaster] #
                    }
#         #
# #         #saveSfdcLeadsToMaster, saveSfdcContactsToMaster, saveSfdcOpportunitiesToMaster, saveSfdcCampaignsToMaster, 
# #         # saveSfdcLeadsToMaster, saveSfdcContactsToMaster, saveSfdcOpportunitiesToMaster, saveSfdcCampaignsToMaster, saveSfdcAccountsToMaster
# #         #collect all relevant tasks in one list and retrieve metadata as well
        for source in existingIntegration.integrations.keys():
            #change metadata depending on source system
            if source == 'sfdc':
                metadata_objects = ['user', 'lead', 'contact', 'campaign', 'opportunity', 'task', 'account'] #[] #objects for which metadata should be collected
            elif source == 'mkto':
                metadata_objects = ['lead', 'activity', 'campaign'] #[] #objects for which metadata should be collected
            elif source == 'hspt':
                metadata_objects = ['lead'] #objects for which metadata should be collected
            else:
                metadata_objects = [] #objects for which metadata should be collected
            # if sfdc, explicitly refresh the access token
            if source == 'sfdc':
                sfdc = Salesforce()
                sfdc.refresh_token(company_id)
            #collect meta data
            url = host + '/api/v1/company/' + str(company_id) + '/integrations/metadata/'
#             for object in metadata_objects:
#                 _superJobMonitorAddTask(superJobMonitor, source, "Retrieval of metadata for " + object + " started")
#                 params = {'code': source, 'object': object}
#                 resp = s.get(url, params=params)  # get metadata about activities
#                 if not resp.status_code == 200:
#                     _superJobMonitorAddTask(superJobMonitor, source, "Retrieval of metadata for " + object + " failed")
#                     continue
#                 else:
#                     _superJobMonitorAddTask(superJobMonitor, source, "Retrieval of metadata for " + object + " completed")
                         
            #collect retrieval tasks
            print 'starting retrieval tasks'
            tasks = []
            if source in task_map:
                tasks.extend(task_map[source])
            #now run the tasks
            for task in tasks:
                _superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " started to store temp data")
                print 'starting task ' + str(task)
                task(user_id=user_id, company_id=company_id, job_id=superJobMonitor.id, run_type=run_type, sinceDateTime=sinceDateTime)
                _superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " completed")
                  
            print 'starting save tasks'    
            #collect save tasks
            tasks = []
            if source in final_task_map:
                tasks.extend(final_task_map[source])
            #now run the tasks
            for task in tasks:
                _superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " started to save master data")
                print 'starting task ' + str(task)
                task(user_id=user_id, company_id=company_id, job_id=superJobMonitor.id, run_type=run_type)
                _superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " completed")
             
        #return #REMOVE THIS IN PRODUCTION  
        #if initial run, delete all analytics data
#         if run_type == 'initial':
#             _superJobMonitorAddTask(superJobMonitor, 'Claritix', "Deletion of analytics data started")
#             count1 = AnalyticsData.objects(company_id=company_id).count() #ensure that website_traffic chart data is not deleted
#             AnalyticsData.objects(company_id=company_id).delete() #ensure that website_traffic chart data is not deleted
#             count2 = AnalyticsIds.objects(company_id=company_id).count()
#             AnalyticsIds.objects(company_id=company_id).delete()
#             _superJobMonitorAddTask(superJobMonitor, 'Claritix', str(count1 + count2) + " records deleted from analytics tables")
#                
        # call chart calculate tasks
        charts = [\
#                     {'chart_name': 'sources_bar', 'system_type': 'MA', 'chart_title':'Timeline', 'mode': run_type, 'start_date': sinceDateTime}, \
#                     #{'chart_name': 'pipeline_duration', 'system_type': 'MA', 'chart_title':'Pipeline Duration', 'mode': run_type, 'start_date': sinceDateTime}, \
#                     #{'chart_name': 'contacts_distr', 'system_type': 'MA', 'chart_title':'Contacts Distribution', 'mode': run_type, 'start_date': sinceDateTime}, \
#                     {'chart_name': 'source_pie', 'system_type': 'MA', 'chart_title':'Source Distribution', 'mode': run_type, 'start_date': sinceDateTime}, \
#                     {'chart_name': 'revenue_source_pie', 'system_type': 'MA', 'chart_title':'Revenue by Source', 'mode': run_type, 'start_date': sinceDateTime}, \
#                     #{'chart_name': 'multichannel_leads', 'system_type': 'MA', 'chart_title':'Multichannel Leads', 'mode': run_type, 'start_date': sinceDateTime}, \
#                     {'chart_name': 'tw_performance', 'system_type': 'SO', 'chart_title':'Twitter Performance', 'mode': run_type, 'start_date': sinceDateTime}, \
#                     {'chart_name': 'fb_performance', 'system_type': 'SO', 'chart_title':'Facebook_Performance', 'mode': run_type, 'start_date': sinceDateTime}, \
#                     {'chart_name': 'google_analytics', 'system_type': 'AD', 'chart_title':'Google Analytics', 'mode': run_type, 'start_date': sinceDateTime}, \
#                      {'chart_name': 'campaign_email_performance', 'system_type': 'MA', 'chart_title':'Campaign Performance by Email', 'mode': run_type, 'start_date': sinceDateTime}, \
#                      {'chart_name': 'email_cta_performance', 'system_type': 'MA', 'chart_title':'Email Performance by CTA', 'mode': run_type, 'start_date': sinceDateTime}, \
#                 
                ]
        
        url = host + '/api/v1/company/' + str(company_id) + '/analytics/calculate/'
        for chart in charts:
            print 'starting chart ' + str(chart['chart_title'])
            _superJobMonitorAddTask(superJobMonitor, 'Claritix', "Started calculating " + str(chart['chart_title']))
            resp = s.get(url, params=chart)
            if not resp.status_code == 200:
                print 'incorrect status code was ' + str(resp.status_code)
                _superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Could not retrieve data for ' + chart['chart_title'])
                continue
            else:
                _superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Retrieved data for ' + chart['chart_title'])    
        
        # call dashboard calculate tasks
        dashboards = [\
#                     {'chart_name': 'social_roi', 'system_type': 'MA', 'chart_title':'Social Performance', 'mode': run_type, 'start_date': sinceDateTime}, \
#                     {'chart_name': 'funnel', 'system_type': 'MA', 'chart_title':'Funnel', 'mode': run_type, 'start_date': sinceDateTime}, \
#                     {'chart_name': 'opp_funnel', 'system_type': 'CRM', 'chart_title':'Opportunity Funnel', 'mode': run_type, 'start_date': sinceDateTime}, \
#                     {'chart_name': 'waterfall_chart', 'system_type': 'MA', 'chart_title':'Waterfall Chart', 'mode': run_type, 'start_date': sinceDateTime}, \
#                     {'chart_name': 'form_fills', 'system_type': 'MA', 'chart_title':'Form Fills', 'mode': run_type, 'start_date': sinceDateTime}, \
                    ]
        
        url = host + '/api/v1/company/' + str(company_id) + '/dashboards/calculate/'
        for dashboard in dashboards:
            print 'starting dashboard ' + str(dashboard['chart_title'])
            _superJobMonitorAddTask(superJobMonitor, 'Claritix', "Started calculating " + str(dashboard['chart_title']))
            resp = s.get(url, params=dashboard)
            if not resp.status_code == 200:
                print 'incorrect status code was ' + str(resp.status_code)
                _superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Could not retrieve data for ' + dashboard['chart_title'])
                continue
            else:
                _superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Retrieved data for ' + dashboard['chart_title'])    
        
        #delete all data from past successful initial runs in Temp Table
#         if run_type == 'initial':
#             _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of temp data table started") 
#             successfulJobs = SuperJobMonitor.objects(Q(company_id=company_id) & Q(type='initial') & Q(status='Completed'))
#             successfulJobsListTemp = list(successfulJobs)
#             print 'found job ids ' + str(len(successfulJobsListTemp))
#             successfulJobsList = [i.id for i in successfulJobsListTemp]
#             count = TempData.objects(job_id__in=successfulJobsList).count()
#             TempData.objects(job_id__in=successfulJobsList).delete()
#             _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " records deleted from temp data table")
        #delete all data from past successful delta runs in Temp Table
        if run_type == 'delta':
            _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of temp data delta table started") 
            successfulJobs = SuperJobMonitor.objects(Q(company_id=company_id) & Q(type='delta') & Q(status='Completed'))
            successfulJobsListTemp = list(successfulJobs)
            print 'found delta job ids ' + str(len(successfulJobsListTemp))
            successfulJobsList = [i.id for i in successfulJobsListTemp]
            count = TempDataDelta.objects(job_id__in=successfulJobsList).count()
            TempDataDelta.objects(job_id__in=successfulJobsList).delete()
            _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " records deleted from temp data delta table")
        #update last run date for initial run in Company Integration record
        
        _superJobMonitorEnd(superJobMonitor, existingIntegration, run_type, 'Completed', 'All tasks completed successfully') 
        return True

    except Exception as e:
        if superJobMonitor is not None and existingIntegration is not None:
            _superJobMonitorEnd(superJobMonitor, existingIntegration, run_type, 'Failed', str(e)) 
        print str(e)
        return False