def retrieveSfdcCampaigns(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): try: sdfc = Salesforce() campaignList = sdfc.get_campaigns_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type) saveSfdcCampaigns(user_id=user_id, company_id=company_id, campaignList=campaignList, job_id=job_id, run_type=run_type) try: message = 'Campaigns retrieved from Salesforce' notification = Notification() #notification.company_id = company_id notification.owner = user_id notification.module = 'Campaigns' notification.type = 'Background task' notification.method = os.path.basename(__file__) notification.message = message notification.success = True notification.read = False notification.save() except Exception as e: send_notification(dict( type='error', success=False, message=str(e) )) return campaignList except Exception as e: print 'error while retrieving sfdc campaigns: ' + str(e) send_notification(dict( type='error', success=False, message=str(e) ))
def retrieveSfdcAccounts(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): try: if sinceDateTime is None: sinceDateTime = (datetime.now() - timedelta(days=1)).date() sfdc = Salesforce() accountList = sfdc.get_accounts_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type) print 'got back accounts ' + str(len(accountList['records'])) saveSfdcAccounts(user_id=user_id, company_id=company_id, accountList=accountList, job_id=job_id, run_type=run_type) try: message = 'Accounts retrieved from Salesforce' notification = Notification() #notification.company_id = company_id notification.owner = user_id notification.module = 'Accounts' notification.type = 'Background task' notification.method = os.path.basename(__file__) notification.message = message notification.success = True notification.read = False notification.save() except Exception as e: send_notification(dict( type='error', success=False, message=str(e) )) return accountList except Exception as e: send_notification(dict(type='error', success=False, message=str(e)))
def retrieveSfdcContacts(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): try: sdfc = Salesforce() if sinceDateTime is None: sinceDateTime = (datetime.now() - timedelta(days=30)).date() contactList = sdfc.get_contacts_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type) print "got back contacts " + str(len(contactList["records"])) saveSfdcContacts( user_id=user_id, company_id=company_id, contactList=contactList, job_id=job_id, run_type=run_type ) try: message = "Contacts retrieved from Salesforce" notification = Notification() # notification.company_id = company_id notification.owner = user_id notification.module = "Contacts" notification.type = "Background task" notification.method = os.path.basename(__file__) notification.message = message notification.success = True notification.read = False notification.save() except Exception as e: send_notification(dict(type="error", success=False, message=str(e))) return contactList except Exception as e: send_notification(dict(type="error", success=False, message=str(e)))
def filtertwInteractionsBufr(user_id, company_id, start_date, end_date, interaction_type, series_type, query_type, page_number, items_per_page, system_type, offset, code, chart_name): #print 'start is ' + str(time.time()) if start_date is not None: local_start_date_naive = datetime.fromtimestamp(float(start_date)) local_start_date = _str_from_date(local_start_date_naive, "short") #local_start_date = get_current_timezone().localize(local_start_date_naive, is_dst=None) #print 'start2 is ' + str(time.time()) if end_date is not None: local_end_date_naive = datetime.fromtimestamp(float(end_date)) local_end_date = _str_from_date(local_end_date_naive, "short") #print 'start3 is ' + str(time.time()) #local_end_date = get_current_timezone().localize(local_end_date_naive, is_dst=None) #print 'filter start us ' + str(local_start_date) + ' and edn is ' + str(local_end_date) #code = _get_code(company_id, system_type) try: interactions = [] company_field_qry = 'company_id' chart_name_qry = 'chart_name' system_type_qry = 'system_type' date_qry = 'date' querydict = {company_field_qry: company_id, system_type_qry: system_type, date_qry: local_start_date, chart_name_qry: chart_name} print 'qd is ' + str(querydict) analyticsIds = AnalyticsIds.objects(**querydict).only('results').first() #print 'start3 is ' + str(time.time()) if analyticsIds is None: return [] print 'interaction tupe is ' + interaction_type ids = analyticsIds['results'].get(interaction_type, None) print 'ids is ' + str(ids) publishedTweets = PublishedTweet.objects(interaction_id__in=ids).skip(offset).limit(items_per_page).order_by('published_date') #print 'start5 is ' + str(time.time()) #now do the calculations total = PublishedTweet.objects(interaction_id__in=ids).count() #len(leads) #print 'start6 is ' + str(time.time()) serializer = PublishedTweetSerializer(publishedTweets, many=True) return JsonResponse({'count' : total, 'results': serializer.data}) except Exception as e: return JsonResponse({'Error' : str(e)})
def retrieveSfdcOpportunities(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): try: #company_id = request.user.company_id existingIntegration = CompanyIntegration.objects(company_id = company_id).first() # code commented out since we are no longer getting only Mkto related opportunities into Cx # if existingIntegration is not None and 'mkto' in existingIntegration['integrations']: # if this system is connected to Marketo # company_qry = 'company_id' # type_field_qry = 'leads__mkto__exists' # sfdc_account_field_qry = 'leads__mkto__sfdcAccountId__ne' # querydict = {company_qry: company_id, type_field_qry: True, sfdc_account_field_qry: None} # leads_with_sfdc_opps = Lead.objects(**querydict).only('mkto_id').only('leads__mkto__sfdcAccountId') # sfdc = Salesforce() # code commented out since we are no longer getting only Mkto related opportunities into Cx # account_list = '(' # for lead in leads_with_sfdc_opps: # account_list += '\'' + lead['leads']['mkto']['sfdcAccountId'] + '\'' + ', ' # account_list = account_list[:-2] # account_list += ')' if sinceDateTime is None: sinceDateTime = (datetime.now() - timedelta(days=30)).date() oppList = sfdc.get_opportunities_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type) print 'got opps ' + str(len(oppList['records'])) #create list of Opp IDs to send for get_contacts call oppid_list = '(' for opp in oppList['records']: oppid_list += '\'' + opp['Id'] + '\'' + ', ' oppid_list = oppid_list[:-2] oppid_list += ')' contactList = sfdc.get_contacts_for_opportunities(user_id, company_id, oppid_list) # needed because SFDC does not have the Contact ID within the Opp record print 'got contacts for opps ' + str(len(contactList['records'])) saveSfdcOpportunities(user_id=user_id, company_id=company_id, oppList=oppList, contactList=contactList, job_id=job_id, run_type=run_type) try: message = 'Opportunities retrieved from Salesforce' notification = Notification() #notification.company_id = company_id notification.owner = user_id notification.module = 'Opportunities' notification.type = 'Background task' notification.method = os.path.basename(__file__) notification.message = message notification.success = True notification.read = False notification.save() except Exception as e: send_notification(dict( type='error', success=False, message=str(e) )) return oppList except Exception as e: send_notification(dict(type='error', success=False, message=str(e)))
def getOpportunities(request, id): try: company_id = request.user.company_id page_number = int(request.GET.get('page_number')) items_per_page = int(request.GET.get('per_page')) offset = (page_number - 1) * items_per_page start_date = int(request.GET.get('start_date')) end_date = int(request.GET.get('end_date')) sub_view = request.GET.get('subview') filters = request.GET.get('filters') filters = json.loads(filters) superfilters = request.GET.get('superfilters') super_filters = json.loads(superfilters) #print 'super filters are ' + str(super_filters) date_field = None querydict_filters = {} #match_filters = {} company_field_qry = 'company_id' opp_field_qry = 'opportunities__sfdc__exists' subview_field_qry = '' original_date_field = '' projection = {'$project': {'_id': '$opportunities.sfdc.Id', 'created_date': '$opportunities.sfdc.CreatedDate', 'close_date': '$opportunities.sfdc.CloseDate', 'account_name': '$source_name', 'name': '$opportunities.sfdc.Name', 'amount': '$opportunities.sfdc.Amount', 'account_id': '$sfdc_id', 'closed': '$opportunities.sfdc.IsClosed', 'won': '$opportunities.sfdc.IsWon', 'owner_id': '$opportunities.sfdc.OwnerId', 'stage': '$opportunities.sfdc.StageName' } } match = {'$match' : { }} if super_filters is not None: #print 'sf ' + str(super_filters) if 'date_types' in super_filters: # need to filter by a certain type of date date_field = super_filters['date_types'] if start_date is not None: start_date = datetime.fromtimestamp(float(start_date) / 1000) #utc_day_start_epoch = datetime.fromtimestamp(float(start_date / 1000)) #utc_day_start_epoch = str('{0:f}'.format(utc_day_start_epoch).rstrip('0').rstrip('.')) #print 'utc start epoch is ' + str(utc_day_start_epoch) #local_start_date = get_current_timezone().localize(local_start_date_naive, is_dst=None) #print 'start2 is ' + str(time.time()) if end_date is not None: end_date = datetime.fromtimestamp(float(end_date) / 1000) #utc_day_end_epoch = datetime.fromtimestamp(float(end_date / 1000)) #utc_day_end_epoch = str('{0:f}'.format(utc_day_end_epoch).rstrip('0').rstrip('.')) #print 'utc end epoch is ' + str(utc_day_end_epoch) #utc_day_start_string = datetime.strftime(utc_day_start_epoch, '%Y-%m-%dT%H-%M-%S.000+0000') #utc_day_end_string = datetime.strftime(utc_day_end_epoch, '%Y-%m-%dT%H-%M-%S.000+0000') local_start_date = get_current_timezone().localize(start_date, is_dst=None) utc_day_start = local_start_date.astimezone(pytz.timezone('UTC')) utc_day_start_string = datetime.strftime(utc_day_start, '%Y-%m-%dT%H-%M-%S.000+0000') local_end_date = get_current_timezone().localize(end_date, is_dst=None) utc_day_end = local_end_date.astimezone(pytz.timezone('UTC')) utc_day_end_string = datetime.strftime(utc_day_end, '%Y-%m-%dT%H-%M-%S.000+0000') print 'utc start string is ' + str(utc_day_start_string) #print 'utc end string is ' + str(utc_day_end_string) #remove the date_types item #super_filters.pop('date_types') if filters is not None: for key, value in filters.items(): if value is not None and value != '': querydict_filters['opportunities__sfdc__' + key] = value #creates an additional querydict that can be added to the main qd match['$match']['opportunities.sfdc.' + key] = value if date_field is None: #if there's no date filter querydict = {opp_field_qry: True, company_field_qry: company_id} querydict.update(querydict_filters) opps = Account.objects(**querydict).aggregate({'$unwind': '$opportunities.sfdc'}, match, projection) #, {'$match': {'opportunities.sfdc.Id' : {'$ne': None}}} # else: #if date filter is used if date_field == 'opportunities.sfdc.CloseDate': #change to Last Modified Date because CloseDate in Opp may not be correctly updated by user date_field = 'opportunities.sfdc.LastModifiedDate' original_date_field = date_field date_field = date_field.replace('.', '__') # needed to change embedded field format for querydict date_field_start_qry = date_field + '__gte' date_field_end_qry = date_field + '__lte' match['$match'][original_date_field] = {'$gte': utc_day_start_string, '$lte': utc_day_end_string} if original_date_field == 'opportunities.sfdc.LastModifiedDate': #if close add, add a filter for 'IsClosed' isclosed_field_qry = 'opportunities__sfdc__IsClosed' querydict = {company_field_qry: company_id, date_field_start_qry: utc_day_start_string, date_field_end_qry: utc_day_end_string, isclosed_field_qry: True} querydict.update(querydict_filters) match['$match']['opportunities.sfdc.IsClosed'] = True opps = Account.objects(**querydict).aggregate({'$unwind': '$opportunities.sfdc'}, match, projection) #, {'$match': {'opportunities.sfdc.Id' : {'$ne': None}}} # else: querydict = {company_field_qry: company_id, date_field_start_qry: utc_day_start_string, date_field_end_qry: utc_day_end_string} querydict.update(querydict_filters) opps = Account.objects(**querydict).aggregate({'$unwind': '$opportunities.sfdc'}, match, projection) #, {'$match': {'opportunities.sfdc.Id' : {'$ne': None}}} # #print 'qd is ' + str(querydict) #print 'start time was ' + str(time.time()) #total = Account.objects(**querydict).count() #print 'start time2 was ' + str(time.time()) opps_list = list(opps) #see if there's a subview if sub_view == 'closedbeforecreated': #find Opps that have a Close Date before Created Date opps_list[:] = [opp for opp in opps_list if opp['close_date'] < _str_from_date(_date_from_str(opp['created_date']).replace(tzinfo=pytz.utc).astimezone(tz.tzlocal()), 'short')] #compare the short forms of both dates as strings after they Created Date is converted to local times elif sub_view == 'nocontact': #find Opps that don't have a contact opps2 = Lead.objects(**querydict).aggregate({'$unwind': '$opportunities.sfdc'}, match, projection) opps_list2 = list(opps2) #print 'opps 2 are ' + str(list(opps2)) #opps_all = _make_hashable(opps_list) #opps_with_contacts = _make_hashable(list(opps2)) #opps_list = [dict(x) for x in set(opps_all).difference(opps_with_contacts)] for opp2 in opps_list2: opps_list[:] = [opp for opp in opps_list if opp['_id'] != opp2['_id']] total = len(opps_list) opps_list = opps_list[offset:offset + items_per_page] #print 'start time3 was ' + str(time.time()) for opp in opps_list: opp['multiple_occurences'] = False #needed due to analytical drilldown on Opps opp['created_date'] = _str_from_date(_date_from_str(opp['created_date']).replace(tzinfo=pytz.utc).astimezone(tz.tzlocal()), 'short') #convert date to local timezone #opp['owner_name'] = _map_sfdc_userid_name(company_id, opp['owner_id']) #print 'start time4 was ' + str(time.time()) opps_list = _map_sfdc_userid_name(company_id, opps_list) serializer = OpportunitySerializer(opps_list, many=True) #print 'start time6 was ' + str(time.time()) type = 'opps' return JsonResponse({'count' : total, 'results': serializer.data, 'type': type, 'source_system': 'sfdc'}) except Exception as e: print 'exception while getting all accounts ' + str(e) return JsonResponse({'Error' : str(e)})
def retrieveSfdcOppStageHistory(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): #needs to be changed - satya try: #delete later #job_id_new = job_id #job_id = ObjectId("56a690e98afb006883048e7e") #set variables sfdc = Salesforce() #for leads if run_type == 'initial': opps = TempData.objects(Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) & else: opps = TempDataDelta.objects(Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) & oppListTemp = list(opps) if not oppListTemp: print 'no opps found' return oppList = [i['source_record'] for i in oppListTemp] batch_size = 500 #10 Activity Types at a time activitiesList = [] for i in range(0, len(oppList), batch_size): opp_list = '(' for opp in oppList[i:i+batch_size]: opp_list += '\'' + opp['Id'] + '\'' + ', ' opp_list = opp_list[:-2] opp_list += ')' activitiesList.extend(sfdc.get_stage_history_for_opportunity(user_id, company_id, opp_list, _str_from_date(sinceDateTime))) print 'got back stage history for SFDC opportunities ' + str(len(activitiesList)) #delete later #job_id = job_id_new saveSfdcOppStageHistory(user_id=user_id, company_id=company_id, activityList=activitiesList, job_id=job_id, run_type=run_type) try: message = 'Opportunity stage history retrieved from Salesforce' notification = Notification() #notification.company_id = company_id notification.owner = user_id notification.module = 'Opportunities' notification.type = 'Background task' notification.method = os.path.basename(__file__) notification.message = message notification.success = True notification.read = False notification.save() except Exception as e: send_notification(dict( type='error', success=False, message=str(e) )) return opp_list except Exception as e: print 'exception while retrieving SFDC opportunity stage history: ' + str(e) send_notification(dict(type='error', success=False, message=str(e)))
def retrieveSfdcLeadHistory(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): #needs to be changed - satya try: #delete later #job_id_new = job_id #job_id = ObjectId("56a690e98afb006883048e7e") #set variables sfdc = Salesforce() #for leads if run_type == 'initial': leads = TempData.objects(Q(record_type='lead') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) & else: leads = TempDataDelta.objects(Q(record_type='lead') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) & leadListTemp = list(leads) if not leadListTemp: print 'no leads found' return leadList = [i['source_record'] for i in leadListTemp] batch_size = 500 #10 Activity Types at a time activitiesList = [] for i in range(0, len(leadList), batch_size): lead_list = '(' for lead in leadList[i:i+batch_size]: lead_list += '\'' + lead['Id'] + '\'' + ', ' lead_list = lead_list[:-2] lead_list += ')' activitiesList.extend(sfdc.get_history_for_lead(user_id, company_id, lead_list, _str_from_date(sinceDateTime))) print 'got back history for SFDC leads ' + str(len(activitiesList)) #delete later #job_id = job_id_new saveSfdcLeadHistory(user_id=user_id, company_id=company_id, activityList=activitiesList, job_id=job_id, run_type=run_type) # #for contacts # if run_type == 'initial': # contacts = TempData.objects(Q(record_type='contact') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) & # else: # contacts = TempDataDelta.objects(Q(record_type='contact') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) & # # contactListTemp = list(contacts) # contactList = [i['source_record'] for i in contactListTemp] # # sfdc = Salesforce() # # batch_size = 500 #10 Activity Types at a time # activitiesList = [] # # for i in range(0, len(contactList), batch_size): # contact_list = '(' # for contact in contactList[i:i+batch_size]: # contact_list += '\'' + contact['Id'] + '\'' + ', ' # contact_list = contact_list[:-2] # contact_list += ')' # activitiesList.extend(sfdc.get_history_for_contact(user_id, company_id, contact_list, _str_from_date(sinceDateTime))) # # print 'got back history for SFDC contacts ' + str(len(activitiesList)) # saveSfdcHistory(user_id=user_id, company_id=company_id, activityList=activitiesList, job_id=job_id, run_type=run_type) # try: message = 'Lead history retrieved from Salesforce' notification = Notification() #notification.company_id = company_id notification.owner = user_id notification.module = 'Leads' notification.type = 'Background task' notification.method = os.path.basename(__file__) notification.message = message notification.success = True notification.read = False notification.save() except Exception as e: send_notification(dict( type='error', success=False, message=str(e) )) return leadList except Exception as e: print 'exception while retrieving SFDC lead history: ' + str(e) send_notification(dict(type='error', success=False, message=str(e)))
def dailyCronJob(): print 'in cron' try: logs = [] #holds all error messages for the job # first get the superadmin user and the companies user = _get_superadmin() if user is None: mail_admins('Could not find super admin!', 'Check settings') return # no superadmin found # remotely login the user host = settings.BASE_URL url = host + '/api/v1/auth/login/' creds = {'email': '*****@*****.**', 'password':'******'} s = requests.Session() resp = s.post(url, data=json.dumps(creds)) print 'resp is ' + str(resp.status_code) if not resp.status_code == 200: mail_admins('Could not login super admin!', 'Check credentials') logs.append('Could not login super admin!') return else: logs.append('Superadmin logged in') cookies = dict(sessionid = resp.cookies['sessionid']) url = host + '/api/v1/users/' resp = s.get(url, cookies=cookies) print 'resp2 is ' + str(resp.status_code) print str(logs) querydict = {'company_id__ne' : 0} companies = Company.objects(**querydict) print 'found companies ' + str(len(companies)) #now loop through each company find which systems are connected for company in companies: company_id = company.company_id company_name = company.name print 'in company ' + company.name existingIntegration = CompanyIntegration.objects(company_id = company_id).first() if existingIntegration is None: # no integration found so move to next company logs.append('No integration record for company ' + str(company_name)) continue else: #skip this company if initial or delta run are in progress if 'initial_run_in_process' in existingIntegration and existingIntegration['initial_run_in_process'] == True: logs.append('Initial run in process for company ' + str(company_name)) continue if 'delta_run_in_process' in existingIntegration and existingIntegration['delta_run_in_process'] == True: logs.append('Delta run in process for company ' + str(company_name)) continue # look for either the last delta run date or the last initial run date sinceDateTime = None if 'delta_run_done' in existingIntegration: if 'delta_run_last_date' in existingIntegration: sinceDateTime = existingIntegration['delta_run_last_date'] if sinceDateTime is None: if 'initial_run_done' in existingIntegration: if 'initial_run_last_date' in existingIntegration: sinceDateTime = existingIntegration['initial_run_last_date'] if sinceDateTime is None: logs.append('No start date for delta run for company ' + str(company_name)) continue sinceDateTime = int(time.mktime(time.strptime(_str_from_date(sinceDateTime), '%Y-%m-%dT%H:%M:%SZ'))) sinceDateTime -= int(12*60*60) #move back 12 hours as a safety measure sinceDateTime = sinceDateTime * 1000 print 'calling extract with ' + str(sinceDateTime) companyDataExtract(user_id=None, company_id=company_id, run_type='delta', sinceDateTime=sinceDateTime) except Exception as e: logs.append(str(e)) print 'exception is ' + str(e)