def retrieveSfdcCampaigns(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
sdfc = Salesforce()
campaignList = sdfc.get_campaigns_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type)
saveSfdcCampaigns(user_id=user_id, company_id=company_id, campaignList=campaignList, job_id=job_id, run_type=run_type)
try:
message = 'Campaigns retrieved from Salesforce'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Campaigns'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return campaignList
except Exception as e:
print 'error while retrieving sfdc campaigns: ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def retrieveSfdcAccounts(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
if sinceDateTime is None:
sinceDateTime = (datetime.now() - timedelta(days=1)).date()
sfdc = Salesforce()
accountList = sfdc.get_accounts_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type)
print 'got back accounts ' + str(len(accountList['records']))
saveSfdcAccounts(user_id=user_id, company_id=company_id, accountList=accountList, job_id=job_id, run_type=run_type)
try:
message = 'Accounts retrieved from Salesforce'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Accounts'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return accountList
except Exception as e:
send_notification(dict(type='error', success=False, message=str(e)))
def retrieveSfdcContacts(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
sdfc = Salesforce()
if sinceDateTime is None:
sinceDateTime = (datetime.now() - timedelta(days=30)).date()
contactList = sdfc.get_contacts_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type)
print "got back contacts " + str(len(contactList["records"]))
saveSfdcContacts(
user_id=user_id, company_id=company_id, contactList=contactList, job_id=job_id, run_type=run_type
)
try:
message = "Contacts retrieved from Salesforce"
notification = Notification()
# notification.company_id = company_id
notification.owner = user_id
notification.module = "Contacts"
notification.type = "Background task"
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(type="error", success=False, message=str(e)))
return contactList
except Exception as e:
send_notification(dict(type="error", success=False, message=str(e)))
def filtertwInteractionsBufr(user_id, company_id, start_date, end_date, interaction_type, series_type, query_type, page_number, items_per_page, system_type, offset, code, chart_name):
#print 'start is ' + str(time.time())
if start_date is not None:
local_start_date_naive = datetime.fromtimestamp(float(start_date))
local_start_date = _str_from_date(local_start_date_naive, "short")
#local_start_date = get_current_timezone().localize(local_start_date_naive, is_dst=None)
#print 'start2 is ' + str(time.time())
if end_date is not None:
local_end_date_naive = datetime.fromtimestamp(float(end_date))
local_end_date = _str_from_date(local_end_date_naive, "short")
#print 'start3 is ' + str(time.time())
#local_end_date = get_current_timezone().localize(local_end_date_naive, is_dst=None)
#print 'filter start us ' + str(local_start_date) + ' and edn is ' + str(local_end_date)
#code = _get_code(company_id, system_type)
try:
interactions = []
company_field_qry = 'company_id'
chart_name_qry = 'chart_name'
system_type_qry = 'system_type'
date_qry = 'date'
querydict = {company_field_qry: company_id, system_type_qry: system_type, date_qry: local_start_date, chart_name_qry: chart_name}
print 'qd is ' + str(querydict)
analyticsIds = AnalyticsIds.objects(**querydict).only('results').first()
#print 'start3 is ' + str(time.time())
if analyticsIds is None:
return []
print 'interaction tupe is ' + interaction_type
ids = analyticsIds['results'].get(interaction_type, None)
print 'ids is ' + str(ids)
publishedTweets = PublishedTweet.objects(interaction_id__in=ids).skip(offset).limit(items_per_page).order_by('published_date')
#print 'start5 is ' + str(time.time())
#now do the calculations
total = PublishedTweet.objects(interaction_id__in=ids).count() #len(leads)
#print 'start6 is ' + str(time.time())
serializer = PublishedTweetSerializer(publishedTweets, many=True)
return JsonResponse({'count' : total, 'results': serializer.data})
except Exception as e:
return JsonResponse({'Error' : str(e)})
def retrieveSfdcOpportunities(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
#company_id = request.user.company_id
existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
# code commented out since we are no longer getting only Mkto related opportunities into Cx
# if existingIntegration is not None and 'mkto' in existingIntegration['integrations']: # if this system is connected to Marketo
# company_qry = 'company_id'
# type_field_qry = 'leads__mkto__exists'
# sfdc_account_field_qry = 'leads__mkto__sfdcAccountId__ne'
# querydict = {company_qry: company_id, type_field_qry: True, sfdc_account_field_qry: None}
# leads_with_sfdc_opps = Lead.objects(**querydict).only('mkto_id').only('leads__mkto__sfdcAccountId')
#
sfdc = Salesforce()
# code commented out since we are no longer getting only Mkto related opportunities into Cx
# account_list = '('
# for lead in leads_with_sfdc_opps:
# account_list += '\'' + lead['leads']['mkto']['sfdcAccountId'] + '\'' + ', '
# account_list = account_list[:-2]
# account_list += ')'
if sinceDateTime is None:
sinceDateTime = (datetime.now() - timedelta(days=30)).date()
oppList = sfdc.get_opportunities_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type)
print 'got opps ' + str(len(oppList['records']))
#create list of Opp IDs to send for get_contacts call
oppid_list = '('
for opp in oppList['records']:
oppid_list += '\'' + opp['Id'] + '\'' + ', '
oppid_list = oppid_list[:-2]
oppid_list += ')'
contactList = sfdc.get_contacts_for_opportunities(user_id, company_id, oppid_list) # needed because SFDC does not have the Contact ID within the Opp record
print 'got contacts for opps ' + str(len(contactList['records']))
saveSfdcOpportunities(user_id=user_id, company_id=company_id, oppList=oppList, contactList=contactList, job_id=job_id, run_type=run_type)
try:
message = 'Opportunities retrieved from Salesforce'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Opportunities'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return oppList
except Exception as e:
send_notification(dict(type='error', success=False, message=str(e)))
def getOpportunities(request, id):
try:
company_id = request.user.company_id
page_number = int(request.GET.get('page_number'))
items_per_page = int(request.GET.get('per_page'))
offset = (page_number - 1) * items_per_page
start_date = int(request.GET.get('start_date'))
end_date = int(request.GET.get('end_date'))
sub_view = request.GET.get('subview')
filters = request.GET.get('filters')
filters = json.loads(filters)
superfilters = request.GET.get('superfilters')
super_filters = json.loads(superfilters)
#print 'super filters are ' + str(super_filters)
date_field = None
querydict_filters = {}
#match_filters = {}
company_field_qry = 'company_id'
opp_field_qry = 'opportunities__sfdc__exists'
subview_field_qry = ''
original_date_field = ''
projection = {'$project': {'_id': '$opportunities.sfdc.Id', 'created_date': '$opportunities.sfdc.CreatedDate', 'close_date': '$opportunities.sfdc.CloseDate', 'account_name': '$source_name', 'name': '$opportunities.sfdc.Name', 'amount': '$opportunities.sfdc.Amount', 'account_id': '$sfdc_id', 'closed': '$opportunities.sfdc.IsClosed', 'won': '$opportunities.sfdc.IsWon', 'owner_id': '$opportunities.sfdc.OwnerId', 'stage': '$opportunities.sfdc.StageName' } }
match = {'$match' : { }}
if super_filters is not None:
#print 'sf ' + str(super_filters)
if 'date_types' in super_filters: # need to filter by a certain type of date
date_field = super_filters['date_types']
if start_date is not None:
start_date = datetime.fromtimestamp(float(start_date) / 1000)
#utc_day_start_epoch = datetime.fromtimestamp(float(start_date / 1000))
#utc_day_start_epoch = str('{0:f}'.format(utc_day_start_epoch).rstrip('0').rstrip('.'))
#print 'utc start epoch is ' + str(utc_day_start_epoch)
#local_start_date = get_current_timezone().localize(local_start_date_naive, is_dst=None)
#print 'start2 is ' + str(time.time())
if end_date is not None:
end_date = datetime.fromtimestamp(float(end_date) / 1000)
#utc_day_end_epoch = datetime.fromtimestamp(float(end_date / 1000))
#utc_day_end_epoch = str('{0:f}'.format(utc_day_end_epoch).rstrip('0').rstrip('.'))
#print 'utc end epoch is ' + str(utc_day_end_epoch)
#utc_day_start_string = datetime.strftime(utc_day_start_epoch, '%Y-%m-%dT%H-%M-%S.000+0000')
#utc_day_end_string = datetime.strftime(utc_day_end_epoch, '%Y-%m-%dT%H-%M-%S.000+0000')
local_start_date = get_current_timezone().localize(start_date, is_dst=None)
utc_day_start = local_start_date.astimezone(pytz.timezone('UTC'))
utc_day_start_string = datetime.strftime(utc_day_start, '%Y-%m-%dT%H-%M-%S.000+0000')
local_end_date = get_current_timezone().localize(end_date, is_dst=None)
utc_day_end = local_end_date.astimezone(pytz.timezone('UTC'))
utc_day_end_string = datetime.strftime(utc_day_end, '%Y-%m-%dT%H-%M-%S.000+0000')
print 'utc start string is ' + str(utc_day_start_string)
#print 'utc end string is ' + str(utc_day_end_string)
#remove the date_types item
#super_filters.pop('date_types')
if filters is not None:
for key, value in filters.items():
if value is not None and value != '':
querydict_filters['opportunities__sfdc__' + key] = value #creates an additional querydict that can be added to the main qd
match['$match']['opportunities.sfdc.' + key] = value
if date_field is None: #if there's no date filter
querydict = {opp_field_qry: True, company_field_qry: company_id}
querydict.update(querydict_filters)
opps = Account.objects(**querydict).aggregate({'$unwind': '$opportunities.sfdc'}, match, projection) #, {'$match': {'opportunities.sfdc.Id' : {'$ne': None}}} #
else: #if date filter is used
if date_field == 'opportunities.sfdc.CloseDate': #change to Last Modified Date because CloseDate in Opp may not be correctly updated by user
date_field = 'opportunities.sfdc.LastModifiedDate'
original_date_field = date_field
date_field = date_field.replace('.', '__') # needed to change embedded field format for querydict
date_field_start_qry = date_field + '__gte'
date_field_end_qry = date_field + '__lte'
match['$match'][original_date_field] = {'$gte': utc_day_start_string, '$lte': utc_day_end_string}
if original_date_field == 'opportunities.sfdc.LastModifiedDate': #if close add, add a filter for 'IsClosed'
isclosed_field_qry = 'opportunities__sfdc__IsClosed'
querydict = {company_field_qry: company_id, date_field_start_qry: utc_day_start_string, date_field_end_qry: utc_day_end_string, isclosed_field_qry: True}
querydict.update(querydict_filters)
match['$match']['opportunities.sfdc.IsClosed'] = True
opps = Account.objects(**querydict).aggregate({'$unwind': '$opportunities.sfdc'}, match, projection) #, {'$match': {'opportunities.sfdc.Id' : {'$ne': None}}} #
else:
querydict = {company_field_qry: company_id, date_field_start_qry: utc_day_start_string, date_field_end_qry: utc_day_end_string}
querydict.update(querydict_filters)
opps = Account.objects(**querydict).aggregate({'$unwind': '$opportunities.sfdc'}, match, projection) #, {'$match': {'opportunities.sfdc.Id' : {'$ne': None}}} #
#print 'qd is ' + str(querydict)
#print 'start time was ' + str(time.time())
#total = Account.objects(**querydict).count()
#print 'start time2 was ' + str(time.time())
opps_list = list(opps)
#see if there's a subview
if sub_view == 'closedbeforecreated': #find Opps that have a Close Date before Created Date
opps_list[:] = [opp for opp in opps_list if opp['close_date'] < _str_from_date(_date_from_str(opp['created_date']).replace(tzinfo=pytz.utc).astimezone(tz.tzlocal()), 'short')] #compare the short forms of both dates as strings after they Created Date is converted to local times
elif sub_view == 'nocontact': #find Opps that don't have a contact
opps2 = Lead.objects(**querydict).aggregate({'$unwind': '$opportunities.sfdc'}, match, projection)
opps_list2 = list(opps2)
#print 'opps 2 are ' + str(list(opps2))
#opps_all = _make_hashable(opps_list)
#opps_with_contacts = _make_hashable(list(opps2))
#opps_list = [dict(x) for x in set(opps_all).difference(opps_with_contacts)]
for opp2 in opps_list2:
opps_list[:] = [opp for opp in opps_list if opp['_id'] != opp2['_id']]
total = len(opps_list)
opps_list = opps_list[offset:offset + items_per_page]
#print 'start time3 was ' + str(time.time())
for opp in opps_list:
opp['multiple_occurences'] = False #needed due to analytical drilldown on Opps
opp['created_date'] = _str_from_date(_date_from_str(opp['created_date']).replace(tzinfo=pytz.utc).astimezone(tz.tzlocal()), 'short') #convert date to local timezone
#opp['owner_name'] = _map_sfdc_userid_name(company_id, opp['owner_id'])
#print 'start time4 was ' + str(time.time())
opps_list = _map_sfdc_userid_name(company_id, opps_list)
serializer = OpportunitySerializer(opps_list, many=True)
#print 'start time6 was ' + str(time.time())
type = 'opps'
return JsonResponse({'count' : total, 'results': serializer.data, 'type': type, 'source_system': 'sfdc'})
except Exception as e:
print 'exception while getting all accounts ' + str(e)
return JsonResponse({'Error' : str(e)})
def retrieveSfdcOppStageHistory(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): #needs to be changed - satya
try:
#delete later
#job_id_new = job_id
#job_id = ObjectId("56a690e98afb006883048e7e")
#set variables
sfdc = Salesforce()
#for leads
if run_type == 'initial':
opps = TempData.objects(Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
else:
opps = TempDataDelta.objects(Q(record_type='opportunity') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
oppListTemp = list(opps)
if not oppListTemp:
print 'no opps found'
return
oppList = [i['source_record'] for i in oppListTemp]
batch_size = 500 #10 Activity Types at a time
activitiesList = []
for i in range(0, len(oppList), batch_size):
opp_list = '('
for opp in oppList[i:i+batch_size]:
opp_list += '\'' + opp['Id'] + '\'' + ', '
opp_list = opp_list[:-2]
opp_list += ')'
activitiesList.extend(sfdc.get_stage_history_for_opportunity(user_id, company_id, opp_list, _str_from_date(sinceDateTime)))
print 'got back stage history for SFDC opportunities ' + str(len(activitiesList))
#delete later
#job_id = job_id_new
saveSfdcOppStageHistory(user_id=user_id, company_id=company_id, activityList=activitiesList, job_id=job_id, run_type=run_type)
try:
message = 'Opportunity stage history retrieved from Salesforce'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Opportunities'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return opp_list
except Exception as e:
print 'exception while retrieving SFDC opportunity stage history: ' + str(e)
send_notification(dict(type='error', success=False, message=str(e)))
def retrieveSfdcLeadHistory(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None): #needs to be changed - satya
try:
#delete later
#job_id_new = job_id
#job_id = ObjectId("56a690e98afb006883048e7e")
#set variables
sfdc = Salesforce()
#for leads
if run_type == 'initial':
leads = TempData.objects(Q(record_type='lead') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
else:
leads = TempDataDelta.objects(Q(record_type='lead') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
leadListTemp = list(leads)
if not leadListTemp:
print 'no leads found'
return
leadList = [i['source_record'] for i in leadListTemp]
batch_size = 500 #10 Activity Types at a time
activitiesList = []
for i in range(0, len(leadList), batch_size):
lead_list = '('
for lead in leadList[i:i+batch_size]:
lead_list += '\'' + lead['Id'] + '\'' + ', '
lead_list = lead_list[:-2]
lead_list += ')'
activitiesList.extend(sfdc.get_history_for_lead(user_id, company_id, lead_list, _str_from_date(sinceDateTime)))
print 'got back history for SFDC leads ' + str(len(activitiesList))
#delete later
#job_id = job_id_new
saveSfdcLeadHistory(user_id=user_id, company_id=company_id, activityList=activitiesList, job_id=job_id, run_type=run_type)
# #for contacts
# if run_type == 'initial':
# contacts = TempData.objects(Q(record_type='contact') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
# else:
# contacts = TempDataDelta.objects(Q(record_type='contact') & Q(source_system='sfdc') & Q(job_id=job_id)) #Q(job_id=job_id) &
#
# contactListTemp = list(contacts)
# contactList = [i['source_record'] for i in contactListTemp]
#
# sfdc = Salesforce()
#
# batch_size = 500 #10 Activity Types at a time
# activitiesList = []
#
# for i in range(0, len(contactList), batch_size):
# contact_list = '('
# for contact in contactList[i:i+batch_size]:
# contact_list += '\'' + contact['Id'] + '\'' + ', '
# contact_list = contact_list[:-2]
# contact_list += ')'
# activitiesList.extend(sfdc.get_history_for_contact(user_id, company_id, contact_list, _str_from_date(sinceDateTime)))
#
# print 'got back history for SFDC contacts ' + str(len(activitiesList))
# saveSfdcHistory(user_id=user_id, company_id=company_id, activityList=activitiesList, job_id=job_id, run_type=run_type)
#
try:
message = 'Lead history retrieved from Salesforce'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Leads'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return leadList
except Exception as e:
print 'exception while retrieving SFDC lead history: ' + str(e)
send_notification(dict(type='error', success=False, message=str(e)))
def dailyCronJob():
print 'in cron'
try:
logs = [] #holds all error messages for the job
# first get the superadmin user and the companies
user = _get_superadmin()
if user is None:
mail_admins('Could not find super admin!', 'Check settings')
return # no superadmin found
# remotely login the user
host = settings.BASE_URL
url = host + '/api/v1/auth/login/'
creds = {'email': '*****@*****.**', 'password':'******'}
s = requests.Session()
resp = s.post(url, data=json.dumps(creds))
print 'resp is ' + str(resp.status_code)
if not resp.status_code == 200:
mail_admins('Could not login super admin!', 'Check credentials')
logs.append('Could not login super admin!')
return
else:
logs.append('Superadmin logged in')
cookies = dict(sessionid = resp.cookies['sessionid'])
url = host + '/api/v1/users/'
resp = s.get(url, cookies=cookies)
print 'resp2 is ' + str(resp.status_code)
print str(logs)
querydict = {'company_id__ne' : 0}
companies = Company.objects(**querydict)
print 'found companies ' + str(len(companies))
#now loop through each company find which systems are connected
for company in companies:
company_id = company.company_id
company_name = company.name
print 'in company ' + company.name
existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
if existingIntegration is None: # no integration found so move to next company
logs.append('No integration record for company ' + str(company_name))
continue
else: #skip this company if initial or delta run are in progress
if 'initial_run_in_process' in existingIntegration and existingIntegration['initial_run_in_process'] == True:
logs.append('Initial run in process for company ' + str(company_name))
continue
if 'delta_run_in_process' in existingIntegration and existingIntegration['delta_run_in_process'] == True:
logs.append('Delta run in process for company ' + str(company_name))
continue
# look for either the last delta run date or the last initial run date
sinceDateTime = None
if 'delta_run_done' in existingIntegration:
if 'delta_run_last_date' in existingIntegration:
sinceDateTime = existingIntegration['delta_run_last_date']
if sinceDateTime is None:
if 'initial_run_done' in existingIntegration:
if 'initial_run_last_date' in existingIntegration:
sinceDateTime = existingIntegration['initial_run_last_date']
if sinceDateTime is None:
logs.append('No start date for delta run for company ' + str(company_name))
continue
sinceDateTime = int(time.mktime(time.strptime(_str_from_date(sinceDateTime), '%Y-%m-%dT%H:%M:%SZ')))
sinceDateTime -= int(12*60*60) #move back 12 hours as a safety measure
sinceDateTime = sinceDateTime * 1000
print 'calling extract with ' + str(sinceDateTime)
companyDataExtract(user_id=None, company_id=company_id, run_type='delta', sinceDateTime=sinceDateTime)
except Exception as e:
logs.append(str(e))
print 'exception is ' + str(e)
