def list(self, request, company_id=None): #retrieve all data extract jobs across all companies
if company_id != '0':
return HttpResponse('Company is incorrect', status=status.HTTP_400_BAD_REQUEST)
try:
page_number = int(request.GET.get('page_number'))
items_per_page = int(request.GET.get('per_page'))
offset = (page_number - 1) * items_per_page
jobs = SuperJobMonitor.objects().order_by('-started_date')
totalCount = SuperJobMonitor.objects().count()
totalCountSuccess = SuperJobMonitor.objects(status='Completed').count()
totalCountFailure = SuperJobMonitor.objects(status='Failed').count()
qlist = list(jobs)
result = qlist[offset:offset+items_per_page]
serializedList = SuperJobMonitorSerializer(result, many=True)
initialCount = SuperJobMonitor.objects(type='initial').count()
initialCountSuccess = SuperJobMonitor.objects(Q(type='initial') & Q(status='Completed')).count()
initialCountFailure = SuperJobMonitor.objects(Q(type='initial') & Q(status='Failed')).count()
deltaCount = SuperJobMonitor.objects(type='delta').count()
deltaCountSuccess = SuperJobMonitor.objects(Q(type='delta') & Q(status='Completed')).count()
deltaCountFailure = SuperJobMonitor.objects(Q(type='delta') & Q(status='Failed')).count()
return JsonResponse({'totalCount': totalCount, 'totalCountSuccess': totalCountSuccess, \
'totalCountFailure': totalCountFailure, 'initialCount': initialCount, \
'initialCountSuccess': initialCountSuccess, 'initialCountFailure': initialCountFailure, \
'deltaCount': deltaCount, 'deltaCountSuccess': deltaCountSuccess, \
'deltaCountFailure': deltaCountFailure, 'results': serializedList.data})
except Exception as e:
return HttpResponse(str(e), status=status.HTTP_400_BAD_REQUEST)
def companyDataExtract(user_id=None, company_id=None, run_type=None, sinceDateTime=None):
superJobMonitor = None
existingIntegration = None
if run_type is None or company_id is None or sinceDateTime is None:
return
try:
sinceDateTime = datetime.fromtimestamp(float(sinceDateTime) / 1000)
#sinceDateTime = datetime.now() - timedelta(days=30)
print 'start date is ' + str(sinceDateTime)
print 'run type is ' + run_type
print 'company id is ' + str(company_id)
#create an entry in the Job Monitor
superJobMonitor = SuperJobMonitor(company_id=company_id, type=run_type, started_date=datetime.utcnow())
superJobMonitor.save()
#do pre-check
_superJobMonitorAddTask(superJobMonitor, "Claritix", "Pre-check started")
# get the integration record
existingIntegration = CompanyIntegration.objects(company_id=company_id).first()
if existingIntegration is None:
_superJobMonitorEnd(superJobMonitor, None, 'Failed', 'No integration record found')
mail_admins('Could not find integration record for company ' + company_id , 'Check settings')
return False
if run_type == 'initial':
existingIntegration['initial_run_in_process'] = True #set the flag
else: #delta
existingIntegration['delta_run_in_process'] = True #set the flag
existingIntegration.save() # save the flag
# set up the Request and Cookie
user = _get_superadmin()
if user is None:
_superJobMonitorEnd(superJobMonitor, existingIntegration, 'Failed', 'No super admin found')
mail_admins('Could not find super admin!', 'Check settings')
return False
# remotely login the user
host = settings.BASE_URL
url = host + '/api/v1/auth/login/'
creds = {'email': '*****@*****.**', 'password':'******'}
s = requests.Session()
resp = s.post(url, data=json.dumps(creds))
if not resp.status_code == 200:
_superJobMonitorEnd(superJobMonitor, existingIntegration, 'Failed', 'Could not login super admin!')
mail_admins('Could not login super admin!', 'Check credentials')
return False
#do cookie thing - refer to SuperAdmin Cron Job Task for details
cookies = dict(sessionid = resp.cookies['sessionid'])
url = host + '/api/v1/users/'
resp = s.get(url, cookies=cookies)
_superJobMonitorAddTask(superJobMonitor, "Claritix", "Pre-check completed")
# #delete data in Lead Master
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of Leads, Contacts, Opportunities and Activities started")
# count = Lead.objects(company_id=company_id).count()
# Lead.objects(company_id=company_id).delete()
# _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " Leads, Contacts, Opportunities and Activities deleted")
#
# #delete data in Campaign Master
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of Campaigns started")
# count = Campaign.objects(company_id=company_id).count()
# Campaign.objects(company_id=company_id).delete()
# _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " Campaigns deleted")
#
# #delete data in Account Master
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of Accounts started")
# count = Account.objects(company_id=company_id).count()
# Account.objects(company_id=company_id).delete()
# _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " Accounts deleted")
#
# find out which systems are integrated and therefore which tasks should be run
task_map = {
#"mkto" : [retrieveMktoCampaigns, retrieveMktoLeadsByProgram], #retrieveMktoLeadCreatedActivities, retrieveMktoLeads, retrieveMktoActivities, retrieveMktoCampaigns, retrieveMktoLeadsByProgram], #IMPORTANT - Lead Created Activities has to be before Leads
#"hspt" : [retrieveHsptCampaigns], # , ], # retrieveHsptLeads, retrieveHsptOpportunities, retrieveHsptWebsiteTraffic, ,
"prdt" : [retrievePrdtLeads],
# "sfdc" : [retrieveSfdcCampaigns] #retrieveSfdcLeads, retrieveSfdcContacts, retrieveSfdcCampaigns, retrieveSfdcAccounts, retrieveSfdcOpportunities, retrieveSfdcLeadHistory, retrieveSfdcContactHistory, retrieveSfdcOppHistory, retrieveSfdcOppStageHistory],
# "sugr" : [retrieveSugrLeads],
# "bufr" : [retrieveBufrTwInteractions],
# "goog" : [retrieveGoogWebsiteTraffic], \
# "fbok" : [retrieveFbokPageStats, retrieveFbokAdStats]#, retrieveFbokPostStats] # , ]
}
# # for future use - retrieveMktoContacts, retrieveMktoOpportunities, retrieveSfdcActivities,
final_task_map = {
# "mkto" : [saveMktoCampaignsToMaster, saveMktoLeadsByProgramToMaster], #saveMktoLeadsToMaster, saveMktoActivitiesToMaster, saveMktoCampaignsToMaster, saveMktoLeadsByProgramToMaster],#mergeMktoSfdcLeads, deleteLeads, deleteDuplicateMktoIdLeads #
#"hspt" : [saveHsptCampaignsToMaster ], #saveHsptLeadsToMaster, saveHsptOpportunitiesToMaster, saveHsptWebsiteTrafficToMaster, , ], # ,
# "sfdc" : [saveSfdcCampaignsToMaster], #saveSfdcLeadsToMaster, saveSfdcContactsToMaster, saveSfdcCampaignsToMaster, saveSfdcAccountsToMaster, saveSfdcOpportunitiesToMaster, saveSfdcLeadHistoryToMaster, saveSfdcContactHistoryToMaster, saveSfdcOppHistoryToMaster, saveSfdcOppStageHistoryToMaster], #
# "sugr" : [saveSugrLeadsToMaster],
# "bufr" : [saveBufrTwInteractionsToMaster],
# "goog": [saveGoogleWebsiteTrafficToMaster],
# "fbok": [saveFbokPageStatsToMaster, saveFbokAdStatsToMaster]#, saveFbokPostStatsToMaster] #
}
# #
# # #saveSfdcLeadsToMaster, saveSfdcContactsToMaster, saveSfdcOpportunitiesToMaster, saveSfdcCampaignsToMaster,
# # # saveSfdcLeadsToMaster, saveSfdcContactsToMaster, saveSfdcOpportunitiesToMaster, saveSfdcCampaignsToMaster, saveSfdcAccountsToMaster
# # #collect all relevant tasks in one list and retrieve metadata as well
for source in existingIntegration.integrations.keys():
#change metadata depending on source system
if source == 'sfdc':
metadata_objects = ['user', 'lead', 'contact', 'campaign', 'opportunity', 'task', 'account'] #[] #objects for which metadata should be collected
elif source == 'mkto':
metadata_objects = ['lead', 'activity', 'campaign'] #[] #objects for which metadata should be collected
elif source == 'hspt':
metadata_objects = ['lead'] #objects for which metadata should be collected
else:
metadata_objects = [] #objects for which metadata should be collected
# if sfdc, explicitly refresh the access token
if source == 'sfdc':
sfdc = Salesforce()
sfdc.refresh_token(company_id)
#collect meta data
url = host + '/api/v1/company/' + str(company_id) + '/integrations/metadata/'
# for object in metadata_objects:
# _superJobMonitorAddTask(superJobMonitor, source, "Retrieval of metadata for " + object + " started")
# params = {'code': source, 'object': object}
# resp = s.get(url, params=params) # get metadata about activities
# if not resp.status_code == 200:
# _superJobMonitorAddTask(superJobMonitor, source, "Retrieval of metadata for " + object + " failed")
# continue
# else:
# _superJobMonitorAddTask(superJobMonitor, source, "Retrieval of metadata for " + object + " completed")
#collect retrieval tasks
print 'starting retrieval tasks'
tasks = []
if source in task_map:
tasks.extend(task_map[source])
#now run the tasks
for task in tasks:
_superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " started to store temp data")
print 'starting task ' + str(task)
task(user_id=user_id, company_id=company_id, job_id=superJobMonitor.id, run_type=run_type, sinceDateTime=sinceDateTime)
_superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " completed")
print 'starting save tasks'
#collect save tasks
tasks = []
if source in final_task_map:
tasks.extend(final_task_map[source])
#now run the tasks
for task in tasks:
_superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " started to save master data")
print 'starting task ' + str(task)
task(user_id=user_id, company_id=company_id, job_id=superJobMonitor.id, run_type=run_type)
_superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " completed")
#return #REMOVE THIS IN PRODUCTION
#if initial run, delete all analytics data
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, 'Claritix', "Deletion of analytics data started")
# count1 = AnalyticsData.objects(company_id=company_id).count() #ensure that website_traffic chart data is not deleted
# AnalyticsData.objects(company_id=company_id).delete() #ensure that website_traffic chart data is not deleted
# count2 = AnalyticsIds.objects(company_id=company_id).count()
# AnalyticsIds.objects(company_id=company_id).delete()
# _superJobMonitorAddTask(superJobMonitor, 'Claritix', str(count1 + count2) + " records deleted from analytics tables")
#
# call chart calculate tasks
charts = [\
# {'chart_name': 'sources_bar', 'system_type': 'MA', 'chart_title':'Timeline', 'mode': run_type, 'start_date': sinceDateTime}, \
# #{'chart_name': 'pipeline_duration', 'system_type': 'MA', 'chart_title':'Pipeline Duration', 'mode': run_type, 'start_date': sinceDateTime}, \
# #{'chart_name': 'contacts_distr', 'system_type': 'MA', 'chart_title':'Contacts Distribution', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'source_pie', 'system_type': 'MA', 'chart_title':'Source Distribution', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'revenue_source_pie', 'system_type': 'MA', 'chart_title':'Revenue by Source', 'mode': run_type, 'start_date': sinceDateTime}, \
# #{'chart_name': 'multichannel_leads', 'system_type': 'MA', 'chart_title':'Multichannel Leads', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'tw_performance', 'system_type': 'SO', 'chart_title':'Twitter Performance', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'fb_performance', 'system_type': 'SO', 'chart_title':'Facebook_Performance', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'google_analytics', 'system_type': 'AD', 'chart_title':'Google Analytics', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'campaign_email_performance', 'system_type': 'MA', 'chart_title':'Campaign Performance by Email', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'email_cta_performance', 'system_type': 'MA', 'chart_title':'Email Performance by CTA', 'mode': run_type, 'start_date': sinceDateTime}, \
#
]
url = host + '/api/v1/company/' + str(company_id) + '/analytics/calculate/'
for chart in charts:
print 'starting chart ' + str(chart['chart_title'])
_superJobMonitorAddTask(superJobMonitor, 'Claritix', "Started calculating " + str(chart['chart_title']))
resp = s.get(url, params=chart)
if not resp.status_code == 200:
print 'incorrect status code was ' + str(resp.status_code)
_superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Could not retrieve data for ' + chart['chart_title'])
continue
else:
_superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Retrieved data for ' + chart['chart_title'])
# call dashboard calculate tasks
dashboards = [\
# {'chart_name': 'social_roi', 'system_type': 'MA', 'chart_title':'Social Performance', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'funnel', 'system_type': 'MA', 'chart_title':'Funnel', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'opp_funnel', 'system_type': 'CRM', 'chart_title':'Opportunity Funnel', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'waterfall_chart', 'system_type': 'MA', 'chart_title':'Waterfall Chart', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'form_fills', 'system_type': 'MA', 'chart_title':'Form Fills', 'mode': run_type, 'start_date': sinceDateTime}, \
]
url = host + '/api/v1/company/' + str(company_id) + '/dashboards/calculate/'
for dashboard in dashboards:
print 'starting dashboard ' + str(dashboard['chart_title'])
_superJobMonitorAddTask(superJobMonitor, 'Claritix', "Started calculating " + str(dashboard['chart_title']))
resp = s.get(url, params=dashboard)
if not resp.status_code == 200:
print 'incorrect status code was ' + str(resp.status_code)
_superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Could not retrieve data for ' + dashboard['chart_title'])
continue
else:
_superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Retrieved data for ' + dashboard['chart_title'])
#delete all data from past successful initial runs in Temp Table
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of temp data table started")
# successfulJobs = SuperJobMonitor.objects(Q(company_id=company_id) & Q(type='initial') & Q(status='Completed'))
# successfulJobsListTemp = list(successfulJobs)
# print 'found job ids ' + str(len(successfulJobsListTemp))
# successfulJobsList = [i.id for i in successfulJobsListTemp]
# count = TempData.objects(job_id__in=successfulJobsList).count()
# TempData.objects(job_id__in=successfulJobsList).delete()
# _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " records deleted from temp data table")
#delete all data from past successful delta runs in Temp Table
if run_type == 'delta':
_superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of temp data delta table started")
successfulJobs = SuperJobMonitor.objects(Q(company_id=company_id) & Q(type='delta') & Q(status='Completed'))
successfulJobsListTemp = list(successfulJobs)
print 'found delta job ids ' + str(len(successfulJobsListTemp))
successfulJobsList = [i.id for i in successfulJobsListTemp]
count = TempDataDelta.objects(job_id__in=successfulJobsList).count()
TempDataDelta.objects(job_id__in=successfulJobsList).delete()
_superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " records deleted from temp data delta table")
#update last run date for initial run in Company Integration record
_superJobMonitorEnd(superJobMonitor, existingIntegration, run_type, 'Completed', 'All tasks completed successfully')
return True
except Exception as e:
if superJobMonitor is not None and existingIntegration is not None:
_superJobMonitorEnd(superJobMonitor, existingIntegration, run_type, 'Failed', str(e))
print str(e)
return False