def list(self, request, company_id=None): #retrieve all data extract jobs across all companies if company_id != '0': return HttpResponse('Company is incorrect', status=status.HTTP_400_BAD_REQUEST) try: page_number = int(request.GET.get('page_number')) items_per_page = int(request.GET.get('per_page')) offset = (page_number - 1) * items_per_page jobs = SuperJobMonitor.objects().order_by('-started_date') totalCount = SuperJobMonitor.objects().count() totalCountSuccess = SuperJobMonitor.objects(status='Completed').count() totalCountFailure = SuperJobMonitor.objects(status='Failed').count() qlist = list(jobs) result = qlist[offset:offset+items_per_page] serializedList = SuperJobMonitorSerializer(result, many=True) initialCount = SuperJobMonitor.objects(type='initial').count() initialCountSuccess = SuperJobMonitor.objects(Q(type='initial') & Q(status='Completed')).count() initialCountFailure = SuperJobMonitor.objects(Q(type='initial') & Q(status='Failed')).count() deltaCount = SuperJobMonitor.objects(type='delta').count() deltaCountSuccess = SuperJobMonitor.objects(Q(type='delta') & Q(status='Completed')).count() deltaCountFailure = SuperJobMonitor.objects(Q(type='delta') & Q(status='Failed')).count() return JsonResponse({'totalCount': totalCount, 'totalCountSuccess': totalCountSuccess, \ 'totalCountFailure': totalCountFailure, 'initialCount': initialCount, \ 'initialCountSuccess': initialCountSuccess, 'initialCountFailure': initialCountFailure, \ 'deltaCount': deltaCount, 'deltaCountSuccess': deltaCountSuccess, \ 'deltaCountFailure': deltaCountFailure, 'results': serializedList.data}) except Exception as e: return HttpResponse(str(e), status=status.HTTP_400_BAD_REQUEST)
def companyDataExtract(user_id=None, company_id=None, run_type=None, sinceDateTime=None): superJobMonitor = None existingIntegration = None if run_type is None or company_id is None or sinceDateTime is None: return try: sinceDateTime = datetime.fromtimestamp(float(sinceDateTime) / 1000) #sinceDateTime = datetime.now() - timedelta(days=30) print 'start date is ' + str(sinceDateTime) print 'run type is ' + run_type print 'company id is ' + str(company_id) #create an entry in the Job Monitor superJobMonitor = SuperJobMonitor(company_id=company_id, type=run_type, started_date=datetime.utcnow()) superJobMonitor.save() #do pre-check _superJobMonitorAddTask(superJobMonitor, "Claritix", "Pre-check started") # get the integration record existingIntegration = CompanyIntegration.objects(company_id=company_id).first() if existingIntegration is None: _superJobMonitorEnd(superJobMonitor, None, 'Failed', 'No integration record found') mail_admins('Could not find integration record for company ' + company_id , 'Check settings') return False if run_type == 'initial': existingIntegration['initial_run_in_process'] = True #set the flag else: #delta existingIntegration['delta_run_in_process'] = True #set the flag existingIntegration.save() # save the flag # set up the Request and Cookie user = _get_superadmin() if user is None: _superJobMonitorEnd(superJobMonitor, existingIntegration, 'Failed', 'No super admin found') mail_admins('Could not find super admin!', 'Check settings') return False # remotely login the user host = settings.BASE_URL url = host + '/api/v1/auth/login/' creds = {'email': '*****@*****.**', 'password':'******'} s = requests.Session() resp = s.post(url, data=json.dumps(creds)) if not resp.status_code == 200: _superJobMonitorEnd(superJobMonitor, existingIntegration, 'Failed', 'Could not login super admin!') mail_admins('Could not login super admin!', 'Check credentials') return False #do cookie thing - refer to SuperAdmin Cron Job Task for details cookies = dict(sessionid = resp.cookies['sessionid']) url = host + '/api/v1/users/' resp = s.get(url, cookies=cookies) _superJobMonitorAddTask(superJobMonitor, "Claritix", "Pre-check completed") # #delete data in Lead Master # if run_type == 'initial': # _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of Leads, Contacts, Opportunities and Activities started") # count = Lead.objects(company_id=company_id).count() # Lead.objects(company_id=company_id).delete() # _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " Leads, Contacts, Opportunities and Activities deleted") # # #delete data in Campaign Master # if run_type == 'initial': # _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of Campaigns started") # count = Campaign.objects(company_id=company_id).count() # Campaign.objects(company_id=company_id).delete() # _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " Campaigns deleted") # # #delete data in Account Master # if run_type == 'initial': # _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of Accounts started") # count = Account.objects(company_id=company_id).count() # Account.objects(company_id=company_id).delete() # _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " Accounts deleted") # # find out which systems are integrated and therefore which tasks should be run task_map = { #"mkto" : [retrieveMktoCampaigns, retrieveMktoLeadsByProgram], #retrieveMktoLeadCreatedActivities, retrieveMktoLeads, retrieveMktoActivities, retrieveMktoCampaigns, retrieveMktoLeadsByProgram], #IMPORTANT - Lead Created Activities has to be before Leads #"hspt" : [retrieveHsptCampaigns], # , ], # retrieveHsptLeads, retrieveHsptOpportunities, retrieveHsptWebsiteTraffic, , "prdt" : [retrievePrdtLeads], # "sfdc" : [retrieveSfdcCampaigns] #retrieveSfdcLeads, retrieveSfdcContacts, retrieveSfdcCampaigns, retrieveSfdcAccounts, retrieveSfdcOpportunities, retrieveSfdcLeadHistory, retrieveSfdcContactHistory, retrieveSfdcOppHistory, retrieveSfdcOppStageHistory], # "sugr" : [retrieveSugrLeads], # "bufr" : [retrieveBufrTwInteractions], # "goog" : [retrieveGoogWebsiteTraffic], \ # "fbok" : [retrieveFbokPageStats, retrieveFbokAdStats]#, retrieveFbokPostStats] # , ] } # # for future use - retrieveMktoContacts, retrieveMktoOpportunities, retrieveSfdcActivities, final_task_map = { # "mkto" : [saveMktoCampaignsToMaster, saveMktoLeadsByProgramToMaster], #saveMktoLeadsToMaster, saveMktoActivitiesToMaster, saveMktoCampaignsToMaster, saveMktoLeadsByProgramToMaster],#mergeMktoSfdcLeads, deleteLeads, deleteDuplicateMktoIdLeads # #"hspt" : [saveHsptCampaignsToMaster ], #saveHsptLeadsToMaster, saveHsptOpportunitiesToMaster, saveHsptWebsiteTrafficToMaster, , ], # , # "sfdc" : [saveSfdcCampaignsToMaster], #saveSfdcLeadsToMaster, saveSfdcContactsToMaster, saveSfdcCampaignsToMaster, saveSfdcAccountsToMaster, saveSfdcOpportunitiesToMaster, saveSfdcLeadHistoryToMaster, saveSfdcContactHistoryToMaster, saveSfdcOppHistoryToMaster, saveSfdcOppStageHistoryToMaster], # # "sugr" : [saveSugrLeadsToMaster], # "bufr" : [saveBufrTwInteractionsToMaster], # "goog": [saveGoogleWebsiteTrafficToMaster], # "fbok": [saveFbokPageStatsToMaster, saveFbokAdStatsToMaster]#, saveFbokPostStatsToMaster] # } # # # # #saveSfdcLeadsToMaster, saveSfdcContactsToMaster, saveSfdcOpportunitiesToMaster, saveSfdcCampaignsToMaster, # # # saveSfdcLeadsToMaster, saveSfdcContactsToMaster, saveSfdcOpportunitiesToMaster, saveSfdcCampaignsToMaster, saveSfdcAccountsToMaster # # #collect all relevant tasks in one list and retrieve metadata as well for source in existingIntegration.integrations.keys(): #change metadata depending on source system if source == 'sfdc': metadata_objects = ['user', 'lead', 'contact', 'campaign', 'opportunity', 'task', 'account'] #[] #objects for which metadata should be collected elif source == 'mkto': metadata_objects = ['lead', 'activity', 'campaign'] #[] #objects for which metadata should be collected elif source == 'hspt': metadata_objects = ['lead'] #objects for which metadata should be collected else: metadata_objects = [] #objects for which metadata should be collected # if sfdc, explicitly refresh the access token if source == 'sfdc': sfdc = Salesforce() sfdc.refresh_token(company_id) #collect meta data url = host + '/api/v1/company/' + str(company_id) + '/integrations/metadata/' # for object in metadata_objects: # _superJobMonitorAddTask(superJobMonitor, source, "Retrieval of metadata for " + object + " started") # params = {'code': source, 'object': object} # resp = s.get(url, params=params) # get metadata about activities # if not resp.status_code == 200: # _superJobMonitorAddTask(superJobMonitor, source, "Retrieval of metadata for " + object + " failed") # continue # else: # _superJobMonitorAddTask(superJobMonitor, source, "Retrieval of metadata for " + object + " completed") #collect retrieval tasks print 'starting retrieval tasks' tasks = [] if source in task_map: tasks.extend(task_map[source]) #now run the tasks for task in tasks: _superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " started to store temp data") print 'starting task ' + str(task) task(user_id=user_id, company_id=company_id, job_id=superJobMonitor.id, run_type=run_type, sinceDateTime=sinceDateTime) _superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " completed") print 'starting save tasks' #collect save tasks tasks = [] if source in final_task_map: tasks.extend(final_task_map[source]) #now run the tasks for task in tasks: _superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " started to save master data") print 'starting task ' + str(task) task(user_id=user_id, company_id=company_id, job_id=superJobMonitor.id, run_type=run_type) _superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " completed") #return #REMOVE THIS IN PRODUCTION #if initial run, delete all analytics data # if run_type == 'initial': # _superJobMonitorAddTask(superJobMonitor, 'Claritix', "Deletion of analytics data started") # count1 = AnalyticsData.objects(company_id=company_id).count() #ensure that website_traffic chart data is not deleted # AnalyticsData.objects(company_id=company_id).delete() #ensure that website_traffic chart data is not deleted # count2 = AnalyticsIds.objects(company_id=company_id).count() # AnalyticsIds.objects(company_id=company_id).delete() # _superJobMonitorAddTask(superJobMonitor, 'Claritix', str(count1 + count2) + " records deleted from analytics tables") # # call chart calculate tasks charts = [\ # {'chart_name': 'sources_bar', 'system_type': 'MA', 'chart_title':'Timeline', 'mode': run_type, 'start_date': sinceDateTime}, \ # #{'chart_name': 'pipeline_duration', 'system_type': 'MA', 'chart_title':'Pipeline Duration', 'mode': run_type, 'start_date': sinceDateTime}, \ # #{'chart_name': 'contacts_distr', 'system_type': 'MA', 'chart_title':'Contacts Distribution', 'mode': run_type, 'start_date': sinceDateTime}, \ # {'chart_name': 'source_pie', 'system_type': 'MA', 'chart_title':'Source Distribution', 'mode': run_type, 'start_date': sinceDateTime}, \ # {'chart_name': 'revenue_source_pie', 'system_type': 'MA', 'chart_title':'Revenue by Source', 'mode': run_type, 'start_date': sinceDateTime}, \ # #{'chart_name': 'multichannel_leads', 'system_type': 'MA', 'chart_title':'Multichannel Leads', 'mode': run_type, 'start_date': sinceDateTime}, \ # {'chart_name': 'tw_performance', 'system_type': 'SO', 'chart_title':'Twitter Performance', 'mode': run_type, 'start_date': sinceDateTime}, \ # {'chart_name': 'fb_performance', 'system_type': 'SO', 'chart_title':'Facebook_Performance', 'mode': run_type, 'start_date': sinceDateTime}, \ # {'chart_name': 'google_analytics', 'system_type': 'AD', 'chart_title':'Google Analytics', 'mode': run_type, 'start_date': sinceDateTime}, \ # {'chart_name': 'campaign_email_performance', 'system_type': 'MA', 'chart_title':'Campaign Performance by Email', 'mode': run_type, 'start_date': sinceDateTime}, \ # {'chart_name': 'email_cta_performance', 'system_type': 'MA', 'chart_title':'Email Performance by CTA', 'mode': run_type, 'start_date': sinceDateTime}, \ # ] url = host + '/api/v1/company/' + str(company_id) + '/analytics/calculate/' for chart in charts: print 'starting chart ' + str(chart['chart_title']) _superJobMonitorAddTask(superJobMonitor, 'Claritix', "Started calculating " + str(chart['chart_title'])) resp = s.get(url, params=chart) if not resp.status_code == 200: print 'incorrect status code was ' + str(resp.status_code) _superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Could not retrieve data for ' + chart['chart_title']) continue else: _superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Retrieved data for ' + chart['chart_title']) # call dashboard calculate tasks dashboards = [\ # {'chart_name': 'social_roi', 'system_type': 'MA', 'chart_title':'Social Performance', 'mode': run_type, 'start_date': sinceDateTime}, \ # {'chart_name': 'funnel', 'system_type': 'MA', 'chart_title':'Funnel', 'mode': run_type, 'start_date': sinceDateTime}, \ # {'chart_name': 'opp_funnel', 'system_type': 'CRM', 'chart_title':'Opportunity Funnel', 'mode': run_type, 'start_date': sinceDateTime}, \ # {'chart_name': 'waterfall_chart', 'system_type': 'MA', 'chart_title':'Waterfall Chart', 'mode': run_type, 'start_date': sinceDateTime}, \ # {'chart_name': 'form_fills', 'system_type': 'MA', 'chart_title':'Form Fills', 'mode': run_type, 'start_date': sinceDateTime}, \ ] url = host + '/api/v1/company/' + str(company_id) + '/dashboards/calculate/' for dashboard in dashboards: print 'starting dashboard ' + str(dashboard['chart_title']) _superJobMonitorAddTask(superJobMonitor, 'Claritix', "Started calculating " + str(dashboard['chart_title'])) resp = s.get(url, params=dashboard) if not resp.status_code == 200: print 'incorrect status code was ' + str(resp.status_code) _superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Could not retrieve data for ' + dashboard['chart_title']) continue else: _superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Retrieved data for ' + dashboard['chart_title']) #delete all data from past successful initial runs in Temp Table # if run_type == 'initial': # _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of temp data table started") # successfulJobs = SuperJobMonitor.objects(Q(company_id=company_id) & Q(type='initial') & Q(status='Completed')) # successfulJobsListTemp = list(successfulJobs) # print 'found job ids ' + str(len(successfulJobsListTemp)) # successfulJobsList = [i.id for i in successfulJobsListTemp] # count = TempData.objects(job_id__in=successfulJobsList).count() # TempData.objects(job_id__in=successfulJobsList).delete() # _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " records deleted from temp data table") #delete all data from past successful delta runs in Temp Table if run_type == 'delta': _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of temp data delta table started") successfulJobs = SuperJobMonitor.objects(Q(company_id=company_id) & Q(type='delta') & Q(status='Completed')) successfulJobsListTemp = list(successfulJobs) print 'found delta job ids ' + str(len(successfulJobsListTemp)) successfulJobsList = [i.id for i in successfulJobsListTemp] count = TempDataDelta.objects(job_id__in=successfulJobsList).count() TempDataDelta.objects(job_id__in=successfulJobsList).delete() _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " records deleted from temp data delta table") #update last run date for initial run in Company Integration record _superJobMonitorEnd(superJobMonitor, existingIntegration, run_type, 'Completed', 'All tasks completed successfully') return True except Exception as e: if superJobMonitor is not None and existingIntegration is not None: _superJobMonitorEnd(superJobMonitor, existingIntegration, run_type, 'Failed', str(e)) print str(e) return False