def saveBufrTwInteractionsToMaster(user_id=None, company_id=None, job_id=None, run_type=None): #behaves differently because it directly saves the data to the AnalyticsData collection if run_type == 'initial': tw_interactions = TempData.objects(Q(company_id=company_id) & Q(record_type='tw_interaction') & Q(source_system='bufr') & Q(job_id=job_id) ).only('source_record') else: tw_interactions = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='tw_interaction') & Q(source_system='bufr') & Q(job_id=job_id) ).only('source_record') company_query = 'company_id' interaction_id_query = 'interaction_id' tw_interactionsList = list(tw_interactions) tw_interactionsList = [i['source_record'] for i in tw_interactionsList] try: for interaction in tw_interactionsList: profile_id = interaction['profile_id'] published_date = datetime.fromtimestamp(float(interaction['sent_at'])) local_published_date = get_current_timezone().localize(published_date, is_dst=None) date = local_published_date.strftime('%Y-%m-%d') queryDict = {company_query : company_id, interaction_id_query: interaction['id']} publishedTweet = PublishedTweet.objects(**queryDict).first() if publishedTweet is None: #this tweet's record not found publishedTweet = PublishedTweet() publishedTweet.company_id = company_id publishedTweet.interaction_id = interaction['id'] publishedTweet.published_date = date publishedTweet.published_timestamp = interaction['sent_at'] publishedTweet.data = interaction publishedTweet.save() # except Exception as e: send_notification(dict(type='error', success=False, message=str(e)))