def get_queryset(self):
status = self.kwargs['status']
company_id = self.request.user.get_company()
print 'co id is ' + str(company_id)
record_id = CompanyIntegration.objects(company_id = company_id ).only('id').first()
existingIntegration = {}
existingIntegration['integrations'] = {}
map = Code("function () {"
" for (var key in this.integrations) emit(key, this.integrations[key]['access_token']); } ")
reduce = Code("function (key, values) { return null; } ")
results = CompanyIntegration.objects(company_id=company_id).map_reduce(map, reduce, "inline")
results = list(results)
for result in results:
existingIntegration['integrations'][result.key] = {'access_token': result.value}
# for key in existingIntegration.integrations.keys():
# if 'metadata' in existingIntegration.integrations[key]:
# existingIntegration.integrations[key]['metadata'] = {} #empty the metadata object so that excess data is not sent back
# if 'metadata' in existingIntegration.integrations[key]:
# existingIntegration.integrations[key]['metadata'] = {} #empty the metadata object so that excess data is not sent back
if existingIntegration['integrations'] != {}: #assume there's onlg one entry
if status == 'new':
queryset = SuperIntegration.objects(code__nin=existingIntegration['integrations'].keys())
for obj in queryset:
#company_info = CompanyGenericIntegrationSerializer()
obj.company_info = {}
elif status == 'existing':
queryset = SuperIntegration.objects(code__in=existingIntegration['integrations'].keys())
for obj in queryset:
#company_info = CompanyGenericIntegrationSerializer(existingIntegration.integrations[obj.code])
# print "code is " + str(existingIntegration.integrations[obj.code])
obj.company_info = existingIntegration['integrations'][obj.code]
obj.company_info["code"] = obj.code
obj.company_info['record_id'] = json.dumps(record_id['id'], default=json_util.default)
return queryset
# if we are here, there are no records for the company
if status == 'new':
print 'new sttatus'
queryset_new = SuperIntegration.objects.all()
elif status == 'existing':
queryset_new = None
return queryset_new
def get_tw_handles_buffer(request, id):
try:
company = Company.objects.filter(company_id=id).first()
existingIntegration = CompanyIntegration.objects(company_id = id ).first()
if 'bufr' in existingIntegration['integrations']: # if Buffer is present and configured
client_id = existingIntegration['integrations']['bufr']['client_id']
client_secret = existingIntegration['integrations']['bufr']['client_secret']
access_token = existingIntegration['integrations']['bufr']['access_token']
buffer = Buffer()
api = Buffer.get_api(buffer, client_id=client_id, client_secret=client_secret, access_token=access_token)
profiles = Buffer.get_twitter_profiles(buffer, api)
results = []
for profile in profiles:
new_profile = {}
new_profile['id'] = profile['id']
new_profile['service'] = profile['service']
new_profile['service_id'] = profile['service_id']
new_profile['service_username'] = profile['service_username']
results.append(new_profile)
#serializedList = BufferProfileSerializer(profiles, many=True)
return JsonResponse({'results' :results})
else:
return JsonResponse({'error' : 'No integration found with Buffer'})
except Exception as e:
return JsonResponse({'Error' : str(e)})
def publishMl(request, id, masterlist_id):
try:
company = Company.objects.filter(company_id=id).first()
ml = TweetMasterList.objects(Q(company=company.id) & Q(id=masterlist_id)).first()
if ml is None:
return HttpResponse("Could not find the Master List for publishing", status=status.HTTP_400_BAD_REQUEST)
tweets = ml['tweets']
shuffle(tweets)
#buffer_profile_id = '558f150b7409ab382f11a39e' #change to parameter later
existingIntegration = CompanyIntegration.objects(company_id = id ).first()
if 'bufr' in existingIntegration['integrations']: # if Buffer is present and configured
client_id = existingIntegration['integrations']['bufr']['client_id']
client_secret = existingIntegration['integrations']['bufr']['client_secret']
access_token = existingIntegration['integrations']['bufr']['access_token']
buffer = Buffer()
api = Buffer.get_api(buffer, client_id=client_id, client_secret=client_secret, access_token=access_token)
#profile = Buffer.get_twitter_profile(buffer, api)
profile = Buffer.get_twitter_profile_by_id(buffer, api, ml.buffer_profile_id)
#print 'posting to profile ' + profile['id']
for tweet in tweets:
try:
post = quote_plus(tweet['text'])
Buffer.post_to_twitter(buffer, api, post, profile)
except Exception as e:
continue
TweetMasterList.objects(Q(company=company.id) & Q(id=masterlist_id)).update(published=True)
TweetMasterList.objects(Q(company=company.id) & Q(id=masterlist_id)).update(published_date=datetime.utcnow())
return HttpResponse("Tweets posted", status=status.HTTP_200_OK)
else:
return HttpResponse("No publishing integration found", status=status.HTTP_400_BAD_REQUEST)
except Exception as e:
return JsonResponse({'Error' : str(e)})
def filterTwInteractions(request, id):
user_id = request.user.id
company_id = request.user.company_id
start_date = request.GET.get('start_date')
end_date = request.GET.get('end_date')
interaction_type = request.GET.get('interaction_type')
series_type = request.GET.get('series_type')
query_type = request.GET.get('query_type')
page_number = int(request.GET.get('page_number'))
items_per_page = int(request.GET.get('per_page'))
system_type = request.GET.get('system_type')
chart_name = request.GET.get('chart_name')
offset = (page_number - 1) * items_per_page
user_id = request.user.id
company_id = request.user.company_id
existingIntegration = CompanyIntegration.objects(company_id = company_id ).first()
try:
code = None
if existingIntegration is not None:
for source in existingIntegration.integrations.keys():
defined_system_type = SuperIntegration.objects(Q(code = source) & Q(system_type = system_type)).first()
if defined_system_type is not None:
code = source
#print 'found code' + str(code)
if code is None:
raise ValueError("No integrations defined")
elif code == 'bufr':
result = filtertwInteractionsBufr(user_id=user_id, company_id=company_id, start_date=start_date, end_date=end_date, interaction_type=interaction_type, series_type=series_type, query_type=query_type, page_number=page_number, items_per_page=items_per_page, system_type=system_type, offset=offset, code=code, chart_name=chart_name)
else:
result = 'Nothing to report'
return result
except Exception as e:
return JsonResponse({'Error' : str(e)})
def retrieveFbokAdCampaignStats(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
print 'starting retrieveFbokAdCampaignStats for company ' + str(company_id)
existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
if 'fbok' not in existingIntegration['integrations']: # if Buffer is present and configured
print 'did not find fbok'
raise Exception('Facebook integration not found')
integration = existingIntegration.integrations['fbok']
if integration['access_token'] is None:
raise Exception('Facebook access token not found')
fbok = Facebook(integration['host'], integration['client_id'], integration['client_secret'], integration['redirect_uri'])
if fbok is None:
raise Exception('Facebook object could not be created')
print 'calling campaigns'
campaigns = fbok.get_campaign_stats(company_id, run_type)
print 'found FB campaigns: ' + str(campaigns)
saveFbokAdCampaignStats(user_id=user_id, company_id=company_id, results=campaigns, job_id=job_id, run_type=run_type)
except Exception as e:
print 'exception was ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def getUserSlackMembership(request, company_id):
try:
#print 'in user room'
user_id = request.user.id
#company_id = request.user.company
slck_auth_needed = False
slck_user_auth_needed = False
# company = Company.objects(company_id=company_id).first()
# company_id = company.id
print 'company id is ' + str(company_id)
existingIntegration= CompanyIntegration.objects(company_id=company_id).first()
if existingIntegration is None or 'slck' not in existingIntegration['integrations']:
return JsonResponse({'slck_auth_needed' : True})
#token = existingIntegration['integrations']['slck']
userOauthRecord = UserOauth.objects(user_id= ObjectId(user_id)).first()
if userOauthRecord is None:
return JsonResponse({'slck_user_auth_needed' : True})
if 'slck_access_token' in userOauthRecord and userOauthRecord['slck_access_token'] is not None and userOauthRecord['slck_access_token'] != "":
token = userOauthRecord['slck_access_token']
else:
return JsonResponse({'slck_user_auth_needed' : True})
slck = Slack(None, None, None, None, token)
channels = json.loads(slck.api_call("channels.list"))
groups = json.loads(slck.api_call("groups.list"))
ims = json.loads(slck.api_call("im.list"))
users = json.loads(slck.api_call("users.list"))
if not users['ok']:
raise ValueError('Error while retrieving users from Slack')
rtm = slck.rtm_connect()
return JsonResponse({'slack_channels': channels, 'slack_groups': groups, 'slack_ims': ims, 'users': users['members'], 'rtm': rtm, 'slck_auth_needed': False, 'slck_user_auth_needed': slck_user_auth_needed })
except Exception as e:
return Response('Error: ' + str(e))
def getViews(request, company_id):
company_id = request.user.company_id
existingIntegration = CompanyIntegration.objects(company_id = company_id ).first()
try:
code = None
views_temp = []
views = []
# if existingIntegration is not None:
# sources = set()
# defined_system_types = set()
# for source in existingIntegration.integrations.keys():
# sources.add(source)
# for source in sources:
# #print 'source is ' + str(source)
# defined_system = SuperIntegration.objects(code = source).first()
# defined_system_types.add(defined_system.system_type)
# for defined_system_type in defined_system_types:
# #print 'def system is ' + str(defined_system.system_type)
# if defined_system_type is not None:
# dashboards_temp = SuperDashboards.objects(Q(system_type = defined_system_type) & Q(status__ne='Inactive')).all()
# for dashboard_temp in list(dashboards_temp):
# serializer = SuperDashboardsSerializer(dashboard_temp, many=False)
# dashboards.append(serializer.data)
views_temp = SuperViews.objects().all()
for view_temp in list(views_temp):
serializer = SuperViewsSerializer(view_temp, many=False)
views.append(serializer.data)
return JsonResponse({"results": views}, safe=False)
except Exception as e:
return JsonResponse({'Error' : str(e)})
def list(self, request, id=None):
try:
company = CompanyIntegration.objects(company_id=id).only('initial_run_in_process').first()
serializedList = CompanyIntegrationSerializer(company, many=False)
return Response(serializedList.data)
except Exception as e:
return Response(str(e))
def retrieveSfdcOpportunities(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
#company_id = request.user.company_id
existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
# code commented out since we are no longer getting only Mkto related opportunities into Cx
# if existingIntegration is not None and 'mkto' in existingIntegration['integrations']: # if this system is connected to Marketo
# company_qry = 'company_id'
# type_field_qry = 'leads__mkto__exists'
# sfdc_account_field_qry = 'leads__mkto__sfdcAccountId__ne'
# querydict = {company_qry: company_id, type_field_qry: True, sfdc_account_field_qry: None}
# leads_with_sfdc_opps = Lead.objects(**querydict).only('mkto_id').only('leads__mkto__sfdcAccountId')
#
sfdc = Salesforce()
# code commented out since we are no longer getting only Mkto related opportunities into Cx
# account_list = '('
# for lead in leads_with_sfdc_opps:
# account_list += '\'' + lead['leads']['mkto']['sfdcAccountId'] + '\'' + ', '
# account_list = account_list[:-2]
# account_list += ')'
if sinceDateTime is None:
sinceDateTime = (datetime.now() - timedelta(days=30)).date()
oppList = sfdc.get_opportunities_delta(user_id, company_id, _str_from_date(sinceDateTime), run_type)
print 'got opps ' + str(len(oppList['records']))
#create list of Opp IDs to send for get_contacts call
oppid_list = '('
for opp in oppList['records']:
oppid_list += '\'' + opp['Id'] + '\'' + ', '
oppid_list = oppid_list[:-2]
oppid_list += ')'
contactList = sfdc.get_contacts_for_opportunities(user_id, company_id, oppid_list) # needed because SFDC does not have the Contact ID within the Opp record
print 'got contacts for opps ' + str(len(contactList['records']))
saveSfdcOpportunities(user_id=user_id, company_id=company_id, oppList=oppList, contactList=contactList, job_id=job_id, run_type=run_type)
try:
message = 'Opportunities retrieved from Salesforce'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Opportunities'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return oppList
except Exception as e:
send_notification(dict(type='error', success=False, message=str(e)))
def get(self, *args, **kwargs):
template_name = 'integrations/new.html'
form_class = IntegrationBaseForm
instance = CompanyIntegration.objects(id=self.kwargs['id']).first()
if instance is not None:
queryset = instance.integrations[self.kwargs['code']];
#print 'qset ' + queryset["host"]
form = IntegrationBaseForm(queryset)
return render_to_response('integrations/new.html', {'form':form})
return HttpResponse("No editable record for " + self.kwargs['code'] + " found!", status=status.HTTP_400_BAD_REQUEST)
def user_create_slack_message(request): #this is only for messages with attachments (e.g. Snapshots). Simple messages are sent via websocket from the client
try:
post_data = json.loads(request.body)
channel_id = post_data['channel_id']
message = post_data['message']
#company_id = post_data['company_id']
user_id = request.user.id
company_id = request.user.company
if 'snapshot_id' in post_data:
snapshot_id = post_data['snapshot_id']
else:
snapshot_id = None
print 'snap id ' + str(snapshot_id)
existingIntegration= CompanyIntegration.objects(company_id=company_id).first()
if existingIntegration is None or 'slck' not in existingIntegration['integrations']:
return JsonResponse(None)
#token = existingIntegration['integrations']['slck']
userOauthRecord = UserOauth.objects(user_id= ObjectId(user_id)).first()
if userOauthRecord is None:
return JsonResponse(None)
if 'slck_access_token' in userOauthRecord and userOauthRecord['slck_access_token'] is not None:
token = userOauthRecord['slck_access_token']
else:
return JsonResponse(None)
slck = Slack(None, None, None, None, token)
attachments = []
if snapshot_id is not None:
snapshot = Snapshot.objects(id=ObjectId(snapshot_id)).first()
if snapshot is None:
raise ValueError('Snapshot not found!')
attachment = {
"fallback": "New Claritix" + snapshot['chart_name'] + " chart snapshot - " + settings.BASE_URL + "/snapshots/" + snapshot_id,
"pretext": "New Claritix chart snapshot",
"title": "Claritix snapshot - " + snapshot['chart_name'] + " as of " + strftime('%Y-%m-%d %H:%M:%S', snapshot['updated_date'].timetuple()),
"title_link": settings.BASE_URL + "/snapshots/" + snapshot_id,
"text": "Click above link to view the snapshot within Claritix",
"color": "#0491c3",
"author_name": "Claritix",
"author_link": "http://claritix.io",
"author_icon": settings.BASE_URL + "/static/images/logo-icon-16x16.png"
}
attachments.append(attachment)
print 'attachme ' + str(attachments)
params = {'channel': channel_id, 'text': message, 'as_user': True, "attachments": json.dumps(attachments)}
result = json.loads(slck.api_call("chat.postMessage", **params))
return JsonResponse({"result": result}, safe=False)
except Exception as e:
return JsonResponse({'Error' : str(e)})
def ajax(self, request):
print 'in ajax'
code = self.kwargs['code']
#print 'new code is ' + code
self.form_class = self.get_form_class();
#print 'class is ' + str(self.form_class)
form = self.form_class(data=json.loads(request.body))
if form.is_valid():
company_id = request.user.get_company()
existingIntegration = CompanyIntegration.objects(company_id = company_id ).first()
if existingIntegration is not None:
if code in existingIntegration.integrations: # record for this system found - update
existingDict = existingIntegration.integrations
existingDict[code] = form.cleaned_data
existingDict[code]['access_token'] = ''
CompanyIntegration.objects(company_id = company_id ).update(integrations=existingDict)
print "Updated data for " + code
return HttpResponse("Updated data for " + code, status=status.HTTP_200_OK)
else:
#existingIntegration.integrations[code] = form.cleaned_data
existingDict = existingIntegration.integrations
existingDict[code] = form.cleaned_data
#existingDict[code] [code]['access_token'] = ''
CompanyIntegration.objects(company_id = company_id ).update(integrations=existingDict)
return HttpResponse('New integration added to existing data', status=status.HTTP_200_OK)
companyIntegration = CompanyIntegration()
companyIntegration.company_id = company_id
companyIntegration.integrations[code] = form.cleaned_data
try:
#print 'saving'
companyIntegration.save()
return HttpResponse('New integration added', status=status.HTTP_200_OK)
except Exception as e:
print str(e)
return HttpResponse(str(e), status=status.HTTP_400_BAD_REQUEST)
else:
response_data = {'errors': form.errors} #, 'success_url': force_text(self.success_url)
return HttpResponse(json.dumps(response_data), content_type="application/json")
def getUserSlackMessages(request, company_id):
try:
#print 'in user room'
user_id = request.user.id
#company_id = request.user.company
slack_id = request.GET.get("id")
slack_type = request.GET.get("type")
# company = Company.objects(company_id=company_id).first()
# company_id = company.id
existingIntegration= CompanyIntegration.objects(company_id=company_id).first()
if existingIntegration is None or 'slck' not in existingIntegration['integrations']:
return JsonResponse(None)
#token = existingIntegration['integrations']['slck']
userOauthRecord = UserOauth.objects(user_id= ObjectId(user_id)).first()
if userOauthRecord is None:
return JsonResponse(None)
if 'slck_access_token' in userOauthRecord and userOauthRecord['slck_access_token'] is not None:
token = userOauthRecord['slck_access_token']
else:
return JsonResponse(None)
slck = Slack(None, None, None, None, token)
messages = None
params = {'channel': slack_id, 'inclusive': 1, 'count': 1000}
if slack_type == 'group':
messages = json.loads(slck.api_call("groups.history", **params))
elif slack_type == 'channel':
messages = json.loads(slck.api_call("channels.history", **params))
elif slack_type == 'im':
messages = json.loads(slck.api_call("im.history", **params))
if not messages['ok']:
raise ValueError('Error while retrieving messages from Slack')
messages_list = messages['messages']
rtm = slck.rtm_connect()
users = json.loads(slck.api_call("users.list"))
if not users['ok']:
raise ValueError('Error while retrieving users from Slack')
# for message in messages_list: #join the user info to each message
# #print 'msg is ' + str(message['user'])
# for user in users['members']:
# if 'user' in message and message['user'] == user['id']:
# message['user_real_name'] = user['profile']['real_name']
# message['user_name'] = user['name']
# message['user_image_url'] = user['profile']['image_72']
# break
return JsonResponse({'slack_messages': messages, 'users': users['members'], 'rtm': rtm}, safe=False)
except Exception as e:
print 'error occurred: ' + str(e)
return Response('Error: ' + str(e))
def retrieveFbokPageStats(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
print 'starting retrieveFbokPageStats for company ' + str(company_id)
#print 'timestamp ' + str(sinceDateTime)
if (datetime.now() - sinceDateTime).days > 89:
print 'FB does not allow for more than 89 days of data'
sinceDateTime = datetime.now() - timedelta(days=89)
sinceTimestamp = calendar.timegm(sinceDateTime.timetuple())
sinceTimestamp = str('{0:f}'.format(sinceTimestamp).rstrip('0').rstrip('.'))
untilTimestamp = time.mktime(datetime.now().timetuple())
untilTimestamp = str('{0:f}'.format(untilTimestamp).rstrip('0').rstrip('.'))
#print 'since ' + str(sinceTimestamp) + ' until ' + str(untilTimestamp)
existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
if 'fbok' not in existingIntegration['integrations']: # if Buffer is present and configured
print 'did not find fbok'
raise Exception('Facebook integration not found')
integration = existingIntegration.integrations['fbok']
if integration['access_token'] is None:
raise Exception('Facebook access token not found')
fbok = FacebookPage(integration['access_token'])
if fbok is None:
raise Exception('Facebook Page object could not be created')
definedPages = integration.get('pages', None)
print 'calling pages'
pages = fbok.get_pages()['data']
print 'found FB pages: ' + str(pages)
for page in pages:
page_token = page['access_token']
page_id = page['id']
if not any(d['id'] == page_id for d in definedPages): #if page not defined in company instance, skip it
continue
print 'page id is ' + str(page_id) + ' and token is ' + str(page_token)
page_insights = fbok.get_page_insights(page_id, page_token, sinceTimestamp, untilTimestamp)
#print 'page insights for ' + page['name'] + ': ' + str(page_insights)
page_insights_cleaned = json.loads(json.dumps(page_insights['data']), object_hook = replace_dots)
results = {'page_id': page_id, 'insights': page_insights_cleaned}
saveFbokPageStats(user_id=user_id, company_id=company_id, results=results, job_id=job_id, run_type=run_type)
#saveFbokAdStats(user_id=user_id, company_id=company_id, results=campaigns, job_id=job_id, run_type=run_type)
except Exception as e:
print 'exception was ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def _map_sfdc_userid_name(company_id, input_list):
users = CompanyIntegration.objects(company_id=company_id).aggregate({'$unwind': '$integrations.sfdc.users.records'}, {'$project': {'_id':'$integrations.sfdc.users.records.Id', 'Fname':'$integrations.sfdc.users.records.FirstName', 'Lname':'$integrations.sfdc.users.records.LastName'}}) #{'$match': {'integrations.sfdc.users.records.Id':opp['owner_id']}},
users = list(users)
for item in input_list:
for user in users:
user_found = False
if user['_id'] == item['owner_id']:
item['owner_name'] = str(user['Fname']) + ' ' + str(user['Lname'])
user_found = True
break
if not user_found:
item['owner_name'] = 'Unknown'
return input_list
def delete(self, *args, **kwargs):
queryset = CompanyIntegration.objects(id=self.kwargs['id']).first()
if queryset is not None and self.request.method == 'DELETE':
try:
integration_backup = CompanyIntegrationDeleted()
integration_backup.company_id = queryset.company_id
integration_backup.integrations[self.kwargs['code']] = queryset.integrations[self.kwargs['code']]
integration_backup.save()
queryset.integrations.pop(self.kwargs['code'])
queryset.save()
except Exception as e:
return HttpResponse(str(e), status=status.HTTP_400_BAD_REQUEST)
return HttpResponse('Integration for ' + self.kwargs['code'] + ' deleted', status=status.HTTP_200_OK)
def _get_code(company_id, system_type):
existingIntegration = CompanyIntegration.objects(company_id = company_id ).first()
try:
code = None
if existingIntegration is not None:
for source in existingIntegration.integrations.keys():
defined_system_type = SuperIntegration.objects(Q(code = source) & Q(system_type = system_type)).first()
if defined_system_type is not None:
code = source
#print 'found code' + str(code)
if code is None:
raise ValueError("No integrations defined")
else:
return code
except Exception as e:
return JsonResponse({'Error' : str(e)})
def filterCampaignEmailEventsByType(request, id):
user_id = request.user.id
company_id = request.user.company_id
start_date = request.GET.get('start_date')
end_date = request.GET.get('end_date')
event_type = request.GET.get('event_type')
campaign_guid = request.GET.get('campaign_guid')
email_id = request.GET.get('email_id')
query_type = request.GET.get('query_type')
page_number = int(request.GET.get('page_number'))
items_per_page = int(request.GET.get('per_page'))
system_type = request.GET.get('system_type')
chart_name = request.GET.get('chart_name')
export_type = request.GET.get('export_type')
offset = (page_number - 1) * items_per_page
print ' in filter 22'
user_id = request.user.id
company_id = request.user.company_id
existingIntegration = CompanyIntegration.objects(company_id = company_id ).first()
try:
code = None
if existingIntegration is not None:
for source in existingIntegration.integrations.keys():
defined_system_type = SuperIntegration.objects(Q(code = source) & Q(system_type = system_type)).first()
if defined_system_type is not None:
code = source
client_secret = existingIntegration['integrations'][code]['client_secret']
print 'found code' + str(code)
if code is None:
raise ValueError("No integrations defined")
elif code == 'mkto':
pass
elif code == 'sfdc':
pass
#result = filterLeadsSfdc(user_id=user_id, company_id=company_id, start_date=start_date, end_date=end_date, lead_type=lead_type, query_type=query_type, page_number=page_number, items_per_page=items_per_page, system_type=system_type, offset=offset, code=code)
elif code == 'hspt':
result = filterCampaignEmailEventsByTypeHspt(user_id=user_id, company_id=company_id, start_date=start_date, end_date=end_date, event_type=event_type, email_id=email_id, query_type=query_type, page_number=page_number, items_per_page=items_per_page, system_type=system_type, offset=offset, code=code, campaign_guid=campaign_guid, export_type=export_type)
result['portal_id'] = client_secret
else:
result = 'Nothing to report'
return JsonResponse(result, safe=False)
except Exception as e:
return JsonResponse({'Error' : str(e)})
def retrieveBufrTwInteractions(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
print 'starting retrieveBufrTwInteractions for company ' + str(company_id)
existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
if 'bufr' in existingIntegration['integrations']: # if Buffer is present and configured
print 'found buffer'
client_id = existingIntegration['integrations']['bufr']['client_id']
client_secret = existingIntegration['integrations']['bufr']['client_secret']
access_token = existingIntegration['integrations']['bufr']['access_token']
buffer = Buffer()
api = Buffer.get_api(buffer, client_id=client_id, client_secret=client_secret, access_token=access_token)
profiles = Buffer.get_twitter_profiles(buffer, api)
for profile in profiles:
results = buffer.get_twitter_updates(profile)
saveBufrTwInteractions(user_id=user_id, company_id=company_id, results=results, job_id=job_id, run_type=run_type)
#print 'Tw results are ' + str(results)
else:
print 'No integration found with Buffer'
return JsonResponse({'error' : 'No integration found with Buffer'})
try:
message = 'Twitter interactions retrieved from Buffer'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Social'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def _get_system(company_id=None, system_type=None):
'''Find the appropriate system code e.g. mkto or sfdc for a given system type e.g. ma or crm '''
if company_id is None or system_type is None:
return None
map = Code("function () {"
" for (var key in this.integrations) emit(key, null); } ")
reduce = Code("function (key, values) { return null; } ")
results = CompanyIntegration.objects(company_id=company_id).map_reduce(map, reduce, "inline")
results = list(results)
systems = SuperIntegration.objects(system_type=system_type).only('code')
systems = list(systems)
for system in systems:
for result in results:
if result.key == system['code']:
return system['code']
return None
def retrieveFbokPostStats(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
try:
print 'starting retrieveFbokPostStats for company ' + str(company_id)
existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
if 'fbok' not in existingIntegration['integrations']: # if Buffer is present and configured
print 'did not find fbok'
raise Exception('Facebook integration not found')
integration = existingIntegration.integrations['fbok']
if integration['access_token'] is None:
raise Exception('Facebook access token not found')
fbok = FacebookPage(integration['access_token'])
if fbok is None:
raise Exception('Facebook Page object could not be created')
print 'calling pages'
pages = fbok.get_pages()['data']
print 'found FB pages: ' + str(pages)
for page in pages:
page_token = page['access_token']
page_id = page['id']
posts = fbok.get_posts(page_id, page_token)['data']
print 'got back #posts ' + str(len(posts))
for post in posts:
post_insights = fbok.get_post_insights(post['id'], page_token)
#print 'post insights for ' + post['id'] + ': ' + str(post_insights)
post_insights_cleaned = json.loads(json.dumps(post_insights['data']), object_hook = replace_dots)
results = {'page_id': page_id, 'post_id' : post['id'], 'created_time': post['created_time'], 'insights': post_insights_cleaned}
saveFbokPostStats(user_id=user_id, company_id=company_id, results=results, job_id=job_id, run_type=run_type)
#saveFbokAdStats(user_id=user_id, company_id=company_id, results=campaigns, job_id=job_id, run_type=run_type)
except Exception as e:
print 'exception was ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def retrieveViews(request, company_id):
start_date = request.GET.get('start_date')
end_date = request.GET.get('end_date')
view_name = request.GET.get('view_name')
system_type = request.GET.get('system_type')
user_id = request.user.id
company_id = request.user.company_id
existingIntegration = CompanyIntegration.objects(company_id = company_id ).first()
try:
#first see if generic view i.e. not dependent on a specific system
generic_views = {'contacts': getLeads, 'campaigns': getCampaigns, 'accounts': getAccounts, 'opps': getOpportunities}
if view_name in generic_views:
result = generic_views[view_name](request, company_id)
return result #assume that the view will convert to JSONResponse
else: #if view is dependent on type of system
code = None
if existingIntegration is not None:
for source in existingIntegration.integrations.keys():
defined_system_type = SuperIntegration.objects(Q(code = source) & Q(system_type = system_type)).first()
if defined_system_type is not None:
code = source
#print 'found code' + str(code)
if code is None:
raise ValueError("No integrations defined")
elif code == 'hspt':
pass
#result = retrieveHsptDashboards(user_id=user_id, company_id=company_id, start_date=start_date, end_date=end_date, dashboard_name=dashboard_name)
elif code == 'mkto':
pass
#result = retrieveMktoDashboards(user_id=user_id, company_id=company_id, start_date=start_date, end_date=end_date, dashboard_name=dashboard_name)
else:
result = {'Error': 'No view found'}
return JsonResponse(result, safe=False)
except Exception as e:
return JsonResponse({'Error' : str(e)})
def retrieveMktoActivities(user_id=None, company_id=None, job_id=None, run_type=None, sinceDateTime=None):
#return
try:
print 'getting mkto activities'
#company_id = request.user.company_id
existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
activityTypeIds = []
# if existingIntegration is not None:
# activityTypeArray = existingIntegration.integrations['mkto']['metadata']['activity']
# for i in range(len(activityTypeArray)):
# if (activityTypeArray[i]['name'] == 'Send Email' or activityTypeArray[i]['name'] == 'Email Delivered' or activityTypeArray[i]['name'] == 'Open Email' or activityTypeArray[i]['name'] == 'Visit Webpage' or activityTypeArray[i]['name'] == 'Fill out Form' or activityTypeArray[i]['name'] == 'Click Link' or activityTypeArray[i]['name'] == 'Email Bounced' or activityTypeArray[i]['name'] == 'Email Unsubscribed' or activityTypeArray[i]['name'] == 'Change Data Value'):
# activityTypeIds.append(str(activityTypeArray[i]['id']))
if existingIntegration is not None:
activityTypeArray = existingIntegration.integrations['mkto']['metadata']['activity']
for i in range(len(activityTypeArray)):
activityTypeIds.append(str(activityTypeArray[i]['id']))
if activityTypeIds is None:
return []
if sinceDateTime is None:
sinceDateTime = datetime.now() - timedelta(days=30) #change to 365
mkto = Marketo(company_id)
#get the 'Claritix Lead List' in order to only get activities for leads in that list
listList = mkto.get_lists(id=None , name=['Claritix Leads List'], programName=None, workspaceName=None, batchSize=None)
if listList and listList[0]:
leadListId = listList[0]['id']
else:
raise ValueError('Claritix Leads List not found')
batch_size = 10 #10 Activity Types at a time
activityList = []
for i in range(0, len(activityTypeIds), batch_size):
activityTypeIdsTemp = activityTypeIds[i:i+batch_size]
print 'gettng activities for ' + str(activityTypeIdsTemp)
activityList.extend(mkto.get_lead_activity(activityTypeIdsTemp, sinceDatetime=sinceDateTime, leadListId=leadListId))
#delete leads from lead list in Marketo
deleteList = mkto.get_leads_by_listId(listId=leadListId)
deleteLeadIds = [str(e['id']) for e in deleteList]
print 'leads to be removed from CX List are ' + str(deleteLeadIds)
batch_size = 300
for i in range(0, len(deleteLeadIds), batch_size):
mkto.remove_leads_from_list(listId=leadListId, leadsIds = deleteLeadIds[i:i+batch_size])
print 'leads removed from Mkto CX List'
print 'going to save mkto activities - count ' + str(len(activityList))
saveMktoActivities(user_id=user_id, company_id=company_id, activityList=activityList, activityTypeArray=activityTypeArray, job_id=job_id, run_type=run_type)
try:
message = 'Activities retrieved from Marketo'
notification = Notification()
#notification.company_id = company_id
notification.owner = user_id
notification.module = 'Activities'
notification.type = 'Background task'
notification.method = os.path.basename(__file__)
notification.message = message
notification.success = True
notification.read = False
notification.save()
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
return activityList
except Exception as e:
print 'Error while retrieving activities from Marketo ' + str(e)
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def dailyCronJob():
print 'in cron'
try:
logs = [] #holds all error messages for the job
# first get the superadmin user and the companies
user = _get_superadmin()
if user is None:
mail_admins('Could not find super admin!', 'Check settings')
return # no superadmin found
# remotely login the user
host = settings.BASE_URL
url = host + '/api/v1/auth/login/'
creds = {'email': '*****@*****.**', 'password':'******'}
s = requests.Session()
resp = s.post(url, data=json.dumps(creds))
print 'resp is ' + str(resp.status_code)
if not resp.status_code == 200:
mail_admins('Could not login super admin!', 'Check credentials')
logs.append('Could not login super admin!')
return
else:
logs.append('Superadmin logged in')
cookies = dict(sessionid = resp.cookies['sessionid'])
url = host + '/api/v1/users/'
resp = s.get(url, cookies=cookies)
print 'resp2 is ' + str(resp.status_code)
print str(logs)
querydict = {'company_id__ne' : 0}
companies = Company.objects(**querydict)
print 'found companies ' + str(len(companies))
#now loop through each company find which systems are connected
for company in companies:
company_id = company.company_id
company_name = company.name
print 'in company ' + company.name
existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
if existingIntegration is None: # no integration found so move to next company
logs.append('No integration record for company ' + str(company_name))
continue
else: #skip this company if initial or delta run are in progress
if 'initial_run_in_process' in existingIntegration and existingIntegration['initial_run_in_process'] == True:
logs.append('Initial run in process for company ' + str(company_name))
continue
if 'delta_run_in_process' in existingIntegration and existingIntegration['delta_run_in_process'] == True:
logs.append('Delta run in process for company ' + str(company_name))
continue
# look for either the last delta run date or the last initial run date
sinceDateTime = None
if 'delta_run_done' in existingIntegration:
if 'delta_run_last_date' in existingIntegration:
sinceDateTime = existingIntegration['delta_run_last_date']
if sinceDateTime is None:
if 'initial_run_done' in existingIntegration:
if 'initial_run_last_date' in existingIntegration:
sinceDateTime = existingIntegration['initial_run_last_date']
if sinceDateTime is None:
logs.append('No start date for delta run for company ' + str(company_name))
continue
sinceDateTime = int(time.mktime(time.strptime(_str_from_date(sinceDateTime), '%Y-%m-%dT%H:%M:%SZ')))
sinceDateTime -= int(12*60*60) #move back 12 hours as a safety measure
sinceDateTime = sinceDateTime * 1000
print 'calling extract with ' + str(sinceDateTime)
companyDataExtract(user_id=None, company_id=company_id, run_type='delta', sinceDateTime=sinceDateTime)
except Exception as e:
logs.append(str(e))
print 'exception is ' + str(e)
def dailyCronJob_Deprecated():
try:
logs = [] #holds all error messages for the job
# first get the superadmin user and the companies
user = _get_superadmin()
if user is None:
mail_admins('Could not find super admin!', 'Check settings')
return # no superadmin found
# remotely login the user
host = settings.BASE_URL
url = host + '/api/v1/auth/login/'
creds = {'email': '*****@*****.**', 'password':'******'}
s = requests.Session()
resp = s.post(url, data=json.dumps(creds))
if not resp.status_code == 200:
mail_admins('Could not login super admin!', 'Check credentials')
logs.append('Could not login super admin!')
return
else:
logs.append('Superadmin logged in')
cookies = dict(sessionid = resp.cookies['sessionid'])
url = host + '/api/v1/users/'
resp = s.get(url, cookies=cookies)
companies = Company.objects.filter(~Q(company_id=0))
#now loop through each company find which systems are connected
for company in companies:
company_id = company.company_id
company_name = company.name
existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
if existingIntegration is None: # no integration found so move to next company
continue
#loop through the different source systems and call their respective tasks
for source in existingIntegration.integrations.keys():
if source == 'mkto':
#start by calling Mkto Lead Daily Retrieval (it calls Mkto Activity Daily Retrieval so no need to call Activity again)
# get meta data for activities
url = host + '/api/v1/company/' + str(company_id) + '/integrations/metadata/'
params = {'code': source, 'object': 'activity'}
resp = s.get(url, params=params) # get metadata about activities
if not resp.status_code == 200:
logs.append('Could not retrieve metadata about activities from Marketo for company ' + str(company_name))
continue
else:
logs.append('Retrieved metadata about activities from Marketo for company ' + str(company_name))
# get meta data for leads
params = {'code': source, 'object': 'lead'}
resp = s.get(url, params=params) # get metadata about leads
if not resp.status_code == 200:
logs.append('Could not retrieve metadata about leads from Marketo for company ' + str(company_name))
continue
else:
logs.append('Retrieved metadata about leads from Marketo for company ' + str(company_name))
# get activities and leads
url = host + '/api/v1/company/' + str(company_id) + '/leads/retrieve/daily/'
params = {'code': source}
resp = s.get(url, params=params) # get leads and activities
if not resp.status_code == 200:
logs.append('Could not retrieve activities and leads from Marketo for company ' + str(company_name))
continue
else:
logs.append('Retrieved activities and leads from Marketo for company ' + str(company_name))
#if integrated to Salesforce
if source == 'sfdc':
#get metadata about contact
url = host + '/api/v1/company/' + str(company_id) + '/integrations/metadata/'
params = {'code': source, 'object': 'contact'}
resp = s.get(url, params=params) # get metadata
if not resp.status_code == 200:
logs.append('Could not retrieve contact metadata from SFDC for company ' + str(company_name))
continue
else:
logs.append('Retrieved metadata about contacts from SFDC for company ' + str(company_name))
#get contacts
url = host + '/api/v1/company/' + str(company_id) + '/contacts/retrieve/daily/'
params = {'code': source}
resp = s.get(url, params=params) # get metadata
if not resp.status_code == 200:
logs.append('Could not retrieve contacts from SFDC for company ' + str(company_name))
continue
else:
logs.append('Retrieved contacts from SFDC for company ' + str(company_name))
#get metadata about campaign
url = host + '/api/v1/company/' + str(company_id) + '/integrations/metadata/'
params = {'code': source, 'object': 'campaign'}
resp = s.get(url, params=params) # get metadata
if not resp.status_code == 200:
logs.append('Could not retrieve campaign metadata from SFDC for company ' + str(company_name))
continue
else:
logs.append('Retrieved metadata about campaigns from SFDC for company ' + str(company_name))
#get contacts
url = host + '/api/v1/company/' + str(company_id) + '/campaigns/retrieve/daily/'
params = {'code': source}
resp = s.get(url, params=params) # get metadata
if not resp.status_code == 200:
logs.append('Could not retrieve campaigns from SFDC for company ' + str(company_name))
continue
else:
logs.append('Retrieved campaigns from SFDC for company ' + str(company_name))
# run the daily analytics extract jobs
# timeline chart
url = host + '/api/v1/company/' + str(company_id) + '/analytics/calculate/'
chart_title = 'Timeline'
params = {'chart_name': 'sources_bar', 'system_type': 'MA', 'chart_title':chart_title, 'mode': 'Daily'}
resp = s.get(url, params=params)
if not resp.status_code == 200:
logs.append('Could not retrieve data for ' + chart_title + ' for company ' + str(company_name))
continue
else:
logs.append('Retrieved data for ' + chart_title + ' for company ' + str(company_name))
# pipeline duration chart
chart_title = 'Pipeline Duration'
params = {'chart_name': 'pipeline_duration', 'system_type': 'MA', 'chart_title':chart_title, 'mode': 'Daily'}
resp = s.get(url, params=params)
if not resp.status_code == 200:
logs.append('Could not retrieve data for ' + chart_title + ' for company ' + str(company_name))
continue
else:
logs.append('Retrieved data for ' + chart_title + ' for company ' + str(company_name))
# contacts distribution chart
chart_title = 'Contacts Distribution'
params = {'chart_name': 'contacts_distr', 'system_type': 'MA', 'chart_title':chart_title, 'mode': 'Daily'}
resp = s.get(url, params=params)
if not resp.status_code == 200:
logs.append('Could not retrieve data for ' + chart_title + ' for company ' + str(company_name))
continue
else:
logs.append('Retrieved data for ' + chart_title + ' for company ' + str(company_name))
# source pie chart
chart_title = 'Source Breakdown'
params = {'chart_name': 'source_pie', 'system_type': 'MA', 'chart_title':chart_title, 'mode': 'Daily'}
resp = s.get(url, params=params)
if not resp.status_code == 200:
logs.append('Could not retrieve data for ' + chart_title + ' for company ' + str(company_name))
continue
else:
logs.append('Retrieved data for ' + chart_title + ' for company ' + str(company_name))
# revenue source pie chart
chart_title = 'Revenue by Source'
params = {'chart_name': 'revenue_source_pie', 'system_type': 'MA', 'chart_title':chart_title, 'mode': 'Daily'}
resp = s.get(url, params=params)
if not resp.status_code == 200:
logs.append('Could not retrieve data for ' + chart_title + ' for company ' + str(company_name))
continue
else:
logs.append('Retrieved data for ' + chart_title + ' for company ' + str(company_name))
mail_admins('Daily extract job completed', '\n'.join(str(elem) for elem in logs))
print 'Daily extract job completed'
except Exception as e:
logs.append(str(e))
def saveMktoActivitiesToMaster(user_id=None, company_id=None, job_id=None, run_type=None):
#job_id = ObjectId("56a2dd408afb006f9e7cb851")
if run_type == 'initial':
#activities = TempData.objects(Q(company_id=company_id) & Q(record_type='activity') & Q(source_system='mkto') & Q(job_id=job_id) ).only('source_record')
collection = TempData._get_collection()
activities = collection.find({'company_id': int(company_id), 'record_type': 'activity', 'source_system': 'mkto', 'job_id': job_id}, projection={'source_record': True}, batch_size=1000)
else:
collection = TempDataDelta._get_collection()
activities = collection.find({'company_id': int(company_id), 'record_type': 'activity', 'source_system': 'mkto', 'job_id': job_id}, projection={'source_record': True}, batch_size=1000)
#activities = TempDataDelta.objects(Q(company_id=company_id) & Q(record_type='activity') & Q(source_system='mkto') & Q(job_id=job_id) ).only('source_record')
# activityListTemp = list(activities)
# activityList = [i['source_record'] for i in activityListTemp]
existingIntegration = CompanyIntegration.objects(company_id = company_id).first()
if existingIntegration is not None:
activityTypeArray = existingIntegration.integrations['mkto']['metadata']['activity']
else:
print 'No activity type metadata found for Marketo'
raise ValueError('No activity type metadata found for Marketo')
try:
for i in range(len(activityTypeArray)):
if activityTypeArray[i]['name'] == 'Change Data Value':
changeActivityId = activityTypeArray[i]['id']
print 'change id is ' + str(changeActivityId)
for activity in activities:
newActivity = activity['source_record']
addThisActivity = True
#company_id = request.user.company_id
mkto_id = str(newActivity['leadId'])
print 'doing lead ' + mkto_id
existingLead = Lead.objects(Q(mkto_id = str(mkto_id)) & Q(company_id = company_id)).first()
if existingLead is not None: # we found this lead to attach the activities
if 'mkto' in existingLead.activities:
currentActivities = existingLead.activities['mkto']
for i in range(len(currentActivities)):
if currentActivities[i]['id'] == newActivity['id']: #check if this activity already exists in the lead dict
addThisActivity = False
break
if addThisActivity == True:
for i in range(len(activityTypeArray)):
if activityTypeArray[i]['id'] == newActivity['activityTypeId']:
newActivity['activityTypeName'] = activityTypeArray[i]['name']
break
currentActivities.append(newActivity)
existingLead.update(activities__mkto = currentActivities)
else:
currentActivities = []
for i in range(len(activityTypeArray)):
if activityTypeArray[i]['id'] == newActivity['activityTypeId']:
newActivity['activityTypeName'] = activityTypeArray[i]['name']
break
currentActivities.append(newActivity)
existingLead.update(activities__mkto = currentActivities)
#addThisActivity == True and
if addThisActivity == True and newActivity['activityTypeId'] == changeActivityId and newActivity['primaryAttributeValue'] == 'Lead Status':
print 'processing status activity for id ' + mkto_id
#statusRecord = [];
newStatus = ''
oldStatus = ''
for attribute in newActivity['attributes']:
if attribute['name'] == 'New Value':
newStatus = attribute['value']
elif attribute['name'] == 'Old Value':
oldStatus = attribute['value']
# elif attribute['name'] == 'Reason':
# reason = attribute['value']
#break
#statusRecord.append({'status': newStatus, 'date': newActivity['activityDate']})
newActivity['newStatus'] = newStatus
newActivity['oldStatus'] = oldStatus
newActivity['date'] = newActivity['activityDate']
if 'mkto' in existingLead.statuses:
currentStatuses = existingLead.statuses['mkto']
currentStatuses.append(newActivity) # changed on 1/22/2016 {'status': newStatus, 'date': newActivity['activityDate']})
existingLead.update(statuses__mkto = currentStatuses)
else:
currentStatuses = []
currentStatuses.append(newActivity) # changed on 1/22/2016{'status': newStatus, 'date': newActivity['activityDate']})
existingLead.update(statuses__mkto = currentStatuses)
# if addThisActivity == True: # this activity was not foudn in the lead, so add it
# existingLead.activities['mkto'].append(newActivity)
# existingLead.save() # no concept of saving the activity if the lead does not exist
except Exception as e:
send_notification(dict(
type='error',
success=False,
message=str(e)
))
def companyDataExtract(user_id=None, company_id=None, run_type=None, sinceDateTime=None):
superJobMonitor = None
existingIntegration = None
if run_type is None or company_id is None or sinceDateTime is None:
return
try:
sinceDateTime = datetime.fromtimestamp(float(sinceDateTime) / 1000)
#sinceDateTime = datetime.now() - timedelta(days=30)
print 'start date is ' + str(sinceDateTime)
print 'run type is ' + run_type
print 'company id is ' + str(company_id)
#create an entry in the Job Monitor
superJobMonitor = SuperJobMonitor(company_id=company_id, type=run_type, started_date=datetime.utcnow())
superJobMonitor.save()
#do pre-check
_superJobMonitorAddTask(superJobMonitor, "Claritix", "Pre-check started")
# get the integration record
existingIntegration = CompanyIntegration.objects(company_id=company_id).first()
if existingIntegration is None:
_superJobMonitorEnd(superJobMonitor, None, 'Failed', 'No integration record found')
mail_admins('Could not find integration record for company ' + company_id , 'Check settings')
return False
if run_type == 'initial':
existingIntegration['initial_run_in_process'] = True #set the flag
else: #delta
existingIntegration['delta_run_in_process'] = True #set the flag
existingIntegration.save() # save the flag
# set up the Request and Cookie
user = _get_superadmin()
if user is None:
_superJobMonitorEnd(superJobMonitor, existingIntegration, 'Failed', 'No super admin found')
mail_admins('Could not find super admin!', 'Check settings')
return False
# remotely login the user
host = settings.BASE_URL
url = host + '/api/v1/auth/login/'
creds = {'email': '*****@*****.**', 'password':'******'}
s = requests.Session()
resp = s.post(url, data=json.dumps(creds))
if not resp.status_code == 200:
_superJobMonitorEnd(superJobMonitor, existingIntegration, 'Failed', 'Could not login super admin!')
mail_admins('Could not login super admin!', 'Check credentials')
return False
#do cookie thing - refer to SuperAdmin Cron Job Task for details
cookies = dict(sessionid = resp.cookies['sessionid'])
url = host + '/api/v1/users/'
resp = s.get(url, cookies=cookies)
_superJobMonitorAddTask(superJobMonitor, "Claritix", "Pre-check completed")
# #delete data in Lead Master
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of Leads, Contacts, Opportunities and Activities started")
# count = Lead.objects(company_id=company_id).count()
# Lead.objects(company_id=company_id).delete()
# _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " Leads, Contacts, Opportunities and Activities deleted")
#
# #delete data in Campaign Master
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of Campaigns started")
# count = Campaign.objects(company_id=company_id).count()
# Campaign.objects(company_id=company_id).delete()
# _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " Campaigns deleted")
#
# #delete data in Account Master
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of Accounts started")
# count = Account.objects(company_id=company_id).count()
# Account.objects(company_id=company_id).delete()
# _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " Accounts deleted")
#
# find out which systems are integrated and therefore which tasks should be run
task_map = {
#"mkto" : [retrieveMktoCampaigns, retrieveMktoLeadsByProgram], #retrieveMktoLeadCreatedActivities, retrieveMktoLeads, retrieveMktoActivities, retrieveMktoCampaigns, retrieveMktoLeadsByProgram], #IMPORTANT - Lead Created Activities has to be before Leads
#"hspt" : [retrieveHsptCampaigns], # , ], # retrieveHsptLeads, retrieveHsptOpportunities, retrieveHsptWebsiteTraffic, ,
"prdt" : [retrievePrdtLeads],
# "sfdc" : [retrieveSfdcCampaigns] #retrieveSfdcLeads, retrieveSfdcContacts, retrieveSfdcCampaigns, retrieveSfdcAccounts, retrieveSfdcOpportunities, retrieveSfdcLeadHistory, retrieveSfdcContactHistory, retrieveSfdcOppHistory, retrieveSfdcOppStageHistory],
# "sugr" : [retrieveSugrLeads],
# "bufr" : [retrieveBufrTwInteractions],
# "goog" : [retrieveGoogWebsiteTraffic], \
# "fbok" : [retrieveFbokPageStats, retrieveFbokAdStats]#, retrieveFbokPostStats] # , ]
}
# # for future use - retrieveMktoContacts, retrieveMktoOpportunities, retrieveSfdcActivities,
final_task_map = {
# "mkto" : [saveMktoCampaignsToMaster, saveMktoLeadsByProgramToMaster], #saveMktoLeadsToMaster, saveMktoActivitiesToMaster, saveMktoCampaignsToMaster, saveMktoLeadsByProgramToMaster],#mergeMktoSfdcLeads, deleteLeads, deleteDuplicateMktoIdLeads #
#"hspt" : [saveHsptCampaignsToMaster ], #saveHsptLeadsToMaster, saveHsptOpportunitiesToMaster, saveHsptWebsiteTrafficToMaster, , ], # ,
# "sfdc" : [saveSfdcCampaignsToMaster], #saveSfdcLeadsToMaster, saveSfdcContactsToMaster, saveSfdcCampaignsToMaster, saveSfdcAccountsToMaster, saveSfdcOpportunitiesToMaster, saveSfdcLeadHistoryToMaster, saveSfdcContactHistoryToMaster, saveSfdcOppHistoryToMaster, saveSfdcOppStageHistoryToMaster], #
# "sugr" : [saveSugrLeadsToMaster],
# "bufr" : [saveBufrTwInteractionsToMaster],
# "goog": [saveGoogleWebsiteTrafficToMaster],
# "fbok": [saveFbokPageStatsToMaster, saveFbokAdStatsToMaster]#, saveFbokPostStatsToMaster] #
}
# #
# # #saveSfdcLeadsToMaster, saveSfdcContactsToMaster, saveSfdcOpportunitiesToMaster, saveSfdcCampaignsToMaster,
# # # saveSfdcLeadsToMaster, saveSfdcContactsToMaster, saveSfdcOpportunitiesToMaster, saveSfdcCampaignsToMaster, saveSfdcAccountsToMaster
# # #collect all relevant tasks in one list and retrieve metadata as well
for source in existingIntegration.integrations.keys():
#change metadata depending on source system
if source == 'sfdc':
metadata_objects = ['user', 'lead', 'contact', 'campaign', 'opportunity', 'task', 'account'] #[] #objects for which metadata should be collected
elif source == 'mkto':
metadata_objects = ['lead', 'activity', 'campaign'] #[] #objects for which metadata should be collected
elif source == 'hspt':
metadata_objects = ['lead'] #objects for which metadata should be collected
else:
metadata_objects = [] #objects for which metadata should be collected
# if sfdc, explicitly refresh the access token
if source == 'sfdc':
sfdc = Salesforce()
sfdc.refresh_token(company_id)
#collect meta data
url = host + '/api/v1/company/' + str(company_id) + '/integrations/metadata/'
# for object in metadata_objects:
# _superJobMonitorAddTask(superJobMonitor, source, "Retrieval of metadata for " + object + " started")
# params = {'code': source, 'object': object}
# resp = s.get(url, params=params) # get metadata about activities
# if not resp.status_code == 200:
# _superJobMonitorAddTask(superJobMonitor, source, "Retrieval of metadata for " + object + " failed")
# continue
# else:
# _superJobMonitorAddTask(superJobMonitor, source, "Retrieval of metadata for " + object + " completed")
#collect retrieval tasks
print 'starting retrieval tasks'
tasks = []
if source in task_map:
tasks.extend(task_map[source])
#now run the tasks
for task in tasks:
_superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " started to store temp data")
print 'starting task ' + str(task)
task(user_id=user_id, company_id=company_id, job_id=superJobMonitor.id, run_type=run_type, sinceDateTime=sinceDateTime)
_superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " completed")
print 'starting save tasks'
#collect save tasks
tasks = []
if source in final_task_map:
tasks.extend(final_task_map[source])
#now run the tasks
for task in tasks:
_superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " started to save master data")
print 'starting task ' + str(task)
task(user_id=user_id, company_id=company_id, job_id=superJobMonitor.id, run_type=run_type)
_superJobMonitorAddTask(superJobMonitor, source, task.__name__ + " completed")
#return #REMOVE THIS IN PRODUCTION
#if initial run, delete all analytics data
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, 'Claritix', "Deletion of analytics data started")
# count1 = AnalyticsData.objects(company_id=company_id).count() #ensure that website_traffic chart data is not deleted
# AnalyticsData.objects(company_id=company_id).delete() #ensure that website_traffic chart data is not deleted
# count2 = AnalyticsIds.objects(company_id=company_id).count()
# AnalyticsIds.objects(company_id=company_id).delete()
# _superJobMonitorAddTask(superJobMonitor, 'Claritix', str(count1 + count2) + " records deleted from analytics tables")
#
# call chart calculate tasks
charts = [\
# {'chart_name': 'sources_bar', 'system_type': 'MA', 'chart_title':'Timeline', 'mode': run_type, 'start_date': sinceDateTime}, \
# #{'chart_name': 'pipeline_duration', 'system_type': 'MA', 'chart_title':'Pipeline Duration', 'mode': run_type, 'start_date': sinceDateTime}, \
# #{'chart_name': 'contacts_distr', 'system_type': 'MA', 'chart_title':'Contacts Distribution', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'source_pie', 'system_type': 'MA', 'chart_title':'Source Distribution', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'revenue_source_pie', 'system_type': 'MA', 'chart_title':'Revenue by Source', 'mode': run_type, 'start_date': sinceDateTime}, \
# #{'chart_name': 'multichannel_leads', 'system_type': 'MA', 'chart_title':'Multichannel Leads', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'tw_performance', 'system_type': 'SO', 'chart_title':'Twitter Performance', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'fb_performance', 'system_type': 'SO', 'chart_title':'Facebook_Performance', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'google_analytics', 'system_type': 'AD', 'chart_title':'Google Analytics', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'campaign_email_performance', 'system_type': 'MA', 'chart_title':'Campaign Performance by Email', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'email_cta_performance', 'system_type': 'MA', 'chart_title':'Email Performance by CTA', 'mode': run_type, 'start_date': sinceDateTime}, \
#
]
url = host + '/api/v1/company/' + str(company_id) + '/analytics/calculate/'
for chart in charts:
print 'starting chart ' + str(chart['chart_title'])
_superJobMonitorAddTask(superJobMonitor, 'Claritix', "Started calculating " + str(chart['chart_title']))
resp = s.get(url, params=chart)
if not resp.status_code == 200:
print 'incorrect status code was ' + str(resp.status_code)
_superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Could not retrieve data for ' + chart['chart_title'])
continue
else:
_superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Retrieved data for ' + chart['chart_title'])
# call dashboard calculate tasks
dashboards = [\
# {'chart_name': 'social_roi', 'system_type': 'MA', 'chart_title':'Social Performance', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'funnel', 'system_type': 'MA', 'chart_title':'Funnel', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'opp_funnel', 'system_type': 'CRM', 'chart_title':'Opportunity Funnel', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'waterfall_chart', 'system_type': 'MA', 'chart_title':'Waterfall Chart', 'mode': run_type, 'start_date': sinceDateTime}, \
# {'chart_name': 'form_fills', 'system_type': 'MA', 'chart_title':'Form Fills', 'mode': run_type, 'start_date': sinceDateTime}, \
]
url = host + '/api/v1/company/' + str(company_id) + '/dashboards/calculate/'
for dashboard in dashboards:
print 'starting dashboard ' + str(dashboard['chart_title'])
_superJobMonitorAddTask(superJobMonitor, 'Claritix', "Started calculating " + str(dashboard['chart_title']))
resp = s.get(url, params=dashboard)
if not resp.status_code == 200:
print 'incorrect status code was ' + str(resp.status_code)
_superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Could not retrieve data for ' + dashboard['chart_title'])
continue
else:
_superJobMonitorAddTask(superJobMonitor, 'Claritix', 'Retrieved data for ' + dashboard['chart_title'])
#delete all data from past successful initial runs in Temp Table
# if run_type == 'initial':
# _superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of temp data table started")
# successfulJobs = SuperJobMonitor.objects(Q(company_id=company_id) & Q(type='initial') & Q(status='Completed'))
# successfulJobsListTemp = list(successfulJobs)
# print 'found job ids ' + str(len(successfulJobsListTemp))
# successfulJobsList = [i.id for i in successfulJobsListTemp]
# count = TempData.objects(job_id__in=successfulJobsList).count()
# TempData.objects(job_id__in=successfulJobsList).delete()
# _superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " records deleted from temp data table")
#delete all data from past successful delta runs in Temp Table
if run_type == 'delta':
_superJobMonitorAddTask(superJobMonitor, "Claritix", "Deletion of temp data delta table started")
successfulJobs = SuperJobMonitor.objects(Q(company_id=company_id) & Q(type='delta') & Q(status='Completed'))
successfulJobsListTemp = list(successfulJobs)
print 'found delta job ids ' + str(len(successfulJobsListTemp))
successfulJobsList = [i.id for i in successfulJobsListTemp]
count = TempDataDelta.objects(job_id__in=successfulJobsList).count()
TempDataDelta.objects(job_id__in=successfulJobsList).delete()
_superJobMonitorAddTask(superJobMonitor, "Claritix", str(count) + " records deleted from temp data delta table")
#update last run date for initial run in Company Integration record
_superJobMonitorEnd(superJobMonitor, existingIntegration, run_type, 'Completed', 'All tasks completed successfully')
return True
except Exception as e:
if superJobMonitor is not None and existingIntegration is not None:
_superJobMonitorEnd(superJobMonitor, existingIntegration, run_type, 'Failed', str(e))
print str(e)
return False
def getSuperFilters(request, company_id):
user_id = request.user.id
company_id = request.user.company_id
object_type = request.GET.get('object_type')
system_type = request.GET.get('system_type')
#existingIntegration = CompanyIntegration.objects(company_id = company_id ).first()
existingIntegration = {}
existingIntegration['integrations'] = {}
map = Code("function () {"
" for (var key in this.integrations) emit(key, this.integrations[key]['filters']); } ")
reduce = Code("function (key, values) { return null; } ")
results = CompanyIntegration.objects(company_id=company_id).map_reduce(map, reduce, "inline")
results = list(results)
for result in results:
existingIntegration['integrations'][result.key] = {'filters': result.value}
try:
code = None
if existingIntegration is not None:
for source in existingIntegration['integrations'].keys():
defined_system_type = SuperIntegration.objects(Q(code = source) & Q(system_type = system_type)).first()
if defined_system_type is not None:
code = source
print 'found code' + str(code)
if code is None:
raise ValueError("No integrations defined")
else:
super_filters = SuperFilters.objects(source_system = code).first()
if super_filters is None:
result = []
else:
if object_type not in super_filters['filters']:
result = []
else:
temp = super_filters['filters'].get(object_type, None)
filters = existingIntegration['integrations'][code].get('filters', None)
if filters is not None:
filter_obj = filters.get(object_type, None)
if filter_obj is None:
return JsonResponse({'results': temp}, safe=False)
for filter, values in filter_obj.items():
if filter in temp:
if filter == 'OwnerId': #reduce the users list to only those with opportunities
temp_values = {}
opp_field_qry = 'opportunities__sfdc__exists'
company_field_qry = 'company_id'
projection = {'$project': {'owner_id': '$opportunities.sfdc.OwnerId' } }
querydict = {opp_field_qry: True, company_field_qry: company_id}
opps = Account.objects(**querydict).aggregate({'$unwind': '$opportunities.sfdc'}, projection)
opps = list(opps)
opps_owner_ids = [opp['owner_id'] for opp in opps]
print 'opp owner ids ' + str(opps_owner_ids)
tempValues = [value for value in values['values'] if value['value'] in opps_owner_ids]
print 'temp values2 is ' + str(tempValues)
temp_values['values'] = tempValues
temp_values['label'] = values['label']
values = temp_values
values['values'].sort()
temp[filter] = values
result = {'results': temp}
#result = 'Nothing to report'
return JsonResponse(result, safe=False)
except Exception as e:
return JsonResponse({'Error' : str(e)})
def getCampaigns(request, id):
try:
company_id = request.user.company_id
system_type = request.GET.get('system_type')
page_number = int(request.GET.get('page_number'))
items_per_page = int(request.GET.get('per_page'))
start_date = int(request.GET.get('start_date'))
end_date = int(request.GET.get('end_date'))
sub_view = request.GET.get('subview')
filters = request.GET.get('filters')
filters = json.loads(filters)
superfilters = request.GET.get('superfilters')
super_filters = json.loads(superfilters)
#print 'super filters are ' + str(super_filters)
date_field = None
querydict_filters = {}
offset = (page_number - 1) * items_per_page
# existingIntegration = CompanyIntegration.objects(company_id = company_id ).first()
# code = None
# if existingIntegration is not None:
# for source in existingIntegration.integrations.keys():
# defined_system_type = SuperIntegration.objects(Q(code = source) & Q(system_type = system_type)).first()
# if defined_system_type is not None:
# code = source
#
# if code is None:
# return JsonResponse({'Error' : 'Marketing Automation system not found'})
#projection = {'$project': {'_id': '$campaigns.' + code + '.id', 'created_date': '$campaigns.' + code + '.createdAt', 'name': '$campaigns.' + code + '.name', 'description': '$campaigns.' + code + '.description', 'url': '$campaigns.' + code + '.url', 'type': '$campaigns.' + code + '.type', 'channel': '$campaigns.' + code + '.channel', } }
match = {'$match' : { }}
date_field = None
collection = Campaign._get_collection()
company_field_qry = 'company_id'
querydict = {company_field_qry: company_id}
code = ''
if super_filters is not None:
if 'date_types' in super_filters: # need to filter by a certain type of date
date_field = super_filters['date_types']
if date_field is not None:
if start_date is not None:
start_date = datetime.fromtimestamp(float(start_date) / 1000)
if end_date is not None:
end_date = datetime.fromtimestamp(float(end_date) / 1000)
local_start_date = get_current_timezone().localize(start_date, is_dst=None)
utc_day_start = local_start_date.astimezone(pytz.timezone('UTC'))
utc_day_start_string = datetime.strftime(utc_day_start, '%Y-%m-%dT%H:%M:%SZ+0000')
utc_day_start_string_crm = datetime.strftime(utc_day_start, '%Y-%m-%dT%H:%M:%S.000+0000')
local_end_date = get_current_timezone().localize(end_date, is_dst=None)
utc_day_end = local_end_date.astimezone(pytz.timezone('UTC'))
utc_day_end_string = datetime.strftime(utc_day_end, '%Y-%m-%dT%H:%M:%SZ+0000')
utc_day_end_string_crm = datetime.strftime(utc_day_end, '%Y-%m-%dT%H:%M:%S.000+0000')
#print 'utc start string is ' + str(utc_day_start_string)
#print 'utc end string is ' + str(utc_day_end_string)
#remove the date_types item
#super_filters.pop('date_types')
date_field_original = date_field
date_field = date_field.replace('.', '__')
date_field_start_qry = date_field + '__gte'
date_field_end_qry = date_field + '__lte'
if filters is not None:
for key, value in filters.items():
if value is not None and value != '':
querydict_filters['campaigns__' + code + '__' + key] = value #creates an additional querydict that can be added to the main qd
match['$match']['campaigns.' + code + '.' + key] = value
if sub_view == 'allcampaigns':
if date_field is None:
total = collection.find({'company_id': int(company_id)}).count() #.hint('company_id_1')
queryset = Campaign.objects(**querydict).skip(offset).limit(items_per_page)
else:
total = collection.find({'company_id': int(company_id), date_field_original: {'$gte':utc_day_start_string, '$lte':utc_day_end_string}}).count() #.hint('company_id_1')
querydict[date_field_start_qry] = utc_day_start_string
querydict[date_field_end_qry] = utc_day_end_string
queryset = Campaign.objects(**querydict).skip(offset).limit(items_per_page)
elif sub_view == 'onlyma' or sub_view == 'onlycrm':
if sub_view == 'onlyma':
code = _get_system(company_id, 'MA')
else:
code = _get_system(company_id, 'CRM')
if code is None:
return JsonResponse({'Error' : 'No source system found'})
querydict['source_system'] = code
if date_field is None:
total = collection.find({'company_id': int(company_id), 'source_system': code}).count() #.hint('company_id_1')
queryset = Campaign.objects(**querydict).skip(offset).limit(items_per_page)
else:
if sub_view == 'onlycrm' and code == 'sfdc':
if date_field_original == 'campaigns.mkto.createdAt':
date_field_original = 'campaigns.sfdc.CreatedDate'
date_field_start_qry = 'campaigns__sfdc__CreatedDate__gte'
date_field_end_qry = 'campaigns__sfdc__CreatedDate__lte'
elif date_field_original == 'campaigns.mkto.updatedAt':
date_field_original = 'campaigns.sfdc.LastModifiedDate'
date_field_start_qry = 'campaigns__sfdc__LastModifiedDate__gte'
date_field_end_qry = 'campaigns__sfdc__LastModifiedDate__lte'
total = collection.find({'company_id': int(company_id), 'source_system': code, date_field_original: {'$gte':utc_day_start_string, '$lte':utc_day_end_string}}).count() #.hint('company_id_1')
querydict[date_field_start_qry] = utc_day_start_string
querydict[date_field_end_qry] = utc_day_end_string
queryset = Campaign.objects(**querydict).skip(offset).limit(items_per_page)
elif sub_view == 'crmfromma' or sub_view == 'crmnotma':
code = _get_system(company_id, 'CRM')
if code is None:
return JsonResponse({'Error' : 'No source system found'})
querydict['source_system'] = code
mapping = CompanyIntegration.objects(company_id=company_id).only('mapping').first()
print 'mapping is ' + str(mapping)
if mapping is None or len(mapping) == 0:
return JsonResponse({'Error' : 'No mapping found in company settings'})
ma_user = None
ma_code = _get_system(company_id, 'MA')
if ma_code == 'mkto':
ma_user = mapping['mapping'].get('mkto_sync_user', None)
if ma_user is None or ma_code is None:
return JsonResponse({'Error' : 'No marketing automation details found'})
if code == 'sfdc':
if sub_view == 'crmfromma':
user_field_qry = 'campaigns.sfdc.CreatedById'
querydict['campaigns__sfdc__CreatedById'] = ma_user
else:
user_field_qry = 'campaigns.sfdc.CreatedById__ne'
querydict['campaigns__sfdc__CreatedById__ne'] = ma_user
if date_field is None:
total = collection.find({'company_id': int(company_id), 'source_system': code, user_field_qry: ma_user}).count() #.hint('company_id_1')
queryset = Campaign.objects(**querydict).skip(offset).limit(items_per_page)
else:
if date_field_original == 'campaigns.mkto.createdAt':
date_field_original = 'campaigns.sfdc.CreatedDate'
date_field_start_qry = 'campaigns__sfdc__CreatedDate__gte'
date_field_end_qry = 'campaigns__sfdc__CreatedDate__lte'
elif date_field_original == 'campaigns.mkto.updatedAt':
date_field_original = 'campaigns.sfdc.LastModifiedDate'
date_field_start_qry = 'campaigns__sfdc__LastModifiedDate__gte'
date_field_end_qry = 'campaigns__sfdc__LastModifiedDate__lte'
total = collection.find({'company_id': int(company_id), 'source_system': code, user_field_qry: ma_user, date_field_original: {'$gte':utc_day_start_string, '$lte':utc_day_end_string}}).count() #.hint('company_id_1')
querydict[date_field_start_qry] = utc_day_start_string
querydict[date_field_end_qry] = utc_day_end_string
queryset = Campaign.objects(**querydict).skip(offset).limit(items_per_page)
serializer = CampaignSerializer(queryset, many=True)
type = 'campaigns'
return JsonResponse({'count' : total, 'results': serializer.data, 'type': type})
except Exception as e:
return JsonResponse({'Error' : str(e)})
def saveSfdcContactsToMaster(user_id=None, company_id=None, job_id=None, run_type=None):
# delete later
# job_id = ObjectId("569adcfc8afb00205c799f28")
if run_type == "initial":
contacts = TempData.objects(
Q(company_id=company_id) & Q(record_type="contact") & Q(source_system="sfdc") & Q(job_id=job_id)
).only(
"source_record"
) # & Q(job_id=job_id)
else:
contacts = TempDataDelta.objects(
Q(company_id=company_id) & Q(record_type="contact") & Q(source_system="sfdc") & Q(job_id=job_id)
).only(
"source_record"
) # & Q(job_id=job_id)
print "co id is " + str(company_id)
contactListTemp = list(contacts)
contactList = [i["source_record"] for i in contactListTemp]
# print 'saving sfdc contacts'
try:
# get the custom field for Contact Status, if it exists
existingIntegration = CompanyIntegration.objects(company_id=company_id).first()
contact_status = None
if "sfdc" in existingIntegration["integrations"]:
contact_status = existingIntegration["mapping"].get("sfdc_contact_status", None)
for newContact in contactList: # ['records']:
# company_id = request.user.company_id
sfdc_contact_Id = str(newContact["Id"])
print "contact id is " + sfdc_contact_Id
# sfdc_mkto_id = str(newLead['sfdcLeadId']) #check if there is a corresponding lead from MKTO
existingLeadMkto = None
existingLeadSfdc = None
existingLeadHspt = None
existingContact = Lead.objects(Q(company_id=company_id) & Q(sfdc_contact_id=sfdc_contact_Id)).first()
if existingContact is not None: # we found this contact already in the DB
print "found contact match for " + str(sfdc_contact_Id)
if "sfdc" in existingContact.contacts:
existingContact.source_first_name = newContact["FirstName"]
existingContact.source_last_name = newContact["LastName"]
existingContact.source_email = newContact["Email"]
# existingContact.source_created_date = str(newContact['CreatedDate'])
existingContact.source_source = newContact["LeadSource"]
if contact_status is not None and contact_status in newContact:
existingContact.source_status = newContact[contact_status]
existingContact.contacts["sfdc"] = newContact
existingContact.save()
# Lead.objects(Q(company_id=company_id) & Q(sfdc_contact_id=sfdc_contact_Id)).update(contacts__sfdc=newContact)
else:
existingContact.contacts["sfdc"] = {}
existingContact.contacts["sfdc"] = newContact
existingContact.save()
# elif existingContact is None: # this lead does not exist
else:
existingLeadSfdc = (
Lead.objects(Q(company_id=company_id) & Q(leads__sfdc__ConvertedContactId=sfdc_contact_Id))
.hint("company_id_1_leads.sfdc.convertedContactId_1")
.first()
)
if existingLeadSfdc is not None:
print "found match for sfdc lead for contact " + str(sfdc_contact_Id)
# existingLeadSfdcList = list(existingLeadSfdc)
existingLeadSfdc.sfdc_contact_id = sfdc_contact_Id
if contact_status is not None and contact_status in newContact:
existingLeadSfdc.source_status = newContact[contact_status]
existingLeadSfdc.contacts = {}
existingLeadSfdc.contacts["sfdc"] = newContact
existingLeadSfdc.save()
# remove below comments after figuring out how Mkto stored SFDC contact ID
else:
existingLeadMkto = Lead.objects(
Q(company_id=company_id) & Q(leads__mkto__sfdcContactId=sfdc_contact_Id)
).first()
if (
existingLeadMkto is not None
): # we found a MKto lead record which is matched to this new Sfdc lead
print "found mkto lead" + existingLeadMkto.mkto_id
existingLeadMkto.sfdc_contact_id = sfdc_contact_Id
# existingLeadMkto.contacts = {}
existingLeadMkto.contacts["sfdc"] = newContact
existingLeadMkto.save()
else:
existingLeadHspt = Lead.objects(
Q(company_id=company_id) & Q(leads__hspt__properties__salesforcecontactid=sfdc_contact_Id)
).first()
if (
existingLeadHspt is not None
): # we found a MKto lead record which is matched to this new Sfdc lead
existingLeadHspt.sfdc_contact_id = sfdc_contact_Id
existingLeadHspt.contacts = {}
existingLeadHspt.contacts["sfdc"] = newContact
existingLeadHspt.save()
if (
existingLeadSfdc is None
and existingLeadMkto is None
and existingLeadHspt is None
and existingContact is None
): # no matches found so save new record
lead = Lead()
lead.sfdc_contact_id = sfdc_contact_Id
lead.company_id = company_id
lead.source_first_name = newContact["FirstName"]
lead.source_last_name = newContact["LastName"]
lead.source_email = newContact["Email"]
lead.source_created_date = str(newContact["CreatedDate"])
lead.source_source = newContact["LeadSource"]
if contact_status is not None and contact_status in newContact:
lead.source_status = newContact[contact_status]
lead.contacts = {}
lead.contacts["sfdc"] = newContact
lead.save()
# lead = Lead()
# company_id = request.user.company_id
# derived_id = 'sfdc_' + str(newLead['Id'])
# Lead.objects(derived_id = derived_id).modify(upsert=True, new=True, set__leads__sfdc = newLead, set_on_insert__derived_id = derived_id, set_on_insert__company_id = company_id)
#
# oldLead = Lead.objects(derived_id = lead.derived_id)
# if oldLead.count() == 0:
# lead.leads["sfdc"] = newLead
# lead.save()
# else:
# oldLead.leads["sfdc"] = newLead
# Lead.objects(derived_id = lead.derived_id).update(oldLead)
except Exception as e:
print "exception while saving SFDC contact " + str(e)
send_notification(dict(type="error", success=False, message=str(e)))