def recall_dsp(project,device_id,created_at,ids):
history_count = find_recall_history(project=project,device_id=device_id,created_at=created_at)
if history_count == 0 or admin.aso_dsp_callback_repeat is True:
results,count = find_recall_url(project=project,device_id=device_id,created_at=created_at)
for result in results:
all_json = {}
result_json = 'no_callback_url'
if result[0] and result[0]!='':
result_json = get_json_from_api(url=result[0].replace('"','').replace('{{ATYPE}}','activate').replace('{{AVALUE}}','0').replace('\\u0026','&'))
org_distinct_id = result[2]
all_json['src'] = json.loads(result[1])
all_json['org_distinct_id'] = org_distinct_id
all_json['ids'] = ids
all_json['recall_result'] = result_json
insert_count = insert_event(table=project,alljson=json.dumps(all_json,ensure_ascii=False),track_id=0,distinct_id=org_distinct_id.replace('"',''),lib='ghost_sa',event='$is_channel_callback_event',type_1='ghost_sa_func',User_Agent=None,Host=None,Connection=None,Pragma=None,Cache_Control=None,Accept=None,Accept_Encoding=None,Accept_Language=None,ip=None,ip_city=None,ip_asn=None,url=None,referrer=None,remark='normal',ua_platform=None,ua_browser=None,ua_version=None,ua_language=None,created_at=created_at)
if admin.use_properties is True:
properties_key = []
for keys in all_json.keys():
properties_key.append(keys)
insert_properties(project=project,lib='ghost_sa',remark='normal',event='$is_channel_callback_event',properties=json.dumps(properties_key),properties_len=len(all_json.keys()),created_at=created_at,updated_at=created_at)
return insert_count
else:
return 0
def insert_installation_track(project, data_decode, User_Agent, Host, Connection, Pragma, Cache_Control, Accept, Accept_Encoding, Accept_Language, ip, ip_city,
ip_asn, url, referrer, remark, ua_platform, ua_browser, ua_version, ua_language, ip_is_good, ip_asn_is_good, created_at=None, updated_at=None,use_kafka=admin.use_kafka):
start_time = time.time()
timenow16 = int(round(time.time() * 1000))
distinct_id = 'undefined'
track_id = 0
dist_id_name = ['IDFA','androidid','IMEI','Idfa','Imei','imei','idfa']
for i in dist_id_name:
if i in data_decode['properties'].keys() and data_decode['properties'][i] and data_decode['properties'][i]!='':
distinct_id = data_decode['properties'][i]
if 'ts' in data_decode['properties']:
track_id = re.sub("\D","",data_decode['properties']['ts'])
if track_id == '':
track_id = 0
if 'properties' in data_decode and 'is_offerwall' in data_decode['properties'] and data_decode['properties']['is_offerwall']=='1':
count_event = insert_event(table=project,alljson=json.dumps(data_decode),track_id=track_id,distinct_id=distinct_id,lib='ghost_sa',event='$AppChannelMatching',type_1='installation_track',User_Agent=User_Agent,Host=Host,Connection=Connection,Pragma=Pragma,Cache_Control=Cache_Control,Accept=Accept,Accept_Encoding=Accept_Encoding,Accept_Language=Accept_Language,ip=ip,ip_city=ip_city,ip_asn=ip_asn,url=url,referrer=referrer,remark=remark,ua_platform=ua_platform,ua_browser=ua_browser,ua_version=ua_version,ua_language=ua_language,created_at=created_at if created_at else start_time)
count_user = insert_user_db(project=project,distinct_id=distinct_id,lib='ghost_sa',map_id='',original_id='',user_id='',all_user_profile=json.dumps(data_decode),update_params='',created_at=created_at if created_at else start_time,updated_at=created_at if created_at else start_time)
time_cost = time.time()-start_time
if admin.use_properties is True:
properties_key = []
for keys in data_decode['properties'].keys():
properties_key.append(keys)
insert_properties(project=project,lib='ghost_sa',remark=remark,event='$AppChannelMatching',properties=json.dumps(properties_key),properties_len=len(data_decode['properties'].keys()),created_at=created_at if created_at else start_time,updated_at=created_at if created_at else start_time)
return count_event,count_user,time_cost
elif use_kafka is False:
insert_event(table=project,alljson=json.dumps(data_decode),track_id=track_id,distinct_id=distinct_id,lib='ghost_sa',event='$AppChannelMatching',type_1='installation_track',User_Agent=User_Agent,Host=Host,Connection=Connection,Pragma=Pragma,Cache_Control=Cache_Control,Accept=Accept,Accept_Encoding=Accept_Encoding,Accept_Language=Accept_Language,ip=ip,ip_city=ip_city,ip_asn=ip_asn,url=url,referrer=referrer,remark=remark,ua_platform=ua_platform,ua_browser=ua_browser,ua_version=ua_version,ua_language=ua_language,created_at=created_at if created_at else start_time)
insert_user_db(project=project,distinct_id=distinct_id,lib='ghost_sa',map_id='',original_id='',user_id='',all_user_profile=json.dumps(data_decode),update_params='',created_at=created_at if created_at else start_time,updated_at=created_at if created_at else start_time)
if admin.use_properties is True:
properties_key = []
for keys in data_decode['properties'].keys():
properties_key.append(keys)
insert_properties(project=project,lib='ghost_sa',remark=remark,event='$AppChannelMatching',properties=json.dumps(properties_key),properties_len=len(data_decode['properties'].keys()),created_at=created_at if created_at else start_time,updated_at=created_at if created_at else start_time)
print(time.time()-start_time)
elif use_kafka is True:
msg = {"group":"installation_track","timestamp":timenow16,"data":{"project":project,"data_decode":data_decode,"User_Agent":User_Agent,"Host":Host,"Connection":Connection,"Pragma":Pragma,"Cache_Control":Cache_Control,"Accept":Accept,"Accept_Encoding":Accept_Encoding,"Accept_Language":Accept_Language,"ip":ip,"ip_city":ip_city,"ip_asn":ip_asn,"url":url,"referrer":referrer,"remark":remark,"ua_platform":ua_platform,"ua_browser":ua_browser,"ua_version":ua_version,"ua_language":ua_language,"ip_is_good":ip_is_good,"ip_asn_is_good":ip_asn_is_good,"created_at":created_at if created_at else start_time,"updated_at":created_at if created_at else start_time}}
insert_message_to_kafka(msg=msg)
print(time.time()-start_time)
def insert_data(project,
data_decode,
User_Agent,
Host,
Connection,
Pragma,
Cache_Control,
Accept,
Accept_Encoding,
Accept_Language,
ip,
ip_city,
ip_asn,
url,
referrer,
remark,
ua_platform,
ua_browser,
ua_version,
ua_language,
ip_is_good,
ip_asn_is_good,
created_at=None,
updated_at=None,
use_kafka=admin.use_kafka):
start_time = time.time()
jsondump = json.dumps(data_decode, ensure_ascii=False)
if '_track_id' in data_decode:
track_id = data_decode['_track_id']
else:
track_id = 0
distinct_id = data_decode['distinct_id']
if 'event' in data_decode:
event = data_decode['event']
else:
event = None
if remark:
remark = remark
else:
remark = ''
type_1 = data_decode['type'] if 'type' in data_decode else None
# lib = data_decode['lib']['$lib'] if '$lib' in data_decode['lib'] else None
lib = None
if 'lib' in data_decode:
if '$lib' in data_decode['lib']:
lib = data_decode['lib']['$lib']
elif 'properties' in data_decode:
if '$lib' in data_decode['properties']:
lib = data_decode['properties']['$lib']
# else:
# lib = None
if use_kafka is False:
try:
# count = insert_event(table=project,alljson=jsondump.replace('\\','\\\\').replace("'","\\'"),track_id=track_id,distinct_id=distinct_id,lib=lib,event=event,type_1=type_1,User_Agent=User_Agent,Host=Host,Connection=Connection,Pragma=Pragma,Cache_Control=Cache_Control,Accept=Accept,Accept_Encoding=Accept_Encoding,Accept_Language=Accept_Language,ip=ip,ip_city=ip_city,ip_asn=ip_asn,url=url,referrer=referrer,remark=remark,ua_platform=ua_platform,ua_browser=ua_browser,ua_version=ua_version,ua_language=ua_language)
count = insert_event(table=project,
alljson=jsondump,
track_id=track_id,
distinct_id=distinct_id,
lib=lib,
event=event,
type_1=type_1,
User_Agent=User_Agent,
Host=Host,
Connection=Connection,
Pragma=Pragma,
Cache_Control=Cache_Control,
Accept=Accept,
Accept_Encoding=Accept_Encoding,
Accept_Language=Accept_Language,
ip=ip,
ip_city=ip_city,
ip_asn=ip_asn,
url=url,
referrer=referrer,
remark=remark,
ua_platform=ua_platform,
ua_browser=ua_browser,
ua_version=ua_version,
ua_language=ua_language,
created_at=created_at)
# print('插入行数：'+str(count))
insert_device(project=project,
data_decode=data_decode,
user_agent=User_Agent,
accept_language=Accept_Language,
ip=ip,
ip_city=ip_city,
ip_is_good=ip_is_good,
ip_asn=ip_asn,
ip_asn_is_good=ip_asn_is_good,
ua_platform=ua_platform,
ua_browser=ua_browser,
ua_version=ua_version,
ua_language=ua_language,
created_at=created_at)
properties_key = []
for keys in data_decode['properties'].keys():
properties_key.append(keys)
if event and admin.use_properties is True:
insert_properties(project=project,
lib=lib,
remark=remark,
event=event,
properties=json.dumps(properties_key),
properties_len=len(
data_decode['properties'].keys()),
created_at=created_at,
updated_at=updated_at)
except Exception:
error = traceback.format_exc()
write_to_log(filename='api', defname='insert_date', result=error)
# if type_1 == 'profile_set' or type_1 == 'track_signup' or type_1 =='profile_set_once' or event == '$SignUp':
if type_1 == 'profile_set' or type_1 == 'track_signup' or type_1 == 'profile_set_once':
try:
insert_user(project=project,
data_decode=data_decode,
created_at=created_at)
except Exception:
error = traceback.format_exc()
write_to_log(filename='api',
defname='insert_date',
result=error)
elif use_kafka is True:
timenow = int(time.time())
timenow16 = int(round(time.time() * 1000))
msg = {
"timestamp": timenow16,
"data": {
"project": project,
"data_decode": data_decode,
"User_Agent": User_Agent,
"Host": Host,
"Connection": Connection,
"Pragma": Pragma,
"Cache_Control": Cache_Control,
"Accept": Accept,
"Accept_Encoding": Accept_Encoding,
"Accept_Language": Accept_Language,
"ip": ip,
"ip_city": ip_city,
"ip_asn": ip_asn,
"url": url,
"referrer": referrer,
"remark": remark,
"ua_platform": ua_platform,
"ua_browser": ua_browser,
"ua_version": ua_version,
"ua_language": ua_language,
"ip_is_good": ip_is_good,
"ip_asn_is_good": ip_asn_is_good,
"created_at": timenow,
"updated_at": timenow
}
}
insert_message_to_kafka(msg=msg)
print(time.time() - start_time)
def check_exist_distinct_id():
start_time = time.time()
password = request.args.get('password')
project = request.args.get('project')
distinct_id = request.args.get('distinct_id')
query_from = request.args.get('query_from')
User_Agent = request.headers.get('User-Agent')[0:2047] if request.headers.get('User-Agent') else None #Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36
Host = request.headers.get('Host') #: 10.16.5.241:5000
Connection = request.headers.get('Connection')#: keep-alive
Pragma = request.headers.get('Pragma')#: no-cache
Cache_Control = request.headers.get('Cache-Control')#: no-cache
Accept = request.headers.get('Accept')[0:254] if request.headers.get('Accept') else None#: image/webp,image/apng,image/*,*/*;q=0.8
Accept_Encoding = request.headers.get('Accept-Encoding')[0:254] if request.headers.get('Accept-Encoding') else None #: gzip, deflate
remark = request.args.get('remark') if 'remark' in request.args else 'normal'
Accept_Language = request.headers.get('Accept-Language')[0:254] if request.headers.get('Accept-Language') else None#: zh-CN,zh;q=0.9
ua_platform = request.user_agent.platform #客户端操作系统
ua_browser = request.user_agent.browser #客户端的浏览器
ua_version = request.user_agent.version #客户端浏览器的版本
ua_language = request.user_agent.language #客户端浏览器的语言
ext = request.args.get('ext')
url = request.url
args = request.args.to_dict(request.args)
data = {"properties":args}
# ip = '124.115.214.179' #测试西安bug
# ip = '36.5.99.68' #测试安徽bug
if 'ip' in args and len(args['ip']) - len( args['ip'].replace('.','') ) == 3:#判断IP里是否存在IP地址
ip = args['ip']
elif request.headers.get('X-Forwarded-For') is not None:
ip = request.headers.get('X-Forwarded-For') #获取SLB真实地址
else:
ip = request.remote_addr#服务器直接暴露
ip_city,ip_is_good = get_addr(ip)
ip_asn,ip_asn_is_good = get_asn(ip)
if ip_is_good ==0:
ip_city = '{}'
if ip_asn_is_good ==0:
ip_asn = '{}'
referrer = request.referrer
if password == admin.admin_password and project and distinct_id and query_from:#只有正确的密码才能触发动作
try:
if ',' in distinct_id:
#兼容七麦多个id一次性递入，返回格式兼容七麦
distinct_id_list = distinct_id.split(',')
returnjson = {'result':'success','results_count':len(distinct_id_list),'query_from':query_from}
for distinct_id in distinct_id_list:
results_count= distinct_id_query(distinct_id=distinct_id,project=project)
returnjson[distinct_id] = results_count
time_cost = time.time() - start_time
returnjson['timecost'] = round(time_cost,4)
data['returnjson'] = returnjson
insert_event(table=project,alljson=json.dumps(data),track_id=0,distinct_id=query_from,lib='ghost_sa',event='check_exist',type_1='ghost_sa_func',User_Agent=User_Agent,Host=Host,Connection=Connection,Pragma=Pragma,Cache_Control=Cache_Control,Accept=Accept,Accept_Encoding=Accept_Encoding,Accept_Language=Accept_Language,ip=ip,ip_city=ip_city,ip_asn=ip_asn,url=url,referrer=referrer,remark=remark,ua_platform=ua_platform,ua_browser=ua_browser,ua_version=ua_version,ua_language=ua_language,created_at=start_time)
if admin.use_properties is True:
properties_key = []
for keys in data.keys():
properties_key.append(keys)
insert_properties(project=project,lib='ghost_sa',remark=remark,event='check_exist',properties=json.dumps(properties_key),properties_len=len(data.keys()),created_at=start_time,updated_at=start_time)
return jsonify(returnjson)
else:
#正常一条一递，兼容有米和七麦的标准
results_count= distinct_id_query(distinct_id=distinct_id,project=project)
# key=['distinct_id','event','type','all_json','host','user_agent','ip','url','remark','created_at']
if results_count== 0 :
row = {}
# pending_result.append(dict(zip(key,item)))
time_cost = time.time() - start_time
# returnjson = {'result':'success','results_count':results_count,'code':0,'msg':'not_exists'}
returnjson = {'result':'success','results_count':results_count,'timecost':round(time_cost,4),'code':0,'msg':'not_exists','query_from':query_from}
returnjson[distinct_id] = results_count
data['returnjson'] = returnjson
insert_event(table=project,alljson=json.dumps(data),track_id=0,distinct_id=query_from,lib='ghost_sa',event='check_exist',type_1='ghost_sa_func',User_Agent=User_Agent,Host=Host,Connection=Connection,Pragma=Pragma,Cache_Control=Cache_Control,Accept=Accept,Accept_Encoding=Accept_Encoding,Accept_Language=Accept_Language,ip=ip,ip_city=ip_city,ip_asn=ip_asn,url=url,referrer=referrer,remark=remark,ua_platform=ua_platform,ua_browser=ua_browser,ua_version=ua_version,ua_language=ua_language,created_at=start_time)
if admin.use_properties is True:
properties_key = []
for keys in data.keys():
properties_key.append(keys)
insert_properties(project=project,lib='ghost_sa',remark=remark,event='check_exist',properties=json.dumps(properties_key),properties_len=len(data.keys()),created_at=start_time,updated_at=start_time)
return jsonify(returnjson)
time_cost = time.time() - start_time
returnjson = {'result':'success','results_count':results_count,'timecost':round(time_cost,4),'code':-1,'msg':'exists','query_from':query_from}
returnjson[distinct_id] = results_count
# returnjson = {'result':'success','results_count':results_count,'code':-1,'msg':'exists'}
data['returnjson'] = returnjson
insert_event(table=project,alljson=json.dumps(data),track_id=0,distinct_id=query_from,lib='ghost_sa',event='check_exist',type_1='ghost_sa_func',User_Agent=User_Agent,Host=Host,Connection=Connection,Pragma=Pragma,Cache_Control=Cache_Control,Accept=Accept,Accept_Encoding=Accept_Encoding,Accept_Language=Accept_Language,ip=ip,ip_city=ip_city,ip_asn=ip_asn,url=url,referrer=referrer,remark=remark,ua_platform=ua_platform,ua_browser=ua_browser,ua_version=ua_version,ua_language=ua_language,created_at=start_time)
if admin.use_properties is True:
properties_key = []
for keys in data.keys():
properties_key.append(keys)
insert_properties(project=project,lib='ghost_sa',remark=remark,event='check_exist',properties=json.dumps(properties_key),properties_len=len(data.keys()),created_at=start_time,updated_at=start_time)
return jsonify(returnjson)
except Exception:
error = traceback.format_exc()
write_to_log(filename='api',defname='check_exist_distinct_id',result=error)
else:
return jsonify('参数不正确')