def insert_installation_track(project, data_decode, User_Agent, Host, Connection, Pragma, Cache_Control, Accept, Accept_Encoding, Accept_Language, ip, ip_city,
ip_asn, url, referrer, remark, ua_platform, ua_browser, ua_version, ua_language, ip_is_good, ip_asn_is_good, created_at=None, updated_at=None,use_kafka=admin.use_kafka):
start_time = time.time()
timenow16 = int(round(time.time() * 1000))
distinct_id = 'undefined'
track_id = 0
dist_id_name = ['IDFA','androidid','IMEI','Idfa','Imei','imei','idfa']
for i in dist_id_name:
if i in data_decode['properties'].keys() and data_decode['properties'][i] and data_decode['properties'][i]!='':
distinct_id = data_decode['properties'][i]
if 'ts' in data_decode['properties']:
track_id = re.sub("\D","",data_decode['properties']['ts'])
if track_id == '':
track_id = 0
if 'properties' in data_decode and 'is_offerwall' in data_decode['properties'] and data_decode['properties']['is_offerwall']=='1':
count_event = insert_event(table=project,alljson=json.dumps(data_decode),track_id=track_id,distinct_id=distinct_id,lib='ghost_sa',event='$AppChannelMatching',type_1='installation_track',User_Agent=User_Agent,Host=Host,Connection=Connection,Pragma=Pragma,Cache_Control=Cache_Control,Accept=Accept,Accept_Encoding=Accept_Encoding,Accept_Language=Accept_Language,ip=ip,ip_city=ip_city,ip_asn=ip_asn,url=url,referrer=referrer,remark=remark,ua_platform=ua_platform,ua_browser=ua_browser,ua_version=ua_version,ua_language=ua_language,created_at=created_at if created_at else start_time)
count_user = insert_user_db(project=project,distinct_id=distinct_id,lib='ghost_sa',map_id='',original_id='',user_id='',all_user_profile=json.dumps(data_decode),update_params='',created_at=created_at if created_at else start_time,updated_at=created_at if created_at else start_time)
time_cost = time.time()-start_time
if admin.use_properties is True:
properties_key = []
for keys in data_decode['properties'].keys():
properties_key.append(keys)
insert_properties(project=project,lib='ghost_sa',remark=remark,event='$AppChannelMatching',properties=json.dumps(properties_key),properties_len=len(data_decode['properties'].keys()),created_at=created_at if created_at else start_time,updated_at=created_at if created_at else start_time)
return count_event,count_user,time_cost
elif use_kafka is False:
insert_event(table=project,alljson=json.dumps(data_decode),track_id=track_id,distinct_id=distinct_id,lib='ghost_sa',event='$AppChannelMatching',type_1='installation_track',User_Agent=User_Agent,Host=Host,Connection=Connection,Pragma=Pragma,Cache_Control=Cache_Control,Accept=Accept,Accept_Encoding=Accept_Encoding,Accept_Language=Accept_Language,ip=ip,ip_city=ip_city,ip_asn=ip_asn,url=url,referrer=referrer,remark=remark,ua_platform=ua_platform,ua_browser=ua_browser,ua_version=ua_version,ua_language=ua_language,created_at=created_at if created_at else start_time)
insert_user_db(project=project,distinct_id=distinct_id,lib='ghost_sa',map_id='',original_id='',user_id='',all_user_profile=json.dumps(data_decode),update_params='',created_at=created_at if created_at else start_time,updated_at=created_at if created_at else start_time)
if admin.use_properties is True:
properties_key = []
for keys in data_decode['properties'].keys():
properties_key.append(keys)
insert_properties(project=project,lib='ghost_sa',remark=remark,event='$AppChannelMatching',properties=json.dumps(properties_key),properties_len=len(data_decode['properties'].keys()),created_at=created_at if created_at else start_time,updated_at=created_at if created_at else start_time)
print(time.time()-start_time)
elif use_kafka is True:
msg = {"group":"installation_track","timestamp":timenow16,"data":{"project":project,"data_decode":data_decode,"User_Agent":User_Agent,"Host":Host,"Connection":Connection,"Pragma":Pragma,"Cache_Control":Cache_Control,"Accept":Accept,"Accept_Encoding":Accept_Encoding,"Accept_Language":Accept_Language,"ip":ip,"ip_city":ip_city,"ip_asn":ip_asn,"url":url,"referrer":referrer,"remark":remark,"ua_platform":ua_platform,"ua_browser":ua_browser,"ua_version":ua_version,"ua_language":ua_language,"ip_is_good":ip_is_good,"ip_asn_is_good":ip_asn_is_good,"created_at":created_at if created_at else start_time,"updated_at":created_at if created_at else start_time}}
insert_message_to_kafka(msg=msg)
print(time.time()-start_time)
def insert_data(project,
data_decode,
User_Agent,
Host,
Connection,
Pragma,
Cache_Control,
Accept,
Accept_Encoding,
Accept_Language,
ip,
ip_city,
ip_asn,
url,
referrer,
remark,
ua_platform,
ua_browser,
ua_version,
ua_language,
ip_is_good,
ip_asn_is_good,
created_at=None,
updated_at=None,
use_kafka=admin.use_kafka):
start_time = time.time()
jsondump = json.dumps(data_decode, ensure_ascii=False)
if '_track_id' in data_decode:
track_id = data_decode['_track_id']
else:
track_id = 0
distinct_id = data_decode['distinct_id']
if 'event' in data_decode:
event = data_decode['event']
else:
event = None
if remark:
remark = remark
else:
remark = ''
type_1 = data_decode['type'] if 'type' in data_decode else None
# lib = data_decode['lib']['$lib'] if '$lib' in data_decode['lib'] else None
lib = None
if 'lib' in data_decode:
if '$lib' in data_decode['lib']:
lib = data_decode['lib']['$lib']
elif 'properties' in data_decode:
if '$lib' in data_decode['properties']:
lib = data_decode['properties']['$lib']
# else:
# lib = None
if use_kafka is False:
try:
# count = insert_event(table=project,alljson=jsondump.replace('\\','\\\\').replace("'","\\'"),track_id=track_id,distinct_id=distinct_id,lib=lib,event=event,type_1=type_1,User_Agent=User_Agent,Host=Host,Connection=Connection,Pragma=Pragma,Cache_Control=Cache_Control,Accept=Accept,Accept_Encoding=Accept_Encoding,Accept_Language=Accept_Language,ip=ip,ip_city=ip_city,ip_asn=ip_asn,url=url,referrer=referrer,remark=remark,ua_platform=ua_platform,ua_browser=ua_browser,ua_version=ua_version,ua_language=ua_language)
count = insert_event(table=project,
alljson=jsondump,
track_id=track_id,
distinct_id=distinct_id,
lib=lib,
event=event,
type_1=type_1,
User_Agent=User_Agent,
Host=Host,
Connection=Connection,
Pragma=Pragma,
Cache_Control=Cache_Control,
Accept=Accept,
Accept_Encoding=Accept_Encoding,
Accept_Language=Accept_Language,
ip=ip,
ip_city=ip_city,
ip_asn=ip_asn,
url=url,
referrer=referrer,
remark=remark,
ua_platform=ua_platform,
ua_browser=ua_browser,
ua_version=ua_version,
ua_language=ua_language,
created_at=created_at)
# print('插入行数：'+str(count))
insert_device(project=project,
data_decode=data_decode,
user_agent=User_Agent,
accept_language=Accept_Language,
ip=ip,
ip_city=ip_city,
ip_is_good=ip_is_good,
ip_asn=ip_asn,
ip_asn_is_good=ip_asn_is_good,
ua_platform=ua_platform,
ua_browser=ua_browser,
ua_version=ua_version,
ua_language=ua_language,
created_at=created_at)
properties_key = []
for keys in data_decode['properties'].keys():
properties_key.append(keys)
if event and admin.use_properties is True:
insert_properties(project=project,
lib=lib,
remark=remark,
event=event,
properties=json.dumps(properties_key),
properties_len=len(
data_decode['properties'].keys()),
created_at=created_at,
updated_at=updated_at)
except Exception:
error = traceback.format_exc()
write_to_log(filename='api', defname='insert_date', result=error)
# if type_1 == 'profile_set' or type_1 == 'track_signup' or type_1 =='profile_set_once' or event == '$SignUp':
if type_1 == 'profile_set' or type_1 == 'track_signup' or type_1 == 'profile_set_once':
try:
insert_user(project=project,
data_decode=data_decode,
created_at=created_at)
except Exception:
error = traceback.format_exc()
write_to_log(filename='api',
defname='insert_date',
result=error)
elif use_kafka is True:
timenow = int(time.time())
timenow16 = int(round(time.time() * 1000))
msg = {
"timestamp": timenow16,
"data": {
"project": project,
"data_decode": data_decode,
"User_Agent": User_Agent,
"Host": Host,
"Connection": Connection,
"Pragma": Pragma,
"Cache_Control": Cache_Control,
"Accept": Accept,
"Accept_Encoding": Accept_Encoding,
"Accept_Language": Accept_Language,
"ip": ip,
"ip_city": ip_city,
"ip_asn": ip_asn,
"url": url,
"referrer": referrer,
"remark": remark,
"ua_platform": ua_platform,
"ua_browser": ua_browser,
"ua_version": ua_version,
"ua_language": ua_language,
"ip_is_good": ip_is_good,
"ip_asn_is_good": ip_asn_is_good,
"created_at": timenow,
"updated_at": timenow
}
}
insert_message_to_kafka(msg=msg)
print(time.time() - start_time)
def insert_installation_track(project,
data_decode,
User_Agent,
Host,
Connection,
Pragma,
Cache_Control,
Accept,
Accept_Encoding,
Accept_Language,
ip,
ip_city,
ip_asn,
url,
referrer,
remark,
ua_platform,
ua_browser,
ua_version,
ua_language,
ip_is_good,
ip_asn_is_good,
created_at=None,
updated_at=None,
use_kafka=admin.use_kafka):
start_time = time.time()
timenow16 = int(round(time.time() * 1000))
distinct_id = 'undefined'
track_id = 0
dist_id_name = ['idfa', 'IDFA', 'imei', 'IMEI', 'Idfa', 'Imei']
for i in dist_id_name:
if i in data_decode.keys():
distinct_id = data_decode[i]
if 'ts' in data_decode:
track_id = data_decode['ts']
if use_kafka is False:
insert_event(table=project,
alljson=json.dumps(data_decode),
track_id=track_id,
distinct_id=distinct_id,
lib='ghost_sa',
event='admaster',
type_1='installation_track',
User_Agent=User_Agent,
Host=Host,
Connection=Connection,
Pragma=Pragma,
Cache_Control=Cache_Control,
Accept=Accept,
Accept_Encoding=Accept_Encoding,
Accept_Language=Accept_Language,
ip=ip,
ip_city=ip_city,
ip_asn=ip_asn,
url=url,
referrer=referrer,
remark=remark,
ua_platform=ua_platform,
ua_browser=ua_browser,
ua_version=ua_version,
ua_language=ua_language,
created_at=created_at if created_at else start_time)
insert_user_db(project=project,
distinct_id=distinct_id,
lib='ghost_sa',
map_id='',
original_id='',
user_id='',
all_user_profile=json.dumps(data_decode),
update_params='',
created_at=created_at if created_at else start_time,
updated_at=created_at if created_at else start_time)
print(time.time() - start_time)
elif use_kafka is True:
msg = {
"group": "installation_track",
"timestamp": timenow16,
"data": {
"project": project,
"data_decode": data_decode,
"User_Agent": User_Agent,
"Host": Host,
"Connection": Connection,
"Pragma": Pragma,
"Cache_Control": Cache_Control,
"Accept": Accept,
"Accept_Encoding": Accept_Encoding,
"Accept_Language": Accept_Language,
"ip": ip,
"ip_city": ip_city,
"ip_asn": ip_asn,
"url": url,
"referrer": referrer,
"remark": remark,
"ua_platform": ua_platform,
"ua_browser": ua_browser,
"ua_version": ua_version,
"ua_language": ua_language,
"ip_is_good": ip_is_good,
"ip_asn_is_good": ip_asn_is_good,
"created_at": created_at if created_at else start_time,
"updated_at": created_at if created_at else start_time
}
}
insert_message_to_kafka(msg=msg)
print(time.time() - start_time)