def get_yesterday_effect(delt_name):
client = MongoClient(mongo_server_uri)
# 连接所需数据库, ai_explore为数据库名
db = client.get_database(os.environ['mongo_dbname'])
out = db.insights.find({
'delt_name': delt_name,
'update': {
'$gte': tool.get_yesterday() + ' 00:00:00',
'$lte': tool.get_yesterday() + ' 23:59:59'
}
}).distinct("ad_id")
fetch_out = db.incomes.find(
{
"ad_id": {
"$in": list(out)
},
'report_date': tool.get_yesterday(),
'daily_install_count': {
'$gte': 5
},
'roi': {
'$gte': 0.02
}
}, {
'ad_id': 1,
'daily_install_count': 1,
'roi': 1,
'_id': 0
})
tmp = list(fetch_out)
client.close()
return tmp
def handle_page(url, city, province, r):
"""
处理每一个页面的url以及数据
:param url:
:param city:
:param province:
:param r:
:return:
"""
global device_id, tim, sign, header, is_break
res = requests.get(url, headers=header)
res_data = dict(
eval(
res.text.replace('false',
'False').replace('true',
'True').replace('null', 'None')))
if '200' not in str(res_data):
while '200' not in str(res_data):
sleep(10)
sign_tmp = sign
tim_tmp = tim
print('handle_page', res.text)
print('权限不足或者accessToken失效，sign失败')
device_id, tim, sign, header = more_get_token()
url = url.replace(sign_tmp, sign).replace().replace(tim_tmp, tim)
res = requests.get(url, headers=header, timeout=10)
res_data = dict(
eval(
res.text.replace('false',
'False').replace('true', 'True').replace(
'null', 'None')))
res.close()
try:
qiye_data = res_data.get('result').get('Result')
except Exception as e:
print(e)
return
if not qiye_data:
is_break = True
return
write_list = []
for qiye in qiye_data:
if qiye.get('StartDate') != get_yesterday():
is_break = True
continue
qiye['City'] = city
qiye['Province'] = province
del qiye['ImageUrl']
del qiye['HitReason']
write_list.append(qiye)
r.set(province + ":" + city + ':' + qiye.get('KeyNo'), str(qiye))
threading.Thread(target=write_data,
args=(get_yesterday() + "-data.txt", write_list)).start()
def new_enterprise_main():
"""
获取新增企业数据
:return:
"""
r = connect_redis(0, 110)
for url, city, province in creat_url():
handle_page(url, city, province, r)
print(get_yesterday(), 'end')
def stats_trigger():
client = MongoClient(mongo_server_uri)
# 连接所需数据库,test为数据库名
db = client.get_database(os.environ['mongo_dbname'])
out = db.income_logs.find({"report_date": tool.get_yesterday()})
tmp = list(out)
client.close()
if len(tmp) > 0:
return True
else:
return False
def new_enterprise_main():
"""
获取新增企业数据
:return:
"""
global device_id, tim, sign, header
device_id, tim, sign, header = more_get_token()
r = connect_redis(0, 110)
for url, city, province in creat_url():
handle_page(url, city, province, r)
sleep(1.5)
print(get_yesterday(), 'end')
def handle_page(url, city, province, r):
"""
处理每一个页面的url以及数据
:param url:
:param city:
:param province:
:param r:
:return:
"""
global device_id, tim, sign, header
res = requests.get(url, headers=header)
print(city, province)
res_data = dict(
eval(
res.text.replace('false',
'False').replace('true',
'True').replace('null', 'None')))
if '200' not in str(res_data):
while '200' not in str(res_data):
sleep(10)
print('handle_page', res.text)
print('权限不足或者accessToken失效，sign失败')
device_id, tim, sign, header = more_get_token()
res = requests.get(url, headers=header, timeout=10)
res_data = dict(
eval(
res.text.replace('false',
'False').replace('true', 'True').replace(
'null', 'None')))
res.close()
try:
qiye_data = res_data.get('result').get('Result')
except Exception as e:
print(e)
return
for qiye in qiye_data:
if qiye.get('StartDate') != get_yesterday(): continue
qiye['City'] = city
qiye['Province'] = province
r.set(province + city + ':' + qiye.get('KeyNo'), str(qiye))
def main():
print(get_yesterday(), 'start')
Process(target=new_enterprise_main).start()