def get_data(self, osd, oed):
headers = {
'Accept':
'application/json, text/javascript, */*; q=0.01',
'Cookie':
self.cookies,
'Host':
'unicorn.wcd.qq.com',
'Referer':
'https://unicorn.wcd.qq.com/paydata/paydatafree.html',
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"
}
# 请求1
productUrl = 'https://unicorn.wcd.qq.com/mcare/product.html'
productRes = get(productUrl, headers=headers, timeout=120)
if not productRes.get('is_success'):
return {'succ': False, 'data': "{'msg': 'request time out'}"}
productResJson = productRes.get('msg').json()
productRes = [
x for x in productResJson.get('data') if int(x.get('key')) > 0
]
# 请求2
vendorUrl = 'https://unicorn.wcd.qq.com/mcare/vendor.html'
vendorRes = get(vendorUrl, headers=headers, timeout=120)
if not vendorRes.get('is_success'):
return {'succ': False, 'data': "{'msg': 'request time out'}"}
vendorResJson = vendorRes.get('msg').json()
vendorRes = [
x.get('key') for x in vendorResJson.get('data')
if int(x.get('key')) > 0
]
# 请求3
channelUrl = "https://unicorn.wcd.qq.com/mcare/channel.html"
channelRes = get(channelUrl, headers=headers)
if not channelRes.get('is_success'):
return {'succ': False, 'data': "{'msg': 'request time out'}"}
channelResStr = channelRes.get('msg').json()
# channelRes = [x for x in channelResStr.get('data') if int(x.get('key')) > 0]
# 请求4
self.get_product_detail(productRes, vendorRes, headers, osd, oed)
return {
'succ': True,
'msg': {
'channelList': channelResStr,
'productList': productResJson,
'vendorList': vendorResJson
}
}
def get_uid_another_version(self):
url = 'https://e.qq.com/gaea/api.php'
cookie = self.d.get_cookies()
cookie = {e['name']: e['value'] for e in cookie}
skey = cookie.get('skey') if not cookie.get(
'gdt_protect') else cookie.get('gdt_protect')
g_tk = self.getGTK(skey)
params = {'mod': 'customer', 'act': 'getlist', 'g_tk': g_tk}
cookie = '; '.join(['%s=%s' % (k, v) for k, v in cookie.items()])
headers = {
'accept':
'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01',
'cookie':
cookie,
'referer':
'https://e.qq.com/ads/',
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3569.0 Safari/537.36'
}
res = get(url, params=params, headers=headers)
if not res.get('is_success'):
return {'succ': False, 'msg': res.get('msg')}
res = res.get('msg').json()
if res.get('ret') == 0:
uid = res.get('data').get('list')[0].get('uid')
return {'succ': True, 'msg': 'login success', 'uid': uid}
else:
return {
'succ': False,
'msg': 'login failed',
'desc': res.get('msg')
}
def get_product(self, sd, ed):
url = 'https://e.qq.com/ec/api.php'
params = {
'mod': 'report',
'act': 'getproduct',
'g_tk': str(self.gtk),
'sdate': sd,
'edate': ed,
'searchtype': 'product',
'product_type': '20'
}
headers = {
'Accept':
'application/json, text/javascript, */*; q=0.01',
'referer':
'https://e.qq.com/atlas/%s/report/producttype' % self.uid,
'cookie':
self.cookies_str,
'user-agent':
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"
}
res = get(url, params=params, headers=headers)
if not res.get('is_success'):
logger.error(res.get('msg').json())
data = res.get('msg').json()
total_num = data.get('data').get('conf').get('totalnum')
if total_num == 0:
return {'succ': False, 'msg': 'no data'}
data_list = data.get('data').get('list')
data = [{
'pname': e.get('pname'),
'pid': e.get('product_id'),
'cost': e.get('cost')
} for e in data_list]
return {'succ': True, 'msg': data}
def get_balance(self, uid):
url = "https://e.qq.com/ec/api.php"
querystring = {
"act": "dashboard",
"g_tk": self.gtk,
"owner": uid,
"advertiser_id": uid,
"mod": "account",
"unicode": "1"
}
headers = {
'accept': "application/json, text/javascript, */*; q=0.01",
'cookie': self.cookies.get('cookie_str'),
'referer': f"https://e.qq.com/atlas/{uid}/",
'user-agent':
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36",
'Host': "e.qq.com",
}
res = get(url, params=querystring, headers=headers)
if not res.get('is_success'):
logger.error(res.get('msg'), exc_info=1)
data = res.get('msg').json()
if not data.get('ret') == 0:
logger.error(data.get('msg'))
return {'succ': False}
accounts = data.get('data').get('accounts')
return {'succ': True, 'msg': accounts}
def get_vc_img(self):
try:
url = self.d.find_element_by_class_name(
'geetest_item_img').get_attribute('src')
except:
return {
'succ': False,
'msg': 'slider verify, refresh to click verify'
}
logger.info('img_url: %s' % url)
headers = {
'accept':
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
'user-agent':
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
}
res = get(url, headers=headers)
if not res.get('is_success'):
return {'succ': False, 'msg': 'net error'}
img = res.get('msg').content
with open(join(IMG_PATH, 'click_vc.png'), 'bw') as i:
i.write(img)
vc = u.rc.rk_create(img, 6906, timeout=90)
del (img)
# if not vc.get('Result'):
# u.rc.rk_report_error(vc.get('Id'))
# return self.get_vc_img()
return {'succ': True, 'msg': vc}
def get_data_another_version(self, sd, ed, data, tid, tname):
logger.info('get into (self.get_data_another_version)function')
# 文件名
fname = '%(productType)s_%(productId)s_%(productName)s_%(sd)s_%(ed)s.json' % {
'productType': tname, 'productId': data.get('pid'), 'productName': data.get('pname'), 'sd': sd, 'ed': ed
}
# 构造请求参数
url = "https://e.qq.com/ec/api.php"
params = {"g_tk": self.g_tk, 'owner': self.uid, "product_id": data.get('pid'), "product_type": tid, "sdate": sd,
"edate": ed}
headers = {
"cookie": self.cookie_str,
"referer": "http://e.qq.com/atlas/%(uid)s/report/analytic2?product_id=%(pid)s&product_type=%(ptype)s" % {
'uid': self.uid, 'pid': data.get('pid'), 'ptype': tid
}
}
self.params.update(params)
self._headers.update(headers)
data = get(url, params=self.params, headers=self._headers)
if not data['is_success']:
return {'succ': False, 'msg': data['msg']}
file_name = os.path.join(self.dir_path, fname)
data = data['msg'].json()
cost = data.get('data').get('total').get('cost').replace(',', '')
if float(cost) == 0:
return {'succ': False, 'msg': 'no data'}
data['account'] = self.acc
with open(file_name, 'w', encoding='utf-8') as f:
json.dump(data, f)
logger.info('crawled data: %s' % data)
return {'succ': True}
def is_login(self, cookie, vc, im):
url = 'http://adv.aiclk.com/user?version=%s' % time.strftime('%Y%m%d%H%M')
cookie = '; '.join(['%s=%s' % (e.get('name'), e.get('value')) for e in cookie])
headers = {
'Accept': "application/json, text/javascript, */*; q=0.01",
'Cookie': cookie,
'Host': "adv.aiclk.com",
'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
}
res = get(url, headers=headers)
if not res.get('is_success'): # 网络异常
return {'succ': False, 'msg': res.get('msg')}
data = json.loads(res.get('msg').content)
# logger.info(data)
if data.get('errorType') == 'NoLogin': # 登陆失败，两个key: retdesc, retcode
msg = self.d.find_element_by_class_name('ui-dialog-text').text
if msg == '开户资料正在审核中':
return {'succ': True, 'msg': msg}
elif msg == '验证码错误':
u.rc.rk_report_error(vc.get('Id'))
return {'succ': False, 'msg': 'verify_code_error'}
elif msg == '用户名或者密码错误':
return {'succ': False, 'msg': 'account_or_password_error'}
else:
return {'succ': False, 'msg': data.get('message')}
else:
u.rc.rk_report(im, 3040, vc.get('Result'), vc_type=self.user_info.get('platform'))
return {'succ': True, 'msg': 'login success'}
def is_login(self, data):
cookie = '; '.join(['%s=%s' % (e['name'], e['value']) for e in data])
url = 'https://dev.vivo.com.cn/webapi/user/info?timestamp=%s' % int(
time.time() * 1000)
haeders = {
'Accept':
'application/json, text/plain, */*',
'Accept-Encoding':
'gzip, deflate, br',
'Accept-Language':
'zh-CN,zh;q=0.9',
'Content-Type':
'application/json;charset=UTF-8',
'Cookie':
cookie,
'Host':
'dev.vivo.com.cn',
'Origin':
'https://dev.vivo.com.cn',
'Referer':
'https://dev.vivo.com.cn/home',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
}
res = get(url, headers=haeders, timeout=10)
if not res.get('is_success'):
raise Exception(res.get('msg'))
res = json.loads(res.get('msg').content)
if res.get('code') != 0:
logger.info('login failed')
return {'succ': False, 'msg': 'login failed'}
logger.info('login success')
return {'succ': True, 'msg': 'login success'}
def is_login(self, cookie):
cookie = '; '.join(
['%s=%s' % (e.get('name'), e.get('value')) for e in cookie])
yesterday = int(time.strftime('%d')) - 1
day = '%s-%s' % (time.strftime('%Y-%m'), yesterday)
url = 'http://agent.e.sogou.com/main/plot.html?start=%s&end=%s&type=2' % (
day, day)
headers = {
'Accept': "application/json, text/javascript, */*; q=0.01",
'Cookie': cookie,
'Host': "agent.e.sogou.com",
'Referer': "http://agent.e.sogou.com/main.html",
'User-Agent':
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
'Content-Type': "application/x-www-form-urlencoded"
}
res = get(url, headers=headers)
if not res.get('is_success'): # 网络异常
return {'succ': False, 'msg': res.get('msg')}
data = json.loads(res.get('msg').content)
# logger.info(data)
if 'plotList' in data.keys(): # 正常情况下，只有一个key: plotList
return {'succ': True, 'msg': 'login succ'}
elif data.get('retcode') == 401: # 登陆失败，两个key: retdesc, retcode
return {'succ': False, 'msg': data.get('retdesc')}
else:
return {'succ': False, 'msg': 'unknown error'}
def save_pic(self, src, pic_name):
from requests import get
headers = {
'accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36'
}
res = get(src, headers=headers)
with open(pic_name, 'bw') as f:
f.write(res.content)
def get_main_content(self, p_cf, cookie, sd, ed):
# 获取主账号的数据，并且完成截图
# get_data------ids
url = 'https://e.yunos.com/api/campaign/list/layout'
headers = {
'accept':
"application/json, text/javascript",
'content-type':
"application/x-www-form-urlencoded",
'cookie':
cookie,
'referer':
"https://e.yunos.com/",
'user-agent':
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
}
data = get(url, headers=headers)
if not data.get('is_success'):
return data
id_list = json.loads(data.get('msg').content.decode('utf-8'))
if id_list.get('total_count') == 0:
return {'succ': False, 'msg': 'no data', 'account': self.acc}
ids = [(e.get('id'), e.get('title')) for e in id_list.get('data')
if '东方' in e.get('title')]
# get_data------data
filename = join(self.dir_path, 'mainContent_#%s_#%s.json' %
(sd, ed)) # change file name
url2 = 'https://e.yunos.com/api/rpt/list'
payload = {
'param':
json.dumps({
"page": 1,
"page_size": 99999,
"date_range_type": 0,
"campaign_id": [i[0] for i in ids],
"report_level": 1,
"report_data_type": 1,
"start_ds": sd,
"end_ds": ed
}),
'p_cf':
p_cf
}
data = post(url2, data=payload, headers=headers)
if not data.get('is_success'):
return {'succ': False, 'msg': data.get('msg')}
data = json.loads(data.get('msg').content.decode('utf-8'))
if data.get('total_count') == 0:
return {'succ': False, 'msg': 'no data', 'account': self.acc}
with open(filename, 'w', encoding='utf-8') as f:
json.dump(data, f)
return {'succ': True, 'msg': data, 'ids': ids, 'sd': sd, 'ed': ed}
def get_child_accounts(self, cookie, reget=True, psize='20'):
"""获取子账号列表"""
url = 'https://e.yunos.com/api/member/agency/ref/list?keyword=&page_size=%s&page=1' % psize
headers = {'cookie': cookie, 'referer': 'https://e.yunos.com/'}
self.headers.update(headers)
data = get(url, headers=self.headers)
if not data['is_success']:
return {'succ': False, 'msg': data['msg']}
data = json.loads(data['msg'].content.decode())
if reget:
total_count = data.get('total_count')
return self.get_child_accounts(cookie, reget=False, psize=total_count)
accounts = [(x['username'], x['company']) for x in data['data'] if x['status'] == 1 and x['privilege'] == 1 and x['ref_status'] == 1]
return {'succ': True, 'msg': accounts}
def get_type_ids(self):
"""获取类型id"""
url = 'https://e.qq.com/ec/api.php'
querystring = {"mod": "product", "act": "getproducttypelist", "g_tk": self.g_tk, "owner": self.uid}
headers = {
'accept': "application/json, text/javascript, */*; q=0.01",
'cookie': self.cookie_str,
'referer': "https://e.qq.com/atlas/%s/report/producttype" % self.uid,
'user-agent': "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3569.0 Safari/537.36"
}
res = get(url, params=querystring, headers=headers)
if not res.get('is_success'):
return {'succ': False, 'msg': res.get('msg')}
data = res.get('msg').json()
ids = [(x.get('id'), x.get('name')) for x in data.get('data')]
return {'succ': True, 'msg': ids}
def get_product_detail(self, productRes, vendorRes, headers, osd, oed):
# 请求4
for item in productRes:
curPage = 1
total = -1
pageSize = 100
alldata = []
productId = item.get('key')
logger.info('开始产品：%s' % item.get('value'))
while total == -1 or curPage <= math.ceil(total / pageSize):
url = 'https://unicorn.wcd.qq.com/free/getPayDataTable.html'
params = {
'timeArea':
osd.replace('-', '') + ' - ' + oed.replace('-', ''),
'product': productId,
'platform': 'android,iphone,ipad,apad,windows',
'vendor': ','.join(vendorRes),
# 'channel': ','.join(channelRes),
'channel': None,
'datatype': 'd',
'dataValue': 'newUser',
'fee': '1,0',
'fileType': '',
'pageSize': pageSize,
'curPage': curPage
}
logger.info('product:%s, curPage:%d, total:%d' %
(item.get('value'), curPage, total))
dataRes = get(url, params=params, headers=headers)
if not dataRes.get('is_success'):
logger.error(
'product:%s, curPage:%d, total:%d, got failed' %
(item.get('value'), curPage, total))
curPage += 1
continue
dataRes = dataRes.get('msg').json()
alldata = alldata + dataRes.get('data')
total = dataRes['total']
curPage += 1
if not alldata:
continue
product_file_name = os.path.join(
self.dir_path, '%s_%s_%s.json' % (osd, oed, item.get('value')))
with open(product_file_name, 'w', encoding='utf-8') as f:
f.write(json.dumps(alldata, ensure_ascii=False))
def get_cid(self, identity):
"""
获取组id
:param identity: 子账号 需要编码
:return: cid list
"""
identity = quote(identity)
url = 'https://e.yunos.com/api/campaign/list/layout?identity=%s' % identity
headers = {"referer": "https://e.yunos.com/?identity=%s" % identity}
self._headers.update(headers)
data = get(url, headers=self._headers)
if not data['is_success']:
return {'succ': False, 'msg': data['msg']}
data = json.loads(data['msg'].content.decode())
cid = [x['id']
for x in data['data']] if data['total_count'] > 0 else []
return {'succ': True, 'msg': cid}
def get_child_accounts(self):
"""获取子账号列表"""
url = 'http://e.mi.com/v2/account/agent/customers'
headers = {'Cookie': self.cookies, 'Referer': "http://e.mi.com/v2/dist/agent.html"}
self._headers.update(headers)
querystring = {"agent": self.agent} # 替换为agent
data = get(url, params=querystring, headers=self._headers)
if not data['is_success']:
return {'succ': False, 'msg': data['msg']}
data = data['msg'].json()
if not data.get('success'):
return {'succ': False, 'msg': data.get('failMsg'), 'desc': 'is_child_account'}
# if reget:
# return self.get_child_accounts(agent, reget=False)
accounts = [x.get('customerId') for x in data.get('result').get('list') if x.get('status') == 2]
data = [i for i in data.get('result').get('list') if i.get('customerId') in self.none_cost]
self.balance_data = {'list': data}
return {'succ': True, 'accounts': accounts}
def has_data(self, sd, ed, productId, vendor, channel):
cookie = '; '.join([
'%s=%s' % (e.get('name'), e.get('value'))
for e in self.d.get_cookies()
])
headers = {
'Accept':
'application/json, text/javascript, */*; q=0.01',
'Cookie':
cookie,
'Host':
'unicorn.wcd.qq.com',
'Referer':
'https://unicorn.wcd.qq.com/paydata/paydatafree.html',
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"
}
url = 'https://unicorn.wcd.qq.com/free/getPayDataTable.html'
params = {
'timeArea': '%s - %s' % (sd, ed),
'product': ','.join(productId),
'platform': 'android,iphone,ipad,apad,windows',
'vendor': ','.join(vendor),
'channel': channel,
'datatype': 'd',
'dataValue': 'newUser',
'fee': '1,0',
'fileType': '',
'pageSize': 50,
'curPage': 1
}
dataRes = get(url, params=params, headers=headers)
if not dataRes.get('is_success'):
return False
dataRes = dataRes.get('msg').json()
time.sleep(0.2)
if dataRes.get('total') == 0:
return False
return dataRes.get('total')
def get_channels(self, cookie, v_g_tk):
headers = {
'Accept':
'*/*',
'Cookie':
cookie,
'Referer':
'https://op.open.qq.com/game_channel/atistic',
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"
}
url = "https://op.open.qq.com/game_channel/info"
querystring = {"g_tk": v_g_tk, "act": "all"}
channel_id = get(url, params=querystring, headers=headers)
if not channel_id.get('is_success'):
return {
'succ':
False,
'data':
"{'msg': 'detail msg like this: %s'}" % channel_id.get('msg')
}
channel_id = channel_id.get('msg').json()
return {'succ': True, 'channel_ids': channel_id.get('data')}
def get_data(self, sd, ed, child_account, pid=None, pname=None, total=False):
"""
获取数据
:return:
"""
# 处理文件名
if total:
pid, pname = ('', '')
fname = '%s_%s_%s.json' % (child_account, sd, ed)
else:
fname = '%(childAcc)s_%(pname)s_%(start_date)s_%(end_date)s.json' % {
'childAcc': child_account, 'pname': pname.strip(), 'start_date': sd, 'end_date': ed
}
file_name = os.path.join(self.dir_path, fname)
if not pid and not total: # 没有数据, 没有产品
return {'succ': False, 'msg': 'no data'}
# 处理请求参数
url = 'http://e.mi.com/v2/report/dashboard'
querystring = {"customerId": child_account,"serviceToken": self.service_token, "xiaomiId": self.acc,
"sdate": sd, "edate": ed, "campaign": pid}
# 发送请求
data = get(url, params=querystring, headers=self._headers)
if not data['is_success']:
return {'succ': False, 'msg': data['msg']}
# 获取数据总数
data = data['msg'].content.decode('utf-8')
t_count = json.loads(data).get('result').get('conf').get('totalnum')
if t_count == 0: # 无消耗结算
return {'succ': False, 'msg': 'no data'}
# 写入文件
with open(file_name, 'w', encoding='utf-8') as f:
try:
f.write(data)
except Exception as e:
logger.error(e, exc_info=1)
logger.info('crawled data: --------' + data)
return {'succ': True}
def get_data(self, cookie, osd, oed, v_g_tk, cid):
headers = {
'Accept':
'*/*',
'cookie':
cookie,
'referer':
'http://op.open.qq.com/game_channel/atistic',
"user-agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"
}
url = "https://op.open.qq.com/game_channel/statis_get"
querystring = {
"channel_id": cid.get('channel_id'),
"g_tk": v_g_tk,
"begintime": osd.replace('-', ''),
"endtime": oed.replace('-', '')
}
statisres = get(url, params=querystring, headers=headers)
if statisres['msg'].status_code != 200:
return {
'succ': False,
'data': {
'msg': 'detail status_code not 200'
}
}
statisres = statisres.get('msg').json()
if statisres['ret'] != 0:
logger.info('url:%s, header:%s, detailres:%s' %
(url, headers, statisres))
return {'succ': False, 'data': {'msg': 'detail data error'}}
if len(statisres['data']) == 0:
logger.info('url:%s, header:%s, detailres:%s' %
(url, headers, statisres))
return {'succ': True, 'data': 'no data'}
return {'succ': True, 'data': statisres}
