def get_data(self, osd, oed): headers = { 'Accept': 'application/json, text/javascript, */*; q=0.01', 'Cookie': self.cookies, 'Host': 'unicorn.wcd.qq.com', 'Referer': 'https://unicorn.wcd.qq.com/paydata/paydatafree.html', "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36" } # 请求1 productUrl = 'https://unicorn.wcd.qq.com/mcare/product.html' productRes = get(productUrl, headers=headers, timeout=120) if not productRes.get('is_success'): return {'succ': False, 'data': "{'msg': 'request time out'}"} productResJson = productRes.get('msg').json() productRes = [ x for x in productResJson.get('data') if int(x.get('key')) > 0 ] # 请求2 vendorUrl = 'https://unicorn.wcd.qq.com/mcare/vendor.html' vendorRes = get(vendorUrl, headers=headers, timeout=120) if not vendorRes.get('is_success'): return {'succ': False, 'data': "{'msg': 'request time out'}"} vendorResJson = vendorRes.get('msg').json() vendorRes = [ x.get('key') for x in vendorResJson.get('data') if int(x.get('key')) > 0 ] # 请求3 channelUrl = "https://unicorn.wcd.qq.com/mcare/channel.html" channelRes = get(channelUrl, headers=headers) if not channelRes.get('is_success'): return {'succ': False, 'data': "{'msg': 'request time out'}"} channelResStr = channelRes.get('msg').json() # channelRes = [x for x in channelResStr.get('data') if int(x.get('key')) > 0] # 请求4 self.get_product_detail(productRes, vendorRes, headers, osd, oed) return { 'succ': True, 'msg': { 'channelList': channelResStr, 'productList': productResJson, 'vendorList': vendorResJson } }
def get_uid_another_version(self): url = 'https://e.qq.com/gaea/api.php' cookie = self.d.get_cookies() cookie = {e['name']: e['value'] for e in cookie} skey = cookie.get('skey') if not cookie.get( 'gdt_protect') else cookie.get('gdt_protect') g_tk = self.getGTK(skey) params = {'mod': 'customer', 'act': 'getlist', 'g_tk': g_tk} cookie = '; '.join(['%s=%s' % (k, v) for k, v in cookie.items()]) headers = { 'accept': 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01', 'cookie': cookie, 'referer': 'https://e.qq.com/ads/', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3569.0 Safari/537.36' } res = get(url, params=params, headers=headers) if not res.get('is_success'): return {'succ': False, 'msg': res.get('msg')} res = res.get('msg').json() if res.get('ret') == 0: uid = res.get('data').get('list')[0].get('uid') return {'succ': True, 'msg': 'login success', 'uid': uid} else: return { 'succ': False, 'msg': 'login failed', 'desc': res.get('msg') }
def get_product(self, sd, ed): url = 'https://e.qq.com/ec/api.php' params = { 'mod': 'report', 'act': 'getproduct', 'g_tk': str(self.gtk), 'sdate': sd, 'edate': ed, 'searchtype': 'product', 'product_type': '20' } headers = { 'Accept': 'application/json, text/javascript, */*; q=0.01', 'referer': 'https://e.qq.com/atlas/%s/report/producttype' % self.uid, 'cookie': self.cookies_str, 'user-agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36" } res = get(url, params=params, headers=headers) if not res.get('is_success'): logger.error(res.get('msg').json()) data = res.get('msg').json() total_num = data.get('data').get('conf').get('totalnum') if total_num == 0: return {'succ': False, 'msg': 'no data'} data_list = data.get('data').get('list') data = [{ 'pname': e.get('pname'), 'pid': e.get('product_id'), 'cost': e.get('cost') } for e in data_list] return {'succ': True, 'msg': data}
def get_balance(self, uid): url = "https://e.qq.com/ec/api.php" querystring = { "act": "dashboard", "g_tk": self.gtk, "owner": uid, "advertiser_id": uid, "mod": "account", "unicode": "1" } headers = { 'accept': "application/json, text/javascript, */*; q=0.01", 'cookie': self.cookies.get('cookie_str'), 'referer': f"https://e.qq.com/atlas/{uid}/", 'user-agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36", 'Host': "e.qq.com", } res = get(url, params=querystring, headers=headers) if not res.get('is_success'): logger.error(res.get('msg'), exc_info=1) data = res.get('msg').json() if not data.get('ret') == 0: logger.error(data.get('msg')) return {'succ': False} accounts = data.get('data').get('accounts') return {'succ': True, 'msg': accounts}
def get_vc_img(self): try: url = self.d.find_element_by_class_name( 'geetest_item_img').get_attribute('src') except: return { 'succ': False, 'msg': 'slider verify, refresh to click verify' } logger.info('img_url: %s' % url) headers = { 'accept': "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", 'user-agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36", } res = get(url, headers=headers) if not res.get('is_success'): return {'succ': False, 'msg': 'net error'} img = res.get('msg').content with open(join(IMG_PATH, 'click_vc.png'), 'bw') as i: i.write(img) vc = u.rc.rk_create(img, 6906, timeout=90) del (img) # if not vc.get('Result'): # u.rc.rk_report_error(vc.get('Id')) # return self.get_vc_img() return {'succ': True, 'msg': vc}
def get_data_another_version(self, sd, ed, data, tid, tname): logger.info('get into (self.get_data_another_version)function') # 文件名 fname = '%(productType)s_%(productId)s_%(productName)s_%(sd)s_%(ed)s.json' % { 'productType': tname, 'productId': data.get('pid'), 'productName': data.get('pname'), 'sd': sd, 'ed': ed } # 构造请求参数 url = "https://e.qq.com/ec/api.php" params = {"g_tk": self.g_tk, 'owner': self.uid, "product_id": data.get('pid'), "product_type": tid, "sdate": sd, "edate": ed} headers = { "cookie": self.cookie_str, "referer": "http://e.qq.com/atlas/%(uid)s/report/analytic2?product_id=%(pid)s&product_type=%(ptype)s" % { 'uid': self.uid, 'pid': data.get('pid'), 'ptype': tid } } self.params.update(params) self._headers.update(headers) data = get(url, params=self.params, headers=self._headers) if not data['is_success']: return {'succ': False, 'msg': data['msg']} file_name = os.path.join(self.dir_path, fname) data = data['msg'].json() cost = data.get('data').get('total').get('cost').replace(',', '') if float(cost) == 0: return {'succ': False, 'msg': 'no data'} data['account'] = self.acc with open(file_name, 'w', encoding='utf-8') as f: json.dump(data, f) logger.info('crawled data: %s' % data) return {'succ': True}
def is_login(self, cookie, vc, im): url = 'http://adv.aiclk.com/user?version=%s' % time.strftime('%Y%m%d%H%M') cookie = '; '.join(['%s=%s' % (e.get('name'), e.get('value')) for e in cookie]) headers = { 'Accept': "application/json, text/javascript, */*; q=0.01", 'Cookie': cookie, 'Host': "adv.aiclk.com", 'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36", } res = get(url, headers=headers) if not res.get('is_success'): # 网络异常 return {'succ': False, 'msg': res.get('msg')} data = json.loads(res.get('msg').content) # logger.info(data) if data.get('errorType') == 'NoLogin': # 登陆失败,两个key: retdesc, retcode msg = self.d.find_element_by_class_name('ui-dialog-text').text if msg == '开户资料正在审核中': return {'succ': True, 'msg': msg} elif msg == '验证码错误': u.rc.rk_report_error(vc.get('Id')) return {'succ': False, 'msg': 'verify_code_error'} elif msg == '用户名或者密码错误': return {'succ': False, 'msg': 'account_or_password_error'} else: return {'succ': False, 'msg': data.get('message')} else: u.rc.rk_report(im, 3040, vc.get('Result'), vc_type=self.user_info.get('platform')) return {'succ': True, 'msg': 'login success'}
def is_login(self, data): cookie = '; '.join(['%s=%s' % (e['name'], e['value']) for e in data]) url = 'https://dev.vivo.com.cn/webapi/user/info?timestamp=%s' % int( time.time() * 1000) haeders = { 'Accept': 'application/json, text/plain, */*', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'zh-CN,zh;q=0.9', 'Content-Type': 'application/json;charset=UTF-8', 'Cookie': cookie, 'Host': 'dev.vivo.com.cn', 'Origin': 'https://dev.vivo.com.cn', 'Referer': 'https://dev.vivo.com.cn/home', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36', } res = get(url, headers=haeders, timeout=10) if not res.get('is_success'): raise Exception(res.get('msg')) res = json.loads(res.get('msg').content) if res.get('code') != 0: logger.info('login failed') return {'succ': False, 'msg': 'login failed'} logger.info('login success') return {'succ': True, 'msg': 'login success'}
def is_login(self, cookie): cookie = '; '.join( ['%s=%s' % (e.get('name'), e.get('value')) for e in cookie]) yesterday = int(time.strftime('%d')) - 1 day = '%s-%s' % (time.strftime('%Y-%m'), yesterday) url = 'http://agent.e.sogou.com/main/plot.html?start=%s&end=%s&type=2' % ( day, day) headers = { 'Accept': "application/json, text/javascript, */*; q=0.01", 'Cookie': cookie, 'Host': "agent.e.sogou.com", 'Referer': "http://agent.e.sogou.com/main.html", 'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36", 'Content-Type': "application/x-www-form-urlencoded" } res = get(url, headers=headers) if not res.get('is_success'): # 网络异常 return {'succ': False, 'msg': res.get('msg')} data = json.loads(res.get('msg').content) # logger.info(data) if 'plotList' in data.keys(): # 正常情况下,只有一个key: plotList return {'succ': True, 'msg': 'login succ'} elif data.get('retcode') == 401: # 登陆失败,两个key: retdesc, retcode return {'succ': False, 'msg': data.get('retdesc')} else: return {'succ': False, 'msg': 'unknown error'}
def save_pic(self, src, pic_name): from requests import get headers = { 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36' } res = get(src, headers=headers) with open(pic_name, 'bw') as f: f.write(res.content)
def get_main_content(self, p_cf, cookie, sd, ed): # 获取主账号的数据,并且完成截图 # get_data------ids url = 'https://e.yunos.com/api/campaign/list/layout' headers = { 'accept': "application/json, text/javascript", 'content-type': "application/x-www-form-urlencoded", 'cookie': cookie, 'referer': "https://e.yunos.com/", 'user-agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", } data = get(url, headers=headers) if not data.get('is_success'): return data id_list = json.loads(data.get('msg').content.decode('utf-8')) if id_list.get('total_count') == 0: return {'succ': False, 'msg': 'no data', 'account': self.acc} ids = [(e.get('id'), e.get('title')) for e in id_list.get('data') if '东方' in e.get('title')] # get_data------data filename = join(self.dir_path, 'mainContent_#%s_#%s.json' % (sd, ed)) # change file name url2 = 'https://e.yunos.com/api/rpt/list' payload = { 'param': json.dumps({ "page": 1, "page_size": 99999, "date_range_type": 0, "campaign_id": [i[0] for i in ids], "report_level": 1, "report_data_type": 1, "start_ds": sd, "end_ds": ed }), 'p_cf': p_cf } data = post(url2, data=payload, headers=headers) if not data.get('is_success'): return {'succ': False, 'msg': data.get('msg')} data = json.loads(data.get('msg').content.decode('utf-8')) if data.get('total_count') == 0: return {'succ': False, 'msg': 'no data', 'account': self.acc} with open(filename, 'w', encoding='utf-8') as f: json.dump(data, f) return {'succ': True, 'msg': data, 'ids': ids, 'sd': sd, 'ed': ed}
def get_child_accounts(self, cookie, reget=True, psize='20'): """获取子账号列表""" url = 'https://e.yunos.com/api/member/agency/ref/list?keyword=&page_size=%s&page=1' % psize headers = {'cookie': cookie, 'referer': 'https://e.yunos.com/'} self.headers.update(headers) data = get(url, headers=self.headers) if not data['is_success']: return {'succ': False, 'msg': data['msg']} data = json.loads(data['msg'].content.decode()) if reget: total_count = data.get('total_count') return self.get_child_accounts(cookie, reget=False, psize=total_count) accounts = [(x['username'], x['company']) for x in data['data'] if x['status'] == 1 and x['privilege'] == 1 and x['ref_status'] == 1] return {'succ': True, 'msg': accounts}
def get_type_ids(self): """获取类型id""" url = 'https://e.qq.com/ec/api.php' querystring = {"mod": "product", "act": "getproducttypelist", "g_tk": self.g_tk, "owner": self.uid} headers = { 'accept': "application/json, text/javascript, */*; q=0.01", 'cookie': self.cookie_str, 'referer': "https://e.qq.com/atlas/%s/report/producttype" % self.uid, 'user-agent': "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3569.0 Safari/537.36" } res = get(url, params=querystring, headers=headers) if not res.get('is_success'): return {'succ': False, 'msg': res.get('msg')} data = res.get('msg').json() ids = [(x.get('id'), x.get('name')) for x in data.get('data')] return {'succ': True, 'msg': ids}
def get_product_detail(self, productRes, vendorRes, headers, osd, oed): # 请求4 for item in productRes: curPage = 1 total = -1 pageSize = 100 alldata = [] productId = item.get('key') logger.info('开始产品:%s' % item.get('value')) while total == -1 or curPage <= math.ceil(total / pageSize): url = 'https://unicorn.wcd.qq.com/free/getPayDataTable.html' params = { 'timeArea': osd.replace('-', '') + ' - ' + oed.replace('-', ''), 'product': productId, 'platform': 'android,iphone,ipad,apad,windows', 'vendor': ','.join(vendorRes), # 'channel': ','.join(channelRes), 'channel': None, 'datatype': 'd', 'dataValue': 'newUser', 'fee': '1,0', 'fileType': '', 'pageSize': pageSize, 'curPage': curPage } logger.info('product:%s, curPage:%d, total:%d' % (item.get('value'), curPage, total)) dataRes = get(url, params=params, headers=headers) if not dataRes.get('is_success'): logger.error( 'product:%s, curPage:%d, total:%d, got failed' % (item.get('value'), curPage, total)) curPage += 1 continue dataRes = dataRes.get('msg').json() alldata = alldata + dataRes.get('data') total = dataRes['total'] curPage += 1 if not alldata: continue product_file_name = os.path.join( self.dir_path, '%s_%s_%s.json' % (osd, oed, item.get('value'))) with open(product_file_name, 'w', encoding='utf-8') as f: f.write(json.dumps(alldata, ensure_ascii=False))
def get_cid(self, identity): """ 获取组id :param identity: 子账号 需要编码 :return: cid list """ identity = quote(identity) url = 'https://e.yunos.com/api/campaign/list/layout?identity=%s' % identity headers = {"referer": "https://e.yunos.com/?identity=%s" % identity} self._headers.update(headers) data = get(url, headers=self._headers) if not data['is_success']: return {'succ': False, 'msg': data['msg']} data = json.loads(data['msg'].content.decode()) cid = [x['id'] for x in data['data']] if data['total_count'] > 0 else [] return {'succ': True, 'msg': cid}
def get_child_accounts(self): """获取子账号列表""" url = 'http://e.mi.com/v2/account/agent/customers' headers = {'Cookie': self.cookies, 'Referer': "http://e.mi.com/v2/dist/agent.html"} self._headers.update(headers) querystring = {"agent": self.agent} # 替换为agent data = get(url, params=querystring, headers=self._headers) if not data['is_success']: return {'succ': False, 'msg': data['msg']} data = data['msg'].json() if not data.get('success'): return {'succ': False, 'msg': data.get('failMsg'), 'desc': 'is_child_account'} # if reget: # return self.get_child_accounts(agent, reget=False) accounts = [x.get('customerId') for x in data.get('result').get('list') if x.get('status') == 2] data = [i for i in data.get('result').get('list') if i.get('customerId') in self.none_cost] self.balance_data = {'list': data} return {'succ': True, 'accounts': accounts}
def has_data(self, sd, ed, productId, vendor, channel): cookie = '; '.join([ '%s=%s' % (e.get('name'), e.get('value')) for e in self.d.get_cookies() ]) headers = { 'Accept': 'application/json, text/javascript, */*; q=0.01', 'Cookie': cookie, 'Host': 'unicorn.wcd.qq.com', 'Referer': 'https://unicorn.wcd.qq.com/paydata/paydatafree.html', "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36" } url = 'https://unicorn.wcd.qq.com/free/getPayDataTable.html' params = { 'timeArea': '%s - %s' % (sd, ed), 'product': ','.join(productId), 'platform': 'android,iphone,ipad,apad,windows', 'vendor': ','.join(vendor), 'channel': channel, 'datatype': 'd', 'dataValue': 'newUser', 'fee': '1,0', 'fileType': '', 'pageSize': 50, 'curPage': 1 } dataRes = get(url, params=params, headers=headers) if not dataRes.get('is_success'): return False dataRes = dataRes.get('msg').json() time.sleep(0.2) if dataRes.get('total') == 0: return False return dataRes.get('total')
def get_channels(self, cookie, v_g_tk): headers = { 'Accept': '*/*', 'Cookie': cookie, 'Referer': 'https://op.open.qq.com/game_channel/atistic', "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36" } url = "https://op.open.qq.com/game_channel/info" querystring = {"g_tk": v_g_tk, "act": "all"} channel_id = get(url, params=querystring, headers=headers) if not channel_id.get('is_success'): return { 'succ': False, 'data': "{'msg': 'detail msg like this: %s'}" % channel_id.get('msg') } channel_id = channel_id.get('msg').json() return {'succ': True, 'channel_ids': channel_id.get('data')}
def get_data(self, sd, ed, child_account, pid=None, pname=None, total=False): """ 获取数据 :return: """ # 处理文件名 if total: pid, pname = ('', '') fname = '%s_%s_%s.json' % (child_account, sd, ed) else: fname = '%(childAcc)s_%(pname)s_%(start_date)s_%(end_date)s.json' % { 'childAcc': child_account, 'pname': pname.strip(), 'start_date': sd, 'end_date': ed } file_name = os.path.join(self.dir_path, fname) if not pid and not total: # 没有数据, 没有产品 return {'succ': False, 'msg': 'no data'} # 处理请求参数 url = 'http://e.mi.com/v2/report/dashboard' querystring = {"customerId": child_account,"serviceToken": self.service_token, "xiaomiId": self.acc, "sdate": sd, "edate": ed, "campaign": pid} # 发送请求 data = get(url, params=querystring, headers=self._headers) if not data['is_success']: return {'succ': False, 'msg': data['msg']} # 获取数据总数 data = data['msg'].content.decode('utf-8') t_count = json.loads(data).get('result').get('conf').get('totalnum') if t_count == 0: # 无消耗结算 return {'succ': False, 'msg': 'no data'} # 写入文件 with open(file_name, 'w', encoding='utf-8') as f: try: f.write(data) except Exception as e: logger.error(e, exc_info=1) logger.info('crawled data: --------' + data) return {'succ': True}
def get_data(self, cookie, osd, oed, v_g_tk, cid): headers = { 'Accept': '*/*', 'cookie': cookie, 'referer': 'http://op.open.qq.com/game_channel/atistic', "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36" } url = "https://op.open.qq.com/game_channel/statis_get" querystring = { "channel_id": cid.get('channel_id'), "g_tk": v_g_tk, "begintime": osd.replace('-', ''), "endtime": oed.replace('-', '') } statisres = get(url, params=querystring, headers=headers) if statisres['msg'].status_code != 200: return { 'succ': False, 'data': { 'msg': 'detail status_code not 200' } } statisres = statisres.get('msg').json() if statisres['ret'] != 0: logger.info('url:%s, header:%s, detailres:%s' % (url, headers, statisres)) return {'succ': False, 'data': {'msg': 'detail data error'}} if len(statisres['data']) == 0: logger.info('url:%s, header:%s, detailres:%s' % (url, headers, statisres)) return {'succ': True, 'data': 'no data'} return {'succ': True, 'data': statisres}