def main(): spider_list = get_code() for spider in spider_list: slug = spider["slug"] spider_coin_record_id = spider["id"] url = "https://dncapi.bqiapp.com/api/v2/coin/bigevent?coincode=%s&webp=1" % slug restexts = common._request(url, "get", {"User-Agent": common.get_randomUa()}) parse(restexts, spider_coin_record_id)
def requests_get(url): try: headers = {"User-Agent": common.get_randomUa()} r = requests.get(url=url, headers=headers) if r.status_code == 200: # print(r.text) return r.text else: print("错误的状态码:{}".format(r.status_code)) return False except Exception as e: print(e) return False
def get_data(slug): url = "https://dncapi.bqiapp.com/api/coin/cointrades-web?code=%s&webp=1" % slug try: headers = {"User-Agent": common.get_randomUa()} r = requests.get(url=url, headers=headers) if r.status_code == 200: # print(r.text) return r.text else: print("错误的状态码:{}".format(r.status_code)) return False except Exception as e: print(e) return False
def process_request(self, request, spider): ua = common.get_randomUa() # print(ua) request.headers['User-Agent'] = ua