def test_configure(self): # default values are empty self.assertDictEqual({}, get_proxies()) # configure fake proxy configure_proxy_provider( SingleProxyProvider(dict(http="http-proxy", https="https-proxy"))) self.assertEqual("http-proxy", get_proxies()['http']) self.assertEqual("https-proxy", get_proxies()['https']) # reset proxy reset_proxy_provider() self.assertDictEqual({}, get_proxies())
def fetch(self, year: int, month: int, sid: str, retry: int = 5): params = {'date': '%d%02d01' % (year, month), 'stockNo': sid} for retry_i in range(retry): qtime = datetime.datetime.now() sleep_time = random.random() / 2 + 0.5 if (qtime - self.last_query_time ).total_seconds() < self.min_seconds_per_query: sleep_time += (qtime - self.last_query_time).total_seconds() print(f'sleep for {sleep_time}') time.sleep(sleep_time) print('awake') r = requests.get(self.REPORT_URL, params=params, proxies=get_proxies()) self.last_query_time = datetime.datetime.now() try: data = r.json() except JSONDecodeError: continue else: break else: # Fail in all retries data = {'stat': '', 'data': []} if data['stat'] == 'OK': data['data'] = self.purify(data) else: data['data'] = [] return data
def fetch(self, year: int, month: int, sid: str, retry: int=5): params = {'date': '%d%02d01' % (year, month), 'stockNo': sid} for retry_i in range(retry): if retry_i > 0: print('retry %d ...' % retry_i) while (time.time() - self.last_fectch_time) < 5.0: time.sleep(0.5) self.last_fectch_time = time.time() try: r = requests.get(self.REPORT_URL, params=params, proxies=get_proxies()) # try: data = r.json() except JSONDecodeError: continue except requests.exceptions.ConnectionError: continue else: break else: # Fail in all retries data = {'stat': '', 'data': []} if data['stat'] == 'OK': data['data'] = self.purify(data) else: data['data'] = [] return data
def fetch_data(url): r = requests.get(url, proxies=get_proxies()) root = etree.HTML(r.text) trs = root.xpath('//tr')[1:] result = [] typ = '' for tr in trs: tr = list(map(lambda x: x.text, tr.iter())) if len(tr) == 4: # This is type typ = tr[2].strip(' ') else: # This is the row data result.append(make_row_tuple(typ, tr)) return result
def get_raw(stocks) -> dict: req = requests.Session() req.get(SESSION_URL, proxies=get_proxies()) r = req.get( STOCKINFO_URL.format(stock_id=_join_stock_id(stocks), time=int(time.time()) * 1000)) if sys.version_info < (3, 5): try: return r.json() except ValueError: return {'rtmessage': 'json decode error', 'rtcode': '5000'} else: try: return r.json() except json.decoder.JSONDecodeError: return {'rtmessage': 'json decode error', 'rtcode': '5000'}
def fetch(self, year: int, month: int, sid: str, retry: int = 5): params = {'d': '%d/%d' % (year - 1911, month), 'stkno': sid} for retry_i in range(retry): r = requests.get(self.REPORT_URL, params=params, proxies=get_proxies()) try: data = r.json() except JSONDecodeError: continue else: break else: # Fail in all retries data = {'aaData': []} data['data'] = [] if data['aaData']: data['data'] = self.purify(data) return data
def fetch(self, year: int, month: int, sid: str, retry: int = 5): params = {'date': '%d%02d01' % (year, month), 'stockNo': sid} for retry_i in range(retry): r = requests.get(self.REPORT_URL, params=params, proxies=get_proxies()) try: data = r.json() except JSONDecodeError: continue else: break else: # Fail in all retries data = {'stat': '', 'data': []} if data['stat'] == 'OK': data['data'] = self.purify(data) else: data['data'] = [] return data