def test_configure(self):
# default values are empty
self.assertDictEqual({}, get_proxies())
# configure fake proxy
configure_proxy_provider(
SingleProxyProvider(dict(http="http-proxy", https="https-proxy")))
self.assertEqual("http-proxy", get_proxies()['http'])
self.assertEqual("https-proxy", get_proxies()['https'])
# reset proxy
reset_proxy_provider()
self.assertDictEqual({}, get_proxies())
def fetch(self, year: int, month: int, sid: str, retry: int = 5):
params = {'date': '%d%02d01' % (year, month), 'stockNo': sid}
for retry_i in range(retry):
qtime = datetime.datetime.now()
sleep_time = random.random() / 2 + 0.5
if (qtime - self.last_query_time
).total_seconds() < self.min_seconds_per_query:
sleep_time += (qtime - self.last_query_time).total_seconds()
print(f'sleep for {sleep_time}')
time.sleep(sleep_time)
print('awake')
r = requests.get(self.REPORT_URL,
params=params,
proxies=get_proxies())
self.last_query_time = datetime.datetime.now()
try:
data = r.json()
except JSONDecodeError:
continue
else:
break
else:
# Fail in all retries
data = {'stat': '', 'data': []}
if data['stat'] == 'OK':
data['data'] = self.purify(data)
else:
data['data'] = []
return data
def fetch(self, year: int, month: int, sid: str, retry: int=5):
params = {'date': '%d%02d01' % (year, month), 'stockNo': sid}
for retry_i in range(retry):
if retry_i > 0:
print('retry %d ...' % retry_i)
while (time.time() - self.last_fectch_time) < 5.0:
time.sleep(0.5)
self.last_fectch_time = time.time()
try:
r = requests.get(self.REPORT_URL, params=params,
proxies=get_proxies())
# try:
data = r.json()
except JSONDecodeError:
continue
except requests.exceptions.ConnectionError:
continue
else:
break
else:
# Fail in all retries
data = {'stat': '', 'data': []}
if data['stat'] == 'OK':
data['data'] = self.purify(data)
else:
data['data'] = []
return data
def fetch_data(url):
r = requests.get(url, proxies=get_proxies())
root = etree.HTML(r.text)
trs = root.xpath('//tr')[1:]
result = []
typ = ''
for tr in trs:
tr = list(map(lambda x: x.text, tr.iter()))
if len(tr) == 4:
# This is type
typ = tr[2].strip(' ')
else:
# This is the row data
result.append(make_row_tuple(typ, tr))
return result
def get_raw(stocks) -> dict:
req = requests.Session()
req.get(SESSION_URL, proxies=get_proxies())
r = req.get(
STOCKINFO_URL.format(stock_id=_join_stock_id(stocks),
time=int(time.time()) * 1000))
if sys.version_info < (3, 5):
try:
return r.json()
except ValueError:
return {'rtmessage': 'json decode error', 'rtcode': '5000'}
else:
try:
return r.json()
except json.decoder.JSONDecodeError:
return {'rtmessage': 'json decode error', 'rtcode': '5000'}
def fetch(self, year: int, month: int, sid: str, retry: int = 5):
params = {'d': '%d/%d' % (year - 1911, month), 'stkno': sid}
for retry_i in range(retry):
r = requests.get(self.REPORT_URL,
params=params,
proxies=get_proxies())
try:
data = r.json()
except JSONDecodeError:
continue
else:
break
else:
# Fail in all retries
data = {'aaData': []}
data['data'] = []
if data['aaData']:
data['data'] = self.purify(data)
return data
def fetch(self, year: int, month: int, sid: str, retry: int = 5):
params = {'date': '%d%02d01' % (year, month), 'stockNo': sid}
for retry_i in range(retry):
r = requests.get(self.REPORT_URL,
params=params,
proxies=get_proxies())
try:
data = r.json()
except JSONDecodeError:
continue
else:
break
else:
# Fail in all retries
data = {'stat': '', 'data': []}
if data['stat'] == 'OK':
data['data'] = self.purify(data)
else:
data['data'] = []
return data
