def download_day_k_data(self, response):
path = response.meta['path']
item = response.meta['item']
trading_dates = []
try:
tmp_str = response.text
json_str = tmp_str[tmp_str.index('{'):tmp_str.index('}') + 1]
tmp_json = json.loads(json_str)
# parse the trading dates
dates = [items[0] for items in tmp_json['kline']]
trading_dates = [
datetime.strptime(str(the_date),
'%Y%m%d').strftime(TIME_FORMAT_DAY)
for the_date in dates
]
except Exception as e:
self.logger.error(
'error when getting k data url={} error={}'.format(
response.url, e))
if len(trading_dates) > 0:
try:
with open(get_trading_dates_path_sse(item), "w") as f:
json.dump(trading_dates, f)
except Exception as e:
self.logger.error(
'error when saving trading dates url={} path={} error={}'.
format(response.url, path, e))
def download_day_k_data(self, response):
path = response.meta['path']
item = response.meta['item']
trading_dates = []
try:
tmp_str = response.text
json_str = tmp_str[tmp_str.index('{'):tmp_str.index('}') + 1]
tmp_json = json.loads(json_str)
# parse the trading dates
dates = [items[0] for items in tmp_json['kline']]
trading_dates = [datetime.strptime(str(the_date), '%Y%m%d').strftime(TIME_FORMAT_DAY) for the_date in dates]
except Exception as e:
self.logger.error('error when getting k data url={} error={}'.format(response.url, e))
if len(trading_dates) > 0:
try:
with open(get_trading_dates_path_sse(item), "w") as f:
json.dump(trading_dates, f)
except Exception as e:
self.logger.error(
'error when saving trading dates url={} path={} error={}'.format(response.url, path, e))
def yield_request(self, item):
data_path = get_trading_dates_path_sse(item) # get day k data
url = self.get_k_data_url(item['exchange'], item['code'])
yield Request(url=url, headers=SSE_KDATA_HEADER,
meta={'path': data_path, 'item': item},
callback=self.download_day_k_data)
def yield_request(self, item):
data_path = get_trading_dates_path_sse(item) # get day k data
url = self.get_k_data_url(item['exchange'], item['code'])
yield Request(url=url,
headers=SSE_KDATA_HEADER,
meta={
'path': data_path,
'item': item
},
callback=self.download_day_k_data)