def download_day_k_data(self, response): path = response.meta['path'] item = response.meta['item'] trading_dates = [] try: tmp_str = response.text json_str = tmp_str[tmp_str.index('{'):tmp_str.index('}') + 1] tmp_json = json.loads(json_str) # parse the trading dates dates = [items[0] for items in tmp_json['kline']] trading_dates = [ datetime.strptime(str(the_date), '%Y%m%d').strftime(TIME_FORMAT_DAY) for the_date in dates ] except Exception as e: self.logger.error( 'error when getting k data url={} error={}'.format( response.url, e)) if len(trading_dates) > 0: try: with open(get_trading_dates_path_sse(item), "w") as f: json.dump(trading_dates, f) except Exception as e: self.logger.error( 'error when saving trading dates url={} path={} error={}'. format(response.url, path, e))
def download_day_k_data(self, response): path = response.meta['path'] item = response.meta['item'] trading_dates = [] try: tmp_str = response.text json_str = tmp_str[tmp_str.index('{'):tmp_str.index('}') + 1] tmp_json = json.loads(json_str) # parse the trading dates dates = [items[0] for items in tmp_json['kline']] trading_dates = [datetime.strptime(str(the_date), '%Y%m%d').strftime(TIME_FORMAT_DAY) for the_date in dates] except Exception as e: self.logger.error('error when getting k data url={} error={}'.format(response.url, e)) if len(trading_dates) > 0: try: with open(get_trading_dates_path_sse(item), "w") as f: json.dump(trading_dates, f) except Exception as e: self.logger.error( 'error when saving trading dates url={} path={} error={}'.format(response.url, path, e))
def yield_request(self, item): data_path = get_trading_dates_path_sse(item) # get day k data url = self.get_k_data_url(item['exchange'], item['code']) yield Request(url=url, headers=SSE_KDATA_HEADER, meta={'path': data_path, 'item': item}, callback=self.download_day_k_data)
def yield_request(self, item): data_path = get_trading_dates_path_sse(item) # get day k data url = self.get_k_data_url(item['exchange'], item['code']) yield Request(url=url, headers=SSE_KDATA_HEADER, meta={ 'path': data_path, 'item': item }, callback=self.download_day_k_data)