def download_tick(self, response): content_type_header = response.headers.get('content-type', None) if content_type_header.decode( "utf-8" ) == 'application/vnd.ms-excel' or "当天没有数据" in response.body.decode( 'GB2312'): trading_date = response.meta['trading_date'] security_item = response.meta['item'] if content_type_header.decode( "utf-8") == 'application/vnd.ms-excel': content = response.body else: kdata_df = get_kdata(security_item, the_date=trading_date) if trading_date in kdata_df.index: json_data = kdata_df.loc[trading_date, :] content = kdata_to_tick(json_data) self.logger.info( "{} {} generate tick from kdata {}".format( security_item['code'], trading_date, content)) content = content.encode('GB2312') sina_tick_to_csv(security_item, io.BytesIO(content), trading_date) else: self.logger.exception( "get tick error:url={} content type={} body={}".format( response.url, content_type_header, response.body))
def handle_error_tick(): for index, security_item in get_security_list().iterrows(): dir = get_tick_dir(security_item) if os.path.exists(dir): files = [os.path.join(dir, f) for f in os.listdir(dir) if (('fatal' in f or 'error' in f) and os.path.isfile(os.path.join(dir, f)))] for f in files: try: the_date = get_file_name(f) csv_path = get_tick_path(security_item, the_date) if not os.path.exists(csv_path): logger.info("{} to {}".format(f, csv_path)) sina_tick_to_csv(security_item, f, the_date) except Exception as e: logger.warn(e) os.rename(f, f + ".fatal")
def handle_error_tick(): for index, security_item in get_security_list().iterrows(): dir = get_tick_dir(security_item) if os.path.exists(dir): files = [os.path.join(dir, f) for f in os.listdir(dir) if (('fatal' in f or 'error' in f) and os.path.isfile(os.path.join(dir, f)))] for f in files: try: the_date = get_file_name(f) csv_path = get_tick_path(security_item, the_date) if not os.path.exists(csv_path): logger.info("{} to {}".format(f, csv_path)) sina_tick_to_csv(security_item, f, the_date) except Exception as e: logger.warn(e) os.rename(f, f + ".fatal")
def download_tick(self, response): content_type_header = response.headers.get('content-type', None) if content_type_header.decode("utf-8") == 'application/vnd.ms-excel' or "当天没有数据" in response.body.decode( 'GB2312'): trading_date = response.meta['trading_date'] security_item = response.meta['item'] if content_type_header.decode("utf-8") == 'application/vnd.ms-excel': content = response.body else: kdata_json = get_kdata(security_item, trading_date).to_json() content = kdata_to_tick(kdata_json) self.logger.info("{} {} generate tick from kdata {}", security_item['code'], trading_date, content) content = content.encode('GB2312') sina_tick_to_csv(security_item, io.BytesIO(content), trading_date) else: self.logger.error( "get tick error:url={} content type={} body={}".format(response.url, content_type_header, response.body))