def download_tick(self, response):
content_type_header = response.headers.get('content-type', None)
if content_type_header.decode(
"utf-8"
) == 'application/vnd.ms-excel' or "当天没有数据" in response.body.decode(
'GB2312'):
trading_date = response.meta['trading_date']
security_item = response.meta['item']
if content_type_header.decode(
"utf-8") == 'application/vnd.ms-excel':
content = response.body
else:
kdata_df = get_kdata(security_item, the_date=trading_date)
if trading_date in kdata_df.index:
json_data = kdata_df.loc[trading_date, :]
content = kdata_to_tick(json_data)
self.logger.info(
"{} {} generate tick from kdata {}".format(
security_item['code'], trading_date, content))
content = content.encode('GB2312')
sina_tick_to_csv(security_item, io.BytesIO(content), trading_date)
else:
self.logger.exception(
"get tick error:url={} content type={} body={}".format(
response.url, content_type_header, response.body))
def handle_error_tick():
for index, security_item in get_security_list().iterrows():
dir = get_tick_dir(security_item)
if os.path.exists(dir):
files = [os.path.join(dir, f) for f in os.listdir(dir) if
(('fatal' in f or 'error' in f) and os.path.isfile(os.path.join(dir, f)))]
for f in files:
try:
the_date = get_file_name(f)
csv_path = get_tick_path(security_item, the_date)
if not os.path.exists(csv_path):
logger.info("{} to {}".format(f, csv_path))
sina_tick_to_csv(security_item, f, the_date)
except Exception as e:
logger.warn(e)
os.rename(f, f + ".fatal")
def handle_error_tick():
for index, security_item in get_security_list().iterrows():
dir = get_tick_dir(security_item)
if os.path.exists(dir):
files = [os.path.join(dir, f) for f in os.listdir(dir) if
(('fatal' in f or 'error' in f) and os.path.isfile(os.path.join(dir, f)))]
for f in files:
try:
the_date = get_file_name(f)
csv_path = get_tick_path(security_item, the_date)
if not os.path.exists(csv_path):
logger.info("{} to {}".format(f, csv_path))
sina_tick_to_csv(security_item, f, the_date)
except Exception as e:
logger.warn(e)
os.rename(f, f + ".fatal")
def download_tick(self, response):
content_type_header = response.headers.get('content-type', None)
if content_type_header.decode("utf-8") == 'application/vnd.ms-excel' or "当天没有数据" in response.body.decode(
'GB2312'):
trading_date = response.meta['trading_date']
security_item = response.meta['item']
if content_type_header.decode("utf-8") == 'application/vnd.ms-excel':
content = response.body
else:
kdata_json = get_kdata(security_item, trading_date).to_json()
content = kdata_to_tick(kdata_json)
self.logger.info("{} {} generate tick from kdata {}", security_item['code'], trading_date, content)
content = content.encode('GB2312')
sina_tick_to_csv(security_item, io.BytesIO(content), trading_date)
else:
self.logger.error(
"get tick error:url={} content type={} body={}".format(response.url, content_type_header,
response.body))