def sina_tick_to_csv(security_item, the_content, the_date):
csv_path = get_tick_path(security_item, the_date)
df = read_csv(the_content, "GB2312", sep='\s+')
df = df.loc[:, ['成交时间', '成交价', '成交量(手)', '成交额(元)', '性质']]
df.columns = TICK_COLUNM
df['direction'] = df['direction'].apply(lambda x: direction_to_int(x))
df.to_csv(csv_path, index=False)
def sina_tick_to_csv(security_item, the_content, the_date):
csv_path = get_tick_path(security_item, the_date)
df = read_csv(the_content, "GB2312", sep='\s+')
df = df.loc[:, ['成交时间', '成交价', '成交量(手)', '成交额(元)', '性质']]
df.columns = TICK_COLUNM
df['direction'] = df['direction'].apply(lambda x: direction_to_int(x))
df.to_csv(csv_path, index=False)
def handle_error_tick():
for index, security_item in get_security_list().iterrows():
dir = get_tick_dir(security_item)
if os.path.exists(dir):
files = [os.path.join(dir, f) for f in os.listdir(dir) if
(('fatal' in f or 'error' in f) and os.path.isfile(os.path.join(dir, f)))]
for f in files:
try:
the_date = get_file_name(f)
csv_path = get_tick_path(security_item, the_date)
if not os.path.exists(csv_path):
logger.info("{} to {}".format(f, csv_path))
sina_tick_to_csv(security_item, f, the_date)
except Exception as e:
logger.warn(e)
os.rename(f, f + ".fatal")
def handle_error_tick():
for index, security_item in get_security_list().iterrows():
dir = get_tick_dir(security_item)
if os.path.exists(dir):
files = [os.path.join(dir, f) for f in os.listdir(dir) if
(('fatal' in f or 'error' in f) and os.path.isfile(os.path.join(dir, f)))]
for f in files:
try:
the_date = get_file_name(f)
csv_path = get_tick_path(security_item, the_date)
if not os.path.exists(csv_path):
logger.info("{} to {}".format(f, csv_path))
sina_tick_to_csv(security_item, f, the_date)
except Exception as e:
logger.warn(e)
os.rename(f, f + ".fatal")
def get_ticks(security_item, the_date=None, start_date=None, end_date=None):
"""
get the ticks.
Parameters
----------
security_item : SecurityItem or str
the security item,id or code
the_date : TimeStamp str or TimeStamp
get the tick for the exact date
start_date : TimeStamp str or TimeStamp
start date
end_date: TimeStamp str or TimeStamp
end date
Yields
-------
DataFrame
"""
security_item = to_security_item(security_item)
if the_date:
the_date = to_time_str(the_date)
tick_path = files_contract.get_tick_path(security_item, the_date)
yield _parse_tick(tick_path, security_item)
else:
tick_dir = files_contract.get_tick_dir(security_item)
if start_date or end_date:
if not start_date:
start_date = security_item['listDate']
if not end_date:
end_date = datetime.datetime.today()
tick_paths = [
os.path.join(tick_dir, f) for f in os.listdir(tick_dir)
if get_file_name(f) in pd.date_range(start=start_date,
end=end_date)
]
else:
tick_paths = [
os.path.join(tick_dir, f) for f in os.listdir(tick_dir)
]
for tick_path in sorted(tick_paths):
yield _parse_tick(tick_path, security_item)
def yield_request(self, item, trading_dates=None):
if not trading_dates:
trading_dates = get_trading_dates(item)
for trading_date in trading_dates:
if get_datetime(trading_date) < get_datetime(settings.START_TICK_DATE) or get_datetime(
trading_date) < get_datetime(settings.AVAILABLE_TICK_DATE):
continue
path = get_tick_path(item, trading_date)
if os.path.exists(path):
continue
yield Request(url=self.get_tick_url(trading_date, item['exchange'] + item['code']),
meta={'proxy': None,
'path': path,
'trading_date': trading_date,
'item': item},
headers=DEFAULT_TICK_HEADER,
callback=self.download_tick)
def get_ticks(security_item, the_date=None, start=None, end=None):
if the_date:
tick_path = files_contract.get_tick_path(security_item, the_date)
return parse_tick(tick_path)
else:
tick_dir = files_contract.get_tick_dir(security_item)
if start or end:
if not start:
start = security_item['listDate']
if not end:
end = datetime.datetime.today()
tick_paths = [os.path.join(tick_dir, f) for f in
os.listdir(tick_dir) if
get_file_name(f) in pd.date_range(start=start, end=end)]
else:
tick_paths = [os.path.join(tick_dir, f) for f in
os.listdir(tick_dir)]
for tick_path in sorted(tick_paths):
yield parse_tick(tick_path, security_item)
def get_ticks(security_item, the_date=None, start=None, end=None):
if the_date:
tick_path = files_contract.get_tick_path(security_item, the_date)
return parse_tick(tick_path)
else:
tick_dir = files_contract.get_tick_dir(security_item)
if start or end:
if not start:
start = security_item['listDate']
if not end:
end = datetime.datetime.today()
tick_paths = [os.path.join(tick_dir, f) for f in
os.listdir(tick_dir) if
get_file_name(f) in pd.date_range(start=start, end=end)]
else:
tick_paths = [os.path.join(tick_dir, f) for f in
os.listdir(tick_dir)]
for tick_path in sorted(tick_paths):
yield parse_tick(tick_path, security_item)
def get_ticks(security_item, the_date=None, start=None, end=None):
"""
get the ticks.
Parameters
----------
security_item : SecurityItem
the security item
the_date : TimeStamp str or TimeStamp
get the tick for the exact date
start : TimeStamp str or TimeStamp
start date
end: TimeStamp str or TimeStamp
end date
Returns
-------
DataFrame
"""
if the_date:
tick_path = files_contract.get_tick_path(security_item, the_date)
return _parse_tick(tick_path)
else:
tick_dir = files_contract.get_tick_dir(security_item)
if start or end:
if not start:
start = security_item['listDate']
if not end:
end = datetime.datetime.today()
tick_paths = [os.path.join(tick_dir, f) for f in
os.listdir(tick_dir) if
get_file_name(f) in pd.date_range(start=start, end=end)]
else:
tick_paths = [os.path.join(tick_dir, f) for f in
os.listdir(tick_dir)]
for tick_path in sorted(tick_paths):
yield _parse_tick(tick_path, security_item)
