def remove_old_tick():
for index, security_item in get_security_list().iterrows():
dir = get_tick_dir(security_item)
if os.path.exists(dir):
files = [os.path.join(dir, f) for f in os.listdir(dir) if
('xls' in f and 'lock' not in f and 'error' not in f and os.path.isfile(os.path.join(dir, f)))]
for f in files:
logger.info("remove {}".format(f))
os.remove(f)
def remove_old_tick():
for index, security_item in get_security_list().iterrows():
dir = get_tick_dir(security_item)
if os.path.exists(dir):
files = [os.path.join(dir, f) for f in os.listdir(dir) if
('xls' in f and 'lock' not in f and 'error' not in f and os.path.isfile(os.path.join(dir, f)))]
for f in files:
logger.info("remove {}".format(f))
os.remove(f)
def check_result():
for index, security_item in get_security_list().iterrows():
for fuquan in ('bfq', 'hfq'):
dayk_path = get_kdata_path(security_item, fuquan=fuquan)
if not os.path.exists(dayk_path):
logger.warn(get_security_dir(security_item))
dir = get_tick_dir(security_item)
if os.path.exists(dir):
files = [os.path.join(dir, f) for f in os.listdir(dir) if
('csv' in f and os.path.isfile(os.path.join(dir, f)))]
if not files:
logger.warn(get_security_dir(security_item))
def check_result():
for index, security_item in get_security_list().iterrows():
for fuquan in ('bfq', 'hfq'):
dayk_path = get_kdata_path(security_item, fuquan=fuquan)
if not os.path.exists(dayk_path):
logger.warn(get_security_dir(security_item))
dir = get_tick_dir(security_item)
if os.path.exists(dir):
files = [os.path.join(dir, f) for f in os.listdir(dir) if
('csv' in f and os.path.isfile(os.path.join(dir, f)))]
if not files:
logger.warn(get_security_dir(security_item))
def handle_error_tick():
for index, security_item in get_security_list().iterrows():
dir = get_tick_dir(security_item)
if os.path.exists(dir):
files = [os.path.join(dir, f) for f in os.listdir(dir) if
(('fatal' in f or 'error' in f) and os.path.isfile(os.path.join(dir, f)))]
for f in files:
try:
the_date = get_file_name(f)
csv_path = get_tick_path(security_item, the_date)
if not os.path.exists(csv_path):
logger.info("{} to {}".format(f, csv_path))
sina_tick_to_csv(security_item, f, the_date)
except Exception as e:
logger.warn(e)
os.rename(f, f + ".fatal")
def handle_error_tick():
for index, security_item in get_security_list().iterrows():
dir = get_tick_dir(security_item)
if os.path.exists(dir):
files = [os.path.join(dir, f) for f in os.listdir(dir) if
(('fatal' in f or 'error' in f) and os.path.isfile(os.path.join(dir, f)))]
for f in files:
try:
the_date = get_file_name(f)
csv_path = get_tick_path(security_item, the_date)
if not os.path.exists(csv_path):
logger.info("{} to {}".format(f, csv_path))
sina_tick_to_csv(security_item, f, the_date)
except Exception as e:
logger.warn(e)
os.rename(f, f + ".fatal")
def get_ticks(security_item, the_date=None, start_date=None, end_date=None):
"""
get the ticks.
Parameters
----------
security_item : SecurityItem or str
the security item,id or code
the_date : TimeStamp str or TimeStamp
get the tick for the exact date
start_date : TimeStamp str or TimeStamp
start date
end_date: TimeStamp str or TimeStamp
end date
Yields
-------
DataFrame
"""
security_item = to_security_item(security_item)
if the_date:
the_date = to_time_str(the_date)
tick_path = files_contract.get_tick_path(security_item, the_date)
yield _parse_tick(tick_path, security_item)
else:
tick_dir = files_contract.get_tick_dir(security_item)
if start_date or end_date:
if not start_date:
start_date = security_item['listDate']
if not end_date:
end_date = datetime.datetime.today()
tick_paths = [
os.path.join(tick_dir, f) for f in os.listdir(tick_dir)
if get_file_name(f) in pd.date_range(start=start_date,
end=end_date)
]
else:
tick_paths = [
os.path.join(tick_dir, f) for f in os.listdir(tick_dir)
]
for tick_path in sorted(tick_paths):
yield _parse_tick(tick_path, security_item)
def mkdir_for_security(item):
finance_dir = get_finance_dir(item)
if not os.path.exists(finance_dir):
os.makedirs(finance_dir)
tick_dir = get_tick_dir(item)
if not os.path.exists(tick_dir):
os.makedirs(tick_dir)
event_dir = get_event_dir(item)
if not os.path.exists(event_dir):
os.makedirs(event_dir)
bfq_kdata_dir = get_kdata_dir(item, 'bfq')
if not os.path.exists(bfq_kdata_dir):
os.makedirs(bfq_kdata_dir)
hfq_kdata_dir = get_kdata_dir(item, 'hfq')
if not os.path.exists(hfq_kdata_dir):
os.makedirs(hfq_kdata_dir)
def get_ticks(security_item, the_date=None, start=None, end=None):
if the_date:
tick_path = files_contract.get_tick_path(security_item, the_date)
return parse_tick(tick_path)
else:
tick_dir = files_contract.get_tick_dir(security_item)
if start or end:
if not start:
start = security_item['listDate']
if not end:
end = datetime.datetime.today()
tick_paths = [os.path.join(tick_dir, f) for f in
os.listdir(tick_dir) if
get_file_name(f) in pd.date_range(start=start, end=end)]
else:
tick_paths = [os.path.join(tick_dir, f) for f in
os.listdir(tick_dir)]
for tick_path in sorted(tick_paths):
yield parse_tick(tick_path, security_item)
def get_ticks(security_item, the_date=None, start=None, end=None):
if the_date:
tick_path = files_contract.get_tick_path(security_item, the_date)
return parse_tick(tick_path)
else:
tick_dir = files_contract.get_tick_dir(security_item)
if start or end:
if not start:
start = security_item['listDate']
if not end:
end = datetime.datetime.today()
tick_paths = [os.path.join(tick_dir, f) for f in
os.listdir(tick_dir) if
get_file_name(f) in pd.date_range(start=start, end=end)]
else:
tick_paths = [os.path.join(tick_dir, f) for f in
os.listdir(tick_dir)]
for tick_path in sorted(tick_paths):
yield parse_tick(tick_path, security_item)
def mkdir_for_security(item):
finance_dir = get_finance_dir(item)
if not os.path.exists(finance_dir):
os.makedirs(finance_dir)
tick_dir = get_tick_dir(item)
if not os.path.exists(tick_dir):
os.makedirs(tick_dir)
event_dir = get_event_dir(item)
if not os.path.exists(event_dir):
os.makedirs(event_dir)
bfq_kdata_dir = get_kdata_dir(item, 'bfq')
if not os.path.exists(bfq_kdata_dir):
os.makedirs(bfq_kdata_dir)
hfq_kdata_dir = get_kdata_dir(item, 'hfq')
if not os.path.exists(hfq_kdata_dir):
os.makedirs(hfq_kdata_dir)
def get_ticks(security_item, the_date=None, start=None, end=None):
"""
get the ticks.
Parameters
----------
security_item : SecurityItem
the security item
the_date : TimeStamp str or TimeStamp
get the tick for the exact date
start : TimeStamp str or TimeStamp
start date
end: TimeStamp str or TimeStamp
end date
Returns
-------
DataFrame
"""
if the_date:
tick_path = files_contract.get_tick_path(security_item, the_date)
return _parse_tick(tick_path)
else:
tick_dir = files_contract.get_tick_dir(security_item)
if start or end:
if not start:
start = security_item['listDate']
if not end:
end = datetime.datetime.today()
tick_paths = [os.path.join(tick_dir, f) for f in
os.listdir(tick_dir) if
get_file_name(f) in pd.date_range(start=start, end=end)]
else:
tick_paths = [os.path.join(tick_dir, f) for f in
os.listdir(tick_dir)]
for tick_path in sorted(tick_paths):
yield _parse_tick(tick_path, security_item)
def get_available_tick_dates(security_item):
dir = files_contract.get_tick_dir(security_item)
return [get_file_name(f) for f in os.listdir(dir)]
def get_available_tick_dates(security_item):
dir = files_contract.get_tick_dir(security_item)
return [get_file_name(f) for f in os.listdir(dir)]
