def remove_old_tick(): for index, security_item in get_security_list().iterrows(): dir = get_tick_dir(security_item) if os.path.exists(dir): files = [os.path.join(dir, f) for f in os.listdir(dir) if ('xls' in f and 'lock' not in f and 'error' not in f and os.path.isfile(os.path.join(dir, f)))] for f in files: logger.info("remove {}".format(f)) os.remove(f)
def check_result(): for index, security_item in get_security_list().iterrows(): for fuquan in ('bfq', 'hfq'): dayk_path = get_kdata_path(security_item, fuquan=fuquan) if not os.path.exists(dayk_path): logger.warn(get_security_dir(security_item)) dir = get_tick_dir(security_item) if os.path.exists(dir): files = [os.path.join(dir, f) for f in os.listdir(dir) if ('csv' in f and os.path.isfile(os.path.join(dir, f)))] if not files: logger.warn(get_security_dir(security_item))
def handle_error_tick(): for index, security_item in get_security_list().iterrows(): dir = get_tick_dir(security_item) if os.path.exists(dir): files = [os.path.join(dir, f) for f in os.listdir(dir) if (('fatal' in f or 'error' in f) and os.path.isfile(os.path.join(dir, f)))] for f in files: try: the_date = get_file_name(f) csv_path = get_tick_path(security_item, the_date) if not os.path.exists(csv_path): logger.info("{} to {}".format(f, csv_path)) sina_tick_to_csv(security_item, f, the_date) except Exception as e: logger.warn(e) os.rename(f, f + ".fatal")
def get_ticks(security_item, the_date=None, start_date=None, end_date=None): """ get the ticks. Parameters ---------- security_item : SecurityItem or str the security item,id or code the_date : TimeStamp str or TimeStamp get the tick for the exact date start_date : TimeStamp str or TimeStamp start date end_date: TimeStamp str or TimeStamp end date Yields ------- DataFrame """ security_item = to_security_item(security_item) if the_date: the_date = to_time_str(the_date) tick_path = files_contract.get_tick_path(security_item, the_date) yield _parse_tick(tick_path, security_item) else: tick_dir = files_contract.get_tick_dir(security_item) if start_date or end_date: if not start_date: start_date = security_item['listDate'] if not end_date: end_date = datetime.datetime.today() tick_paths = [ os.path.join(tick_dir, f) for f in os.listdir(tick_dir) if get_file_name(f) in pd.date_range(start=start_date, end=end_date) ] else: tick_paths = [ os.path.join(tick_dir, f) for f in os.listdir(tick_dir) ] for tick_path in sorted(tick_paths): yield _parse_tick(tick_path, security_item)
def mkdir_for_security(item): finance_dir = get_finance_dir(item) if not os.path.exists(finance_dir): os.makedirs(finance_dir) tick_dir = get_tick_dir(item) if not os.path.exists(tick_dir): os.makedirs(tick_dir) event_dir = get_event_dir(item) if not os.path.exists(event_dir): os.makedirs(event_dir) bfq_kdata_dir = get_kdata_dir(item, 'bfq') if not os.path.exists(bfq_kdata_dir): os.makedirs(bfq_kdata_dir) hfq_kdata_dir = get_kdata_dir(item, 'hfq') if not os.path.exists(hfq_kdata_dir): os.makedirs(hfq_kdata_dir)
def get_ticks(security_item, the_date=None, start=None, end=None): if the_date: tick_path = files_contract.get_tick_path(security_item, the_date) return parse_tick(tick_path) else: tick_dir = files_contract.get_tick_dir(security_item) if start or end: if not start: start = security_item['listDate'] if not end: end = datetime.datetime.today() tick_paths = [os.path.join(tick_dir, f) for f in os.listdir(tick_dir) if get_file_name(f) in pd.date_range(start=start, end=end)] else: tick_paths = [os.path.join(tick_dir, f) for f in os.listdir(tick_dir)] for tick_path in sorted(tick_paths): yield parse_tick(tick_path, security_item)
def get_ticks(security_item, the_date=None, start=None, end=None): """ get the ticks. Parameters ---------- security_item : SecurityItem the security item the_date : TimeStamp str or TimeStamp get the tick for the exact date start : TimeStamp str or TimeStamp start date end: TimeStamp str or TimeStamp end date Returns ------- DataFrame """ if the_date: tick_path = files_contract.get_tick_path(security_item, the_date) return _parse_tick(tick_path) else: tick_dir = files_contract.get_tick_dir(security_item) if start or end: if not start: start = security_item['listDate'] if not end: end = datetime.datetime.today() tick_paths = [os.path.join(tick_dir, f) for f in os.listdir(tick_dir) if get_file_name(f) in pd.date_range(start=start, end=end)] else: tick_paths = [os.path.join(tick_dir, f) for f in os.listdir(tick_dir)] for tick_path in sorted(tick_paths): yield _parse_tick(tick_path, security_item)
def get_available_tick_dates(security_item): dir = files_contract.get_tick_dir(security_item) return [get_file_name(f) for f in os.listdir(dir)]