def yield_request(self, item, trading_dates=None): if not trading_dates: trading_dates = get_trading_dates(item) for trading_date in trading_dates: if get_datetime(trading_date) < get_datetime(settings.START_TICK_DATE) or get_datetime( trading_date) < get_datetime(settings.AVAILABLE_TICK_DATE): continue path = get_tick_path(item, trading_date) if os.path.exists(path): continue yield Request(url=self.get_tick_url(trading_date, item['exchange'] + item['code']), meta={'proxy': None, 'path': path, 'trading_date': trading_date, 'item': item}, headers=DEFAULT_TICK_HEADER, callback=self.download_tick)
def merge_ths_kdata(security_item, dates): ths_kdata = {} ths_fuquan_kdata = {} try: with open(get_kdata_path_ths(security_item)) as data_file: ths_items = json.load(data_file) for item in ths_items: if item["timestamp"] in dates: ths_kdata[item["timestamp"]] = item with open(get_kdata_path_ths(security_item, True)) as data_file: ths_items = json.load(data_file) for item in ths_items: if item["timestamp"] in dates: ths_fuquan_kdata[item["timestamp"]] = item year_quarter_map_dates = {} for the_date in dates: year, quarter = get_year_quarter(get_datetime(the_date)) year_quarter_map_dates.setdefault((year, quarter), []) year_quarter_map_dates.get((year, quarter)).append(the_date) for year, quarter in year_quarter_map_dates.keys(): for fuquan in (False, True): data_path = get_kdata_path_old(security_item, year, quarter, fuquan) data_exist = os.path.isfile(data_path) if data_exist: with open(data_path) as data_file: k_items = json.load(data_file) if fuquan: for the_date in year_quarter_map_dates.get( (year, quarter)): k_items.append(ths_fuquan_kdata[the_date]) else: for the_date in year_quarter_map_dates.get( (year, quarter)): k_items.append(ths_kdata[the_date]) k_items = sorted(k_items, key=lambda item: item["timestamp"], reverse=True) with open(data_path, "w") as f: json.dump(k_items, f) except Exception as e: logger.error(e)
def merge_ths_kdata(security_item, dates): ths_kdata = {} ths_fuquan_kdata = {} try: with open(get_kdata_path_ths(security_item)) as data_file: ths_items = json.load(data_file) for item in ths_items: if item["timestamp"] in dates: ths_kdata[item["timestamp"]] = item with open(get_kdata_path_ths(security_item, True)) as data_file: ths_items = json.load(data_file) for item in ths_items: if item["timestamp"] in dates: ths_fuquan_kdata[item["timestamp"]] = item year_quarter_map_dates = {} for the_date in dates: year, quarter = get_year_quarter(get_datetime(the_date)) year_quarter_map_dates.setdefault((year, quarter), []) year_quarter_map_dates.get((year, quarter)).append(the_date) for year, quarter in year_quarter_map_dates.keys(): for fuquan in (False, True): data_path = get_kdata_path_old(security_item, year, quarter, fuquan) data_exist = os.path.isfile(data_path) if data_exist: with open(data_path) as data_file: k_items = json.load(data_file) if fuquan: for the_date in year_quarter_map_dates.get((year, quarter)): k_items.append(ths_fuquan_kdata[the_date]) else: for the_date in year_quarter_map_dates.get((year, quarter)): k_items.append(ths_kdata[the_date]) k_items = sorted(k_items, key=lambda item: item["timestamp"], reverse=True) with open(data_path, "w") as f: json.dump(k_items, f) except Exception as e: logger.error(e)