def yield_request(self, item, trading_dates=None):
if not trading_dates:
trading_dates = get_trading_dates(item)
for trading_date in trading_dates:
if get_datetime(trading_date) < get_datetime(settings.START_TICK_DATE) or get_datetime(
trading_date) < get_datetime(settings.AVAILABLE_TICK_DATE):
continue
path = get_tick_path(item, trading_date)
if os.path.exists(path):
continue
yield Request(url=self.get_tick_url(trading_date, item['exchange'] + item['code']),
meta={'proxy': None,
'path': path,
'trading_date': trading_date,
'item': item},
headers=DEFAULT_TICK_HEADER,
callback=self.download_tick)
def merge_ths_kdata(security_item, dates):
ths_kdata = {}
ths_fuquan_kdata = {}
try:
with open(get_kdata_path_ths(security_item)) as data_file:
ths_items = json.load(data_file)
for item in ths_items:
if item["timestamp"] in dates:
ths_kdata[item["timestamp"]] = item
with open(get_kdata_path_ths(security_item, True)) as data_file:
ths_items = json.load(data_file)
for item in ths_items:
if item["timestamp"] in dates:
ths_fuquan_kdata[item["timestamp"]] = item
year_quarter_map_dates = {}
for the_date in dates:
year, quarter = get_year_quarter(get_datetime(the_date))
year_quarter_map_dates.setdefault((year, quarter), [])
year_quarter_map_dates.get((year, quarter)).append(the_date)
for year, quarter in year_quarter_map_dates.keys():
for fuquan in (False, True):
data_path = get_kdata_path_old(security_item, year, quarter,
fuquan)
data_exist = os.path.isfile(data_path)
if data_exist:
with open(data_path) as data_file:
k_items = json.load(data_file)
if fuquan:
for the_date in year_quarter_map_dates.get(
(year, quarter)):
k_items.append(ths_fuquan_kdata[the_date])
else:
for the_date in year_quarter_map_dates.get(
(year, quarter)):
k_items.append(ths_kdata[the_date])
k_items = sorted(k_items,
key=lambda item: item["timestamp"],
reverse=True)
with open(data_path, "w") as f:
json.dump(k_items, f)
except Exception as e:
logger.error(e)
def merge_ths_kdata(security_item, dates):
ths_kdata = {}
ths_fuquan_kdata = {}
try:
with open(get_kdata_path_ths(security_item)) as data_file:
ths_items = json.load(data_file)
for item in ths_items:
if item["timestamp"] in dates:
ths_kdata[item["timestamp"]] = item
with open(get_kdata_path_ths(security_item, True)) as data_file:
ths_items = json.load(data_file)
for item in ths_items:
if item["timestamp"] in dates:
ths_fuquan_kdata[item["timestamp"]] = item
year_quarter_map_dates = {}
for the_date in dates:
year, quarter = get_year_quarter(get_datetime(the_date))
year_quarter_map_dates.setdefault((year, quarter), [])
year_quarter_map_dates.get((year, quarter)).append(the_date)
for year, quarter in year_quarter_map_dates.keys():
for fuquan in (False, True):
data_path = get_kdata_path_old(security_item, year, quarter, fuquan)
data_exist = os.path.isfile(data_path)
if data_exist:
with open(data_path) as data_file:
k_items = json.load(data_file)
if fuquan:
for the_date in year_quarter_map_dates.get((year, quarter)):
k_items.append(ths_fuquan_kdata[the_date])
else:
for the_date in year_quarter_map_dates.get((year, quarter)):
k_items.append(ths_kdata[the_date])
k_items = sorted(k_items, key=lambda item: item["timestamp"], reverse=True)
with open(data_path, "w") as f:
json.dump(k_items, f)
except Exception as e:
logger.error(e)