def yield_request(self, item, trading_dates=[], fuquan=None):
the_quarters = []
force_download = False
if trading_dates:
force_download = True
for the_date in trading_dates:
the_quarters.append(get_year_quarter(the_date))
else:
the_quarters = get_quarters(item['listDate'])
the_quarters = set(the_quarters)
if fuquan:
fuquans = [fuquan]
else:
fuquans = ['bfq', 'hfq']
# get day k data
for year, quarter in the_quarters:
for fuquan in fuquans:
data_path = get_kdata_path(item, source='sina', year=year, quarter=quarter, fuquan=fuquan)
data_exist = os.path.exists(data_path) or kdata_exist(item, year, quarter, fuquan, source='sina')
if not data_exist or force_download:
url = self.get_k_data_url(item['code'], year, quarter, fuquan)
yield Request(url=url, headers=DEFAULT_KDATA_HEADER,
meta={'path': data_path, 'item': item, 'fuquan': fuquan},
callback=self.download_day_k_data)
def yield_request(self, item, trading_dates=[], fuquan=None):
the_quarters = []
force_download = False
if trading_dates:
force_download = True
for the_date in trading_dates:
the_quarters.append(get_year_quarter(the_date))
else:
the_quarters = get_quarters(item['listDate'])
the_quarters = set(the_quarters)
if fuquan:
fuquans = [fuquan]
else:
fuquans = ['bfq', 'hfq']
# get day k data
for year, quarter in the_quarters:
for fuquan in fuquans:
data_path = get_kdata_path(item, source='sina', year=year, quarter=quarter, fuquan=fuquan)
data_exist = os.path.exists(data_path) or kdata_exist(item, year, quarter, fuquan, source='sina')
if not data_exist or force_download:
url = self.get_k_data_url(item['code'], year, quarter, fuquan)
yield Request(url=url, headers=DEFAULT_KDATA_HEADER,
meta={'path': data_path, 'item': item, 'fuquan': fuquan},
callback=self.download_day_k_data)
def merge_ths_kdata(security_item, dates):
ths_kdata = {}
ths_fuquan_kdata = {}
try:
with open(get_kdata_path_ths(security_item)) as data_file:
ths_items = json.load(data_file)
for item in ths_items:
if item["timestamp"] in dates:
ths_kdata[item["timestamp"]] = item
with open(get_kdata_path_ths(security_item, True)) as data_file:
ths_items = json.load(data_file)
for item in ths_items:
if item["timestamp"] in dates:
ths_fuquan_kdata[item["timestamp"]] = item
year_quarter_map_dates = {}
for the_date in dates:
year, quarter = get_year_quarter(get_datetime(the_date))
year_quarter_map_dates.setdefault((year, quarter), [])
year_quarter_map_dates.get((year, quarter)).append(the_date)
for year, quarter in year_quarter_map_dates.keys():
for fuquan in (False, True):
data_path = get_kdata_path_old(security_item, year, quarter,
fuquan)
data_exist = os.path.isfile(data_path)
if data_exist:
with open(data_path) as data_file:
k_items = json.load(data_file)
if fuquan:
for the_date in year_quarter_map_dates.get(
(year, quarter)):
k_items.append(ths_fuquan_kdata[the_date])
else:
for the_date in year_quarter_map_dates.get(
(year, quarter)):
k_items.append(ths_kdata[the_date])
k_items = sorted(k_items,
key=lambda item: item["timestamp"],
reverse=True)
with open(data_path, "w") as f:
json.dump(k_items, f)
except Exception as e:
logger.error(e)
def merge_ths_kdata(security_item, dates):
ths_kdata = {}
ths_fuquan_kdata = {}
try:
with open(get_kdata_path_ths(security_item)) as data_file:
ths_items = json.load(data_file)
for item in ths_items:
if item["timestamp"] in dates:
ths_kdata[item["timestamp"]] = item
with open(get_kdata_path_ths(security_item, True)) as data_file:
ths_items = json.load(data_file)
for item in ths_items:
if item["timestamp"] in dates:
ths_fuquan_kdata[item["timestamp"]] = item
year_quarter_map_dates = {}
for the_date in dates:
year, quarter = get_year_quarter(get_datetime(the_date))
year_quarter_map_dates.setdefault((year, quarter), [])
year_quarter_map_dates.get((year, quarter)).append(the_date)
for year, quarter in year_quarter_map_dates.keys():
for fuquan in (False, True):
data_path = get_kdata_path_old(security_item, year, quarter, fuquan)
data_exist = os.path.isfile(data_path)
if data_exist:
with open(data_path) as data_file:
k_items = json.load(data_file)
if fuquan:
for the_date in year_quarter_map_dates.get((year, quarter)):
k_items.append(ths_fuquan_kdata[the_date])
else:
for the_date in year_quarter_map_dates.get((year, quarter)):
k_items.append(ths_kdata[the_date])
k_items = sorted(k_items, key=lambda item: item["timestamp"], reverse=True)
with open(data_path, "w") as f:
json.dump(k_items, f)
except Exception as e:
logger.error(e)