def yield_request(self, item, trading_dates=[], fuquan=None): the_quarters = [] force_download = False if trading_dates: force_download = True for the_date in trading_dates: the_quarters.append(get_year_quarter(the_date)) else: the_quarters = get_quarters(item['listDate']) the_quarters = set(the_quarters) if fuquan: fuquans = [fuquan] else: fuquans = ['bfq', 'hfq'] # get day k data for year, quarter in the_quarters: for fuquan in fuquans: data_path = get_kdata_path(item, source='sina', year=year, quarter=quarter, fuquan=fuquan) data_exist = os.path.exists(data_path) or kdata_exist(item, year, quarter, fuquan, source='sina') if not data_exist or force_download: url = self.get_k_data_url(item['code'], year, quarter, fuquan) yield Request(url=url, headers=DEFAULT_KDATA_HEADER, meta={'path': data_path, 'item': item, 'fuquan': fuquan}, callback=self.download_day_k_data)
def yield_request(self, item, trading_dates=[], fuquan=None): the_quarters = [] force_download = False if trading_dates: force_download = True for the_date in trading_dates: the_quarters.append(get_year_quarter(the_date)) else: the_quarters = get_quarters(item['listDate']) the_quarters = set(the_quarters) if fuquan: fuquans = [fuquan] else: fuquans = ['bfq', 'hfq'] # get day k data for year, quarter in the_quarters: for fuquan in fuquans: data_path = get_kdata_path(item, source='sina', year=year, quarter=quarter, fuquan=fuquan) data_exist = os.path.exists(data_path) or kdata_exist(item, year, quarter, fuquan, source='sina') if not data_exist or force_download: url = self.get_k_data_url(item['code'], year, quarter, fuquan) yield Request(url=url, headers=DEFAULT_KDATA_HEADER, meta={'path': data_path, 'item': item, 'fuquan': fuquan}, callback=self.download_day_k_data)
def merge_ths_kdata(security_item, dates): ths_kdata = {} ths_fuquan_kdata = {} try: with open(get_kdata_path_ths(security_item)) as data_file: ths_items = json.load(data_file) for item in ths_items: if item["timestamp"] in dates: ths_kdata[item["timestamp"]] = item with open(get_kdata_path_ths(security_item, True)) as data_file: ths_items = json.load(data_file) for item in ths_items: if item["timestamp"] in dates: ths_fuquan_kdata[item["timestamp"]] = item year_quarter_map_dates = {} for the_date in dates: year, quarter = get_year_quarter(get_datetime(the_date)) year_quarter_map_dates.setdefault((year, quarter), []) year_quarter_map_dates.get((year, quarter)).append(the_date) for year, quarter in year_quarter_map_dates.keys(): for fuquan in (False, True): data_path = get_kdata_path_old(security_item, year, quarter, fuquan) data_exist = os.path.isfile(data_path) if data_exist: with open(data_path) as data_file: k_items = json.load(data_file) if fuquan: for the_date in year_quarter_map_dates.get( (year, quarter)): k_items.append(ths_fuquan_kdata[the_date]) else: for the_date in year_quarter_map_dates.get( (year, quarter)): k_items.append(ths_kdata[the_date]) k_items = sorted(k_items, key=lambda item: item["timestamp"], reverse=True) with open(data_path, "w") as f: json.dump(k_items, f) except Exception as e: logger.error(e)
def merge_ths_kdata(security_item, dates): ths_kdata = {} ths_fuquan_kdata = {} try: with open(get_kdata_path_ths(security_item)) as data_file: ths_items = json.load(data_file) for item in ths_items: if item["timestamp"] in dates: ths_kdata[item["timestamp"]] = item with open(get_kdata_path_ths(security_item, True)) as data_file: ths_items = json.load(data_file) for item in ths_items: if item["timestamp"] in dates: ths_fuquan_kdata[item["timestamp"]] = item year_quarter_map_dates = {} for the_date in dates: year, quarter = get_year_quarter(get_datetime(the_date)) year_quarter_map_dates.setdefault((year, quarter), []) year_quarter_map_dates.get((year, quarter)).append(the_date) for year, quarter in year_quarter_map_dates.keys(): for fuquan in (False, True): data_path = get_kdata_path_old(security_item, year, quarter, fuquan) data_exist = os.path.isfile(data_path) if data_exist: with open(data_path) as data_file: k_items = json.load(data_file) if fuquan: for the_date in year_quarter_map_dates.get((year, quarter)): k_items.append(ths_fuquan_kdata[the_date]) else: for the_date in year_quarter_map_dates.get((year, quarter)): k_items.append(ths_kdata[the_date]) k_items = sorted(k_items, key=lambda item: item["timestamp"], reverse=True) with open(data_path, "w") as f: json.dump(k_items, f) except Exception as e: logger.error(e)