def run(self): for category, url in self.category_map_url.items(): resp = requests.get(url) results = json_callback_param(resp.text) for result in results: items = result.split(',') code = items[1] name = items[2] id = 'index_cn_{}'.format(code) if id in self.index_ids: continue self.session.add( Index(id=id, entity_id=id, entity_type='index', exchange='cn', code=code, name=name, category=category.value)) self.session.commit() indices = get_entities(session=self.session, entity_type='index', return_type='domain', filters=[ Index.category.in_([ StockCategory.concept.value, StockCategory.industry.value ]) ], provider=self.provider) for index_item in indices: resp = requests.get( self.category_stocks_url.format(index_item.code, '1')) try: results = json_callback_param(resp.text) the_list = [] for result in results: items = result.split(',') stock_code = items[1] stock_id = china_stock_code_to_id(stock_code) index_id = index_item.id the_list.append({ 'id': '{}_{}'.format(index_id, stock_id), 'index_id': index_id, 'stock_id': stock_id }) if the_list: df = pd.DataFrame.from_records(the_list) df_to_db(data_schema=self.data_schema, df=df, provider=self.provider) self.logger.info('finish recording index:{},{}'.format( index_item.category, index_item.name)) except Exception as e: self.logger.error("error:,resp.text:", e, resp.text) self.sleep()
def record(self, entity, start, end, size, timestamps): the_url = self.url.format("{}".format(entity.code), level_flag(self.level), size, now_time_str(fmt=TIME_FORMAT_DAY1)) resp = requests.get(the_url) results = json_callback_param(resp.text) kdatas = [] if results: klines = results['data']['klines'] # TODO: ignore the last unfinished kdata now,could control it better if need for result in klines[:-1]: # "2000-01-28,1005.26,1012.56,1173.12,982.13,3023326,3075552000.00" # time,open,close,high,low,volume,turnover fields = result.split(',') the_timestamp = to_pd_timestamp(fields[0]) the_id = generate_kdata_id(entity_id=entity.id, timestamp=the_timestamp, level=self.level) kdatas.append(dict(id=the_id, timestamp=the_timestamp, entity_id=entity.id, code=entity.code, name=entity.name, level=self.level.value, open=to_float(fields[1]), close=to_float(fields[2]), high=to_float(fields[3]), low=to_float(fields[4]), volume=to_float(fields[5]), turnover=to_float(fields[6]))) return kdatas
def run(self): for category, url in self.category_map_url.items(): resp = requests.get(url) results = json_callback_param(resp.text) the_list = [] for result in results: items = result.split(',') code = items[1] name = items[2] entity_id = f'block_cn_{code}' the_list.append({ 'id': entity_id, 'entity_id': entity_id, 'entity_type': 'block', 'exchange': 'cn', 'code': code, 'name': name, 'category': category.value }) if the_list: df = pd.DataFrame.from_records(the_list) df_to_db(data_schema=self.data_schema, df=df, provider=self.provider, force_update=True) self.logger.info(f"finish record sina blocks:{category.value}")
def record(self, entity, start, end, size, time_array): if entity.type == 'index': id_flag = "{}1".format(entity.code) elif entity.type == 'stock': if entity.exchange == 'sh': id_flag = "{}1".format(entity.code) if entity.exchange == 'sz': id_flag = "{}2".format(entity.code) the_url = self.url.format("{}".format(id_flag), eastmoney_map_zvt_trading_level(self.level), now_time_str(fmt=TIME_FORMAT_MINUTE), size) resp = requests.get(the_url) results = json_callback_param(resp.text) kdatas = [] for result in results: the_timestamp = to_pd_timestamp(result['time']) the_id = generate_kdata_id(entity_id=entity.id, timestamp=the_timestamp, level=self.level) if not data_exist(self.session, self.kdata_schema, the_id): kdatas.append( self.kdata_schema(id=the_id, timestamp=the_timestamp, entity_id=entity.id, code=entity.code, name=entity.name, level=self.level, open=to_float(result['open']), close=to_float(result['close']), high=to_float(result['high']), low=to_float(result['low']), volume=to_float(result['volume']), turnover=to_float(result['amount']), turnover_rate=to_float( result['turnoverrate']))) return kdatas
def record(self, entity, start, end, size, timestamps): resp = requests.get(self.category_stocks_url.format(entity.code, '1')) try: results = json_callback_param(resp.text) the_list = [] for result in results: items = result.split(',') stock_code = items[1] stock_id = china_stock_code_to_id(stock_code) block_id = entity.id the_list.append({ 'id': '{}_{}'.format(block_id, stock_id), 'entity_id': block_id, 'entity_type': 'block', 'exchange': entity.exchange, 'code': entity.code, 'name': entity.name, 'timestamp': now_pd_timestamp(), 'stock_id': stock_id, 'stock_code': stock_code, 'stock_name': items[2], }) if the_list: df = pd.DataFrame.from_records(the_list) df_to_db(data_schema=self.data_schema, df=df, provider=self.provider, force_update=True) self.logger.info('finish recording block:{},{}'.format( entity.category, entity.name)) except Exception as e: self.logger.error("error:,resp.text:", e, resp.text) self.sleep()