def test_basic_get_securities(): items = get_entities(entity_type='stock', provider='eastmoney') print(items) items = get_entities(entity_type='index', provider='eastmoney') print(items) items = get_entities(entity_type='coin', provider='ccxt') print(items)
def run(self): for category, url in self.category_map_url.items(): resp = requests.get(url) results = json_callback_param(resp.text) for result in results: items = result.split(',') code = items[1] name = items[2] id = 'index_cn_{}'.format(code) if id in self.index_ids: continue self.session.add( Index(id=id, entity_id=id, entity_type='index', exchange='cn', code=code, name=name, category=category.value)) self.session.commit() indices = get_entities(session=self.session, entity_type='index', return_type='domain', filters=[ Index.category.in_([ StockCategory.concept.value, StockCategory.industry.value ]) ], provider=self.provider) for index_item in indices: resp = requests.get( self.category_stocks_url.format(index_item.code, '1')) try: results = json_callback_param(resp.text) the_list = [] for result in results: items = result.split(',') stock_code = items[1] stock_id = china_stock_code_to_id(stock_code) index_id = index_item.id the_list.append({ 'id': '{}_{}'.format(index_id, stock_id), 'index_id': index_id, 'stock_id': stock_id }) if the_list: df = pd.DataFrame.from_records(the_list) df_to_db(data_schema=self.data_schema, df=df, provider=self.provider) self.logger.info('finish recording index:{},{}'.format( index_item.category, index_item.name)) except Exception as e: self.logger.error("error:,resp.text:", e, resp.text) self.sleep()
def init_entities(self): self.entity_session = get_db_session(provider=self.entity_provider, data_schema=self.entity_schema) self.entities = get_entities(session=self.entity_session, entity_type='index', exchanges=self.exchanges, codes=self.codes, entity_ids=self.entity_ids, return_type='domain', provider=self.provider, # 只抓概念和行业 filters=[Index.category.in_( [StockCategory.industry.value, StockCategory.concept.value])])
def __init__(self, batch_size=10, force_update=False, sleeping_time=10, codes=None) -> None: super().__init__(batch_size, force_update, sleeping_time) self.codes = codes if not self.force_update: self.entities = get_entities(session=self.session, entity_type='stock', exchanges=['sh', 'sz'], codes=self.codes, filters=[Stock.profile.is_(None)], return_type='domain', provider=self.provider)
def __init__(self, batch_size=10, force_update=False, sleeping_time=10) -> None: super().__init__(batch_size, force_update, sleeping_time) self.indices = get_entities(session=self.session, entity_type='index', exchanges=['cn'], return_type='domain', provider=self.provider) self.index_ids = [index_item.id for index_item in self.indices]
def run(self): # get stock category from sina for category, url in self.category_map_url.items(): resp = requests.get(url) resp.encoding = 'GBK' tmp_str = resp.text json_str = tmp_str[tmp_str.index('{'):tmp_str.index('}') + 1] tmp_json = json.loads(json_str) for code in tmp_json: name = tmp_json[code].split(',')[1] id = 'index_cn_{}'.format(code) if id in self.index_ids: continue self.session.add(Index(id=id, entity_type='index', exchange='cn', code=code, name=name, category=category.value)) self.session.commit() indices = get_entities(session=self.session, entity_type='index', return_type='domain', filters=[ Index.category.in_([StockCategory.industry.value, StockCategory.concept.value])], provider=self.provider) for index_item in indices: for page in range(1, 5): resp = requests.get(self.category_stocks_url.format(page, index_item.code)) try: if resp.text == 'null' or resp.text is None: break category_jsons = demjson.decode(resp.text) the_list = [] for category in category_jsons: stock_code = category['code'] stock_id = china_stock_code_to_id(stock_code) index_id = index_item.id the_list.append({ 'id': '{}_{}'.format(index_id, stock_id), 'index_id': index_id, 'stock_id': stock_id }) if the_list: df = pd.DataFrame.from_records(the_list) df_to_db(data_schema=self.data_schema, df=df, provider=self.provider) self.logger.info('finish recording index:{},{}'.format(index_item.category, index_item.name)) except Exception as e: self.logger.error("error:,resp.text:", e, resp.text) self.sleep()
def __init__(self, batch_size=10, force_update=False, sleeping_time=10, codes=None) -> None: super().__init__(batch_size, force_update, sleeping_time) # get list at first EastmoneyChinaStockListRecorder().run() self.codes = codes if not self.force_update: self.entities = get_entities(session=self.session, entity_type='stock_detail', exchanges=['sh', 'sz'], codes=self.codes, filters=[StockDetail.profile.is_(None)], return_type='domain', provider=self.provider)