def test_basic_get_securities():
items = get_entities(entity_type='stock', provider='eastmoney')
print(items)
items = get_entities(entity_type='index', provider='eastmoney')
print(items)
items = get_entities(entity_type='coin', provider='ccxt')
print(items)
def run(self):
for category, url in self.category_map_url.items():
resp = requests.get(url)
results = json_callback_param(resp.text)
for result in results:
items = result.split(',')
code = items[1]
name = items[2]
id = 'index_cn_{}'.format(code)
if id in self.index_ids:
continue
self.session.add(
Index(id=id,
entity_id=id,
entity_type='index',
exchange='cn',
code=code,
name=name,
category=category.value))
self.session.commit()
indices = get_entities(session=self.session,
entity_type='index',
return_type='domain',
filters=[
Index.category.in_([
StockCategory.concept.value,
StockCategory.industry.value
])
],
provider=self.provider)
for index_item in indices:
resp = requests.get(
self.category_stocks_url.format(index_item.code, '1'))
try:
results = json_callback_param(resp.text)
the_list = []
for result in results:
items = result.split(',')
stock_code = items[1]
stock_id = china_stock_code_to_id(stock_code)
index_id = index_item.id
the_list.append({
'id': '{}_{}'.format(index_id, stock_id),
'index_id': index_id,
'stock_id': stock_id
})
if the_list:
df = pd.DataFrame.from_records(the_list)
df_to_db(data_schema=self.data_schema,
df=df,
provider=self.provider)
self.logger.info('finish recording index:{},{}'.format(
index_item.category, index_item.name))
except Exception as e:
self.logger.error("error:,resp.text:", e, resp.text)
self.sleep()
def init_entities(self):
self.entity_session = get_db_session(provider=self.entity_provider, data_schema=self.entity_schema)
self.entities = get_entities(session=self.entity_session, entity_type='index',
exchanges=self.exchanges,
codes=self.codes,
entity_ids=self.entity_ids,
return_type='domain', provider=self.provider,
# 只抓概念和行业
filters=[Index.category.in_(
[StockCategory.industry.value, StockCategory.concept.value])])
def __init__(self, batch_size=10, force_update=False, sleeping_time=10, codes=None) -> None:
super().__init__(batch_size, force_update, sleeping_time)
self.codes = codes
if not self.force_update:
self.entities = get_entities(session=self.session,
entity_type='stock',
exchanges=['sh', 'sz'],
codes=self.codes,
filters=[Stock.profile.is_(None)],
return_type='domain',
provider=self.provider)
def __init__(self,
batch_size=10,
force_update=False,
sleeping_time=10) -> None:
super().__init__(batch_size, force_update, sleeping_time)
self.indices = get_entities(session=self.session,
entity_type='index',
exchanges=['cn'],
return_type='domain',
provider=self.provider)
self.index_ids = [index_item.id for index_item in self.indices]
def run(self):
# get stock category from sina
for category, url in self.category_map_url.items():
resp = requests.get(url)
resp.encoding = 'GBK'
tmp_str = resp.text
json_str = tmp_str[tmp_str.index('{'):tmp_str.index('}') + 1]
tmp_json = json.loads(json_str)
for code in tmp_json:
name = tmp_json[code].split(',')[1]
id = 'index_cn_{}'.format(code)
if id in self.index_ids:
continue
self.session.add(Index(id=id, entity_type='index', exchange='cn', code=code, name=name,
category=category.value))
self.session.commit()
indices = get_entities(session=self.session, entity_type='index',
return_type='domain', filters=[
Index.category.in_([StockCategory.industry.value, StockCategory.concept.value])],
provider=self.provider)
for index_item in indices:
for page in range(1, 5):
resp = requests.get(self.category_stocks_url.format(page, index_item.code))
try:
if resp.text == 'null' or resp.text is None:
break
category_jsons = demjson.decode(resp.text)
the_list = []
for category in category_jsons:
stock_code = category['code']
stock_id = china_stock_code_to_id(stock_code)
index_id = index_item.id
the_list.append({
'id': '{}_{}'.format(index_id, stock_id),
'index_id': index_id,
'stock_id': stock_id
})
if the_list:
df = pd.DataFrame.from_records(the_list)
df_to_db(data_schema=self.data_schema, df=df, provider=self.provider)
self.logger.info('finish recording index:{},{}'.format(index_item.category, index_item.name))
except Exception as e:
self.logger.error("error:,resp.text:", e, resp.text)
self.sleep()
def __init__(self, batch_size=10, force_update=False, sleeping_time=10, codes=None) -> None:
super().__init__(batch_size, force_update, sleeping_time)
# get list at first
EastmoneyChinaStockListRecorder().run()
self.codes = codes
if not self.force_update:
self.entities = get_entities(session=self.session,
entity_type='stock_detail',
exchanges=['sh', 'sz'],
codes=self.codes,
filters=[StockDetail.profile.is_(None)],
return_type='domain',
provider=self.provider)
