def test_basic_get_securities():
items = get_securities(security_type='stock', provider='eastmoney')
print(items)
items = get_securities(security_type='index', provider='eastmoney')
print(items)
items = get_securities(security_type='coin', provider='ccxt')
print(items)
def __init__(self,
security_type=SecurityType.stock,
exchanges=['sh', 'sz'],
codes=None,
batch_size=10,
force_update=False,
sleeping_time=10) -> None:
super().__init__(batch_size=batch_size,
force_update=force_update,
sleeping_time=sleeping_time)
assert self.meta_provider is not None
assert self.meta_schema is not None
self.meta_category = get_store_category(data_schema=self.meta_schema)
# setup the securities you want to record
self.security_type = security_type
self.exchanges = exchanges
self.codes = codes
self.meta_session = get_db_session(provider=self.meta_provider,
store_category=self.meta_category)
# init the security list
self.securities = get_securities(session=self.meta_session,
security_type=self.security_type,
exchanges=self.exchanges,
codes=self.codes,
return_type='domain',
provider=self.meta_provider)
def get_codes_input(security_type='stock', id=None):
if security_type == 'stock':
df = get_securities(security_type=security_type, columns=[Stock.code])
codes = df['code'].tolist()
if security_type == 'coin':
codes = COIN_PAIRS
return dcc.Dropdown(id=id, options=[{'label': item, 'value': item} for item in codes],
value=None,
multi=True)
def __init__(self,
batch_size=10,
force_update=False,
sleeping_time=10) -> None:
super().__init__(batch_size, force_update, sleeping_time)
self.indices = get_securities(session=self.session,
security_type=SecurityType.index,
exchanges=['cn'],
return_type='domain',
provider=self.provider)
self.index_ids = [index_item.id for index_item in self.indices]
def __init__(self,
security_type=SecurityType.stock,
exchanges=['sh', 'sz'],
codes=None,
batch_size=10,
force_update=False,
sleeping_time=10) -> None:
"""
:param security_type:
:type security_type:SecurityType
:param exchanges:the exchanges for recording
:type exchanges:list of str
:param codes:the codes for recording
:type codes:list of str
:param batch_size:batch size to saving to db
:type batch_size:int
:param force_update: whether force update the data even if it exists
:type force_update:bool
:param sleeping_time:sleeping seconds for recoding loop
:type sleeping_time:int
"""
# setup the securities you want to record
self.security_type = security_type
self.exchanges = exchanges
self.codes = codes
self.batch_size = batch_size
self.force_update = force_update
self.sleeping_time = sleeping_time
# using to do db operations
self.session = get_db_session(
provider=self.provider,
store_category=self.store_category) # type: sqlalchemy.orm.Session
if self.need_securities:
if self.store_category != StoreCategory.meta:
self.meta_session = get_db_session(
provider=self.meta_provider,
store_category=StoreCategory.meta)
else:
self.meta_session = self.session
self.securities = get_securities(session=self.meta_session,
security_type=self.security_type,
exchanges=self.exchanges,
codes=self.codes,
return_type='domain',
provider=self.meta_provider)
def run(self):
# get stock category from sina
for category, url in self.category_map_url.items():
resp = requests.get(url)
resp.encoding = 'GBK'
tmp_str = resp.text
json_str = tmp_str[tmp_str.index('{'):tmp_str.index('}') + 1]
tmp_json = json.loads(json_str)
for code in tmp_json:
name = tmp_json[code].split(',')[1]
id = 'index_cn_{}'.format(code)
if id in self.index_ids:
continue
self.session.add(
Index(id=id,
type='index',
exchange='cn',
code=code,
name=name,
category=category.value))
self.session.commit()
indices = get_securities(
session=self.session,
security_type=SecurityType.index,
return_type='domain',
filters=[Index.category != StockCategory.main.value],
provider=self.provider)
for index_item in indices:
for page in range(1, 5):
resp = requests.get(
self.category_stocks_url.format(page, index_item.code))
try:
if resp.text == 'null' or resp.text is None:
break
category_jsons = demjson.decode(resp.text)
the_list = []
for category in category_jsons:
stock_code = category['code']
stock_id = china_stock_code_to_id(stock_code)
index_id = index_item.id
the_list.append({
'id':
'{}_{}'.format(index_id, stock_id),
'index_id':
index_id,
'stock_id':
stock_id
})
if the_list:
df = pd.DataFrame.from_records(the_list)
df_to_db(data_schema=self.data_schema,
df=df,
provider=self.provider)
self.logger.info('finish recording index:{},{}'.format(
index_item.category, index_item.name))
except Exception as e:
self.logger.error("error:,resp.text:", e, resp.text)
self.sleep()