def get_income_sheet(code, end_date):
connect("mongodb://localhost:27017/stock?", alias="income_sheet")
df = pro.income(ts_code=code, start_date='20050101', end_date=end_date)
df = add_key_(df)
arr = df.values
if len(arr) == 0:
print('非交易日：' + time.strftime("%Y%m%d", time.localtime(time.time())))
pass
else:
for i in tqdm(range(len(arr))):
IncomeSheet(
arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3], arr[i, 4],
arr[i, 5], arr[i, 6], arr[i, 7], arr[i, 8], arr[i, 9],
arr[i, 10], arr[i, 11], arr[i, 12], arr[i, 13], arr[i, 14],
arr[i, 15], arr[i, 16], arr[i, 17], arr[i, 18], arr[i, 19],
arr[i, 20], arr[i, 21], arr[i, 22], arr[i, 23], arr[i, 24],
arr[i, 25], arr[i, 26], arr[i, 27], arr[i, 28], arr[i, 29],
arr[i, 30], arr[i, 31], arr[i, 32], arr[i, 33], arr[i, 34],
arr[i, 35], arr[i, 36], arr[i, 37], arr[i, 38], arr[i, 39],
arr[i, 40], arr[i, 41], arr[i, 42], arr[i, 43], arr[i, 44],
arr[i, 45], arr[i, 46], arr[i, 47], arr[i, 48], arr[i, 49],
arr[i, 50], arr[i, 51], arr[i, 52], arr[i, 53], arr[i, 54],
arr[i, 55], arr[i, 56], arr[i, 57], arr[i,
58], arr[i,
59], arr[i,
60],
arr[i, 61], arr[i, 62], arr[i, 63], arr[i,
64], arr[i,
65]).save()
print('利润表数据导入完成：', end_date)
def custom_connection(self, **kwargs):
# mix kwargs with variables
variables = self.variables
for key, value in kwargs.items():
variables[key] = value
db = variables.get('database', self._defaultdb)
uri = "mongodb://%s:%s/%s" % (
variables.get('host'), variables.get('port'), db)
# if db == self._defaultdb:
# mongodb.connect(uri)
# obj = mongodb._get_connection()
# else:
mongodb.connect(uri, alias=db)
link = mongodb._get_connection(alias=db)
log.debug("Connected to db %s", db)
class obj:
connection = link
return obj
def get_cashflow_sheet(code, end_date):
connect("mongodb://localhost:27017/stock?", alias="cashflow_sheet")
df = pro.cashflow(ts_code=code, start_date='20050101', end_date=end_date)
df = add_key_(df)
arr = df.values
if len(arr) == 0:
print('非交易日：' + time.strftime("%Y%m%d", time.localtime(time.time())))
pass
else:
for i in tqdm(range(len(arr))):
CashFlowSheet(
arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3], arr[i, 4],
arr[i, 5], arr[i, 6], arr[i, 7], arr[i, 8], arr[i, 9], arr[i,
10],
arr[i, 11], arr[i, 12], arr[i, 13], arr[i, 14], arr[i, 15],
arr[i, 16], arr[i, 17], arr[i, 18], arr[i, 19], arr[i, 20],
arr[i, 21], arr[i, 22], arr[i, 23], arr[i, 24], arr[i, 25],
arr[i, 26], arr[i, 27], arr[i, 28], arr[i, 29], arr[i, 30],
arr[i, 31], arr[i, 32], arr[i, 33], arr[i, 34], arr[i, 35],
arr[i, 36], arr[i, 37], arr[i, 38], arr[i, 39], arr[i, 40],
arr[i, 41], arr[i, 42], arr[i, 43], arr[i, 44], arr[i, 45],
arr[i, 46], arr[i, 47], arr[i, 48], arr[i, 49], arr[i, 50],
arr[i, 51], arr[i, 52], arr[i, 53], arr[i, 54], arr[i, 55],
arr[i, 56], arr[i, 57], arr[i, 58], arr[i, 59], arr[i, 60],
arr[i, 61], arr[i, 62], arr[i, 63], arr[i, 64], arr[i, 65],
arr[i, 66], arr[i, 67], arr[i, 68], arr[i, 69], arr[i, 70],
arr[i, 71], arr[i, 72], arr[i, 73], arr[i, 74], arr[i, 75],
arr[i, 76], arr[i, 77], arr[i, 78], arr[i, 79], arr[i, 80],
arr[i, 81], arr[i, 82], arr[i, 83], arr[i, 84], arr[i, 85],
arr[i, 86], arr[i, 87], arr[i, 88], arr[i,
89], arr[i,
90]).save()
print('现金流量表数据导入完成：', code, end_date)
def test_connect_lazily_with_index(self):
heartbeat_listener = HeartbeatStartedListener()
create_indexes_listener = WhiteListEventListener('createIndexes')
connect('mongodb://localhost:27017/foo',
'foo-connection',
connect=False,
event_listeners=[heartbeat_listener, create_indexes_listener])
client = _get_connection('foo-connection').database.client
class Article(MongoModel):
title = CharField()
class Meta:
connection_alias = 'foo-connection'
indexes = [
IndexModel([('title', 1)])
]
# Creating the class didn't create a connection, or any indexes.
self.assertEqual(len(heartbeat_listener.results), 0)
self.assertEqual(len(create_indexes_listener.results['started']), 0)
# The connection and indexes are created on the first query.
self.assertEqual(Article.objects.count(), 0)
self.assertGreaterEqual(len(heartbeat_listener.results), 1)
self.assertGreaterEqual(len(create_indexes_listener.results['started']), 1)
def custom_connection(self, **kwargs):
##################
# mix kwargs with variables
variables = self.variables
for key, value in kwargs.items():
variables[key] = value
##################
# connect for authentication if required
uri = "mongodb://{}:{}/{}".format(
variables.get('host'),
variables.get('port'),
AUTH_DB,
)
mongodb.connect(uri, alias=AUTH_DB)
##################
db = variables.get('database', 'UNKNOWN')
uri = "mongodb://{}:{}/{}".format(variables.get('host'),
variables.get('port'), db)
mongodb.connect(uri, alias=db)
link = mongodb._get_connection(alias=db)
log.verbose("Connected to db {}", db)
class obj:
connection = link
return obj
def test_connect_with_kwargs(self):
connect('mongodb://localhost:27017/foo?maxPoolSize=42',
'foo-connection',
minpoolsize=10)
client = _get_connection('foo-connection').database.client
self.assertEqual(42, client.max_pool_size)
self.assertEqual(10, client.min_pool_size)
def load_pipe(storage, name):
pipe = None
error = "Could not load pipeline"
if storage == "m":
try:
pipe = load_pipe_from_db(name)
except ValueError as exc:
connect(app.config['mongo_db_url'])
pipe = load_pipe_from_db(name)
elif storage == "a":
try:
pipe = app.config['pipe_objects'][name]
except KeyError as ke:
# Todo pretty error handling
error = ke
elif storage == "f":
try:
pipe_path = app.config['pipe_files'][name]
pipe = MDBHelper.load_results(pipe_path)
except KeyError as ke:
# Todo File not Found
error = ke
except Exception as e:
# Todo: handle file does not exist
debug = True
if not pipe or not isinstance(pipe, MDBHyperpipe):
# return "Could not load pipeline"
return error
return pipe
def get_field(SecuCode, collection_name):
if collection_name == 'hk_secumain':
print('Connect the Mongodb...')
connect(
"mongodb://*****:*****@192.168.0.119:27017/jydb?authSource=admin",
alias="initial_update")
SecuCodes_class = hk_secumain.objects.get_SecuCode(SecuCode)
SecuCodes = []
for secu in SecuCodes_class:
SecuCodes.append(eval('secu.' + SecuCode))
print('Download Completed.')
return SecuCodes
elif collection_name == 'hk_adjustedfactor':
print('Connect the Mongodb...')
connect(
"mongodb://*****:*****@192.168.0.119:27017/jydb?authSource=admin",
alias="initial_update_adjustedinfo")
SecuCodes_class = hk_adjustedfactor.objects.get_SecuCode(SecuCode)
SecuCodes = []
for secu in SecuCodes_class:
SecuCodes.append(eval('secu.' + SecuCode))
print('Download Completed.')
return SecuCodes
def get_stock_list(date):
connect("mongodb://localhost:27017/stock?", alias="stock_list")
# df = pro.stock_basic()
data_L = pro.stock_basic(
exchange='',
list_status='L',
fields=
'ts_code,name,area,industry,list_date,exchange,list_status,delist_date,is_hs'
)
data_D = pro.stock_basic(
exchange='',
list_status='D',
fields=
'ts_code,name,area,industry,list_date,exchange,list_status,delist_date,is_hs'
)
data_P = pro.stock_basic(
exchange='',
list_status='P',
fields=
'ts_code,name,area,industry,list_date,exchange,list_status,delist_date,is_hs'
)
df = pd.concat([data_L, data_D, data_P], axis=0).reset_index(drop=True)
arr = df.values
if len(arr) == 0:
print('非交易日：' + time.strftime("%Y%m%d", time.localtime(time.time())))
pass
else:
for i in tqdm(range(len(arr))):
StockList(arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3], arr[i, 4],
arr[i, 5], arr[i, 6], arr[i, 7], arr[i, 8]).save()
print('股票列表数据导入完成：', date)
def test_connect_with_kwargs(self):
connect('mongodb://localhost:27017/foo?maxPoolSize=42',
'foo-connection',
minpoolsize=10)
client = _get_connection('foo-connection').database.client
self.assertEqual(42, client.max_pool_size)
self.assertEqual(10, client.min_pool_size)
def test_connect_lazily_with_index(self):
heartbeat_listener = HeartbeatStartedListener()
create_indexes_listener = WhiteListEventListener('createIndexes')
connect('mongodb://localhost:27017/foo',
'foo-connection',
connect=False,
event_listeners=[heartbeat_listener, create_indexes_listener])
client = _get_connection('foo-connection').database.client
class Article(MongoModel):
title = CharField()
class Meta:
connection_alias = 'foo-connection'
indexes = [IndexModel([('title', 1)])]
# Creating the class didn't create a connection, or any indexes.
self.assertEqual(len(heartbeat_listener.results), 0)
self.assertEqual(len(create_indexes_listener.results['started']), 0)
# The connection and indexes are created on the first query.
self.assertEqual(Article.objects.count(), 0)
self.assertGreaterEqual(len(heartbeat_listener.results), 1)
self.assertGreaterEqual(
len(create_indexes_listener.results['started']), 1)
def load_pipe(storage, name):
pipe = None
error = "Could not load pipeline"
if storage == "m":
try:
pipe = load_pipe_from_db(name)
except ValueError as exc:
connect(application.config['mongo_db_url'], alias='photon_core')
pipe = load_pipe_from_db(name)
if storage == "w":
pipe = load_pipe_from_wizard(name)
elif storage == "a":
try:
pipe = application.config['pipe_objects'][name]
except KeyError as ke:
# Todo pretty error handling
error = ke
elif storage == "f":
try:
pipe_path = application.config['pipe_files'][name]
pipe = MDBHelper.load_results(pipe_path)
except KeyError as ke:
# Todo File not Found
error = ke
except Exception as e:
# Todo: handle file does not exist
debug = True
if not pipe:
session["error_msg"] = "Could not load result object."
abort(500)
return pipe
def get_financial_indicator(code):
connect("mongodb://localhost:27017/stock?", alias="financial_indicator")
df = pro.fina_indicator(ts_code=code)
df = add_key_(df)
arr = df.values
if len(arr) == 0:
print('非交易日：' + time.strftime("%Y%m%d", time.localtime(time.time())))
pass
else:
for i in tqdm(range(len(arr))):
FinancialIndicator(arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3],
arr[i, 4], arr[i, 5], arr[i, 6], arr[i,
7], arr[i,
8],
arr[i, 9], arr[i, 10], arr[i, 11], arr[i, 12],
arr[i, 13], arr[i, 14], arr[i, 15], arr[i, 16],
arr[i, 17], arr[i, 18], arr[i, 19], arr[i, 20],
arr[i, 21], arr[i, 22], arr[i, 23], arr[i, 24],
arr[i, 25], arr[i, 26], arr[i, 27], arr[i, 28],
arr[i, 29], arr[i, 30], arr[i, 31], arr[i, 32],
arr[i, 33], arr[i, 34], arr[i, 35], arr[i, 36],
arr[i, 37], arr[i, 38], arr[i, 39], arr[i, 40],
arr[i, 41], arr[i, 42], arr[i, 43], arr[i, 44],
arr[i, 45], arr[i, 46], arr[i, 47], arr[i, 48],
arr[i, 49], arr[i, 50], arr[i, 51], arr[i, 52],
arr[i, 53], arr[i, 54], arr[i, 55], arr[i, 56],
arr[i, 57], arr[i, 58], arr[i, 59], arr[i, 60],
arr[i, 61], arr[i, 62], arr[i, 63], arr[i, 64],
arr[i, 65], arr[i, 66], arr[i, 67], arr[i, 68],
arr[i, 69], arr[i, 70], arr[i, 71], arr[i, 72],
arr[i, 73], arr[i, 74], arr[i, 75], arr[i, 76],
arr[i, 77], arr[i, 78], arr[i, 79], arr[i, 80],
arr[i, 81], arr[i,
82], arr[i,
83], arr[i,
84], arr[i,
85],
arr[i, 86], arr[i,
87], arr[i,
88], arr[i,
89], arr[i,
90],
arr[i, 91], arr[i,
92], arr[i,
93], arr[i,
94], arr[i,
95],
arr[i, 96], arr[i,
97], arr[i,
98], arr[i,
99], arr[i,
100],
arr[i, 101], arr[i, 102], arr[i, 103], arr[i,
104],
arr[i,
105], arr[i,
106], arr[i,
107], arr[i,
108]).save()
print('财务指标数据导入完成：', code)
def _setup_db(args):
""" Setup db connection. """
runtime_config = args.config + "/runtime.cfg"
config = configparser.ConfigParser()
config.read(runtime_config)
connect(config.get('general', 'mongodb'), ssl_cert_reqs=ssl.CERT_NONE)
def clear_db(app):
with app.app_context():
connect(current_app.config["MONGO_URI"])
db = PyMongo(app).db
db.sample.delete_many({})
db.step.delete_many({})
db.step_instance.delete_many({})
db.workflow.delete_many({})
def __init__(self, url="mongodb://localhost:27017/hive"):
try:
connect(url, alias="default")
except Exception as e:
print("Connection with MongoDB Failed!")
print(e)
return
def connect_db(uri, db_name=''):
try:
logger.debug('DB uri: {}, database name: {}'.format(uri, db_name))
connect(uri)
logger.info('Connected to database')
except Exception as e:
logger.exception(
'Failed to connect to database. Exception: {}'.format(e))
raise e
def _connect(self) -> None:
"""
This method will connect to the MongoDB
via URL using a MongoClient
:return: None
"""
if not self.database_uri:
raise KeyError(CONFIG_NOT_FOUND.format("MONGODB_DATABASE_URI"))
connect(self.database_uri, connect=True, **self.config)
def __init__(self):
'''
Start queue instance
'''
connect(os.environ.get('MONGO_CLIENT', ''), retryWrites=False)
self.thread = None
MessageQueue.instanciated = True
self.queue = Queue()
self.driver = WhatsappMessaging()
def tst(session):
os.environ["APPLICATION_ENV"] = "testing"
application = make_app()
session.client = application.test_client()
session.context = application.test_request_context()
with application.app_context():
connection.connect(application.config["MONGO_URL"])
yield session
client = connection._get_connection().database.client
client.drop_database(application.config["MONGO_DB"])
def _setup_db(args):
""" Setup db connection. """
runtime_config = args.config + "/runtime.cfg"
config = configparser.ConfigParser()
config.read(runtime_config)
mongodb_uri = config.get('general', 'mongodb', fallback=None)
if mongodb_uri:
connect(mongodb_uri, ssl_cert_reqs=ssl.CERT_NONE)
def load_available_pipes():
available_pipes = dict()
available_pipes['RAM'] = app.config['pipe_objects'].keys()
available_pipes['FILES'] = app.config['pipe_files'].keys()
available_pipes['MONGO'] = []
if 'mongo_db_url' in app.config:
try:
load_mongo_pipes(available_pipes)
except ValueError as exc:
connect(app.config['mongo_db_url'])
load_mongo_pipes(available_pipes)
return available_pipes
def get_adjfactor(date):
connect("mongodb://localhost:27017/stock?", alias="adjust_factor")
df = pro.adj_factor(ts_code='', trade_date=date) # 复权因子无论从任何时间去下载都是一样的
df = add_key_(df)
arr = df.values
if len(arr) == 0:
print('非交易日：' + time.strftime("%Y%m%d", time.localtime(time.time())))
pass
else:
for i in tqdm(range(len(arr))):
AdjustFactor(arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3]).save()
print('复权因子数据导入完成：', date)
def get_tradedays(start, end):
connect("mongodb://localhost:27017/stock?", alias="trade_days")
df = pro.trade_cal(exchange='', start_date=start, end_date=end)
df = df[df['is_open'] == 1]
arr = df.values
if len(arr) == 0:
print('非交易日：' + time.strftime("%Y%m%d", time.localtime(time.time())))
pass
else:
for i in tqdm(range(len(arr))):
TradeDays(arr[i, 0], arr[i, 1], arr[i, 2]).save()
print('日历数据导入完成：', end)
def load_pipe_from_wizard(obj_id):
try:
connect("mongodb://trap-umbriel:27017/photon_results", alias="photon_core")
pipe = (
MDBHyperpipe.objects.order_by([("computation_start_time", DESCENDING)])
.raw({"wizard_object_id": ObjectId(obj_id)})
.first()
)
return pipe
except DoesNotExist as dne:
# Todo: pretty error handling
return dne
def get_suspend_list(date):
connect("mongodb://localhost:27017/stock?", alias="suspend_list")
df = pro.suspend(suspend_date=date)
df = add_key_(df)
arr = df.values
if len(arr) == 0:
print('非交易日：' + time.strftime("%Y%m%d", time.localtime(time.time())))
pass
else:
for i in tqdm(range(len(arr))):
SuspendList(arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3],
arr[i, 4]).save()
print('停牌数据导入完成：', date)
def get_index_bar(code):
connect("mongodb://localhost:27017/stock?", alias="index_bar")
df = pro.index_daily(ts_code=code)
df = add_key_(df)
arr = df.values
if len(arr) == 0:
print('非交易日：' + time.strftime("%Y%m%d", time.localtime(time.time())))
pass
else:
for i in tqdm(range(len(arr))):
IndexBar(arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3], arr[i, 4],
arr[i, 5], arr[i, 6], arr[i, 7], arr[i, 8], arr[i, 9],
arr[i, 10], arr[i, 11]).save()
print('指数行情数据导入完成：', code)
def connect(self, **kwargs):
variables = self.variables.copy()
variables.update(kwargs)
MongoExt.DATABASE = variables.get("database", "rapydo")
uri = self._get_uri(variables)
mongodb.connect(uri, alias=MongoExt.DATABASE)
self.connection = mongodb._get_connection(alias=MongoExt.DATABASE)
TopLevelMongoModel.save = catch_db_exceptions(TopLevelMongoModel.save)
return self
def get_dailybar(date):
connect("mongodb://localhost:27017/stock?", alias="origin_daily_bar")
df = pro.daily(trade_date=date)
df = add_key_(df)
arr = df.values
if len(arr) == 0:
print('非交易日：' + time.strftime("%Y%m%d", time.localtime(time.time())))
pass
else:
for i in tqdm(range(len(arr))):
OriginDailyBar(arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3],
arr[i, 4], arr[i, 5], arr[i, 6], arr[i, 7],
arr[i, 8], arr[i, 9], arr[i, 10], arr[i,
11]).save()
print('行情数据导入完成：', date)
def get_client(self, retry=True):
from pymodm.connection import connect
from pymongo import MongoClient
from pymongo.errors import ServerSelectionTimeoutError
dburl = "mongodb://localhost:27017/dbtest"
client = MongoClient(host=[dburl], serverSelectionTimeoutMS=200)
try:
client.server_info()
connect(dburl)
return client
except ServerSelectionTimeoutError:
if not retry:
raise
check_call('sudo systemctl start mongodb'.split())
sleep(1.0)
return self.get_client(False)
def get_stock_main(date):
connect("mongodb://localhost:27017/stock?", alias="stock_main")
df_sh = pro.stock_company(exchange='SSE')
df_sz = pro.stock_company(exchange='SZSE')
df = pd.concat([df_sh, df_sz], axis=0).reset_index(drop=True)
df['employees'] = df['employees'].fillna(0)
arr = df.values
if len(arr) == 0:
print('非交易日：' + time.strftime("%Y%m%d", time.localtime(time.time())))
pass
else:
for i in tqdm(range(len(arr))):
StocksMain(arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3], arr[i, 4],
arr[i, 5], arr[i, 6], arr[i, 7], arr[i, 8], arr[i, 9],
arr[i, 10], arr[i, 11]).save()
print('股票主要信息数据导入完成：', date)
def get_index_weight(code, date):
connect("mongodb://localhost:27017/stock?", alias="index_weight")
if code == '000300.SH':
code = '399300.SZ'
elif code == '000905.SH':
code = '399905.SZ'
print('已将上交所指数代码改为深交所指数代码.')
df = pro.index_weight(index_code=code, trade_date=date)
df = add_key_(df)
arr = df.values
if len(arr) == 0:
print('非交易日：' + time.strftime("%Y%m%d", time.localtime(time.time())))
pass
else:
for i in tqdm(range(len(arr))):
IndexWeight(arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3],
arr[i, 4]).save()
print('指数权重数据导入完成：', code)
def test_connect_lazily(self):
heartbeat_listener = HeartbeatStartedListener()
connect('mongodb://localhost:27017/foo',
'foo-connection',
connect=False,
event_listeners=[heartbeat_listener])
client = _get_connection('foo-connection').database.client
class Article(MongoModel):
title = CharField()
class Meta:
connection_alias = 'foo-connection'
# Creating the class didn't create a connection.
self.assertEqual(len(heartbeat_listener.results), 0)
# The connection is created on the first query.
self.assertEqual(Article.objects.count(), 0)
self.assertGreaterEqual(len(heartbeat_listener.results), 1)
def setUpClass(cls):
cls.db_name = 'alternate-db'
connect(MONGO_URI + '/' + cls.db_name, 'backups')
cls.db = CLIENT[cls.db_name]
import sys
import unittest
import pymongo
from pymodm.connection import connect
PY3 = sys.version_info[0] == 3
MONGO_URI = os.environ.get('MONGO_URI', 'mongodb://localhost:27017')
CLIENT = pymongo.MongoClient(MONGO_URI)
DB = CLIENT.odm_test
connect('%s/%s' % (MONGO_URI, DB.name))
class ODMTestCase(unittest.TestCase):
def tearDown(self):
CLIENT.drop_database(DB.name)
if not PY3:
# assertRaisesRegexp is deprecated in Python 3 but is all we have in
# Python 2.
def assertRaisesRegex(self, *args, **kwargs):
return self.assertRaisesRegexp(*args, **kwargs)
def assertEqualsModel(self, expected, model_instance):
"""Assert that a Model instance equals the expected document."""
def connect_to_test_DB(alias=None):
if alias is None:
connect('%s/%s' % (MONGO_URI, DB.name))
else:
connect('%s/%s' % (MONGO_URI, DB.name), alias=alias)
