def get_income_sheet(code, end_date): connect("mongodb://localhost:27017/stock?", alias="income_sheet") df = pro.income(ts_code=code, start_date='20050101', end_date=end_date) df = add_key_(df) arr = df.values if len(arr) == 0: print('非交易日:' + time.strftime("%Y%m%d", time.localtime(time.time()))) pass else: for i in tqdm(range(len(arr))): IncomeSheet( arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3], arr[i, 4], arr[i, 5], arr[i, 6], arr[i, 7], arr[i, 8], arr[i, 9], arr[i, 10], arr[i, 11], arr[i, 12], arr[i, 13], arr[i, 14], arr[i, 15], arr[i, 16], arr[i, 17], arr[i, 18], arr[i, 19], arr[i, 20], arr[i, 21], arr[i, 22], arr[i, 23], arr[i, 24], arr[i, 25], arr[i, 26], arr[i, 27], arr[i, 28], arr[i, 29], arr[i, 30], arr[i, 31], arr[i, 32], arr[i, 33], arr[i, 34], arr[i, 35], arr[i, 36], arr[i, 37], arr[i, 38], arr[i, 39], arr[i, 40], arr[i, 41], arr[i, 42], arr[i, 43], arr[i, 44], arr[i, 45], arr[i, 46], arr[i, 47], arr[i, 48], arr[i, 49], arr[i, 50], arr[i, 51], arr[i, 52], arr[i, 53], arr[i, 54], arr[i, 55], arr[i, 56], arr[i, 57], arr[i, 58], arr[i, 59], arr[i, 60], arr[i, 61], arr[i, 62], arr[i, 63], arr[i, 64], arr[i, 65]).save() print('利润表数据导入完成:', end_date)
def custom_connection(self, **kwargs): # mix kwargs with variables variables = self.variables for key, value in kwargs.items(): variables[key] = value db = variables.get('database', self._defaultdb) uri = "mongodb://%s:%s/%s" % ( variables.get('host'), variables.get('port'), db) # if db == self._defaultdb: # mongodb.connect(uri) # obj = mongodb._get_connection() # else: mongodb.connect(uri, alias=db) link = mongodb._get_connection(alias=db) log.debug("Connected to db %s", db) class obj: connection = link return obj
def get_cashflow_sheet(code, end_date): connect("mongodb://localhost:27017/stock?", alias="cashflow_sheet") df = pro.cashflow(ts_code=code, start_date='20050101', end_date=end_date) df = add_key_(df) arr = df.values if len(arr) == 0: print('非交易日:' + time.strftime("%Y%m%d", time.localtime(time.time()))) pass else: for i in tqdm(range(len(arr))): CashFlowSheet( arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3], arr[i, 4], arr[i, 5], arr[i, 6], arr[i, 7], arr[i, 8], arr[i, 9], arr[i, 10], arr[i, 11], arr[i, 12], arr[i, 13], arr[i, 14], arr[i, 15], arr[i, 16], arr[i, 17], arr[i, 18], arr[i, 19], arr[i, 20], arr[i, 21], arr[i, 22], arr[i, 23], arr[i, 24], arr[i, 25], arr[i, 26], arr[i, 27], arr[i, 28], arr[i, 29], arr[i, 30], arr[i, 31], arr[i, 32], arr[i, 33], arr[i, 34], arr[i, 35], arr[i, 36], arr[i, 37], arr[i, 38], arr[i, 39], arr[i, 40], arr[i, 41], arr[i, 42], arr[i, 43], arr[i, 44], arr[i, 45], arr[i, 46], arr[i, 47], arr[i, 48], arr[i, 49], arr[i, 50], arr[i, 51], arr[i, 52], arr[i, 53], arr[i, 54], arr[i, 55], arr[i, 56], arr[i, 57], arr[i, 58], arr[i, 59], arr[i, 60], arr[i, 61], arr[i, 62], arr[i, 63], arr[i, 64], arr[i, 65], arr[i, 66], arr[i, 67], arr[i, 68], arr[i, 69], arr[i, 70], arr[i, 71], arr[i, 72], arr[i, 73], arr[i, 74], arr[i, 75], arr[i, 76], arr[i, 77], arr[i, 78], arr[i, 79], arr[i, 80], arr[i, 81], arr[i, 82], arr[i, 83], arr[i, 84], arr[i, 85], arr[i, 86], arr[i, 87], arr[i, 88], arr[i, 89], arr[i, 90]).save() print('现金流量表数据导入完成:', code, end_date)
def test_connect_lazily_with_index(self): heartbeat_listener = HeartbeatStartedListener() create_indexes_listener = WhiteListEventListener('createIndexes') connect('mongodb://localhost:27017/foo', 'foo-connection', connect=False, event_listeners=[heartbeat_listener, create_indexes_listener]) client = _get_connection('foo-connection').database.client class Article(MongoModel): title = CharField() class Meta: connection_alias = 'foo-connection' indexes = [ IndexModel([('title', 1)]) ] # Creating the class didn't create a connection, or any indexes. self.assertEqual(len(heartbeat_listener.results), 0) self.assertEqual(len(create_indexes_listener.results['started']), 0) # The connection and indexes are created on the first query. self.assertEqual(Article.objects.count(), 0) self.assertGreaterEqual(len(heartbeat_listener.results), 1) self.assertGreaterEqual(len(create_indexes_listener.results['started']), 1)
def custom_connection(self, **kwargs): ################## # mix kwargs with variables variables = self.variables for key, value in kwargs.items(): variables[key] = value ################## # connect for authentication if required uri = "mongodb://{}:{}/{}".format( variables.get('host'), variables.get('port'), AUTH_DB, ) mongodb.connect(uri, alias=AUTH_DB) ################## db = variables.get('database', 'UNKNOWN') uri = "mongodb://{}:{}/{}".format(variables.get('host'), variables.get('port'), db) mongodb.connect(uri, alias=db) link = mongodb._get_connection(alias=db) log.verbose("Connected to db {}", db) class obj: connection = link return obj
def test_connect_with_kwargs(self): connect('mongodb://localhost:27017/foo?maxPoolSize=42', 'foo-connection', minpoolsize=10) client = _get_connection('foo-connection').database.client self.assertEqual(42, client.max_pool_size) self.assertEqual(10, client.min_pool_size)
def load_pipe(storage, name): pipe = None error = "Could not load pipeline" if storage == "m": try: pipe = load_pipe_from_db(name) except ValueError as exc: connect(app.config['mongo_db_url']) pipe = load_pipe_from_db(name) elif storage == "a": try: pipe = app.config['pipe_objects'][name] except KeyError as ke: # Todo pretty error handling error = ke elif storage == "f": try: pipe_path = app.config['pipe_files'][name] pipe = MDBHelper.load_results(pipe_path) except KeyError as ke: # Todo File not Found error = ke except Exception as e: # Todo: handle file does not exist debug = True if not pipe or not isinstance(pipe, MDBHyperpipe): # return "Could not load pipeline" return error return pipe
def get_field(SecuCode, collection_name): if collection_name == 'hk_secumain': print('Connect the Mongodb...') connect( "mongodb://*****:*****@192.168.0.119:27017/jydb?authSource=admin", alias="initial_update") SecuCodes_class = hk_secumain.objects.get_SecuCode(SecuCode) SecuCodes = [] for secu in SecuCodes_class: SecuCodes.append(eval('secu.' + SecuCode)) print('Download Completed.') return SecuCodes elif collection_name == 'hk_adjustedfactor': print('Connect the Mongodb...') connect( "mongodb://*****:*****@192.168.0.119:27017/jydb?authSource=admin", alias="initial_update_adjustedinfo") SecuCodes_class = hk_adjustedfactor.objects.get_SecuCode(SecuCode) SecuCodes = [] for secu in SecuCodes_class: SecuCodes.append(eval('secu.' + SecuCode)) print('Download Completed.') return SecuCodes
def get_stock_list(date): connect("mongodb://localhost:27017/stock?", alias="stock_list") # df = pro.stock_basic() data_L = pro.stock_basic( exchange='', list_status='L', fields= 'ts_code,name,area,industry,list_date,exchange,list_status,delist_date,is_hs' ) data_D = pro.stock_basic( exchange='', list_status='D', fields= 'ts_code,name,area,industry,list_date,exchange,list_status,delist_date,is_hs' ) data_P = pro.stock_basic( exchange='', list_status='P', fields= 'ts_code,name,area,industry,list_date,exchange,list_status,delist_date,is_hs' ) df = pd.concat([data_L, data_D, data_P], axis=0).reset_index(drop=True) arr = df.values if len(arr) == 0: print('非交易日:' + time.strftime("%Y%m%d", time.localtime(time.time()))) pass else: for i in tqdm(range(len(arr))): StockList(arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3], arr[i, 4], arr[i, 5], arr[i, 6], arr[i, 7], arr[i, 8]).save() print('股票列表数据导入完成:', date)
def test_connect_lazily_with_index(self): heartbeat_listener = HeartbeatStartedListener() create_indexes_listener = WhiteListEventListener('createIndexes') connect('mongodb://localhost:27017/foo', 'foo-connection', connect=False, event_listeners=[heartbeat_listener, create_indexes_listener]) client = _get_connection('foo-connection').database.client class Article(MongoModel): title = CharField() class Meta: connection_alias = 'foo-connection' indexes = [IndexModel([('title', 1)])] # Creating the class didn't create a connection, or any indexes. self.assertEqual(len(heartbeat_listener.results), 0) self.assertEqual(len(create_indexes_listener.results['started']), 0) # The connection and indexes are created on the first query. self.assertEqual(Article.objects.count(), 0) self.assertGreaterEqual(len(heartbeat_listener.results), 1) self.assertGreaterEqual( len(create_indexes_listener.results['started']), 1)
def load_pipe(storage, name): pipe = None error = "Could not load pipeline" if storage == "m": try: pipe = load_pipe_from_db(name) except ValueError as exc: connect(application.config['mongo_db_url'], alias='photon_core') pipe = load_pipe_from_db(name) if storage == "w": pipe = load_pipe_from_wizard(name) elif storage == "a": try: pipe = application.config['pipe_objects'][name] except KeyError as ke: # Todo pretty error handling error = ke elif storage == "f": try: pipe_path = application.config['pipe_files'][name] pipe = MDBHelper.load_results(pipe_path) except KeyError as ke: # Todo File not Found error = ke except Exception as e: # Todo: handle file does not exist debug = True if not pipe: session["error_msg"] = "Could not load result object." abort(500) return pipe
def get_financial_indicator(code): connect("mongodb://localhost:27017/stock?", alias="financial_indicator") df = pro.fina_indicator(ts_code=code) df = add_key_(df) arr = df.values if len(arr) == 0: print('非交易日:' + time.strftime("%Y%m%d", time.localtime(time.time()))) pass else: for i in tqdm(range(len(arr))): FinancialIndicator(arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3], arr[i, 4], arr[i, 5], arr[i, 6], arr[i, 7], arr[i, 8], arr[i, 9], arr[i, 10], arr[i, 11], arr[i, 12], arr[i, 13], arr[i, 14], arr[i, 15], arr[i, 16], arr[i, 17], arr[i, 18], arr[i, 19], arr[i, 20], arr[i, 21], arr[i, 22], arr[i, 23], arr[i, 24], arr[i, 25], arr[i, 26], arr[i, 27], arr[i, 28], arr[i, 29], arr[i, 30], arr[i, 31], arr[i, 32], arr[i, 33], arr[i, 34], arr[i, 35], arr[i, 36], arr[i, 37], arr[i, 38], arr[i, 39], arr[i, 40], arr[i, 41], arr[i, 42], arr[i, 43], arr[i, 44], arr[i, 45], arr[i, 46], arr[i, 47], arr[i, 48], arr[i, 49], arr[i, 50], arr[i, 51], arr[i, 52], arr[i, 53], arr[i, 54], arr[i, 55], arr[i, 56], arr[i, 57], arr[i, 58], arr[i, 59], arr[i, 60], arr[i, 61], arr[i, 62], arr[i, 63], arr[i, 64], arr[i, 65], arr[i, 66], arr[i, 67], arr[i, 68], arr[i, 69], arr[i, 70], arr[i, 71], arr[i, 72], arr[i, 73], arr[i, 74], arr[i, 75], arr[i, 76], arr[i, 77], arr[i, 78], arr[i, 79], arr[i, 80], arr[i, 81], arr[i, 82], arr[i, 83], arr[i, 84], arr[i, 85], arr[i, 86], arr[i, 87], arr[i, 88], arr[i, 89], arr[i, 90], arr[i, 91], arr[i, 92], arr[i, 93], arr[i, 94], arr[i, 95], arr[i, 96], arr[i, 97], arr[i, 98], arr[i, 99], arr[i, 100], arr[i, 101], arr[i, 102], arr[i, 103], arr[i, 104], arr[i, 105], arr[i, 106], arr[i, 107], arr[i, 108]).save() print('财务指标数据导入完成:', code)
def _setup_db(args): """ Setup db connection. """ runtime_config = args.config + "/runtime.cfg" config = configparser.ConfigParser() config.read(runtime_config) connect(config.get('general', 'mongodb'), ssl_cert_reqs=ssl.CERT_NONE)
def clear_db(app): with app.app_context(): connect(current_app.config["MONGO_URI"]) db = PyMongo(app).db db.sample.delete_many({}) db.step.delete_many({}) db.step_instance.delete_many({}) db.workflow.delete_many({})
def __init__(self, url="mongodb://localhost:27017/hive"): try: connect(url, alias="default") except Exception as e: print("Connection with MongoDB Failed!") print(e) return
def connect_db(uri, db_name=''): try: logger.debug('DB uri: {}, database name: {}'.format(uri, db_name)) connect(uri) logger.info('Connected to database') except Exception as e: logger.exception( 'Failed to connect to database. Exception: {}'.format(e)) raise e
def _connect(self) -> None: """ This method will connect to the MongoDB via URL using a MongoClient :return: None """ if not self.database_uri: raise KeyError(CONFIG_NOT_FOUND.format("MONGODB_DATABASE_URI")) connect(self.database_uri, connect=True, **self.config)
def __init__(self): ''' Start queue instance ''' connect(os.environ.get('MONGO_CLIENT', ''), retryWrites=False) self.thread = None MessageQueue.instanciated = True self.queue = Queue() self.driver = WhatsappMessaging()
def tst(session): os.environ["APPLICATION_ENV"] = "testing" application = make_app() session.client = application.test_client() session.context = application.test_request_context() with application.app_context(): connection.connect(application.config["MONGO_URL"]) yield session client = connection._get_connection().database.client client.drop_database(application.config["MONGO_DB"])
def _setup_db(args): """ Setup db connection. """ runtime_config = args.config + "/runtime.cfg" config = configparser.ConfigParser() config.read(runtime_config) mongodb_uri = config.get('general', 'mongodb', fallback=None) if mongodb_uri: connect(mongodb_uri, ssl_cert_reqs=ssl.CERT_NONE)
def load_available_pipes(): available_pipes = dict() available_pipes['RAM'] = app.config['pipe_objects'].keys() available_pipes['FILES'] = app.config['pipe_files'].keys() available_pipes['MONGO'] = [] if 'mongo_db_url' in app.config: try: load_mongo_pipes(available_pipes) except ValueError as exc: connect(app.config['mongo_db_url']) load_mongo_pipes(available_pipes) return available_pipes
def get_adjfactor(date): connect("mongodb://localhost:27017/stock?", alias="adjust_factor") df = pro.adj_factor(ts_code='', trade_date=date) # 复权因子无论从任何时间去下载都是一样的 df = add_key_(df) arr = df.values if len(arr) == 0: print('非交易日:' + time.strftime("%Y%m%d", time.localtime(time.time()))) pass else: for i in tqdm(range(len(arr))): AdjustFactor(arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3]).save() print('复权因子数据导入完成:', date)
def get_tradedays(start, end): connect("mongodb://localhost:27017/stock?", alias="trade_days") df = pro.trade_cal(exchange='', start_date=start, end_date=end) df = df[df['is_open'] == 1] arr = df.values if len(arr) == 0: print('非交易日:' + time.strftime("%Y%m%d", time.localtime(time.time()))) pass else: for i in tqdm(range(len(arr))): TradeDays(arr[i, 0], arr[i, 1], arr[i, 2]).save() print('日历数据导入完成:', end)
def load_pipe_from_wizard(obj_id): try: connect("mongodb://trap-umbriel:27017/photon_results", alias="photon_core") pipe = ( MDBHyperpipe.objects.order_by([("computation_start_time", DESCENDING)]) .raw({"wizard_object_id": ObjectId(obj_id)}) .first() ) return pipe except DoesNotExist as dne: # Todo: pretty error handling return dne
def get_suspend_list(date): connect("mongodb://localhost:27017/stock?", alias="suspend_list") df = pro.suspend(suspend_date=date) df = add_key_(df) arr = df.values if len(arr) == 0: print('非交易日:' + time.strftime("%Y%m%d", time.localtime(time.time()))) pass else: for i in tqdm(range(len(arr))): SuspendList(arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3], arr[i, 4]).save() print('停牌数据导入完成:', date)
def get_index_bar(code): connect("mongodb://localhost:27017/stock?", alias="index_bar") df = pro.index_daily(ts_code=code) df = add_key_(df) arr = df.values if len(arr) == 0: print('非交易日:' + time.strftime("%Y%m%d", time.localtime(time.time()))) pass else: for i in tqdm(range(len(arr))): IndexBar(arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3], arr[i, 4], arr[i, 5], arr[i, 6], arr[i, 7], arr[i, 8], arr[i, 9], arr[i, 10], arr[i, 11]).save() print('指数行情数据导入完成:', code)
def connect(self, **kwargs): variables = self.variables.copy() variables.update(kwargs) MongoExt.DATABASE = variables.get("database", "rapydo") uri = self._get_uri(variables) mongodb.connect(uri, alias=MongoExt.DATABASE) self.connection = mongodb._get_connection(alias=MongoExt.DATABASE) TopLevelMongoModel.save = catch_db_exceptions(TopLevelMongoModel.save) return self
def get_dailybar(date): connect("mongodb://localhost:27017/stock?", alias="origin_daily_bar") df = pro.daily(trade_date=date) df = add_key_(df) arr = df.values if len(arr) == 0: print('非交易日:' + time.strftime("%Y%m%d", time.localtime(time.time()))) pass else: for i in tqdm(range(len(arr))): OriginDailyBar(arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3], arr[i, 4], arr[i, 5], arr[i, 6], arr[i, 7], arr[i, 8], arr[i, 9], arr[i, 10], arr[i, 11]).save() print('行情数据导入完成:', date)
def get_client(self, retry=True): from pymodm.connection import connect from pymongo import MongoClient from pymongo.errors import ServerSelectionTimeoutError dburl = "mongodb://localhost:27017/dbtest" client = MongoClient(host=[dburl], serverSelectionTimeoutMS=200) try: client.server_info() connect(dburl) return client except ServerSelectionTimeoutError: if not retry: raise check_call('sudo systemctl start mongodb'.split()) sleep(1.0) return self.get_client(False)
def get_stock_main(date): connect("mongodb://localhost:27017/stock?", alias="stock_main") df_sh = pro.stock_company(exchange='SSE') df_sz = pro.stock_company(exchange='SZSE') df = pd.concat([df_sh, df_sz], axis=0).reset_index(drop=True) df['employees'] = df['employees'].fillna(0) arr = df.values if len(arr) == 0: print('非交易日:' + time.strftime("%Y%m%d", time.localtime(time.time()))) pass else: for i in tqdm(range(len(arr))): StocksMain(arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3], arr[i, 4], arr[i, 5], arr[i, 6], arr[i, 7], arr[i, 8], arr[i, 9], arr[i, 10], arr[i, 11]).save() print('股票主要信息数据导入完成:', date)
def get_index_weight(code, date): connect("mongodb://localhost:27017/stock?", alias="index_weight") if code == '000300.SH': code = '399300.SZ' elif code == '000905.SH': code = '399905.SZ' print('已将上交所指数代码改为深交所指数代码.') df = pro.index_weight(index_code=code, trade_date=date) df = add_key_(df) arr = df.values if len(arr) == 0: print('非交易日:' + time.strftime("%Y%m%d", time.localtime(time.time()))) pass else: for i in tqdm(range(len(arr))): IndexWeight(arr[i, 0], arr[i, 1], arr[i, 2], arr[i, 3], arr[i, 4]).save() print('指数权重数据导入完成:', code)
def test_connect_lazily(self): heartbeat_listener = HeartbeatStartedListener() connect('mongodb://localhost:27017/foo', 'foo-connection', connect=False, event_listeners=[heartbeat_listener]) client = _get_connection('foo-connection').database.client class Article(MongoModel): title = CharField() class Meta: connection_alias = 'foo-connection' # Creating the class didn't create a connection. self.assertEqual(len(heartbeat_listener.results), 0) # The connection is created on the first query. self.assertEqual(Article.objects.count(), 0) self.assertGreaterEqual(len(heartbeat_listener.results), 1)
def setUpClass(cls): cls.db_name = 'alternate-db' connect(MONGO_URI + '/' + cls.db_name, 'backups') cls.db = CLIENT[cls.db_name]
import sys import unittest import pymongo from pymodm.connection import connect PY3 = sys.version_info[0] == 3 MONGO_URI = os.environ.get('MONGO_URI', 'mongodb://localhost:27017') CLIENT = pymongo.MongoClient(MONGO_URI) DB = CLIENT.odm_test connect('%s/%s' % (MONGO_URI, DB.name)) class ODMTestCase(unittest.TestCase): def tearDown(self): CLIENT.drop_database(DB.name) if not PY3: # assertRaisesRegexp is deprecated in Python 3 but is all we have in # Python 2. def assertRaisesRegex(self, *args, **kwargs): return self.assertRaisesRegexp(*args, **kwargs) def assertEqualsModel(self, expected, model_instance): """Assert that a Model instance equals the expected document."""
def connect_to_test_DB(alias=None): if alias is None: connect('%s/%s' % (MONGO_URI, DB.name)) else: connect('%s/%s' % (MONGO_URI, DB.name), alias=alias)