def prj(request): if request.param == "arctic": connstr = "mongodb://localhost:27017/" name = "test_project" arc = arctic.Arctic(connstr) if name in [lib.split(".")[0] for lib in arc.list_libraries()]: connector = pst.ArcticConnector(name, connstr) prj = pst.PastaStore(name, connector) else: connector = pst.ArcticConnector(name, connstr) prj = initialize_project(connector) elif request.param == "pystore": name = "test_project" path = "./tests/data/pystore" pystore.set_path(path) if name in pystore.list_stores(): connector = pst.PystoreConnector(name, path) prj = pst.PastaStore(name, connector) else: connector = pst.PystoreConnector(name, path) prj = initialize_project(connector) elif request.param == "dict": name = "test_project" connector = pst.DictConnector(name) prj = initialize_project(connector) prj.type = request.param # added here for defining test dependencies yield prj
def __init__(self, library, host='127.0.0.1', key=None, numeric_type=float, quota=0, **kwargs): """ library: str arctic library. Will be created if does not exist. key: str setting key lets you override the symbol name. The defaults are related to the data being stored, i.e. trade, funding, etc quota: int absolute number of bytes that this library is limited to. The default of 0 means that the storage size is unlimited. kwargs: if library needs to be created you can specify the lib_type in the kwargs. Default is VersionStore, but you can set to chunkstore with lib_type=arctic.CHUNK_STORE """ con = arctic.Arctic(host) if library not in con.list_libraries(): lib_type = kwargs.get('lib_type', arctic.VERSION_STORE) con.initialize_library(library, lib_type=lib_type) con.set_quota(library, quota) self.lib = con[library] self.key = key if key else self.default_key self.numeric_type = numeric_type
def to_arctic(self, connstr, libname, verbose=False): """Write ObsCollection to MongoDB using Arctic Parameters ---------- connstr : str database connection string libname : str name of the library to store data verbose : bool, optional show progress bar, by default False """ import arctic from tqdm import tqdm arc = arctic.Arctic(connstr) # get library try: lib = arc.get_library(libname) except arctic.exceptions.LibraryNotFoundException: print("Library '{}' not found, initializing library...".format( libname)) arc.initialize_library(libname) lib = arc[libname] # write data for o in (tqdm(self.obs) if verbose else self.obs): metadata = o.meta lib.write(o.name, o, metadata=metadata)
def __init__(self, name: str, connstr: str): """Create an ArcticConnector object that connects to a running MongoDB database via Arctic. Parameters ---------- connstr : str connection string (e.g. 'mongodb://localhost:27017/') name : str name of the project """ try: import arctic except ModuleNotFoundError as e: print("Please install arctic (also requires " "a MongoDB instance running somewhere, e.g. " "MongoDB Community: \n" "https://docs.mongodb.com/manual/administration" "/install-community/)!") raise e self.connstr = connstr self.name = name self.libs: dict = {} self.arc = arctic.Arctic(connstr) self._initialize() self.models = ModelAccessor(self) # for older versions of PastaStore, if oseries_models library is empty # populate oseries - models database self._update_all_oseries_model_links()
def delete_arctic(connstr: str, name: str, libraries: Optional[List[str]] = None) -> None: """Delete libraries from arctic database. Parameters ---------- connstr : str connection string to the database name : str name of the database libraries : Optional[List[str]], optional list of library names to delete, by default None which deletes all libraries """ try: import arctic except ModuleNotFoundError as e: print("Please install `arctic`!") raise e arc = arctic.Arctic(connstr) print(f"Deleting database: '{name}' ...") # get library names if libraries is None: libs = [] for ilib in arc.list_libraries(): if ilib.split(".")[0] == name: libs.append(ilib) else: libs = [name + "." + ilib for ilib in libraries] for l in libs: arc.delete_library(l) print(f" - deleted: {l}") print("... Done!")
def __init__(self, name: str, connstr: str): """Create an ArcticConnector object that connects to a running MongoDB database via Arctic. Parameters ---------- connstr : str connection string (e.g. 'mongodb://localhost:27017/') name : str name of the project """ try: import arctic except ModuleNotFoundError as e: print("Please install arctic (also requires " "a MongoDB instance running somewhere, e.g. " "MongoDB Community: \n" "https://docs.mongodb.com/manual/administration" "/install-community/)!") raise e self.connstr = connstr self.name = name self.libs: dict = {} self.arc = arctic.Arctic(connstr) self._initialize()
def get_store(): """ get Arctic store connection :return: arctic connection """ mongo_host = conf.MONGO_HOST store = arctic.Arctic(mongo_host) return store
def delete_arctic_connector(conn=None, connstr: Optional[str] = None, name: Optional[str] = None, libraries: Optional[List[str]] = None) -> None: """Delete libraries from arctic database. Parameters ---------- conn : pastastore.ArcticConnector ArcticConnector object connstr : str, optional connection string to the database name : str, optional name of the database libraries : Optional[List[str]], optional list of library names to delete, by default None which deletes all libraries """ import arctic if conn is not None: name = conn.name connstr = conn.connstr elif name is None or connstr is None: raise ValueError("Provide 'name' and 'connstr' OR 'conn'!") arc = arctic.Arctic(connstr) print(f"Deleting ArcticConnector database: '{name}' ... ", end="") # get library names if libraries is None: libs = [] for ilib in arc.list_libraries(): if ilib.split(".")[0] == name: libs.append(ilib) elif name is not None: libs = [name + "." + ilib for ilib in libraries] else: raise ValueError("Provide 'name' and 'connstr' OR 'conn'!") for lib in libs: arc.delete_library(lib) if libraries is not None: print() print(f" - deleted: {lib}") print("Done!")
def main(): a = arctic.Arctic('127.0.0.1') lib = a['BITMEX'] it = lib.iterator('l2_book-XBTUSD') book = {'bid': {}, 'ask': {}} for chunk in it: for row in chunk.iterrows(): timestamp = row[0] side, price, size, delta = row[1].values if size == 0: del book[side][price] else: book[side][price] = size if delta: print(f"Time: {timestamp} L2 Book: {book}")
def __init__(self, library, host='127.0.0.1', key=None, **kwargs): """ library: str arctic library. Will be created if does not exist. key: str setting key lets you override the symbol name. The defaults are related to the data being stored, i.e. trade, funding, etc kwargs: if library needs to be created you can specify the lib_type in the kwargs. Default is VersionStore, but you can set to chunkstore with lib_type=arctic.CHUNK_STORE """ con = arctic.Arctic(host) if library not in con.list_libraries(): lib_type = kwargs.get('lib_type', arctic.VERSION_STORE) con.initialize_library(library, lib_type=lib_type) self.lib = con[library] self.key = key
def get_arctic_store_ex(config): """ get arctic store """ store = None try: conn = pymongo.MongoClient(config.get("MONGODB", "MONGO_SERVER")) # authenticate auth = authenticate(conn[config.get("MONGODB", "MONGO_DATABASE")],\ config.get("MONGODB", "MONGO_USER"),\ config.get("MONGODB", "MONGO_PASSWORD")) if auth == True: store = arctic.Arctic(conn) except Exception as ex: print('Exception in getting arctic store {}'.format(str(ex.args))) finally: return store
def read_arctic(connstr, libname, ObsClass, verbose=False): """Read all timeseries from Arctic database. Parameters ---------- connstr : str connection string to MongoDB instance libname : str name of library to read data from ObsClass : Obs type of Obs to load data as verbose : bool, optional progressbar and timing info, by default False Returns ------- obs_list : list of Obs list of Obs """ arc = arctic.Arctic(connstr) lib = arc.get_library(libname) start = default_timer() obs_list = [] rows_read = 0 for sym in (tqdm(lib.list_symbols()) if verbose else lib.list_symbols()): item = lib.read(sym) o = ObsClass(item.data, name=sym, meta=item.metadata) obs_list.append(o) if verbose: rows_read += len(item.data.index) end = default_timer() if verbose: print("Symbols: {0:.0f} " "Rows: {1:,.0f} " "Time: {2:.2f}s " "Rows/s: {3:,.1f}".format(len(lib.list_symbols()), rows_read, (end - start), rows_read / (end - start))) return obs_list
def main(): a = arctic.Arctic('127.0.0.1') lib = a['BITMEX'] it = lib.iterator('l2_book-XBTUSD') book = {'bid': sd(), 'ask': sd()} for chunk in it: for row in chunk.iterrows(): timestamp = row[0] side, price, size, receipt_timestamp, delta = row[1].values if size == 0: del book[side][price] else: book[side][price] = size if delta: print(f"Time: {timestamp} L2 Book: {book}") best_bid, best_ask = book['bid'].keys()[-1], book['ask'].keys()[0] print("Best Bid:", best_bid) print("Best Ask:", best_ask) print("\n")
def scrape_to_arctic(self, library, start=None, end=None, host='localhost', channel='trade', **kwargs): if not start: start = '11/22/2014' if not end: end = (pd.datetime.utcnow() - pd.Timedelta(days=1, hours=6)).date() date_range = pd.date_range(start, end) con = arctic.Arctic(host) if library not in con.list_libraries(): lib_type = kwargs.get('lib_type', arctic.CHUNK_STORE) con.initialize_library(library, lib_type=lib_type) lib = con[library] for date in tqdm(date_range): df = self.get_df(date, channel) lib.update(channel, df, upsert=True)
def get_arctic_store(config): """ get arctic connection """ return arctic.Arctic(config.get("MONGODB", "MONGO_SERVER"))
def connect(self): self._Arctic = arctic.Arctic(self.IPAddr) return 0
def arctic_inf_init(self): # handle the pd data, like panel dataframe series # with arctic to hadnle mongodb local self.store = arctic.Arctic('mongodb://127.0.0.1') return Const.SUCC
def __init__(self, hostname='localhost'): self.arctic = arctic.Arctic(hostname)
def get_arctic_store(config): ''' get arctic connection ''' return arctic.Arctic(config.get('MONGODB', 'MONGO_SERVER'))
import arctic, pyramid, swamp, atlantis scenes = [[arctic.Arctic(), atlantis.Atlantis()], [pyramid.Pyramid(), swamp.Swamp()]] temperature = float(input("Temperatur: ")) humidity = float(input("Luftfeuchtigkeit: ")) if temperature > 20.0: temperature = 1 else: temperature = 0 if humidity > 20.0: humidity = 1 else: humidity = 0 cur_scenario = scenes[temperature][humidity] print(cur_scenario)
from arctic.hooks import register_get_auth_hook from pandas import Series from typing import Dict def setPWD(user=None, passwd=None): if user is not None: auth = lambda: None auth.user = user auth.password = passwd register_get_auth_hook(lambda *x, **y: auth) else: register_get_auth_hook(lambda *x, **y: None) arctic.Arctic("192.168.1.135:19931") setPWD("liangxuanshuo", "}[;.KLJI*(%^5g6") client = arctic.Arctic("192.168.1.135:19931") lib = client['liangxuanshuo.demoFuture'] data = lib.read("rb1901", ) data = lib.read(symbol='rb1901', date_range=DateRange(start='2018-09-10', end='2018-11-13')) data.sort_index() # 将数据按每天分类,并将前一天的夜盘并到当天 da = {} # type: Dict[str, data] days = [] starttime = pd.to_datetime(range(60), unit='D', origin=pd.Timestamp('2018-09-27')) def f(x): return x.strftime("%m%d")
# Make sure MongoDB is running in the data folder import arctic import pandas as pd import datafeed import datetime # load configuration config = pd.read_csv('arctic_config,csv') bloomberg = pd.read_csv('cfg_blgterminal.csv') localstore = arctic.Arctic('localhost') success = [] fail = [] # Add new entries to the database for entry in config.iterrows(): index = entry[0] record = entry[1] sendmethod = record['method'] csvpath = record['filepath'] # Download data from Mongo or csv and set index for the dataframe if sendmethod == 'csv': csvpath = 'Data/' + csvpath pricedata = pd.read_csv(csvpath, index_col=0) if sendmethod == 'mongo': client = datafeed.QT_mongoclient(record['mongourl']) pricedata = datafeed.QT_mongo2df(client, record['mongoname'], record['mongocollection']) pricedata.set_index(record['indexname'], inplace=True)