def prj(request):
if request.param == "arctic":
connstr = "mongodb://localhost:27017/"
name = "test_project"
arc = arctic.Arctic(connstr)
if name in [lib.split(".")[0] for lib in arc.list_libraries()]:
connector = pst.ArcticConnector(name, connstr)
prj = pst.PastaStore(name, connector)
else:
connector = pst.ArcticConnector(name, connstr)
prj = initialize_project(connector)
elif request.param == "pystore":
name = "test_project"
path = "./tests/data/pystore"
pystore.set_path(path)
if name in pystore.list_stores():
connector = pst.PystoreConnector(name, path)
prj = pst.PastaStore(name, connector)
else:
connector = pst.PystoreConnector(name, path)
prj = initialize_project(connector)
elif request.param == "dict":
name = "test_project"
connector = pst.DictConnector(name)
prj = initialize_project(connector)
prj.type = request.param # added here for defining test dependencies
yield prj
def __init__(self,
library,
host='127.0.0.1',
key=None,
numeric_type=float,
quota=0,
**kwargs):
"""
library: str
arctic library. Will be created if does not exist.
key: str
setting key lets you override the symbol name.
The defaults are related to the data
being stored, i.e. trade, funding, etc
quota: int
absolute number of bytes that this library is limited to.
The default of 0 means that the storage size is unlimited.
kwargs:
if library needs to be created you can specify the
lib_type in the kwargs. Default is VersionStore, but you can
set to chunkstore with lib_type=arctic.CHUNK_STORE
"""
con = arctic.Arctic(host)
if library not in con.list_libraries():
lib_type = kwargs.get('lib_type', arctic.VERSION_STORE)
con.initialize_library(library, lib_type=lib_type)
con.set_quota(library, quota)
self.lib = con[library]
self.key = key if key else self.default_key
self.numeric_type = numeric_type
def to_arctic(self, connstr, libname, verbose=False):
"""Write ObsCollection to MongoDB using Arctic
Parameters
----------
connstr : str
database connection string
libname : str
name of the library to store data
verbose : bool, optional
show progress bar, by default False
"""
import arctic
from tqdm import tqdm
arc = arctic.Arctic(connstr)
# get library
try:
lib = arc.get_library(libname)
except arctic.exceptions.LibraryNotFoundException:
print("Library '{}' not found, initializing library...".format(
libname))
arc.initialize_library(libname)
lib = arc[libname]
# write data
for o in (tqdm(self.obs) if verbose else self.obs):
metadata = o.meta
lib.write(o.name, o, metadata=metadata)
def __init__(self, name: str, connstr: str):
"""Create an ArcticConnector object that connects to a running MongoDB
database via Arctic.
Parameters
----------
connstr : str
connection string (e.g. 'mongodb://localhost:27017/')
name : str
name of the project
"""
try:
import arctic
except ModuleNotFoundError as e:
print("Please install arctic (also requires "
"a MongoDB instance running somewhere, e.g. "
"MongoDB Community: \n"
"https://docs.mongodb.com/manual/administration"
"/install-community/)!")
raise e
self.connstr = connstr
self.name = name
self.libs: dict = {}
self.arc = arctic.Arctic(connstr)
self._initialize()
self.models = ModelAccessor(self)
# for older versions of PastaStore, if oseries_models library is empty
# populate oseries - models database
self._update_all_oseries_model_links()
def delete_arctic(connstr: str,
name: str,
libraries: Optional[List[str]] = None) -> None:
"""Delete libraries from arctic database.
Parameters
----------
connstr : str
connection string to the database
name : str
name of the database
libraries : Optional[List[str]], optional
list of library names to delete, by default None which deletes
all libraries
"""
try:
import arctic
except ModuleNotFoundError as e:
print("Please install `arctic`!")
raise e
arc = arctic.Arctic(connstr)
print(f"Deleting database: '{name}' ...")
# get library names
if libraries is None:
libs = []
for ilib in arc.list_libraries():
if ilib.split(".")[0] == name:
libs.append(ilib)
else:
libs = [name + "." + ilib for ilib in libraries]
for l in libs:
arc.delete_library(l)
print(f" - deleted: {l}")
print("... Done!")
def __init__(self, name: str, connstr: str):
"""Create an ArcticConnector object that connects to a running MongoDB
database via Arctic.
Parameters
----------
connstr : str
connection string (e.g. 'mongodb://localhost:27017/')
name : str
name of the project
"""
try:
import arctic
except ModuleNotFoundError as e:
print("Please install arctic (also requires "
"a MongoDB instance running somewhere, e.g. "
"MongoDB Community: \n"
"https://docs.mongodb.com/manual/administration"
"/install-community/)!")
raise e
self.connstr = connstr
self.name = name
self.libs: dict = {}
self.arc = arctic.Arctic(connstr)
self._initialize()
def get_store():
"""
get Arctic store connection
:return: arctic connection
"""
mongo_host = conf.MONGO_HOST
store = arctic.Arctic(mongo_host)
return store
def delete_arctic_connector(conn=None,
connstr: Optional[str] = None,
name: Optional[str] = None,
libraries: Optional[List[str]] = None) -> None:
"""Delete libraries from arctic database.
Parameters
----------
conn : pastastore.ArcticConnector
ArcticConnector object
connstr : str, optional
connection string to the database
name : str, optional
name of the database
libraries : Optional[List[str]], optional
list of library names to delete, by default None which deletes
all libraries
"""
import arctic
if conn is not None:
name = conn.name
connstr = conn.connstr
elif name is None or connstr is None:
raise ValueError("Provide 'name' and 'connstr' OR 'conn'!")
arc = arctic.Arctic(connstr)
print(f"Deleting ArcticConnector database: '{name}' ... ", end="")
# get library names
if libraries is None:
libs = []
for ilib in arc.list_libraries():
if ilib.split(".")[0] == name:
libs.append(ilib)
elif name is not None:
libs = [name + "." + ilib for ilib in libraries]
else:
raise ValueError("Provide 'name' and 'connstr' OR 'conn'!")
for lib in libs:
arc.delete_library(lib)
if libraries is not None:
print()
print(f" - deleted: {lib}")
print("Done!")
def main():
a = arctic.Arctic('127.0.0.1')
lib = a['BITMEX']
it = lib.iterator('l2_book-XBTUSD')
book = {'bid': {}, 'ask': {}}
for chunk in it:
for row in chunk.iterrows():
timestamp = row[0]
side, price, size, delta = row[1].values
if size == 0:
del book[side][price]
else:
book[side][price] = size
if delta:
print(f"Time: {timestamp} L2 Book: {book}")
def __init__(self, library, host='127.0.0.1', key=None, **kwargs):
"""
library: str
arctic library. Will be created if does not exist.
key: str
setting key lets you override the symbol name.
The defaults are related to the data
being stored, i.e. trade, funding, etc
kwargs:
if library needs to be created you can specify the
lib_type in the kwargs. Default is VersionStore, but you can
set to chunkstore with lib_type=arctic.CHUNK_STORE
"""
con = arctic.Arctic(host)
if library not in con.list_libraries():
lib_type = kwargs.get('lib_type', arctic.VERSION_STORE)
con.initialize_library(library, lib_type=lib_type)
self.lib = con[library]
self.key = key
def get_arctic_store_ex(config):
""" get arctic store """
store = None
try:
conn = pymongo.MongoClient(config.get("MONGODB", "MONGO_SERVER"))
# authenticate
auth = authenticate(conn[config.get("MONGODB", "MONGO_DATABASE")],\
config.get("MONGODB", "MONGO_USER"),\
config.get("MONGODB", "MONGO_PASSWORD"))
if auth == True:
store = arctic.Arctic(conn)
except Exception as ex:
print('Exception in getting arctic store {}'.format(str(ex.args)))
finally:
return store
def read_arctic(connstr, libname, ObsClass, verbose=False):
"""Read all timeseries from Arctic database.
Parameters
----------
connstr : str
connection string to MongoDB instance
libname : str
name of library to read data from
ObsClass : Obs
type of Obs to load data as
verbose : bool, optional
progressbar and timing info, by default False
Returns
-------
obs_list : list of Obs
list of Obs
"""
arc = arctic.Arctic(connstr)
lib = arc.get_library(libname)
start = default_timer()
obs_list = []
rows_read = 0
for sym in (tqdm(lib.list_symbols()) if verbose else lib.list_symbols()):
item = lib.read(sym)
o = ObsClass(item.data, name=sym, meta=item.metadata)
obs_list.append(o)
if verbose:
rows_read += len(item.data.index)
end = default_timer()
if verbose:
print("Symbols: {0:.0f} "
"Rows: {1:,.0f} "
"Time: {2:.2f}s "
"Rows/s: {3:,.1f}".format(len(lib.list_symbols()), rows_read,
(end - start),
rows_read / (end - start)))
return obs_list
def main():
a = arctic.Arctic('127.0.0.1')
lib = a['BITMEX']
it = lib.iterator('l2_book-XBTUSD')
book = {'bid': sd(), 'ask': sd()}
for chunk in it:
for row in chunk.iterrows():
timestamp = row[0]
side, price, size, receipt_timestamp, delta = row[1].values
if size == 0:
del book[side][price]
else:
book[side][price] = size
if delta:
print(f"Time: {timestamp} L2 Book: {book}")
best_bid, best_ask = book['bid'].keys()[-1], book['ask'].keys()[0]
print("Best Bid:", best_bid)
print("Best Ask:", best_ask)
print("\n")
def scrape_to_arctic(self,
library,
start=None,
end=None,
host='localhost',
channel='trade',
**kwargs):
if not start:
start = '11/22/2014'
if not end:
end = (pd.datetime.utcnow() - pd.Timedelta(days=1, hours=6)).date()
date_range = pd.date_range(start, end)
con = arctic.Arctic(host)
if library not in con.list_libraries():
lib_type = kwargs.get('lib_type', arctic.CHUNK_STORE)
con.initialize_library(library, lib_type=lib_type)
lib = con[library]
for date in tqdm(date_range):
df = self.get_df(date, channel)
lib.update(channel, df, upsert=True)
def get_arctic_store(config):
""" get arctic connection """
return arctic.Arctic(config.get("MONGODB", "MONGO_SERVER"))
def connect(self):
self._Arctic = arctic.Arctic(self.IPAddr)
return 0
def arctic_inf_init(self):
# handle the pd data, like panel dataframe series
# with arctic to hadnle mongodb local
self.store = arctic.Arctic('mongodb://127.0.0.1')
return Const.SUCC
def __init__(self, hostname='localhost'):
self.arctic = arctic.Arctic(hostname)
def get_arctic_store(config):
''' get arctic connection '''
return arctic.Arctic(config.get('MONGODB', 'MONGO_SERVER'))
import arctic, pyramid, swamp, atlantis
scenes = [[arctic.Arctic(), atlantis.Atlantis()],
[pyramid.Pyramid(), swamp.Swamp()]]
temperature = float(input("Temperatur: "))
humidity = float(input("Luftfeuchtigkeit: "))
if temperature > 20.0:
temperature = 1
else:
temperature = 0
if humidity > 20.0:
humidity = 1
else:
humidity = 0
cur_scenario = scenes[temperature][humidity]
print(cur_scenario)
from arctic.hooks import register_get_auth_hook
from pandas import Series
from typing import Dict
def setPWD(user=None, passwd=None):
if user is not None:
auth = lambda: None
auth.user = user
auth.password = passwd
register_get_auth_hook(lambda *x, **y: auth)
else:
register_get_auth_hook(lambda *x, **y: None)
arctic.Arctic("192.168.1.135:19931")
setPWD("liangxuanshuo", "}[;.KLJI*(%^5g6")
client = arctic.Arctic("192.168.1.135:19931")
lib = client['liangxuanshuo.demoFuture']
data = lib.read("rb1901", )
data = lib.read(symbol='rb1901', date_range=DateRange(start='2018-09-10', end='2018-11-13'))
data.sort_index()
# 将数据按每天分类，并将前一天的夜盘并到当天
da = {} # type: Dict[str, data]
days = []
starttime = pd.to_datetime(range(60), unit='D', origin=pd.Timestamp('2018-09-27'))
def f(x):
return x.strftime("%m%d")
# Make sure MongoDB is running in the data folder
import arctic
import pandas as pd
import datafeed
import datetime
# load configuration
config = pd.read_csv('arctic_config,csv')
bloomberg = pd.read_csv('cfg_blgterminal.csv')
localstore = arctic.Arctic('localhost')
success = []
fail = []
# Add new entries to the database
for entry in config.iterrows():
index = entry[0]
record = entry[1]
sendmethod = record['method']
csvpath = record['filepath']
# Download data from Mongo or csv and set index for the dataframe
if sendmethod == 'csv':
csvpath = 'Data/' + csvpath
pricedata = pd.read_csv(csvpath, index_col=0)
if sendmethod == 'mongo':
client = datafeed.QT_mongoclient(record['mongourl'])
pricedata = datafeed.QT_mongo2df(client, record['mongoname'],
record['mongocollection'])
pricedata.set_index(record['indexname'], inplace=True)
