def __init__(self, db_name, domain, raise_if_table_doesnt_exist=True, only_enabled=False, only_users=False): self.db = None self.table = None self.only_enabled = (Query().enabled.exists() if only_enabled else Query().ntlmhash.exists()) & ( Query().enabled == True if only_enabled else Query().ntlmhash.exists()) self.only_users = (Query().username.exists() if only_users else Query( ).ntlmhash.exists()) & (Query().username.test(lambda v: not v.endswith( "$")) if only_users else Query().ntlmhash.exists()) serialization = SerializationMiddleware() serialization.register_serializer(DateTimeSerializer(), "datetime") self.db = TinyDB(db_name, storage=CachingMiddleware(serialization)) tables = list(self.db.tables()) if raise_if_table_doesnt_exist and domain not in tables: raise DomainDoesntExist( "Hashes for domain '{}' do not exist in database.".format( domain), tables) self.table = self.db.table(domain)
def test_serialisation_of_pandas_dataframe(tmpdir): from sacred.observers.tinydb_hashfs import (DataFrameSerializer, SeriesSerializer) from tinydb_serialization import SerializationMiddleware import numpy as np import pandas as pd # Setup Serialisation object for non list/dict objects serialization_store = SerializationMiddleware() serialization_store.register_serializer(DataFrameSerializer(), 'TinyDataFrame') serialization_store.register_serializer(SeriesSerializer(), 'TinySeries') db = TinyDB(os.path.join(tmpdir.strpath, 'metadata.json'), storage=serialization_store) df = pd.DataFrame(np.eye(3), columns=list('ABC')) series = pd.Series(np.ones(5)) document = { 'foo': 'bar', 'some_dataframe': df, 'nested': { 'ones': series } } db.insert(document) returned_doc = db.all()[0] assert returned_doc['foo'] == 'bar' assert (returned_doc['some_dataframe'] == df).all().all() assert (returned_doc['nested']['ones'] == series).all()
def test_serialisation_of_numpy_ndarray(tmpdir): from sacred.observers.tinydb_hashfs import NdArraySerializer from tinydb_serialization import SerializationMiddleware import numpy as np # Setup Serialisation object for non list/dict objects serialization_store = SerializationMiddleware() serialization_store.register_serializer(NdArraySerializer(), 'TinyArray') db = TinyDB(os.path.join(tmpdir.strpath, 'metadata.json'), storage=serialization_store) eye_mat = np.eye(3) ones_array = np.ones(5) document = { 'foo': 'bar', 'some_array': eye_mat, 'nested': { 'ones': ones_array } } db.insert(document) returned_doc = db.all()[0] assert returned_doc['foo'] == 'bar' assert (returned_doc['some_array'] == eye_mat).all() assert (returned_doc['nested']['ones'] == ones_array).all()
def initdb(name): # TODO get serialization to work, it seems borken serialization = SerializationMiddleware() serialization.register_serializer(DateTimeSerializer(), 'TinyDate') return tinydb.TinyDB(name, storage=serialization)
def create(path='./runs_db', overwrite=None): root_dir = os.path.abspath(path) if not os.path.exists(root_dir): os.makedirs(root_dir) fs = HashFS(os.path.join(root_dir, 'hashfs'), depth=3, width=2, algorithm='md5') # Setup Serialisation object for non list/dict objects serialization_store = SerializationMiddleware() serialization_store.register_serializer(DateTimeSerializer(), 'TinyDate') serialization_store.register_serializer(FileSerializer(fs), 'TinyFile') if opt.has_numpy: serialization_store.register_serializer(NdArraySerializer(), 'TinyArray') if opt.has_pandas: serialization_store.register_serializer(DataFrameSerializer(), 'TinyDataFrame') serialization_store.register_serializer(SeriesSerializer(), 'TinySeries') db = TinyDB(os.path.join(root_dir, 'metadata.json'), storage=serialization_store) return TinyDbObserver(db, fs, overwrite=overwrite, root=root_dir)
def __init__(self, path): root_dir = os.path.abspath(path) if not os.path.exists(root_dir): raise IOError('Path does not exist: %s' % path) fs = HashFS(os.path.join(root_dir, 'hashfs'), depth=3, width=2, algorithm='md5') # Setup Serialisation for non list/dict objects serialization_store = SerializationMiddleware() serialization_store.register_serializer(DateTimeSerializer(), 'TinyDate') serialization_store.register_serializer(FileSerializer(fs), 'TinyFile') if opt.has_numpy: serialization_store.register_serializer(NdArraySerializer(), 'TinyArray') if opt.has_pandas: serialization_store.register_serializer(DataFrameSerializer(), 'TinyDataFrame') serialization_store.register_serializer(SeriesSerializer(), 'TinySeries') db = TinyDB(os.path.join(root_dir, 'metadata.json'), storage=serialization_store) self.db = db self.runs = db.table('runs') self.fs = fs
def opendb(): """ Open TinyDB database """ serialization = SerializationMiddleware() serialization.register_serializer(DateSerializer(), 'Date Serializer') return tinydb.TinyDB(DEFAULT_DATABASE, storage=serialization)
def test_serialisation_of_pandas_dataframe(tmpdir): from sacred.observers.tinydb_hashfs.bases import DataFrameSerializer from sacred.observers.tinydb_hashfs.bases import SeriesSerializer from tinydb_serialization import SerializationMiddleware import numpy as np import pandas as pd # Setup Serialisation object for non list/dict objects serialization_store = SerializationMiddleware() serialization_store.register_serializer(DataFrameSerializer(), "TinyDataFrame") serialization_store.register_serializer(SeriesSerializer(), "TinySeries") db = TinyDB(os.path.join(tmpdir.strpath, "metadata.json"), storage=serialization_store) df = pd.DataFrame(np.eye(3), columns=list("ABC")) series = pd.Series(np.ones(5)) document = {"foo": "bar", "some_dataframe": df, "nested": {"ones": series}} db.insert(document) returned_doc = db.all()[0] assert returned_doc["foo"] == "bar" assert (returned_doc["some_dataframe"] == df).all().all() assert (returned_doc["nested"]["ones"] == series).all()
def test_serialisation_of_numpy_ndarray(tmpdir): from sacred.observers.tinydb_hashfs.bases import NdArraySerializer from tinydb_serialization import SerializationMiddleware import numpy as np # Setup Serialisation object for non list/dict objects serialization_store = SerializationMiddleware() serialization_store.register_serializer(NdArraySerializer(), "TinyArray") db = TinyDB(os.path.join(tmpdir.strpath, "metadata.json"), storage=serialization_store) eye_mat = np.eye(3) ones_array = np.ones(5) document = { "foo": "bar", "some_array": eye_mat, "nested": { "ones": ones_array } } db.insert(document) returned_doc = db.all()[0] assert returned_doc["foo"] == "bar" assert (returned_doc["some_array"] == eye_mat).all() assert (returned_doc["nested"]["ones"] == ones_array).all()
def initialize_tinydb(dbfile): """ """ serialization = SerializationMiddleware() serialization.register_serializer(PendulumDateTimeSerializer(), 'T') # Time serialization.register_serializer(PendulumDateSerializer(), 'D') # Date serialization.register_serializer(PendulumDurationSerializer(), 'I') # Interval serialization.register_serializer(PendulumWeekdaySerializer(), 'W') # Wkday serialization.register_serializer(MaskSerializer(), 'M') # Mask if tinydb_version >= '4.0.0': db = TinyDB(dbfile, storage=serialization, indent=1, ensure_ascii=False) db.default_table_name = 'items' else: db = TinyDB(dbfile, storage=serialization, default_table='items', indent=1, ensure_ascii=False) return db
def __init__( self, database_url: str = "", search_exception: Optional[bool] = None, ) -> None: """Initialize the repository attributes. Args: database_url: URL specifying the connection to the database. """ super().__init__(database_url, search_exception) self.database_file = os.path.expanduser( database_url.replace("tinydb://", "")) if not os.path.isfile(self.database_file): try: with open(self.database_file, "a", encoding="utf-8") as file_cursor: file_cursor.close() except FileNotFoundError as error: raise ConnectionError( f"Could not create the database file: {self.database_file}" ) from error serialization = SerializationMiddleware(JSONStorage) serialization.register_serializer(DateTimeSerializer(), "TinyDate") self.db_ = TinyDB(self.database_file, storage=serialization, sort_keys=True, indent=4) self.staged: Dict[str, List[Any]] = {"add": [], "remove": []}
def _new_db(path: Path) -> TinyDB: """ Construct a TinyDB with our customizations """ serialization = SerializationMiddleware() serialization.register_serializer(DateTimeSerializer(), "TinyDate") return TinyDB(path, storage=serialization)
def initialize_tinydb(dbfile): """ """ serialization = SerializationMiddleware() serialization.register_serializer(PendulumDateTimeSerializer(), 'T') # Time serialization.register_serializer(PendulumDateSerializer(), 'D') # Date serialization.register_serializer(PendulumDurationSerializer(), 'I') # Interval serialization.register_serializer(PendulumWeekdaySerializer(), 'W') # Wkday serialization.register_serializer(MaskSerializer(), 'M') # Mask return TinyDB(dbfile, storage=serialization, default_table='items', indent=1, ensure_ascii=False)
def get_db(table=None): serializer = SerializationMiddleware(JSONStorage) serializer.register_serializer(DateSerializer(), 'TinyDate') db = TinyDB(DATABASE, storage=serializer, indent=4) if table is None: return db else: return db.table(table)
def open_or_create_db(path): """Get a TinyDB database object for the recipy database. This opens the DB, creating it if it doesn't exist. """ serialization = SerializationMiddleware() serialization.register_serializer(DateTimeSerializer(), 'TinyDate') # if not os.path.exists(os.path.dirname(path)): # os.mkdir(os.path.dirname(path)) db = TinyDB(path, storage=serialization) return db
def test_serializer_nondestructive(tmpdir): path = str(tmpdir.join('db.json')) serializer = SerializationMiddleware(JSONStorage) serializer.register_serializer(DateTimeSerializer(), 'TinyDate') db = TinyDB(path, storage=serializer) data = {'date': datetime.utcnow(), 'int': 3, 'list': []} data_before = dict(data) # implicitly copy db.insert(data) assert data == data_before
def get_db(table=None): if 'db' not in g: serializer = SerializationMiddleware(JSONStorage) serializer.register_serializer(DateSerializer(), 'TinyDate') g.db = TinyDB(app.config['DATABASE'], storage=serializer, indent=4) if table is None: return g.db else: return g.db.table(table)
def test_serializer_nondestructive(tmpdir): path = str(tmpdir.join('db.json')) serializer = SerializationMiddleware(JSONStorage) serializer.register_serializer(DateTimeSerializer(), 'TinyDate') db = TinyDB(path, storage=serializer) data = {'date': datetime.utcnow(), 'int': 3} data_before = dict(data) # implicitly copy db.insert(data) assert data == data_before
def __init__(self, db=SONG_DB_PATH): CachingMiddleware.WRITE_CACHE_SIZE = 100000 # For context manager to be on save sight to make a cache clear serialization = SerializationMiddleware(CachingMiddleware(JSONStorage)) serialization.register_serializer(DateTimeSerializer(), 'TinyDate') super().__init__(db, storage=serialization, sort_keys=True, indent=4, separators=(',', ': ')) self._query = Query()
def __init__(self): super().__init__() loop = asyncio.get_event_loop() serialization = SerializationMiddleware() serialization.register_serializer(DateTimeSerializer(), 'TinyDate') serialization.register_serializer(AddressSerializer(), 'TinyAddress') self.db = TinyDB(os.path.join(settings.DATADIR, 'db.json'), storage=serialization) self.maildb = TinyDB(os.path.join(settings.DATADIR, 'maildb.json'))
def test_serializer(tmpdir): path = str(tmpdir.join('db.json')) serializer = SerializationMiddleware(JSONStorage) serializer.register_serializer(DateTimeSerializer(), 'TinyDate') db = TinyDB(path, storage=serializer) date = datetime(2000, 1, 1, 12, 0, 0) db.insert({'date': date}) db.insert({'int': 2}) assert db.count(where('date') == date) == 1 assert db.count(where('int') == 2) == 1
def get_collection(cls): """ Get collection (create one if it doesn't exist yet). :return: collection reference object """ collection = None if server_env == "localhost": # datetime serializer for TinyDB. Make sure to add tinydb-serialization in requirements.txt from datetime import datetime from tinydb_serialization import Serializer, SerializationMiddleware class DateTimeSerializer(Serializer): OBJ_CLASS = datetime # The class this serializer handles def encode(self, obj): return obj.strftime('%Y-%m-%dT%H:%M:%S') def decode(self, s): return datetime.strptime(s, '%Y-%m-%dT%H:%M:%S') serialization = SerializationMiddleware() serialization.register_serializer(DateTimeSerializer(), 'TinyDate') tiny_db_name = 'db.json' if os.getenv("TESTING"): tiny_db_name = 'test_db.json' db = TinyDB(tiny_db_name, storage=serialization) collection = db.table(cls.__name__) elif server_env == "gae": if gae_database == "datastore": db = datastore.Client() return db # no collections in the Datastore else: db = firestore.client() collection = db.collection(cls.__name__) elif server_env == "azure": db = client["my-database"] # these two env vars should be created automatically when you launch Cosmos DB (with MongoDB API) on Azure # if not, add them manually db.authenticate(name=os.getenv("APPSETTING_MONGO_USERNAME"), password=os.getenv("APPSETTING_MONGO_PASSWORD")) collection = db[cls.__name__] elif server_env == "heroku": db_name = os.getenv("MONGODB_URI").split(":")[1].replace( "//", "") # get the username out, which is also the db name db = client[db_name] collection = db[cls.__name__] return collection
def __init__(self, datadir="data/", **kwargs): super().__init__(**kwargs) # Post init_settings things if not os.path.isdir(datadir): os.makedirs(datadir) # TODO put this in start() serialization = SerializationMiddleware() serialization.register_serializer(DateTimeSerializer(), "TinyDate") serialization.register_serializer(AddressSerializer(), "TinyAddress") self.db = TinyDB(os.path.join(datadir, "db.json"), storage=serialization) self.maildb = TinyDB(os.path.join(datadir, "maildb.json"))
def test_serializer_recursive(tmpdir): path = str(tmpdir.join('db.json')) serializer = SerializationMiddleware(JSONStorage) serializer.register_serializer(DateTimeSerializer(), 'TinyDate') db = TinyDB(path, storage=serializer) date = datetime(2000, 1, 1, 12, 0, 0) datenow = datetime.utcnow() dates = [{'date': date, 'hp': 100}, {'date': datenow, 'hp': 1}] data = {'dates': dates, 'int': 10} db.insert(data) db.insert({'int': 2}) assert db.count(where('dates').any(where('date') == date)) == 1 assert db.count(where('int') == 2) == 1
def main(args): serialization = SerializationMiddleware(JSONStorage) serialization.register_serializer(DateTimeSerializer(), "TinyDate") db_path = args.output_dir / "feeds.json" db = TinyDB(db_path, sort_keys=True, indent=4, separators=(",", ": "), storage=serialization) feeds = db.table("feeds") if args.import_opml: import_opml(feeds, args.import_opml) update_feeds(feeds, force_updates=args.force_updates) for feed in feeds.all(): download_feed(feed, args.output_dir, args.max_episodes)
def serialization_config() -> SerializationMiddleware: def full_name(t: Type): return "%s.%s" % (t.__module__, t.__qualname__) serializers = [ DateTimeSerializer, SkypeUserNameSerializer, ] middleware = SerializationMiddleware() for serializer in serializers: logger.debug("Registering serializer '%s' for type '%s'...", full_name(serializer), full_name(serializer.OBJ_CLASS)) middleware.register_serializer(serializer(), full_name(serializer)) logger.debug("Serializer '%s' registered", full_name(serializer)) return middleware
def generate_connection(db_conf=None): """ create a connection from th given conf or from config.py (if db_conf == None) """ if db_conf == None: tinydb = database.get("tinydb", None) else: tinydb = db_conf serialization = SerializationMiddleware() serialization.register_serializer(DateTimeSerializer(), 'TinyDate') if tinydb: conn_str = tinydb["dbname"] #print(" ... setting it up for tinyDB: " + conn_str) tdb= TinyDB(conn_str, storage=serialization) return Dbinfo(db=tdb, collection=None) else: raise Exception("I had a problem setting up tinyDB")
def __init__(self, db='nonedb.json'): # Storage and serialization serializer = SerializationMiddleware(tinydb.storages.JSONStorage) serializer.register_serializer(DateTimeSerializer(), 'TinyDateTime') # A reference to the actual database object. self._conn = tinydb.TinyDB(db, storage=serializer) # Activat SmartCache self._conn.table_class = SmartCacheTable # A shortcut to ``tinydb.TinyDB.table`` method. # See http://tinydb.readthedocs.org/en/latest/usage.html#tables # for reference. self.table = self._conn.table # A shortcut to ``tinydb.where`` object. # See http://tinydb.readthedocs.org/en/latest/usage.html#queries # for reference. self.where = tinydb.where
class TinyDBDal(): def __init__(self): self.serialization = SerializationMiddleware() self.serialization.register_serializer(DateTimeSerializer(), 'TinyDate') self.db = TinyDB('db.json', storage=self.serialization) self.pastes = self.db.table('pastes') self.settings = self.db.table('settings') def insert_new_paste(self, paste): self.pastes.insert(paste) def get_last_crawl_date(self): settings = self.settings.all() return settings and settings[0].get('last_crawl_date') def update_last_crawl_date(self, date): if not self.get_last_crawl_date(): self.settings.insert({'last_crawl_date': date}) else: self.settings.update({'last_crawl_date': date})
def get_db_file_manager(root_dir) -> Tuple[TinyDB, HashFS]: root_dir = Path(root_dir) fs = HashFS(root_dir / "hashfs", depth=3, width=2, algorithm="md5") # Setup Serialisation object for non list/dict objects serialization_store = SerializationMiddleware() serialization_store.register_serializer(DateTimeSerializer(), "TinyDate") serialization_store.register_serializer(FileSerializer(fs), "TinyFile") if opt.has_numpy: serialization_store.register_serializer(NdArraySerializer(), "TinyArray") if opt.has_pandas: serialization_store.register_serializer(DataFrameSerializer(), "TinyDataFrame") serialization_store.register_serializer(SeriesSerializer(), "TinySeries") db = TinyDB(os.path.join(root_dir, "metadata.json"), storage=serialization_store) return db, fs
def get_db_file_manager(root_dir): fs = HashFS(os.path.join(root_dir, 'hashfs'), depth=3, width=2, algorithm='md5') # Setup Serialisation object for non list/dict objects serialization_store = SerializationMiddleware() serialization_store.register_serializer(DateTimeSerializer(), 'TinyDate') serialization_store.register_serializer(FileSerializer(fs), 'TinyFile') if opt.has_numpy: serialization_store.register_serializer(NdArraySerializer(), 'TinyArray') if opt.has_pandas: serialization_store.register_serializer(DataFrameSerializer(), 'TinyDataFrame') serialization_store.register_serializer(SeriesSerializer(), 'TinySeries') db = TinyDB(os.path.join(root_dir, 'metadata.json'), storage=serialization_store) return db, fs
# # # from testapp.config import database from tinydb import TinyDB, Query from tinydb.storages import JSONStorage from tinydb_serialization import SerializationMiddleware from models.tinydb.serializer import DateTimeSerializer tinydb = database.get("tinydb", None) serialization = SerializationMiddleware() serialization.register_serializer(DateTimeSerializer(), 'TinyDate') if tinydb: conn_str = tinydb["dbname"] print(" ... setting it up for tinyDB: " + conn_str) tinydb = TinyDB(conn_str, storage=serialization) else: raise Exception("I had a problem setting up tinyDB")
def init_db(self, dbfile): """ Init repository """ serialization = SerializationMiddleware() serialization.register_serializer(DateTimeSerializer(), 'TinyDate') self.database = TinyDB(dbfile, storage=serialization)
import functools import re from .dict_riv import generate_riv from .dict_riv import RIV from .vec_perms import Permutations as Perms from .serializers import RIVSerializer, PermutationSerializer from tinydb_serialization import SerializationMiddleware from tinydb import TinyDB, where from tinyrecord import transaction import random serialization = SerializationMiddleware() serialization.register_serializer(RIVSerializer(), 'RIV') serialization.register_serializer(PermutationSerializer(), 'Permutations') def _cycle_context(length, nums, position): return nums[position + 1:length] + nums[:position] def _merge_update(destination, update): for k in update: if k in destination: destination[k].destructive_add(update[k]) else: destination[k] = update[k] def _open_db(db_path): return TinyDB(db_path, storage=serialization)
from datetime import datetime from tinydb.storages import JSONStorage from tinydb_serialization import Serializer, SerializationMiddleware class DateTimeSerializer(Serializer): OBJ_CLASS = datetime # The class this serializer handles FORMAT = '%Y-%m-%dT%H:%M:%S' def encode(self, obj): return obj.strftime(self.FORMAT) def decode(self, s): return datetime.strptime(s, self.FORMAT) serializer = SerializationMiddleware() serializer.register_serializer(DateTimeSerializer(), 'TinyDate')
def _storage(self): serialization = SerializationMiddleware() serialization.register_serializer(DateTimeSerializer(), 'TinyDate') # TODO: Read custom serializers from settings and extensions return serialization