def test_serialisation_of_numpy_ndarray(tmpdir): from sacred.observers.tinydb_hashfs import NdArraySerializer from tinydb_serialization import SerializationMiddleware import numpy as np # Setup Serialisation object for non list/dict objects serialization_store = SerializationMiddleware() serialization_store.register_serializer(NdArraySerializer(), 'TinyArray') db = TinyDB(os.path.join(tmpdir.strpath, 'metadata.json'), storage=serialization_store) eye_mat = np.eye(3) ones_array = np.ones(5) document = { 'foo': 'bar', 'some_array': eye_mat, 'nested': { 'ones': ones_array } } db.insert(document) returned_doc = db.all()[0] assert returned_doc['foo'] == 'bar' assert (returned_doc['some_array'] == eye_mat).all() assert (returned_doc['nested']['ones'] == ones_array).all()
def test_serialisation_of_pandas_dataframe(tmpdir): from sacred.observers.tinydb_hashfs import (DataFrameSerializer, SeriesSerializer) from tinydb_serialization import SerializationMiddleware import numpy as np import pandas as pd # Setup Serialisation object for non list/dict objects serialization_store = SerializationMiddleware() serialization_store.register_serializer(DataFrameSerializer(), 'TinyDataFrame') serialization_store.register_serializer(SeriesSerializer(), 'TinySeries') db = TinyDB(os.path.join(tmpdir.strpath, 'metadata.json'), storage=serialization_store) df = pd.DataFrame(np.eye(3), columns=list('ABC')) series = pd.Series(np.ones(5)) document = { 'foo': 'bar', 'some_dataframe': df, 'nested': { 'ones': series } } db.insert(document) returned_doc = db.all()[0] assert returned_doc['foo'] == 'bar' assert (returned_doc['some_dataframe'] == df).all().all() assert (returned_doc['nested']['ones'] == series).all()
def opendb(): """ Open TinyDB database """ serialization = SerializationMiddleware() serialization.register_serializer(DateSerializer(), 'Date Serializer') return tinydb.TinyDB(DEFAULT_DATABASE, storage=serialization)
def test_serializer_nondestructive(tmpdir): path = str(tmpdir.join('db.json')) serializer = SerializationMiddleware(JSONStorage) serializer.register_serializer(DateTimeSerializer(), 'TinyDate') db = TinyDB(path, storage=serializer) data = {'date': datetime.utcnow(), 'int': 3} data_before = dict(data) # implicitly copy db.insert(data) assert data == data_before
def test_serializer(tmpdir): path = str(tmpdir.join('db.json')) serializer = SerializationMiddleware(JSONStorage) serializer.register_serializer(DateTimeSerializer(), 'TinyDate') db = TinyDB(path, storage=serializer) date = datetime(2000, 1, 1, 12, 0, 0) db.insert({'date': date}) db.insert({'int': 2}) assert db.count(where('date') == date) == 1 assert db.count(where('int') == 2) == 1
def test_serializer_recursive(tmpdir): path = str(tmpdir.join('db.json')) serializer = SerializationMiddleware(JSONStorage) serializer.register_serializer(DateTimeSerializer(), 'TinyDate') db = TinyDB(path, storage=serializer) date = datetime(2000, 1, 1, 12, 0, 0) datenow = datetime.utcnow() dates = [{'date': date, 'hp': 100}, {'date': datenow, 'hp': 1}] data = {'dates': dates, 'int': 10} db.insert(data) db.insert({'int': 2}) assert db.count(where('dates').any(where('date') == date)) == 1 assert db.count(where('int') == 2) == 1
from datetime import datetime from tinydb.storages import JSONStorage from tinydb_serialization import Serializer, SerializationMiddleware class DateTimeSerializer(Serializer): OBJ_CLASS = datetime # The class this serializer handles FORMAT = '%Y-%m-%dT%H:%M:%S' def encode(self, obj): return obj.strftime(self.FORMAT) def decode(self, s): return datetime.strptime(s, self.FORMAT) serializer = SerializationMiddleware() serializer.register_serializer(DateTimeSerializer(), 'TinyDate')
def create(path='./runs_db', overwrite=None): root_dir = os.path.abspath(path) if not os.path.exists(root_dir): os.makedirs(root_dir) fs = HashFS(os.path.join(root_dir, 'hashfs'), depth=3, width=2, algorithm='md5') # Setup Serialisation object for non list/dict objects serialization_store = SerializationMiddleware() serialization_store.register_serializer(DateTimeSerializer(), 'TinyDate') serialization_store.register_serializer(FileSerializer(fs), 'TinyFile') if opt.has_numpy: serialization_store.register_serializer(NdArraySerializer(), 'TinyArray') if opt.has_pandas: serialization_store.register_serializer(DataFrameSerializer(), 'TinyDataFrame') serialization_store.register_serializer(SeriesSerializer(), 'TinySeries') db = TinyDB(os.path.join(root_dir, 'metadata.json'), storage=serialization_store) return TinyDbObserver(db, fs, overwrite=overwrite, root=root_dir)
def __init__(self, path): root_dir = os.path.abspath(path) if not os.path.exists(root_dir): raise IOError('Path does not exist: %s' % path) fs = HashFS(os.path.join(root_dir, 'hashfs'), depth=3, width=2, algorithm='md5') # Setup Serialisation for non list/dict objects serialization_store = SerializationMiddleware() serialization_store.register_serializer(DateTimeSerializer(), 'TinyDate') serialization_store.register_serializer(FileSerializer(fs), 'TinyFile') if opt.has_numpy: serialization_store.register_serializer(NdArraySerializer(), 'TinyArray') if opt.has_pandas: serialization_store.register_serializer(DataFrameSerializer(), 'TinyDataFrame') serialization_store.register_serializer(SeriesSerializer(), 'TinySeries') db = TinyDB(os.path.join(root_dir, 'metadata.json'), storage=serialization_store) self.db = db self.runs = db.table('runs') self.fs = fs
def initialize_tinydb(dbfile): """ """ serialization = SerializationMiddleware() serialization.register_serializer(PendulumDateTimeSerializer(), 'T') # Time serialization.register_serializer(PendulumDateSerializer(), 'D') # Date serialization.register_serializer(PendulumDurationSerializer(), 'I') # Interval serialization.register_serializer(PendulumWeekdaySerializer(), 'W') # Wkday serialization.register_serializer(MaskSerializer(), 'M') # Mask return TinyDB(dbfile, storage=serialization, default_table='items', indent=1, ensure_ascii=False)
def init_db(self, dbfile): """ Init repository """ serialization = SerializationMiddleware() serialization.register_serializer(DateTimeSerializer(), 'TinyDate') self.database = TinyDB(dbfile, storage=serialization)
def _storage(self): serialization = SerializationMiddleware() serialization.register_serializer(DateTimeSerializer(), 'TinyDate') # TODO: Read custom serializers from settings and extensions return serialization
def get_db_file_manager(root_dir): fs = HashFS(os.path.join(root_dir, 'hashfs'), depth=3, width=2, algorithm='md5') # Setup Serialisation object for non list/dict objects serialization_store = SerializationMiddleware() serialization_store.register_serializer(DateTimeSerializer(), 'TinyDate') serialization_store.register_serializer(FileSerializer(fs), 'TinyFile') if opt.has_numpy: serialization_store.register_serializer(NdArraySerializer(), 'TinyArray') if opt.has_pandas: serialization_store.register_serializer(DataFrameSerializer(), 'TinyDataFrame') serialization_store.register_serializer(SeriesSerializer(), 'TinySeries') db = TinyDB(os.path.join(root_dir, 'metadata.json'), storage=serialization_store) return db, fs
import functools import re from .dict_riv import generate_riv from .dict_riv import RIV from .vec_perms import Permutations as Perms from .serializers import RIVSerializer, PermutationSerializer from tinydb_serialization import SerializationMiddleware from tinydb import TinyDB, where from tinyrecord import transaction import random serialization = SerializationMiddleware() serialization.register_serializer(RIVSerializer(), 'RIV') serialization.register_serializer(PermutationSerializer(), 'Permutations') def _cycle_context(length, nums, position): return nums[position + 1:length] + nums[:position] def _merge_update(destination, update): for k in update: if k in destination: destination[k].destructive_add(update[k]) else: destination[k] = update[k] def _open_db(db_path): return TinyDB(db_path, storage=serialization)
""" @Authour : Lindokuhle Magudulela The scripts sets up database and all relavant tables """ from tinydb import TinyDB from serializer import DateTimeSerializer from tinydb_serialization import SerializationMiddleware from tinydb_serialization import Serializer serialization = SerializationMiddleware() serialization.register_serializer(DateTimeSerializer(), 'TinyDate') data_base = TinyDB('testdb.json', storage=serialization) # Creating tables, tinyDB tables act the same as mongoDB collections data_base.table('energy') data_base.table('water') data_base.table('solarHeating')