def test_serialisation_of_numpy_ndarray(tmpdir):
from sacred.observers.tinydb_hashfs import NdArraySerializer
from tinydb_serialization import SerializationMiddleware
import numpy as np
# Setup Serialisation object for non list/dict objects
serialization_store = SerializationMiddleware()
serialization_store.register_serializer(NdArraySerializer(), 'TinyArray')
db = TinyDB(os.path.join(tmpdir.strpath, 'metadata.json'),
storage=serialization_store)
eye_mat = np.eye(3)
ones_array = np.ones(5)
document = {
'foo': 'bar',
'some_array': eye_mat,
'nested': {
'ones': ones_array
}
}
db.insert(document)
returned_doc = db.all()[0]
assert returned_doc['foo'] == 'bar'
assert (returned_doc['some_array'] == eye_mat).all()
assert (returned_doc['nested']['ones'] == ones_array).all()
def test_serialisation_of_pandas_dataframe(tmpdir):
from sacred.observers.tinydb_hashfs import (DataFrameSerializer,
SeriesSerializer)
from tinydb_serialization import SerializationMiddleware
import numpy as np
import pandas as pd
# Setup Serialisation object for non list/dict objects
serialization_store = SerializationMiddleware()
serialization_store.register_serializer(DataFrameSerializer(),
'TinyDataFrame')
serialization_store.register_serializer(SeriesSerializer(),
'TinySeries')
db = TinyDB(os.path.join(tmpdir.strpath, 'metadata.json'),
storage=serialization_store)
df = pd.DataFrame(np.eye(3), columns=list('ABC'))
series = pd.Series(np.ones(5))
document = {
'foo': 'bar',
'some_dataframe': df,
'nested': {
'ones': series
}
}
db.insert(document)
returned_doc = db.all()[0]
assert returned_doc['foo'] == 'bar'
assert (returned_doc['some_dataframe'] == df).all().all()
assert (returned_doc['nested']['ones'] == series).all()
def opendb():
"""
Open TinyDB database
"""
serialization = SerializationMiddleware()
serialization.register_serializer(DateSerializer(), 'Date Serializer')
return tinydb.TinyDB(DEFAULT_DATABASE, storage=serialization)
def test_serializer_nondestructive(tmpdir):
path = str(tmpdir.join('db.json'))
serializer = SerializationMiddleware(JSONStorage)
serializer.register_serializer(DateTimeSerializer(), 'TinyDate')
db = TinyDB(path, storage=serializer)
data = {'date': datetime.utcnow(), 'int': 3}
data_before = dict(data) # implicitly copy
db.insert(data)
assert data == data_before
def test_serializer(tmpdir):
path = str(tmpdir.join('db.json'))
serializer = SerializationMiddleware(JSONStorage)
serializer.register_serializer(DateTimeSerializer(), 'TinyDate')
db = TinyDB(path, storage=serializer)
date = datetime(2000, 1, 1, 12, 0, 0)
db.insert({'date': date})
db.insert({'int': 2})
assert db.count(where('date') == date) == 1
assert db.count(where('int') == 2) == 1
def test_serializer_recursive(tmpdir):
path = str(tmpdir.join('db.json'))
serializer = SerializationMiddleware(JSONStorage)
serializer.register_serializer(DateTimeSerializer(), 'TinyDate')
db = TinyDB(path, storage=serializer)
date = datetime(2000, 1, 1, 12, 0, 0)
datenow = datetime.utcnow()
dates = [{'date': date, 'hp': 100}, {'date': datenow, 'hp': 1}]
data = {'dates': dates, 'int': 10}
db.insert(data)
db.insert({'int': 2})
assert db.count(where('dates').any(where('date') == date)) == 1
assert db.count(where('int') == 2) == 1
from datetime import datetime
from tinydb.storages import JSONStorage
from tinydb_serialization import Serializer, SerializationMiddleware
class DateTimeSerializer(Serializer):
OBJ_CLASS = datetime # The class this serializer handles
FORMAT = '%Y-%m-%dT%H:%M:%S'
def encode(self, obj):
return obj.strftime(self.FORMAT)
def decode(self, s):
return datetime.strptime(s, self.FORMAT)
serializer = SerializationMiddleware()
serializer.register_serializer(DateTimeSerializer(), 'TinyDate')
def create(path='./runs_db', overwrite=None):
root_dir = os.path.abspath(path)
if not os.path.exists(root_dir):
os.makedirs(root_dir)
fs = HashFS(os.path.join(root_dir, 'hashfs'), depth=3,
width=2, algorithm='md5')
# Setup Serialisation object for non list/dict objects
serialization_store = SerializationMiddleware()
serialization_store.register_serializer(DateTimeSerializer(),
'TinyDate')
serialization_store.register_serializer(FileSerializer(fs),
'TinyFile')
if opt.has_numpy:
serialization_store.register_serializer(NdArraySerializer(),
'TinyArray')
if opt.has_pandas:
serialization_store.register_serializer(DataFrameSerializer(),
'TinyDataFrame')
serialization_store.register_serializer(SeriesSerializer(),
'TinySeries')
db = TinyDB(os.path.join(root_dir, 'metadata.json'),
storage=serialization_store)
return TinyDbObserver(db, fs, overwrite=overwrite, root=root_dir)
def __init__(self, path):
root_dir = os.path.abspath(path)
if not os.path.exists(root_dir):
raise IOError('Path does not exist: %s' % path)
fs = HashFS(os.path.join(root_dir, 'hashfs'), depth=3,
width=2, algorithm='md5')
# Setup Serialisation for non list/dict objects
serialization_store = SerializationMiddleware()
serialization_store.register_serializer(DateTimeSerializer(),
'TinyDate')
serialization_store.register_serializer(FileSerializer(fs),
'TinyFile')
if opt.has_numpy:
serialization_store.register_serializer(NdArraySerializer(),
'TinyArray')
if opt.has_pandas:
serialization_store.register_serializer(DataFrameSerializer(),
'TinyDataFrame')
serialization_store.register_serializer(SeriesSerializer(),
'TinySeries')
db = TinyDB(os.path.join(root_dir, 'metadata.json'),
storage=serialization_store)
self.db = db
self.runs = db.table('runs')
self.fs = fs
def initialize_tinydb(dbfile):
"""
"""
serialization = SerializationMiddleware()
serialization.register_serializer(PendulumDateTimeSerializer(), 'T') # Time
serialization.register_serializer(PendulumDateSerializer(), 'D') # Date
serialization.register_serializer(PendulumDurationSerializer(), 'I') # Interval
serialization.register_serializer(PendulumWeekdaySerializer(), 'W') # Wkday
serialization.register_serializer(MaskSerializer(), 'M') # Mask
return TinyDB(dbfile, storage=serialization, default_table='items', indent=1, ensure_ascii=False)
def __init__(self, path):
root_dir = os.path.abspath(path)
if not os.path.exists(root_dir):
raise IOError('Path does not exist: %s' % path)
fs = HashFS(os.path.join(root_dir, 'hashfs'),
depth=3,
width=2,
algorithm='md5')
# Setup Serialisation for non list/dict objects
serialization_store = SerializationMiddleware()
serialization_store.register_serializer(DateTimeSerializer(),
'TinyDate')
serialization_store.register_serializer(FileSerializer(fs), 'TinyFile')
if opt.has_numpy:
serialization_store.register_serializer(NdArraySerializer(),
'TinyArray')
if opt.has_pandas:
serialization_store.register_serializer(DataFrameSerializer(),
'TinyDataFrame')
serialization_store.register_serializer(SeriesSerializer(),
'TinySeries')
db = TinyDB(os.path.join(root_dir, 'metadata.json'),
storage=serialization_store)
self.db = db
self.runs = db.table('runs')
self.fs = fs
def init_db(self, dbfile):
""" Init repository """
serialization = SerializationMiddleware()
serialization.register_serializer(DateTimeSerializer(), 'TinyDate')
self.database = TinyDB(dbfile, storage=serialization)
def _storage(self):
serialization = SerializationMiddleware()
serialization.register_serializer(DateTimeSerializer(), 'TinyDate')
# TODO: Read custom serializers from settings and extensions
return serialization
def get_db_file_manager(root_dir):
fs = HashFS(os.path.join(root_dir, 'hashfs'),
depth=3,
width=2,
algorithm='md5')
# Setup Serialisation object for non list/dict objects
serialization_store = SerializationMiddleware()
serialization_store.register_serializer(DateTimeSerializer(), 'TinyDate')
serialization_store.register_serializer(FileSerializer(fs), 'TinyFile')
if opt.has_numpy:
serialization_store.register_serializer(NdArraySerializer(),
'TinyArray')
if opt.has_pandas:
serialization_store.register_serializer(DataFrameSerializer(),
'TinyDataFrame')
serialization_store.register_serializer(SeriesSerializer(),
'TinySeries')
db = TinyDB(os.path.join(root_dir, 'metadata.json'),
storage=serialization_store)
return db, fs
def create(path='./runs_db', overwrite=None):
root_dir = os.path.abspath(path)
if not os.path.exists(root_dir):
os.makedirs(root_dir)
fs = HashFS(os.path.join(root_dir, 'hashfs'),
depth=3,
width=2,
algorithm='md5')
# Setup Serialisation object for non list/dict objects
serialization_store = SerializationMiddleware()
serialization_store.register_serializer(DateTimeSerializer(),
'TinyDate')
serialization_store.register_serializer(FileSerializer(fs), 'TinyFile')
if opt.has_numpy:
serialization_store.register_serializer(NdArraySerializer(),
'TinyArray')
if opt.has_pandas:
serialization_store.register_serializer(DataFrameSerializer(),
'TinyDataFrame')
serialization_store.register_serializer(SeriesSerializer(),
'TinySeries')
db = TinyDB(os.path.join(root_dir, 'metadata.json'),
storage=serialization_store)
return TinyDbObserver(db, fs, overwrite=overwrite, root=root_dir)
import functools
import re
from .dict_riv import generate_riv
from .dict_riv import RIV
from .vec_perms import Permutations as Perms
from .serializers import RIVSerializer, PermutationSerializer
from tinydb_serialization import SerializationMiddleware
from tinydb import TinyDB, where
from tinyrecord import transaction
import random
serialization = SerializationMiddleware()
serialization.register_serializer(RIVSerializer(), 'RIV')
serialization.register_serializer(PermutationSerializer(), 'Permutations')
def _cycle_context(length, nums, position):
return nums[position + 1:length] + nums[:position]
def _merge_update(destination, update):
for k in update:
if k in destination:
destination[k].destructive_add(update[k])
else:
destination[k] = update[k]
def _open_db(db_path):
return TinyDB(db_path, storage=serialization)
"""
@Authour : Lindokuhle Magudulela
The scripts sets up database and all relavant tables
"""
from tinydb import TinyDB
from serializer import DateTimeSerializer
from tinydb_serialization import SerializationMiddleware
from tinydb_serialization import Serializer
serialization = SerializationMiddleware()
serialization.register_serializer(DateTimeSerializer(), 'TinyDate')
data_base = TinyDB('testdb.json', storage=serialization)
# Creating tables, tinyDB tables act the same as mongoDB collections
data_base.table('energy')
data_base.table('water')
data_base.table('solarHeating')
