Example #1
0
def test_serializer(tmpdir):
    path = str(tmpdir.join('db.json'))

    serializer = SerializationMiddleware(JSONStorage)
    serializer.register_serializer(DateTimeSerializer(), 'TinyDate')
    db = TinyDB(path, storage=serializer)

    date = datetime(2000, 1, 1, 12, 0, 0)

    db.insert({'date': date})
    db.insert({'int': 2})
    assert db.count(where('date') == date) == 1
    assert db.count(where('int') == 2) == 1
def test_serializer_recursive(tmpdir):
    path = str(tmpdir.join('db.json'))

    serializer = SerializationMiddleware(JSONStorage)
    serializer.register_serializer(DateTimeSerializer(), 'TinyDate')
    db = TinyDB(path, storage=serializer)

    date = datetime(2000, 1, 1, 12, 0, 0)
    datenow = datetime.utcnow()
    dates = [{'date': date, 'hp': 100}, {'date': datenow, 'hp': 1}]
    data = {'dates': dates, 'int': 10}
    db.insert(data)
    db.insert({'int': 2})
    assert db.count(where('dates').any(where('date') == date)) == 1
    assert db.count(where('int') == 2) == 1
Example #3
0
def test_upgrade(tmpdir):
    db_file = tmpdir.join('db.json')
    db_file.write(v1_0)

    # Run upgrade
    assert migrate(str(db_file)) is True
    db = TinyDB(str(db_file))

    assert db.count(where('key') == 'value') == 1
Example #4
0
def test_cutom_mapping_type_with_json(tmpdir):
    from tinydb.database import Mapping

    class CustomDocument(Mapping):
        def __init__(self, data):
            self.data = data

        def __getitem__(self, key):
            return self.data[key]

        def __iter__(self):
            return iter(self.data)

        def __len__(self):
            return len(self.data)

    # Insert
    db = TinyDB(str(tmpdir.join('test.db')))
    db.purge()
    db.insert(CustomDocument({'int': 1, 'char': 'a'}))
    assert db.count(where('int') == 1) == 1

    # Insert multiple
    db.insert_multiple([
        CustomDocument({'int': 2, 'char': 'a'}),
        CustomDocument({'int': 3, 'char': 'a'})
    ])
    assert db.count(where('int') == 1) == 1
    assert db.count(where('int') == 2) == 1
    assert db.count(where('int') == 3) == 1

    # Write back
    doc_id = db.get(where('int') == 3).doc_id
    db.write_back([CustomDocument({'int': 4, 'char': 'a'})], [doc_id])
    assert db.count(where('int') == 3) == 0
    assert db.count(where('int') == 4) == 1
class ThoughtCache(object):
    """
    Quickly and simply cache python objects (inc. functions and classes) into a NoSQL
    database for later tag-based retrieval
    """
    def __init__(self, path='tc_db.json',
                 encoder=jsonpickle.encode, decoder=jsonpickle.decode):
        """
        Initialize.
        If specifying encoder and decoder, it must accept a signature of f(o).  If not,
            use `partial`, lambda, etc. to wrap it.
        :param path: path to TinyDB json database
        :param encoder: function to encode objects
        :param decoder: function to decode objects
        :return: None
        """
        self.db = TinyDB(path)
        self.encode, self.decode = encoder, decoder
        self._tags = collections.Counter()


    def store(self, o, metadata):
        """
        Store object `o` in the database as `metadata['name']`.

        `metadata['name']` must be provided, but need not be unique.

        If no UUID is provided in the metadata, a random one is created and used.
        UUIDs are the authoritative source of identity, so they are the only basis on
        which a record will be overwritten without regard to any other fields.  If an
        existing UUID is specified, *ALL* fields will be overwritten, not only the ones
        specified.

        UUID may be a valid UUID string or None.  If you don't intend to overwrite an
        existing record, simply remove the UUID key from the dict entirely, or leave as
        None.

        :param o: The actual object to be stored.
        :param metadata: dict of metadata with the following format:
            {
            'time':time.time(),
            'location':'work',
            'tag':['general networking'],
            'name':'ExtendedPingResult'  # 'name' is mandatory because it cannot be
                                           reliably inferred at runtime.
            'uuid':'85280d8e-66bf-4e65-814f-507a65c0375c'
            }
        :return:
        """

        store = self._store

        if 'uuid' not in metadata:
            metadata['uuid'] = str(uuid.uuid4())
        elif self._test_uuid(metadata['uuid']):
            store = self._update

        metadata['json'] = self.encode(o)

        store(metadata)

    def retrieve_json(self, metadata_filter):
        """
        Retrieve list of JSON objects that match the given filter.

        If uuid is specified, all other criteria are ignored.

        All non-uuid fields that are specified are, unless otherwise noted,
            treated as a series of 'OR' clauses that are 'AND'ed together.

        `if (location in locations) and (tag in tags): `
            etc...

        :param metadata_filter: dict of values to filter against.  Currently supported:
        {
        'time': day, month, year or range #NOT IMPLEMENTED
        'location': 'work', 'home', etc...
        'name': where('name').contains Value.  Case Sensitive
        'tag': single entry or list of entries
        'uuid': single entry or list of entries
        }
        :return: list of JSON strings representing the objects
        """
        query = metadata_filter_to_query(metadata_filter)
        return [x['json'] for x in sequenceify(self.db.search(query))]

    def retrieve(self, metadata_filter):
        """
        Returns reconstructed objects that match filter.
        :param metadata_filter: See ThoughtCache.retrieve_json.__doc__
        :return:
        """
        json_list = self.retrieve_json(metadata_filter)
        return list(map(self.decode, json_list))

    def tag_search(self, *tags):
        """
        Given a string or list of strings, return entries that with one or more
            matching tags (if A in B)
        :param tags: string or list of strings
        :return: list of json strings
        """
        metadata_filter = {'tag':sequenceify(tags)}
        return self.retrieve(metadata_filter)

    def update_tags(self):
        """
        updates internal tag Counter.
        :return: None
        """
        self._tags.clear()
        elements = self.db.search(where('tag'))
        for element in elements:
            self._tags.update(element['tag'])

    @property
    def tags(self):
        """
        Returns list of all tags currently in use in the database
        :return: list of tags (str).
        """
        self.update_tags()
        return list(self._tags.keys())

    def matching_tags(self, substr):
        """
        Returns list of all tags currently in use in the database that contain
            substr.

        :param substr: substring to search for.  Case insensitive.
        :return: list of tags (str).
        """
        return [x for x in self.tags if (x.lower().find(substr) > -1)]

    def _test_uuid(self, o_uuid):
        """
        Tests for the existence of a provided object UUID string in the
            database.
        :param o_uuid: object UUID string
        :return: True if UUID exists, else False
        """
        i = self.db.count(where('uuid') == o_uuid)  # number of matching queries
        assert i < 2, "uuid {o_uuid} is not unique in database!"
        return bool(i)

    def _store(self, metadata):
        """
        Store new entry into db using provided metadata
        :param json: JSON Object
        :param metadata: Dict of metadata.  Must include 'uuid', 'name', and 'json'.
        :return:
        """
        self.db.insert(metadata)

    def _update(self, metadata):
        """
        update existing db record.  Preserves TinyDB behavior:
            overwrites existing fields, adds new fields, does not delete fields.
        :param metadata: Dict of metadata, must include 'name', and 'uuid', and 'json'
        :return:
        """
        element = self.db.get(where('uuid') == metadata['uuid'])
        self.db.update(metadata, eids=[element.eid])
Example #6
0
#!/usr/bin/env python

from tinydb import TinyDB
import numpy as np

tdb = TinyDB(parse_args = False)
tdb.arg_parser().add_argument("-o", required = True)
args = tdb.parse_args()

z = np.zeros(tdb.dim(), np.int64)
for i in tdb.chunks():
	z += np.fromstring(i, np.uint8)
z = np.float64(z) / tdb.count()

open(args.o, "w").write(" ".join([repr(i) for i in z.flatten()]))
Example #7
0
#!/usr/bin/env python

from tinydb import TinyDB
import numpy as np

tdb = TinyDB(parse_args=False)
tdb.arg_parser().add_argument("-o", required=True)
tdb.arg_parser().add_argument("--mean", required=True)
args = tdb.parse_args()

# read the mean for each dimension
mean = np.array([float(i) for i in open(args.mean).readline().strip().split(" ")], np.float64)
assert len(mean) == 3072

z = np.zeros(tdb.dim(), np.float64)
for i in tdb.chunks():
    z += np.power(np.float64(np.fromstring(i, np.uint8)) - mean, 2)
z = np.sqrt(z / tdb.count())

open(args.o, "w").write(" ".join([repr(i) for i in z.flatten()]))
Example #8
0
from tinydb import TinyDB, Query

# https://tinydb.readthedocs.io/en/latest/usage.html

db = TinyDB("score.json")
db.purge()


db.insert({"name": "math", "score": 100})
db.insert({"name": "english", "score": 90})
db.insert({"name": "japanese", "score": 80})

S = Query()
print(db.count(S.score >= 90))
print(db.search(S.name.matches(".*h$")))
print(db.search(S.name.search("h$")))

print(db.search(S.name.test(lambda v: v in ["math", "japanese"])))
print(db.search((S.name == "math") | (S.name == "japanese")))
print(db.search((S.name == "math") & (S.score >= 80)))

# for collection, using all,any
Example #9
0
CHANNELS = 3
DIM = WIDTH * HEIGHT * CHANNELS

tdb = TinyDB(dimensions = DIM, parse_args = None)
tdb.arg_parser().add_argument("--mean", required = True)
tdb.arg_parser().add_argument("--std", required = True)
tdb.arg_parser().add_argument("--rows", type = int, default = 20000)
tdb.arg_parser().add_argument("-o", required = True)
args = tdb.parse_args()

# read the mean for each dimension
mean = np.array([float(i) for i in open(args.mean).readline().strip().split(" ")], np.float64)
assert(len(mean) == DIM)

# read the standard deviation for each dimension
std = np.array([float(i) for i in open(args.std).readline().strip().split(" ")], np.float64)
assert(len(std) == DIM)


def compute(m):
	k = np.matrix([np.fromstring(i, np.uint8) for i in m]) - mean
	k = k / std
	return k.transpose() * k

jobs = process(tdb.groups(args.rows), compute)
m = reduce(lambda acc, x: acc + x, jobs, np.zeros((DIM, DIM), np.float64))

print >> sys.stderr, "processed rows:", tdb.count()

sio.savemat(args.o, {"c": m, "n": tdb.count()}, do_compression = True)
#rows = db.get(('task'), where('priority') == 10)
#rows = db.search(where('id'))
start = datetime.datetime.now()
for x in range(0, 1000):
	db.insert({'id': 1, 'task': 'Read A-byte-of-python to get a good introduction into Python', 'status': '0'})
	db.insert({'id': 1, 'task': 'Read A-byte-of-python to get a good introduction into Python', 'status': '0', 'priority': 10})
end = datetime.datetime.now()
exc_time = end - start
print ("Execution time : ", exc_time)

stime = datetime.datetime.now()
rows = db.search(where('id'))
etime = datetime.datetime.now()
total_time = etime - stime
print ("Fecthing time : ",  total_time)
count = db.count(where('id'))
print (count)

#import sqlite3
#import datetime
#con = sqlite3.connect('homework_sqlite.db')
#cursor = con.cursor()

#start = datetime.datetime.now()
#for x in range(0, 10000):
#con.execute("CREATE TABLE Homework (id INTEGER PRIMARY KEY, task char(100) NOT NULL, status bool NOT NULL)")
#	con.execute("INSERT INTO Homework (task,status) VALUES ('Finish software testing Homework',0)")
#	con.execute("INSERT INTO Homework (task,status) VALUES ('Finished SQlite part',1)")
#con.commit()
#end = datetime.datetime.now()
#exc_time = end - start