def setUp(self):
with open(os.path.join(BASE_DIR, 'tests', 'fixtures', 'test.json')) as f:
self.j = json.load(f)
self.database_name = os.path.join(os.getcwd(), 'test.db')
db = TinyDB(self.database_name)
db.insert_multiple(self.j)
db.close
def write_to_cache_file(data,
cache_path,
type_id=0,
region_id=0,
logger=LOGGER):
"""save data to tinyDB
Args:
data (:obj:`pandas.DataFrame`): data to write out
cache_path (str): path to cache file
type_id (int, optional): EVE Online type_id
region_id (int, optional): EVE Online region_id
logger (:obj:`logging.logger`, optional): logging handle
Returns:
None
"""
## get DB ##
logger.info('Writing data to cache')
tdb = TinyDB(cache_path)
date_min = data['date'].min()
logger.debug(date_min)
## clean out existing entries ##
if (type_id and region_id):
logger.info('--Removing old cache entries')
tdb.remove((Query().region_id == region_id)
& (Query().type_id == type_id) & (Query().date >= date_min))
caching_data_str = data.to_json(date_format='iso', orient='records')
cache_data = json.loads(caching_data_str)
logger.info('--Writing to cache file')
tdb.insert_multiple(cache_data)
class JsonDB(object):
def __init__(self, tb_name):
"""
使用特殊的表名
:param table_name:
:return:
"""
self.db = TinyDB(DATABASES['json_db']['NAME'])
self.Q = Query()
self.tb = self.db.table(tb_name)
def insert(self, one_json):
"""
插入单个json数据
:param one_json:
:return:
"""
try:
self.db.insert(one_json)
except Exception as err:
print "无法插入json数据: ", one_json
raise err
def insert_multiple(self, list_of_json):
"""
插入一组json数据
:param list_of_json:
:return:
"""
try:
self.db.insert_multiple(list_of_json)
except Exception as err:
print "无法批量插入json数据: ", list_of_json
raise err
def _open_db(self, dbpath):
self.verbose(f"Opening merge database: '{dbpath}'")
# copy the existing database into memory
db = TinyDB(dbpath)
db_ram = TinyDB(storage=MemoryStorage)
db_ram.insert_multiple(db.all())
return db_ram
def _initialize(self):
"""
Compute camera poses and K matrix.
Store them in local variables
:return:
:rtype:
"""
self._image_type_map = {
"rgb": 'rgb',
'depth_float32': 'depth_32F',
'depth_int16': 'depth_16U',
'label': 'label',
'mask': 'mask'
}
self._camera_pose_dict = dict()
self._camera_matrix_dict = dict()
for camera_name in self.camera_names:
self._camera_pose_dict[camera_name] = self.camera_pose(camera_name)
self._camera_matrix_dict[camera_name] = self.camera_K_matrix(
camera_name)
# make TinyDB dataset for label etc
keys = ['label_db', 'mask_db']
for key in keys:
if key in self._non_image_data['metadata']:
data_list = self._non_image_data['metadata'][key]
db = TinyDB(storage=MemoryStorage)
db.insert_multiple(data_list)
self.mask_labels = []
for entry in self.mask_db:
self.mask_labels.append(int(entry['label']))
class TinyDBBackend:
def __init__(self, conf):
from tinydb import TinyDB
self.con = TinyDB(conf['path'])
def save(self, dicts):
self._cast_unserializable(dicts)
self.con.insert_multiple(dicts)
def update(self, dicts):
self._cast_unserializable(dicts)
for d in dicts:
self.con.update(d, eid=[d.eid])
def get(self, **kwargs):
raise Exception('Not implemented')
@staticmethod
def _cast_unserializable(dicts):
# tinydb cannot serialize some objects to json
# cast them before saving
import datetime
for d in dicts:
for k, v in d.items():
if isinstance(v, datetime.datetime):
d[k] = v.strftime("%Y-%m-%d %H:%M:%S")
class Proxy(object):
def __init__(self, config):
self.c = config
self.ldb = None
self.rdb = None
self.tag = Query()
self.req = None
if config.local:
try:
self.ldb = TinyDB(config.local,
storage=CachingMiddleware(JSONStorage))
except:
self.ldb = TinyDB(storage=MemoryStorage)
else:
self.ldb = TinyDB(storage=MemoryStorage)
if config.url:
auth = None
if config.user:
auth = (config.user, click.prompt('password', hide_input=True))
if config.url.startswith('http'):
dbclass = CouchDB
elif config.url.startswith('mongodb'):
dbclass = MongoDB
try:
self.rdb = dbclass(config.url, auth=auth, verify=config.verify)
except:
self.rdb = None
def set_tag(self, tag=None):
self.tag = (where('tag').search(tag)) if tag else Query()
def insert_multiple(self, docs):
self.ldb.insert_multiple(docs)
def contains(self, q=None, **kargs):
if q is None: q = self.tag
for k in kargs:
q &= (where(k) == kargs[k])
if self.rdb:
return self.rdb.contains(q.hashval, **kargs)
return self.ldb.contains(q)
def search(self, q=None, **kargs):
if q is None: q = self.tag
for k in kargs:
q &= (where(k) == kargs[k])
if self.rdb:
return list(self.rdb.search(q.hashval, **kargs))
return self.ldb.search(q)
def get(self, q=None, **kargs):
if q is None: q = self.tag
for k in kargs:
q &= (where(k) == kargs[k])
if self.rdb:
return self.rdb.get(q.hashval, **kargs)
return self.ldb.get(q)
def close(self):
self.ldb.close()
def _get_quote_intraday_by_date(self, symbol, date):
db = TinyDB('data/iex_db_%s.json' % symbol)
quote_query = Query()
datestr = date.strftime("%Y%m%d")
quotes = db.search(quote_query.date == datestr)
if quotes and len(quotes) > 0:
print("from db: %s" % len(quotes))
return quotes
url = "https://api.iextrading.com/1.0/stock/%s/chart/date/%s" % (
symbol.lower(), datestr)
print(url)
with urllib.request.urlopen(url) as response:
content = response.read()
# resp, content = self.client.request(url, "GET")
# print(content)
data = json.loads(content.decode('utf-8'))
quotes = data
# print(quotes)
print(len(quotes))
if len(quotes) > 0:
current_datetime = datetime.now()
if date.date() < current_datetime.date(
) or current_datetime > current_datetime.replace(
hour=16, minute=0, second=0, microsecond=0):
print("storing quotes:")
db.insert_multiple(quotes)
return quotes
def create_subset_db(file_index_fn,
subset_file_index_fn,
processed=True,
extension=".hdf5"):
""" From a main DB, create a subset DB of the main DB, by only extracting
the hdf5 files. If 'processed' input argument is indicated (as True
or False), only the processed or non processed hdf5 files will be
added to the freshly created DB"""
if os.path.exists(subset_file_index_fn):
os.remove(subset_file_index_fn)
directory = os.path.dirname(file_index_fn) + "/"
subset_file_index_fn = directory + subset_file_index_fn
file_index_db = TinyDB(file_index_fn,
storage=CachingMiddleware(JSONStorage))
subset_file_index_db = TinyDB(subset_file_index_fn,
storage=CachingMiddleware(JSONStorage))
subset_file_index_db.purge()
files = Query()
if processed is True or processed is False:
query_cmd = ((files.extension == extension) &
(files.processed == processed))
else:
query_cmd = (files.extension == extension)
records = file_index_db.search(query_cmd)
subset_file_index_db.insert_multiple(records)
file_index_db.close()
return subset_file_index_db
def create_tiny_db():
import json
from pprint import pprint
from dnf_game.util.tinydb import InMemoryBZ2PStorage
"""
source = _type['source']
"""
_type = OBJECTS["bestiary"]
dest = _type['dest']
old_data = packer.unpack_json(dest)
# obj = {value for value in old_data.values()}
# pprint(obj, indent=4)
# exit()
bz2p = dest.replace(".bzp", ".bz2p")
# packer.pack_data(obj, bz2p)
from tinydb import TinyDB, Query
db = TinyDB(bz2p, storage=InMemoryBZ2PStorage)
db.insert_multiple([value for value in old_data.values()])
print(db._storage.memory)
db._storage.write_to_disk()
def mock_db(mocker, mock_cfg):
mocker.patch('passpie.cli.ensure_is_database')
credentials = [
{
'login': '******',
'name': 'bar',
'fullname': 'foo@bar',
'password': '',
'comment': ''
},
{
'login': '******',
'name': 'bazzy',
'fullname': 'foa@bazzy',
'password': '',
'comment': ''
},
{
'login': '******',
'name': 'egg',
'fullname': 'spam@egg',
'password': '',
'comment': ''
},
]
database = TinyDB(storage=MemoryStorage)
database.insert_multiple(credentials)
MockDB = mock.MagicMock(return_value=database)
mocker.patch('passpie.cli.Database', MockDB)
return database
def db_smartcache():
db_ = TinyDB(storage=MemoryStorage)
db_.purge_tables()
db_.table_class = SmartCacheTable
db_ = db_.table('_default')
db_.insert_multiple({'int': 1, 'char': c} for c in 'abc')
return db_
def saveCookie2File():
cookies = driver.get_cookies()
try:
db = TinyDB(cookieFile)
db.purge_tables()
db.insert_multiple(cookies)
db.close()
except Exception as err:
print(f'Error in save cookie File: {err}')
def db_smartcache():
db_ = TinyDB(storage=MemoryStorage)
db_.purge_tables()
db_.table_class = SmartCacheTable
db_ = db_.table('_default')
db_.insert_multiple({'int': 1, 'char': c} for c in 'abc')
return db_
def saveCookie2File():
cookies = driver.get_cookies()
logger.info('cookies:' + str(cookies))
try:
db = TinyDB(cookieFile)
db.purge_tables()
db.insert_multiple(cookies)
db.close()
except Exception as err:
logger.error(f'Error in save cookie File: {err}')
def get_db(smart_cache=False):
db_ = TinyDB(storage=MemoryStorage)
db_.purge_tables()
if smart_cache:
db_.table_class = SmartCacheTable
db_ = db_.table('_default')
db_.insert_multiple({'int': 1, 'char': c} for c in 'abc')
return db_
def db(request):
with tempfile.TemporaryDirectory() as tmpdir:
if request.param == 'json':
db_ = TinyDB(os.path.join(tmpdir, 'test.db'), storage=JSONStorage)
else:
db_ = TinyDB(storage=MemoryStorage)
db_.drop_tables()
db_.insert_multiple({'int': 1, 'char': c} for c in 'abc')
yield db_
def multi_db():
db = TinyDB(storage=MemoryStorage).table()
pairs = [
[1, 2],
[2, 3],
[2, 4],
[3, 4],
[3, 5],
]
shuffle(pairs)
db.insert_multiple([{'k': k, 't': t} for k, t in pairs])
return db
class UsersDAO:
def __init__(self, dbpath):
self.dbpath = dbpath
self.db = TinyDB(os.path.join(dbpath, 'users.json'))
if len(self.db) == 0:
self.db.insert_multiple([
{
'username': '******',
'password': '******',
'currsym': '₹'
},
{
'username': '******',
'password': '******',
'currsym': '$'
},
])
self.expdao_cache = {}
self.portdao_cache = {}
def get(self, username):
UserQ = Query()
user = self.db.get(UserQ.username == username)
if user:
del user['password']
return user
def verify(self, form):
UserQ = Query()
user = self.db.get(UserQ.username == form['username'])
if user and user['password'] == form['password']:
return self.get(user['username'])
return False
def get_expdao(self, user):
username = user['username']
if username in self.expdao_cache:
return self.expdao_cache[username]
dbfile = '%s.json' % username
dao = ExpensesDAO(os.path.join(self.dbpath, 'expenses', dbfile))
self.expdao_cache[username] = dao
return dao
def get_portdao(self, user):
username = user['username']
if username in self.portdao_cache:
return self.portdao_cache[username]
dbfile = '%s.json' % username
dao = PortfolioDAO(os.path.join(self.dbpath, 'portfolios', dbfile))
self.portdao_cache[username] = dao
return dao
class PastebinDBService(object):
def __init__(self):
self.db = TinyDB('db.json')
def create(self, pastebin_data_list):
self.db.insert(pastebin_data_list)
def add_bulk(self, pastebin_data_list):
self.db.insert_multiple(pastebin_data_list)
def get_existing_ids(self, ids: List[str]):
paste = Query()
search = self.db.search(paste.id.one_of(ids))
return search
def mock_db(mocker):
credentials = [
{'login': '******', 'name': 'bar', 'fullname': 'foo@bar',
'password': '', 'comment': ''},
{'login': '******', 'name': 'bazzy', 'fullname': 'foa@bazzy',
'password': '', 'comment': ''},
{'login': '******', 'name': 'egg', 'fullname': 'spam@egg',
'password': '', 'comment': ''},
]
database = TinyDB(storage=MemoryStorage)
database.insert_multiple(credentials)
MockDB = mock.MagicMock(return_value=database)
mocker.patch('passpie.cli.Database', MockDB)
return database
def test_query_cache():
db = TinyDB(storage=MemoryStorage)
db.insert_multiple([
{'name': 'foo', 'value': 42},
{'name': 'bar', 'value': -1337}
])
query = where('value') > 0
results = db.search(query)
assert len(results) == 1
# Now, modify the result ist
results.extend([1])
assert db.search(query) == [{'name': 'foo', 'value': 42}]
class TinyDBIndexer(Executor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.db = TinyDB('db.json')
@requests(on='/index')
def index(self, docs: DocumentArray, **kwargs):
self.db.insert_multiple(docs.get_attributes('tags'))
@requests(on='/search')
def search(self, docs: DocumentArray, **kwargs):
for doc in docs:
r = self.db.search(where(doc.tags__key) == doc.tags__value)
if len(r) > 0:
doc.matches = [Document(tags=r[0])]
def populate_db(count, dbname):
"""Create batch IAM profile v2 objects and insert them in the database."""
path = os.path.dirname(os.path.abspath(__file__))
if not dbname.endswith('.json'):
dbname = '{0}.json'.format(dbname)
click.echo('Creating database {0}'.format(dbname))
db = TinyDB(os.path.join(path, dbname))
users = V2ProfileFactory().create_batch(count, export_json=False)
db.insert_multiple(users)
click.echo('Added {0} profiles in database {1}.'.format(count, dbname))
def make_dep_json(self):
"""
Get the dependencies of all packages installed and cache the result in a json.
:return: a tinydb database
"""
deptree = subprocess.getoutput(
'pipdeptree -j'
) # run pipdeptree (python module) on the terminal: outputs json
pack_json = json.loads(deptree) # load json to python environment
pack_db = TinyDB("pack_db.json")
pack_db.purge(
) # the method clears the database on every call, avoiding rewrites of packages (duplicates)
pack_db.insert_multiple(pack_json)
return pack_db
def recreate_db():
# Base
import uuid
# Flask
from flask import Flask, jsonify, request
from flask_cors import CORS
# DB
from tinydb import TinyDB, Query
db = TinyDB('./database.json')
db.purge()
db.insert_multiple([{
'id': uuid.uuid4().hex,
'name': 'Jack Kerouac',
'email': '*****@*****.**',
'address': 'Park Drive,Boston, MA',
'lat': 42.341590,
'long': -71.097740,
'request': 'Lack of diapers in my area, can anyone help?',
'requestType': 'shopping',
'needHelp': False,
'canHelp': False,
}, {
'id': uuid.uuid4().hex,
'name': 'J. K. Rowling',
'email': '*****@*****.**',
'address': '23 Aldie St., Allston, MA 02134',
'lat': 42.358960,
'long': -71.135920,
'request': 'Can someone watch my kid from 2-3pm tomorrow?',
'requestType': 'inHouseHelp',
'needHelp': False,
'canHelp': False
}, {
'id': uuid.uuid4().hex,
'name': 'Ben B',
'email': '*****@*****.**',
'address': '1000 Commonwealth Ave., Boston, MA 02135',
'lat': 42.349420,
'long': -71.132920,
'request':
'I need a baby sitter this Friday (3/20/2020) from 1-2pm/ Is anyone available?',
'requestType': 'inHouseHelp',
'needHelp': False,
'canHelp': False
}])
class TinyDBIndexer(Executor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.db = TinyDB('db.json')
@requests(on='/index')
def index(self, docs: DocumentArray, **kwargs):
self.db.insert_multiple(docs[:, 'tags'])
@requests(on='/search')
def search(self, docs: DocumentArray, **kwargs):
for doc in docs:
r = self.db.search(where(doc.tags['key']) == doc.tags['value'])
if len(r) > 0:
doc.matches = DocumentArray()
doc.matches.append(Document(tags=r[0]))
def time_tinyDBmem():
try:
del tDB
except:
pass
T0 = time.time()
tDB = TinyDB(storage=MemoryStorage)
tDB.insert_multiple(data)
TC = time.time()-T0
T0 = time.time()
tDB.search(where('iri')==30)
TQ = time.time() - T0
sys.stdout.write('.');sys.stdout.flush()
return TC,TQ
def get_labels_to_mask(self):
"""
Returns a set of labels that represent slider object in the scene
:return: TinyDB database
"""
# write sql query
label_db = self._sim_diagram.get_label_db()
# return all objects that have matching model_instance_id in the list
Label = Query()
labels_to_mask = label_db.search(
Label.model_instance_id.one_of(int(self._slider_model_id)))
# create new TinyDB with just those
mask_db = TinyDB(storage=MemoryStorage)
mask_db.insert_multiple(labels_to_mask)
return mask_db
def write_db(file_list):
'''sanitize and write file_list to tinydb database
values that are not in (str, int, list, float or type(None)) are converted to str representations
Args:
file_list(`list` of `dict`)
Returns:
list of records'''
logging.debug(f'writing DB to {constants.DATABASE_PATH}')
## FIXME -- purge entries older than N weeks
try:
db = TinyDB(constants.DATABASE_PATH)
except (OSError) as e:
logging.error(f'failed to write database {constants.DATABASE_PATH}: {e}')
return False
cleaned = []
for file in file_list:
temp_item = {}
for key, value in file.items():
if isinstance(value, (str, int, list, float, type(None))):
temp_item.update({key: value})
else:
temp_item.update({key: str(value)})
cleaned.append(temp_item)
doc_ids = db.insert_multiple(cleaned)
return doc_ids
def load(release):
if release not in RELEASES_URLS:
raise ValueError('{} not in in {}'.format(release,
list(RELEASES_URLS.keys())))
db = TinyDB(storage=MemoryStorage)
for dataset_name in get_release_datasets_dir(release):
for filepath in get_dataset_files(release, dataset_name):
entries_dicts = read_webnlg_file(dataset_name, filepath)
db.insert_multiple(entries_dicts)
return WebNLGCorpus(release, db)
def insert_ratings_db(stock_symbol, data_dict):
stock_firstLetter = stock_symbol[0]
if stock_firstLetter.isalpha():
stock_firstLetter = stock_firstLetter.lower()
else:
stock_firstLetter = ''
dbFilePath = str(dbPath) + '/db/ratingsDB_' + stock_firstLetter + '.json'
print('dbFilePath: ', dbFilePath)
ratings_db = TinyDB(dbFilePath)
print("record to be inserted: ", data_dict)
print("first remove... ")
ratings_db.update(delete('stockSymbol'),
where('stockSymbol') == stock_symbol)
print("... then insert ")
ratings_db.insert_multiple(data_dict)
return
def get_labels_to_mask(self):
"""
Returns a set of labels that represent objects of interest in the scene
Usually this is the object being pushed
:return: TinyDB database
"""
# write sql query
label_db = self.get_label_db()
# return all objects that have matching model_name in the list
Label = Query()
labels_to_mask = label_db.search(Label.model_name.one_of(self._model_names_to_mask))
# create new TinyDB with just those
mask_db = TinyDB(storage=MemoryStorage)
mask_db.insert_multiple(labels_to_mask)
return mask_db
def test_query_cache():
db = TinyDB(storage=MemoryStorage)
db.insert_multiple([
{'name': 'foo', 'value': 42},
{'name': 'bar', 'value': -1337}
])
query = where('value') > 0
results = db.search(query)
assert len(results) == 1
# Modify the db instance to not return any results when
# bypassing the query cache
db._table_cache[TinyDB.DEFAULT_TABLE]._read = lambda: {}
# Make sure we got an independent copy of the result list
results.extend([1])
assert db.search(query) == [{'name': 'foo', 'value': 42}]
def write_to_cache_file(
data,
cache_path,
type_id=0,
region_id=0,
logger=logging.getLogger(PROGNAME)
):
"""save data to tinyDB
Args:
data (:obj:`pandas.DataFrame`): data to write out
cache_path (str): path to cache file
type_id (int, optional): EVE Online type_id
region_id (int, optional): EVE Online region_id
logger (:obj:`logging.logger`): logging handle
Returns:
None
"""
## get DB ##
logger.info('Writing data to cache')
tdb = TinyDB(cache_path)
date_min = data['date'].min()
logger.debug(date_min)
## clean out existing entries ##
if (type_id and region_id):
logger.info('--Removing old cache entries')
tdb.remove(
(Query().region_id == region_id) &
(Query().type_id == type_id) &
(Query().date >= date_min)
)
caching_data_str = data.to_json(
date_format='iso',
orient='records'
)
cache_data = json.loads(caching_data_str)
logger.info('--Writing to cache file')
tdb.insert_multiple(cache_data)
def database():
"""Initialise a TinyDB database at ./db.json and extends article model
to include read_count and popularity."""
global db
if os.path.isfile("db.json"):
db = TinyDB("db.json")
return None
req = requests.get("https://s3.eu-central-1.amazonaws.com/mm-dmu/arbetsprov/article_data.json")
articles = req.json()["articles"]
for article in articles:
article["read_count"] = 0.0
article["popularity"] = 0.0
db = TinyDB("db.json")
db.insert_multiple(articles)
update_all_articles_popularity()
def test_cutom_mapping_type_with_json(tmpdir):
from tinydb.database import Mapping
class CustomDocument(Mapping):
def __init__(self, data):
self.data = data
def __getitem__(self, key):
return self.data[key]
def __iter__(self):
return iter(self.data)
def __len__(self):
return len(self.data)
# Insert
db = TinyDB(str(tmpdir.join('test.db')))
db.purge()
db.insert(CustomDocument({'int': 1, 'char': 'a'}))
assert db.count(where('int') == 1) == 1
# Insert multiple
db.insert_multiple([
CustomDocument({'int': 2, 'char': 'a'}),
CustomDocument({'int': 3, 'char': 'a'})
])
assert db.count(where('int') == 1) == 1
assert db.count(where('int') == 2) == 1
assert db.count(where('int') == 3) == 1
# Write back
doc_id = db.get(where('int') == 3).doc_id
db.write_back([CustomDocument({'int': 4, 'char': 'a'})], [doc_id])
assert db.count(where('int') == 3) == 0
assert db.count(where('int') == 4) == 1
def db():
db = TinyDB(storage=MemoryStorage)
db.insert_multiple({'int': 1, 'char': c} for c in 'abc')
return db
def multi_db():
db = TinyDB(storage=MemoryStorage).table()
pairs = [[1, 2], [2, 3], [2, 4], [3, 4], [3, 5]]
shuffle(pairs)
db.insert_multiple([{"k": k, "t": t} for k, t in pairs])
return db
class Database(unittest.TestCase):
DB_NAME = 'test_database.json'
def setUp(self):
self.db = TinyDB(self.DB_NAME)
self.db.insert_multiple([
{"skills": 'python', "github": 'https://github.com/kennethreitz', "time": '0', "points": '',
"email": '*****@*****.**', "username": '******', 'slack_id': 'U123456789'},
{"skills": 'Duchess', "github": 'https://bitbucket.org/madhatter', "time": '5', "points": '1871',
"email": '*****@*****.**', "username": '******', 'slack_id': 'U18651871'}
])
self.User = Query()
database.db = self.db
database.User = self.User
def tearDown(self):
self.db.close()
import os
os.remove(self.DB_NAME)
def test_insert_user(self):
self.assertDictEqual(
json.loads(database.insert_user(email='*****@*****.**', skills='magic, rabbits'))['response'],
dict(skills='magic, rabbits', github='', time='', email='*****@*****.**', username='', slack_id='', points='0')
)
def test_update_user(self):
self.assertDictEqual(
json.loads(database.update_user(params={'skills': 'requests'}, slack_id='U123456789'))['response']['value'],
{"skills": 'requests', "github": 'https://github.com/kennethreitz', "time": '0', "points": '',
"email": '*****@*****.**', "username": '******', 'slack_id': 'U123456789'}
)
self.assertDictEqual(
json.loads(database.update_user(params={'username': '******'}, slack_id='U123456789'))['response']['value'],
{"skills": 'requests', "github": 'https://github.com/kennethreitz', "time": '0', "points": '',
"email": '*****@*****.**', "username": '******', 'slack_id': 'U123456789'}
)
self.assertEqual(
json.loads(database.update_user(params={'skills': 'django'}, slack_id='nonExisting'))['response']['value'],
None
)
def test_get_user(self):
self.assertDictEqual(
json.loads(database.get_user(slack_id='U18651871'))['response'],
{"skills": 'Duchess', "github": 'https://bitbucket.org/madhatter', "time": '5', "points": '1871',
"email": '*****@*****.**', "username": '******', 'slack_id': 'U18651871'}
)
self.assertDictEqual(
json.loads(database.get_user(slack_id='U123456789', email='*****@*****.**'))['response'],
{"skills": 'python', "github": 'https://github.com/kennethreitz', "time": '0', "points": '',
"email": '*****@*****.**', "username": '******', 'slack_id': 'U123456789'}
)
self.assertDictEqual(
json.loads(database.get_user(email='*****@*****.**'))['response'],
{'Error': 'User not found'}
)
def test_get_all_users(self):
self.assertListEqual(
json.loads(database.get_all_users())['response'],
[
{"skills": 'python', "github": 'https://github.com/kennethreitz', "time": '0', "points": '',
"email": '*****@*****.**', "username": '******', 'slack_id': 'U123456789'},
{"skills": 'Duchess', "github": 'https://bitbucket.org/madhatter', "time": '5', "points": '1871',
"email": '*****@*****.**', "username": '******', 'slack_id': 'U18651871'}
]
)
def test_delete_user(self):
self.assertEqual(
json.loads(database.delete_user(slack_id='U123456789', email='*****@*****.**', username='******'))['response']['id'],
'U123456789'
)
self.assertEqual(
json.loads(database.delete_user(email='*****@*****.**', username='******'))['response']['id'],
'cheshirecat'
)
reviews_db = TinyDB('data/reviews.json')
# Load key from yaml file
with open('key.yaml', 'r') as f:
key = yaml.load(f.read())['google-key']
# load language parameter
with open('config.yaml', 'r') as f:
language = yaml.load(f.read())['language']
# template url used to send requests
url_template = ('https://maps.googleapis.com/maps/api/place/details/json?'
'key={key}&placeid={placeid}&language={language}')
# load details for every place
for i in xrange(1, len(places_db)):
# build dic with params to send to the API
params = {'key': key,
'placeid': places_db.get(eid=i)['place_id'],
'language': language}
# send request
res = requests.get(url_template.format(**params))
# obtain reviews if any
try:
reviews = json.loads(res.content)['result']['reviews']
except Exception, e:
print 'Place does not have reviews, skipping...'
else:
# insert reviews in json file
reviews_db.insert_multiple(reviews)
from tinydb import TinyDB
# Load key from yaml file
with open('key.yaml', 'r') as f:
key = yaml.load(f.read())['google-key']
# load configuration parameters
with open('config.yaml', 'r') as f:
config = yaml.load(f.read())
# Google maps API template
url_template = ('https://maps.googleapis.com/maps/api/place/radarsearch/json'
'?key={key}&location={location}&radius={radius}&types={types}')
db = TinyDB('data/places.json')
# generate a request for every type of place
for a_type in config['types'].split('|'):
# Parameters to be included in the request
params = {'location': config['location'],
'key': key,
'radius': config['radius'],
'types': a_type}
# Make the request
res = requests.get(url_template.format(**params))
places = json.loads(res.content)['results']
# Save results in a json file
db.insert_multiple(places)
def db():
db = TinyDB(storage=MemoryStorage).table()
db.insert_multiple([{"k": i} for i in [4, 2, 1, 3, 5]])
return db
def db():
db_ = TinyDB(storage=MemoryStorage)
db_.purge_tables()
db_.insert_multiple({'int': 1, 'char': c} for c in 'abc')
return db_
class Database(object): #pylint: disable=R0902
"""
Structured thermodynamic and/or kinetic data.
Attributes
----------
elements : list
List of elements in database.
species : list
List of species in database.
phases : dict
Phase objects indexed by their system-local name.
symbols : dict
SymPy objects indexed by their name (FUNCTIONs in Thermo-Calc).
references : dict
Reference objects indexed by their system-local identifier.
Examples
--------
>>> mydb = Database(open('crfeni_mie.tdb'))
>>> mydb = Database('crfeni_mie.tdb')
>>> f = StringIO(u'$a complete TDB file as a string\n')
>>> mydb = Database(f)
"""
def __new__(cls, *args):
if len(args) == 0:
obj = super(Database, cls).__new__(cls, *args)
# Should elements be rolled into a special case of species?
obj.elements = set()
obj.species = set()
obj.phases = {}
obj._structure_dict = {} # System-local phase names to global IDs
obj._parameters = TinyDB(storage=MemoryStorage)
obj.symbols = {}
obj.references = {}
# Note: No public typedefs here (from TDB files)
# Instead we put that information in the model_hint for phases
return obj
elif len(args) == 1:
fname = args[0]
# Backwards compatibility: assume TDB by default
fmt = 'tdb'
# Attempt to auto-detect the correct format based on the file extension
try:
path, ext = os.path.splitext(fname)
if '.' in ext and ext[1:].lower() in format_registry:
fmt = ext[1:].lower()
except AttributeError:
pass
if hasattr(fname, 'read'):
# File descriptor
return cls.from_file(fname, fmt=fmt)
elif fname.find('\n') == -1:
# Single-line string; it's probably a filename
return cls.from_file(fname, fmt=fmt)
else:
# Newlines found: probably a full database string
return cls.from_string(fname, fmt=fmt)
else:
raise ValueError('Invalid number of parameters: '+len(args))
def __getstate__(self):
pickle_dict = {}
for key, value in self.__dict__.items():
if key == '_parameters':
pickle_dict[key] = value.all()
else:
pickle_dict[key] = value
return pickle_dict
def __setstate__(self, state):
for key, value in state.items():
if key == '_parameters':
self._parameters = TinyDB(storage=MemoryStorage)
self._parameters.insert_multiple(value)
else:
setattr(self, key, value)
@staticmethod
def register_format(fmt, read=None, write=None):
"""
Add support for reading and/or writing the specified format.
Parameters
----------
fmt: str
Format.
read : callable, optional
Read function with arguments (Database, file_descriptor)
write : callable, optional
Write function with arguments (Database, file_descriptor)
Examples
--------
None yet.
"""
format_registry[fmt.lower()] = DatabaseFormat(read=read, write=write)
@staticmethod
def from_file(fname, fmt=None):
"""
Create a Database from a file.
Parameters
----------
fname: str or file-like
File name/descriptor to read.
fmt : str, optional
File format. If not specified, an attempt at auto-detection is made.
Returns
-------
dbf : Database
Database from file.
Examples
--------
None yet.
"""
if fmt is None:
# Attempt to auto-detect the correct format based on the file extension
try:
path, ext = os.path.splitext(fname)
except AttributeError:
# fname isn't actually a path, so we don't know the correct format
raise ValueError('\'fmt\' keyword argument must be specified when passing a file descriptor.')
if '.' in ext and ext[1:].lower() in format_registry:
fmt = ext[1:].lower()
else:
fmt = fmt.lower()
if fmt not in format_registry or format_registry[fmt].read is None:
supported_reads = [key for key, value in format_registry.items() if value.read is not None]
raise NotImplementedError('Unsupported read format \'{0}\'. Supported formats: {1}'.format(fmt,
supported_reads))
# Is it a file descriptor?
if hasattr(fname, 'read'):
fd = fname
need_to_close = False
else:
# It's not file-like, so it's probably a filename
need_to_close = True
fd = open(fname, mode='r')
try:
dbf = Database()
format_registry[fmt.lower()].read(dbf, fd)
finally:
# Close file descriptors created in this routine
# Otherwise that's left up to the calling function
if need_to_close:
fd.close()
return dbf
@classmethod
def from_string(cls, data, **kwargs):
"""
Returns Database from a string in the specified format.
This function is a wrapper for calling `from_file` with StringIO.
Parameters
----------
data : str
Raw database string in the specified format.
kwargs : optional
See keyword arguments for `from_file`.
Returns
-------
dbf : Database
"""
return cls.from_file(StringIO(data), **kwargs)
def to_file(self, fname, fmt=None, if_exists='raise', **write_kwargs):
"""
Write the Database to a file.
Parameters
----------
fname: str or file-like
File name/descriptor to write.
fmt : str, optional
File format. If not specified, an attempt at auto-detection is made.
if_exists : string, optional ['raise', 'rename', 'overwrite']
Strategy if 'fname' already exists.
The 'raise' option (default) will raise a FileExistsError.
The 'rename' option will append the date/time to the filename.
The 'overwrite' option will overwrite the file.
This argument is ignored if 'fname' is file-like.
write_kwargs : optional
Keyword arguments to pass to write function.
Examples
--------
None yet.
"""
if fmt is None:
# Attempt to auto-detect the correct format based on the file extension
try:
path, ext = os.path.splitext(fname)
except AttributeError:
# fname isn't actually a path, so we don't know the correct format
raise ValueError('\'fmt\' keyword argument must be specified when passing a file descriptor.')
if '.' in ext and ext[1:].lower() in format_registry:
fmt = ext[1:].lower()
else:
fmt = fmt.lower()
if fmt not in format_registry or format_registry[fmt].write is None:
supported_writes = [key for key, value in format_registry.items() if value.write is not None]
raise NotImplementedError('Unsupported write format \'{0}\'. Supported formats: {1}'.format(fmt,
supported_writes))
# Is this a file descriptor?
if hasattr(fname, 'write'):
format_registry[fmt].write(self, fname, **write_kwargs)
else:
if os.path.exists(fname) and if_exists != 'overwrite':
if if_exists == 'raise':
raise FileExistsError('File {} already exists'.format(fname))
elif if_exists == 'rename':
writetime = datetime.now()
fname = os.path.splitext(fname)
fname = fname[0] + "." + writetime.strftime("%Y-%m-%d-%H-%M") + fname[1]
with open(fname, mode='w') as fd:
format_registry[fmt].write(self, fd, **write_kwargs)
def to_string(self, **kwargs):
"""
Returns Database as a string.
This function is a wrapper for calling `to_file` with StringIO.
Parameters
----------
kwargs : optional
See keyword arguments for `to_file`.
Returns
-------
result : str
"""
result = StringIO()
self.to_file(result, **kwargs)
return result.getvalue()
def __str__(self):
result = 'Elements: {0}\n'.format(sorted(self.elements))
result += 'Species: {0}\n'.format(sorted(self.species))
for name, phase in sorted(self.phases.items()):
result += str(phase)+'\n'
result += '{0} symbols in database\n'.format(len(self.symbols))
result += '{0} parameters in database\n'.format(len(self._parameters))
return result
def __eq__(self, other):
if self is other:
return True
elif type(self) != type(other):
return False
elif sorted(self.__dict__.keys()) != sorted(other.__dict__.keys()):
return False
else:
def param_sort_key(x):
return x['phase_name'], x['parameter_type'], x['constituent_array'], \
x['parameter_order'], x['diffusing_species']
for key in self.__dict__.keys():
if key == '_parameters':
# Special handling for TinyDB objects
if len(self._parameters.all()) != len(other._parameters.all()):
return False
self_params = sorted(self._parameters.all(), key=param_sort_key)
other_params = sorted(other._parameters.all(), key=param_sort_key)
if self_params != other_params:
return False
elif self.__dict__[key] != other.__dict__[key]:
return False
return True
def __ne__(self, other):
return not self.__eq__(other)
def add_structure_entry(self, local_name, global_name):
"""
Define a relation between the system-local name of a phase and a
"global" identifier. This is used to link crystallographically
similar phases known by different colloquial names.
Parameters
----------
local_name : string
System-local name of the phase.
global_name : object
Abstract representation of symbol, e.g., in SymPy format.
Examples
--------
None yet.
"""
self._structure_dict[local_name] = global_name
def add_parameter(self, param_type, phase_name, #pylint: disable=R0913
constituent_array, param_order,
param, ref=None, diffusing_species=None):
"""
Add a parameter.
Parameters
----------
param_type : str
Type name of the parameter, e.g., G, L, BMAGN.
phase_name : str
Name of the phase.
constituent_array : list
Configuration of the sublattices (elements and/or species).
param_order : int
Polynomial order of the parameter.
param : object
Abstract representation of the parameter, e.g., in SymPy format.
ref : str, optional
Reference for the parameter.
diffusing_species : str, optional
(If kinetic parameter) Diffusing species for this parameter.
Examples
--------
None yet.
"""
new_parameter = {
'phase_name': phase_name,
'constituent_array': _to_tuple(constituent_array), # must be hashable type
'parameter_type': param_type,
'parameter_order': param_order,
'parameter': param,
'diffusing_species': diffusing_species,
'reference': ref
}
param_id = self._parameters.insert(new_parameter)
return param_id
def add_phase(self, phase_name, model_hints, sublattices):
"""
Add a phase.
Parameters
----------
phase_name : string
System-local name of the phase.
model_hints : list
Structured "hints" for a Model trying to read this phase.
Hints for major constituents and typedefs (Thermo-Calc) go here.
sublattices : list
Site ratios of sublattices.
Examples
--------
None yet.
"""
new_phase = Phase()
new_phase.name = phase_name
# Need to convert from ParseResults or else equality testing will break
new_phase.sublattices = tuple(sublattices)
new_phase.model_hints = model_hints
self.phases[phase_name] = new_phase
def add_phase_constituents(self, phase_name, constituents):
"""
Add a phase.
Parameters
----------
phase_name : string
System-local name of the phase.
constituents : list
Possible phase constituents (elements and/or species).
Examples
--------
None yet.
"""
try:
# Need to convert constituents from ParseResults
# Otherwise equality testing will be broken
self.phases[phase_name].constituents = tuple(map(frozenset, constituents))
except KeyError:
print("Undefined phase "+phase_name)
raise
def search(self, query):
"""
Search for parameters matching the specified query.
Parameters
----------
query : object
Structured database query in TinyDB format.
Examples
--------
>>>> from tinydb import where
>>>> db = Database()
>>>> eid = db.add_parameter(...) #TODO
>>>> db.search(where('eid') == eid)
"""
return self._parameters.search(query)
