def __init__(self, _settings, _app, *args, **kwargs):
"""perform the initial load"""
super().__init__(*args, **kwargs)
# Bring Dynaconf instance value to Flask Config
Config.update(self, _settings.store)
self._settings = _settings
self._app = _app
def __init__(self, _settings, *args, **kwargs):
"""perform the initial load"""
super(DynaconfConfig, self).__init__(*args, **kwargs)
Config.update(self, _settings.store)
self._settings = _settings
class DatacatCore(object):
def __init__(self, config=None):
self.config = Config()
if config is not None:
self.config.update(config)
@property
def db(self):
if getattr(self, '_db', None) is None:
self._db = connect(**self.config['DATABASE'])
self._db.autocommit = False
return self._db
@property
def admin_db(self):
if getattr(self, '_admin_db', None) is None:
self._admin_db = connect(**self.config['DATABASE'])
self._admin_db.autocommit = True
return self._admin_db
def create_tables(self):
create_tables(self.admin_db)
def drop_tables(self):
drop_tables(self.admin_db)
# ------------------------------------------------------------
# Resource data CRUD
# ------------------------------------------------------------
def resource_data_iter(self):
"""Iterate over resource attributes"""
with self.db, self.db.cursor() as cur:
cur.execute("SELECT * FROM resource_data")
for row in cur.fetchall():
yield row
def resource_data_create(self, stream, metadata=None, mimetype=None):
"""Create resource data from a stream"""
resource_hash = hashlib.sha1()
with self.db, self.db.cursor() as cur:
lobj = self.db.lobject(oid=0, mode='wb')
oid = lobj.oid
for chunk in file_read_chunks(stream):
lobj.write(chunk)
resource_hash.update(chunk)
lobj.close()
data = {
'ctime': datetime.now(),
'mtime': datetime.now(),
'metadata': json.dumps(metadata),
'mimetype': mimetype or 'application/octet-stream',
'data_oid': oid,
'hash': 'sha1:{0}'.format(resource_hash.hexdigest()),
}
query = querybuilder.insert('resource_data', data)
cur.execute(query, data)
resource_data_id = cur.fetchone()[0]
return resource_data_id
def resource_data_get_info(self, objid):
query = querybuilder.select_pk('resource_data')
with self.db, self.db.cursor() as cur:
cur.execute(query, {'id': objid})
return cur.fetchone()
def resource_data_read(self, objid):
resource = self.resource_data_get_info(objid)
with self.db:
lobj = self.db.lobject(oid=resource['data_oid'], mode='rb')
return lobj.read()
def resource_data_copy(self, objid, dest):
resource = self.resource_data_get_info(objid)
with self.db:
lobj = self.db.lobject(oid=resource['data_oid'], mode='rb')
for chunk in file_read_chunks(lobj):
dest.write(chunk)
def resource_data_update(self, objid, stream=None, metadata=None,
mimetype=None):
# Get the original object, to check for its existence
# and to get the oid of the lobject holding the data.
original = self.resource_data_get_info(objid)
data = {
'id': objid,
'mtime': datetime.now(),
}
if metadata is not None:
data['metadata'] = json.dumps(metadata)
if mimetype is not None:
data['mimetype'] = mimetype
with self.db, self.db.cursor() as cur:
if stream is not None:
# Update the lobject with data from the stream
resource_hash = hashlib.sha1()
lobj = self.db.lobject(oid=original['data_oid'], mode='wb')
for chunk in file_read_chunks(stream):
lobj.write(chunk)
resource_hash.update(chunk)
lobj.close()
data['hash'] = 'sha1:{0}'.format(resource_hash.hexdigest())
query = querybuilder.update('resource_data', data)
cur.execute(query, data)
def resource_data_remove(self, objid):
# We need the OID to remove the lobject
original = self.resource_data_get_info(objid)
with self.db, self.db.cursor() as cur:
lobject = db.lobject(oid=resource['data_oid'], mode='rb')
data = lobject.read()
lobject.close()
cur.execute(querybuilder.delete('resource_data'), {'id': objid})
# ------------------------------------------------------------
# Dataset / resource CRUD
# ------------------------------------------------------------
def create_resource(self, resource):
return self._dsres_create('resource', resource)
def update_resource(self, resource_id, resource):
return self._dsres_update('resource', resource_id, resource)
def get_resource(self, resource_id):
return self._dsres_get('resource', resource_id)
def list_resources(self, offset=None, limit=None):
return self._dsres_list('resource', offset=offset, limit=limit)
def delete_resource(self, resource_id):
return self._dsres_delete('resource', resource_id)
def create_dataset(self, dataset):
return self._dsres_create('dataset', dataset)
def update_dataset(self, dataset_id, dataset):
return self._dsres_update('dataset', dataset_id, dataset)
def get_dataset(self, dataset_id):
return self._dsres_get('dataset', dataset_id)
def list_datasets(self, offset=None, limit=None):
return self._dsres_list('dataset', offset=offset, limit=limit)
def delete_dataset(self, dataset_id):
return self._dsres_delete('dataset', dataset_id)
def add_dataset_resource(self, dataset_id, resource_id, order=0):
data = {
'dataset_id': dataset_id,
'resource_id': resource_id,
'order': order}
query = querybuilder.insert(
'dataset_resource', data=data, table_key=None)
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
def delete_dataset_resource(self, dataset_id, resource_id):
data = {'dataset_id': dataset_id, 'resource_id': resource_id}
query = ("DELETE FROM dataset_resource"
" WHERE dataset_id=%(dataset_id)s"
" AND resource_id=%(resource_id)s")
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
def move_dataset_resource(self, dataset_id, resource_id, order):
data = {'dataset_id': dataset_id,
'resource_id': resource_id,
'order': order}
query = ("UPDATE dataset_resource"
" SET order=%(order)s"
" WHERE dataset_id=%(dataset_id)s"
" AND resource_id=%(resource_id)s")
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
# ------------------------------------------------------------
# Rows and datasets have the same schema:
# let's use some common functions for them..
# ------------------------------------------------------------
def _dsres_create(self, name, obj):
data = {
'configuration': json.dumps(obj),
'ctime': datetime.now(),
'mtime': datetime.now(),
}
query = querybuilder.insert(name, data)
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
return cur.fetchone()[0]
def _dsres_update(self, name, obj_id, obj):
data = {
'id': obj_id,
'configuration': json.dumps(obj),
'mtime': datetime.now(),
}
query = querybuilder.update(name, data)
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
def _dsres_get(self, name, obj_id):
query = querybuilder.select_pk(name)
with self.db, self.db.cursor() as cur:
cur.execute(query, dict(id=obj_id))
row = cur.fetchone()
if row is None:
raise NotFound()
return self._row_to_obj(row)
def _dsres_list(self, name, offset=None, limit=None):
query = querybuilder.select_paged(
name, offset=offset, limit=limit)
with self.db, self.db.cursor() as cur:
cur.execute(query)
for row in cur.fetchall():
yield self._row_to_obj(row)
def _dsres_from_row(self, row):
obj = row['configuration']
obj['_id'] = row['id']
obj['_ctime'] = row['ctime']
obj['_mtime'] = row['mtime']
return obj
def _dsres_delete(self, name, obj_id):
query = querybuilder.delete(name)
with self.db, self.db.cursor() as cur:
cur.execute(query, dict(id=obj_id))
class DatacatCore(object):
def __init__(self, config=None):
self.config = Config()
if config is not None:
self.config.update(config)
@property
def db(self):
if getattr(self, '_db', None) is None:
self._db = connect(**self.config['DATABASE'])
self._db.autocommit = False
return self._db
@property
def admin_db(self):
if getattr(self, '_admin_db', None) is None:
self._admin_db = connect(**self.config['DATABASE'])
self._admin_db.autocommit = True
return self._admin_db
def create_tables(self):
create_tables(self.admin_db)
def drop_tables(self):
drop_tables(self.admin_db)
# ------------------------------------------------------------
# Resource data CRUD
# ------------------------------------------------------------
def resource_data_iter(self):
"""Iterate over resource attributes"""
with self.db, self.db.cursor() as cur:
cur.execute("SELECT * FROM resource_data")
for row in cur.fetchall():
yield row
def resource_data_create(self, stream, metadata=None, mimetype=None):
"""Create resource data from a stream"""
resource_hash = hashlib.sha1()
with self.db, self.db.cursor() as cur:
lobj = self.db.lobject(oid=0, mode='wb')
oid = lobj.oid
for chunk in file_read_chunks(stream):
lobj.write(chunk)
resource_hash.update(chunk)
lobj.close()
data = {
'ctime': datetime.now(),
'mtime': datetime.now(),
'metadata': json.dumps(metadata),
'mimetype': mimetype or 'application/octet-stream',
'data_oid': oid,
'hash': 'sha1:{0}'.format(resource_hash.hexdigest()),
}
query = querybuilder.insert('resource_data', data)
cur.execute(query, data)
resource_data_id = cur.fetchone()[0]
return resource_data_id
def resource_data_get_info(self, objid):
query = querybuilder.select_pk('resource_data')
with self.db, self.db.cursor() as cur:
cur.execute(query, {'id': objid})
return cur.fetchone()
def resource_data_read(self, objid):
resource = self.resource_data_get_info(objid)
with self.db:
lobj = self.db.lobject(oid=resource['data_oid'], mode='rb')
return lobj.read()
def resource_data_copy(self, objid, dest):
resource = self.resource_data_get_info(objid)
with self.db:
lobj = self.db.lobject(oid=resource['data_oid'], mode='rb')
for chunk in file_read_chunks(lobj):
dest.write(chunk)
def resource_data_update(self,
objid,
stream=None,
metadata=None,
mimetype=None):
# Get the original object, to check for its existence
# and to get the oid of the lobject holding the data.
original = self.resource_data_get_info(objid)
data = {
'id': objid,
'mtime': datetime.now(),
}
if metadata is not None:
data['metadata'] = json.dumps(metadata)
if mimetype is not None:
data['mimetype'] = mimetype
with self.db, self.db.cursor() as cur:
if stream is not None:
# Update the lobject with data from the stream
resource_hash = hashlib.sha1()
lobj = self.db.lobject(oid=original['data_oid'], mode='wb')
for chunk in file_read_chunks(stream):
lobj.write(chunk)
resource_hash.update(chunk)
lobj.close()
data['hash'] = 'sha1:{0}'.format(resource_hash.hexdigest())
query = querybuilder.update('resource_data', data)
cur.execute(query, data)
def resource_data_remove(self, objid):
# We need the OID to remove the lobject
original = self.resource_data_get_info(objid)
with self.db, self.db.cursor() as cur:
lobject = db.lobject(oid=resource['data_oid'], mode='rb')
data = lobject.read()
lobject.close()
cur.execute(querybuilder.delete('resource_data'), {'id': objid})
# ------------------------------------------------------------
# Dataset / resource CRUD
# ------------------------------------------------------------
def create_resource(self, resource):
return self._dsres_create('resource', resource)
def update_resource(self, resource_id, resource):
return self._dsres_update('resource', resource_id, resource)
def get_resource(self, resource_id):
return self._dsres_get('resource', resource_id)
def list_resources(self, offset=None, limit=None):
return self._dsres_list('resource', offset=offset, limit=limit)
def delete_resource(self, resource_id):
return self._dsres_delete('resource', resource_id)
def create_dataset(self, dataset):
return self._dsres_create('dataset', dataset)
def update_dataset(self, dataset_id, dataset):
return self._dsres_update('dataset', dataset_id, dataset)
def get_dataset(self, dataset_id):
return self._dsres_get('dataset', dataset_id)
def list_datasets(self, offset=None, limit=None):
return self._dsres_list('dataset', offset=offset, limit=limit)
def delete_dataset(self, dataset_id):
return self._dsres_delete('dataset', dataset_id)
def add_dataset_resource(self, dataset_id, resource_id, order=0):
data = {
'dataset_id': dataset_id,
'resource_id': resource_id,
'order': order
}
query = querybuilder.insert('dataset_resource',
data=data,
table_key=None)
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
def delete_dataset_resource(self, dataset_id, resource_id):
data = {'dataset_id': dataset_id, 'resource_id': resource_id}
query = ("DELETE FROM dataset_resource"
" WHERE dataset_id=%(dataset_id)s"
" AND resource_id=%(resource_id)s")
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
def move_dataset_resource(self, dataset_id, resource_id, order):
data = {
'dataset_id': dataset_id,
'resource_id': resource_id,
'order': order
}
query = ("UPDATE dataset_resource"
" SET order=%(order)s"
" WHERE dataset_id=%(dataset_id)s"
" AND resource_id=%(resource_id)s")
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
# ------------------------------------------------------------
# Rows and datasets have the same schema:
# let's use some common functions for them..
# ------------------------------------------------------------
def _dsres_create(self, name, obj):
data = {
'configuration': json.dumps(obj),
'ctime': datetime.now(),
'mtime': datetime.now(),
}
query = querybuilder.insert(name, data)
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
return cur.fetchone()[0]
def _dsres_update(self, name, obj_id, obj):
data = {
'id': obj_id,
'configuration': json.dumps(obj),
'mtime': datetime.now(),
}
query = querybuilder.update(name, data)
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
def _dsres_get(self, name, obj_id):
query = querybuilder.select_pk(name)
with self.db, self.db.cursor() as cur:
cur.execute(query, dict(id=obj_id))
row = cur.fetchone()
if row is None:
raise NotFound()
return self._row_to_obj(row)
def _dsres_list(self, name, offset=None, limit=None):
query = querybuilder.select_paged(name, offset=offset, limit=limit)
with self.db, self.db.cursor() as cur:
cur.execute(query)
for row in cur.fetchall():
yield self._row_to_obj(row)
def _dsres_from_row(self, row):
obj = row['configuration']
obj['_id'] = row['id']
obj['_ctime'] = row['ctime']
obj['_mtime'] = row['mtime']
return obj
def _dsres_delete(self, name, obj_id):
query = querybuilder.delete(name)
with self.db, self.db.cursor() as cur:
cur.execute(query, dict(id=obj_id))
def __init__(self, _config, *args, **kwargs):
super(_Config, self).__init__(*args, **kwargs)
Config.update(self, _config)
p, ss_user, pwd_host, port_db = db_url.split(":")
pwd, host = pwd_host.split('@')
port, db = port_db.split("/")
user = ss_user[2:]
config.update(
dict(db_host=host,
db_port=int(port),
db_user=user,
db_password=pwd,
db_name=db))
config = Config(root_path=os.path.abspath('.'))
config.update(
dict(testing=False,
redis_host='localhost',
redis_port=6379,
redis_pwd=None,
redis_db=0,
db_host='localhost',
db_name='honest-ab_development',
db_user='******',
db_password='******',
db_port=5432,
login_manager_secret_key='lakjsdlfkjasdlfkjasd',
batch_size=30,
stats_base_dir=os.path.abspath(os.path.join('.', 'stats'))))
apply_redis(config)
apply_db(config)
