def clear_datastore(self, ds_name=None, prefix=None):
"""
Clears a datastore or a set of datastores of common prefix
"""
ds = DatastoreFactory.get_datastore(config=self.config, scope=self.sysname)
try:
if ds_name:
try:
ds.delete_datastore(ds_name)
except NotFound:
try:
# Try the unscoped version
ds1 = DatastoreFactory.get_datastore(config=self.config)
ds1.delete_datastore(ds_name)
except NotFound:
pass
elif prefix:
prefix = prefix.lower()
ds_noscope = DatastoreFactory.get_datastore(config=self.config)
for dsn in ds_noscope.list_datastores():
if dsn.lower().startswith(prefix):
ds_noscope.delete_datastore(dsn)
else:
log.warn("Cannot clear datastore without prefix or datastore name")
finally:
ds.close()
def clear_datastore(self, ds_name=None, prefix=None):
"""
Clears a datastore or a set of datastores of common prefix
"""
ds = DatastoreFactory.get_datastore(config=self.config,
scope=self.sysname)
try:
if ds_name:
try:
ds.delete_datastore(ds_name)
except NotFound:
try:
# Try the unscoped version
ds1 = DatastoreFactory.get_datastore(
config=self.config)
ds1.delete_datastore(ds_name)
except NotFound:
pass
elif prefix:
prefix = prefix.lower()
ds_noscope = DatastoreFactory.get_datastore(config=self.config)
for dsn in ds_noscope.list_datastores():
if dsn.lower().startswith(prefix):
ds_noscope.delete_datastore(dsn)
else:
log.warn(
"Cannot clear datastore without prefix or datastore name")
finally:
ds.close()
def tearDownClass(cls):
# Removes temporary files
# Comment this out if you need to inspect the HDF5 files.
shutil.rmtree(cls.working_dir)
span_store = DatastoreFactory.get_datastore(datastore_name="coverage_spans", config=CFG)
coverage_store = DatastoreFactory.get_datastore(datastore_name="coverage", config=CFG)
if span_store is None:
raise RuntimeError("Unable to load datastore for coverage_spans")
if coverage_store is None:
raise RuntimeError("Unable to load datastore for coverages")
for guid in cls.coverages:
with span_store.pool.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
cur.execute("DELETE FROM %s WHERE coverage_id='%s'" % (span_store._get_datastore_name(), guid))
cur.execute("DELETE FROM %s WHERE id='%s'" % (coverage_store._get_datastore_name(), guid))
def _clear_couch(config, prefix, verbose=False, sysname=None):
cfg_copy = dict(config)
if "password" in cfg_copy:
cfg_copy["password"] = "******"
print 'clear_couch: Clearing CouchDB databases using config=', cfg_copy
db_server = DatastoreFactory.get_datastore(config=config)
import logging
db_server.log = logging
if verbose:
print "clear_couch: Connected to couch server with config %s" % (config)
db_list = db_server.list_datastores()
ignored_num = 0
for db_name in db_list:
if (prefix == '*' and not db_name.startswith('_')) or db_name.lower().startswith(prefix.lower()):
db_server.delete_datastore(db_name)
print 'clear_couch: Dropped couch database: %s' % db_name
else:
if verbose:
print 'clear_couch: Ignored couch database: %s' % db_name
ignored_num += 1
print 'clear_couch: Ignored %s existing databases' % ignored_num
db_server.close()
def _dump_datastore(self, outpath_base, ds_name, clear_dir=True):
ds = DatastoreFactory.get_datastore(datastore_name=ds_name,
config=self.config,
scope=self.sysname)
try:
if not ds.datastore_exists(ds_name):
log.warn("Datastore does not exist: %s" % ds_name)
return
if not os.path.exists(outpath_base):
os.makedirs(outpath_base)
outpath = "%s/%s" % (outpath_base, ds_name)
if not os.path.exists(outpath):
os.makedirs(outpath)
if clear_dir:
[
os.remove(os.path.join(outpath, f))
for f in os.listdir(outpath)
]
objs = ds.find_docs_by_view("_all_docs", None, id_only=False)
compact_obj = [obj for obj_id, obj_key, obj in objs]
compact_obj = ["COMPACTDUMP", compact_obj]
with open("%s/%s_compact.json" % (outpath, ds_name), 'w') as f:
json.dump(compact_obj, f)
numwrites = len(objs)
log.info("Wrote %s files to %s" % (numwrites, outpath))
finally:
ds.close()
def dump_datastore(self, path=None, ds_name=None, clear_dir=True):
"""
Dumps CouchDB datastores into a directory as YML files.
@param ds_name Logical name (such as "resources") of an ION datastore
@param path Directory to put dumped datastores into (defaults to
"res/preload/local/dump_[timestamp]")
@param clear_dir if True, delete contents of datastore dump dirs
@param compact if True, saves all objects in one big YML file
"""
if not path:
dtstr = datetime.datetime.today().strftime('%Y%m%d_%H%M%S')
path = "res/preload/local/dump_%s" % dtstr
if ds_name:
ds = DatastoreFactory.get_datastore(datastore_name=ds_name,
config=self.config,
scope=self.sysname)
if ds.datastore_exists(ds_name):
self._dump_datastore(path, ds_name, clear_dir)
else:
log.warn("Datastore does not exist")
ds.close()
else:
ds_list = ['resources', 'objects', 'state', 'events']
for dsn in ds_list:
self._dump_datastore(path, dsn, clear_dir)
def __init__(self, sysname=None, orgname=None, config=None):
self.orgname = orgname or get_safe(config, 'system.root_org', 'ION')
sysname = sysname or get_default_sysname()
self.datastore_name = "resources"
self.datastore = DatastoreFactory.get_datastore(datastore_name=self.datastore_name, config=config,
scope=sysname, profile=DataStore.DS_PROFILE.RESOURCES,
variant=DatastoreFactory.DS_BASE)
def _dump_datastore(self, outpath_base, ds_name, clear_dir=True):
ds = DatastoreFactory.get_datastore(datastore_name=ds_name, config=self.config, scope=self.sysname)
try:
if not ds.datastore_exists(ds_name):
log.warn("Datastore does not exist: %s" % ds_name)
return
if not os.path.exists(outpath_base):
os.makedirs(outpath_base)
outpath = "%s/%s" % (outpath_base, ds_name)
if not os.path.exists(outpath):
os.makedirs(outpath)
if clear_dir:
[os.remove(os.path.join(outpath, f)) for f in os.listdir(outpath)]
objs = ds.find_docs_by_view("_all_docs", None, id_only=False)
compact_obj = [obj for obj_id, obj_key, obj in objs]
compact_obj= ["COMPACTDUMP", compact_obj]
with open("%s/%s_compact.json" % (outpath, ds_name), 'w') as f:
json.dump(compact_obj, f)
numwrites = len(objs)
log.info("Wrote %s files to %s" % (numwrites, outpath))
finally:
ds.close()
def _force_clean(cls, recreate=False, initial=False):
# Database resources
from pyon.core.bootstrap import get_sys_name, CFG
from pyon.datastore.datastore_common import DatastoreFactory
datastore = DatastoreFactory.get_datastore(config=CFG, variant=DatastoreFactory.DS_BASE, scope=get_sys_name())
if initial:
datastore._init_database(datastore.database)
dbs = datastore.list_datastores()
clean_prefix = '%s_' % get_sys_name().lower()
things_to_clean = [x for x in dbs if x.startswith(clean_prefix)]
try:
for thing in things_to_clean:
datastore.delete_datastore(datastore_name=thing)
if recreate:
datastore.create_datastore(datastore_name=thing)
finally:
datastore.close()
# Broker resources
from putil.rabbitmq.rabbit_util import RabbitManagementUtil
rabbit_util = RabbitManagementUtil(CFG, sysname=bootstrap.get_sys_name())
deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname()
log.info("Deleted %s exchanges, %s queues" % (len(deleted_exchanges), len(deleted_queues)))
# File system
from pyon.util.file_sys import FileSystem
FileSystem._clean(CFG)
def _load_datastore(self, path=None, ds_name=None, ignore_errors=True):
ds = DatastoreFactory.get_datastore(datastore_name=ds_name, config=self.config, scope=self.sysname)
try:
objects = []
for fn in os.listdir(path):
fp = os.path.join(path, fn)
try:
with open(fp, 'r') as f:
json_text = f.read()
obj = json.loads(json_text)
if obj and type(obj) is list and obj[0] == "COMPACTDUMP":
objects.extend(obj[1])
else:
objects.append(obj)
except Exception as ex:
if ignore_errors:
log.warn("load error id=%s err=%s" % (fn, str(ex)))
else:
raise ex
if objects:
for obj in objects:
if "_rev" in obj:
del obj["_rev"]
try:
res = ds.create_doc_mult(objects)
log.info("DatastoreLoader: Loaded %s objects into %s" % (len(res), ds_name))
except Exception as ex:
if ignore_errors:
log.warn("load error err=%s" % (str(ex)))
else:
raise ex
finally:
ds.close()
def __init__(self, sysname=None, orgname=None, config=None):
self.orgname = orgname or get_safe(config, 'system.root_org', 'ION')
sysname = sysname or get_default_sysname()
self.datastore_name = "resources"
self.datastore = DatastoreFactory.get_datastore(datastore_name=self.datastore_name, config=config,
scope=sysname, profile=DataStore.DS_PROFILE.DIRECTORY,
variant=DatastoreFactory.DS_BASE)
def __init__(self, sysname=None, orgname=None, config=None):
self.orgname = orgname or get_safe(config, "system.root_org", "ION")
sysname = sysname or get_default_sysname()
self.datastore_name = "resources"
self.datastore = DatastoreFactory.get_datastore(
datastore_name=self.datastore_name,
config=config,
scope=sysname,
profile=DataStore.DS_PROFILE.DIRECTORY,
variant=DatastoreFactory.DS_BASE,
)
def get_datastore_instance(cls, ds_name, profile=None):
profile = profile or DataStore.DS_PROFILE_MAPPING.get(
ds_name, DataStore.DS_PROFILE.BASIC)
new_ds = DatastoreFactory.get_datastore(
datastore_name=ds_name,
profile=profile,
scope=get_sys_name(),
config=CFG,
variant=DatastoreFactory.DS_FULL)
return new_ds
def get_blame_objects(self):
ds_list = ['resources', 'objects', 'state', 'events']
blame_objs = {}
for ds_name in ds_list:
ret_objs = []
try:
ds = DatastoreFactory.get_datastore(datastore_name=ds_name, config=self.config, scope=self.sysname)
ret_objs = ds.find_docs_by_view("_all_docs", None, id_only=False)
ds.close()
except BadRequest:
continue
objs = []
for obj_id, obj_key, obj in ret_objs:
if "blame_" in obj:
objs.append(obj)
blame_objs[ds_name] = objs
return blame_objs
def test_span_insert(self):
scov, cov_name = self.construct_cov(nt=10)
self.coverages.add(scov.persistence_guid)
self.assertIsNotNone(scov)
span_store = DatastoreFactory.get_datastore(datastore_name='coverage_spans', config=CFG)
if span_store is None:
raise RuntimeError("Unable to load datastore for coverage_spans")
mm = scov._persistence_layer.master_manager
span_addr = []
with span_store.pool.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
cur.execute("SELECT span_address from %s where coverage_id='%s'" % (span_store._get_datastore_name(), mm.guid))
self.assertGreater(cur.rowcount, 0)
for row in cur:
span_addr.append(row['span_address'])
with span_store.pool.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
for addr in span_addr:
cur.execute("DELETE FROM %s WHERE span_address='%s'" % (span_store._get_datastore_name(), addr))
def _force_clean(cls, recreate=False):
from pyon.core.bootstrap import get_sys_name, CFG
from pyon.datastore.datastore_common import DatastoreFactory
datastore = DatastoreFactory.get_datastore(config=CFG, variant=DatastoreFactory.DS_BASE, scope=get_sys_name())
#datastore = DatastoreFactory.get_datastore(config=CFG, variant=DatastoreFactory.DS_BASE)
dbs = datastore.list_datastores()
things_to_clean = filter(lambda x: x.startswith('%s_' % get_sys_name().lower()), dbs)
try:
for thing in things_to_clean:
datastore.delete_datastore(datastore_name=thing)
if recreate:
datastore.create_datastore(datastore_name=thing)
finally:
datastore.close()
FileSystem._clean(CFG)
def create_core_datastores(self):
""" Create core datastores, so that they exist when containers start concurrently. """
ds = DatastoreFactory.get_datastore(config=self.config, scope=self.sysname, variant=DatastoreFactory.DS_BASE)
datastores = [
('resources', 'RESOURCES'),
('events', 'EVENTS'),
('state', 'STATE'),
('objects', 'OBJECTS'),
]
count = 0
ds_created = []
for local_dsn, profile in datastores:
if not ds.datastore_exists(local_dsn):
ds.create_datastore(datastore_name=local_dsn, profile=profile)
count += 1
ds_created.append(local_dsn)
if count:
log.info("store_interfaces: Created %s datastores: %s" % (count, ds_created))
def create_core_datastores(self):
"""
Main entry point into creating core datastores
"""
ds = DatastoreFactory.get_datastore(config=self.config, scope=self.sysname, variant=DatastoreFactory.DS_BASE)
datastores = [
('resources', 'RESOURCES'),
('events', 'EVENTS'),
('state', 'STATE'),
('objects', 'OBJECTS'),
]
count = 0
ds_created = []
for local_dsn, profile in datastores:
if not ds.datastore_exists(local_dsn):
ds.create_datastore(datastore_name=local_dsn, profile=profile)
count += 1
ds_created.append(local_dsn)
print "store_interfaces: Created %s datastores: %s" % (count, ds_created)
def _force_clean(cls, recreate=False):
from pyon.core.bootstrap import get_sys_name, CFG
from pyon.datastore.datastore_common import DatastoreFactory
datastore = DatastoreFactory.get_datastore(
config=CFG, variant=DatastoreFactory.DS_BASE, scope=get_sys_name())
#datastore = DatastoreFactory.get_datastore(config=CFG, variant=DatastoreFactory.DS_BASE)
dbs = datastore.list_datastores()
things_to_clean = filter(
lambda x: x.startswith('%s_' % get_sys_name().lower()), dbs)
try:
for thing in things_to_clean:
datastore.delete_datastore(datastore_name=thing)
if recreate:
datastore.create_datastore(datastore_name=thing)
finally:
datastore.close()
if os.environ.get('CEI_LAUNCH_TEST', None) is None:
FileSystem._clean(CFG)
def get_blame_objects(self):
ds_list = ['resources', 'objects', 'state', 'events']
blame_objs = {}
for ds_name in ds_list:
ret_objs = []
try:
ds = DatastoreFactory.get_datastore(datastore_name=ds_name,
config=self.config,
scope=self.sysname)
ret_objs = ds.find_docs_by_view("_all_docs",
None,
id_only=False)
ds.close()
except BadRequest:
continue
objs = []
for obj_id, obj_key, obj in ret_objs:
if "blame_" in obj:
objs.append(obj)
blame_objs[ds_name] = objs
return blame_objs
def create_core_datastores(self):
""" Create core datastores, so that they exist when containers start concurrently. """
ds = DatastoreFactory.get_datastore(config=self.config,
scope=self.sysname,
variant=DatastoreFactory.DS_BASE)
datastores = [
('resources', 'RESOURCES'),
('events', 'EVENTS'),
('state', 'STATE'),
('objects', 'OBJECTS'),
]
count = 0
ds_created = []
for local_dsn, profile in datastores:
if not ds.datastore_exists(local_dsn):
ds.create_datastore(datastore_name=local_dsn, profile=profile)
count += 1
ds_created.append(local_dsn)
if count:
log.info("store_interfaces: Created %s datastores: %s" %
(count, ds_created))
def dump_datastore(self, path=None, ds_name=None, clear_dir=True):
"""
Dumps CouchDB datastores into a directory as YML files.
@param ds_name Logical name (such as "resources") of an ION datastore
@param path Directory to put dumped datastores into (defaults to
"res/preload/local/dump_[timestamp]")
@param clear_dir if True, delete contents of datastore dump dirs
@param compact if True, saves all objects in one big YML file
"""
if not path:
dtstr = datetime.datetime.today().strftime('%Y%m%d_%H%M%S')
path = "res/preload/local/dump_%s" % dtstr
if ds_name:
ds = DatastoreFactory.get_datastore(datastore_name=ds_name, config=self.config, scope=self.sysname)
if ds.datastore_exists(ds_name):
self._dump_datastore(path, ds_name, clear_dir)
else:
log.warn("Datastore does not exist")
ds.close()
else:
ds_list = ['resources', 'objects', 'state', 'events']
for dsn in ds_list:
self._dump_datastore(path, dsn, clear_dir)
def _load_datastore(self, path=None, ds_name=None, ignore_errors=True):
ds = DatastoreFactory.get_datastore(datastore_name=ds_name,
config=self.config,
scope=self.sysname)
try:
objects = []
for fn in os.listdir(path):
fp = os.path.join(path, fn)
try:
with open(fp, 'r') as f:
json_text = f.read()
obj = json.loads(json_text)
if obj and type(obj) is list and obj[0] == "COMPACTDUMP":
objects.extend(obj[1])
else:
objects.append(obj)
except Exception as ex:
if ignore_errors:
log.warn("load error id=%s err=%s" % (fn, str(ex)))
else:
raise ex
if objects:
for obj in objects:
if "_rev" in obj:
del obj["_rev"]
try:
res = ds.create_doc_mult(objects)
log.info("DatastoreLoader: Loaded %s objects into %s" %
(len(res), ds_name))
except Exception as ex:
if ignore_errors:
log.warn("load error err=%s" % (str(ex)))
else:
raise ex
finally:
ds.close()
def clear_db(config, prefix, sysname=None):
config = DatastoreFactory.get_server_config(config)
_clear_db(config=config,
prefix=prefix,
sysname=sysname)
def get_datastore_instance(cls, ds_name, profile=None):
profile = profile or DataStore.DS_PROFILE_MAPPING.get(ds_name, DataStore.DS_PROFILE.BASIC)
new_ds = DatastoreFactory.get_datastore(datastore_name=ds_name, profile=profile, scope=get_sys_name(),
config=CFG, variant=DatastoreFactory.DS_FULL)
return new_ds
def clear_db(config, prefix, sysname=None):
config = DatastoreFactory.get_server_config(config)
_clear_db(config=config, prefix=prefix, sysname=sysname)
