def clear_datastore(cls, ds_name=None, prefix=None):
if CFG.system.mockdb:
log.warn("Cannot clear MockDB")
return
generic_ds = DatastoreManager.get_datastore_instance("")
if ds_name:
# First interpret ds_name as unqualified name
if DatastoreManager.exists(ds_name, scoped=False):
generic_ds.delete_datastore(ds_name)
return
# New interpret as logical name
if DatastoreManager.exists(ds_name, scoped=True):
generic_ds.delete_datastore(ds_name)
else:
log.warn("Datastore does not exist: %s" % ds_name)
elif prefix:
db_list = generic_ds.list_datastores()
cleared, ignored = 0, 0
for db_name in db_list:
if db_name.startswith(prefix):
generic_ds.delete_datastore(db_name)
log.debug("Cleared couch datastore '%s'" % db_name)
cleared += 1
else:
ignored += 1
log.info("Cleared %d couch datastores, ignored %d" %
(cleared, ignored))
else:
log.warn("Cannot clear datastore without prefix or datastore name")
def clear_datastore(cls, ds_name=None, prefix=None):
if CFG.system.mockdb:
log.warn("Cannot clear MockDB")
return
generic_ds = DatastoreManager.get_datastore_instance("")
if ds_name:
# First interpret ds_name as unqualified name
if DatastoreManager.exists(ds_name, scoped=False):
generic_ds.delete_datastore(ds_name)
return
# New interpret as logical name
if DatastoreManager.exists(ds_name, scoped=True):
generic_ds.delete_datastore(ds_name)
else:
log.warn("Datastore does not exist: %s" % ds_name)
elif prefix:
db_list = generic_ds.list_datastores()
cleared, ignored = 0, 0
for db_name in db_list:
if db_name.startswith(prefix):
generic_ds.delete_datastore(db_name)
log.debug("Cleared couch datastore '%s'" % db_name)
cleared += 1
else:
ignored += 1
log.info("Cleared %d couch datastores, ignored %d" % (cleared, ignored))
else:
log.warn("Cannot clear datastore without prefix or datastore name")
def _load_datastore(cls, path=None, ds_name=None, ignore_errors=True):
if not DatastoreManager.exists(ds_name):
log.warn("Datastore does not exist: %s" % ds_name)
ds = DatastoreManager.get_datastore_instance(ds_name)
objects = []
for fn in os.listdir(path):
fp = os.path.join(path, fn)
try:
with open(fp, 'r') as f:
yaml_text = f.read()
obj = yaml.load(yaml_text)
if "_rev" in obj:
del obj["_rev"]
objects.append(obj)
except Exception as ex:
if ignore_errors:
log.warn("load error id=%s err=%s" % (fn, str(ex)))
else:
raise ex
if objects:
try:
res = ds.create_doc_mult(objects, allow_ids=True)
log.info("DatastoreLoader: Loaded %s objects into %s" %
(len(res), ds_name))
except Exception as ex:
if ignore_errors:
log.warn("load error id=%s err=%s" % (fn, str(ex)))
else:
raise ex
def _dump_datastore(cls, outpath_base, ds_name, clear_dir=True):
if not DatastoreManager.exists(ds_name):
log.warn("Datastore does not exist: %s" % ds_name)
return
ds = DatastoreManager.get_datastore_instance(ds_name)
if not os.path.exists(outpath_base):
os.makedirs(outpath_base)
outpath = "%s/%s" % (outpath_base, ds_name)
if not os.path.exists(outpath):
os.makedirs(outpath)
if clear_dir:
[os.remove(os.path.join(outpath, f)) for f in os.listdir(outpath)]
objs = ds.find_by_view("_all_docs",
None,
id_only=False,
convert_doc=False)
numwrites = 0
for obj_id, obj_key, obj in objs:
fn = obj_id
# Some object ids have slashes
fn = obj_id.replace("/", "_")
with open("%s/%s.yml" % (outpath, fn), 'w') as f:
yaml.dump(obj, f, default_flow_style=False)
numwrites += 1
log.info("Wrote %s objects to %s" % (numwrites, outpath))
def dump_datastore(cls, path=None, ds_name=None, clear_dir=True):
"""
Dumps CouchDB datastores into a directory as YML files.
@param ds_name Logical name (such as "resources") of an ION datastore
@param path Directory to put dumped datastores into (defaults to
"res/preload/local/dump_[timestamp]")
@param clear_dir if True, delete contents of datastore dump dirs
"""
if CFG.system.mockdb:
log.warn("Cannot dump from MockDB")
return
if not path:
dtstr = datetime.datetime.today().strftime('%Y%m%d_%H%M%S')
path = "res/preload/local/dump_%s" % dtstr
if ds_name:
if DatastoreManager.exists(ds_name):
cls._dump_datastore(ds_name, path, clear_dir)
else:
log.warn("Datastore does not exist")
else:
ds_list = [
'resources', 'objects', 'state', 'events', 'directory',
'scidata'
]
for ds in ds_list:
cls._dump_datastore(path, ds, clear_dir)
def _dump_datastore(cls, outpath_base, ds_name, clear_dir=True):
if not DatastoreManager.exists(ds_name):
log.warn("Datastore does not exist: %s" % ds_name)
return
ds = DatastoreManager.get_datastore_instance(ds_name)
if not os.path.exists(outpath_base):
os.makedirs(outpath_base)
outpath = "%s/%s" % (outpath_base, ds_name)
if not os.path.exists(outpath):
os.makedirs(outpath)
if clear_dir:
[os.remove(os.path.join(outpath, f)) for f in os.listdir(outpath)]
objs = ds.find_by_view("_all_docs", None, id_only=False, convert_doc=False)
numwrites = 0
for obj_id, obj_key, obj in objs:
fn = obj_id
# Some object ids have slashes
fn = obj_id.replace("/", "_")
with open("%s/%s.yml" % (outpath, fn), "w") as f:
yaml.dump(obj, f, default_flow_style=False)
numwrites += 1
log.info("Wrote %s objects to %s" % (numwrites, outpath))
def _load_datastore(cls, path=None, ds_name=None, ignore_errors=True):
if not DatastoreManager.exists(ds_name):
log.warn("Datastore does not exist: %s" % ds_name)
ds = DatastoreManager.get_datastore_instance(ds_name)
objects = []
for fn in os.listdir(path):
fp = os.path.join(path, fn)
try:
with open(fp, 'r') as f:
yaml_text = f.read()
obj = yaml.load(yaml_text)
if "_rev" in obj:
del obj["_rev"]
objects.append(obj)
except Exception as ex:
if ignore_errors:
log.warn("load error id=%s err=%s" % (fn, str(ex)))
else:
raise ex
if objects:
try:
res = ds.create_doc_mult(objects, allow_ids=True)
log.info("DatastoreLoader: Loaded %s objects into %s" % (len(res), ds_name))
except Exception as ex:
if ignore_errors:
log.warn("load error id=%s err=%s" % (fn, str(ex)))
else:
raise ex
def _load_datastore(cls, path=None, ds_name=None, ignore_errors=True):
if not DatastoreManager.exists(ds_name):
log.warn("Datastore does not exist: %s" % ds_name)
ds = DatastoreManager.get_datastore_instance(ds_name)
for fn in os.listdir(path):
fp = os.path.join(path, fn)
try:
cls._read_and_create_obj(ds, fp)
except Exception as ex:
if ignore_errors:
log.warn("load error id=%s err=%s" % (fn, str(ex)))
else:
raise ex
def dump_datastore(cls, path=None, ds_name=None, clear_dir=True):
"""
Dumps CouchDB datastores into a directory as YML files.
@param ds_name Logical name (such as "resources") of an ION datastore
@param path Directory to put dumped datastores into (defaults to
"res/preload/local/dump_[timestamp]")
@param clear_dir if True, delete contents of datastore dump dirs
"""
if CFG.system.mockdb:
log.warn("Cannot dump from MockDB")
return
if not path:
dtstr = datetime.datetime.today().strftime("%Y%m%d_%H%M%S")
path = "res/preload/local/dump_%s" % dtstr
if ds_name:
if DatastoreManager.exists(ds_name):
cls._dump_datastore(ds_name, path, clear_dir)
else:
log.warn("Datastore does not exist")
else:
ds_list = ["resources", "objects", "state", "events", "directory", "scidata"]
for ds in ds_list:
cls._dump_datastore(path, ds, clear_dir)
