def close(self):
""" Close the file (and compress it if required)
"""
if not self.opened: return
##
if 'sqlite3' == self.dbtype and 'c' == self.mode: write = True
elif 'sqlite3' != self.dbtype and 'e' == self.mode: write = True
else: write = False
if write:
meta = meta_info()
dct = self.get('__metainfo__', {})
dct['Updated at'] = datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S')
dct['Updated by'] = meta.User
dct['Updated with Ostap version'] = meta.Ostap
dct['Updated with Python version'] = meta.Python
dct['Updated with ROOT version'] = meta.ROOT
self['__metainfo__'] = dct
if not self.silent: self.ls()
shelve.Shelf.close(self)
self.__opened = False
##
if self.__compress:
self.__in_place_compress(self.__compress)
## remove the intermediate files
for f in self.__remove:
if os.path.exists(f):
try:
os.remove(f)
except OSError:
pass
def close(self):
""" Close the file (and compress it if required)
"""
if self.flag == 'c':
meta = meta_info()
dct = self.get('__metainfo__', {})
dct['Updated at'] = datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S')
dct['Updated by'] = meta.User
dct['Updated with Ostap version'] = meta.Ostap
dct['Updated with Python version'] = meta.Python
dct['Updated with ROOT version'] = meta.ROOT
self['__metainfo__'] = dct
return SqliteDict.close(self)
def __init__(self,
dbname,
mode='c',
protocol=PROTOCOL,
compress=0,
writeback=False,
silent=True,
keyencoding='utf-8'):
## the mode
mode = _modes_.get(mode.lower(), '')
if not mode:
logger.warning("Unknown opening mode '%s', replace with 'c'")
mode = 'c'
## expand the actual file name
dbname = os.path.expandvars(dbname)
dbname = os.path.expanduser(dbname)
dbname = os.path.expandvars(dbname)
dbname = os.path.expandvars(dbname)
self.__compresslevel = compress
self.__nominal_dbname = dbname
self.__actual_dbname = dbname
self.__compress = ()
self.__remove = ()
self.__silent = silent
self.__opened = False
self.__files = ()
if not self.__silent:
logger.info('Open DB: %s' % dbname)
## filename without extension and the extension itself
fname, ext = os.path.splitext(dbname)
self.__extension = ext
## predefined extension?
if ext.lower() in self.extensions:
fexists = os.path.exists(dbname)
if fexists and 'r' == mode:
## uncompress into the temporary location
tfiles = self.uncompress_file(dbname)
filename = self.dbase_name(tfiles)
self.__remove = tfiles
self.__compress = ()
self.__actual_dbname = filename
self.__files = tfiles
elif fexists and 'r' != mode:
## uncompress locally
tfiles = self.__in_place_uncompress(dbname)
filename = self.dbase_name(tfiles)
self.__compress = tfiles
self.__remove = tfiles
self.__files = tfiles
self.__actual_dbname = filename
else:
##
filename = fname
self.__compress = True
self.__remove = ()
self.__actual_dbname = filename
afiles = tuple([
self.dbname + suffix
for suffix in ('', ',db', '.dir', '.pag', '.dat')
])
ofiles = set([i for i in glob.iglob(self.dbname + '*') if i in afiles])
if 3 <= python_version.major:
shelve.Shelf.__init__(
self,
dbopen(self.dbname,
mode,
decode=lambda s: s,
encode=lambda s: c), protocol, writeback, keyencoding)
else:
shelve.Shelf.__init__(
self,
dbopen(self.dbname,
mode,
decode=lambda s: s,
encode=lambda s: c), protocol, writeback)
self.keyencoding = keyencoding
self.__opened = True
self.__mode = mode
self.sync()
self.__dbtype = whichdb(self.dbname)
nfiles = set([i for i in glob.iglob(self.dbname + '*') if i in afiles
]) - ofiles
if not self.__files:
files = []
f = self.dbname
db = self.dbtype
if os.path.exists ( f ) and os.path.isfile ( f ) and \
db in ( 'dbm.gnu' , 'gdbm' , 'dbhash' , 'bsddb185' , 'bsddb' , 'bsddb3' , 'sqlite3' ) :
files.append(f)
elif f + '.db' in nfiles and db in ('dbm.ndmb', 'dbm'):
files.append(f + '.db')
elif f + '.pag' in nfiles and f + '.dir' in nfiles and db in (
'dbm.ndbm', 'dbm'):
files.append(f + '.pag')
files.append(f + '.dir')
elif f + '.dat' in nfiles and f + '.dir' in nfiles and db in (
'dbm.dumb', 'dumbdbm'):
files.append(f + '.dat')
files.append(f + '.dir')
elif f + '.pag' in nfiles and db in ('dbm.ndbm', 'dbm'):
files.append(f + '.pag')
elif f + '.db' in ofiles and db in ('dbm.ndmb', 'dbm'):
files.append(f + '.db')
elif f + '.pag' in ofiles and f + '.dir' in ofiles and db in (
'dbm.ndbm', 'dbm'):
files.append(f + '.pag')
files.append(f + '.dir')
elif f + '.dat' in ofiles and f + '.dir' in ofiles and db in (
'dbm.dumb', 'dumbdbm'):
files.append(f + '.dat')
files.append(f + '.dir')
elif f + '.pag' in ofiles and db in ('dbm.dumb', 'dumbdbm'):
files.append(f + '.pag')
else:
logger.error('Cannot find DB for %s|%s' %
(self.dbname, self.dbtype))
files.sort()
self.__files = tuple(files)
if self.__compress is True:
self.__compress = self.files
self.__remove = self.files
if 'sqlite3' == self.dbtype and self.mode in ('w', 'n'): write = True
elif 'sqlite3' != self.dbtype and self.mode in ('c', 'n'): write = True
else: write = False
if write:
meta = meta_info()
dct = collections.OrderedDict()
dct['Created by'] = meta.User
dct['Created at'] = datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S')
dct['Created with Ostap version'] = meta.Ostap
dct['Created with Python version'] = meta.Python
dct['Created with ROOT version'] = meta.ROOT
dct['Pickle protocol'] = protocol
dct['Compress level'] = self.__compresslevel
self['__metainfo__'] = dct
if not self.silent:
self.ls()
ff = [os.path.basename(f) for f in self.files]
ff = ff[0] if 1 == len(ff) else ff
logger.info('DB files are %s|%s' % (ff, self.dbtype))
def __init__(
self,
filename=None,
mode='c',
tablename='ostap',
writeback=True, ## original name: "autocommit"
compress_level=zlib.Z_BEST_COMPRESSION,
journal_mode="DELETE",
protocol=PROTOCOL):
"""Initialize a thread-safe sqlite-backed dictionary.
The dictionary will be a table ``tablename`` in database file
``filename``. A single file (=database) may contain multiple tables.
If no ``filename`` is given, a random file in temp will be used
(and deleted from temp once the dict is closed/deleted).
If you enable ``writeback/autocommit`` changes will be committed
after each operation (more inefficient but safer).
Otherwise, changes are committed on
``self.commit()``,
``self.clear()`` and
``self.close()``.
Set ``journal_mode`` to ``OFF``
if you're experiencing sqlite I/O problems
or if you need performance and don't care about crash-consistency.
The `mode` parameter:
- 'c': default mode, open for read/write, creating the db/table if necessary.
- 'w': open for r/w, but drop `tablename` contents first (start with empty table)
- 'n': create a new database (erasing any existing tables, not just `tablename`!).
Modes: %s
""" % _modes_
## the mode
mode = _modes_.get(mode.lower(), '')
if not mode:
logger.warning("Unknown opening mode '%s', replace with 'c'")
mode = 'c'
if not filename is None:
import os
filename = os.path.expandvars(filename)
filename = os.path.expanduser(filename)
filename = os.path.expandvars(filename)
filename = os.path.abspath(filename)
self.__filename = filename
SqliteDict.__init__(self,
filename=filename,
tablename=tablename,
flag=mode,
autocommit=writeback,
journal_mode=journal_mode)
self.__compresslevel = compress_level
self.__protocol = protocol
self.__sizes = {}
if self.flag in ('w', 'n'):
meta = meta_info()
dct = collections.OrderedDict()
dct['Created by'] = meta.User
dct['Created at'] = datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S')
dct['Created with Ostap version'] = meta.Ostap
dct['Created with Python version'] = meta.Python
dct['Created with ROOT version'] = meta.ROOT
dct['Pickle protocol'] = protocol
dct['Compress level'] = self.__compresslevel
self['__metainfo__'] = dct