def local_intermittent_storage(
cache_name: str, ) -> Generator[pt.DbmMapping, None, None]:
"""
The cache is deleted upon a Raspberry Pi restart!
Examples
---------
> with local_intermittent_storage('pwm') as cache:
> assert '1' in cache
> cache['1'] = str(0.5)
Notes
-------
Opening the same cache in a context manager are tricky, and should be avoided. The general rule is:
1. Outer-scoped caches can't access keys created by inner scope caches.
2. The latest value given to a key, regardless of which scoped cache, is the one that is finally written.
See tests in test_utils.py for examples.
"""
try:
cache = ndbm.open(f"/tmp/{cache_name}", "c")
yield cache # type: ignore
finally:
cache.close()
def __init__(self, path, readOnly=False):
super().__init__()
self._extension = '.db'
if path[-3:] != self._extension:
raise DbError(
_("Database %(db)s does not end with %(ext)s") % {
'db': path,
'ext': self._extension
})
self._dbpath = path[:-3] # strip the .db suffix
try:
mode = 'r' if readOnly else 'c'
self._seen = ndbm.open(self._dbpath, mode, 0o644)
'foo%0' in self._seen
except Exception as ex:
raise DbError(
_("Database %(db)s failed to load: %(errmsg)s") % {
'db': path,
'errmsg': str(ex)
}) from ex
# Will replace seen after changes have actually been seen
self._seen_new = {}
def apply_changes(self):
if not self._seen_new:
# Nothing to update
return
def mk(arg):
return self._dbpath + arg + self._extension
old, cur, new = (mk('-old'), mk(''), mk('-new'))
# For reliability and to have backup of the database,
# it is updated in the following steps:
# 1. copy current version to new
if os.path.isfile(cur):
shutil.copy(cur, new)
# 2. apply the changes to new
seen = ndbm.open(self._dbpath + '-new', 'c', 0o644)
for (key, value) in self._seen_new.items():
seen[key] = value
seen.close()
# 3. save current as old, and rename new to current
if os.path.isfile(old):
os.unlink(old)
if os.path.isfile(cur):
os.link(cur, old)
os.rename(new, cur)
def ndbm_test_db(request):
print("creating test ndbm file")
temp_file = tempfile.NamedTemporaryFile()
test_db = ndbm.open(temp_file.name, "n")
test_db[key1] = val1
test_db[key2] = val2
test_db.close()
def delete_test_ndbm():
print("deleting test ndbm file")
temp_file.close()
os.remove(temp_file.name + ".db")
request.addfinalizer(delete_test_ndbm)
return temp_file.name
def local_persistant_storage(
cache_name: str, ) -> Generator[pt.DbmMapping, None, None]:
"""
Values stored in this storage will stay around between RPi restarts, and until overwritten
or deleted.
Examples
---------
> with local_persistant_storage('od_blank') as cache:
> assert '1' in cache
> cache['1'] = str(0.5)
"""
from pioreactor.whoami import is_testing_env
try:
if is_testing_env():
cache = ndbm.open(f".pioreactor/storage/{cache_name}", "c")
else:
cache = ndbm.open(f"/home/pi/.pioreactor/storage/{cache_name}",
"c")
yield cache # type: ignore
finally:
cache.close()
def ndbm_test_db(request):
temp_file = tempfile.NamedTemporaryFile(delete=False)
test_db = ndbm.open(temp_file.name, "n")
test_db[key1] = val1
test_db[key2] = val2
test_db.close()
def delete_test_ndbm():
temp_file.close()
for f in glob.glob("{}*".format(temp_file.name)):
print("deleting test ndbm file {}".format(f))
os.remove(f)
request.addfinalizer(delete_test_ndbm)
return temp_file.name
def ndbm_test_db(request):
temp_file = tempfile.NamedTemporaryFile(delete=False)
test_db = ndbm.open(temp_file.name, "n")
test_db[key1] = val1
test_db[key2] = val2
test_db.close()
def delete_test_ndbm():
temp_file.close()
for f in glob.glob("{}*".format(temp_file.name)):
print("deleting test ndbm file {}".format(f))
os.remove(f)
request.addfinalizer(delete_test_ndbm)
return temp_file.name
def whichdb(filename):
"""Guess which db package to use to open a db file.
Return values:
- None if the database file can't be read;
- empty string if the file can be read but can't be recognized
- the name of the dbm submodule (e.g. "ndbm" or "gnu") if recognized.
Importing the given module may still fail, and opening the
database using that module may still fail.
"""
# Check for ndbm first -- this has a .pag and a .dir file
try:
f = io.open(filename + ".pag", "rb")
f.close()
# dbm linked with gdbm on OS/2 doesn't have .dir file
if not (ndbm.library == "GNU gdbm" and sys.platform == "os2emx"):
f = io.open(filename + ".dir", "rb")
f.close()
return "dbm.ndbm"
except IOError:
# some dbm emulations based on Berkeley DB generate a .db file
# some do not, but they should be caught by the bsd checks
try:
f = io.open(filename + ".db", "rb")
f.close()
# guarantee we can actually open the file using dbm
# kind of overkill, but since we are dealing with emulations
# it seems like a prudent step
if ndbm is not None:
d = ndbm.open(filename)
d.close()
return "dbm.ndbm"
except IOError:
pass
# Check for dumbdbm next -- this has a .dir and a .dat file
try:
# First check for presence of files
os.stat(filename + ".dat")
size = os.stat(filename + ".dir").st_size
# dumbdbm files with no keys are empty
if size == 0:
return "dbm.dumb"
f = io.open(filename + ".dir", "rb")
try:
if f.read(1) in (b"'", b'"'):
return "dbm.dumb"
finally:
f.close()
except (OSError, IOError):
pass
# See if the file exists, return None if not
try:
f = io.open(filename, "rb")
except IOError:
return None
# Read the start of the file -- the magic number
s16 = f.read(16)
f.close()
s = s16[0:4]
# Return "" if not at least 4 bytes
if len(s) != 4:
return ""
# Convert to 4-byte int in native byte order -- return "" if impossible
try:
(magic,) = struct.unpack("=l", s)
except struct.error:
return ""
# Check for GNU dbm
if magic in (0x13579ace, 0x13579acd, 0x13579acf):
return "dbm.gnu"
# Later versions of Berkeley db hash file have a 12-byte pad in
# front of the file type
try:
(magic,) = struct.unpack("=l", s16[-4:])
except struct.error:
return ""
# Unknown
return ""
x['DictionaryKeyIteratorType'] = iter(type.__dict__.keys())
x['DictionaryValueIteratorType'] = iter(type.__dict__.values())
else:
x['DictionaryItemIteratorType'] = type.__dict__.iteritems()
x['DictionaryKeyIteratorType'] = type.__dict__.iterkeys()
x['DictionaryValueIteratorType'] = type.__dict__.itervalues()
# string services (CH 7)
x['StructType'] = struct.Struct('c')
x['CallableIteratorType'] = _srepattern.finditer('')
x['SREMatchType'] = _srepattern.match('')
x['SREScannerType'] = _srepattern.scanner('')
x['StreamReader'] = codecs.StreamReader(_cstrI) #XXX: ... and etc
# python object persistence (CH 11)
# x['DbShelveType'] = shelve.open('foo','n')#,protocol=2) #XXX: delete foo
if HAS_ALL:
x['DbmType'] = dbm.open(_tempfile,'n')
# x['DbCursorType'] = _dbcursor = anydbm.open('foo','n') #XXX: delete foo
# x['DbType'] = _dbcursor.db
# data compression and archiving (CH 12)
x['ZlibCompressType'] = zlib.compressobj()
x['ZlibDecompressType'] = zlib.decompressobj()
# file formats (CH 13)
x['CSVReaderType'] = csv.reader(_cstrI)
x['CSVWriterType'] = csv.writer(_cstrO)
x['CSVDictReaderType'] = csv.DictReader(_cstrI)
x['CSVDictWriterType'] = csv.DictWriter(_cstrO,{})
# cryptographic services (CH 14)
x['HashType'] = hashlib.md5()
x['HMACType'] = hmac.new(_in)
# generic operating system services (CH 15)
if HAS_CURSES: pass
x["DictionaryKeyIteratorType"] = iter(type.__dict__.keys())
x["DictionaryValueIteratorType"] = iter(type.__dict__.values())
else:
x["DictionaryItemIteratorType"] = type.__dict__.iteritems()
x["DictionaryKeyIteratorType"] = type.__dict__.iterkeys()
x["DictionaryValueIteratorType"] = type.__dict__.itervalues()
# string services (CH 7)
x["StructType"] = struct.Struct("c")
x["CallableIteratorType"] = _srepattern.finditer("")
x["SREMatchType"] = _srepattern.match("")
x["SREScannerType"] = _srepattern.scanner("")
x["StreamReader"] = codecs.StreamReader(_cstrI) # XXX: ... and etc
# python object persistence (CH 11)
# x['DbShelveType'] = shelve.open('foo','n')#,protocol=2) #XXX: delete foo
if HAS_ALL:
x["DbmType"] = dbm.open(_tempfile, "n")
# x['DbCursorType'] = _dbcursor = anydbm.open('foo','n') #XXX: delete foo
# x['DbType'] = _dbcursor.db
# data compression and archiving (CH 12)
x["ZlibCompressType"] = zlib.compressobj()
x["ZlibDecompressType"] = zlib.decompressobj()
# file formats (CH 13)
x["CSVReaderType"] = csv.reader(_cstrI)
x["CSVWriterType"] = csv.writer(_cstrO)
x["CSVDictReaderType"] = csv.DictReader(_cstrI)
x["CSVDictWriterType"] = csv.DictWriter(_cstrO, {})
# cryptographic services (CH 14)
x["HashType"] = hashlib.md5()
x["HMACType"] = hmac.new(_in)
# generic operating system services (CH 15)
if HAS_CURSES:
def __init__(self, db_filename='schedules'):
'''Initializes the scheduler and opens the dbm file.'''
db_filename = db_filename
db_filename = os.path.join(options.main.workdir, db_filename)
self.db = dbm.open(db_filename, 'c', 0o600)
def __init__(self, db_filename='schedules'):
'''Initializes the scheduler and opens the dbm file.'''
db_filename = db_filename
db_filename = os.path.join(options.main.workdir, db_filename)
self.db = dbm.open(db_filename, 'c', 0o600)
def __init__(self, datadir):
self.blockDB = dbmd.open(datadir + "/blocks", 'c')
self.currentBlock = 0
self.headers_map = dict()
def __init__(self, datadir):
self.txDB = dbmd.open(datadir + "/transactions", 'c')
def whichdb(filename):
"""Guess which db package to use to open a db file.
Return values:
- None if the database file can't be read;
- empty string if the file can be read but can't be recognized
- the name of the dbm submodule (e.g. "ndbm" or "gnu") if recognized.
Importing the given module may still fail, and opening the
database using that module may still fail.
"""
try:
f = io.open(filename + '.pag', 'rb')
f.close()
f = io.open(filename + '.dir', 'rb')
f.close()
return 'dbm.ndbm'
except OSError:
try:
f = io.open(filename + '.db', 'rb')
f.close()
if ndbm is not None:
d = ndbm.open(filename)
d.close()
return 'dbm.ndbm'
except OSError:
pass
try:
os.stat(filename + '.dat')
size = os.stat(filename + '.dir').st_size
if size == 0:
return 'dbm.dumb'
f = io.open(filename + '.dir', 'rb')
try:
if f.read(1) in (b"'", b'"'):
return 'dbm.dumb'
finally:
f.close()
except OSError:
pass
try:
f = io.open(filename, 'rb')
except OSError:
return None
with f:
s16 = f.read(16)
s = s16[0:4]
if len(s) != 4:
return ''
try:
magic, = struct.unpack('=l', s)
except struct.error:
return ''
if magic in (324508366, 324508365, 324508367):
return 'dbm.gnu'
try:
magic, = struct.unpack('=l', s16[-4:])
except struct.error:
return ''
return ''
def whichdb(filename):
"""Guess which db package to use to open a db file.
Return values:
- None if the database file can't be read;
- empty string if the file can be read but can't be recognized
- the name of the dbm submodule (e.g. "ndbm" or "gnu") if recognized.
Importing the given module may still fail, and opening the
database using that module may still fail.
"""
# Check for ndbm first -- this has a .pag and a .dir file
try:
f = io.open(filename + ".pag", "rb")
f.close()
f = io.open(filename + ".dir", "rb")
f.close()
return "dbm.ndbm"
except OSError:
# some dbm emulations based on Berkeley DB generate a .db file
# some do not, but they should be caught by the bsd checks
try:
f = io.open(filename + ".db", "rb")
f.close()
# guarantee we can actually open the file using dbm
# kind of overkill, but since we are dealing with emulations
# it seems like a prudent step
if ndbm is not None:
d = ndbm.open(filename)
d.close()
return "dbm.ndbm"
except OSError:
pass
# Check for dumbdbm next -- this has a .dir and a .dat file
try:
# First check for presence of files
os.stat(filename + ".dat")
size = os.stat(filename + ".dir").st_size
# dumbdbm files with no keys are empty
if size == 0:
return "dbm.dumb"
f = io.open(filename + ".dir", "rb")
try:
if f.read(1) in (b"'", b'"'):
return "dbm.dumb"
finally:
f.close()
except OSError:
pass
# See if the file exists, return None if not
try:
f = io.open(filename, "rb")
except OSError:
return None
with f:
# Read the start of the file -- the magic number
s16 = f.read(16)
s = s16[0:4]
# Return "" if not at least 4 bytes
if len(s) != 4:
return ""
# Convert to 4-byte int in native byte order -- return "" if impossible
try:
(magic,) = struct.unpack("=l", s)
except struct.error:
return ""
# Check for GNU dbm
if magic in (0x13579ace, 0x13579acd, 0x13579acf):
return "dbm.gnu"
# Later versions of Berkeley db hash file have a 12-byte pad in
# front of the file type
try:
(magic,) = struct.unpack("=l", s16[-4:])
except struct.error:
return ""
# Unknown
return ""
x['DictionaryKeyIteratorType'] = iter(type.__dict__.keys())
x['DictionaryValueIteratorType'] = iter(type.__dict__.values())
else:
x['DictionaryItemIteratorType'] = type.__dict__.iteritems()
x['DictionaryKeyIteratorType'] = type.__dict__.iterkeys()
x['DictionaryValueIteratorType'] = type.__dict__.itervalues()
# string services (CH 7)
x['StructType'] = struct.Struct('c')
x['CallableIteratorType'] = _srepattern.finditer('')
x['SREMatchType'] = _srepattern.match('')
x['SREScannerType'] = _srepattern.scanner('')
x['StreamReader'] = codecs.StreamReader(_cstrI) #XXX: ... and etc
# python object persistence (CH 11)
# x['DbShelveType'] = shelve.open('foo','n')#,protocol=2) #XXX: delete foo
if HAS_ALL:
x['DbmType'] = dbm.open(_tempfile, 'n')
# x['DbCursorType'] = _dbcursor = anydbm.open('foo','n') #XXX: delete foo
# x['DbType'] = _dbcursor.db
# data compression and archiving (CH 12)
x['ZlibCompressType'] = zlib.compressobj()
x['ZlibDecompressType'] = zlib.decompressobj()
# file formats (CH 13)
x['CSVReaderType'] = csv.reader(_cstrI)
x['CSVWriterType'] = csv.writer(_cstrO)
x['CSVDictReaderType'] = csv.DictReader(_cstrI)
x['CSVDictWriterType'] = csv.DictWriter(_cstrO, {})
# cryptographic services (CH 14)
x['HashType'] = hashlib.md5()
x['HMACType'] = hmac.new(_in)
# generic operating system services (CH 15)
if HAS_CURSES: pass
def __init__(self, storepath):
self.storepath = storepath
self.db = dbm.open(storepath, "c", 0o600)
