def test_create_new(self):
with dumbdbm.open(_fname, 'n') as f:
for k in self._dict:
f[k] = self._dict[k]
with dumbdbm.open(_fname, 'n') as f:
self.assertEqual(f.keys(), [])
def test_missing_data(self):
for value in ('r', 'w'):
_delete_files()
with self.assertRaises(FileNotFoundError):
dumbdbm.open(_fname, value)
self.assertFalse(os.path.exists(_fname + '.dir'))
self.assertFalse(os.path.exists(_fname + '.bak'))
def test_write_write_read(self):
# test for bug #482460
with contextlib.closing(dumbdbm.open(_fname)) as f:
f[b'1'] = b'hello'
f[b'1'] = b'hello2'
with contextlib.closing(dumbdbm.open(_fname)) as f:
self.assertEqual(f[b'1'], b'hello2')
def test_missing_index(self):
with dumbdbm.open(_fname, 'n') as f:
pass
os.unlink(_fname + '.dir')
for value in ('r', 'w'):
with self.assertRaises(FileNotFoundError):
dumbdbm.open(_fname, value)
self.assertFalse(os.path.exists(_fname + '.dir'))
self.assertFalse(os.path.exists(_fname + '.bak'))
def test_write_write_read(self):
# test for bug #482460
f = dumbdbm.open(_fname)
f[b'1'] = b'hello'
f[b'1'] = b'hello2'
f.close()
f = dumbdbm.open(_fname)
self.assertEqual(f[b'1'], b'hello2')
f.close()
def test_context_manager(self):
with dumbdbm.open(_fname, 'c') as db:
db["dumbdbm context manager"] = "context manager"
with dumbdbm.open(_fname, 'r') as db:
self.assertEqual(list(db.keys()), [b"dumbdbm context manager"])
with self.assertRaises(dumbdbm.error):
db.keys()
def test_str_write_contains(self):
self.init_db()
with contextlib.closing(dumbdbm.open(_fname)) as f:
f['\u00fc'] = b'!'
f['1'] = 'a'
with contextlib.closing(dumbdbm.open(_fname, 'r')) as f:
self.assertIn('\u00fc', f)
self.assertEqual(f['\u00fc'.encode('utf-8')],
self._dict['\u00fc'.encode('utf-8')])
self.assertEqual(f[b'1'], b'a')
def test_str_write_contains(self):
self.init_db()
f = dumbdbm.open(_fname)
f['\u00fc'] = b'!'
f['1'] = 'a'
f.close()
f = dumbdbm.open(_fname, 'r')
self.assertIn('\u00fc', f)
self.assertEqual(f['\u00fc'.encode('utf-8')],
self._dict['\u00fc'.encode('utf-8')])
self.assertEqual(f[b'1'], b'a')
def test_nonascii_filename(self):
filename = support.TESTFN_NONASCII
for suffix in ['.dir', '.dat', '.bak']:
self.addCleanup(support.unlink, filename + suffix)
with dumbdbm.open(filename, 'c') as db:
db[b'key'] = b'value'
self.assertTrue(os.path.exists(filename + '.dat'))
self.assertTrue(os.path.exists(filename + '.dir'))
with dumbdbm.open(filename, 'r') as db:
self.assertEqual(list(db.keys()), [b'key'])
self.assertTrue(b'key' in db)
self.assertEqual(db[b'key'], b'value')
def test_context_manager(self):
with dumbdbm.open(_fname, 'c') as db:
db["dumbdbm context manager"] = "context manager"
with dumbdbm.open(_fname, 'r') as db:
self.assertEqual(list(db.keys()), [b"dumbdbm context manager"])
# This currently just raises AttributeError rather than a specific
# exception like the GNU or NDBM based implementations. See
# http://bugs.python.org/issue19385 for details.
with self.assertRaises(Exception):
db.keys()
def test_readonly_files(self):
with support.temp_dir() as dir:
fname = os.path.join(dir, 'db')
with dumbdbm.open(fname, 'n') as f:
self.assertEqual(list(f.keys()), [])
for key in self._dict:
f[key] = self._dict[key]
os.chmod(fname + ".dir", stat.S_IRUSR)
os.chmod(fname + ".dat", stat.S_IRUSR)
os.chmod(dir, stat.S_IRUSR|stat.S_IXUSR)
with dumbdbm.open(fname, 'r') as f:
self.assertEqual(sorted(f.keys()), sorted(self._dict))
f.close() # don't write
def test_missing_index(self):
with dumbdbm.open(_fname, 'n') as f:
pass
os.unlink(_fname + '.dir')
for value in ('r', 'w'):
with self.assertWarnsRegex(DeprecationWarning,
"The index file is missing, the "
"semantics of the 'c' flag will "
"be used."):
f = dumbdbm.open(_fname, value)
f.close()
self.assertEqual(os.path.exists(_fname + '.dir'), value == 'w')
self.assertFalse(os.path.exists(_fname + '.bak'))
def test_dumbdbm_creation(self):
f = dumbdbm.open(_fname, 'c')
self.assertEqual(list(f.keys()), [])
for key in self._dict:
f[key] = self._dict[key]
self.read_helper(f)
f.close()
def cache_rm(self, distfilenames):
with contextlib.closing(dumbdbm.open(self.cache, 'w')) as dbm:
for distfilename in distfilenames:
distfilename = os.path.basename(distfilename)
key = self._make_key(distfilename)
if key in dbm:
del dbm[key]
def test_invalid_flag(self):
for flag in ('x', 'rf', None):
with self.assertWarnsRegex(DeprecationWarning,
"Flag must be one of "
"'r', 'w', 'c', or 'n'"):
f = dumbdbm.open(_fname, flag)
f.close()
def test_eval(self):
with open(_fname + '.dir', 'w') as stream:
stream.write("str(print('Hacked!')), 0\n")
with support.captured_stdout() as stdout:
with self.assertRaises(ValueError):
with dumbdbm.open(_fname) as f:
pass
self.assertEqual(stdout.getvalue(), '')
def test_dumbdbm_modification(self):
self.init_db()
with contextlib.closing(dumbdbm.open(_fname, 'w')) as f:
self._dict[b'g'] = f[b'g'] = b"indented"
self.read_helper(f)
# setdefault() works as in the dict interface
self.assertEqual(f.setdefault(b'xxx', b'foo'), b'foo')
self.assertEqual(f[b'xxx'], b'foo')
def cache_print_errors(self):
with contextlib.closing(dumbdbm.open(self.cache, 'r')) as dbm:
for key, value in dbm.items():
if not self._key_match(key):
continue
if value:
wheel = self._key_to_wheel(key)
click.echo(u"{}: {}".format(wheel, value))
def test_warn_on_ignored_flags(self):
for value in ('r', 'w'):
_delete_files()
with self.assertWarnsRegex(DeprecationWarning,
"The database file is missing, the "
"semantics of the 'c' flag will "
"be used."):
f = dumbdbm.open(_fname, value)
f.close()
def test_line_endings(self):
# test for bug #1172763: dumbdbm would die if the line endings
# weren't what was expected.
f = dumbdbm.open(_fname)
f[b'1'] = b'hello'
f[b'2'] = b'hello2'
f.close()
# Mangle the file by changing the line separator to Windows or Unix
data = io.open(_fname + '.dir', 'rb').read()
if os.linesep == '\n':
data = data.replace(b'\n', b'\r\n')
else:
data = data.replace(b'\r\n', b'\n')
io.open(_fname + '.dir', 'wb').write(data)
f = dumbdbm.open(_fname)
self.assertEqual(f[b'1'], b'hello')
self.assertEqual(f[b'2'], b'hello2')
def test_dumbdbm_read(self):
self.init_db()
f = dumbdbm.open(_fname, 'r')
self.read_helper(f)
with self.assertWarnsRegex(DeprecationWarning,
'The database is opened for reading only'):
f[b'g'] = b'x'
with self.assertWarnsRegex(DeprecationWarning,
'The database is opened for reading only'):
del f[b'a']
f.close()
def test_dumbdbm_read(self):
self.init_db()
f = dumbdbm.open(_fname, 'r')
self.read_helper(f)
with self.assertRaisesRegex(ValueError,
'The database is opened for reading only'):
f[b'g'] = b'x'
with self.assertRaisesRegex(ValueError,
'The database is opened for reading only'):
del f[b'a']
f.close()
def test_random(self):
import random
d = {} # mirror the database
for dummy in range(5):
with contextlib.closing(dumbdbm.open(_fname)) as f:
for dummy in range(100):
k = random.choice('abcdefghijklm')
if random.random() < 0.2:
if k in d:
del d[k]
del f[k]
else:
v = random.choice((b'a', b'b', b'c')) * random.randrange(10000)
d[k] = v
f[k] = v
self.assertEqual(f[k], v)
with contextlib.closing(dumbdbm.open(_fname)) as f:
expected = sorted((k.encode("latin-1"), v) for k, v in d.items())
got = sorted(f.items())
self.assertEqual(expected, got)
def upload_dists(self, distfilenames):
to_upload = []
for distfilename in distfilenames:
if os.path.isfile(distfilename) and \
(distfilename.lower().endswith('.whl') or
distfilename.lower().endswith('.tar.gzXXX')):
to_upload.append(distfilename)
else:
_logger.debug("skipped %s: not a python distribution",
distfilename)
with contextlib.closing(dumbdbm.open(self.cache, 'c')) as dbm:
for distfilename in sorted(to_upload, key=_split_filename):
self.upload_dist(distfilename, dbm)
def dumbdbm_test_db(request):
temp_file = tempfile.NamedTemporaryFile(delete=False)
print("creating test dumbdbm file {}".format(temp_file.name))
test_db = dumb.open(temp_file.name, "n")
test_db[key1] = val1
test_db[key2] = val2
test_db.close()
def delete_dumbdbm_test_db():
temp_file.close()
for f in glob.glob("{}*".format(temp_file.name)):
print("deleting test dumbdbm file {}".format(f))
os.remove(f)
request.addfinalizer(delete_dumbdbm_test_db)
return temp_file.name
def test_dumbdbm_read(self):
self.init_db()
with contextlib.closing(dumbdbm.open(_fname, 'r')) as f:
self.read_helper(f)
with self.assertRaisesRegex(dumbdbm.error,
'The database is opened for reading only'):
f[b'g'] = b'x'
with self.assertRaisesRegex(dumbdbm.error,
'The database is opened for reading only'):
del f[b'a']
# get() works as in the dict interface
self.assertEqual(f.get(b'a'), self._dict[b'a'])
self.assertEqual(f.get(b'xxx', b'foo'), b'foo')
self.assertIsNone(f.get(b'xxx'))
with self.assertRaises(KeyError):
f[b'xxx']
def dumbdbm_test_db(request):
print("creating test dumbdbm file")
temp_file = tempfile.NamedTemporaryFile()
test_db = dumb.open(temp_file.name, "n")
test_db[key1] = val1
test_db[key2] = val2
test_db.close()
def delete_dumbdbm_test_db():
print("deleting test dumbdbm file")
temp_file.close()
os.remove(temp_file.name + ".dir")
os.remove(temp_file.name + ".bak")
os.remove(temp_file.name + ".dat")
request.addfinalizer(delete_dumbdbm_test_db)
return temp_file.name
def test_dumbdbm_read(self):
self.init_db()
f = dumbdbm.open(_fname, 'r')
self.read_helper(f)
with self.assertWarnsRegex(DeprecationWarning,
'The database is opened for reading only'):
f[b'g'] = b'x'
with self.assertWarnsRegex(DeprecationWarning,
'The database is opened for reading only'):
del f[b'a']
# get() works as in the dict interface
self.assertEqual(f.get(b'b'), self._dict[b'b'])
self.assertEqual(f.get(b'xxx', b'foo'), b'foo')
self.assertIsNone(f.get(b'xxx'))
with self.assertRaises(KeyError):
f[b'xxx']
f.close()
def test_dumbdbm_creation_mode(self):
try:
old_umask = os.umask(0o002)
f = dumbdbm.open(_fname, 'c', 0o637)
f.close()
finally:
os.umask(old_umask)
expected_mode = 0o635
if os.name != 'posix':
# Windows only supports setting the read-only attribute.
# This shouldn't fail, but doesn't work like Unix either.
expected_mode = 0o666
import stat
st = os.stat(_fname + '.dat')
self.assertEqual(stat.S_IMODE(st.st_mode), expected_mode)
st = os.stat(_fname + '.dir')
self.assertEqual(stat.S_IMODE(st.st_mode), expected_mode)
def test_check_closed(self):
f = dumbdbm.open(_fname, 'c')
f.close()
for meth in (partial(operator.delitem, f),
partial(operator.setitem, f, 'b'),
partial(operator.getitem, f),
partial(operator.contains, f)):
with self.assertRaises(dumbdbm.error) as cm:
meth('test')
self.assertEqual(str(cm.exception),
"DBM object has already been closed")
for meth in (operator.methodcaller('keys'),
operator.methodcaller('iterkeys'),
operator.methodcaller('items'),
len):
with self.assertRaises(dumbdbm.error) as cm:
meth(f)
self.assertEqual(str(cm.exception),
"DBM object has already been closed")
transformers = pickle.load(f)
# * Chunk IDs and process it chunk by chunk
if args.dev:
with shelve.open(str(path_db), 'r') as f:
ids = [index for index in f]
else:
ids = [str(i) for i in range(10000)]
# * Chunk can't be smaller than 1
n_chunks = max(1, len(ids)//chunksize)
id_chunks = np.array_split(ids, n_chunks)
# * Make database
print(get_time(), 'Creating database')
with dumbdbm.open(path_new_db, 'n') as f:
db_train = shelve.Shelf(f)
print(get_time(), 'Database created!')
# * Loop over chunks and save then one at a time
for i_chunk, id_chunk in enumerate(id_chunks):
print('')
print(get_time(), 'Processing chunk %d of %d'%(i_chunk+1, n_chunks))
# * For each chunk, first retrieve data on the n nearest neighbors.
n_nearest_data = get_n_nearest_data(str(path_db), id_chunk, geom_features, path_geom_dict, n_cpus=n_cpus)
# * Now transform all data
events = transform_events(str(path_db), id_chunk, feature_dicts, transformers, n_nearest_data, geom_features, n_cpus=n_cpus)
print(get_time(), 'Saving chunk %d of %d'%(i_chunk+1, n_chunks))
with shelve.open(path_new_db, 'w') as db:
def test_str_read(self):
self.init_db()
f = dumbdbm.open(_fname, 'r')
self.assertEqual(f['\u00fc'], self._dict['\u00fc'.encode('utf-8')])
f.close()
def test_dumbdbm_modification(self):
self.init_db()
f = dumbdbm.open(_fname, 'w')
self._dict[b'g'] = f[b'g'] = b"indented"
self.read_helper(f)
f.close()
def init_db(self):
f = dumbdbm.open(_fname, 'w')
for k in self._dict:
f[k] = self._dict[k]
f.close()
def test_open_with_pathlib_bytes_path(self):
dumbdbm.open(os_helper.FakePath(os.fsencode(_fname)), "c").close()
def __init__(self, datadir):
self.txDB = dbmd.open(datadir + "/transactions", 'c')
def test_write_contains(self):
with contextlib.closing(dumbdbm.open(_fname)) as f:
f[b'1'] = b'hello'
self.assertIn(b'1', f)
def test_write_contains(self):
f = dumbdbm.open(_fname)
f[b'1'] = b'hello'
self.assertIn(b'1', f)
f.close()
def test_invalid_flag(self):
for flag in ('x', 'rf', None):
with self.assertRaisesRegex(
ValueError, "Flag must be one of "
"'r', 'w', 'c', or 'n'"):
dumbdbm.open(_fname, flag)
def test_open_with_bytes_path(self):
dumbdbm.open(os.fsencode(_fname), "c").close()
def test_dumbdbm_creation(self):
with contextlib.closing(dumbdbm.open(_fname, 'c')) as f:
self.assertEqual(list(f.keys()), [])
for key in self._dict:
f[key] = self._dict[key]
self.read_helper(f)
def test_close_twice(self):
f = dumbdbm.open(_fname)
f[b'a'] = b'b'
self.assertEqual(f[b'a'], b'b')
f.close()
f.close()
def __init__(self, datadir):
self.blockDB = dbmd.open(datadir + "/blocks", 'c')
self.currentBlock = 0
self.headers_map = dict()
#!/usr/bin/env python3
from bottle import run, request, response, Bottle, static_file
import simplejson as json
import random
import bs4
import dbm.dumb as dbm
from collections import namedtuple, Counter
from AutoTranslateCommon import transmap_filename
from XLIFFReader import parse_xliff_file
db = dbm.open("kagame.dbm", flag='c')
app = Bottle()
Translation = namedtuple("Translation", ["id", "source", "target"])
availableStrings = None
fileid = None
targetlang = ""
stringIDMap = {} # ID => string
# Client vote counter
clientVotes = Counter()
def extract_strings_from_xliff_soup(filename, soup):
"""
Remove both untranslated and notes from the given soup.
For the untranslated elements, in
"""
results = []
global fileid
def test_dumbdbm_keys(self):
self.init_db()
f = dumbdbm.open(_fname)
keys = self.keys_helper(f)
f.close()
def test_create_new(self):
with dumbdbm.open(_fname, 'n') as f:
for k in self._dict:
f[k] = self._dict[k]
with dumbdbm.open(_fname, 'n') as f:
self.assertEqual(f.keys(), [])
def test_dumbdbm_read(self):
self.init_db()
f = dumbdbm.open(_fname, 'r')
self.read_helper(f)
f.close()
def test_str_read(self):
self.init_db()
with contextlib.closing(dumbdbm.open(_fname, 'r')) as f:
self.assertEqual(f['\u00fc'], self._dict['\u00fc'.encode('utf-8')])
def test_dumbdbm_keys(self):
self.init_db()
with contextlib.closing(dumbdbm.open(_fname)) as f:
keys = self.keys_helper(f)
def init_db(self):
with contextlib.closing(dumbdbm.open(_fname, 'n')) as f:
for k in self._dict:
f[k] = self._dict[k]
def test_open_with_pathlib_path(self):
dumbdbm.open(os_helper.FakePath(_fname), "c").close()