def test_updating_image(self):
IMAGE = {
'imageID': 'name-page',
'documentURL': 'http://localhist:8080/whatever',
'width': 1000,
'height': 2000
}
tmp = tempfile.NamedTemporaryFile(suffix='db', prefix='tmp-mdlg')
with PersistMandlagore(tmp.name) as db:
version = db.ensure_schema(rebuilt=True)
self.assertIsNotNone(version)
db.ensure_images([IMAGE])
data = db.retrieve_image(IMAGE['imageID'])
self.assertTrue(len(data), 1)
self.assertEqual(data, IMAGE)
IMAGE_UPDATE = {'imageID': 'name-page', 'width': 5, 'height': 40}
db.update_images([IMAGE_UPDATE])
IMAGE_UPDATED = IMAGE
IMAGE_UPDATED.update(IMAGE_UPDATE)
data = db.retrieve_image(IMAGE['imageID'])
self.assertTrue(len(data), 1)
self.assertEqual(data, IMAGE_UPDATED)
def test_bnf_dumps(self):
queries = {
"SELECT * FROM classes": True,
"SELECT * FROM classes WHERE classID = 'biche'": True,
"SELECT * FROM classes WHERE classID = 'abeille'": False,
"SELECT * FROM images WHERE imageID = '7823277-1'": True,
"SELECT * FROM images WHERE imageID = '52504824-151'": True,
"SELECT * FROM scenes WHERE mandragoreID = '52957'": True,
"SELECT * FROM scenes WHERE mandragoreID = 181563 AND imageID = '10308906-65'": True,
"SELECT * FROM scenes WHERE mandragoreID = 40999 AND imageID = '10500687-147'": False,
"SELECT * FROM descriptors WHERE classID = 'taureau'": True,
"SELECT * FROM descriptors WHERE classID = 'taureau' AND mandragoreID = 10005": True,
"SELECT * FROM descriptors WHERE classID = 'oreilles' AND mandragoreID = 10005": False,
}
with tempfile.TemporaryDirectory() as tmpdir:
self.prepare_data(tmpdir, TEST_BNF_FILES)
with PersistMandlagore(os.path.join(tmpdir, 'mdlg-test.db')) as db:
db.ensure_schema(True)
mng = MandragoreDumpManager(tmpdir, db)
# now try to import the files
mng.load_bnf_data()
self.verify_db_content(db, queries)
def reset(mdlgenv: MdlgEnv):
# ensure the DB file is wiped-out
pathdb = mdlgenv.db_filename()
if os.path.exists(pathdb):
if not os.path.isfile(pathdb):
raise Exception("the root dir provided contains already a dir named {} - it should be a simple file".format(pathdb))
os.remove(pathdb)
with PersistMandlagore(pathdb) as db:
version = db.ensure_schema(rebuilt=True)
click.echo("Mdlg DB rebuilt on file : {} - version {}".format(mdlgenv.db_filename(), version))
def galactica(mdlgenv: MdlgEnv, images, scenes, descriptors, limit, dryrun, faked):
# complete download informations from Galactica : images and size of images
# we should have filters:
# -all to download everything
# -scenes to complete only images involved in scenes that are located (have a defined position in the image)
# -size for size only
pathdb = mdlgenv.db_filename()
filter = [build_filter_from_option('images', f) for f in images]
filter += [build_filter_from_option('scenes', f) for f in scenes]
filter += [build_filter_from_option('descriptors', f) for f in descriptors]
with PersistMandlagore(pathdb) as db, GalacticaSession() as gal:
imgr = ImagesManager(mdlgenv.source_images_galactica_dirname(), db, gal)
imgr.ensure_content_images(filter, limit, dryrun, faked)
def mandragore(mdlgenv: MdlgEnv):
# load last downloaded dumps in the DB
pathdb = mdlgenv.db_filename()
with PersistMandlagore(pathdb) as db:
dname = mdlgenv.dump_data_dirname()
mng = MandragoreDumpManager(dname, db)
report = mng.load_bnf_data()
click.echo("Mdlg Mandragore dump files loaded into the DB")
for file, report, warnings in report:
click.echo("%s : %s" % (file, report))
if len(warnings) > 0:
click.echo("%d warning reported, 10 first warnings below:" % len(warnings))
for w in warnings[:10]:
click.echo(" -- W -- %s" % w)
def labels(mdlgenv: MdlgEnv):
# load labels frol all the VIA annotation files available in the import folder
pathdb = mdlgenv.db_filename()
with PersistMandlagore(pathdb) as db, GalacticaSession() as gal:
dname = mdlgenv.via_annotation_dirname()
vlm = ViaLabelManager(dname, db, gal)
report = vlm.import_labels()
click.echo("Mdlg labels (VIA annotations) imported into the DB")
for file, warnings, report in report:
click.echo("%s : %s" % (file, report))
if len(warnings) > 0:
click.echo("%d warning reported at preparation, 10 first warnings below:" % len(warnings))
for w in warnings[:10]:
click.echo(" -- W -- %s" % w)
def test_build_filtered_query(self):
TEST = {
'one-query-simple-where':
[(('images', 'ID', '"an-of-an-image"'), ), False],
'dbl-join-query-simple-where-localized':
[(('scenes', 'localized', None),
('images', 'ID:like', '"%images-names"'), (
'descriptors',
'ID:list',
(('"m-one"', '"m-two"'), ('"class-two"', '"class-one"')),
)), False],
'one-query-no-where': [(), True],
'one-query-simple-where-field': [[('images', 'height', 14)],
False],
'one-query-simple-where-like':
[(('images', 'ID:like', '"%images-names"'), ), False],
'one-query-simple-where-list':
[(('images', 'ID:list', ['"id-1"', '"id-2"', '"id-3"']), ), False],
'one-query-simple-where-localized': [(('images', 'localized',
None), ), False],
'sev-query-simple-where-localized': [
(('images', 'localized', None), ('images', 'ID:like',
['"%images-names"'])), False
],
'join-query-simple-where-localized': [(('scenes', 'localized',
None), (
'images',
'ID:like',
('"%images-names"'),
)), False],
}
tmp = tempfile.NamedTemporaryFile(suffix='db', prefix='tmp-mdlg')
with PersistMandlagore(tmp.name) as db:
version = db.ensure_schema(rebuilt=True)
self.assertIsNotNone(version)
for k, ts in TEST.items():
sql = SQLBuilder.build_filtered_query('images',
'*',
ts[0],
limit=None)
try:
db.conn.execute(sql)
except Exception as e:
self.fail("%s - query %s failed : %s " % (k, sql, str(e)))
def test_reorganized_dumps(self):
queries = {
"SELECT * FROM classes": True,
"SELECT * FROM classes WHERE classID = 'abeille'": True,
"SELECT * FROM classes WHERE classID = 'lion'": False,
"SELECT * FROM images WHERE imageID = '7842457-1'": True,
"SELECT * FROM images WHERE imageID = '52504824-10'": True,
"SELECT * FROM scenes WHERE mandragoreID = '51326'": True,
"SELECT * FROM scenes WHERE mandragoreID = 40999 AND imageID = '52504824-10'": True,
"SELECT * FROM scenes WHERE mandragoreID = 40999 AND imageID = '7842457-1'": False,
"SELECT * FROM descriptors WHERE classID = 'abeille'": True,
"SELECT * FROM descriptors WHERE classID = 'abeille' AND mandragoreID = 51125": True,
"SELECT * FROM descriptors WHERE classID = 'oreilles' AND mandragoreID = 51125": False,
}
with tempfile.TemporaryDirectory() as tmpdir:
self.prepare_data(tmpdir, TEST_FILES)
with PersistMandlagore(os.path.join(tmpdir, 'mdlg-test.db')) as db:
db.ensure_schema(True)
mng = MandragoreDumpManager(tmpdir, db)
# now try to import the files
mng.load_basic_data()
self.verify_db_content(db, queries)
def test_creating_db(self):
tmp = tempfile.NamedTemporaryFile(suffix='db', prefix='tmp-mdlg')
with PersistMandlagore(tmp.name) as db:
version = db.ensure_schema(rebuilt=True)
self.assertIsNotNone(version)