def cmd_lock(args):
if not args.quiet:
print("Locking database {0}".format(args.database))
if not args.dry_run:
database = Database.instance(args.database)
mapper = DatabaseMapper(database)
mapper.set_destroy_lock(True)
class LockExpirer(object): """ Expires locks in the database """ def __init__(self, database): self._dbmapper = DatabaseMapper(Database.instance(database)) def run(self): self._dbmapper.expire_locks()
def main():
training_file = open('training.txt','w')
testing_file = open('testing.txt','w')
training_images = {'blur':list(), 'noblur':list()}
testing_images = {'blur':list(), 'noblur':list()}
parser = argparse.ArgumentParser(description='generates training/testing files for blur')
parser.add_argument('-l', '--limit', type=int, metavar='COUNT', required=False, help='Maximum number of images to use')
parser.add_argument('-r', '--random', action="store_true", default=False, required=False, help='Fetch images ordered randomly if limit is active')
parser.add_argument('--tag_require', action='append', dest='tags_require', default=None, required=False, help='Tag that must be present on selected images')
parser.add_argument('--tag_exclude', action='append', dest='tags_exclude', default=None, required=False, help='Tag that must not be present on selected images')
parser.add_argument('-p', '--percent_training', dest='percent', default=0.25, required=False, help='Tag indicating what percent of images for training')
parser.add_argument('database', help='Name of database to use')
args = parser.parse_args()
db = Database.instance(args.database)
db_mapper = DatabaseMapper(db)
images = db_mapper.get_images_for_analysis(kDomain, limit=args.limit, random=args.random, tags_require=args.tags_require, tags_exclude=args.tags_exclude)
blur_images = list()
noblur_images = list()
for image in images:
if image['annotations'][0]['model'] == 'blur':
blur_images.append(image)
else:
noblur_images.append(image)
random.shuffle(blur_images)
random.shuffle(noblur_images)
blur_training_len = int(len(blur_images)*float(args.percent))
noblur_training_len = int(len(noblur_images)*float(args.percent))
training_images['blur'] = blur_images[:blur_training_len]
testing_images['blur'] = blur_images[blur_training_len:]
training_images['noblur'] = noblur_images[:noblur_training_len]
testing_images['noblur'] = noblur_images[noblur_training_len:]
for file,image_dict in ((training_file,training_images),(testing_file,testing_images)):
for model in image_dict.keys():
for image in image_dict[model]:
file.write('{}\t{}\n'.format(rigor.imageops.find(image), model))
def main():
parser = argparse.ArgumentParser(
description='Runs text detector on relevant images')
parser.add_argument('classifier_file', help='Path to classifier CLF')
parser.add_argument('-l',
'--limit',
type=int,
metavar='COUNT',
required=False,
help='Maximum number of images to use')
parser.add_argument(
'-r',
'--random',
action="store_true",
default=False,
required=False,
help='Fetch images ordered randomly if limit is active')
parser.add_argument('database', help='Database to use')
args = parser.parse_args()
parameters["classifier_file"] = args.classifier_file
i = rigor.runner.Runner('text',
parameters,
limit=args.limit,
random=args.random)
database_mapper = DatabaseMapper(Database.instance(args.database))
for result in i.run():
detected = result[1]
expected = result[2]
image = database_mapper.get_image_by_id(result[0])
cv_image = rigor.imageops.fetch(image)
cv2.polylines(cv_image, expected, True, cv2.RGB(0, 255, 0))
cv2.polylines(cv_image, detected, True, cv2.RGB(255, 255, 0))
cv2.imwrite(".".join((str(image["id"]), image["format"])), cv_image)
def cmd_unlock(args):
if not args.quiet:
print("Unlocking database {0}".format(args.database))
if not args.dry_run:
database = Database.instance(args.database)
mapper = DatabaseMapper(database)
mapper.set_destroy_lock(False)
def cmd_clone(args):
if not args.quiet:
print("Cloning database {0} to {1}".format(args.source, args.destination))
if not args.dry_run:
Database.cls().clone(args.source, args.destination)
database = Database.instance(args.destination)
mapper = DatabaseMapper(database)
mapper.set_destroy_lock(False) # new databases are always unlocked
def cmd_patch(args): database = Database.instance(args.database) mapper = DatabaseMapper(database) start_level = mapper.get_patch_level() + 1 stop_level = None if args.level: stop_level = args.level patch(mapper, args.patch_dir, start_level, stop_level, args.dry_run, args.quiet)
def cmd_clone(args):
if not args.quiet:
print("Cloning database {0} to {1}".format(args.source,
args.destination))
if not args.dry_run:
Database.cls().clone(args.source, args.destination)
database = Database.instance(args.destination)
mapper = DatabaseMapper(database)
mapper.set_destroy_lock(False) # new databases are always unlocked
def cmd_patch(args):
database = Database.instance(args.database)
mapper = DatabaseMapper(database)
start_level = mapper.get_patch_level() + 1
stop_level = None
if args.level:
stop_level = args.level
patch(mapper, args.patch_dir, start_level, stop_level, args.dry_run,
args.quiet)
def cmd_destroy(args):
if not args.quiet:
print("Destroying database {0}".format(args.database))
database = Database.instance(args.database)
mapper = DatabaseMapper(database)
if mapper.get_destroy_lock():
sys.stderr.write("Error: database is locked\n")
sys.exit(2)
mapper = None
database = None
if not args.dry_run:
Database.cls().drop(args.database)
def cmd_destroy(args):
if not args.quiet:
print("Destroying database {0}".format(args.database))
database = Database.instance(args.database)
mapper = DatabaseMapper(database)
if mapper.get_destroy_lock():
sys.stderr.write("Error: database is locked\n")
sys.exit(2)
mapper = None
database = None
if not args.dry_run:
Database.cls().drop(args.database)
def __init__(self, directory, database, move=False):
"""
directory is which directory to scan for files; move is whether to move
files to the repository, as opposed to just copying them
"""
self._directory = directory
self._logger = rigor.logger.get_logger('.'.join((__name__, self.__class__.__name__)))
self._move = move
self._metadata = dict()
self._database = rigor.database.Database.instance(database)
self._database_mapper = DatabaseMapper(self._database)
try:
import pyexiv2 # pylint: disable=W0612
except ImportError:
self._logger.warning("Unable to import pyexiv2; sensor data should be supplied in metadata")
os.umask(002)
def main():
parser = argparse.ArgumentParser(description='Runs text detector on relevant images')
parser.add_argument('classifier_file', help='Path to classifier CLF')
parser.add_argument('-l', '--limit', type=int, metavar='COUNT', required=False, help='Maximum number of images to use')
parser.add_argument('-r', '--random', action="store_true", default=False, required=False, help='Fetch images ordered randomly if limit is active')
parser.add_argument('database', help='Database to use')
args = parser.parse_args()
parameters["classifier_file"] = args.classifier_file
i = rigor.runner.Runner('text', parameters, limit=args.limit, random=args.random)
database_mapper = DatabaseMapper(Database.instance(args.database))
for result in i.run():
detected = result[1]
expected = result[2]
image = database_mapper.get_image_by_id(result[0])
cv_image = rigor.imageops.fetch(image)
cv2.polylines(cv_image, expected, True, cv2.RGB(0, 255, 0))
cv2.polylines(cv_image, detected, True, cv2.RGB(255, 255, 0))
cv2.imwrite(".".join((str(image["id"]), image["format"])), cv_image)
def main():
rigor.domain.money.init(parameters)
logger = rigor.logger.getLogger(__file__)
database_mapper = DatabaseMapper(Database.instance(kDatabase))
logger.debug('Fetching image IDs from database')
images = database_mapper.get_images_for_analysis(kDomain, kLimit, False)
for parameter_set in get_parameters():
timestamp = datetime.utcnow().strftime("{0}-%Y%m%d_%H%M%S%f".format(kDomain))
with open("{0}.params".format(timestamp), "w") as parameter_file:
json.dump(parameter_set, parameter_file)
parameter_file.write("\n")
with open("{0}.results".format(timestamp), "w") as result_file:
image_config = partial(rigor.domain.money.run, parameters=parameter_set)
logger.debug('Processing {0} images'.format(len(images)))
for result in map(image_config, images):
result_file.write("\t".join([str(x) for x in result]))
result_file.write("\n")
def destroy_image(database, image):
""" Completely Removes an image from the database, along with its tags, annotations, and any anotation tags. Also deletes the image file from disk."""
mapper = DatabaseMapper(database)
with database.get_cursor() as cursor:
sql = "DELETE FROM tag WHERE image_id = %s;"
cursor.execute(sql, (image['id'],))
annotations = mapper._get_annotations_by_image_id(cursor, image['id']) # pylint: disable=W0212
for annotation in annotations:
sql = "DELETE FROM annotation_tag WHERE annotation_id = %s"
cursor.execute(sql, (annotation['id'],))
mapper._delete_annotation(cursor, annotation['id']) # pylint: disable=W0212
sql = "DELETE FROM image WHERE id = %s;"
cursor.execute(sql, (image['id'],))
image_path = find(image)
os.unlink(image_path)
for thumbnail_dir in os.listdir(config.get('thumbnail', 'image_repository')):
image_thumbnail_path = build_path(image, os.path.join(config.get('thumbnail', 'image_repository'), thumbnail_dir))
os.unlink(image_thumbnail_path)
def destroy_image(database, image):
""" Completely Removes an image from the database, along with its tags, annotations, and any anotation tags. Also deletes the image file from disk."""
mapper = DatabaseMapper(database)
with database.get_cursor() as cursor:
sql = "DELETE FROM tag WHERE image_id = %s;"
cursor.execute(sql, (image['id'], ))
annotations = mapper._get_annotations_by_image_id(cursor, image['id']) # pylint: disable=W0212
for annotation in annotations:
sql = "DELETE FROM annotation_tag WHERE annotation_id = %s"
cursor.execute(sql, (annotation['id'], ))
mapper._delete_annotation(cursor, annotation['id']) # pylint: disable=W0212
sql = "DELETE FROM image WHERE id = %s;"
cursor.execute(sql, (image['id'], ))
image_path = find(image)
os.unlink(image_path)
for thumbnail_dir in os.listdir(
config.get('thumbnail', 'image_repository')):
image_thumbnail_path = build_path(
image,
os.path.join(config.get('thumbnail', 'image_repository'),
thumbnail_dir))
os.unlink(image_thumbnail_path)
def cmd_create(args):
if not args.quiet:
print("Creating database {0}".format(args.database))
if not args.dry_run:
Database.cls().create(args.database)
stop_level = None
if args.level:
stop_level = args.level
try:
database = Database.instance(args.database)
mapper = DatabaseMapper(database)
patch(mapper, args.patch_dir, 0, stop_level, args.dry_run, True)
except:
# Save exception for later
exc_info = sys.exc_info()
try:
Database.cls().drop(args.database)
except:
pass
raise exc_info[0], exc_info[1], exc_info[2]
def main():
training_file = open('training.txt', 'w')
testing_file = open('testing.txt', 'w')
training_images = {'blur': list(), 'noblur': list()}
testing_images = {'blur': list(), 'noblur': list()}
parser = argparse.ArgumentParser(
description='generates training/testing files for blur')
parser.add_argument('-l',
'--limit',
type=int,
metavar='COUNT',
required=False,
help='Maximum number of images to use')
parser.add_argument(
'-r',
'--random',
action="store_true",
default=False,
required=False,
help='Fetch images ordered randomly if limit is active')
parser.add_argument('--tag_require',
action='append',
dest='tags_require',
default=None,
required=False,
help='Tag that must be present on selected images')
parser.add_argument('--tag_exclude',
action='append',
dest='tags_exclude',
default=None,
required=False,
help='Tag that must not be present on selected images')
parser.add_argument(
'-p',
'--percent_training',
dest='percent',
default=0.25,
required=False,
help='Tag indicating what percent of images for training')
parser.add_argument('database', help='Name of database to use')
args = parser.parse_args()
db = Database.instance(args.database)
db_mapper = DatabaseMapper(db)
images = db_mapper.get_images_for_analysis(kDomain,
limit=args.limit,
random=args.random,
tags_require=args.tags_require,
tags_exclude=args.tags_exclude)
blur_images = list()
noblur_images = list()
for image in images:
if image['annotations'][0]['model'] == 'blur':
blur_images.append(image)
else:
noblur_images.append(image)
random.shuffle(blur_images)
random.shuffle(noblur_images)
blur_training_len = int(len(blur_images) * float(args.percent))
noblur_training_len = int(len(noblur_images) * float(args.percent))
training_images['blur'] = blur_images[:blur_training_len]
testing_images['blur'] = blur_images[blur_training_len:]
training_images['noblur'] = noblur_images[:noblur_training_len]
testing_images['noblur'] = noblur_images[noblur_training_len:]
for file, image_dict in ((training_file, training_images),
(testing_file, testing_images)):
for model in image_dict.keys():
for image in image_dict[model]:
file.write('{}\t{}\n'.format(rigor.imageops.find(image),
model))
class Importer(object):
""" Class containing methods for importing images into the Rigor framework """
extensions = ('jpg', 'png')
modes = {'1': 1, 'L': 8, 'RGB': 24, 'RGBA': 32}
def __init__(self, directory, database, move=False):
"""
directory is which directory to scan for files; move is whether to move
files to the repository, as opposed to just copying them
"""
self._directory = directory
self._logger = rigor.logger.get_logger('.'.join((__name__, self.__class__.__name__)))
self._move = move
self._metadata = dict()
self._database = rigor.database.Database.instance(database)
self._database_mapper = DatabaseMapper(self._database)
try:
import pyexiv2 # pylint: disable=W0612
except ImportError:
self._logger.warning("Unable to import pyexiv2; sensor data should be supplied in metadata")
os.umask(002)
def run(self):
""" Imports all images from the directory, returning the number processed """
metadata = self._read_global_metadata()
for entry in os.listdir(self._directory):
absfile = os.path.abspath(os.path.join(self._directory, entry))
if not os.path.isfile(absfile):
# Not a file
continue
(basename, sep, extension) = entry.rpartition(os.extsep)
if not sep:
# No separating dot
self._logger.warn("Could not find separator for {0}".format(entry))
continue
if extension.lower() not in self.extensions:
# Extension not in known list
continue
# Looks like we have an image file
self.import_image(absfile, basename, metadata)
def import_image(self, path, basename, metadata):
""" Reads the metadata for an invididual image and returns an object ready to insert """
image = dict()
image['locator'] = uuid.uuid4().hex
image['hash'] = rigor.hash.sha1_hash(path)
data = rigor.imageops.read(path)
image['resolution'] = (data.shape[1], data.shape[0])
image['format'] = imghdr.what(path)
if not image['format']:
image['format'] = 'jpeg' # TODO: why is imghdr.what returning None for /home/kaolin/data/ICDAR2005SceneCompetition/wltang_15.08.2002/IMG_0001.JPG
if len(data.shape) == 2:
image['depth'] = 8
else:
image['depth'] = data.shape[2] * 8
if not metadata:
metadata = {};
new_metadata = metadata.copy()
new_metadata.update(self._read_local_metadata(basename))
if 'timestamp' in new_metadata:
image['stamp'] = datetime.strptime(new_metadata['timestamp'], config.get('import', 'timestamp_format'))
else:
image['stamp'] = datetime.utcfromtimestamp(os.path.getmtime(path))
if 'source_id' in new_metadata:
image['source_id'] = new_metadata['source_id']
else:
image['source_id'] = None
image['tags'] = list()
if 'tags' in new_metadata:
image['tags'] = new_metadata['tags']
has_sensor = False
for tag in image['tags']:
if tag.startswith('sensor:'):
has_sensor = True
break
if not has_sensor:
try:
exif = pyexiv2.ImageMetadata(path) # pylint: disable=E0602
exif.read()
if 'Exif.Image.Make' in exif and 'Exif.Image.Model' in exif:
sensor_tag = 'sensor:' + '_'.join((exif['Exif.Image.Make'].value, exif['Exif.Image.Model'].value))
image['tags'] += sensor_tag.lower()
except NameError:
self._logger.warning("Image at {0}: No sensor listed in metadata, and EXIF data is not available.".format(path))
if 'location' in new_metadata:
image['location'] = new_metadata['location']
else:
image['location'] = None
annotations = list()
if 'annotations' in new_metadata:
for annotation in new_metadata['annotations']:
new_annotations = dict()
for key in ('boundary', 'domain', 'model', 'confidence', 'annotation_tags'):
if key in annotation:
new_annotations[key] = annotation[key]
else:
new_annotations[key] = None
if 'timestamp' in annotation:
new_annotations['stamp'] = datetime.strptime(annotation['timestamp'], config.get('import', 'timestamp_format'))
else:
new_annotations['stamp'] = image['stamp']
annotations.append(new_annotations)
image['annotations'] = annotations
destination = os.path.join(config.get('import', 'upload_repository'), image['locator'][0:2], image['locator'][2:4], os.extsep.join((image['locator'], image['format'])))
# We take control of the transaction here so we can fail if copying/moving the file fails
try:
with self._database_mapper._db.get_cursor() as cursor: # pylint: disable=W0212
self._database_mapper._create_image(cursor, image) # pylint: disable=W0212
# Create destination directory, if it doesn't already exist. try/except
# structure avoids race condition
try:
os.makedirs(os.path.dirname(destination))
except OSError as err:
if err.errno == errno.EEXIST:
pass
else:
raise
if self._move:
shutil.move(path, destination)
else:
shutil.copy2(path, destination)
os.chmod(destination, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH)
self._logger.info("Imported image {0}".format(image['locator']))
return image
except rigor.database.IntegrityError:
self._logger.warn("The image at '{0}' already exists in the database".format(path))
def _read_local_metadata(self, basename):
""" Reads the metadata file for the image and sets defaults """
metadata_file = os.path.join(self._directory, "{0}.json".format(basename))
if not os.path.isfile(metadata_file):
return dict()
with open(metadata_file, 'r') as metadata:
return json.load(metadata)
def _read_global_metadata(self):
""" Reads the metadata file for the image directory and sets defaults """
metadata_file = os.path.join(self._directory, config.get('import', 'metadata_file'))
if not os.path.isfile(metadata_file):
return
with open(metadata_file, 'r') as metadata:
return json.load(metadata)
def __init__(self, database): self._dbmapper = DatabaseMapper(Database.instance(database))
import rigor.database
import rigor.imageops
from rigor.dbmapper import DatabaseMapper
from rigor.config import config
import argparse
parser = argparse.ArgumentParser(description="Tool to delete ICDAR images")
parser.add_argument("database", help="Database to connect to")
args = parser.parse_args()
gDb = rigor.database.Database.instance(args.database)
gDbMapper = DatabaseMapper(gDb)
def main():
source_to_delete = 'ICDAR2003'
images = gDbMapper.get_images_by_source(source_to_delete)
print('Found {0} images'.format(len(images)))
for image in images:
print('testing: not deleteing image id {0}'.format(image['id']))
#rigor.imageops.destroy_image(gDb, image)
if __name__ == "__main__":
main()
""""
Script to delete ground truth (image, thumbnail, and all!)
"""
import argparse
import rigor.imageops
from rigor.dbmapper import DatabaseMapper
from rigor.database import Database
parser = argparse.ArgumentParser(description='Deletes ground truth (image, thumbnail, and all!)')
parser.add_argument('database', help='Name of database to use')
parser.add_argument('delete_ids', metavar='delete_id', nargs='+', type=int, help='ID(s) of images to delete')
args = parser.parse_args()
db = Database.instance(args.database)
db_mapper = DatabaseMapper(db)
for image_id in args.delete_ids:
image = db_mapper.get_image_by_id(image_id)
print("OBLITERATING {}".format(image['id']))
rigor.imageops.destroy_image(db, image)
""""
Script to delete ground truth (image, thumbnail, and all!)
"""
import argparse
import rigor.imageops
from rigor.dbmapper import DatabaseMapper
from rigor.database import Database
parser = argparse.ArgumentParser(
description='Deletes ground truth (image, thumbnail, and all!)')
parser.add_argument('database', help='Name of database to use')
parser.add_argument('delete_ids',
metavar='delete_id',
nargs='+',
type=int,
help='ID(s) of images to delete')
args = parser.parse_args()
db = Database.instance(args.database)
db_mapper = DatabaseMapper(db)
for image_id in args.delete_ids:
image = db_mapper.get_image_by_id(image_id)
print("OBLITERATING {}".format(image['id']))
rigor.imageops.destroy_image(db, image)