def __init__(self, database):
self._convert = os.path.abspath(config.get('thumbnail',
'convert_path'))
self._lock_file = os.path.abspath(
config.get('thumbnail', 'lock_file') + '-' + database)
self._size = int(config.get('thumbnail', 'image_size'))
self._database = rigor.database.Database.instance(database)
os.umask(002)
def __init__(self, move=True):
self._source = os.path.abspath(config.get('import', 'upload_repository')) + os.sep
self._destination = os.path.abspath(config.get('global', 'image_repository')) + os.sep
self._move = move
rsync = os.path.abspath(config.get('import', 'rsync_path'))
args = [rsync, '-q', '-r', '-p', '--chmod=ugo+r,Dugo+x,ug+w', '-t', '-O']
if move:
args.append('--remove-source-files')
args.append(self._source)
args.append(self._destination)
self._args = args
os.umask(002)
def find_thumbnail(image, size):
"""
Given a base path, constructs a path to the image's thumbnail
"""
return build_thumbnail_path(image,
config.get('thumbnail', 'image_repository'),
size)
def _read_global_metadata(self):
""" Reads the metadata file for the image directory and sets defaults """
metadata_file = os.path.join(self._directory, config.get('import', 'metadata_file'))
if not os.path.isfile(metadata_file):
return
with open(metadata_file, 'r') as metadata:
return json.load(metadata)
def destroy_image(database, image):
""" Completely Removes an image from the database, along with its tags, annotations, and any anotation tags. Also deletes the image file from disk."""
mapper = DatabaseMapper(database)
with database.get_cursor() as cursor:
sql = "DELETE FROM tag WHERE image_id = %s;"
cursor.execute(sql, (image['id'],))
annotations = mapper._get_annotations_by_image_id(cursor, image['id']) # pylint: disable=W0212
for annotation in annotations:
sql = "DELETE FROM annotation_tag WHERE annotation_id = %s"
cursor.execute(sql, (annotation['id'],))
mapper._delete_annotation(cursor, annotation['id']) # pylint: disable=W0212
sql = "DELETE FROM image WHERE id = %s;"
cursor.execute(sql, (image['id'],))
image_path = find(image)
os.unlink(image_path)
for thumbnail_dir in os.listdir(config.get('thumbnail', 'image_repository')):
image_thumbnail_path = build_path(image, os.path.join(config.get('thumbnail', 'image_repository'), thumbnail_dir))
os.unlink(image_thumbnail_path)
def build_dsn(database):
""" Builds the database connection string from config values """
dsn = "dbname='{0}' host='{1}'".format(database, config.get('database', 'host'))
try:
ssl = config.getboolean('database', 'ssl')
if ssl:
dsn += " sslmode='require'"
except ConfigParser.Error:
pass
try:
username = config.get('database', 'username')
dsn += " user='******'".format(username)
except ConfigParser.Error:
pass
try:
password = config.get('database', 'password')
dsn += " password='******'".format(password)
except ConfigParser.Error:
pass
return dsn
def build_dsn(database):
""" Builds the database connection parameters from config values """
dsn = {'database': database, 'host': config.get('database', 'host')}
try:
ssl = config.getboolean('database', 'ssl')
if ssl:
dsn['ssl'] = ssl
except ConfigParser.Error:
pass
try:
username = config.get('database', 'username')
dsn['user'] = username
except ConfigParser.Error:
pass
try:
password = config.get('database', 'password')
dsn['password'] = password
except ConfigParser.Error:
pass
return dsn
def __init__(self, algorithm, domain, arguments=None):
"""
The domain dictates which images to use as sources. The limit is an
optional maximum number of images to use as sources. If random, images
will be pulled in random order, up to limit; otherwise, images will be
pulled in sequential order. Tags are used to control the image selection
further.
"""
DatabaseRunner.__init__(self, algorithm, arguments)
self._domain = domain
self._image_id = None
if kMaxWorkers > 1:
self._pool = Pool(int(config.get('global', 'max_workers')))
def destroy_image(database, image):
""" Completely Removes an image from the database, along with its tags, annotations, and any anotation tags. Also deletes the image file from disk."""
mapper = DatabaseMapper(database)
with database.get_cursor() as cursor:
sql = "DELETE FROM tag WHERE image_id = %s;"
cursor.execute(sql, (image['id'], ))
annotations = mapper._get_annotations_by_image_id(cursor, image['id']) # pylint: disable=W0212
for annotation in annotations:
sql = "DELETE FROM annotation_tag WHERE annotation_id = %s"
cursor.execute(sql, (annotation['id'], ))
mapper._delete_annotation(cursor, annotation['id']) # pylint: disable=W0212
sql = "DELETE FROM image WHERE id = %s;"
cursor.execute(sql, (image['id'], ))
image_path = find(image)
os.unlink(image_path)
for thumbnail_dir in os.listdir(
config.get('thumbnail', 'image_repository')):
image_thumbnail_path = build_path(
image,
os.path.join(config.get('thumbnail', 'image_repository'),
thumbnail_dir))
os.unlink(image_thumbnail_path)
def main():
rigor.domain.money.init(parameters)
logger = rigor.logger.getLogger(__file__)
database_mapper = DatabaseMapper(Database.instance(kDatabase))
pool = Pool(int(config.get('global', 'max_workers')))
logger.debug('Fetching image IDs from database')
images = database_mapper.get_images_for_analysis(kDomain, kLimit, False)
for parameter_set in get_parameters():
timestamp = datetime.utcnow().strftime("{0}-%Y%m%d_%H%M%S%f".format(kDomain))
with open("{0}.params".format(timestamp), "w") as parameter_file:
json.dump(parameter_set, parameter_file)
parameter_file.write("\n")
with open("{0}.results".format(timestamp), "w") as result_file:
image_config = partial(rigor.domain.money.run, parameters=parameter_set)
logger.debug('Processing {0} images'.format(len(images)))
for result in pool.map(image_config, images):
result_file.write("\t".join([str(x) for x in result]))
result_file.write("\n")
def main():
rigor.domain.money.init(parameters)
logger = rigor.logger.getLogger(__file__)
database_mapper = DatabaseMapper(Database.instance(kDatabase))
pool = Pool(int(config.get('global', 'max_workers')))
logger.debug('Fetching image IDs from database')
images = database_mapper.get_images_for_analysis(kDomain, kLimit, False)
for parameter_set in get_parameters():
timestamp = datetime.utcnow().strftime(
"{0}-%Y%m%d_%H%M%S%f".format(kDomain))
with open("{0}.params".format(timestamp), "w") as parameter_file:
json.dump(parameter_set, parameter_file)
parameter_file.write("\n")
with open("{0}.results".format(timestamp), "w") as result_file:
image_config = partial(rigor.domain.money.run,
parameters=parameter_set)
logger.debug('Processing {0} images'.format(len(images)))
for result in pool.map(image_config, images):
result_file.write("\t".join([str(x) for x in result]))
result_file.write("\n")
def __init__(self, database):
self._convert = os.path.abspath(config.get('thumbnail', 'convert_path'))
self._lock_file = os.path.abspath(config.get('thumbnail', 'lock_file') + '-' + database)
self._size = int(config.get('thumbnail', 'image_size'))
self._database = rigor.database.Database.instance(database)
os.umask(002)
def import_image(self, path, basename, metadata):
""" Reads the metadata for an invididual image and returns an object ready to insert """
image = dict()
image['locator'] = uuid.uuid4().hex
image['hash'] = rigor.hash.sha1_hash(path)
data = rigor.imageops.read(path)
image['resolution'] = (data.shape[1], data.shape[0])
image['format'] = imghdr.what(path)
if not image['format']:
image['format'] = 'jpeg' # TODO: why is imghdr.what returning None for /home/kaolin/data/ICDAR2005SceneCompetition/wltang_15.08.2002/IMG_0001.JPG
if len(data.shape) == 2:
image['depth'] = 8
else:
image['depth'] = data.shape[2] * 8
if not metadata:
metadata = {};
new_metadata = metadata.copy()
new_metadata.update(self._read_local_metadata(basename))
if 'timestamp' in new_metadata:
image['stamp'] = datetime.strptime(new_metadata['timestamp'], config.get('import', 'timestamp_format'))
else:
image['stamp'] = datetime.utcfromtimestamp(os.path.getmtime(path))
if 'source_id' in new_metadata:
image['source_id'] = new_metadata['source_id']
else:
image['source_id'] = None
image['tags'] = list()
if 'tags' in new_metadata:
image['tags'] = new_metadata['tags']
has_sensor = False
for tag in image['tags']:
if tag.startswith('sensor:'):
has_sensor = True
break
if not has_sensor:
try:
exif = pyexiv2.ImageMetadata(path) # pylint: disable=E0602
exif.read()
if 'Exif.Image.Make' in exif and 'Exif.Image.Model' in exif:
sensor_tag = 'sensor:' + '_'.join((exif['Exif.Image.Make'].value, exif['Exif.Image.Model'].value))
image['tags'] += sensor_tag.lower()
except NameError:
self._logger.warning("Image at {0}: No sensor listed in metadata, and EXIF data is not available.".format(path))
if 'location' in new_metadata:
image['location'] = new_metadata['location']
else:
image['location'] = None
annotations = list()
if 'annotations' in new_metadata:
for annotation in new_metadata['annotations']:
new_annotations = dict()
for key in ('boundary', 'domain', 'model', 'confidence', 'annotation_tags'):
if key in annotation:
new_annotations[key] = annotation[key]
else:
new_annotations[key] = None
if 'timestamp' in annotation:
new_annotations['stamp'] = datetime.strptime(annotation['timestamp'], config.get('import', 'timestamp_format'))
else:
new_annotations['stamp'] = image['stamp']
annotations.append(new_annotations)
image['annotations'] = annotations
destination = os.path.join(config.get('import', 'upload_repository'), image['locator'][0:2], image['locator'][2:4], os.extsep.join((image['locator'], image['format'])))
# We take control of the transaction here so we can fail if copying/moving the file fails
try:
with self._database_mapper._db.get_cursor() as cursor: # pylint: disable=W0212
self._database_mapper._create_image(cursor, image) # pylint: disable=W0212
# Create destination directory, if it doesn't already exist. try/except
# structure avoids race condition
try:
os.makedirs(os.path.dirname(destination))
except OSError as err:
if err.errno == errno.EEXIST:
pass
else:
raise
if self._move:
shutil.move(path, destination)
else:
shutil.copy2(path, destination)
os.chmod(destination, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH)
self._logger.info("Imported image {0}".format(image['locator']))
return image
except rigor.database.IntegrityError:
self._logger.warn("The image at '{0}' already exists in the database".format(path))
def cls():
adapter_name = config.get('database', 'adapter')
adapter = __import__("rigor.adapters.{}_adapter".format(adapter_name), fromlist=["rigor.adapters", ])
return adapter.Database
def find(image):
"""
Returns the location in the repository where this image can be found.
"""
return build_path(image, config.get('global', 'image_repository'))
def find(image):
"""
Returns the location in the repository where this image can be found.
"""
return build_path(image, config.get('global', 'image_repository'))
"""
Runs an algorithm across a set of images. Result is a report containing image
ID, detected model, expected model, and elapsed time
"""
from rigor.config import config
import rigor.logger
import rigor.database
import rigor.dbmapper
import abc
import json
import argparse
kMaxWorkers = int(config.get('global', 'max_workers'))
if kMaxWorkers > 1:
from multiprocessing.pool import Pool
class BaseRunner(object):
"""
The base class for Runner objects, which fetch a set of images and apply
the Algorithm instance to each
"""
__metaclass__ = abc.ABCMeta
def __init__(self, algorithm, arguments=None):
self._logger = rigor.logger.get_logger('.'.join((__name__, self.__class__.__name__)))
self._algorithm = algorithm
self._parameters = None
self._arguments = None
self.parse_arguments(arguments)
def __init__(self, database):
super(Database, self).__init__(database)
register_type(psycopg2.extensions.UNICODE)
register_uuid()
dsn = Database.build_dsn(database)
self._pool = ThreadedConnectionPool(config.get('database', 'min_database_connections'), config.get('database', 'max_database_connections'), dsn)
def find_thumbnail(image, size):
"""
Given a base path, constructs a path to the image's thumbnail
"""
return build_thumbnail_path(image, config.get('thumbnail', 'image_repository'), size)