def __init__(self, database): self._convert = os.path.abspath(config.get('thumbnail', 'convert_path')) self._lock_file = os.path.abspath( config.get('thumbnail', 'lock_file') + '-' + database) self._size = int(config.get('thumbnail', 'image_size')) self._database = rigor.database.Database.instance(database) os.umask(002)
def __init__(self, move=True): self._source = os.path.abspath(config.get('import', 'upload_repository')) + os.sep self._destination = os.path.abspath(config.get('global', 'image_repository')) + os.sep self._move = move rsync = os.path.abspath(config.get('import', 'rsync_path')) args = [rsync, '-q', '-r', '-p', '--chmod=ugo+r,Dugo+x,ug+w', '-t', '-O'] if move: args.append('--remove-source-files') args.append(self._source) args.append(self._destination) self._args = args os.umask(002)
def find_thumbnail(image, size): """ Given a base path, constructs a path to the image's thumbnail """ return build_thumbnail_path(image, config.get('thumbnail', 'image_repository'), size)
def _read_global_metadata(self): """ Reads the metadata file for the image directory and sets defaults """ metadata_file = os.path.join(self._directory, config.get('import', 'metadata_file')) if not os.path.isfile(metadata_file): return with open(metadata_file, 'r') as metadata: return json.load(metadata)
def destroy_image(database, image): """ Completely Removes an image from the database, along with its tags, annotations, and any anotation tags. Also deletes the image file from disk.""" mapper = DatabaseMapper(database) with database.get_cursor() as cursor: sql = "DELETE FROM tag WHERE image_id = %s;" cursor.execute(sql, (image['id'],)) annotations = mapper._get_annotations_by_image_id(cursor, image['id']) # pylint: disable=W0212 for annotation in annotations: sql = "DELETE FROM annotation_tag WHERE annotation_id = %s" cursor.execute(sql, (annotation['id'],)) mapper._delete_annotation(cursor, annotation['id']) # pylint: disable=W0212 sql = "DELETE FROM image WHERE id = %s;" cursor.execute(sql, (image['id'],)) image_path = find(image) os.unlink(image_path) for thumbnail_dir in os.listdir(config.get('thumbnail', 'image_repository')): image_thumbnail_path = build_path(image, os.path.join(config.get('thumbnail', 'image_repository'), thumbnail_dir)) os.unlink(image_thumbnail_path)
def build_dsn(database): """ Builds the database connection string from config values """ dsn = "dbname='{0}' host='{1}'".format(database, config.get('database', 'host')) try: ssl = config.getboolean('database', 'ssl') if ssl: dsn += " sslmode='require'" except ConfigParser.Error: pass try: username = config.get('database', 'username') dsn += " user='******'".format(username) except ConfigParser.Error: pass try: password = config.get('database', 'password') dsn += " password='******'".format(password) except ConfigParser.Error: pass return dsn
def build_dsn(database): """ Builds the database connection parameters from config values """ dsn = {'database': database, 'host': config.get('database', 'host')} try: ssl = config.getboolean('database', 'ssl') if ssl: dsn['ssl'] = ssl except ConfigParser.Error: pass try: username = config.get('database', 'username') dsn['user'] = username except ConfigParser.Error: pass try: password = config.get('database', 'password') dsn['password'] = password except ConfigParser.Error: pass return dsn
def __init__(self, algorithm, domain, arguments=None): """ The domain dictates which images to use as sources. The limit is an optional maximum number of images to use as sources. If random, images will be pulled in random order, up to limit; otherwise, images will be pulled in sequential order. Tags are used to control the image selection further. """ DatabaseRunner.__init__(self, algorithm, arguments) self._domain = domain self._image_id = None if kMaxWorkers > 1: self._pool = Pool(int(config.get('global', 'max_workers')))
def destroy_image(database, image): """ Completely Removes an image from the database, along with its tags, annotations, and any anotation tags. Also deletes the image file from disk.""" mapper = DatabaseMapper(database) with database.get_cursor() as cursor: sql = "DELETE FROM tag WHERE image_id = %s;" cursor.execute(sql, (image['id'], )) annotations = mapper._get_annotations_by_image_id(cursor, image['id']) # pylint: disable=W0212 for annotation in annotations: sql = "DELETE FROM annotation_tag WHERE annotation_id = %s" cursor.execute(sql, (annotation['id'], )) mapper._delete_annotation(cursor, annotation['id']) # pylint: disable=W0212 sql = "DELETE FROM image WHERE id = %s;" cursor.execute(sql, (image['id'], )) image_path = find(image) os.unlink(image_path) for thumbnail_dir in os.listdir( config.get('thumbnail', 'image_repository')): image_thumbnail_path = build_path( image, os.path.join(config.get('thumbnail', 'image_repository'), thumbnail_dir)) os.unlink(image_thumbnail_path)
def main(): rigor.domain.money.init(parameters) logger = rigor.logger.getLogger(__file__) database_mapper = DatabaseMapper(Database.instance(kDatabase)) pool = Pool(int(config.get('global', 'max_workers'))) logger.debug('Fetching image IDs from database') images = database_mapper.get_images_for_analysis(kDomain, kLimit, False) for parameter_set in get_parameters(): timestamp = datetime.utcnow().strftime("{0}-%Y%m%d_%H%M%S%f".format(kDomain)) with open("{0}.params".format(timestamp), "w") as parameter_file: json.dump(parameter_set, parameter_file) parameter_file.write("\n") with open("{0}.results".format(timestamp), "w") as result_file: image_config = partial(rigor.domain.money.run, parameters=parameter_set) logger.debug('Processing {0} images'.format(len(images))) for result in pool.map(image_config, images): result_file.write("\t".join([str(x) for x in result])) result_file.write("\n")
def main(): rigor.domain.money.init(parameters) logger = rigor.logger.getLogger(__file__) database_mapper = DatabaseMapper(Database.instance(kDatabase)) pool = Pool(int(config.get('global', 'max_workers'))) logger.debug('Fetching image IDs from database') images = database_mapper.get_images_for_analysis(kDomain, kLimit, False) for parameter_set in get_parameters(): timestamp = datetime.utcnow().strftime( "{0}-%Y%m%d_%H%M%S%f".format(kDomain)) with open("{0}.params".format(timestamp), "w") as parameter_file: json.dump(parameter_set, parameter_file) parameter_file.write("\n") with open("{0}.results".format(timestamp), "w") as result_file: image_config = partial(rigor.domain.money.run, parameters=parameter_set) logger.debug('Processing {0} images'.format(len(images))) for result in pool.map(image_config, images): result_file.write("\t".join([str(x) for x in result])) result_file.write("\n")
def __init__(self, database): self._convert = os.path.abspath(config.get('thumbnail', 'convert_path')) self._lock_file = os.path.abspath(config.get('thumbnail', 'lock_file') + '-' + database) self._size = int(config.get('thumbnail', 'image_size')) self._database = rigor.database.Database.instance(database) os.umask(002)
def import_image(self, path, basename, metadata): """ Reads the metadata for an invididual image and returns an object ready to insert """ image = dict() image['locator'] = uuid.uuid4().hex image['hash'] = rigor.hash.sha1_hash(path) data = rigor.imageops.read(path) image['resolution'] = (data.shape[1], data.shape[0]) image['format'] = imghdr.what(path) if not image['format']: image['format'] = 'jpeg' # TODO: why is imghdr.what returning None for /home/kaolin/data/ICDAR2005SceneCompetition/wltang_15.08.2002/IMG_0001.JPG if len(data.shape) == 2: image['depth'] = 8 else: image['depth'] = data.shape[2] * 8 if not metadata: metadata = {}; new_metadata = metadata.copy() new_metadata.update(self._read_local_metadata(basename)) if 'timestamp' in new_metadata: image['stamp'] = datetime.strptime(new_metadata['timestamp'], config.get('import', 'timestamp_format')) else: image['stamp'] = datetime.utcfromtimestamp(os.path.getmtime(path)) if 'source_id' in new_metadata: image['source_id'] = new_metadata['source_id'] else: image['source_id'] = None image['tags'] = list() if 'tags' in new_metadata: image['tags'] = new_metadata['tags'] has_sensor = False for tag in image['tags']: if tag.startswith('sensor:'): has_sensor = True break if not has_sensor: try: exif = pyexiv2.ImageMetadata(path) # pylint: disable=E0602 exif.read() if 'Exif.Image.Make' in exif and 'Exif.Image.Model' in exif: sensor_tag = 'sensor:' + '_'.join((exif['Exif.Image.Make'].value, exif['Exif.Image.Model'].value)) image['tags'] += sensor_tag.lower() except NameError: self._logger.warning("Image at {0}: No sensor listed in metadata, and EXIF data is not available.".format(path)) if 'location' in new_metadata: image['location'] = new_metadata['location'] else: image['location'] = None annotations = list() if 'annotations' in new_metadata: for annotation in new_metadata['annotations']: new_annotations = dict() for key in ('boundary', 'domain', 'model', 'confidence', 'annotation_tags'): if key in annotation: new_annotations[key] = annotation[key] else: new_annotations[key] = None if 'timestamp' in annotation: new_annotations['stamp'] = datetime.strptime(annotation['timestamp'], config.get('import', 'timestamp_format')) else: new_annotations['stamp'] = image['stamp'] annotations.append(new_annotations) image['annotations'] = annotations destination = os.path.join(config.get('import', 'upload_repository'), image['locator'][0:2], image['locator'][2:4], os.extsep.join((image['locator'], image['format']))) # We take control of the transaction here so we can fail if copying/moving the file fails try: with self._database_mapper._db.get_cursor() as cursor: # pylint: disable=W0212 self._database_mapper._create_image(cursor, image) # pylint: disable=W0212 # Create destination directory, if it doesn't already exist. try/except # structure avoids race condition try: os.makedirs(os.path.dirname(destination)) except OSError as err: if err.errno == errno.EEXIST: pass else: raise if self._move: shutil.move(path, destination) else: shutil.copy2(path, destination) os.chmod(destination, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) self._logger.info("Imported image {0}".format(image['locator'])) return image except rigor.database.IntegrityError: self._logger.warn("The image at '{0}' already exists in the database".format(path))
def cls(): adapter_name = config.get('database', 'adapter') adapter = __import__("rigor.adapters.{}_adapter".format(adapter_name), fromlist=["rigor.adapters", ]) return adapter.Database
def find(image): """ Returns the location in the repository where this image can be found. """ return build_path(image, config.get('global', 'image_repository'))
def find(image): """ Returns the location in the repository where this image can be found. """ return build_path(image, config.get('global', 'image_repository'))
""" Runs an algorithm across a set of images. Result is a report containing image ID, detected model, expected model, and elapsed time """ from rigor.config import config import rigor.logger import rigor.database import rigor.dbmapper import abc import json import argparse kMaxWorkers = int(config.get('global', 'max_workers')) if kMaxWorkers > 1: from multiprocessing.pool import Pool class BaseRunner(object): """ The base class for Runner objects, which fetch a set of images and apply the Algorithm instance to each """ __metaclass__ = abc.ABCMeta def __init__(self, algorithm, arguments=None): self._logger = rigor.logger.get_logger('.'.join((__name__, self.__class__.__name__))) self._algorithm = algorithm self._parameters = None self._arguments = None self.parse_arguments(arguments)
def __init__(self, database): super(Database, self).__init__(database) register_type(psycopg2.extensions.UNICODE) register_uuid() dsn = Database.build_dsn(database) self._pool = ThreadedConnectionPool(config.get('database', 'min_database_connections'), config.get('database', 'max_database_connections'), dsn)
def find_thumbnail(image, size): """ Given a base path, constructs a path to the image's thumbnail """ return build_thumbnail_path(image, config.get('thumbnail', 'image_repository'), size)