Exemple #1
0
    def load(self):
        """Load plugins from config file.
        """
        # If plugins have been loaded then return
        if self.loaded == True:
            return

        plugin_list = load_plugin_config()
        for plugin in plugin_list:
            plugin_lower = plugin.lower()
            try:
                # We attempt to do the following.
                #  1. Load the module of the plugin.
                #  2. Instantiate an object of the plugin's class.
                #  3. Add the plugin to the list of plugins.
                #
                #  #3 should only happen if #2 doesn't throw an error
                this_module = import_module('elodie.plugins.{}.{}'.format(
                    plugin_lower, plugin_lower))
                self.classes[plugin] = getattr(this_module, plugin)()
                # We only append to self.plugins if we're able to load the class
                self.plugins.append(plugin)
            except:
                log.error(
                    'An error occurred initiating plugin {}'.format(plugin))
                log.error(format_exc())

        self.loaded = True
Exemple #2
0
def _generate_db(source):
    """Regenerate the hash.json database which contains all of the sha1 signatures of media files.
    """
    result = Result()
    source = os.path.abspath(os.path.expanduser(source))

    extensions = set()
    all_files = set()
    valid_files = set()

    if not os.path.isdir(source):
        log.error('Source is not a valid directory %s' % source)
        sys.exit(1)
        
    subclasses = get_all_subclasses(Base)
    for cls in subclasses:
        extensions.update(cls.extensions)

    all_files.update(FILESYSTEM.get_all_files(source, None))

    db = Db()
    db.backup_hash_db()
    db.reset_hash_db()

    for current_file in all_files:
        if os.path.splitext(current_file)[1][1:].lower() not in extensions:
            log.info('Skipping invalid file %s' % current_file)
            result.append((current_file, False))
            continue

        result.append((current_file, True))
        db.add_hash(db.checksum(current_file), current_file)
    
    db.update_hash_db()
    result.write()
Exemple #3
0
def update_location(media, file_path, location_name):
    """Update location exif metadata of media.
    """
    location_coords = geolocation.coordinates_by_name(location_name)

    if location_coords and 'latitude' in location_coords and \
            'longitude' in location_coords:
        location_status = media.set_location(location_coords[
            'latitude'], location_coords['longitude'])
        if not location_status:
            log.error('Failed to update location')
            log.all(('{"source":"%s",' % file_path,
                       '"error_msg":"Failed to update location"}'))
            sys.exit(1)
    return True
Exemple #4
0
def update_time(media, file_path, time_string):
    """Update time exif metadata of media.
    """
    time_format = '%Y-%m-%d %H:%M:%S'
    if re.match(r'^\d{4}-\d{2}-\d{2}$', time_string):
        time_string = '%s 00:00:00' % time_string
    elif re.match(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}\d{2}$', time_string):
        msg = ('Invalid time format. Use YYYY-mm-dd hh:ii:ss or YYYY-mm-dd')
        log.error(msg)
        log.all('{"source":"%s", "error_msg":"%s"}' % (file_path, msg))
        sys.exit(1)

    time = datetime.strptime(time_string, time_format)
    media.set_date_taken(time)
    return True
Exemple #5
0
def update_time(media, file_path, time_string):
    """Update time exif metadata of media.
    """
    time_format = '%Y-%m-%d %H:%M:%S'
    if re.match(r'^\d{4}-\d{2}-\d{2}$', time_string):
        time_string = '%s 00:00:00' % time_string
    elif re.match(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}\d{2}$', time_string):
        msg = ('Invalid time format. Use YYYY-mm-dd hh:ii:ss or YYYY-mm-dd')
        log.error(msg)
        print('{"source":"%s", "error_msg":"%s"}' % (file_path, msg))
        sys.exit(1)

    time = datetime.strptime(time_string, time_format)
    media.set_date_taken(time)
    return True
Exemple #6
0
def update_location(media, file_path, location_name):
    """Update location exif metadata of media.
    """
    location_coords = geolocation.coordinates_by_name(location_name)

    if location_coords and 'latitude' in location_coords and \
            'longitude' in location_coords:
        location_status = media.set_location(location_coords[
            'latitude'], location_coords['longitude'])
        if not location_status:
            log.error('Failed to update location')
            print(('{"source":"%s",' % file_path,
                '"error_msg":"Failed to update location"}'))
            sys.exit(1)
    return True
Exemple #7
0
    def get_date_taken(self):
        """Get the date which the photo was taken.

        The date value returned is defined by the min() of mtime and ctime.

        :returns: time object or None for non-photo files or 0 timestamp
        """
        if (not self.is_valid()):
            return None

        source = self.source
        seconds_since_epoch = min(os.path.getmtime(source),
                                  os.path.getctime(source))  # noqa

        exif = self.get_exiftool_attributes()
        if not exif:
            return None  # seconds_since_epoch

        # We need to parse a string from EXIF into a timestamp.
        # EXIF DateTimeOriginal and EXIF DateTime are both stored
        #   in %Y:%m:%d %H:%M:%S format
        # we split on a space and then r':|-' -> convert to int -> .timetuple()
        #   the conversion in the local timezone
        # EXIF DateTime is already stored as a timestamp
        # Sourced from https://github.com/photo/frontend/blob/master/src/libraries/models/Photo.php#L500  # noqa
        for key in self.exif_map['date_taken']:
            try:
                if (key in exif):
                    if (re.match('\d{4}(-|:)\d{2}(-|:)\d{2}', exif[key])
                            is not None):  # noqa
                        dt, tm = exif[key].split(' ')
                        dt_list = compile(r'-|:').split(dt)
                        dt_list = dt_list + compile(r'-|:').split(tm)
                        dt_list = map(int, dt_list)
                        time_tuple = datetime(*dt_list).timetuple()
                        seconds_since_epoch = time.mktime(time_tuple)
                        break
                # None of the date-taken keys were present
                return None
            except BaseException as e:
                log.error(e)
                pass

        if (seconds_since_epoch == 0):
            return None

        return time.gmtime(seconds_since_epoch)
Exemple #8
0
    def parse_metadata_line(self):
        if isinstance(self.metadata_line, dict):
            return self.metadata_line

        source = self.source
        if source is None:
            return None

        with open(source, "r") as f:
            first_line = f.readline().strip()

        try:
            parsed_json = loads(first_line)
            if isinstance(parsed_json, dict):
                self.metadata_line = parsed_json
        except ValueError:
            log.error("Could not parse JSON from first line: %s" % first_line)
            pass
Exemple #9
0
    def parse_metadata_line(self):
        if isinstance(self.metadata_line, dict):
            return self.metadata_line

        source = self.source
        if source is None:
            return None

        with open(source, 'r') as f:
            first_line = f.readline().strip()

        try:
            parsed_json = loads(first_line)
            if isinstance(parsed_json, dict):
                self.metadata_line = parsed_json
        except ValueError:
            log.error('Could not parse JSON from first line: %s' % first_line)
            pass
Exemple #10
0
    def get_date_taken(self):
        """Get the date which the photo was taken.

        The date value returned is defined by the min() of mtime and ctime.

        :returns: time object or None for non-photo files or 0 timestamp
        """
        if(not self.is_valid()):
            return None

        source = self.source
        seconds_since_epoch = min(os.path.getmtime(source), os.path.getctime(source))  # noqa

        exif = self.get_exiftool_attributes()
        if not exif:
            return seconds_since_epoch

        # We need to parse a string from EXIF into a timestamp.
        # EXIF DateTimeOriginal and EXIF DateTime are both stored
        #   in %Y:%m:%d %H:%M:%S format
        # we split on a space and then r':|-' -> convert to int -> .timetuple()
        #   the conversion in the local timezone
        # EXIF DateTime is already stored as a timestamp
        # Sourced from https://github.com/photo/frontend/blob/master/src/libraries/models/Photo.php#L500  # noqa
        for key in self.exif_map['date_taken']:
            try:
                if(key in exif):
                    if(re.match('\d{4}(-|:)\d{2}(-|:)\d{2}', exif[key]) is not None):  # noqa
                        dt, tm = exif[key].split(' ')
                        dt_list = compile(r'-|:').split(dt)
                        dt_list = dt_list + compile(r'-|:').split(tm)
                        dt_list = map(int, dt_list)
                        time_tuple = datetime(*dt_list).timetuple()
                        seconds_since_epoch = time.mktime(time_tuple)
                        break
            except BaseException as e:
                log.error(e)
                pass

        if(seconds_since_epoch == 0):
            return None

        return time.gmtime(seconds_since_epoch)
Exemple #11
0
 def run_batch(self):
     self.load()
     pass_status = True
     for cls in self.classes:
         this_method = getattr(self.classes[cls], 'batch')
         # We try to call the plugin's `before()` method.
         # If the method explicitly raises an ElodiePluginError we'll fail the import
         #  by setting pass_status to False.
         # If any other error occurs we log the message and proceed as usual.
         # By default, plugins don't change behavior.
         try:
             this_method()
             log.info('Called batch() for {}'.format(cls))
         except ElodiePluginError as err:
             log.warn('Plugin {} raised an exception in run_batch: {}'.format(cls, err))
             log.error(format_exc())
             pass_status = False
         except:
             log.error(format_exc())
     return pass_status
Exemple #12
0
def lookup(**kwargs):
    if ('location' not in kwargs and 'lat' not in kwargs
            and 'lon' not in kwargs):
        return None

    key = get_key()
    prefer_english_names = get_prefer_english_names()

    if (key is None):
        return None

    try:
        params = {'format': 'json', 'key': key}
        params.update(kwargs)
        path = '/geocoding/v1/address'
        if ('lat' in kwargs and 'lon' in kwargs):
            path = '/nominatim/v1/reverse.php'
        url = '%s%s?%s' % (constants.mapquest_base_url, path,
                           urllib.parse.urlencode(params))
        headers = {}
        if (prefer_english_names):
            headers = {'Accept-Language': 'en-EN,en;q=0.8'}
        r = requests.get(url, headers=headers)
        return parse_result(r.json())
    except requests.exceptions.RequestException as e:
        log.error(e)
        return None
    except ValueError as e:
        log.error(r.text)
        log.error(e)
        return None
Exemple #13
0
def lookup(**kwargs):
    if(
        'location' not in kwargs and
        'lat' not in kwargs and
        'lon' not in kwargs
    ):
        return None

    key = get_key()

    if(key is None):
        return None

    try:
        params = {'format': 'json', 'key': key}
        params.update(kwargs)
        path = '/geocoding/v1/address'
        if('lat' in kwargs and 'lon' in kwargs):
            path = '/nominatim/v1/reverse.php'
        url = 'http://open.mapquestapi.com%s?%s' % (
                    path,
                    urllib.parse.urlencode(params)
              )
        r = requests.get(url)
        return parse_result(r.json())
    except requests.exceptions.RequestException as e:
        log.error(e)
        return None
    except ValueError as e:
        log.error(r.text)
        log.error(e)
        return None
Exemple #14
0
def lookup(**kwargs):
    if(
        'location' not in kwargs and
        'lat' not in kwargs and
        'lon' not in kwargs
    ):
        return None

    key = get_key()

    if(key is None):
        return None

    try:
        params = {'format': 'json', 'accept-language': constants.accepted_language, 'key': key}
        params.update(kwargs)
        path = '/geocoding/v1/address'
        if('lat' in kwargs and 'lon' in kwargs):
            path = '/nominatim/v1/reverse.php'
        url = 'http://open.mapquestapi.com%s?%s' % (
                    path,
                    urllib.parse.urlencode(params)
              )
        r = requests.get(url)
        return parse_result(r.json())
    except requests.exceptions.RequestException as e:
        log.error(e)
        return None
    except ValueError as e:
        log.error(r.text)
        log.error(e)
        return None
Exemple #15
0
 def run_all_after(self, file_path, destination_folder, final_file_path,
                   metadata):
     """Process `before` methods of each plugin that was loaded.
     """
     self.load()
     pass_status = True
     for cls in self.classes:
         this_method = getattr(self.classes[cls], 'after')
         # We try to call the plugin's `before()` method.
         # If the method explicitly raises an ElodiePluginError we'll fail the import
         #  by setting pass_status to False.
         # If any other error occurs we log the message and proceed as usual.
         # By default, plugins don't change behavior.
         try:
             this_method(file_path, destination_folder, final_file_path,
                         metadata)
             log.info('Called after() for {}'.format(cls))
         except ElodiePluginError as err:
             log.warn('Plugin {} raised an exception in run_all_before: {}'.
                      format(cls, err))
             log.error(format_exc())
             log.error('false')
             pass_status = False
         except:
             log.error(format_exc())
     return pass_status
Exemple #16
0
def _generate_db(source):
    """Regenerate the hash.json database which contains all of the sha1 signatures of media files.
    """
    result = Result()
    source = os.path.abspath(os.path.expanduser(source))

    if not os.path.isdir(source):
        log.error('Source is not a valid directory %s' % source)
        sys.exit(1)

    db = Db()
    db.backup_hash_db()
    db.reset_hash_db()

    for current_file in FILESYSTEM.get_all_files(source):
        result.append((current_file, True))
        db.add_hash(db.checksum(current_file), current_file)
        log.progress()

    db.update_hash_db()
    log.progress('', True)
    result.write()
Exemple #17
0
def _import(source,
            config_path,
            manifest_path,
            allow_duplicates,
            dryrun,
            debug,
            move=False,
            indent_manifest=False,
            no_overwrite_manifest=False):
    """Import files or directories by reading their EXIF and organizing them accordingly.
    """
    start_time = round(time.time())

    constants.debug = debug
    has_errors = False
    result = Result()

    # Load the configuration from the json file.
    config = Config().load_from_file(config_path)

    source = config["sources"][0]  # For now, only one.
    target = config["targets"][
        0]  # For now, only one target allowed...but data structure allows more

    source_file_path = source["file_path"]

    manifest = Manifest()

    if manifest_path is not None:
        manifest.load_from_file(manifest_path)

    log_base_path, _ = os.path.split(manifest.file_path)
    FILESYSTEM.create_directory(os.path.join(log_base_path, '.elodie'))
    log_path = os.path.join(log_base_path, '.elodie',
                            'import_{}.log'.format(utility.timestamp_string()))

    def signal_handler(sig, frame):
        log.warn('[ ] Import cancelled')
        log.write(log_path)
        sys.exit(0)

    signal.signal(signal.SIGINT, signal_handler)

    original_manifest_key_count = len(manifest)

    # destination = _decode(destination)
    # destination = os.path.abspath(os.path.expanduser(destination))

    exiftool_addedargs = [
        # '-overwrite_original',
        u'-config',
        u'"{}"'.format(constants.exiftool_config)
    ]

    file_generator = FILESYSTEM.get_all_files(source_file_path, None)
    source_file_count = 0

    with ExifTool(addedargs=exiftool_addedargs) as et:
        while True:
            file_batch = list(
                itertools.islice(file_generator,
                                 constants.exiftool_batch_size))
            if len(file_batch) == 0: break

            # This will cause slight discrepancies in file counts: since elodie.json is counted but not imported,
            #   each one will set the count off by one.
            source_file_count += len(file_batch)
            metadata_list = et.get_metadata_batch(file_batch)
            if not metadata_list:
                raise Exception("Metadata scrape failed.")
            # Key on the filename to make for easy access,
            metadata_dict = dict((os.path.abspath(el["SourceFile"]), el)
                                 for el in metadata_list)
            for current_file in file_batch:
                # Don't import localized config files.
                if current_file.endswith(
                        "elodie.json"):  # Faster than a os.path.split
                    continue
                try:
                    result = import_file(current_file,
                                         config,
                                         manifest,
                                         metadata_dict,
                                         move=move,
                                         dryrun=dryrun,
                                         allow_duplicates=allow_duplicates)
                except Exception as e:
                    log.warn("[!] Error importing {}: {}".format(
                        current_file, e))
                    result = False
                has_errors = has_errors or not result
        exiftool_waiting_time = et.waiting_time

    manifest.write(indent=indent_manifest,
                   overwrite=(not no_overwrite_manifest))

    manifest_key_count = len(manifest)

    try:
        total_time = round(time.time() - start_time)
        log.info("Statistics:")
        log.info("Source: File Count {}".format(source_file_count))
        log.info("Manifest: New Hashes {}".format(manifest_key_count -
                                                  original_manifest_key_count))
        log.info("Manifest: Total Hashes {}".format(manifest_key_count))
        log.info("Time: Total {}s".format(total_time))
        log.info("Time: Files/sec {}".format(
            round(source_file_count / total_time)))
        log.info("Time: Waiting on ExifTool {}s".format(
            round(exiftool_waiting_time)))
    except Exception as e:
        log.error("[!] Error generating statistics: {}".format(e))

    log.write(log_path)

    if has_errors:
        sys.exit(1)
Exemple #18
0
def _update(album, location, time, title, paths, debug):
    """Update a file's EXIF. Automatically modifies the file's location and file name accordingly.
    """
    constants.debug = debug
    has_errors = False
    result = Result()

    files = set()
    for path in paths:
        path = os.path.expanduser(path)
        if os.path.isdir(path):
            files.update(FILESYSTEM.get_all_files(path, None))
        else:
            files.add(path)

    for current_file in files:
        if not os.path.exists(current_file):
            has_errors = True
            result.append((current_file, False))
            log.warn('Could not find %s' % current_file)
            log.error('{"source":"%s", "error_msg":"Could not find %s"}' % \
                (current_file, current_file))
            continue

        current_file = os.path.expanduser(current_file)

        # The destination folder structure could contain any number of levels
        #  So we calculate that and traverse up the tree.
        # '/path/to/file/photo.jpg' -> '/path/to/file' ->
        #  ['path','to','file'] -> ['path','to'] -> '/path/to'
        current_directory = os.path.dirname(current_file)
        destination_depth = -1 * len(FILESYSTEM.get_folder_path_definition())
        destination = os.sep.join(
            os.path.normpath(current_directory).split(
                os.sep)[:destination_depth])

        media = Media.get_class_by_file(current_file, get_all_subclasses())
        if not media:
            continue

        updated = False
        if location:
            update_location(media, current_file, location)
            updated = True
        if time:
            update_time(media, current_file, time)
            updated = True
        if album:
            media.set_album(album)
            updated = True

        # Updating a title can be problematic when doing it 2+ times on a file.
        # You would end up with img_001.jpg -> img_001-first-title.jpg ->
        # img_001-first-title-second-title.jpg.
        # To resolve that we have to track the prior title (if there was one.
        # Then we massage the updated_media's metadata['base_name'] to remove
        # the old title.
        # Since FileSystem.get_file_name() relies on base_name it will properly
        #  rename the file by updating the title instead of appending it.
        remove_old_title_from_name = False
        if title:
            # We call get_metadata() to cache it before making any changes
            metadata = media.get_metadata()
            title_update_status = media.set_title(title)
            original_title = metadata['title']
            if title_update_status and original_title:
                # @TODO: We should move this to a shared method since
                # FileSystem.get_file_name() does it too.
                original_title = re.sub(r'\W+', '-', original_title.lower())
                original_base_name = metadata['base_name']
                remove_old_title_from_name = True
            updated = True

        if updated:
            updated_media = Media.get_class_by_file(current_file,
                                                    get_all_subclasses())
            # See comments above on why we have to do this when titles
            # get updated.
            if remove_old_title_from_name and len(original_title) > 0:
                updated_media.get_metadata()
                updated_media.set_metadata_basename(
                    original_base_name.replace('-%s' % original_title, ''))

            dest_path = FILESYSTEM.process_file(current_file,
                                                destination,
                                                updated_media,
                                                move=True,
                                                allowDuplicate=True)
            log.info(u'%s -> %s' % (current_file, dest_path))
            log.info('{"source":"%s", "destination":"%s"}' %
                     (current_file, dest_path))
            # If the folder we moved the file out of or its parent are empty
            # we delete it.
            FILESYSTEM.delete_directory_if_empty(os.path.dirname(current_file))
            FILESYSTEM.delete_directory_if_empty(
                os.path.dirname(os.path.dirname(current_file)))
            result.append((current_file, dest_path))
            # Trip has_errors to False if it's already False or dest_path is.
            has_errors = has_errors is True or not dest_path
        else:
            has_errors = False
            result.append((current_file, False))

    result.write()

    if has_errors:
        sys.exit(1)