def _import(destination, source, file, album_from_folder, trash, paths, allow_duplicates): """Import files or directories by reading their EXIF and organizing them accordingly. """ result = Result() destination = os.path.abspath(os.path.expanduser(destination)) files = set() paths = set(paths) if source: paths.add(source) if file: paths.add(file) for path in paths: path = os.path.expanduser(path) if os.path.isdir(path): files.update(FILESYSTEM.get_all_files(path, None)) else: files.add(path) for current_file in files: dest_path = import_file(current_file, destination, album_from_folder, trash, allow_duplicates) result.append((current_file, dest_path)) result.write()
def test_add_multiple_rows_with_success(): expected = """****** SUMMARY ****** Metric Count -------- ------- Success 2 Error 0""" result = Result() result.append(('id1', '/some/path/1')) result.append(('id2', '/some/path/2')) call_result_and_assert(result, expected)
def test_add_multiple_rows_with_failure_and_success(): expected = """****** ERROR DETAILS ****** File ------ id1 ****** SUMMARY ****** Metric Count -------- ------- Success 1 Error 1""" result = Result() result.append(('id1', False)) result.append(('id2', '/some/path')) call_result_and_assert(result, expected)
def test_add_multiple_rows_with_failure(): expected = """****** ERROR DETAILS ****** File ------ id1 id2 ****** SUMMARY ****** Metric Count -------- ------- Success 0 Error 2""" result = Result() result.append(('id1', False)) result.append(('id2', False)) call_result_and_assert(result, expected)
def _generate_db(source): """Regenerate the hash.json database which contains all of the sha1 signatures of media files. """ result = Result() source = os.path.abspath(os.path.expanduser(source)) extensions = set() all_files = set() valid_files = set() if not os.path.isdir(source): log.error('Source is not a valid directory %s' % source) sys.exit(1) subclasses = get_all_subclasses(Base) for cls in subclasses: extensions.update(cls.extensions) all_files.update(FILESYSTEM.get_all_files(source, None)) db = Db() db.backup_hash_db() db.reset_hash_db() for current_file in all_files: if os.path.splitext(current_file)[1][1:].lower() not in extensions: log.info('Skipping invalid file %s' % current_file) result.append((current_file, False)) continue result.append((current_file, True)) db.add_hash(db.checksum(current_file), current_file) db.update_hash_db() result.write()
def _verify(): result = Result() db = Db() for checksum, file_path in db.all(): if not os.path.isfile(file_path): result.append((file_path, False)) continue actual_checksum = db.checksum(file_path) if checksum == actual_checksum: result.append((file_path, True)) else: result.append((file_path, False)) result.write()
def _verify(): result = Result() db = Db() for checksum, file_path in db.all(): if not os.path.isfile(file_path): result.append((file_path, False)) log.progress('x') continue actual_checksum = db.checksum(file_path) if checksum == actual_checksum: result.append((file_path, True)) log.progress() else: result.append((file_path, False)) log.progress('x') log.progress('', True) result.write()
def _import(destination, source, file, album_from_folder, trash, allow_duplicates, debug, paths, move): """Import files or directories by reading their EXIF and organizing them accordingly. """ constants.debug = debug has_errors = False result = Result() move = move destination = _decode(destination) destination = os.path.abspath(os.path.expanduser(destination)) files = set() paths = set(paths) if source: source = _decode(source) paths.add(source) if file: paths.add(file) for path in paths: path = os.path.expanduser(path) if os.path.isdir(path): files.update(FILESYSTEM.get_all_files(path, None)) else: files.add(path) for current_file in files: dest_path = import_file(current_file, destination, album_from_folder, trash, allow_duplicates, move) result.append((current_file, dest_path)) has_errors = has_errors is True or not dest_path result.write() if has_errors: sys.exit(1)
def _import(destination, source, file, album_from_folder, trash, paths, allow_duplicates): """Import files or directories by reading their EXIF and organizing them accordingly. """ result = Result() destination = destination.decode(sys.getfilesystemencoding()) destination = os.path.abspath(os.path.expanduser(destination)) files = set() paths = set(paths) if source: source = source.decode(sys.getfilesystemencoding()) paths.add(source) if file: paths.add(file) for path in paths: path = os.path.expanduser(path) if os.path.isdir(path): files.update(FILESYSTEM.get_all_files(path, None)) else: files.add(path) for current_file in files: dest_path = import_file(current_file, destination, album_from_folder, trash, allow_duplicates) result.append((current_file, dest_path)) result.write()
def main(argv): filesystem = FileSystem() result = Result() subclasses = get_all_subclasses() paths = argv[1:] for path in paths: path = os.path.expanduser(path) if os.path.isdir(path): for source in filesystem.get_all_files(path, None): status = add_original_name(source, subclasses) result.append((_decode(source), status)) else: status = add_original_name(path, subclasses) result.append((_decode(path), status)) result.write()
def _import(destination, source, file, album_from_folder, trash, allow_duplicates, debug, exclude_regex, paths): """Import files or directories by reading their EXIF and organizing them accordingly. """ constants.debug = debug has_errors = False result = Result() destination = _decode(destination) destination = os.path.abspath(os.path.expanduser(destination)) files = set() paths = set(paths) if source: source = _decode(source) paths.add(source) if file: paths.add(file) # if no exclude list was passed in we check if there's a config if len(exclude_regex) == 0: config = load_config() if 'Exclusions' in config: exclude_regex = [ value for key, value in config.items('Exclusions') ] exclude_regex_list = set(exclude_regex) for path in paths: path = os.path.expanduser(path) if os.path.isdir(path): files.update( FILESYSTEM.get_all_files(path, None, exclude_regex_list)) else: if not FILESYSTEM.should_exclude(path, exclude_regex_list, True): files.add(path) for current_file in files: dest_path = import_file(current_file, destination, album_from_folder, trash, allow_duplicates) result.append((current_file, dest_path)) has_errors = has_errors is True or not dest_path result.write() if has_errors: sys.exit(1)
def _generate_db(source): """Regenerate the hash.json database which contains all of the sha1 signatures of media files. """ result = Result() source = os.path.abspath(os.path.expanduser(source)) if not os.path.isdir(source): log.error('Source is not a valid directory %s' % source) sys.exit(1) db = Db() db.backup_hash_db() db.reset_hash_db() for current_file in FILESYSTEM.get_all_files(source): result.append((current_file, True)) db.add_hash(db.checksum(current_file), current_file) log.progress() db.update_hash_db() log.progress('', True) result.write()
def _import(source, config_path, manifest_path, allow_duplicates, dryrun, debug, move=False, indent_manifest=False, no_overwrite_manifest=False): """Import files or directories by reading their EXIF and organizing them accordingly. """ start_time = round(time.time()) constants.debug = debug has_errors = False result = Result() # Load the configuration from the json file. config = Config().load_from_file(config_path) source = config["sources"][0] # For now, only one. target = config["targets"][ 0] # For now, only one target allowed...but data structure allows more source_file_path = source["file_path"] manifest = Manifest() if manifest_path is not None: manifest.load_from_file(manifest_path) log_base_path, _ = os.path.split(manifest.file_path) FILESYSTEM.create_directory(os.path.join(log_base_path, '.elodie')) log_path = os.path.join(log_base_path, '.elodie', 'import_{}.log'.format(utility.timestamp_string())) def signal_handler(sig, frame): log.warn('[ ] Import cancelled') log.write(log_path) sys.exit(0) signal.signal(signal.SIGINT, signal_handler) original_manifest_key_count = len(manifest) # destination = _decode(destination) # destination = os.path.abspath(os.path.expanduser(destination)) exiftool_addedargs = [ # '-overwrite_original', u'-config', u'"{}"'.format(constants.exiftool_config) ] file_generator = FILESYSTEM.get_all_files(source_file_path, None) source_file_count = 0 with ExifTool(addedargs=exiftool_addedargs) as et: while True: file_batch = list( itertools.islice(file_generator, constants.exiftool_batch_size)) if len(file_batch) == 0: break # This will cause slight discrepancies in file counts: since elodie.json is counted but not imported, # each one will set the count off by one. source_file_count += len(file_batch) metadata_list = et.get_metadata_batch(file_batch) if not metadata_list: raise Exception("Metadata scrape failed.") # Key on the filename to make for easy access, metadata_dict = dict((os.path.abspath(el["SourceFile"]), el) for el in metadata_list) for current_file in file_batch: # Don't import localized config files. if current_file.endswith( "elodie.json"): # Faster than a os.path.split continue try: result = import_file(current_file, config, manifest, metadata_dict, move=move, dryrun=dryrun, allow_duplicates=allow_duplicates) except Exception as e: log.warn("[!] Error importing {}: {}".format( current_file, e)) result = False has_errors = has_errors or not result exiftool_waiting_time = et.waiting_time manifest.write(indent=indent_manifest, overwrite=(not no_overwrite_manifest)) manifest_key_count = len(manifest) try: total_time = round(time.time() - start_time) log.info("Statistics:") log.info("Source: File Count {}".format(source_file_count)) log.info("Manifest: New Hashes {}".format(manifest_key_count - original_manifest_key_count)) log.info("Manifest: Total Hashes {}".format(manifest_key_count)) log.info("Time: Total {}s".format(total_time)) log.info("Time: Files/sec {}".format( round(source_file_count / total_time))) log.info("Time: Waiting on ExifTool {}s".format( round(exiftool_waiting_time))) except Exception as e: log.error("[!] Error generating statistics: {}".format(e)) log.write(log_path) if has_errors: sys.exit(1)
def _update(album, location, time, title, files): """Update a file's EXIF. Automatically modifies the file's location and file name accordingly. """ result = Result() for current_file in files: if not os.path.exists(current_file): if constants.debug: print('Could not find %s' % current_file) print('{"source":"%s", "error_msg":"Could not find %s"}' % \ (current_file, current_file)) continue current_file = os.path.expanduser(current_file) destination = os.path.expanduser(os.path.dirname(os.path.dirname( os.path.dirname(current_file)))) media = Media.get_class_by_file(current_file, [Text, Audio, Photo, Video]) if not media: continue updated = False if location: update_location(media, current_file, location) updated = True if time: update_time(media, current_file, time) updated = True if album: media.set_album(album) updated = True # Updating a title can be problematic when doing it 2+ times on a file. # You would end up with img_001.jpg -> img_001-first-title.jpg -> # img_001-first-title-second-title.jpg. # To resolve that we have to track the prior title (if there was one. # Then we massage the updated_media's metadata['base_name'] to remove # the old title. # Since FileSystem.get_file_name() relies on base_name it will properly # rename the file by updating the title instead of appending it. remove_old_title_from_name = False if title: # We call get_metadata() to cache it before making any changes metadata = media.get_metadata() title_update_status = media.set_title(title) original_title = metadata['title'] if title_update_status and original_title: # @TODO: We should move this to a shared method since # FileSystem.get_file_name() does it too. original_title = re.sub(r'\W+', '-', original_title.lower()) original_base_name = metadata['base_name'] remove_old_title_from_name = True updated = True if updated: updated_media = Media.get_class_by_file(current_file, [Text, Audio, Photo, Video]) # See comments above on why we have to do this when titles # get updated. if remove_old_title_from_name and len(original_title) > 0: updated_media.get_metadata() updated_media.set_metadata_basename( original_base_name.replace('-%s' % original_title, '')) dest_path = FILESYSTEM.process_file(current_file, destination, updated_media, move=True, allowDuplicate=True) log.info(u'%s -> %s' % (current_file, dest_path)) print('{"source":"%s", "destination":"%s"}' % (current_file, dest_path)) # If the folder we moved the file out of or its parent are empty # we delete it. FILESYSTEM.delete_directory_if_empty(os.path.dirname(current_file)) FILESYSTEM.delete_directory_if_empty( os.path.dirname(os.path.dirname(current_file))) result.append((current_file, dest_path)) else: result.append((current_file, False)) result.write()
def _update(album, location, time, title, paths, debug): """Update a file's EXIF. Automatically modifies the file's location and file name accordingly. """ constants.debug = debug has_errors = False result = Result() files = set() for path in paths: path = os.path.expanduser(path) if os.path.isdir(path): files.update(FILESYSTEM.get_all_files(path, None)) else: files.add(path) for current_file in files: if not os.path.exists(current_file): has_errors = True result.append((current_file, False)) log.warn('Could not find %s' % current_file) log.all('{"source":"%s", "error_msg":"Could not find %s"}' % (current_file, current_file)) continue current_file = os.path.expanduser(current_file) # The destination folder structure could contain any number of levels # So we calculate that and traverse up the tree. # '/path/to/file/photo.jpg' -> '/path/to/file' -> # ['path','to','file'] -> ['path','to'] -> '/path/to' current_directory = os.path.dirname(current_file) destination_depth = -1 * len(FILESYSTEM.get_folder_path_definition()) destination = os.sep.join( os.path.normpath( current_directory ).split(os.sep)[:destination_depth] ) media = Media.get_class_by_file(current_file, get_all_subclasses()) if not media: continue updated = False if location: update_location(media, current_file, location) updated = True if time: update_time(media, current_file, time) updated = True if album: media.set_album(album) updated = True # Updating a title can be problematic when doing it 2+ times on a file. # You would end up with img_001.jpg -> img_001-first-title.jpg -> # img_001-first-title-second-title.jpg. # To resolve that we have to track the prior title (if there was one. # Then we massage the updated_media's metadata['base_name'] to remove # the old title. # Since FileSystem.get_file_name() relies on base_name it will properly # rename the file by updating the title instead of appending it. remove_old_title_from_name = False if title: # We call get_metadata() to cache it before making any changes metadata = media.get_metadata() title_update_status = media.set_title(title) original_title = metadata['title'] if title_update_status and original_title: # @TODO: We should move this to a shared method since # FileSystem.get_file_name() does it too. original_title = re.sub(r'\W+', '-', original_title.lower()) original_base_name = metadata['base_name'] remove_old_title_from_name = True updated = True if updated: updated_media = Media.get_class_by_file(current_file, get_all_subclasses()) # See comments above on why we have to do this when titles # get updated. if remove_old_title_from_name and len(original_title) > 0: updated_media.get_metadata() updated_media.set_metadata_basename( original_base_name.replace('-%s' % original_title, '')) dest_path = FILESYSTEM.process_file(current_file, destination, updated_media, move=True, allowDuplicate=True) log.info(u'%s -> %s' % (current_file, dest_path)) log.all('{"source":"%s", "destination":"%s"}' % (current_file, dest_path)) # If the folder we moved the file out of or its parent are empty # we delete it. FILESYSTEM.delete_directory_if_empty(os.path.dirname(current_file)) FILESYSTEM.delete_directory_if_empty( os.path.dirname(os.path.dirname(current_file))) result.append((current_file, dest_path)) # Trip has_errors to False if it's already False or dest_path is. has_errors = has_errors is True or not dest_path else: has_errors = False result.append((current_file, False)) result.write() if has_errors: sys.exit(1)
def _update(album, location, time, title, files): """Update a file's EXIF. Automatically modifies the file's location and file name accordingly. """ result = Result() for current_file in files: if not os.path.exists(current_file): if constants.debug: print('Could not find %s' % current_file) print('{"source":"%s", "error_msg":"Could not find %s"}' % \ (current_file, current_file)) continue current_file = os.path.expanduser(current_file) destination = os.path.expanduser( os.path.dirname(os.path.dirname(os.path.dirname(current_file)))) media = Media.get_class_by_file(current_file, [Text, Audio, Photo, Video]) if not media: continue updated = False if location: update_location(media, current_file, location) updated = True if time: update_time(media, current_file, time) updated = True if album: media.set_album(album) updated = True # Updating a title can be problematic when doing it 2+ times on a file. # You would end up with img_001.jpg -> img_001-first-title.jpg -> # img_001-first-title-second-title.jpg. # To resolve that we have to track the prior title (if there was one. # Then we massage the updated_media's metadata['base_name'] to remove # the old title. # Since FileSystem.get_file_name() relies on base_name it will properly # rename the file by updating the title instead of appending it. remove_old_title_from_name = False if title: # We call get_metadata() to cache it before making any changes metadata = media.get_metadata() title_update_status = media.set_title(title) original_title = metadata['title'] if title_update_status and original_title: # @TODO: We should move this to a shared method since # FileSystem.get_file_name() does it too. original_title = re.sub(r'\W+', '-', original_title.lower()) original_base_name = metadata['base_name'] remove_old_title_from_name = True updated = True if updated: updated_media = Media.get_class_by_file( current_file, [Text, Audio, Photo, Video]) # See comments above on why we have to do this when titles # get updated. if remove_old_title_from_name and len(original_title) > 0: updated_media.get_metadata() updated_media.set_metadata_basename( original_base_name.replace('-%s' % original_title, '')) dest_path = FILESYSTEM.process_file(current_file, destination, updated_media, move=True, allowDuplicate=True) log.info(u'%s -> %s' % (current_file, dest_path)) print('{"source":"%s", "destination":"%s"}' % (current_file, dest_path)) # If the folder we moved the file out of or its parent are empty # we delete it. FILESYSTEM.delete_directory_if_empty(os.path.dirname(current_file)) FILESYSTEM.delete_directory_if_empty( os.path.dirname(os.path.dirname(current_file))) result.append((current_file, dest_path)) else: result.append((current_file, False)) result.write()