def flat(origin, destination, replace=True, grain=3, persist=False, delimiter='.', update_time=False, logger=None): """ Handles the movement of files from one location to another. The destination will not be organized; all files will just be dumped into it. The files will be renamed to indicate their origin time. :param origin: the originating directory, which will be copied :param destination: the destination directory where the nesting will be created :param replace: if attempting to move a file and it exists in the destination already, should it be replaced or left alone? :type replace: bool :param grain: how much information to incorporate in the new filename :type grain: int :param persist: whether to leave the original files in-place or delete them :type persist: bool :param delimiter: the string used to split the parts of the date in the new filename :param update_time: whether to update the timestamps on files in the destination :type update_time: bool :param logger: a Management Tools logger to record information """ # Ensure we have some sort of logger. Prevents errors. if not logger: logger = loggers.stream_logger(1) date = formatting.date(grain) # Check that the origin actually, like... exists. if not os.path.isdir(origin): raise RuntimeError("No such origin directory: " + origin) if not os.path.isdir(destination): logger.error("Creating destination directory at: " + destination) os.makedirs(destination) # Find everything in the origin directory. logger.info("Building payload list.") with ChDir(origin): payload = sorted([os.path.abspath(x) for x in os.listdir('.')]) # Do all of the archival. with ChDir(destination): file_prefixes = {} max_file_length = 0 for file in payload: # For each file, pull its timestamp and split it into its different # parts. These will be used to create the appropriate file name for # each file being moved. time = datetime.fromtimestamp(os.path.getmtime(file)) date_parts = time.strftime(date).split('.') prefix = delimiter.join(date_parts) file_prefixes[file] = prefix # This is just used for pretty printing. if len(os.path.basename(file)) > max_file_length: max_file_length = len(os.path.basename(file)) # Start moving/copying the files. # (Moving is used if the files don't need to stay in the origin.) logger.info("{} files to appropriate subdirectories...".format("Moving" if not persist else "Copying")) for file, prefix in file_prefixes.items(): # Each file gets wrapped in a try/except block to ensure that flow # is not interrupted if there's an issue with one of them. try: # Form the new file name. new_name = prefix + delimiter + os.path.basename(file) add = False # Determine whether the file should be put in the destination. if replace: # Definitely add the file if we're okay with replacing it. add = True else: # If we're not okay with replacing it, ensure that the file # does not exist in the destination already. if not os.path.isfile(new_name): add = True # If we're okay with adding the file, then do the thing! if add: logger.info(" {file:>{length}} {dash}> ./{new}".format( length = max_file_length, file = os.path.basename(file), new = new_name, dash = '=' if persist else '-' )) # If the file exists in the destination, delete it before # attempting to move a new copy there. This also accounts # for symbolic links. if os.path.isfile(new_name): os.remove(new_name) # Copy if persisting data; move otherwise. if persist: shutil.copy2(file, new_name) else: shutil.move(file, new_name) # Update the time as needed. if update_time: os.utime(new_name, None) except (IOError, OSError) as e: # These are the most likely errors. logger.error("{}".format(repr(e))) logger.error("Unable to copy file '{}' to path: {}".format(file, path)) except (KeyboardInterrupt, SystemExit): logger.info("Quitting...") break except Exception as e: logger.error("{}".format(repr(e)))
def nested(origin, destination, replace=True, grain=3, persist=False, update_time=False, logger=None): """ Handles the movement of files from one location to another, but the destination will be organized in a nested format, e.g. Destination > Year > Month > Day > file :param origin: the originating directory, which will be copied :param destination: the destination directory where the nesting will be created :param replace: if attempting to move a file and it exists in the destination already, should it be replaced or left alone? :type replace: bool :param grain: how deep to form the nestings :type grain: int :param persist: whether to leave the original files in-place or delete them :type persist: bool :param update_time: whether to update the timestamps on files in the destination :type update_time: bool :param logger: a Management Tools logger to record information """ # Ensure we have some sort of logger. Prevents errors. if not logger: logger = loggers.stream_logger(1) date = formatting.date(grain) # Check that the origin actually, like... exists. if not os.path.isdir(origin): raise RuntimeError("No such origin directory: " + origin) # Destination should probably exist. if not os.path.isdir(destination): logger.error("Creating destination directory at: " + destination) os.makedirs(destination) # Find everything in the origin directory. logger.info("Building payload list.") with ChDir(origin): payload = sorted([os.path.abspath(x) for x in os.listdir('.')]) # Do all of the archival. with ChDir(destination): logger.info("Creating nested directory structure in: {}".format(os.path.abspath(destination))) dirs_needed = [] file_paths = {} max_file_length = 0 for file in payload: # For each file, pull its timestamp and split it into its different # parts. These will be used to create the appropriate directory # structure in the destination. time = datetime.fromtimestamp(os.path.getmtime(file)) dirs = time.strftime(date).split('.') leaf = os.path.join(*dirs) dirs_needed.append(leaf) file_paths[file] = leaf # This is just used for pretty printing. if len(os.path.basename(file)) > max_file_length: max_file_length = len(os.path.basename(file)) # Remove duplicates from the necessary directories. (This avoids errors # where a folder already exists.) Then create the nested folders. dirs_needed = uniquify(dirs_needed) for dir in dirs_needed: if not os.path.isdir(dir): logger.info(" ./" + dir) os.makedirs(dir) # Start moving/copying the files. # (Moving is used if the files don't need to stay in the origin.) logger.info("{} files to appropriate subdirectories...".format("Moving" if not persist else "Copying")) for file, path in file_paths.items(): # Each file gets wrapped in a try/except block to ensure that flow # is not interrupted if there's an issue with one of them. try: # Set the file's destination. file_destination = os.path.join(path, os.path.basename(file)) # Determine whether the file should be put in the destination. add = False if replace: # Definitely add the file if we're okay with replacing it. add = True else: # If we're not okay with replacing it, ensure that the file # does not exist in the destination already. if not os.path.isfile(file_destination): add = True # If we're okay with adding the file, then do the thing! if add: logger.info(" {file:>{length}} {dash}> ./{dest}".format( file = os.path.basename(file), length = max_file_length, dest = file_destination, dash = '=' if persist else '-' )) # If the file exists in the destination, delete it before # attempting to move a new copy there. This also accounts # for symbolic links. if os.path.isfile(file_destination): os.remove(file_destination) # Copy if persisting data; move otherwise. if persist: shutil.copy2(file, path) else: shutil.move(file, path) # Update the time as needed. if update_time: os.utime(file_destination, None) except (IOError, OSError) as e: # These are the most likely errors. logger.error("{}".format(repr(e))) logger.error("Unable to copy file '{}' to path: {}".format(file, path)) except (KeyboardInterrupt, SystemExit): logger.info("Quitting...") break except Exception as e: logger.error("{}".format(repr(e)))