def config_display_string(config, show_exclusions=False):
    """
    Builds a string that contains all relevant information about a given configuration.
    :param config: The configuration object to display information about.
    :param show_exclusions: True if detailed exclusion information should be shown. False by default.
    :return: A string containing formatted information about the configuration.
    """
    # Display this message if there is nothing in the configuration yet
    if config.num_entries() == 0:
        return "NO FOLDERS/FILES SELECTED TO BACKUP"

    # Header: show the configuration name if it exists
    if config.name is None:
        return_str = "CURRENT CONFIGURATION         \n"
    else:
        return_str = "CURRENT CONFIGURATION ({})    \n".format(config.name)

    # Loop through every entry and show information about each
    entry_number = 1
    for input_str, outputs_list in config.get_zipped_entries():
        # Display the size of this entry's input
        total_size, total_files = util.directory_size_with_exclusions(
            input_str, config, entry_number)
        return_str += "\tBACKUP: {} ({}, {} files)".format(
            input_str, util.bytes_to_string(total_size, 2), total_files)

        # If this entry has exclusions, show them
        if config.get_entry(entry_number).num_exclusions() > 0:
            # If show_exclusions is true, show all information, otherwise just show that exclusions exist here
            if show_exclusions:
                return_str += "\n\t\tEXCLUSIONS:\n"
                for exclusion in config.get_entry(entry_number).exclusions:
                    return_str += "\t\t\t{}\n".format(exclusion.to_string())
                    if exclusion.has_limitations():
                        for limitation in exclusion.limitations:
                            return_str += "\t\t\t\tLimit to {}\n".format(
                                limitation.to_string(
                                    config.get_entry(entry_number).input))
            else:
                return_str += " [Contains exclusions]\n"
        else:
            return_str += "\n"

        # Display every output path below the previously displayed input
        for output_str in outputs_list:
            return_str += "\t\tCOPY TO: {}\n".format(output_str)
        entry_number += 1
    return return_str.strip()
示例#2
0
    def process_packet(self, packet):

        print(f"CaptureThread: processing packet: {packet}")
        packet_dict = to_dict(packet, strict=True)
        if "Raw" in packet_dict:
            del packet_dict["Raw"]
        packet_dict["hexdump"] = hexdump(packet, dump=True)
        packet_dict_no_bytes = bytes_to_string(packet_dict)

        pprint(packet_dict_no_bytes)

        print(f"CaptureThread: sending capture: {packet_dict_no_bytes}")
        status_code = send_capture(gethostname(), self.destination,
                                   str(datetime.now())[:-1],
                                   [packet_dict_no_bytes], self.snoop)
        print(f"CaptureThread: capture sent, result={status_code}\n")
def run_backup(config):
    """
    The primary entry function that starts and runs the backup process. This algorithm will go through each
    input-output pair in the configuration individually and run on each. It begins on each pair by marking which
    files will be new, modified, or deleted, then it uses those to check space requirements on the disk, then it
    does the file operations on each file in each list.
    :param config: A configuration containing paths to folders to backup.
    """
    print("Initializing...", end="\r", flush=True)
    set_status("Initializing...")
    reset_backup_number()
    log.log("\n" +
            configuration.config_display_string(config, show_exclusions=True))

    # Loop through every entry in the configuration
    for input_number in range(1, config.num_entries() + 1):
        input_path = config.get_entry(input_number).input
        outputs = config.get_entry(input_number).outputs
        for output_path in outputs:
            # True if backing up only one file, false if backing up a directory
            file_mode = os.path.isfile(input_path)

            # Get the name of the folder to make the backup in
            folder_name = os.path.split(input_path)[1]
            if file_mode:
                backup_folder = output_path
            else:
                backup_folder = os.path.join(
                    output_path, folder_name + " " + BACKUP_FOLDER_SUFFIX)

            # Start the log messages
            log.log("\n" + '/' * 60 + "\n///// INPUT: " + input_path +
                    "\n///// OUTPUT: " + backup_folder + "\n" + '/' * 60 +
                    "\n")

            # Mark all the files needed for the backup process
            print(' ' * 40 +
                  "\nPreparing files for backup from {} to {}...".format(
                      input_path, backup_folder))
            reset_globals()
            start_time = time.time()
            if file_mode:
                # If backing up one file, create the backed-up filename here
                filename_no_ext, filename_ext = os.path.splitext(folder_name)
                output_filename = os.path.join(
                    output_path, filename_no_ext + " " + BACKUP_FOLDER_SUFFIX +
                    filename_ext)
                new_files, changed_files, remove_files = mark_files(
                    input_path, output_filename, config, input_number)
            else:
                new_files, changed_files, remove_files = mark_files(
                    input_path, backup_folder, config, input_number)
            set_num_marked(
                len(new_files) + len(changed_files) + len(remove_files))
            if not file_mode:
                print()
            print("File preparation complete.")
            if NUM_FILES_ERROR > 0:
                log.log_print(
                    "There were {} error(s) reported during file preparation.".
                    format(NUM_FILES_ERROR))
                print(
                    "Please check the log file for more info on the individual errors."
                )

            # Check that doing this backup won't over-fill the disk, if it will then return
            has_space, remaining_space, space_difference =\
                check_space_requirements(new_files, changed_files, remove_files, backup_folder)
            if not has_space:
                drive_letter, tail = os.path.splitdrive(backup_folder)
                error_str = "Copying {} to {} may not fit on the {} drive.".format(
                    input_path, backup_folder, drive_letter)
                error_str += "\nPlease clear up space on the drive you want to copy to and try again."
                error_str += "\nTry clearing at least {} on the {} drive and trying again.".format(
                    util.bytes_to_string(-1 * remaining_space, 3),
                    drive_letter)
                log.log_print("\n" + error_str)
                set_error(error_str)
                set_status(
                    "ERROR: The backup will not fit. Backup process has stopped."
                )
                if not file_mode:
                    if util.dir_empty(backup_folder):
                        util.rmtree(backup_folder)
                return

            # Make changes to the files found in file preparation
            print("Backing up files from {} to {}...".format(
                input_path, backup_folder))
            num_errors = backup_files(new_files, changed_files, remove_files)
            end_time = time.time()

            # Backup is complete, report the time taken, space difference, and if any errors occurred
            complete_str = "Backup complete in {}. ".format(
                util.time_string(end_time - start_time))
            if space_difference == 0:
                if len(remove_files) + len(changed_files) + len(new_files) > 0:
                    complete_str += "({}{})".format(
                        util.sign_string(space_difference),
                        util.bytes_to_string(abs(space_difference),
                                             precision=2))
                else:
                    complete_str += "(No changes)"
            else:
                complete_str += "({}{})".format(
                    util.sign_string(space_difference),
                    util.bytes_to_string(abs(space_difference), precision=2))
            print("\n" + complete_str)
            set_status(complete_str)
            if num_errors > 0:
                log.log_print(
                    "There were {} error(s) reported during the backup.".
                    format(num_errors))
                print(
                    "Please check the log file for more info on the individual errors."
                )

            # Report on any errors and finalize the backup
            final_report_str = "Backup complete: {} files processed, {} new files, {} existing files modified, " + \
                               "{} files removed ({}, {}{})"
            log.log(
                final_report_str.format(
                    NUM_FILES_PROCESSED, NUM_FILES_NEW, NUM_FILES_MODIFIED,
                    NUM_FILES_DELETED,
                    util.bytes_to_string(TOTAL_SIZE_PROCESSED, 2),
                    util.sign_string(space_difference),
                    util.bytes_to_string(abs(space_difference), precision=2)))
            if NUM_FILES_ERROR > 0:
                log.log_print(
                    "There were {} error(s) reported during this backup.".
                    format(NUM_FILES_ERROR))
                print(
                    "Please check the log file for more info on the individual errors."
                )
            if not os.path.isfile(input_path):
                create_backup_text_file(backup_folder)
            increment_backup_number()
def backup_files(new_files, changed_files, remove_files):
    """
    The files provided in the given lists will be backed up. This will first delete all the files in the
    remove_files list, then copy over all the files in the new_files list, then modify each file in the
    changed_files list.
    :param new_files: A list of new files to backup generated by mark_files().
    :param changed_files: A list of changed files to backup generated by mark_files().
    :param remove_files: A list of files to delete from the backup generated by mark_files().
    :return: The number of errors that occurred.
    """
    # If there's no changes to make, display a message
    if len(new_files) == 0 and len(changed_files) == 0 and len(
            remove_files) == 0:
        print("No changes are needed.", end="\r", flush=True)

    # Prepare values that will track the progress of each section of the backup
    num_errors = 0
    count = 0
    limit = NUM_FILES_DELETED

    # Delete every file in the remove list
    for file_tuple in remove_files:
        delete_file_path = file_tuple[0]
        # Use the correct delete function based on if it's a file or folder
        try:
            set_status("Deleting {}".format(
                os.path.split(delete_file_path)[1]))
            if os.path.isdir(delete_file_path):
                deleted_file_count = util.rmtree(delete_file_path)
                for _ in range(deleted_file_count):
                    count += 1
                    print("Deleting old files: {}/{}".format(count, limit) +
                          ' ' * 20,
                          end="\r",
                          flush=True)
                increment_backup_progress()
            else:
                os.remove(delete_file_path)
                increment_backup_progress()
                count += 1
                print("Deleting old files: {}/{}".format(count, limit) +
                      ' ' * 20,
                      end="\r",
                      flush=True)
            log.log("DELETED - " + delete_file_path)
        except PermissionError:
            # Log the exception and indicate that an error occurred
            log.log_exception(delete_file_path, "DELETING")
            num_errors += 1

    # Reset the counter values and copy over every file in the new list
    count = 0
    limit = len(new_files)
    for file_tuple in new_files:
        new_file = file_tuple[0]
        output_path = file_tuple[2]
        try:
            set_status("Copying over {} ({})".format(
                os.path.split(new_file)[1],
                util.bytes_to_string(os.path.getsize(new_file), 2)))
            shutil.copy2(new_file, output_path)
            log.log("NEW - " + output_path)
        except PermissionError:
            # Write the full error to the log file and record that an error occurred
            log.log_exception(output_path, "CREATING")
            num_errors += 1
        count += 1
        increment_backup_progress()
        print("Copying over new files: {}/{}".format(count, limit) + ' ' * 20,
              end="\r",
              flush=True)

    # Reset the counter values and overwrite every file in the changed list
    count = 0
    limit = len(changed_files)
    for file_tuple in changed_files:
        new_file = file_tuple[0]
        output_path = file_tuple[2]
        try:
            set_status("Updating {}, ({})".format(
                os.path.split(new_file)[1],
                util.bytes_to_string(os.path.getsize(new_file), 2)))
            shutil.copy2(new_file, output_path)
            log.log("UPDATED - " + output_path)
        except PermissionError:
            # Write the full error to the log file and record that an error occurred
            log.log_exception(output_path, "UPDATING")
            num_errors += 1
        count += 1
        increment_backup_progress()
        print("Updating existing files: {}/{}".format(count, limit) + ' ' * 20,
              end="\r",
              flush=True)
    return num_errors
def mark_files(input_path, output_path, config, input_number, depth=0):
    """
    This is the file preparation stage of the backup process. The directory to be backed up is walked through, and
    all new files, changed files, and files that should be deleted are compiled into their respective lists,
    essentially "marking" those files for later. While the directory is walked, a directory skeleton structure
    is created in the output, so any directories that will have files sent to them later will exist in the output.
    :param input_path: The file or directory to backup.
    :param output_path: The file or directory in the drive to backup to.
    :param config: The current configuration.
    :param input_number: The index of the entry currently being worked with, starting from 1.
    :param depth: The depth of the recursive search. Will be 0 if not specified.
    :return: A tuple of three lists is returned.
             First is a list of new files. Each element of this list is a tuple of three values: first the absolute
             file path, second that file's size in bytes, and third the absolute file path from the output.
             Second is a list of changed files. Each element of this list is a tuple of four values: first the
             absolute file path from the input, second the size of the file from the input, third the absolute
             file path from the output, and fourth the size of the file from the output.
             Third is a list of files to delete. Each element of this list is a tuple of two values: first the absolute
             file path from the output, and second that file's size in bytes.
    """
    global THREAD_START_DEPTH
    # Don't continue down this path if it should be excluded
    if config.get_entry(input_number).should_exclude(input_path, output_path):
        log.log("EXCLUDED - " + input_path)
        return [], [], []

    # If this is a file, check what to do with it and increment counters as necessary
    if os.path.isfile(input_path):
        file_size = os.path.getsize(input_path)
        if os.path.exists(output_path):
            if not util.file_compare(input_path, output_path):
                # The file has changed and will be added to the update list
                mark_file_processed(file_size, modified=True)
                return [], [(input_path, file_size, output_path,
                             os.path.getsize(output_path))], []
            else:
                # The file needs no attention
                mark_file_processed(file_size)
                return [], [], []
        else:
            # The file is new and will be added to the new list
            mark_file_processed(file_size, is_new=True)
            return [(input_path, file_size, output_path)], [], []

    # Otherwise, it's a directory, so recurse on each child of the directory
    else:
        new_files = []
        changed_files = []
        remove_files = []

        # If this directory doesn't exist in the output, make it
        if not os.path.exists(output_path):
            try:
                os.mkdir(output_path)
                shutil.copymode(input_path, output_path)
            except PermissionError:
                # Log the exception and return so we don't process any of this directory's children
                log.log_exception(output_path, "CREATING DIRECTORY")
                increment_error()
                return [], [], []

        # Initialize values that will help in efficiently gathering names of files to remove
        input_dir_files = os.listdir(input_path)
        output_dir_files = os.listdir(output_path)
        output_dir_idx = 0
        len_output_dir = len(output_dir_files)
        param_list = []

        # Start by sorting the file lists so we can index them and compare them side by side
        input_dir_files.sort()
        output_dir_files.sort()

        try:
            # Check every file in the input
            for input_dir_idx in range(len(input_dir_files)):
                filename = input_dir_files[input_dir_idx]
                new_input = os.path.join(input_path, filename)
                new_output = os.path.join(output_path, filename)

                # Loop to check if this file exists in the output as well by looping through output files
                while output_dir_idx < len_output_dir:
                    # If it does, index over it and leave the loop, leaving the file in the output alone
                    if filename == output_dir_files[output_dir_idx]:
                        output_dir_idx += 1
                        break
                    # If this output file isn't the current input file, add it to the remove list
                    else:
                        # Stop checking if we are beyond where this file would alphabetically be
                        if filename < output_dir_files[output_dir_idx]:
                            break
                        else:
                            output_filename = os.path.join(
                                output_path, output_dir_files[output_dir_idx])
                            # Only add this to the list if it's not the old confirmation file
                            if not output_dir_files[output_dir_idx] == CONFIRMATION_FILENAME or \
                                    not input_path == config.get_entry(input_number).input:
                                if os.path.isdir(output_filename):
                                    delete_size, delete_files = util.directory_size(
                                        output_filename)
                                    for _ in range(delete_files):
                                        mark_file_processed(deleted=True)
                                else:
                                    mark_file_processed(deleted=True)
                                remove_files.append(
                                    (output_filename,
                                     os.path.getsize(output_filename)))
                            output_dir_idx += 1

                # If this is a directory, save parameters to spawn a thread later
                if os.path.isdir(new_input):
                    param_list.append([
                        new_input, new_output, config, input_number, depth + 1
                    ])
                # Otherwise, recurse and process this file here
                else:
                    temp_new, temp_changed, temp_remove = mark_files(
                        new_input, new_output, config, input_number, depth + 1)
                    new_files.extend(temp_new)
                    changed_files.extend(temp_changed)
                    remove_files.extend(temp_remove)

            # In spawn_threads mode, execute each list of parameters on a separate thread and combine the results
            with concurrent.futures.ThreadPoolExecutor() as executor:
                futures = []
                # Don't spawn a new thread if only one set of parameters is in the list
                if len(param_list) == 1:
                    temp_new, temp_changed, temp_remove = mark_files(
                        *param_list[0])
                    new_files.extend(temp_new)
                    changed_files.extend(temp_changed)
                    remove_files.extend(temp_remove)
                else:
                    for params in param_list:
                        # Only spawn new threads around the same depth in the file heirarchy
                        if ACTIVE_THREADS < MAX_THREADS and (
                                depth <= THREAD_START_DEPTH + 1
                                or THREAD_START_DEPTH == -1):
                            if THREAD_START_DEPTH == -1:
                                THREAD_START_DEPTH = depth
                            edit_thread_count(1)
                            futures.append(executor.submit(
                                mark_files, *params))
                        else:
                            temp_new, temp_changed, temp_remove = mark_files(
                                *params)
                            new_files.extend(temp_new)
                            changed_files.extend(temp_changed)
                            remove_files.extend(temp_remove)
            results = [f.result() for f in futures]
            edit_thread_count(-1 * len(results))
            for (temp_new, temp_changed, temp_remove) in results:
                new_files.extend(temp_new)
                changed_files.extend(temp_changed)
                remove_files.extend(temp_remove)

            # If there's still more files in the output that weren't looped over, add them all to the remove list
            if output_dir_idx < len_output_dir:
                for end_output_idx in range(output_dir_idx,
                                            len(output_dir_files)):
                    output_filename = os.path.join(
                        output_path, output_dir_files[end_output_idx])
                    # Only add this to the list if it's not the old confirmation file
                    if not output_dir_files[end_output_idx] == CONFIRMATION_FILENAME or \
                            not input_path == config.get_entry(input_number).input:
                        if os.path.isdir(output_filename):
                            delete_size, delete_files = util.directory_size(
                                output_filename)
                            for _ in range(delete_files):
                                mark_file_processed(deleted=True)
                        else:
                            mark_file_processed(deleted=True)
                        remove_files.append((output_filename,
                                             os.path.getsize(output_filename)))

        except FileNotFoundError as error:
            # Display a warning if long paths need to be enabled on Windows
            if len(input_path) >= 260:
                print("FileNotFoundError: Unable to access " + input_path)
                print(
                    "This is likely because the file path is longer than 260 characters."
                )
                print(
                    "If you are running this on Windows, set LongPathsEnabled to 1 in your registry."
                )
            else:
                print(error)
            exit(1)

        # Show the current progress and return
        progress_str = "{} files found, {} ({} new, {} changed, {} to remove)".format(
            NUM_FILES_PROCESSED, util.bytes_to_string(TOTAL_SIZE_PROCESSED, 2),
            NUM_FILES_NEW, NUM_FILES_MODIFIED, NUM_FILES_DELETED)
        print(progress_str + ' ' * 10, end="\r", flush=True)
        return new_files, changed_files, remove_files