コード例 #1
0
def do_backup(input_file):
    counter = 1
    proposed_backup_folder_path = os.path.join(os.path.dirname(os.path.abspath(input_file)), "backups")
    if os.path.isdir(proposed_backup_folder_path) or not os.path.exists(proposed_backup_folder_path):
        backup_folder_path = proposed_backup_folder_path
    else:
        while os.path.isfile(proposed_backup_folder_path + str(counter)):
            counter += 1
        backup_folder_path = proposed_backup_folder_path + str(counter)
    if not os.path.exists(backup_folder_path):
        os.mkdir(backup_folder_path)
    backup_path = os.path.join(backup_folder_path,
                               os.path.basename(input_file) + '.bak' + "-" + str(time.ctime()).replace(":", "-"))
    shutil.copy(input_file, backup_path)
    clear_old_logs.do_clear(os.path.dirname(backup_path), 50)
    return backup_path
コード例 #2
0
def process(database_connection, folders_database, run_log, emails_table, run_log_directory,
            reporting, processed_files, root, args, version, errors_folder, edi_converter_scratch_folder, settings,
            simple_output=None):
    def update_overlay(overlay_text, dispatch_folder_count, folder_total, dispatch_file_count, file_total, footer):
        if not args.automatic:
            doingstuffoverlay.update_overlay(parent=root,
                                             overlay_text=overlay_text + " folder " +
                                                          str(dispatch_folder_count) + " of " +
                                                          str(folder_total) + "," + " file " +
                                                          str(dispatch_file_count) + " of " +
                                                          str(file_total), footer=footer, overlay_height=120)
        elif simple_output is not None:
            simple_output.configure(text=overlay_text + " folder " +
                                         str(dispatch_folder_count) + " of " +
                                         str(folder_total) + "," + " file " +
                                         str(dispatch_file_count) + " of " +
                                         str(file_total))
        root.update()

    def empty_directory(top):
        for folder_root, dirs, files_delete in os.walk(top, topdown=False):
            for name in files_delete:
                os.remove(os.path.join(folder_root, name))
            for name in dirs:
                os.rmdir(os.path.join(folder_root, name))

    global edi_validator_errors
    global global_edi_validator_error_status
    edi_validator_errors = StringIO()
    global_edi_validator_error_status = False

    def validate_file(input_file, file_name):
        global edi_validator_errors
        global global_edi_validator_error_status
        edi_validator_output, edi_validator_error_status, minor_edi_errors = mtc_edi_validator.report_edi_issues(input_file)
        if minor_edi_errors or edi_validator_error_status and reporting['report_edi_errors']:
            edi_validator_errors.write("\r\nErrors for " + file_name + ":\r\n")
            edi_validator_errors.write(edi_validator_output.getvalue())
            edi_validator_output.close()
            global_edi_validator_error_status = True
        return edi_validator_error_status

    error_counter = 0
    processed_counter = 0
    folder_count = 0
    folder_total_count = folders_database.count(folder_is_active="True")
    # loop over all known active folders, in order of alias name
    for parameters_dict in folders_database.find(folder_is_active="True", order_by="alias"):
        global filename
        global send_filename
        folder_count += 1
        file_count = 0
        file_count_total = 0
        update_overlay("processing folder...\n\n", folder_count, folder_total_count, file_count, file_count_total, "")
        if os.path.isdir(parameters_dict['folder_name']) is True:
            print("entering folder " + parameters_dict['folder_name'] + ", aliased as " + parameters_dict['alias'])
            run_log.write(("\r\n\r\nentering folder " + parameters_dict['folder_name'] + ", aliased as " +
                           parameters_dict['alias'] + "\r\n\r\n").encode())
            os.chdir(parameters_dict['folder_name'])
            # strip potentially invalid send_filename characters from alias string
            cleaned_alias_string = re.sub('[^a-zA-Z0-9 ]', '', parameters_dict['alias'])
            # add iso8601 date/time stamp to send_filename, but filter : for - due to send_filename constraints
            folder_error_log_name_constructor = \
                cleaned_alias_string + " errors." + str(time.ctime()).replace(":", "-") + ".txt"
            folder_error_log_name_full_path = os.path.join(errors_folder['errors_folder'],
                                                           os.path.basename(parameters_dict['folder_name']),
                                                           folder_error_log_name_constructor)
            folder_errors_log = StringIO()
            files = [f for f in os.listdir('.')]  # create list of all files in directory
            filtered_files = []
            file_count_total = len(files)
            run_log.write("Checking for new files\r\n".encode())
            print("Checking for new files")
            for f in files:
                file_count += 1
                update_overlay("processing folder... (checking files)\n\n", folder_count, folder_total_count,
                               file_count, file_count_total, "Checking File: " + f)
                if processed_files.find_one(file_name=os.path.join(os.getcwd(), f),
                                            file_checksum=hashlib.md5(open(f, 'rb').read()).hexdigest()) is None or \
                        processed_files.find_one(file_name=os.path.join(os.getcwd(), f), resend_flag=True):
                    filtered_files.append(f)

            file_count = 0
            errors = False
            if len(files) == 0:  # if there are no files in directory, record in log
                run_log.write("No files in directory\r\n\r\n".encode())
                print("No files in directory")
            if len(filtered_files) == 0 and len(files) > 0:
                run_log.write("No new files in directory\r\n\r\n".encode())
                print("No new files in directory")
            if len(filtered_files) != 0:
                run_log.write((str(len(filtered_files)) + " found\r\n\r\n").encode())
                print(str(len(filtered_files)) + " found")
            file_count_total = len(filtered_files)
            for filename in filtered_files:  # iterate over all files in directory
                empty_directory(edi_converter_scratch_folder['edi_converter_scratch_folder'])
                filename = os.path.abspath(filename)
                original_filename = filename
                file_count += 1
                update_overlay("processing folder...\n\n", folder_count, folder_total_count,
                               file_count, file_count_total, "Sending File: " + os.path.basename(original_filename))
                valid_edi_file = True
                if (parameters_dict['process_edi'] == "True"
                        or parameters_dict['tweak_edi']
                        or parameters_dict['split_edi'])\
                        or parameters_dict['force_edi_validation']:
                    if validate_file(filename, original_filename):
                        valid_edi_file = False
                if parameters_dict['split_edi'] and valid_edi_file:
                    run_log.write(("Splitting edi file " + original_filename + "...\r\n").encode())
                    print("Splitting edi file " + original_filename + "...")
                    try:
                        split_edi_list = split_edi.do_split_edi(filename, edi_converter_scratch_folder[
                            'edi_converter_scratch_folder'])
                        if len(split_edi_list) > 1:
                            run_log.write(
                                ("edi file split into " + str(len(split_edi_list)) + " files\r\n\r\n").encode())
                            print("edi file split into " + str(len(split_edi_list)) + " files")
                    except Exception as error:
                        split_edi_list = [filename]
                        record_error.do(run_log, folder_errors_log,
                                        "splitting edi file failed with error: " + str(error), filename, "edi splitter")
                else:
                    split_edi_list = [filename]
                if len(split_edi_list) <= 1 and parameters_dict['split_edi']:
                    run_log.write("Cannot split edi file\r\n\r\n".encode())
                    print("Cannot split edi file")
                    split_edi_list = [filename]
                for send_filename in split_edi_list:
                    if errors is True:
                        break
                    stripped_filename = re.sub('[^A-Za-z0-9. ]+', '', os.path.basename(send_filename))
                    if os.path.exists(send_filename):
                        if valid_edi_file:
                            if parameters_dict['process_edi'] == "True" and errors is False:
                                # if the current file is recognized as a valid edi file,
                                # then allow conversion, otherwise log and carry on
                                if parameters_dict['convert_to_format'] == "csv":
                                    output_filename = os.path.join(
                                        edi_converter_scratch_folder['edi_converter_scratch_folder'],
                                        os.path.basename(stripped_filename) + ".csv")
                                    if os.path.exists(os.path.dirname(output_filename)) is False:
                                        os.mkdir(os.path.dirname(output_filename))
                                    try:
                                        run_log.write(("converting " + send_filename + " from EDI to CSV\r\n").encode())
                                        print("converting " + send_filename + " from EDI to CSV")
                                        convert_to_csv.edi_convert(send_filename, output_filename,
                                                                   parameters_dict['calculate_upc_check_digit'],
                                                                   parameters_dict['include_a_records'],
                                                                   parameters_dict['include_c_records'],
                                                                   parameters_dict['include_headers'],
                                                                   parameters_dict['filter_ampersand'],
                                                                   parameters_dict['pad_a_records'],
                                                                   parameters_dict['a_record_padding'])
                                        run_log.write("Success\r\n\r\n".encode())
                                        send_filename = output_filename
                                    except Exception as error:
                                        print(str(error))
                                        errors = True
                                        record_error.do(run_log, folder_errors_log, str(error), str(send_filename),
                                                        "EDI Processor")

                            if parameters_dict['tweak_edi'] is True:
                                output_filename = os.path.join(edi_converter_scratch_folder['edi_converter_scratch_folder'],
                                                               os.path.basename(stripped_filename))
                                if os.path.exists(os.path.dirname(output_filename)) is False:
                                    os.mkdir(os.path.dirname(output_filename))
                                try:
                                    edi_tweaks.edi_tweak(send_filename, output_filename,
                                                         parameters_dict['pad_a_records'],
                                                         parameters_dict['a_record_padding'])
                                    send_filename = output_filename
                                except Exception as error:
                                    print(str(error))
                                    errors = True
                                    record_error.do(run_log, folder_errors_log, str(error), str(send_filename),
                                                    "EDI Processor")

                        # the following blocks process the files using the specified backend,
                        # and log in the event of any errors
                        if parameters_dict['process_backend_copy'] is True and errors is False:
                            try:
                                print("sending " + str(send_filename) + " to " +
                                      str(parameters_dict['copy_to_directory']) + " with copy backend")
                                run_log.write(("sending " + str(send_filename) + " to " +
                                               str(parameters_dict[
                                                       'copy_to_directory']) + " with copy backend\r\n\r\n").encode())
                                copy_backend.do(parameters_dict, send_filename)
                                run_log.write("Success\r\n\r\n".encode())
                            except Exception as error:
                                print(str(error))
                                record_error.do(run_log, folder_errors_log, str(error), str(send_filename),
                                                "Copy Backend")
                                errors = True
                        if parameters_dict['process_backend_ftp'] is True and errors is False:
                            try:
                                print(
                                    "sending " + str(send_filename) + " to " + str(parameters_dict['ftp_server']) +
                                    str(parameters_dict['ftp_folder']) + " with FTP backend")
                                run_log.write(
                                    ("sending " + str(send_filename) + " to " + str(parameters_dict['ftp_server']) +
                                     str(parameters_dict['ftp_folder']) + " with FTP backend\r\n\r\n").encode())
                                ftp_backend.do(parameters_dict, send_filename)
                                run_log.write("Success\r\n\r\n".encode())
                            except Exception as error:
                                print(str(error))
                                record_error.do(run_log, folder_errors_log, str(error), str(send_filename),
                                                "FTP Backend")
                                errors = True
                        if parameters_dict['process_backend_email'] is True and errors is False and \
                                settings['enable_email']:
                            try:
                                print("sending " + str(send_filename) + " to " + str(parameters_dict['email_to']) +
                                      " with email backend")
                                run_log.write(
                                    ("sending " + str(send_filename) + " to " + str(parameters_dict['email_to']) +
                                     " with email backend\r\n\r\n").encode())
                                email_backend.do(parameters_dict, settings, send_filename)
                                run_log.write("Success\r\n\r\n".encode())
                            except Exception as error:
                                record_error.do(run_log, folder_errors_log, str(error), str(send_filename),
                                                "Email Backend")
                                errors = True
                if errors is False:
                    try:
                        if processed_files.count(file_name=str(original_filename),
                                                 resend_flag=True) > 0:
                            file_old_id = processed_files.find_one(file_name=str(original_filename), resend_flag=True)
                            processed_files_update = dict(resend_flag=False, id=file_old_id['id'])
                            processed_files.update(processed_files_update, ['id'])

                        processed_files.insert(dict(file_name=str(original_filename),
                                                    folder_id=parameters_dict['id'],
                                                    folder_alias=parameters_dict['alias'],
                                                    file_checksum=hashlib.md5(open(original_filename, 'rb').read())
                                                    .hexdigest(),
                                                    sent_date_time=datetime.datetime.now(),
                                                    copy_destination=parameters_dict['copy_to_directory'] if
                                                    parameters_dict['process_backend_copy'] is True else "N/A",
                                                    ftp_destination=parameters_dict['ftp_server'] +
                                                                    parameters_dict['ftp_folder'] if
                                                    parameters_dict['process_backend_ftp'] is True else "N/A",
                                                    email_destination=parameters_dict['email_to'] if
                                                    parameters_dict['process_backend_email'] is True else "N/A",
                                                    resend_flag=False))
                    except Exception as error:
                        record_error.do(run_log, folder_errors_log, str(error), str(original_filename), "Dispatch")
                        errors = True
                if errors is False:
                    processed_counter += 1
                else:
                    break
            if errors is True:
                error_counter += 1
                if os.path.exists(errors_folder['errors_folder']) is False:
                    record_error.do(run_log, folder_errors_log, "Base errors folder not found",
                                    str(parameters_dict['folder_name']), "Dispatch Error Logger")
                    print("Base error folder not found, " + "making one")
                    os.mkdir(errors_folder['errors_folder'])
                if os.path.exists(os.path.dirname(folder_error_log_name_full_path)) is False:
                    record_error.do(run_log, folder_errors_log, "Error folder Not Found",
                                    str(parameters_dict['folder_name']),
                                    "Dispatch Error Logger")
                    print("Error folder not found for " + parameters_dict['folder_name'] + ", " + "making one")
                    try:
                        os.mkdir(os.path.dirname(folder_error_log_name_full_path))
                    except IOError:  # if we can't create error logs folder, put it in run log directory
                        record_error.do(run_log, folder_errors_log, "Error creating errors folder",
                                        str(parameters_dict['folder_name']), "Dispatch Error Logger")
                        folder_error_log_name_full_path = os.path.join(run_log_directory,
                                                                       folder_error_log_name_constructor)
                try:
                    folder_errors_log_write = open(folder_error_log_name_full_path, 'wb')
                    clear_old_logs.do_clear(os.path.dirname(folder_error_log_name_full_path), 500)
                    folder_errors_log_write.write(("Program Version = " + version + "\r\n\r\n").encode())
                    folder_errors_log_write.write(folder_errors_log.getvalue().encode())
                    if reporting['enable_reporting'] == "True":
                        emails_table.insert(dict(log=folder_error_log_name_full_path,
                                                 folder_alias=parameters_dict['alias'],
                                                 folder_id=parameters_dict['id']))
                except Exception as error:
                    # if file can't be created in either directory, put error log inline in the run log
                    print("can't open error log file,\r\n error is " + str(error) + "\r\ndumping to run log")
                    run_log.write(("can't open error log file,\r\n error is " + str(error) +
                                   "\r\ndumping to run log\r\n").encode())
                    run_log.write(("error log name was: " + folder_error_log_name_constructor + "\r\n\r\n").encode())
                    run_log.write(folder_errors_log.getvalue().encode())
                    run_log.write("\r\n\r\nEnd of Error file\r\n\r\n".encode())
            else:
                database_connection.query("DELETE FROM processed_files WHERE ROWID IN (SELECT id FROM processed_files"
                                          " WHERE folder_id=" + str(parameters_dict['id']) +
                                          " ORDER BY id DESC LIMIT -1 OFFSET 5000)")
                processed_files_update = dict(resend_flag=False, folder_id=parameters_dict['id'])
                processed_files.update(processed_files_update, ['folder_id'])
            folder_errors_log.close()
        else:
            # if the folder is missing, complain and increment error counter
            run_log.write(("\r\nerror: " + os.path.abspath(parameters_dict['folder_name']) + " is missing " + " for " +
                           parameters_dict['alias'] + "\r\n\r\n").encode())
            print("error: " + os.path.abspath(parameters_dict['folder_name']) + " is missing " + " for " +
                  parameters_dict['alias'])
            error_counter += 1
    if global_edi_validator_error_status is True:
        validator_log_name_constructor = "Validator Log " + str(time.ctime()).replace(":", "-") + ".txt"
        validator_log_path = os.path.join(run_log_directory, validator_log_name_constructor)
        validator_log_file = open(validator_log_path, 'wb')
        validator_log_file.write(edi_validator_errors.getvalue().encode())
        edi_validator_errors.close()
        validator_log_file.close()
        if reporting['enable_reporting'] == "True":
            emails_table.insert(dict(log=validator_log_path))
    print(str(processed_counter) + " processed, " + str(error_counter) + " errors")
    run_log.write(
        ("\r\n\r\n" + str(processed_counter) + " processed, " + str(error_counter) + " errors" + "\r\n\r\n").encode())
    if error_counter > 0:
        return True
    else:
        return False