Exemple #1
0
 def test_unblock(self):
     umodedata = ((b("\x00\x00\x00\x0A1234567890") * 72) +
                  b("\x00\x00\x00\x0A\x40\x401234567890"))
     records = mciutil.unblock(umodedata)
     print(len(records))
     print(records[0])
     print(records[72])
Exemple #2
0
def _main(args):
    """
    paramconv main processing

    :param args: argparse arguments
    :return: exit code
    """

    logging.basicConfig(
        level=args.loglevel,
        format='%(asctime)s:%(name)s:%(lineno)s:%(levelname)s:%(message)s'
    )

    # exit if input file does not exist
    if not os.path.isfile(args.input):
        print("Input file not found - {0}".format(args.input))
        exit(8)

    input_filename = args.input
    output_filename = args.output
    if not args.output:
        output_filename = input_filename + ".out"

    # read file to string
    with open(input_filename, 'rb') as infile:
        input_file = infile.read()

    print("{0} bytes read from {1}".format(len(input_file), input_filename))

    # deblock the file
    input_file = unblock(input_file)

    # convert the file
    output_list = [
        _convert(record, args.sourceformat) for record in input_file
    ]

    # re-block the data
    output_data = block(output_list)

    # save to file
    with open(output_filename, "wb") as output_file:
        output_file.write(output_data)

    print("{0} bytes written to {1}".format(len(output_data), output_filename))
    print("{0} records".format(len(output_list)))

    if args.loglevel == logging.DEBUG:
        print("DEBUG:Input first 5000 bytes")
        hexdump(input_file[:5000])
        print("DEBUG:Output first 5000 bytes")
        hexdump(output_data[:5000])

        print("DEBUG:Input last 5000 bytes")
        hexdump(input_file[len(output_data)-5000:len(input_file)])
        print("DEBUG:Output last 5000 bytes")
        hexdump(output_data[len(output_data)-5000:len(output_data)])

    print("Done!")
Exemple #3
0
def _main(args):
    """
    paramconv main processing

    :param args: argparse arguments
    :return: exit code
    """

    logging.basicConfig(
        level=args.loglevel,
        format='%(asctime)s:%(name)s:%(lineno)s:%(levelname)s:%(message)s')

    # exit if input file does not exist
    if not os.path.isfile(args.input):
        print("Input file not found - {0}".format(args.input))
        exit(8)

    input_filename = args.input
    output_filename = args.output
    if not args.output:
        output_filename = input_filename + ".out"

    # read file to string
    with open(input_filename, 'rb') as infile:
        input_file = infile.read()

    print("{0} bytes read from {1}".format(len(input_file), input_filename))

    # deblock the file
    input_file = unblock(input_file)

    # convert the file
    output_list = [
        _convert(record, args.sourceformat) for record in input_file
    ]

    # re-block the data
    output_data = block(output_list)

    # save to file
    with open(output_filename, "wb") as output_file:
        output_file.write(output_data)

    print("{0} bytes written to {1}".format(len(output_data), output_filename))
    print("{0} records".format(len(output_list)))

    if args.loglevel == logging.DEBUG:
        print("DEBUG:Input first 5000 bytes")
        hexdump(input_file[:5000])
        print("DEBUG:Output first 5000 bytes")
        hexdump(output_data[:5000])

        print("DEBUG:Input last 5000 bytes")
        hexdump(input_file[len(output_data) - 5000:len(input_file)])
        print("DEBUG:Output last 5000 bytes")
        hexdump(output_data[len(output_data) - 5000:len(output_data)])

    print("Done!")
Exemple #4
0
 def test_block(self):
     linebreakdata = []
     for x in range(0, 73):
         linebreakdata.append(b("1234567890"))
     output = mciutil.block(linebreakdata)
     hexdump.hexdump(output)
     input = mciutil.unblock(output)
     print(len(input))
     print(input)
Exemple #5
0
def extract_command(args):
    """
    extract command
    :param args: arg object
    :return:
    """

    # Read input file
    with open(args.input, 'rb') as infile:
        input_file = infile.read()
    LOGGER.info("%s bytes read from %s", len(input_file), args.input)

    # Unblock input
    input_file = unblock(input_file)

    # get config filename
    config_filename = get_config_filename("mideu.yml")
    LOGGER.info("Config file: %s", config_filename)

    # load the config from yaml file
    with open(config_filename, 'r') as config_file:
        config = yaml.load(config_file)

    # parse the records
    output_list = [
        get_message_elements(
            record,
            config["bit_config"],
            args.sourceformat
        ) for record in input_file
    ]

    print("\nCompleted processing {0} records".format(len(input_file)))

    # write to output
    if args.mongo or args.mongohost or args.mongodb:
        mongo_config = config["mongo_config"]
        if args.mongohost:
            mongo_config["host"] = args.mongohost
        if args.mongodb:
            mongo_config["db"] = args.mongodb

        add_to_mongo(
            output_list,
            config['output_data_elements'],
            mongo_config)

    else:  # default write to csv
        if args.csvoutputfile:
            csv_output_filename = args.csvoutputfile
        else:
            csv_output_filename = args.input + ".csv"
        add_to_csv(
            output_list,
            config['output_data_elements'],
            csv_output_filename
        )
Exemple #6
0
 def test_block(self):
     linebreakdata = []
     for x in range(0, 73):
         linebreakdata.append(b("1234567890"))
     output = mciutil.block(linebreakdata)
     hexdump.hexdump(output)
     input = mciutil.unblock(output)
     print(len(input))
     print(input)
Exemple #7
0
def extract_command(args):
    """
    extract command
    :param args: arg object
    :return:
    """

    # Read input file
    with open(args.input, 'rb') as infile:
        input_file = infile.read()
    LOGGER.info("%s bytes read from %s", len(input_file), args.input)

    # Unblock input
    input_file = unblock(input_file)

    # get config filename
    config_filename = get_config_filename("mideu.yml")
    LOGGER.info("Config file: %s", config_filename)

    # load the config from yaml file
    with open(config_filename, 'r') as config_file:
        config = yaml.load(config_file)

    # parse the records
    output_list = [
        get_message_elements(record, config["bit_config"], args.sourceformat)
        for record in input_file
    ]

    print("\nCompleted processing {0} records".format(len(input_file)))

    # write to output
    if args.mongo or args.mongohost or args.mongodb:
        mongo_config = config["mongo_config"]
        if args.mongohost:
            mongo_config["host"] = args.mongohost
        if args.mongodb:
            mongo_config["db"] = args.mongodb

        add_to_mongo(output_list, config['output_data_elements'], mongo_config)

    else:  # default write to csv
        if args.csvoutputfile:
            csv_output_filename = args.csvoutputfile
        else:
            csv_output_filename = args.input + ".csv"
        add_to_csv(output_list, config['output_data_elements'],
                   csv_output_filename)
Exemple #8
0
def convert_command(args):
    """
    convert command
    :param args: arg object
    :return:
    """

    # Read input file
    with open(args.input, 'rb') as input_file:
        input_data = input_file.read()
    LOGGER.info("%s bytes read from %s", len(input_data), args.input)

    # Unblock input
    if args.no_1014_blocking:
        input_data = vbs_unpack(input_data)
    else:
        input_data = unblock(input_data)

    # get config filename
    config_filename = get_config_filename("mideu.yml")
    LOGGER.info("Config file: %s", config_filename)

    # load the config from yaml file
    with open(config_filename, 'r') as config_file:
        config = yaml.load(config_file)

    output_records = [
        flip_message_encoding(record, config["bit_config"], args.sourceformat)
        for record in input_data
    ]

    if args.no_1014_blocking:
        output_data = vbs_pack(output_records)
    else:
        output_data = block(output_records)

    print("\nCompleted processing {0} records".format(len(input_data)))

    # save to file
    output_filename = args.input + ".out"
    with open(output_filename, "wb") as output_file:
        output_file.write(output_data)
Exemple #9
0
def convert_command(args):
    """
    convert command
    :param args: arg object
    :return:
    """

    # Read input file
    with open(args.input, 'rb') as input_file:
        input_data = input_file.read()
    LOGGER.info("%s bytes read from %s", len(input_data), args.input)

    # Unblock input
    input_data = unblock(input_data)

    # get config filename
    config_filename = get_config_filename("mideu.yml")
    LOGGER.info("Config file: %s", config_filename)

    # load the config from yaml file
    with open(config_filename, 'r') as config_file:
        config = yaml.load(config_file)

    output_records = [
        flip_message_encoding(
            record,
            config["bit_config"],
            args.sourceformat
        ) for record in input_data
    ]

    output_data = block(output_records)

    print("\nCompleted processing {0} records".format(len(input_data)))

    # save to file
    output_filename = args.input + ".out"
    with open(output_filename, "wb") as output_file:
        output_file.write(output_data)
Exemple #10
0
 def test_unblock(self):
     umodedata = (b("\x00\x00\x00\x0A1234567890") * 72) + b("\x00\x00\x00\x0A\x40\x401234567890")
     records = mciutil.unblock(umodedata)
     print(len(records))
     print(records[0])
     print(records[72])