예제 #1
0
파일: extract.py 프로젝트: adelosa/mciutil
def extract_command(args):
    """
    extract command
    :param args: arg object
    :return:
    """

    # Read input file
    with open(args.input, 'rb') as infile:
        input_file = infile.read()
    LOGGER.info("%s bytes read from %s", len(input_file), args.input)

    # Unblock input
    input_file = unblock(input_file)

    # get config filename
    config_filename = get_config_filename("mideu.yml")
    LOGGER.info("Config file: %s", config_filename)

    # load the config from yaml file
    with open(config_filename, 'r') as config_file:
        config = yaml.load(config_file)

    # parse the records
    output_list = [
        get_message_elements(
            record,
            config["bit_config"],
            args.sourceformat
        ) for record in input_file
    ]

    print("\nCompleted processing {0} records".format(len(input_file)))

    # write to output
    if args.mongo or args.mongohost or args.mongodb:
        mongo_config = config["mongo_config"]
        if args.mongohost:
            mongo_config["host"] = args.mongohost
        if args.mongodb:
            mongo_config["db"] = args.mongodb

        add_to_mongo(
            output_list,
            config['output_data_elements'],
            mongo_config)

    else:  # default write to csv
        if args.csvoutputfile:
            csv_output_filename = args.csvoutputfile
        else:
            csv_output_filename = args.input + ".csv"
        add_to_csv(
            output_list,
            config['output_data_elements'],
            csv_output_filename
        )
예제 #2
0
def extract_command(args):
    """
    extract command
    :param args: arg object
    :return:
    """

    # Read input file
    with open(args.input, 'rb') as infile:
        input_file = infile.read()
    LOGGER.info("%s bytes read from %s", len(input_file), args.input)

    # Unblock input
    input_file = unblock(input_file)

    # get config filename
    config_filename = get_config_filename("mideu.yml")
    LOGGER.info("Config file: %s", config_filename)

    # load the config from yaml file
    with open(config_filename, 'r') as config_file:
        config = yaml.load(config_file)

    # parse the records
    output_list = [
        get_message_elements(record, config["bit_config"], args.sourceformat)
        for record in input_file
    ]

    print("\nCompleted processing {0} records".format(len(input_file)))

    # write to output
    if args.mongo or args.mongohost or args.mongodb:
        mongo_config = config["mongo_config"]
        if args.mongohost:
            mongo_config["host"] = args.mongohost
        if args.mongodb:
            mongo_config["db"] = args.mongodb

        add_to_mongo(output_list, config['output_data_elements'], mongo_config)

    else:  # default write to csv
        if args.csvoutputfile:
            csv_output_filename = args.csvoutputfile
        else:
            csv_output_filename = args.input + ".csv"
        add_to_csv(output_list, config['output_data_elements'],
                   csv_output_filename)
예제 #3
0
    def test_get_message_elements_ebcdic(self):
        message_raw = (_convert_text_asc2eb(b("1144")) +
                       b("\xF0\x10\x05\x42\x84\x61\x80\x02\x02\x00\x00\x04") +
                       b("\x00\x00\x00\x00") + _convert_text_asc2eb(
                           b("164444555544445555111111000000009999201508"
                             "151715123456789012333123423579957991200000"
                             "012306120612345612345657994211111111145BIG"
                             " BOBS\\70 FERNDALE ST\\ANNERLEY\\4103  QLD"
                             "AUS0080001001Y9990160000000000000001123456"
                             "7806999999")))
        message_elements = mciutil.get_message_elements(
            message_raw, CONFIG['data_elements'], 'ebcdic')

        # print message_elements
        self.assertEqual(message_elements["DE2"], b"444455*******555")
        self.assertEqual(message_elements["DE3"], b"111111")
        self.assertEqual(message_elements["DE4"], b"000000009999")
예제 #4
0
    def test_get_message_elements_ascii(self):
        message_raw = b("1144\xF0\x10\x05\x42\x84\x61\x80\x02\x02\x00\x00\x04"
                        "\x00\x00\x00\x00" +
                        "1644445555444455551111110000000099992015081517151234"
                        "5678901233312342357995799120000001230612061234561234"
                        "5657994211111111145BIG BOBS\\70 FERNDALE ST\\ANNERLE"
                        "Y\\4103  QLDAUS0080001001Y99901600000000000000011234"
                        "567806999999")

        message_elements = mciutil.get_message_elements(
            message_raw, CONFIG['data_elements'], 'ascii')

        print(message_elements)

        # print message_elements
        self.assertEqual(message_elements["DE2"], b"444455*******555")
        self.assertEqual(message_elements["DE3"], b"111111")
        self.assertEqual(message_elements["DE4"], b"000000009999")
        self.assertEqual(message_elements["PDS0001"], b"Y")
예제 #5
0
    def test_get_message_elements_ascii(self):
        message_raw = b(
            "1144\xF0\x10\x05\x42\x84\x61\x80\x02\x02\x00\x00\x04"
            "\x00\x00\x00\x00" + "1644445555444455551111110000000099992015081517151234"
            "5678901233312342357995799120000001230612061234561234"
            "5657994211111111145BIG BOBS\\70 FERNDALE ST\\ANNERLE"
            "Y\\4103  QLDAUS0080001001Y99901600000000000000011234"
            "567806999999"
        )

        message_elements = mciutil.get_message_elements(message_raw, CONFIG["data_elements"], "ascii")

        print(message_elements)

        # print message_elements
        self.assertEqual(message_elements["DE2"], b"444455*******555")
        self.assertEqual(message_elements["DE3"], b"111111")
        self.assertEqual(message_elements["DE4"], b"000000009999")
        self.assertEqual(message_elements["PDS0001"], b"Y")
예제 #6
0
    def test_get_message_elements_ebcdic(self):
        message_raw = (
            _convert_text_asc2eb(b("1144"))
            + b("\xF0\x10\x05\x42\x84\x61\x80\x02\x02\x00\x00\x04")
            + b("\x00\x00\x00\x00")
            + _convert_text_asc2eb(
                b(
                    "164444555544445555111111000000009999201508"
                    "151715123456789012333123423579957991200000"
                    "012306120612345612345657994211111111145BIG"
                    " BOBS\\70 FERNDALE ST\\ANNERLEY\\4103  QLD"
                    "AUS0080001001Y9990160000000000000001123456"
                    "7806999999"
                )
            )
        )
        message_elements = mciutil.get_message_elements(message_raw, CONFIG["data_elements"], "ebcdic")

        # print message_elements
        self.assertEqual(message_elements["DE2"], b"444455*******555")
        self.assertEqual(message_elements["DE3"], b"111111")
        self.assertEqual(message_elements["DE4"], b"000000009999")