def extract_command(args):
"""
extract command
:param args: arg object
:return:
"""
# Read input file
with open(args.input, 'rb') as infile:
input_file = infile.read()
LOGGER.info("%s bytes read from %s", len(input_file), args.input)
# Unblock input
input_file = unblock(input_file)
# get config filename
config_filename = get_config_filename("mideu.yml")
LOGGER.info("Config file: %s", config_filename)
# load the config from yaml file
with open(config_filename, 'r') as config_file:
config = yaml.load(config_file)
# parse the records
output_list = [
get_message_elements(
record,
config["bit_config"],
args.sourceformat
) for record in input_file
]
print("\nCompleted processing {0} records".format(len(input_file)))
# write to output
if args.mongo or args.mongohost or args.mongodb:
mongo_config = config["mongo_config"]
if args.mongohost:
mongo_config["host"] = args.mongohost
if args.mongodb:
mongo_config["db"] = args.mongodb
add_to_mongo(
output_list,
config['output_data_elements'],
mongo_config)
else: # default write to csv
if args.csvoutputfile:
csv_output_filename = args.csvoutputfile
else:
csv_output_filename = args.input + ".csv"
add_to_csv(
output_list,
config['output_data_elements'],
csv_output_filename
)
def extract_command(args):
"""
extract command
:param args: arg object
:return:
"""
# Read input file
with open(args.input, 'rb') as infile:
input_file = infile.read()
LOGGER.info("%s bytes read from %s", len(input_file), args.input)
# Unblock input
input_file = unblock(input_file)
# get config filename
config_filename = get_config_filename("mideu.yml")
LOGGER.info("Config file: %s", config_filename)
# load the config from yaml file
with open(config_filename, 'r') as config_file:
config = yaml.load(config_file)
# parse the records
output_list = [
get_message_elements(record, config["bit_config"], args.sourceformat)
for record in input_file
]
print("\nCompleted processing {0} records".format(len(input_file)))
# write to output
if args.mongo or args.mongohost or args.mongodb:
mongo_config = config["mongo_config"]
if args.mongohost:
mongo_config["host"] = args.mongohost
if args.mongodb:
mongo_config["db"] = args.mongodb
add_to_mongo(output_list, config['output_data_elements'], mongo_config)
else: # default write to csv
if args.csvoutputfile:
csv_output_filename = args.csvoutputfile
else:
csv_output_filename = args.input + ".csv"
add_to_csv(output_list, config['output_data_elements'],
csv_output_filename)
def test_get_message_elements_ebcdic(self):
message_raw = (_convert_text_asc2eb(b("1144")) +
b("\xF0\x10\x05\x42\x84\x61\x80\x02\x02\x00\x00\x04") +
b("\x00\x00\x00\x00") + _convert_text_asc2eb(
b("164444555544445555111111000000009999201508"
"151715123456789012333123423579957991200000"
"012306120612345612345657994211111111145BIG"
" BOBS\\70 FERNDALE ST\\ANNERLEY\\4103 QLD"
"AUS0080001001Y9990160000000000000001123456"
"7806999999")))
message_elements = mciutil.get_message_elements(
message_raw, CONFIG['data_elements'], 'ebcdic')
# print message_elements
self.assertEqual(message_elements["DE2"], b"444455*******555")
self.assertEqual(message_elements["DE3"], b"111111")
self.assertEqual(message_elements["DE4"], b"000000009999")
def test_get_message_elements_ascii(self):
message_raw = b("1144\xF0\x10\x05\x42\x84\x61\x80\x02\x02\x00\x00\x04"
"\x00\x00\x00\x00" +
"1644445555444455551111110000000099992015081517151234"
"5678901233312342357995799120000001230612061234561234"
"5657994211111111145BIG BOBS\\70 FERNDALE ST\\ANNERLE"
"Y\\4103 QLDAUS0080001001Y99901600000000000000011234"
"567806999999")
message_elements = mciutil.get_message_elements(
message_raw, CONFIG['data_elements'], 'ascii')
print(message_elements)
# print message_elements
self.assertEqual(message_elements["DE2"], b"444455*******555")
self.assertEqual(message_elements["DE3"], b"111111")
self.assertEqual(message_elements["DE4"], b"000000009999")
self.assertEqual(message_elements["PDS0001"], b"Y")
def test_get_message_elements_ascii(self):
message_raw = b(
"1144\xF0\x10\x05\x42\x84\x61\x80\x02\x02\x00\x00\x04"
"\x00\x00\x00\x00" + "1644445555444455551111110000000099992015081517151234"
"5678901233312342357995799120000001230612061234561234"
"5657994211111111145BIG BOBS\\70 FERNDALE ST\\ANNERLE"
"Y\\4103 QLDAUS0080001001Y99901600000000000000011234"
"567806999999"
)
message_elements = mciutil.get_message_elements(message_raw, CONFIG["data_elements"], "ascii")
print(message_elements)
# print message_elements
self.assertEqual(message_elements["DE2"], b"444455*******555")
self.assertEqual(message_elements["DE3"], b"111111")
self.assertEqual(message_elements["DE4"], b"000000009999")
self.assertEqual(message_elements["PDS0001"], b"Y")
def test_get_message_elements_ebcdic(self):
message_raw = (
_convert_text_asc2eb(b("1144"))
+ b("\xF0\x10\x05\x42\x84\x61\x80\x02\x02\x00\x00\x04")
+ b("\x00\x00\x00\x00")
+ _convert_text_asc2eb(
b(
"164444555544445555111111000000009999201508"
"151715123456789012333123423579957991200000"
"012306120612345612345657994211111111145BIG"
" BOBS\\70 FERNDALE ST\\ANNERLEY\\4103 QLD"
"AUS0080001001Y9990160000000000000001123456"
"7806999999"
)
)
)
message_elements = mciutil.get_message_elements(message_raw, CONFIG["data_elements"], "ebcdic")
# print message_elements
self.assertEqual(message_elements["DE2"], b"444455*******555")
self.assertEqual(message_elements["DE3"], b"111111")
self.assertEqual(message_elements["DE4"], b"000000009999")