def test_unblock(self):
umodedata = ((b("\x00\x00\x00\x0A1234567890") * 72) +
b("\x00\x00\x00\x0A\x40\x401234567890"))
records = mciutil.unblock(umodedata)
print(len(records))
print(records[0])
print(records[72])
def _main(args):
"""
paramconv main processing
:param args: argparse arguments
:return: exit code
"""
logging.basicConfig(
level=args.loglevel,
format='%(asctime)s:%(name)s:%(lineno)s:%(levelname)s:%(message)s'
)
# exit if input file does not exist
if not os.path.isfile(args.input):
print("Input file not found - {0}".format(args.input))
exit(8)
input_filename = args.input
output_filename = args.output
if not args.output:
output_filename = input_filename + ".out"
# read file to string
with open(input_filename, 'rb') as infile:
input_file = infile.read()
print("{0} bytes read from {1}".format(len(input_file), input_filename))
# deblock the file
input_file = unblock(input_file)
# convert the file
output_list = [
_convert(record, args.sourceformat) for record in input_file
]
# re-block the data
output_data = block(output_list)
# save to file
with open(output_filename, "wb") as output_file:
output_file.write(output_data)
print("{0} bytes written to {1}".format(len(output_data), output_filename))
print("{0} records".format(len(output_list)))
if args.loglevel == logging.DEBUG:
print("DEBUG:Input first 5000 bytes")
hexdump(input_file[:5000])
print("DEBUG:Output first 5000 bytes")
hexdump(output_data[:5000])
print("DEBUG:Input last 5000 bytes")
hexdump(input_file[len(output_data)-5000:len(input_file)])
print("DEBUG:Output last 5000 bytes")
hexdump(output_data[len(output_data)-5000:len(output_data)])
print("Done!")
def _main(args):
"""
paramconv main processing
:param args: argparse arguments
:return: exit code
"""
logging.basicConfig(
level=args.loglevel,
format='%(asctime)s:%(name)s:%(lineno)s:%(levelname)s:%(message)s')
# exit if input file does not exist
if not os.path.isfile(args.input):
print("Input file not found - {0}".format(args.input))
exit(8)
input_filename = args.input
output_filename = args.output
if not args.output:
output_filename = input_filename + ".out"
# read file to string
with open(input_filename, 'rb') as infile:
input_file = infile.read()
print("{0} bytes read from {1}".format(len(input_file), input_filename))
# deblock the file
input_file = unblock(input_file)
# convert the file
output_list = [
_convert(record, args.sourceformat) for record in input_file
]
# re-block the data
output_data = block(output_list)
# save to file
with open(output_filename, "wb") as output_file:
output_file.write(output_data)
print("{0} bytes written to {1}".format(len(output_data), output_filename))
print("{0} records".format(len(output_list)))
if args.loglevel == logging.DEBUG:
print("DEBUG:Input first 5000 bytes")
hexdump(input_file[:5000])
print("DEBUG:Output first 5000 bytes")
hexdump(output_data[:5000])
print("DEBUG:Input last 5000 bytes")
hexdump(input_file[len(output_data) - 5000:len(input_file)])
print("DEBUG:Output last 5000 bytes")
hexdump(output_data[len(output_data) - 5000:len(output_data)])
print("Done!")
def test_block(self):
linebreakdata = []
for x in range(0, 73):
linebreakdata.append(b("1234567890"))
output = mciutil.block(linebreakdata)
hexdump.hexdump(output)
input = mciutil.unblock(output)
print(len(input))
print(input)
def extract_command(args):
"""
extract command
:param args: arg object
:return:
"""
# Read input file
with open(args.input, 'rb') as infile:
input_file = infile.read()
LOGGER.info("%s bytes read from %s", len(input_file), args.input)
# Unblock input
input_file = unblock(input_file)
# get config filename
config_filename = get_config_filename("mideu.yml")
LOGGER.info("Config file: %s", config_filename)
# load the config from yaml file
with open(config_filename, 'r') as config_file:
config = yaml.load(config_file)
# parse the records
output_list = [
get_message_elements(
record,
config["bit_config"],
args.sourceformat
) for record in input_file
]
print("\nCompleted processing {0} records".format(len(input_file)))
# write to output
if args.mongo or args.mongohost or args.mongodb:
mongo_config = config["mongo_config"]
if args.mongohost:
mongo_config["host"] = args.mongohost
if args.mongodb:
mongo_config["db"] = args.mongodb
add_to_mongo(
output_list,
config['output_data_elements'],
mongo_config)
else: # default write to csv
if args.csvoutputfile:
csv_output_filename = args.csvoutputfile
else:
csv_output_filename = args.input + ".csv"
add_to_csv(
output_list,
config['output_data_elements'],
csv_output_filename
)
def test_block(self):
linebreakdata = []
for x in range(0, 73):
linebreakdata.append(b("1234567890"))
output = mciutil.block(linebreakdata)
hexdump.hexdump(output)
input = mciutil.unblock(output)
print(len(input))
print(input)
def extract_command(args):
"""
extract command
:param args: arg object
:return:
"""
# Read input file
with open(args.input, 'rb') as infile:
input_file = infile.read()
LOGGER.info("%s bytes read from %s", len(input_file), args.input)
# Unblock input
input_file = unblock(input_file)
# get config filename
config_filename = get_config_filename("mideu.yml")
LOGGER.info("Config file: %s", config_filename)
# load the config from yaml file
with open(config_filename, 'r') as config_file:
config = yaml.load(config_file)
# parse the records
output_list = [
get_message_elements(record, config["bit_config"], args.sourceformat)
for record in input_file
]
print("\nCompleted processing {0} records".format(len(input_file)))
# write to output
if args.mongo or args.mongohost or args.mongodb:
mongo_config = config["mongo_config"]
if args.mongohost:
mongo_config["host"] = args.mongohost
if args.mongodb:
mongo_config["db"] = args.mongodb
add_to_mongo(output_list, config['output_data_elements'], mongo_config)
else: # default write to csv
if args.csvoutputfile:
csv_output_filename = args.csvoutputfile
else:
csv_output_filename = args.input + ".csv"
add_to_csv(output_list, config['output_data_elements'],
csv_output_filename)
def convert_command(args):
"""
convert command
:param args: arg object
:return:
"""
# Read input file
with open(args.input, 'rb') as input_file:
input_data = input_file.read()
LOGGER.info("%s bytes read from %s", len(input_data), args.input)
# Unblock input
if args.no_1014_blocking:
input_data = vbs_unpack(input_data)
else:
input_data = unblock(input_data)
# get config filename
config_filename = get_config_filename("mideu.yml")
LOGGER.info("Config file: %s", config_filename)
# load the config from yaml file
with open(config_filename, 'r') as config_file:
config = yaml.load(config_file)
output_records = [
flip_message_encoding(record, config["bit_config"], args.sourceformat)
for record in input_data
]
if args.no_1014_blocking:
output_data = vbs_pack(output_records)
else:
output_data = block(output_records)
print("\nCompleted processing {0} records".format(len(input_data)))
# save to file
output_filename = args.input + ".out"
with open(output_filename, "wb") as output_file:
output_file.write(output_data)
def convert_command(args):
"""
convert command
:param args: arg object
:return:
"""
# Read input file
with open(args.input, 'rb') as input_file:
input_data = input_file.read()
LOGGER.info("%s bytes read from %s", len(input_data), args.input)
# Unblock input
input_data = unblock(input_data)
# get config filename
config_filename = get_config_filename("mideu.yml")
LOGGER.info("Config file: %s", config_filename)
# load the config from yaml file
with open(config_filename, 'r') as config_file:
config = yaml.load(config_file)
output_records = [
flip_message_encoding(
record,
config["bit_config"],
args.sourceformat
) for record in input_data
]
output_data = block(output_records)
print("\nCompleted processing {0} records".format(len(input_data)))
# save to file
output_filename = args.input + ".out"
with open(output_filename, "wb") as output_file:
output_file.write(output_data)
def test_unblock(self):
umodedata = (b("\x00\x00\x00\x0A1234567890") * 72) + b("\x00\x00\x00\x0A\x40\x401234567890")
records = mciutil.unblock(umodedata)
print(len(records))
print(records[0])
print(records[72])