def extract_command(args):
"""
extract command
:param args: arg object
:return:
"""
# Read input file
with open(args.input, 'rb') as infile:
input_file = infile.read()
LOGGER.info("%s bytes read from %s", len(input_file), args.input)
# Unblock input
input_file = unblock(input_file)
# get config filename
config_filename = get_config_filename("mideu.yml")
LOGGER.info("Config file: %s", config_filename)
# load the config from yaml file
with open(config_filename, 'r') as config_file:
config = yaml.load(config_file)
# parse the records
output_list = [
get_message_elements(
record,
config["bit_config"],
args.sourceformat
) for record in input_file
]
print("\nCompleted processing {0} records".format(len(input_file)))
# write to output
if args.mongo or args.mongohost or args.mongodb:
mongo_config = config["mongo_config"]
if args.mongohost:
mongo_config["host"] = args.mongohost
if args.mongodb:
mongo_config["db"] = args.mongodb
add_to_mongo(
output_list,
config['output_data_elements'],
mongo_config)
else: # default write to csv
if args.csvoutputfile:
csv_output_filename = args.csvoutputfile
else:
csv_output_filename = args.input + ".csv"
add_to_csv(
output_list,
config['output_data_elements'],
csv_output_filename
)
def test_get_config_filename(self):
"""
check that package default config exists, otherwise fail
this will show up on remote build when package is installed
rather than pointing to development environment
"""
filename = get_config_filename('mideu.yml')
self.assertTrue(os.path.exists(filename))
print("config filename={0}".format(filename))
if not os.path.isdir(".git"):
print("Checking that config from site-packages")
self.assertNotEqual(filename.find("site-packages"), -1)
def extract_command(args):
"""
extract command
:param args: arg object
:return:
"""
# Read input file
with open(args.input, 'rb') as infile:
input_file = infile.read()
LOGGER.info("%s bytes read from %s", len(input_file), args.input)
# Unblock input
input_file = unblock(input_file)
# get config filename
config_filename = get_config_filename("mideu.yml")
LOGGER.info("Config file: %s", config_filename)
# load the config from yaml file
with open(config_filename, 'r') as config_file:
config = yaml.load(config_file)
# parse the records
output_list = [
get_message_elements(record, config["bit_config"], args.sourceformat)
for record in input_file
]
print("\nCompleted processing {0} records".format(len(input_file)))
# write to output
if args.mongo or args.mongohost or args.mongodb:
mongo_config = config["mongo_config"]
if args.mongohost:
mongo_config["host"] = args.mongohost
if args.mongodb:
mongo_config["db"] = args.mongodb
add_to_mongo(output_list, config['output_data_elements'], mongo_config)
else: # default write to csv
if args.csvoutputfile:
csv_output_filename = args.csvoutputfile
else:
csv_output_filename = args.input + ".csv"
add_to_csv(output_list, config['output_data_elements'],
csv_output_filename)
def convert_command(args):
"""
convert command
:param args: arg object
:return:
"""
# Read input file
with open(args.input, 'rb') as input_file:
input_data = input_file.read()
LOGGER.info("%s bytes read from %s", len(input_data), args.input)
# Unblock input
if args.no_1014_blocking:
input_data = vbs_unpack(input_data)
else:
input_data = unblock(input_data)
# get config filename
config_filename = get_config_filename("mideu.yml")
LOGGER.info("Config file: %s", config_filename)
# load the config from yaml file
with open(config_filename, 'r') as config_file:
config = yaml.load(config_file)
output_records = [
flip_message_encoding(record, config["bit_config"], args.sourceformat)
for record in input_data
]
if args.no_1014_blocking:
output_data = vbs_pack(output_records)
else:
output_data = block(output_records)
print("\nCompleted processing {0} records".format(len(input_data)))
# save to file
output_filename = args.input + ".out"
with open(output_filename, "wb") as output_file:
output_file.write(output_data)
def convert_command(args):
"""
convert command
:param args: arg object
:return:
"""
# Read input file
with open(args.input, 'rb') as input_file:
input_data = input_file.read()
LOGGER.info("%s bytes read from %s", len(input_data), args.input)
# Unblock input
input_data = unblock(input_data)
# get config filename
config_filename = get_config_filename("mideu.yml")
LOGGER.info("Config file: %s", config_filename)
# load the config from yaml file
with open(config_filename, 'r') as config_file:
config = yaml.load(config_file)
output_records = [
flip_message_encoding(
record,
config["bit_config"],
args.sourceformat
) for record in input_data
]
output_data = block(output_records)
print("\nCompleted processing {0} records".format(len(input_data)))
# save to file
output_filename = args.input + ".out"
with open(output_filename, "wb") as output_file:
output_file.write(output_data)
def test_module_default(self):
filename = get_config_filename("test.conf")
print(filename)
def test_module_default(self):
filename = get_config_filename("test.conf")
print(filename)
