import utils
#load config data from yaml file
with open('configurations/clean_file_config.yaml') as f:
# use safe_load instead load
data_map = yaml.safe_load(f)
with open('configurations/test_filepaths.yaml') as f:
filepaths = yaml.safe_load(f)
ts_schema = pd.DataFrame(json.load(open(
filepaths['ts_schema_json']))) #load TS schema
lar_schema = pd.DataFrame(json.load(open(
filepaths['lar_schema_json']))) #load LAR schema
#instantiate test_file_generator.py to modify clean data so that the resulting files fail specific edits
file_maker = test_data(ts_schema=ts_schema,
lar_schema=lar_schema) #instantiate edit file maker
ts_data, lar_data = utils.read_data_file(
path=filepaths['clean_filepath'].format(
bank_name=data_map["name"]["value"]),
data_file=data_map["clean_file"]["value"]) #load clean data file
file_maker.load_data_frames(
ts_data, lar_data) #pass clean file data to file maker object
#generate a file for each edit function in file maker
edits = []
for func in dir(file_maker): #loop over all data modification functions
if func[:1] in ("s", "v", "q") and func[1:4].isdigit(
) == True: #check if function is a numbered syntax or validity edit
print("applying:", func)
getattr(file_maker,
func)() #apply data modification functions and produce files
def create_files(self, kind):
"""Creates a clean file or a set of edit files specified by the
function call"""
#Creates TS row data.
self.ts_row = self.lar_gen.make_ts_row()
#Creates a TS row dataframe.
ts_df = pd.DataFrame(self.ts_row, index=[1])
#The following produces a clean file.
if kind == 'clean_file':
#Creates a first row of LAR to begin the dataframe.
#All other rows are concatenated to the dataframe until
#the length of the dataframe reaches the file length specified in the
#test filepaths yaml file.
for i in range(0, self.clean_config["file_length"]["value"]):
print('Creating row {i}'.format(i=i))
if i == 0:
first_row = self.make_clean_lar_row(ts_row=self.ts_row)
lar_frame = pd.DataFrame(first_row, index=[1])
else:
new_row = self.make_clean_lar_row(ts_row=self.ts_row)
new_row = pd.DataFrame(new_row, index=[1])
lar_frame = pd.concat([lar_frame, new_row], axis=0)
#Writes the file to a clean filepath specified in the test_filepaths
#configuration.
out_file_path = self.filepaths['clean_filepath'].format(
bank_name=self.clean_config["name"]["value"])
out_file_bank_name = self.filepaths['clean_filename'].format(
row_count=self.clean_config["file_length"]["value"],
bank_name=self.clean_config["name"]["value"])
utils.write_file(ts_input=ts_df,
lar_input=lar_frame,
path=out_file_path,
name=out_file_bank_name)
#For error files.
if kind == 'error_files':
#Modifies clean data and outputs
#resulting files that fail specific edits.
#Instantiates the edit file maker.
file_maker = test_data(ts_schema=self.ts_schema_df,
lar_schema=self.lar_schema_df,
geographic_data=self.geographic_data)
#Pulls in the clean data filepath and name from the
#test filepaths yaml file.
ts_data, lar_data = utils.read_data_file(
path=self.filepaths['clean_filepath'].format(
bank_name=self.clean_config["name"]["value"]),
data_file=self.filepaths["clean_filename"].format(
bank_name=self.clean_config["name"]["value"],
row_count=self.clean_config["file_length"]["value"]))
#Passes clean file data to the file maker object.
file_maker.load_data_frames(ts_data, lar_data)
#Generates a file for each edit function in file maker.
edits = []
#Loops over all data modification functions.
for func in dir(file_maker):
#Checks if function is a numbered syntax or validity edit.
if func[:1] in ("s", "v", "q") and func[1:4].isdigit() == True:
print("applying:", func)
#Applies data modification functions and produces files.
getattr(file_maker, func)()