def transform(mapping_file, main_sheet, input_file, output_file, q):
try:
input_data = pd.read_excel(input_file, dtype=str)
parallelize_tranformation(mapping_file,
main_sheet,
input_data,
output_file,
n_cores=1)
q.put(("Transform_End",
"Transformed file generated at: \n" + output_file))
except Exception as e:
infor.logger.exception(e)
q.put(("Transform_End",
"Something went wrong! Please check the error logs!"))
def main_transformation(
mappingfile=None, mainsheet=None, stagingdata=None, outputfile=None
):
if mappingfile is None:
return "Error: Mapping file path missing"
if os.path.exists(mappingfile) == False:
return "Error: Mapping file does not exist"
if mainsheet is None:
return "Error: Main sheet name is empty"
if stagingdata.empty:
return "Error: Data frame is empty"
try:
return parallelize_tranformation(
mappingfile, mainsheet, stagingdata, outputfile
)
except Exception as ex:
logger.exception(ex)
if hasattr(ex, "message"):
return "There is an error while transforming the records. " + ex.message
else:
return "There is an unknown error while transforming the records. "
sys.exit(1)
def main_transformation_from_db(
mappingfile=None, mainsheet=None, db_config=None, table_name=None, outputfile=None
):
if mappingfile is None:
return "Error: Mapping file path missing"
if os.path.exists(mappingfile) == False:
return "Error: Mapping file does not exist"
if os.path.exists(db_config) == False:
return "Error: DB Configurations does not exist."
else:
is_valid, code, message = validate_db_credentials(db_config)
if not is_valid:
return message
if table_name is None:
return "Error: Table name is missing."
db_cred = get_db_credentials(db_config)
conn_string = (
"DRIVER={"
+ db_cred["driver"]
+ "};"
+ "SERVER={0};DATABASE={1};UID={2};PWD={3}".format(
db_cred["servername"],
db_cred["database"],
db_cred["username"],
db_cred["password"],
)
)
sql_conn = pyodbc.connect(conn_string)
query = "SELECT * FROM {0}".format(table_name)
data = pd.read_sql(query, sql_conn)
try:
return parallelize_tranformation(mappingfile, mainsheet, data, outputfile)
except Exception as ex:
logger.exception(ex)
if hasattr(ex, "message"):
return "There is an error while transforming the records. " + ex.message
else:
return "There is an unknown error while transforming the records. "
sys.exit(1)
def main_transformation(mappingfile=None,
mainsheet=None,
stagingdata=None,
outputfile=None):
if mappingfile is None:
return "Error: Mapping file path missing"
if os.path.exists(mappingfile) == False:
return "Error: Mapping file does not exist"
if mainsheet is None:
return "Error: Main sheet name is empty"
if stagingdata.empty:
return "Error: Data frame is empty"
return parallelize_tranformation(mappingfile, mainsheet, stagingdata,
outputfile)