def create_header(message, filename):
"""
Create header script.
:param message: (dict) pipeline parameters
:param filename: (str) filename with absolute path
:return:
"""
try:
header_template_path = "{0}{1}/header.txt".format(
top_level_directory, TEMPLATE_PATH["TEMPLATE"])
logger.info(
"header_template_path : '{0}'.".format(header_template_path))
header_pattern = {
"$SPARK_CODE_DEPEDENCIES_URL":
CONFIG["SPARK_CODE_DEPEDENCIES_URL"],
"$PIPELINE": str(message)
}
#print ('header_pattern : ', header_pattern)
header_str = Templater.load_template(header_template_path)
header_str = Templater.replace_word(header_str, header_pattern)
Templater.save_as(header_str, filename)
except FileExistsError as fee:
raise fee
except FileNotFoundError as fnfe:
raise fnfe
except Exception as er:
raise er
return True
def create_footer(message, filename):
"""
Create footer script.
:param message: (dict) pipeline parameters
:param filename: (str) filename with absolute path
:return:
"""
try:
footer_template_path = "{0}{1}/footer.txt".format(
top_level_directory, TEMPLATE_PATH["TEMPLATE"])
if message["last_stage"] == "fit":
if message["type"] == "GeneralPipeline":
footer_pattern = FOOTER_PATTERN["GP"]
elif message["type"] == "CrossValidator":
footer_pattern = FOOTER_PATTERN["CV"]
elif message["type"] == "TrainValidationSplit":
footer_pattern = FOOTER_PATTERN["TVS"]
else:
raise KeyError("Unknown key 'type' in message.")
elif message["last_stage"] == "transform":
footer_pattern = FOOTER_PATTERN["TRANSFORM"]
else:
raise KeyError("Unknown key 'last_stage' in message.")
footer_str = Templater.load_template(footer_template_path)
footer_str = Templater.replace_word(footer_str, footer_pattern)
Templater.save_as(footer_str, filename, "a")
except FileExistsError as fee:
raise fee
except FileNotFoundError as fnfe:
raise fnfe
except Exception as er:
raise er
return True
