def download_file2(**kwargs):
print("download_file2()")
lc = LoaderUtils()
downloaded_file = lc.download_file(
kwargs["ti"].xcom_pull(key="file2"),
kwargs["ti"].xcom_pull(key="file_path2"))
kwargs["ti"].xcom_push(key="file2_to_load", value=downloaded_file)
def transfer_file(**kwargs):
print("transfer_file()")
file_to_upload = kwargs["ti"].xcom_pull(key="file_to_upload")
working_dir = kwargs["ti"].xcom_pull(key="working_dir")
file_name = kwargs["ti"].xcom_pull(key="file_name")
lc = LoaderUtils()
lc.upload_file_to_s3(working_dir, file_to_upload, file_name)
def download_file(**kwargs):
print("download_file()")
lc = LoaderUtils()
directory, working_dir = lc.create_working_dir()
file1 = 'file1.csv'
file_path = directory + "/" + file1
downloaded_file = lc.download_file(file1, file_path)
kwargs["ti"].xcom_push(key="file_to_load", value=downloaded_file)
kwargs["ti"].xcom_push(key="temp_directory", value=directory)
def poke(self, context):
lc = LoaderUtils()
if lc.get_stat1_status() == 'Y':
print("Wow... Sensor status is Good to go")
return True
else:
print(
"Well, Not ready to continue the tasks in the pipeline, waiting for the user to check some status"
)
return False
def build_file(**kwargs):
print("build_file()")
lc = LoaderUtils()
directory, working_dir = lc.create_working_dir()
file_name = 'file1.csv'
file_path = directory + "/" + file_name
upload_file = lc.build_file1(file_name, file_path)
kwargs["ti"].xcom_push(key="file_to_upload", value=upload_file)
kwargs["ti"].xcom_push(key="temp_directory", value=directory)
kwargs["ti"].xcom_push(key="working_dir", value=working_dir)
kwargs["ti"].xcom_push(key="file_name", value=file_name)
def prepare_files(**kwargs):
print("prepare_files()")
lc = LoaderUtils()
directory, working_dir = lc.create_working_dir()
file1 = 'file1.csv'
file_path1 = directory + "/" + file1
file2 = 'file2.csv'
file_path2 = directory + "/" + file2
kwargs["ti"].xcom_push(key="file1", value=file1)
kwargs["ti"].xcom_push(key="file_path1", value=file_path1)
kwargs["ti"].xcom_push(key="file2", value=file2)
kwargs["ti"].xcom_push(key="file_path2", value=file_path2)
kwargs["ti"].xcom_push(key="temp_directory", value=directory)
kwargs["ti"].xcom_push(key="working_dir", value=working_dir)
def cleanup(**kwargs):
print("cleanup()")
temp_directory = kwargs["ti"].xcom_pull(key="temp_directory")
lc = LoaderUtils()
lc.cleanup_directory(temp_directory)
def load_file(**kwargs):
print("load_file()")
file_to_load = kwargs["ti"].xcom_pull(key="file_to_load")
lc = LoaderUtils()
lc.load_file1(file_to_load)
def reset_stat1_status(**kwargs):
print("reset_stat1_status()")
lc = LoaderUtils()
lc.reset_stat1_status()
def update_status(**kwargs):
print("update_status()")
lc = LoaderUtils()
lc.update_status()