def validate_log_file(**kwargs): print("validate_log_file:") dp = DagPebbles() if dp.validate_pipeline_log(kwargs["ti"].xcom_pull(key='download_key')): return "pipeline_log_validation_passed" else: return "pipeline_log_validation_failed"
def begin_pipeline(**kwargs): print("begin_pipeline:") s3_bucket = os.environ.get("S3_BUCKET", "") folder_path = kwargs['dag_run'].conf.get('folder_path') s3_file = kwargs['dag_run'].conf.get('s3_file') uuid_mapping_file = kwargs['dag_run'].conf.get('uuid_mapping_file') biobank_file = kwargs['dag_run'].conf.get('biobank_file') mapping_file = kwargs['dag_run'].conf.get('mapping_file') dp = DagPebbles() download_key = dp.get_download_key(s3_bucket, folder_path, s3_file) pipeline_state_args = { "s3_bucket": s3_bucket, "folder_path": folder_path, "s3_file": s3_file, "uuid_mapping_file": uuid_mapping_file, "biobank_file": biobank_file, "mapping_file": mapping_file, "download_key": download_key } dp.save_pipeline_state(**pipeline_state_args) kwargs["ti"].xcom_push(key="folder_path", value=folder_path) kwargs["ti"].xcom_push(key="s3_file", value=s3_file) kwargs["ti"].xcom_push(key="download_key", value=download_key) kwargs["ti"].xcom_push(key="uuid_mapping_file", value=uuid_mapping_file) kwargs["ti"].xcom_push(key="biobank_file", value=biobank_file) kwargs["ti"].xcom_push(key="mapping_file", value=mapping_file)
def pipeline_enable_check(**kwargs): dp = DagPebbles() if dp.pipeline_enable_check('DECRYPT_FILES'): kwargs["ti"].xcom_push(key="S3_BUCKET", value=os.environ.get("S3_BUCKET","")) kwargs["ti"].xcom_push(key="SKIP_DECRYPT_FILES", value="N") return "pipeline_check_passed" else: return "pipeline_check_skipped"
def pipeline_enable_check(**kwargs): dp = DagPebbles() if dp.pipeline_enable_check('DATA_PIPELINE_INITIATOR'): #TODO:: get it from db kwargs["ti"].xcom_push(key="SKIP_DOWNLOAD_LOG_FILE", value='N') kwargs["ti"].xcom_push(key="SKIP_DECRYPT_LOG_FILE", value='N') return "pipeline_check_passed" else: return "pipeline_check_skipped"
def begin_pipeline(**kwargs): dp = DagPebbles() pipeline = dp.get_current_pipeline() print(pipeline) packed_dir=os.environ.get("BCH_HPDS_INTERNAL") kwargs["ti"].xcom_push(key='packed_dir', value=packed_dir) hpds_encrypted_file_name = dp.get_hpds_packed_file_name()+".encrypted" kwargs["ti"].xcom_push(key='hpds_encrypted_file_name', value=hpds_encrypted_file_name) hpds_encrypted_file = packed_dir + '/' + hpds_encrypted_file_name kwargs["ti"].xcom_push(key='hpds_encrypted_file', value=hpds_encrypted_file)
def begin_pipeline(**kwargs): print("begin_pipeline:") dp = DagPebbles() pipeline = dp.get_current_pipeline() s3_bucket = os.environ.get("S3_BUCKET","") folder_path = pipeline['log_file_path'] s3_file = pipeline['log_file_name'] download_key = dp.get_download_key(s3_bucket, folder_path, s3_file) kwargs["ti"].xcom_push(key="folder_path", value=folder_path) kwargs["ti"].xcom_push(key="s3_file", value=s3_file) kwargs["ti"].xcom_push(key="download_key", value=download_key)
def stage_custom_dmp_files(**kwargs): print("stage_custom_dmp_files:") dp = DagPebbles() custom_log_file = kwargs['dag_run'].conf.get('custom_log_file') custom_dmp_file = kwargs['dag_run'].conf.get('custom_dmp_file') pipeline_args = { "custom_log_file": custom_log_file, "custom_dmp_file": custom_dmp_file } dp.save_pipeline_state(**pipeline_args) dp.stage_custom_dmp_files(**pipeline_args)
def begin_pipeline(**kwargs): dp = DagPebbles() pipeline = dp.get_current_pipeline() print(pipeline) current_time = datetime.now() packed_file_name = "hpds_phenotype_" + os.environ.get( "BCH_PIC_SURE_HPDS_ALS_TAG") + "_" + current_time.strftime( '%m_%d_%Y_%H_%M_%S') + "_" + os.environ.get( "BCH_PIC_SURE_HPDS_ETL_TAG") + ".tar.gz" packed_dir = os.environ.get("BCH_HPDS_INTERNAL") kwargs["ti"].xcom_push(key='packed_file_name', value=packed_file_name) kwargs["ti"].xcom_push(key='packed_dir', value=packed_dir) dp.save_hpds_package_file_name(packed_file_name)
t_end = PythonOperator( task_id="end", python_callable=end, provide_context=True, trigger_rule="none_failed", dag=dag, ) t_pipeline_begin >> t_check_pipeline t_check_pipeline >> t_pipeline_check_skipped >> t_end_pipeline t_check_pipeline >> t_pipeline_check_passed try: dp = DagPebbles() pipeline = dp.get_current_pipeline() s3_file = pipeline['log_file_name'] target_log_file = pipeline['log_file_name'].replace(".encrypted", "") decrypt_log_file_cmd = "/opt/bitnami/airflow/airflow-data/scripts/decrypt_s3_file.sh " + s3_file + " {{ ti.xcom_pull(key='SKIP_DECRYPT_FILES')}} " t_decrypt_log_file = BashOperator( task_id='decrypt_log_file', bash_command=decrypt_log_file_cmd, dag=dag) t_pipeline_check_passed >> t_decrypt_log_file files = dp.get_files(log_file_id = None, type = 'decrypt') if files == None or len(files) == 0: t_decrypt_log_file >> t_end_pipeline else:
def notify(**kwargs): dp = DagPebbles() print("notify")
def stage_dmp_files1(**kwargs): print("stage_dmp_files1:") dp = DagPebbles() dp.stage_dmp_files1(log_file_id=None)
def clean_hpds_source_data(**kwargs): dp = DagPebbles() dp.clean_hpds_source_data()
def recreate_hpds_source_data(**kwargs): dp = DagPebbles() dp.recreate_bch_hpds_data()
def begin_pipeline(**kwargs): dp = DagPebbles() pipeline = dp.get_current_pipeline() print(pipeline)
def stage_biobank_file(**kwargs): print("stage_biobank_file:") dp = DagPebbles() dp.stage_biobank_file(log_file_id=None)
def stage_uuid_mapping_file(**kwargs): print("stage_uuid_mapping_file:") dp = DagPebbles() dp.stage_uuid_mapping_file(log_file_id=None)
def save_pipeline_log(**kwargs): print("save_pipeline_log:") dp = DagPebbles() dp.save_pipeline_log(kwargs["ti"].xcom_pull(key='folder_path'), kwargs["ti"].xcom_pull(key='s3_file'))
def pipeline_enable_check(**kwargs): dp = DagPebbles() if dp.pipeline_enable_check('STAGE_CUSTOM_DMP_FILES'): return "pipeline_check_passed" else: return "pipeline_check_skipped"
def pipeline_enable_check(**kwargs): dp = DagPebbles() if dp.pipeline_enable_check('DATA_LOAD'): return "pipeline_check_passed" else: return "pipeline_check_skipped"
def cleanup(**kwargs): dp = DagPebbles() print("cleanup")
def load_data(**kwargs): print("load_data:") dp = DagPebbles() dp.load_data(log_file_id=None)
def end(**kwargs): dp = DagPebbles() print("end")
def pipeline_enable_check(**kwargs): dp = DagPebbles() if dp.pipeline_enable_check('CONCEPT_DIM_MAPPING'): return "pipeline_check_passed" else: return "pipeline_check_skipped"