linqdm_filter 3. Creates all tables in Fact database. linqdm_fdn 4. Creates all tables in Base database for now this database will have only DIH tables. dih """ # Set the batch id from Airflow dag run setbatch = getpythonoperator("BatchId", getBatchId) batchid = "{{ ti.xcom_pull(key='batchId', task_ids='Run_BatchId') }}" for db in database: with TaskGroup(group_id="{}_Tab".format(db)) as run_stage0: stagetaskgrp = [] with TaskGroup(group_id="{}_S2HS".format(db)) as run_stage1: for tabname in database[db]["tabname"]: taskname = "CRT_{}_{}".format(db, tabname) taskid = 'TA_' + taskname commands = "base64 -d <<< {} | kinit {}@{} && ssh -o StrictHostKeyChecking=no -o GSSAPIAuthentication=yes -oGSSAPIDelegateCredentials=yes {}@{} '{}'".format(password, kinitprincipal, kinitdomain, kinitprincipal, edgenodehost, "{} {} {} {} {} {}".format(scriptpaths["hiveload"], tabname , batchid, 'ddl', db, database[db]["type"])) ssh_create_stage = getbashoperator(taskname, False, commands) ssh_create_stage stagetaskgrp.append(run_stage1) run_stage1 group.append(run_stage0) dummyop = DummyOperator(task_id='NoOP') setbatch >> group >> dummyop
edgenodehost = kinitparms["edgenodehost"] password = Variable.get("kinit_passwd") default_args = { 'owner': dset["owner"], 'depends_on_past': dset["depends_on_past"], 'start_date': datetime.utcnow(), 'email': dset["email"], 'email_on_failure': dset["email_on_failure"], 'email_on_retry': dset["email_on_retry"], 'concurrency' : dset["concurrency"], 'retries': dset["retries"], 'retry_delay': timedelta(minutes=dset["retry_delay"]) } with DAG(dset["name"], default_args=default_args, schedule_interval=None, dagrun_timeout=timedelta(minutes=dset['dagrun_timeout'])) as dag: # Set the batch id from Airflow dag run setbatch = getpythonoperator("BatchId", getBatchId) batchid = "{{ ti.xcom_pull(key='batchId', task_ids='Run_BatchId') }}" # Running spark job #command="base64 -d <<< {} | kinit {}@{} && ssh -o StrictHostKeyChecking=no -o GSSAPIAuthentication=yes -o GSSAPIDelegateCredentials=yes {}@{} '{}'".format(password,kinitprincipal, kinitdomain, kinitprincipal, edgenodehost, "cd {} && sh {}".format(scriptpaths["cdcommand"],scriptpaths["sparkscript"])) command="base64 -d <<< {} | kinit {}@{} && ssh -o StrictHostKeyChecking=no -o GSSAPIAuthentication=yes -o GSSAPIDelegateCredentials=yes {}@{} '{}'".format(password, kinitprincipal, kinitdomain, kinitprincipal, edgenodehost, "{}".format(scriptpaths["tag"])) taskname = "spark_transmission_run" atlas_tag = getbashoperator(taskname, False, command) setbatch >> ssh_spark
password = Variable.get("kinit_passwd") default_args = { 'owner': dset["owner"], 'depends_on_past': dset["depends_on_past"], 'start_date': datetime.utcnow(), 'email': dset["email"], 'email_on_failure': dset["email_on_failure"], 'email_on_retry': dset["email_on_retry"], 'concurrency' : dset["concurrency"], 'retries': dset["retries"], 'retry_delay': timedelta(minutes=dset["retry_delay"]) } with DAG(dset["name"], default_args=default_args, schedule_interval=None, dagrun_timeout=timedelta(minutes=dset['dagrun_timeout'])) as dag: # Set the batch id from Airflow dag run setbatch = getpythonoperator("BatchId", getBatchId) batchid = "{{ ti.xcom_pull(key='batchId', task_ids='Run_BatchId') }}" # Running spark job #command="base64 -d <<< {} | kinit {}@{} && ssh -o StrictHostKeyChecking=no -o GSSAPIAuthentication=yes -o GSSAPIDelegateCredentials=yes {}@{} '{}'".format(password,kinitprincipal, kinitdomain, kinitprincipal, edgenodehost, "cd {} && sh {}".format(scriptpaths["cdcommand"],scriptpaths["sparkscript"])) for tabconf in confs: command="base64 -d <<< {} | kinit {}@{} && ssh -o StrictHostKeyChecking=no -o GSSAPIAuthentication=yes -o GSSAPIDelegateCredentials=yes {}@{} '{}'".format(password, kinitprincipal, kinitdomain, kinitprincipal, edgenodehost, "cd {} && sh {} -c {}".format(scriptpaths["cdcommand"],scriptpaths["validationscript"],tabconf)) taskname = "validation_hbase_hdfs" validation = getbashoperator(taskname, False, command) dummyop1 = DummyOperator(task_id='NoOP') setbatch >> ssh_spark >> dummyop1
'retry_delay': timedelta(minutes=dset["retry_delay"]) } with DAG(dset["name"], default_args=default_args, schedule_interval=None, dagrun_timeout=timedelta(minutes=dset['dagrun_timeout'])) as dag: # Set the batch id from Airflow dag run setbatch = getpythonoperator("BatchId", getBatchId) batchid = "{{ ti.xcom_pull(key='batchId', task_ids='Run_BatchId') }}" # Running spark job command="base64 -d <<< {} | kinit {}@{} && ssh -o StrictHostKeyChecking=no -o GSSAPIAuthentication=yes -o GSSAPIDelegateCredentials=yes {}@{} '{}'".format(password,kinitprincipal, kinitdomain, kinitprincipal, edgenodehost, "cd {} && sh {}".format(scriptpaths["cdcommand"],scriptpaths["sparkscript"])) taskname = "spark_transmission_run" ssh_spark = getbashoperator(taskname, False, command) # Task Group for Sqoop, validation and stage table load group =[] for stagegrp in sparkjobs: with TaskGroup(group_id="{}_SparktoFact".format(stagegrp)) as run_stage0: stagetaskgrp = [] with TaskGroup(group_id="{}_SparktoStage".format(stagegrp)) as run_stage1: for landtab in sparkjobs[stagegrp]["landtab"]: dbname, tabname = landtab.split('.')
# for stagedeptab in snowsqljobs[stagegrp]["depstage"]: # dbname, tabname = stagedeptab.split('.') # taskname = "DEPSTG_{}_{}".format(dbname, tabname) # taskid = 'TA_' + taskname # commands = "base64 -d <<< {} | kinit {}@{} && ssh -o StrictHostKeyChecking=no -o GSSAPIAuthentication=yes -o GSSAPIDelegateCredentials=yes {}@{} '{}'".format(password, kinitprincipal, kinitdomain, kinitprincipal, edgenodehost, "{} {} {} {} {} {}".format(scriptpaths["hiveload"], tabname , batchid, 'dml', dbname, 'stage')) # ssh_stage = getbashoperator(taskname, False, commands) # depstagetaskgrp.append(run_depstage) facttaskgrp = [] with TaskGroup( group_id="{}_FactLoad".format(stagegrp)) as run_fact: for table in snowsqljobs[stagegrp]["facttabs"]: dbname, tabname = table.split('.') taskname = "FCT_{}".format(table) taskid = 'TA_' + taskname commands = "base64 -d <<< {} | kinit {}@{} && ssh -o StrictHostKeyChecking=no -o GSSAPIAuthentication=yes -o GSSAPIDelegateCredentials=yes {}@{} '{}'".format( password, kinitprincipal, kinitdomain, kinitprincipal, edgenodehost, "{} {} {} {} {} {}".format(scriptpaths["hiveload"], tabname, batchid, 'dml', dbname, 'fact')) ssh_fact = getbashoperator(taskname, False, commands) facttaskgrp.append(run_fact) run_fact #run_stage1 >> run_depstage >> run_fact group.append(run_stage0) dummyop1 = DummyOperator(task_id='DIHLODCMP') setbatch >> group >> dummyop1
'retries': dset['retries'] } with DAG(dset["name"], default_args=default_args, schedule_interval=None, dagrun_timeout=timedelta(minutes=dset['dagrun_timeout'])) as dag: # Set the batch id from Airflow dag run setbatch = getpythonoperator("BatchId", getBatchId) batchid = "{{ ti.xcom_pull(key='batchId', task_ids='Run_BatchId') }}" # get the Kinit task command = 'echo {} | kinit {}@{}'.format(password, kinitprincipal, kinitdomain) bash_kinit = getbashoperator("KinitEdge", False, command) # Task Group for Sqoop, validation and stage table load group = [] for stagegrp in sqoopjobs: with TaskGroup( group_id="{}_SqooptoFact".format(stagegrp)) as run_stage0: stagetaskgrp = [] with TaskGroup( group_id="{}_SqooptoStage".format(stagegrp)) as run_stage1: for landtab in sqoopjobs[stagegrp]["landtab"]:
dag.doc_md = __doc__ # providing that you have a docstring at the beggining of the DAG dag.doc_md = """ This Dag runs the method validation for all the SQQOP jobs. This runs after all the fact tables are loaded. This job performs rowcount match and data mismatch checks. The results are stored in a hive table for further analysis. """ # Set the batch id from Airflow dag run setbatch = getpythonoperator("BatchId", getBatchId) batchid = "{{ ti.xcom_pull(key='batchId', task_ids='Run_BatchId') }}" for db in database: with TaskGroup(group_id="{}_Db".format(db)) as run_stage0: stagetaskgrp = [] with TaskGroup(group_id="{}_MVAL".format(db)) as run_stage1: for tabname in database[db]["tables"]: taskname = "MVAL_{}_{}".format(db, tabname) taskid = 'TA_' + taskname command="base64 -d <<< {} | kinit {}@{} && ssh -o StrictHostKeyChecking=no -o GSSAPIAuthentication=yes -o GSSAPIDelegateCredentials=yes {}@{} '{}'".format(password, kinitprincipal, kinitdomain, kinitprincipal, edgenodehost, "{} {} {} {} {}".format(scriptpaths["baseval"], db, tabname, dset["src2land"][db], batchid)) ssh_valid = getbashoperator(taskname, False, command) ssh_valid stagetaskgrp.append(run_stage1) run_stage1 group.append(run_stage0) dummyop = DummyOperator(task_id='NoOP') setbatch >> group >> dummyop
'concurrency': dset['concurrency'], 'retries': dset['retries'] } with DAG(dset["name"], default_args=default_args, schedule_interval=None, dagrun_timeout=timedelta(minutes=dset['dagrun_timeout'])) as dag: # Set the batch id from Airflow dag run setbatch = getpythonoperator("BatchId", getBatchId) batchid = "{{ ti.xcom_pull(key='batchId', task_ids='Run_BatchId') }}" # get the Kinit task command = 'echo {} | kinit {}'.format(password, kinitprincipal) bash_kinit = getbashoperator("KinitEdge", False, command) # Task Group for Sqoop, validation and stage table load group = [] for stagegrp in sqoopjobs: with TaskGroup( group_id="{}_SqooptoFact".format(stagegrp)) as run_stage0: stagetaskgrp = [] with TaskGroup( group_id="{}_SqooptoStage".format(stagegrp)) as run_stage1: for landtab in sqoopjobs[stagegrp]["landtab"]: