def make_dag(file): with open(file, 'r') as f: job = json.load(f) if "uid" not in job: raise Exception("UID must be part of the job") uid = job["uid"] if "start_day" in job: start_day = job["start_day"] else: start_day = datetime.combine( datetime.utcfromtimestamp(os.path.getctime(file)) - timedelta(_delta + 1), datetime.min.time()) end_day = datetime.combine(start_day + timedelta(_delta - 1), datetime.min.time()) owner = 'SciDAP' if "author" in job: owner = job['author'] email = '*****@*****.**' if "email" in job: email = job['email'] dag_id = uid + "_bam2bigwig" default_args = { 'owner': owner, 'start_date': start_day, 'email': [email], 'email_on_failure': False, 'email_on_retry': False, 'end_date': end_day, 'on_failure_callback': fail_callback } dag = CWLDAG(dag_id=dag_id, cwl_workflow="workflows/scidap/bam-genomecov-bigwig.cwl", schedule_interval=timedelta(days=_delta), default_args=default_args) dag.create() dag.assign_job_dispatcher( JobDispatcher(task_id="read", read_file=file, dag=dag)) dag.assign_job_cleanup( JobCleanup(task_id="cleanup", outputs=dag.get_output_list(), rm_files=[file], dag=dag)) globals()[dag_id] = dag return dag
def make_dag(file): with open(file, 'r') as f: job = json.load(f) if "uid" not in job: raise Exception("UID must be part of the job") uid = job["uid"] if "start_day" in job: start_day = job["start_day"] else: start_day = datetime.combine(datetime.utcfromtimestamp(os.path.getctime(file)) - timedelta(_delta+1), datetime.min.time()) end_day = datetime.combine(start_day + timedelta(_delta-1), datetime.min.time()) owner = 'SciDAP' if "author" in job: owner = job['author'] email = '*****@*****.**' if "email" in job: email = job['email'] dag_id = uid+"_bam2bigwig" default_args = { 'owner': owner, 'start_date': start_day, 'email': [email], 'email_on_failure': False, 'email_on_retry': False, 'end_date': end_day, 'on_failure_callback': fail_callback } dag = CWLDAG( dag_id=dag_id, cwl_workflow="workflows/scidap/bam-genomecov-bigwig.cwl", schedule_interval=timedelta(days=_delta), default_args=default_args) dag.create() dag.assign_job_dispatcher(JobDispatcher(task_id="read", read_file=file, dag=dag)) dag.assign_job_cleanup(JobCleanup(task_id="cleanup", outputs=dag.get_output_list(), rm_files=[file], dag=dag)) globals()[dag_id] = dag return dag
def make_dag(file): with open(file, 'r') as f: job = json.load(f) if "uid" not in job: raise Exception("UID must be part of the job") uid = job["uid"] owner = 'SciDAP' if "author" in job: owner = job['author'] email = '*****@*****.**' if "email" in job: email = job['email'] dag_id = uid+"_bam2bigwig" default_args = { 'owner': owner, 'start_date': start_day, 'email': [email], 'email_on_failure': False, 'email_on_retry': False, 'end_date': end_day, 'on_failure_callback': fail_callback } dag = CWLDAG( dag_id=dag_id, cwl_workflow="workflows/scidap/bam-genomecov-bigwig.cwl", default_args=default_args) dag.create() dag.assign_job_dispatcher(JobDispatcher(task_id="read", read_file=file, dag=dag)) dag.assign_job_cleanup(JobCleanup(task_id="cleanup", outputs=dag.get_output_list(), rm_files=[file], dag=dag)) globals()[dag_id] = dag return dag
from airflow.models import DAG from airflow.utils import (apply_defaults) from datetime import datetime, timedelta from scidap.cwldag import CWLDAG from scidap.jobdispatcher import JobDispatcher # from scidap.cwlutils import shortname start_day = datetime.combine(datetime.today() - timedelta(1), datetime.min.time()) default_args = { 'owner': 'Andrey, Kartashov', 'start_date': start_day, 'email': ['*****@*****.**'], 'email_on_failure': False, 'email_on_retry': False, # 'schedule_interval': timedelta(minutes = 1), # 'end_date': datetime(2016, 1, 1), } JOB_FOLDER = "/absolute/path/to/the/directory/with/cwl/job/json/files/*" dag = CWLDAG(cwl_workflow="workflows/scidap/bam-genomecov-bigwig.cwl", default_args=default_args) dag.assign_job_dispatcher( JobDispatcher(task_id=dag.dag_id + "_folder", monitor_folder=JOB_FOLDER, dag=dag))
#!/usr/bin/env python from airflow.models import DAG from airflow.utils import (apply_defaults) from datetime import datetime, timedelta from scidap.cwldag import CWLDAG from scidap.jobdispatcher import JobDispatcher # from scidap.cwlutils import shortname start_day = datetime.combine(datetime.today() - timedelta(1), datetime.min.time()) default_args = { 'owner': 'Andrey, Kartashov', 'start_date': start_day, 'email': ['*****@*****.**'], 'email_on_failure': False, 'email_on_retry': False, # 'schedule_interval': timedelta(minutes = 1), # 'end_date': datetime(2016, 1, 1), } JOB_FOLDER = "/absolute/path/to/the/directory/with/cwl/job/json/files/*" dag = CWLDAG( cwl_workflow="workflows/scidap/bam-genomecov-bigwig.cwl", default_args=default_args) dag.assign_job_dispatcher(JobDispatcher(task_id=dag.dag_id + "_folder", monitor_folder=JOB_FOLDER, dag=dag))