"mkdir output;hostname > output/f1.txt")
# The task f
taskF = DagonTask(
TaskType.BATCH, "F",
"echo $RANDOM > f2.txt; cat workflow://wf1-transversal-demo/A/output/f1.txt >> f2.txt; cat workflow:///E/output/f1.txt >> f2.txt"
)
# The task g
taskG = DagonTask(
TaskType.BATCH, "G",
"cat workflow:///F/f2.txt >> f3.txt; cat workflow://wf1-transversal-demo/C/f2.txt >> f3.txt"
)
# add tasks to the workflow 2
workflow2.add_task(taskE)
workflow2.add_task(taskF)
workflow2.add_task(taskG)
workflow2.make_dependencies()
break
except Exception as exe:
logging.debug(exe)
time.sleep(1) #if the transversal workflow does not exist
# run the workflow
workflow2.run()
# if workflow.get_dry() is False:
# # set the result filename
# result_filename = taskD.get_scratch_dir() + "/f3.txt"
"echo Soy Berlin > f2.txt; cat workflow://Tokio/f1.txt >> f2.txt",
"ubuntu")
# The task c
taskC = DockerTask(
"Nairobi",
"echo Soy Nairobi > f2.txt; cat workflow://Tokio/f1.txt >> f2.txt",
"ubuntu")
# The task d
taskD = DockerTask(
"Mosco",
"cat workflow://Berlin/f2.txt workflow://Nairobi/f2.txt > f3.txt",
"ubuntu")
# add tasks to the workflow
workflow.add_task(taskA)
workflow.add_task(taskB)
workflow.add_task(taskC)
workflow.add_task(taskD)
workflow.make_dependencies()
jsonWorkflow = workflow.asJson()
with open('dataflow-demo-docker.json', 'w') as outfile:
stringWorkflow = json.dumps(jsonWorkflow, sort_keys=True, indent=2)
outfile.write(stringWorkflow)
# run the workflow
workflow.run()
wf_b = Workflow("TestTPS_WFb")
taskD = DagonTask(
TaskType.BATCH, "AdquisitionB", "cp -r " + command_dir +
"/launcher/lib $PWD;java -jar " + command_dir +
"launcher/launcher.jar 1 1 01-03-2019 02-03-2019 $PWD/ adq_tps_b")
taskE = DockerTask(
"InterpolationB",
"python /home/Interpolacion.py -i workflow:///AdquisitionB/metadata/FilesNc -w 1 ",
image="module_interpolation:v1")
taskF = DockerTask(
"UploaderB",
"python /home/upload.py -i workflow:///InterpolationB/output -w 1 ",
image="module_todb:v1")
wf_a.add_task(taskA)
wf_a.add_task(taskB)
wf_a.add_task(taskC)
wf_b.add_task(taskD)
wf_b.add_task(taskE)
wf_b.add_task(taskF)
meta_workflow.add_workflow(wf_a)
meta_workflow.add_workflow(wf_b)
meta_workflow.make_dependencies()
end_building = time.time() #end building
start_validation = time.time() #start validate
meta_workflow.Validate_WF()
taskD = DagonTask(TaskType.BATCH, "D", "cat workflow:///B/f2.txt >> f3.txt; cat workflow:///C/f2.txt >> f3.txt")
#second workflow
workflow2 = Workflow("DataFlow-transversal")
workflow2.set_dry(False)
# The task E
taskE = DagonTask(TaskType.BATCH, "E", "mkdir output;cat /tmp/pruebas/merra.csv > output/f1.csv")
# The task f
taskF = DagonTask(TaskType.BATCH, "F", "echo $RANDOM > f2.txt; cat workflow://DataFlow-Demo-Server/A/output/f1.csv >> f2.txt; cat workflow:///E/output/f1.csv >> f2.txt")
# The task g
taskG = DagonTask(TaskType.BATCH, "G", "cat workflow:///F/f2.txt >> f3.txt; cat workflow://DataFlow-Demo-Server/C/f2.txt >> f3.txt")
# add tasks to the workflow 1
workflow.add_task(taskA)
workflow.add_task(taskB)
workflow.add_task(taskC)
workflow.add_task(taskD)
# add tasks to the workflow 2
workflow2.add_task(taskE)
workflow2.add_task(taskF)
workflow2.add_task(taskG)
#list of the workflows
#WF =[workflow,workflow2]
metaworkflow=DAG_TPS("NewDAG")
metaworkflow.add_workflow(workflow)
metaworkflow.add_workflow(workflow2)
metaworkflow.make_dependencies()
"metgrid", command_dir_base + "/metgrid " + i_date +
" workflow:///makeWpsNameList/namelist.wps workflow:///ungrib/FILE\* workflow:///geogrid/geo_em.\*"
)
taskMakeInputNameList = batch.Batch(
"makeInputNameList", command_dir_base + "/makeInputNamelist." +
wrf_model + " " + i_date + " " + f_date + " 3 " + useRestart + " 1440")
# The real task executed using Slurm
taskReal = Slurm(
"real", command_dir_base + "/real " + i_date +
" workflow:///makeInputNameList/namelist.input workflow:///metgrid/met_em.\*",
"hicpu", 1)
# add tasks to the workflow
workflow.add_task(taskMakeWpsNameList)
workflow.add_task(taskGeogrid)
workflow.add_task(taskUngrib)
workflow.add_task(taskMetgrid)
workflow.add_task(taskMakeInputNameList)
workflow.add_task(taskReal)
days = hours / 24
for day in range(0, days):
i_date1 = start_date + datetime.timedelta(hours=24 * day)
f_date1 = start_date + datetime.timedelta(hours=24 * (day + 1))
i_date1s = i_date1.strftime("%Y%m%dZ%H")
f_date1s = f_date1.strftime("%Y%m%dZ%H")
# ACQUISITION PHASE
acq_cores = 16
acq_state = "yuc camp qroo"
acq_partitions = 10
acq_volume = "data"
acq_command = "java -jar acquisition.jar %d %d %s %s" % (acq_cores, acq_partitions, acq_state, acq_volume)
task_acquisition = DagonTask(TaskType.BATCH, "ACQ" ,acq_command, ip="148.247.201.227", ssh_username="******",
working_dir="/home/hreyes/pruebas_dante/tasks/acquisition")
# PARSING PHASE
parsing_input = "workflow:///ACQ/%s/documentos/lote" % acq_volume
parsing_command = "python /home/task/parser.py -i %s -o res"
workflow.add_task(task_acquisition)
for i in range(1, acq_partitions + 1):
input_element = parsing_input + str(i)
command = parsing_command % input_element
task_parsing = DagonTask(TaskType.DOCKER, "P%d" % i, command,
ip="ec2-34-208-132-217.us-west-2.compute.amazonaws.com",
ssh_username="******", keypath="dagon_services.pem", image="ddomizzi/parser")
workflow.add_task(task_parsing)
# TRANSFORM THE DATA
transform_command = "Rscript /home/batman/Transform.R "
for i in range(1, acq_partitions + 1):
TaskType.BATCH, "H",
"mkdir output;hostname > output/f1.txt; cat workflow://WF-2/G/f3.txt >> output/f1.txt"
)
# The task i
taskI = DagonTask(
TaskType.BATCH, "I",
"echo $RANDOM > f2.txt; cat workflow:///H/output/f1.txt >> f2.txt")
# The task j
taskJ = DagonTask(
TaskType.BATCH, "J",
"echo $RANDOM > f3.txt; cat workflow:///I/f2.txt >> f3.txt")
# add tasks to the workflow 1
workflow.add_task(taskA)
workflow.add_task(taskB)
workflow.add_task(taskC)
workflow.add_task(taskD)
# add tasks to the workflow 2
workflow2.add_task(taskE)
workflow2.add_task(taskF)
workflow2.add_task(taskG)
# add tasks to the workflow 2
workflow3.add_task(taskH)
workflow3.add_task(taskI)
workflow3.add_task(taskJ)
#list of the workflows
