import json
from dagon import Workflow
from dagon import batch
from dagon import docker_task as dt
if __name__ == '__main__':
config={
"scratch_dir_base":"/tmp/test",
"remove_dir":False
}
# Create the orchestration workflow
workflow=Workflow("Taskflow-Demo",config)
taskA=dt.Docker("Tokio","/bin/hostname >tokio.out", "ubuntu:latest")
taskB=dt.Docker("Berlin","/bin/date", "ubuntu:latest")
taskC=dt.Docker("Nairobi","/usr/bin/uptime", "ubuntu:latest")
taskD=dt.Docker("Mosco","cat workflow://Tokio/tokio.out", "ubuntu:latest")
workflow.add_task(taskA)
workflow.add_task(taskB)
workflow.add_task(taskC)
workflow.add_task(taskD)
taskB.add_dependency_to(taskA)
taskC.add_dependency_to(taskA)
taskD.add_dependency_to(taskB)
taskD.add_dependency_to(taskC)
jsonWorkflow=workflow.asJson()
with open('taskflow-demo-docker.json', 'w') as outfile:
import os
from dagon import Workflow
from dagon.task import DagonTask, TaskType
# Check if this is the main
if __name__ == '__main__':
config = {
"scratch_dir_base": "/tmp/test6",
"remove_dir": False
}
# Create the orchestration workflow
workflow = Workflow("DataFlow-Demo-Server")
# Set the dry
workflow.set_dry(False)
# The task a
taskA = DagonTask(TaskType.BATCH, "A", "mkdir output;hostname > output/f1.txt")
# The task b
taskB = DagonTask(TaskType.BATCH, "B", "echo $RANDOM > f2.txt; cat workflow:///A/output/f1.txt >> f2.txt")
# The task c
taskC = DagonTask(TaskType.BATCH, "C", "echo $RANDOM > f2.txt; cat workflow:///A/output/f1.txt >> f2.txt")
# The task d
taskD = DagonTask(TaskType.BATCH, "D", "cat workflow:///B/f2.txt >> f3.txt; cat workflow:///C/f2.txt >> f3.txt")
import logging
logging.debug('This message should go to the log file')
logging.info('So should this')
logging.warning('And this, too')
# Check if this is the main
if __name__ == '__main__':
# Create the orchestration workflow
filename = "./jsons/MW-demo.json"
with open(filename, 'r') as f:
datastore = json.load(f)
workflow = Workflow("NEW-DAGT_PS", jsonload=datastore)
# Set the dry
workflow.set_dry(False)
#load json to a workflow
workflow.make_dependencies()
jsonWorkflow = workflow.as_json()
with open('./jsons/NEW-Meta-workflow.json', 'w') as outfile:
stringWorkflow = json.dumps(jsonWorkflow, sort_keys=True, indent=2)
outfile.write(stringWorkflow)
# run the workflow
workflow.run()
# if workflow.get_dry() is False:
import logging
logging.debug('This message should go to the log file')
logging.info('So should this')
logging.warning('And this, too')
# Check if this is the main
if __name__ == '__main__':
config = {
"scratch_dir_base": "/tmp/test6",
"remove_dir": False
}
# Create the orchestration workflow
workflow = Workflow("DataFlow-Demo-Server")
# Set the dry
workflow.set_dry(False)
# The task a
taskA = DagonTask(TaskType.BATCH, "A", "mkdir output;cat /tmp/pruebas/conHeaders.csv > output/f1.csv")
# The task b
taskB = DagonTask(TaskType.BATCH, "B", "echo $RANDOM > f2.txt; cat workflow:///A/output/f1.csv >> f2.txt")
# The task c
taskC = DagonTask(TaskType.BATCH, "C", "echo $RANDOM > f2.txt; cat workflow:///A/output/f1.csv >> f2.txt")
# The task d
taskD = DagonTask(TaskType.BATCH, "D", "cat workflow:///B/f2.txt >> f3.txt; cat workflow:///C/f2.txt >> f3.txt")
if __name__ == '__main__':
command_dir = sys.argv[1]
no_TPS = int(sys.argv[2])
iterations = 1
logfile = open("../logs/" + str(no_TPS) + "_TPS_test.txt", "a+")
logfile.write("building,validation,runtime,extraction,processing,\n")
# Create the orchestration workflow
for i in range(0, iterations):
meta_workflow = DAG_TPS("DAGtp_2WF_" + str(no_TPS) + "TPS")
start_building = time.time()
wf_a = Workflow("TestTPS_WFa")
taskA = DagonTask(
TaskType.BATCH, "AdquisitionA", "cp -r " + command_dir +
"/launcher/lib $PWD;java -jar " + command_dir +
"launcher/launcher.jar 1 1 01-01-2019 02-01-2019 $PWD/ adq_tps_a")
taskB = DockerTask(
"InterpolationA",
"python /home/Interpolacion.py -i workflow:///AdquisitionA/metadata/FilesNc -w 1 ",
image="module_interpolation:v1")
taskC = DockerTask(
"UploaderA",
"python /home/upload.py -i workflow:///InterpolationA/output -w 1 ",
image="module_todb:v1")
wf_b = Workflow("TestTPS_WFb")
taskD = DagonTask(
import logging
logging.debug('This message should go to the log file')
logging.info('So should this')
logging.warning('And this, too')
# Check if this is the main
if __name__ == '__main__':
config = {"scratch_dir_base": "/tmp/test6", "remove_dir": False}
#second workflow
while (True):
try:
workflow2 = Workflow("wf2-transversal-demo")
workflow2.set_dry(False)
# The task E
taskE = DagonTask(TaskType.BATCH, "E",
"mkdir output;hostname > output/f1.txt")
# The task f
taskF = DagonTask(
TaskType.BATCH, "F",
"echo $RANDOM > f2.txt; cat workflow://wf1-transversal-demo/A/output/f1.txt >> f2.txt; cat workflow:///E/output/f1.txt >> f2.txt"
)
# The task g
taskG = DagonTask(
TaskType.BATCH, "G",
"cat workflow:///F/f2.txt >> f3.txt; cat workflow://wf1-transversal-demo/C/f2.txt >> f3.txt"
from dagon.docker_task import DockerTask
from dagon import Status
from dagon.dag_tps import DAG_TPS
import sys
# Check if this is the main
if __name__ == '__main__':
command_dir = sys.argv[1]
init_date = sys.argv[2]
end_date = sys.argv[3]
workflow_id = int(sys.argv[4])
no_workflows = sys.argv[5]
# Create the orchestration workflow
workflow = Workflow("DAGtp_wf_" + str(workflow_id))
#create list of dependencies to interpolation
adquisition_list = "'"
for i in range(1, workflow_id + 1):
workflow_name = "DAGtp_wf_" + str(i)
adquisition_list += "workflow://%s/Adquisition%s/metadata/FilesNc '" % (
workflow_name, i)
adquisition_list = adquisition_list[:-1]
adquisition_list += ","
adquisition_list = adquisition_list[:-1]
adquisition_list += "'"
start_building = time.time()
# -------------- DISTRIBUTED TRANSVERSAL WORKER 1 ------------------ #
#command_dir = "/home/robot/Escritorio/Projects/Crawlers/Merra_Master/"
command_dir = sys.argv[1]
init_date = sys.argv[2]
end_date = sys.argv[3]
iterations = str(sys.argv[4]) #also iterations
# Create the orchestration workflow
if not os.path.exists("../logs/"):
os.makedirs("../logs/")
logfile = open("../logs/TDW1-" + iterations + ".txt", "a+")
logfile.write("building,validation,runtime\n")
for i in range(1, int(iterations) + 1):
workflow = Workflow("TDW1-" + str(i))
# The tasks
start_building = time.time()
taskA = DagonTask(
TaskType.BATCH, "Adquisition",
"cp -r " + command_dir + "/launcher/lib $PWD;\
java -jar " + command_dir + "launcher/launcher.jar 1 1 " +
init_date + " " + end_date + " $PWD/ aqdw_" + str(i))
taskB = DockerTask(
"Interpolation",
"python /home/Interpolacion.py -i workflow:///Adquisition/metadata/FilesNc -w 1 ",
image="module_interpolation:v1")
taskC = DockerTask(
"Uploader",
"python /home/upload.py -i workflow:///Interpolation/output -w 1 ",
from libcloud.compute.types import Provider
from dagon import Workflow, Batch
from dagon.docker_task import DockerTask
from dagon.remote import CloudTask
from dagon.cloud import KeyOptions
from dagon.task import DagonTask, TaskType
if __name__ == '__main__':
# Create the orchestration workflow
workflow = Workflow("CONAGUASlurm5")
# ACQUISITION PHASE
acq_cores = 16
acq_state = "yuc camp qroo"
acq_partitions = 10
acq_volume = "data"
acq_command = "java -jar acquisition.jar %d %d %s %s" % (acq_cores, acq_partitions, acq_state, acq_volume)
task_acquisition = DagonTask(TaskType.BATCH, "ACQ" ,acq_command, ip="148.247.201.227", ssh_username="******",
working_dir="/home/hreyes/pruebas_dante/tasks/acquisition")
# PARSING PHASE
parsing_input = "workflow:///ACQ/%s/documentos/lote" % acq_volume
parsing_command = "python /home/task/parser.py -i %s -o res"
workflow.add_task(task_acquisition)
for i in range(1, acq_partitions + 1):
input_element = parsing_input + str(i)
import logging
logging.debug('This message should go to the log file')
logging.info('So should this')
logging.warning('And this, too')
# Check if this is the main
if __name__ == '__main__':
config = {
"scratch_dir_base": "/tmp/test6",
"remove_dir": False
}
# Create the orchestration workflow
workflow = Workflow("TPw1")
# Set the dry
workflow.set_dry(False)
# The task a
taskA = DagonTask(TaskType.BATCH, "A", "sleep 1; mkdir output;cat /tmp/pruebas/conHeaders.csv > output/f1.csv")
# The task b
taskB = DagonTask(TaskType.BATCH, "B", "sleep 1; echo $RANDOM > f2.txt; cat workflow:///A/output/f1.csv >> f2.txt")
# The task c
taskC = DagonTask(TaskType.BATCH, "C", "sleep 1; echo $RANDOM > f2.txt; cat workflow:///A/output/f1.csv >> f2.txt")
# The task d
taskD = DagonTask(TaskType.BATCH, "D", "sleep 1; cat workflow:///B/f2.txt >> f3.txt; cat workflow:///C/f2.txt >> f3.txt")
from dagon.dag_tps import DAG_TPS
from dagon.task import DagonTask, TaskType
import logging
logging.debug('This message should go to the log file')
logging.info('So should this')
logging.warning('And this, too')
# Check if this is the main
if __name__ == '__main__':
config = {"scratch_dir_base": "/tmp/test6", "remove_dir": False}
# Create the orchestration workflow
workflow = Workflow("wf1-transversal-demo")
# Set the dry
workflow.set_dry(False)
# The task a
taskA = DagonTask(TaskType.BATCH, "A",
"mkdir output;hostname > output/f1.txt")
# The task b
taskB = DagonTask(
TaskType.BATCH, "B",
"echo $RANDOM > f2.txt; cat workflow:///A/output/f1.txt >> f2.txt;sleep 10"
)
# The task c
from dagon.dag_tps import DAG_TPS
from dagon.task import DagonTask, TaskType
import logging
logging.debug('This message should go to the log file')
logging.info('So should this')
logging.warning('And this, too')
# Check if this is the main
if __name__ == '__main__':
config = {"scratch_dir_base": "/tmp/test6", "remove_dir": False}
# Create the orchestration workflow
workflow = Workflow("WF-1")
# Set the dry
workflow.set_dry(False)
# The task a
taskA = DagonTask(
TaskType.BATCH, "A",
"$SC:5 ;mkdir output;hostname > output/f1.txt; cat workflow://WF-3/H/output/f1.txt >> output/f1.txt"
)
# The task b
taskB = DagonTask(
TaskType.BATCH, "B",
"echo $RANDOM > f2.txt; cat workflow:///A/output/f1.txt >> f2.txt")
for i in range(0,iterations):
init_date = sys.argv[2]
end_date = sys.argv[3]
idate = datetime.strptime(init_date, '%d-%m-%Y')
edate = datetime.strptime(end_date, '%d-%m-%Y')
daterange = (edate-idate)/no_workflows
start_building = time.time()
meta_workflow = DAG_TPS("DAGtp_"+str(no_workflows)+"_workflows")
adquisition_list = "'"
for i in range(1,no_workflows+1):
init_date = idate.strftime('%d-%m-%Y')
end_date = (idate + daterange).strftime('%d-%m-%Y')
workflow_name = "Merra-Docker_%s" %(i)
workflow = Workflow(workflow_name)
adquisition_list += "workflow://%s/Adquisition%s/metadata/FilesNc '" %(workflow_name,i)
taskA = DagonTask(TaskType.BATCH, "Adquisition"+str(i) , "cp -r "+command_dir+"/launcher/lib $PWD;java -jar "+command_dir+"launcher/launcher.jar 1 1 "+str(init_date)+" "+str(end_date)+" $PWD/ aqui"+str(i))
# The task b
taskB = DockerTask("Interpolation"+str(i), "python /home/Interpolacion.py -i "+adquisition_list+" -w 1 ", image="module_interpolation:v1")
taskC = DockerTask("Uploader"+str(i), "python /home/upload.py -i workflow:///Interpolation"+str(i)+"/output -w 1 ", image="module_todb:v1")
# add tasks to the workflow
workflow.add_task(taskA)
workflow.add_task(taskB)
workflow.add_task(taskC)
meta_workflow.add_workflow(workflow)
adquisition_list = adquisition_list[:-1]
adquisition_list+=","
idate = datetime.strptime(end_date, '%d-%m-%Y') + timedelta(days=1) ##adding 1 day
import json
import os.path
import time
from dagon import Workflow
from dagon.docker_task import DockerTask
from dagon import Status
# Check if this is the main
if __name__ == '__main__':
# Create the orchestration workflow
workflow = Workflow("DataFlow-Demo-Docker-remote-docker-2")
# The task a
taskA = DockerTask("A",
"mkdir output;hostname > output/f1.txt",
image="ubuntu_curl",
ip="ec2-34-219-189-203.us-west-2.compute.amazonaws.com",
ssh_username="******",
keypath="dagon_services.pem")
# The task b
taskB = DockerTask(
"B",
"echo $RANDOM > f2.txt; cat workflow:///A/output/f1.txt >> f2.txt",
"ubuntu_curl",
ip="ec2-34-219-189-203.us-west-2.compute.amazonaws.com",
ssh_username="******",
keypath="dagon_services.pem")
# The task c
import json
import os.path
import time
from dagon import Workflow
from dagon.docker_task import DockerTask
from dagon import Status
# Check if this is the main
if __name__ == '__main__':
# Create the orchestration workflow
workflow = Workflow("DataFlow-Demo-Docker")
# The task a
taskA = DockerTask("A",
"mkdir output;hostname > output/f1.txt",
image="ubuntu_curl")
# The task b
taskB = DockerTask(
"B",
"echo $RANDOM > f2.txt; cat workflow:///A/output/f1.txt >> f2.txt",
"ubuntu_curl")
# The task c
taskC = DockerTask(
"C",
"echo $RANDOM > f2.txt; cat workflow:///A/output/f1.txt >> f2.txt",
"ubuntu_curl")
# The task d
end_date = start_date + time_delta
# The final date in NCEP format
f_date = end_date.strftime("%Y%m%dZ%H")
# Initialization data path
data_dir = data_dir_base + start_date.strftime(
"%Y%m%d") + "/" + start_date.strftime(
"%Y%m%d") + "Z" + start_date.strftime("%H")
# Initial and final simulation dates in WRF format
initial = start_date.strftime("%Y-%m-%d_%H:%M:%S")
final = end_date.strftime("%Y-%m-%d_%H:%M:%S")
# Create the orchestration workflow
workflow = Workflow("CCMMMA")
# Some beauty logging
workflow.logger.info("initialization date: %s", i_date)
workflow.logger.info("data dir: %s", data_dir)
workflow.logger.info("initial: %s", initial)
workflow.logger.info("final: %s", final)
# The makeWpsNamelist task executed locally
taskMakeWpsNameList = batch.Batch(
"makeWpsNameList", command_dir_base + "/makeWpsNamelist." + wrf_model +
" " + initial + " " + final)
# The geogrid task executed using Slurm
taskGeogrid = Slurm(
"geogrid", command_dir_base + "/geogrid " + i_date +
#Images configuration
defaultflavour = {"image":"ami-0bbe6b35405ecebdb","size":"t1.micro"} #EC2
digitalOceanFlavour = {"image":"38897365","size":"1gb","location":"nyc1"} #Digital ocean
googleFlavour = {"image":'debian-7',"size":"n1-standard-1","location":"us-east1-b"} #google
keysshparams = {"option":KeyOptions.GET,"keyname":"test-key","keypath":"test-key.pem","cloudargs":{"name":"test-key"}}
keysDO = KeyPair.generate_RSA()
googleKeyParams = {"keypath":"googlekey.pem","username":"******","public_key":keysDO[1],"private_key":keysDO[0]}
keysshparams3 = {"option":KeyOptions.GET,"keyname":"dagonkey","keypath":"dagonkey.pem","cloudargs":{"name":"dagonkey"}}
keysshparams2 = {"option":KeyOptions.CREATE,"keypath":"dagonDOkey.pem",
"cloudargs":{"name":"dagonDOkey","public_key":keysDO[1],"private_key":keysDO[0]}}
# Create the orchestration workflow
workflow=Workflow("DataFlow-Demo-Cloud",config)
# The task a: a EC2 Instance
taskA= CloudTask("A", "mkdir output;echo I am A > output/f1.txt", Provider.EC2,"ubuntu",create_instance=False,id="i-021d09715bc0b877c", keyparams=keysshparams3,endpoint="6c24aea0-e208-11e8-8c92-0a1d4c5c824a",flavour=defaultflavour, instance_name="dagon")
# The task B: another EC2 instance
taskB= CloudTask("B", "echo I am B > f2.txt; cat workflow://A/output/f1.txt >> f2.txt", Provider.EC2,"ubuntu",create_instance=False,id="i-03d1c4f2b326ed016", keyparams=keysshparams3,flavour=defaultflavour, instance_name="B")
# The task C: a Docker local container
taskC=DockerTask("C","mkdir output;ls > output/f4.txt;echo I am C > f3.txt; cat workflow://A/output/f1.txt >> f3.txt", image="ubuntu:18.04")
# Task D: another EC2 instance
taskD= CloudTask("D", "echo I am D > f1.txt", Provider.EC2,"ubuntu",create_instance=False,id="i-021d09715bc0b877c", keyparams=keysshparams3,endpoint="6c24aea0-e208-11e8-8c92-0a1d4c5c824a",flavour=defaultflavour, instance_name="dagon")
#Task E: Another EC2 instance
taskE= CloudTask("E", "echo I am E > f4.txt; cat workflow://D/f1.txt >> f4.txt; cat workflow://C/f3.txt >> f4.txt", Provider.EC2,"ubuntu",create_instance=False,id="i-021d09715bc0b877c", keyparams=keysshparams3,endpoint="6c24aea0-e208-11e8-8c92-0a1d4c5c824a",flavour=defaultflavour, instance_name="dagon")
if __name__ == '__main__':
#command_dir = "/home/robot/Escritorio/Projects/Crawlers/Merra_Master/"
command_dir = sys.argv[1]
init_date = sys.argv[2]
end_date = sys.argv[3]
id_test = str(sys.argv[4]) #also iterations
# Create the orchestration workflow
if not os.path.exists("../logs/"):
os.makedirs("../logs/")
logfile= open("../logs/LOG_single_test_No-<"+id_test+".txt", "a+")
logfile.write("building,validation,runtime\n")
for i in range(0,int(id_test)):
workflow = Workflow("Merra-Docker")
# The task a
start_building = time.time()
taskA = DagonTask(TaskType.BATCH, "Adquisition", "cp -r "+command_dir+"/launcher/lib $PWD;\
java -jar "+command_dir+"launcher/launcher.jar 1 1 "+init_date+" "+end_date+" $PWD/ aq_"+id_test)
# The task b
taskB = DockerTask("Interpolation", "python /home/Interpolacion.py -i workflow:///Adquisition/metadata/FilesNc -w 1 ", image="module_interpolation:v1")
taskC = DockerTask("Uploader", "python /home/upload.py -i workflow:///Interpolation/output -w 1 ", image="module_todb:v1")
# add tasks to the workflow
workflow.add_task(taskA)
workflow.add_task(taskB)
workflow.add_task(taskC)
import sys
import json
import os.path
import datetime
from dagon import batch
from dagon import Workflow
from dagon.docker_task import DockerTask
# Check if this is the main
if __name__ == '__main__':
config = {"scratch_dir_base": "/tmp/test/", "remove_dir": False}
# Create the orchestration workflow
workflow = Workflow("DataFlow-Demo-Docker", config)
# The task a
taskA = DockerTask("A", "echo I am A > f1.txt", image="ubuntu:18.04")
#taskA=DockerTask("Tokio","echo Soy Tokio > f1.txt", "ubuntu")
# The task b
taskB = DockerTask(
"Berlin",
"echo Soy Berlin > f2.txt; cat workflow://Tokio/f1.txt >> f2.txt",
"ubuntu")
# The task c
taskC = DockerTask(
"Nairobi",
"echo Soy Nairobi > f2.txt; cat workflow://Tokio/f1.txt >> f2.txt",
from dagon import Workflow
from dagon.task import TaskType, DagonTask
import json
if __name__ == '__main__':
# Create the orchestration workflow
workflow = Workflow("Taskflow-Demo")
taskA = DagonTask(TaskType.BATCH, "Tokio", "/bin/hostname >tokio.out")
taskB = DagonTask(TaskType.BATCH, "Berlin", "/bin/date")
taskC = DagonTask(TaskType.BATCH, "Nairobi", "/usr/bin/uptime")
taskD = DagonTask(TaskType.BATCH, "Mosco",
"cat workflow:///Tokio/tokio.out")
workflow.add_task(taskA)
workflow.add_task(taskB)
workflow.add_task(taskC)
workflow.add_task(taskD)
taskB.add_dependency_to(taskA)
taskC.add_dependency_to(taskA)
taskD.add_dependency_to(taskB)
taskD.add_dependency_to(taskC)
jsonWorkflow = workflow.as_json()
with open('taskflow-demo.json', 'w') as outfile:
stringWorkflow = json.dumps(jsonWorkflow, sort_keys=True, indent=2)
outfile.write(stringWorkflow)
workflow.run()
