def runRscript_file(args):
"""
Generic task to batch submit to R
args: run parameters saved as a text file in json format
The run parameters will be read into R as part of the R script
Users will need to know structure of the parms file
kwargs: keyword arg is the R script filename
"""
task_id = str(runRscript_file.request.id)
resultDir = setup_result_directory(task_id)
with open(resultDir + '/input/args.json', "w") as f:
jsonx.dump(args, f)
#Run R Script
docker_opts = " -v /opt/osucybercom/data/static/campgruber:/script:z -w /script "
docker_cmd = " Rscript /script/simple.R "
result = docker_task(docker_name="cybercom_r",
docker_opts=docker_opts,
docker_command=docker_cmd,
id=task_id)
reportDir = os.path.join('/opt/osucybercom/data/static/campgruber/tasks/',
task_id, 'report')
move("/data/static/campgruber/testing_R.txt",
"{0}/report/testing_R.txt".format(resultDir))
result_url = "http://{0}/campgruber/tasks/{1}".format(
"cybercom-app.hpc.okstate.edu", task_id)
return result_url
def teco_spruce_data_assimilation(pars):
"""
DA TECO Spruce
args: pars - Initial parameters for TECO SPRUCE
kwargs: da_params - Which DA variable and min and max range for 18 variables
"""
task_id = str(teco_spruce_data_assimilation.request.id)
resultDir = setup_result_directory(task_id)
#parm template file
param_filename = create_template('SPRUCE_pars',pars,resultDir,check_params)
da_param_filename = create_template('SPRUCE_da_pars',pars,resultDir,check_params)
#if da_params:
# da_param_filename = create_template('spruce_da_pars',da_params,resultDir,check_params)
#else:
# copyfile("{0}/ecopad_tasks/default/SPRUCE_da_pars.txt".format(basedir),"{0}/SPRUCE_da_pars.txt".format(resultDir))
# da_param_filename ="SPRUCE_da_pars.txt"
#Run Spruce TECO code
host_data_resultDir = "{0}/static/ecopad_tasks/{1}".format(host_data_dir,task_id)
host_data_dir_spruce_data="{0}/local/spruce_data".format(host_data_dir)
docker_opts = "-v {0}:/data:z -v {1}:/spruce_data".format(host_data_resultDir,host_data_dir_spruce_data)
docker_cmd = "{0} {1} {2} {3} {4} {5}".format("/data/{0}".format(param_filename),"/spruce_data/SPRUCE_forcing.txt",
"/spruce_data/SPRUCE_obs.txt",
"/data",1, "/data/{0}".format(da_param_filename))
result = docker_task(docker_name="teco_spruce",docker_opts=docker_opts,docker_command=docker_cmd,id=task_id)
#Run R Plots
docker_opts = "-v {0}:/data:z ".format(host_data_resultDir)
docker_cmd ="Rscript ECOPAD_da_viz.R {0} {1}".format("/data/Paraest.txt","/data")
result = docker_task(docker_name="ecopad_r",docker_opts=docker_opts,docker_command=docker_cmd,id=task_id)
#Clean up result Directory
clean_up(resultDir)
#Create Report
report_data ={'zero_label':'Results','zero_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'histogram.png')}
report_data['title']="SPRUCE Ecological Data Assimilation Task Report"
desc= "Multiple data streams from SPRUCE are assimilated to TECO model using MCMC algorithm. "\
"The current dataset are mainly from pre-treatment measurement from 2011 to 2014. "\
"This will be updated regularly when new data stream is available. 5 out of 18 parameters are constrained from pre-treatment data. "\
"The 18 parameters are (1) specific leaf area, (2) maximum leaf growth rate, (3) maximum root growth rate, "\
"(4) maximum stem growth rate, (5) maximum rate of carboxylation, (6) turnover rate of foliage pool, "\
"(7) turnover rate of woody pool, (8) turnover rate of root pool, (9) turnover rate of fine litter pool, "\
"(10) turnover rate of coarse litter pool, (11) turnover rate of fast soil pool, (12) turnover rate of slow soil pool, "\
"(13) turnover rate of passive soil pool, (14) onset of growing degree days, (15) temperature sensitivity Q10, "\
"(16) baseline leaf respiration, (17) baseline stem respiration, (18) baseline root respiration"
report_data['description']=desc
report_name = create_report('report_da',report_data,resultDir)
return "http://{0}/ecopad_tasks/{1}".format(result['host'],result['task_id'])
def teco_spruce_data_assimilation(pars):
"""
DA TECO Spruce
args: pars - Initial parameters for TECO SPRUCE
kwargs: da_params - Which DA variable and min and max range for 18 variables
"""
task_id = str(teco_spruce_data_assimilation.request.id)
resultDir = setup_result_directory(task_id)
#parm template file
param_filename = create_template('SPRUCE_pars',pars,resultDir,check_params)
da_param_filename = create_template('SPRUCE_da_pars',pars,resultDir,check_params)
#if da_params:
# da_param_filename = create_template('spruce_da_pars',da_params,resultDir,check_params)
#else:
# copyfile("{0}/ecopad_tasks/default/SPRUCE_da_pars.txt".format(basedir),"{0}/SPRUCE_da_pars.txt".format(resultDir))
# da_param_filename ="SPRUCE_da_pars.txt"
#Run Spruce TECO code
host_data_resultDir = "{0}/static/ecopad_tasks/{1}".format(host_data_dir,task_id)
host_data_dir_spruce_data="{0}/local/spruce_data".format(host_data_dir)
docker_opts = "-v {0}:/data:z -v {1}:/spruce_data".format(host_data_resultDir,host_data_dir_spruce_data)
docker_cmd = "{0} {1} {2} {3} {4} {5}".format("/data/{0}".format(param_filename),"/spruce_data/SPRUCE_forcing.txt",
"/spruce_data/SPRUCE_obs.txt",
"/data",1, "/data/{0}".format(da_param_filename))
result = docker_task(docker_name="teco_spruce",docker_opts=docker_opts,docker_command=docker_cmd,id=task_id)
#Run R Plots
docker_opts = "-v {0}:/data:z ".format(host_data_resultDir)
docker_cmd ="Rscript ECOPAD_da_viz.R {0} {1}".format("/data/Paraest.txt","/data")
result = docker_task(docker_name="ecopad_r",docker_opts=docker_opts,docker_command=docker_cmd,id=task_id)
#Clean up result Directory
clean_up(resultDir)
#Create Report
report_data ={'zero_label':'Results','zero_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'histogram.png')}
report_data['title']="SPRUCE Ecological Data Assimilation Task Report"
desc= "Multiple data streams from SPRUCE are assimilated to TECO model using MCMC algorithm. "\
"The current dataset are mainly from pre-treatment measurement from 2011 to 2014. "\
"This will be updated regularly when new data stream is available. 5 out of 18 parameters are constrained from pre-treatment data. "\
"The 18 parameters are (1) specific leaf area, (2) maximum leaf growth rate, (3) maximum root growth rate, "\
"(4) maximum stem growth rate, (5) maximum rate of carboxylation, (6) turnover rate of foliage pool, "\
"(7) turnover rate of woody pool, (8) turnover rate of root pool, (9) turnover rate of fine litter pool, "\
"(10) turnover rate of coarse litter pool, (11) turnover rate of fast soil pool, (12) turnover rate of slow soil pool, "\
"(13) turnover rate of passive soil pool, (14) onset of growing degree days, (15) temperature sensitivity Q10, "\
"(16) baseline leaf respiration, (17) baseline stem respiration, (18) baseline root respiration"
report_data['description']=desc
report_name = create_report('report_da',report_data,resultDir)
return "http://{0}/ecopad_tasks/{1}".format(result['host'],result['task_id'])
def teco_spruce_simulation(pars): # ,model_type="0", da_params=None):
""" Setup task convert parameters from html portal
to file, and store the file in input folder.
call teco_spruce_model.
"""
task_id = str(teco_spruce_simulation.request.id)
resultDir = setup_result_directory(task_id)
#create param file
param_filename = create_template('SPRUCE_pars',pars,resultDir,check_params)
#Run Spruce TECO code
host_data_resultDir = "{0}/static/ecopad_tasks/{1}".format(host_data_dir,task_id)
host_data_dir_spruce_data="{0}/local/spruce_data".format(host_data_dir)
docker_opts = "-v {0}:/data:z -v {1}:/spruce_data:z".format(host_data_resultDir,host_data_dir_spruce_data)
docker_cmd = "{0} {1} {2} {3} {4} {5}".format("/data/{0}".format(param_filename),"/spruce_data/SPRUCE_forcing.txt",
"/spruce_data/SPRUCE_obs.txt",
"/data", 0 , "/spruce_data/SPRUCE_da_pars.txt")
result = docker_task(docker_name="teco_spruce",docker_opts=docker_opts,docker_command=docker_cmd,id=task_id)
#Run R Plots
#os.makedirs("{0}/graphoutput".format(host_data_resultDir)) #make plot directory
docker_opts = "-v {0}:/usr/local/src/myscripts/graphoutput:z ".format(host_data_resultDir)
docker_cmd = None
result = docker_task(docker_name="ecopad_r",docker_opts=docker_opts,docker_command=docker_cmd,id=task_id)
#Clean up result Directory
clean_up(resultDir)
#Create Report
report_data ={'zero_label':'GPP','zero_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'gpp.png'),
'one_label':'ER','one_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'er.png'),
'two_label':'Foliage','two_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'foliage.png'),
'three_label':'Wood','three_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'wood.png'),
'four_label':'Root','four_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'root.png'),
'five_label':'Soil','five_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'soil.png')}
report_data['title']="SPRUCE Ecological Simulation Task Report"
report_data['description']="Simulations of carbon fluxes and pool sizes for SPRUCE experiment based on user defined initial parameters."
report = create_report('report',report_data,resultDir)
result_url ="http://{0}/ecopad_tasks/{1}".format(result['host'],result['task_id'])
#report_url = "http://{0}/ecopad_tasks/{1}/{2}".format(result['host'],result['task_id'],"report.htm")
#{"report":report_url,"data":result_url}
return result_url
def teco_spruce_simulation(pars): # ,model_type="0", da_params=None):
""" Setup task convert parameters from html portal
to file, and store the file in input folder.
call teco_spruce_model.
"""
task_id = str(teco_spruce_simulation.request.id)
resultDir = setup_result_directory(task_id)
#create param file
param_filename = create_template('SPRUCE_pars',pars,resultDir,check_params)
#Run Spruce TECO code
host_data_resultDir = "{0}/static/ecopad_tasks/{1}".format(host_data_dir,task_id)
host_data_dir_spruce_data="{0}/local/spruce_data".format(host_data_dir)
docker_opts = "-v {0}:/data:z -v {1}:/spruce_data:z".format(host_data_resultDir,host_data_dir_spruce_data)
docker_cmd = "{0} {1} {2} {3} {4} {5}".format("/data/{0}".format(param_filename),"/spruce_data/SPRUCE_forcing.txt",
"/spruce_data/SPRUCE_obs.txt",
"/data", 0 , "/spruce_data/SPRUCE_da_pars.txt")
result = docker_task(docker_name="teco_spruce",docker_opts=docker_opts,docker_command=docker_cmd,id=task_id)
#Run R Plots
#os.makedirs("{0}/graphoutput".format(host_data_resultDir)) #make plot directory
docker_opts = "-v {0}:/usr/local/src/myscripts/graphoutput:z ".format(host_data_resultDir)
docker_cmd = None
result = docker_task(docker_name="ecopad_r",docker_opts=docker_opts,docker_command=docker_cmd,id=task_id)
#Clean up result Directory
clean_up(resultDir)
#Create Report
report_data ={'zero_label':'GPP','zero_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'gpp.png'),
'one_label':'ER','one_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'er.png'),
'two_label':'Foliage','two_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'foliage.png'),
'three_label':'Wood','three_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'wood.png'),
'four_label':'Root','four_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'root.png'),
'five_label':'Soil','five_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'soil.png')}
report_data['title']="SPRUCE Ecological Simulation Task Report"
report_data['description']="Simulations of carbon fluxes and pool sizes for SPRUCE experiment based on user defined initial parameters."
report = create_report('report',report_data,resultDir)
result_url ="http://{0}/ecopad_tasks/{1}".format(result['host'],result['task_id'])
#report_url = "http://{0}/ecopad_tasks/{1}/{2}".format(result['host'],result['task_id'],"report.htm")
#{"report":report_url,"data":result_url}
return result_url
def add_usingR(x, y):
task_id = str(add_usingR.request.id)
resultDir = setup_result_directory(task_id)
docker_opts = '-v /opt/osucybercom/data/static/campgruber:/script:z -w /script '
docker_cmd = "Rscript /script/add_usingR.R {0} {1}".format(x, y)
print docker_cmd, docker_opts
result = docker_task(docker_name="cybercom_r",
docker_opts=docker_opts,
docker_command=docker_cmd,
id=task_id)
result_url = "http://{0}/campgruber/tasks/{1}".format(
"cybercom-app.hpc.okstate.edu", task_id)
return result_url
def post(self, request,format=None):
filename = request.DATA.get('filename',None)
#filename =request.POST.get('filename',None)
if filename:
if os.path.isfile(filename):
p = Popen(['md5sum', filename], stdin=PIPE, stdout=PIPE, stderr=PIPE)
output, err = p.communicate()
resultDir = os.path.join("/data/tmp", self.get_username(request),output.split()[0])
#pass
else:
if not check_url_exist(filename):
raise Exception("Please Check URL or Local File Path(local files must be in /data directory) %s" % filename)
resultDir = os.path.join("/data/tmp", self.get_username(request))
map_read = os.path.join(resultDir,filename.split('/')[-1])
make_dirs(resultDir)
logfile= open(resultDir + "/logfile.txt","w")
call(['wget','-O',map_read,filename],stdout=logfile,stderr=logfile)
logfile.close()
p = Popen(['md5sum', map_read], stdin=PIPE, stdout=PIPE, stderr=PIPE)
output, err = p.communicate()
resultDir = os.path.join("/data/tmp", self.get_username(request),output.split()[0])
filename=map_read
make_dirs(resultDir)
else:
raise Exception('Please provide map file filename')
docker_cmd = self.docker_cmd % (filename,resultDir)
result = docker_task(docker_name=self.docker_container,docker_opts=self.docker_opts,docker_command=docker_cmd,id=output.split()[0])
logfile= glob.glob(resultDir + '/*.log')
status = ""
with open(logfile[0]) as f:
data = f.read()
if "No errors" in data:
status="SUCCESS"
else:
status="FAILURE"
return Response({
'status':status,
'log': data,
'local-file': filename
})
def new_task(pars): # ,model_type="0", da_params=None):
""" Setup task convert parameters from html portal
to file, and store the file in input folder.
call teco_spruce_model.
"""
host_data_dir = "/home/vova/ecopad/data"
task_id = str(new_task.request.id)
resultDir = setup_result_directory(task_id)
#create param file
param_filename = create_template('f0_SPRUCE_parameterfile', pars,
resultDir, check_params)
#Run Spruce TECO code
host_data_resultDir = "{0}/static/ecopad_tasks/{1}".format(
host_data_dir, task_id)
host_data_dir_spruce_data = "{0}/local/frc_01_input".format(host_data_dir)
docker_opts = "-v {0}:/data:z -v {1}:/frc_01_input:z".format(
host_data_resultDir, host_data_dir_spruce_data)
docker_cmd = "{0} {1}".format(
"/data/{0}".format(param_filename),
#COMMENT f0 Working
"/frc_01_input/f1_EM1forcing2011-2015.txt") #,
#COMMENT f1 Working
# "/frc_01_input/f2_SPRUCE_Water_Table_Level_2011-2014.txt") #,
#COMMENT f2 is not Working ERROR: At line 4371 of file /source/frc_01.f90 (unit = 111, file = 'fort.111')
# "/frc_01_input/f3_SPRUCE_Snow_Depth_2011-2014.txt") #,
#COMMENT f3 Working
# "/frc_01_input/f4_obs_SPRUCE.txt",
#COMMENT f4 In process
# "/frc_01_input/f5_SPRUCE_hummock_toplayer.txt",
#COMMENT f5 In process
# "/frc_01_input/f6_SPRUCE_soilt.txt",
#COMMENT f6 In process
# "/frc_01_input/f7_daily_soilt_2011-2014.txt",
#COMMENT f7 In process
# "/frc_01_input/f8_daily_watertable_2011-2014.txt",
#COMMENT f8 In process
# "/frc_01_input/f9_obs_CH4_for_MEMCMC.txt")
#COMMENT f9 In process
#!!!!
#Need one more command to send output back (out of container before termination).
#!!!!
result = docker_task(docker_name="frc_01",
docker_opts=docker_opts,
docker_command=docker_cmd,
id=task_id)
#Run R Plots
#os.makedirs("{0}/graphoutput".format(host_data_resultDir)) #make plot directory
# docker_opts = "-v {0}:/usr/local/src/myscripts/graphoutput:z ".format(host_data_resultDir)
# docker_cmd = None
# result = docker_task(docker_name="ecopad_r",docker_opts=docker_opts,docker_command=docker_cmd,id=task_id)
#
#
# #Clean up result Directory
# clean_up(resultDir)
# #Create Report
report_data = {
'zero_label': 'GPP',
'zero_url': '/ecopad_tasks/{0}/plot/{1}'.format(task_id, 'gpp.png'),
'one_label': 'ER',
'one_url': '/ecopad_tasks/{0}/plot/{1}'.format(task_id, 'er.png'),
'two_label': 'Foliage',
'two_url': '/ecopad_tasks/{0}/plot/{1}'.format(task_id, 'foliage.png'),
'three_label': 'Wood',
'three_url': '/ecopad_tasks/{0}/plot/{1}'.format(task_id, 'wood.png'),
'four_label': 'Root',
'four_url': '/ecopad_tasks/{0}/plot/{1}'.format(task_id, 'root.png'),
'five_label': 'Soil',
'five_url': '/ecopad_tasks/{0}/plot/{1}'.format(task_id, 'soil.png')
}
report_data['title'] = "SPRUCE Ecological Simulation Task Report"
report_data[
'description'] = "Simulations of carbon fluxes and pool sizes for SPRUCE experiment based on user defined initial parameters."
#
report = create_report('report', report_data, resultDir)
result_url = "http://{0}/ecopad_tasks/{1}".format(result['host'],
result['task_id'])
#report_url = "http://{0}/ecopad_tasks/{1}/{2}".format(result['host'],result['task_id'],"report.htm")
#{"report":report_url,"data":result_url}
return result_url
def new_code_build_sim(pars): # ,model_type="0", da_params=None):
""" Setup task convert parameters from html portal
to file, and store the file in input folder.
call teco_spruce_model.
"""
task_id = str(new_code_build_sim.request.id)
resultDir = setup_result_directory(task_id)
#create param file
param_filename = create_template('SPRUCE_parameterfile', pars, resultDir,
check_params)
#Run Spruce TECO code
host_data_resultDir = "{0}/static/ecopad_tasks/{1}".format(
host_data_dir, task_id)
host_data_dir_spruce_data = "{0}/local/example_code_mip_data".format(
host_data_dir)
docker_opts = "-v {0}:/data:z -v {1}:/example_code_mip_data:z".format(
host_data_resultDir, host_data_dir_spruce_data)
docker_cmd = "{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10} {11}".format(
"/data/{0}".format(param_filename),
"example_code_mip_data/daily_soilt_2011-2014.txt",
"example_code_mip/daily_watertable2011_2014.txt",
"/example_code_mip_data/EM1forcing2011-2015.txt",
"example_code_mip_data/obs_CH4_for_MEMCMC.txt",
"/example_code_mip_data/obs_SPRUCE.txt",
"example_code_mip_data/SPRUCE_hummock_toplayer.txt",
"example_code_mip_data/SPRUCE_Snow_Depth_2011-2014.txt",
"SPRUCE_soilt.txt", "SPRUCE_Watr_Table_Level_2011-2014.txt", "/data",
0)
result = docker_task(docker_name="new_code_build",
docker_opts=docker_opts,
docker_command=docker_cmd,
id=task_id)
#Run R Plots
#os.makedirs("{0}/graphoutput".format(host_data_resultDir)) #make plot directory
# docker_opts = "-v {0}:/usr/local/src/myscripts/graphoutput:z ".format(host_data_resultDir)
# docker_cmd = None
# result = docker_task(docker_name="ecopad_r",docker_opts=docker_opts,docker_command=docker_cmd,id=task_id)
#
#
# #Clean up result Directory
# clean_up(resultDir)
# #Create Report
report_data = {
'zero_label': 'GPP',
'zero_url': '/ecopad_tasks/{0}/plot/{1}'.format(task_id, 'gpp.png'),
'one_label': 'ER',
'one_url': '/ecopad_tasks/{0}/plot/{1}'.format(task_id, 'er.png'),
'two_label': 'Foliage',
'two_url': '/ecopad_tasks/{0}/plot/{1}'.format(task_id, 'foliage.png'),
'three_label': 'Wood',
'three_url': '/ecopad_tasks/{0}/plot/{1}'.format(task_id, 'wood.png'),
'four_label': 'Root',
'four_url': '/ecopad_tasks/{0}/plot/{1}'.format(task_id, 'root.png'),
'five_label': 'Soil',
'five_url': '/ecopad_tasks/{0}/plot/{1}'.format(task_id, 'soil.png')
}
report_data['title'] = "SPRUCE Ecological Simulation Task Report"
report_data[
'description'] = "Simulations of carbon fluxes and pool sizes for SPRUCE experiment based on user defined initial parameters."
#
report = create_report('report', report_data, resultDir)
result_url = "http://{0}/ecopad_tasks/{1}".format(result['host'],
result['task_id'])
#report_url = "http://{0}/ecopad_tasks/{1}/{2}".format(result['host'],result['task_id'],"report.htm")
#{"report":report_url,"data":result_url}
return result_url
def teco_spruce_forecast(pars,forecast_year,forecast_day,temperature_treatment=0.0,co2_treatment=380.0,da_task_id=None,public=None):
"""
Forecasting
args: pars - Initial parameters for TECO SPRUCE
forecast_year,forecast_day
"""
task_id = str(teco_spruce_forecast.request.id)
resultDir = setup_result_directory(task_id)
param_filename = create_template('SPRUCE_pars',pars,resultDir,check_params)
#da_param_filename = create_template('SPRUCE_da_pars',pars,resultDir,check_params)
da_param_filename ="SPRUCE_da_pars.txt"
host_data_dir_spruce_data="{0}/local/spruce_data".format(host_data_dir)
#Set Param estimation file from DA
if not da_task_id:
try:
copyfile("{0}/Paraest.txt".format(spruce_data_folder),"{0}/Paraest.txt".format(resultDir))
copyfile("{0}/SPRUCE_da_pars.txt".format(spruce_data_folder),"{0}/SPRUCE_da_pars.txt".format(resultDir))
except:
error_file = "{0}/Paraest.txt or SPRUCE_da_pars.txt".format(spruce_data_folder)
raise Exception("Parameter Estimation file location problem. {0} file not found.".format(error_file))
else:
try:
copyfile("{0}/ecopad_tasks/{1}/input/Paraest.txt".format(basedir,da_task_id),"{0}/Paraest.txt".format(resultDir))
copyfile("{0}/ecopad_tasks/{1}/input/SPRUCE_da_pars.txt".format(basedir,da_task_id),"{0}/SPRUCE_da_pars.txt".format(resultDir))
except:
error_file = "{0}/ecopad_tasks/{1}/input/Paraest.txt or SPRUCE_da_pars.txt".format(basedir,da_task_id)
raise Exception("Parameter Estimation file location problem. {0} file not found.".format(error_file))
#Run Spruce TECO code
host_data_resultDir = "{0}/static/ecopad_tasks/{1}".format(host_data_dir,task_id)
host_data_dir_spruce_data="{0}/local/spruce_data".format(host_data_dir)
docker_opts = "-v {0}:/data:z -v {1}:/spruce_data".format(host_data_resultDir,host_data_dir_spruce_data)
docker_cmd = "{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}".format("/data/{0}".format(param_filename),
"/spruce_data/SPRUCE_forcing.txt", "/spruce_data/SPRUCE_obs.txt",
"/data",2, "/data/{0}".format(da_param_filename),
"/spruce_data/Weathergenerate",forecast_year, forecast_day,
temperature_treatment,co2_treatment)
result = docker_task(docker_name="teco_spruce",docker_opts=docker_opts,docker_command=docker_cmd,id=task_id)
#Run R Plots
docker_opts = "-v {0}:/data:z ".format(host_data_resultDir)
docker_cmd ="Rscript ECOPAD_forecast_viz.R {0} {1} {2} {3}".format("obs_file/SPRUCE_obs.txt","/data","/data",100)
result = docker_task(docker_name="ecopad_r",docker_opts=docker_opts,docker_command=docker_cmd,id=task_id)
# Yuanyuan add to reformat output data
docker_opts = "-v {0}:/data:z ".format(host_data_resultDir)
docker_cmd = "Rscript reformat_to_csv.R {0} {1} {2} {3} {4}".format("/data","/data",100,temperature_treatment,co2_treatment)
#docker_opts = "-v {0}:/data:z ".format(host_data_resultDir)
#docker_cmd = "Rscript reformat_to_csv_backup.R {0} {1} {2}".format("/data","/data",100)
# docker_opts = None
# docker_cmd = None
result = docker_task(docker_name="ecopad_r",docker_opts=docker_opts,docker_command=docker_cmd,id=task_id)
#Clean up result Directory
clean_up(resultDir)
#Create Report
report_data ={'zero_label':'GPP Forecast','zero_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'gpp_forecast.png'),
'one_label':'ER Forecast','one_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'er_forecast.png'),
'two_label':'Foliage Forecast','two_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'foliage_forecast.png'),
'three_label':'Wood Forecast','three_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'wood_forecast.png'),
'four_label':'Root Forecast','four_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'root_forecast.png'),
'five_label':'Soil Forecast','five_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'soil_forecast.png')}
report_data['title']="SPRUCE Ecological Forecast Task Report"
desc = "Use constrained parameters from Data Assimilation to predict carbon fluxes and pool sizes. "
desc = desc + "Forcing inputs are genereated by auto-regression model using historical climate data of the SPRUCE site. "
desc = desc + "Allow users to choose which year and day to make predictations of ecosystem in response to treatment effects."
report_data['description']=desc
report_name = create_report('report',report_data,resultDir)
#return {"data":"http://{0}/ecopad_tasks/{1}".format(result['host'],result['task_id']),
# "report": "http://{0}/ecopad_tasks/{1}/{2}".format(result['host'],result['task_id'],report_name)}
result_url = "http://{0}/ecopad_tasks/{1}".format(result['host'],result['task_id'])
if public:
data={'tag':public,'result_url':result_url,'task_id':task_id,'timestamp':datetime.now()}
db=MongoClient("mongodb://*****:*****@cybercom_mongo:27017/?ssl=true&ssl_ca_certs=/ssl/testca/cacert.pem&ssl_certfile=/ssl/client/mongodb.pem",27017)
db.forecast.public.save(data)
return result_url
def teco_spruce_forecast(pars,forecast_year,forecast_day,temperature_treatment=0.0,co2_treatment=380.0,da_task_id=None,public=None):
"""
Forecasting
args: pars - Initial parameters for TECO SPRUCE
forecast_year,forecast_day
"""
task_id = str(teco_spruce_forecast.request.id)
resultDir = setup_result_directory(task_id)
param_filename = create_template('SPRUCE_pars',pars,resultDir,check_params)
#da_param_filename = create_template('SPRUCE_da_pars',pars,resultDir,check_params)
da_param_filename ="SPRUCE_da_pars.txt"
host_data_dir_spruce_data="{0}/local/spruce_data".format(host_data_dir)
#Set Param estimation file from DA
if not da_task_id:
try:
copyfile("{0}/Paraest.txt".format(spruce_data_folder),"{0}/Paraest.txt".format(resultDir))
copyfile("{0}/SPRUCE_da_pars.txt".format(spruce_data_folder),"{0}/SPRUCE_da_pars.txt".format(resultDir))
except:
error_file = "{0}/Paraest.txt or SPRUCE_da_pars.txt".format(spruce_data_folder)
raise Exception("Parameter Estimation file location problem. {0} file not found.".format(error_file))
else:
try:
copyfile("{0}/ecopad_tasks/{1}/input/Paraest.txt".format(basedir,da_task_id),"{0}/Paraest.txt".format(resultDir))
copyfile("{0}/ecopad_tasks/{1}/input/SPRUCE_da_pars.txt".format(basedir,da_task_id),"{0}/SPRUCE_da_pars.txt".format(resultDir))
except:
error_file = "{0}/ecopad_tasks/{1}/input/Paraest.txt or SPRUCE_da_pars.txt".format(basedir,da_task_id)
raise Exception("Parameter Estimation file location problem. {0} file not found.".format(error_file))
#Run Spruce TECO code
host_data_resultDir = "{0}/static/ecopad_tasks/{1}".format(host_data_dir,task_id)
host_data_dir_spruce_data="{0}/local/spruce_data".format(host_data_dir)
docker_opts = "-v {0}:/data:z -v {1}:/spruce_data".format(host_data_resultDir,host_data_dir_spruce_data)
docker_cmd = "{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}".format("/data/{0}".format(param_filename),
"/spruce_data/SPRUCE_forcing.txt", "/spruce_data/SPRUCE_obs.txt",
"/data",2, "/data/{0}".format(da_param_filename),
"/spruce_data/Weathergenerate",forecast_year, forecast_day,
temperature_treatment,co2_treatment)
result = docker_task(docker_name="teco_spruce",docker_opts=docker_opts,docker_command=docker_cmd,id=task_id)
#Run R Plots
docker_opts = "-v {0}:/data:z ".format(host_data_resultDir)
docker_cmd ="Rscript ECOPAD_forecast_viz.R {0} {1} {2} {3}".format("obs_file/SPRUCE_obs.txt","/data","/data",100)
result = docker_task(docker_name="ecopad_r",docker_opts=docker_opts,docker_command=docker_cmd,id=task_id)
# Yuanyuan add to reformat output data
docker_opts = "-v {0}:/data:z ".format(host_data_resultDir)
docker_cmd = "Rscript reformat_to_csv.R {0} {1} {2} {3} {4}".format("/data","/data",100,temperature_treatment,co2_treatment)
#docker_opts = "-v {0}:/data:z ".format(host_data_resultDir)
#docker_cmd = "Rscript reformat_to_csv_backup.R {0} {1} {2}".format("/data","/data",100)
# docker_opts = None
# docker_cmd = None
result = docker_task(docker_name="ecopad_r",docker_opts=docker_opts,docker_command=docker_cmd,id=task_id)
#Clean up result Directory
clean_up(resultDir)
#Create Report
report_data ={'zero_label':'GPP Forecast','zero_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'gpp_forecast.png'),
'one_label':'ER Forecast','one_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'er_forecast.png'),
'two_label':'Foliage Forecast','two_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'foliage_forecast.png'),
'three_label':'Wood Forecast','three_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'wood_forecast.png'),
'four_label':'Root Forecast','four_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'root_forecast.png'),
'five_label':'Soil Forecast','five_url':'/ecopad_tasks/{0}/plot/{1}'.format(task_id,'soil_forecast.png')}
report_data['title']="SPRUCE Ecological Forecast Task Report"
desc = "Use constrained parameters from Data Assimilation to predict carbon fluxes and pool sizes. "
desc = desc + "Forcing inputs are genereated by auto-regression model using historical climate data of the SPRUCE site. "
desc = desc + "Allow users to choose which year and day to make predictations of ecosystem in response to treatment effects."
report_data['description']=desc
report_name = create_report('report',report_data,resultDir)
#return {"data":"http://{0}/ecopad_tasks/{1}".format(result['host'],result['task_id']),
# "report": "http://{0}/ecopad_tasks/{1}/{2}".format(result['host'],result['task_id'],report_name)}
result_url = "http://{0}/ecopad_tasks/{1}".format(result['host'],result['task_id'])
if public:
data={'tag':public,'result_url':result_url,'task_id':task_id,'timestamp':datetime.now()}
db=MongoClient('ecopad_mongo',27017)
db.forecast.public.save(data)
return result_url