def task_runner(sa_engine, task):
# Get crop_name and mgmt_code
crop_no = task["crop_no"]
lat = float(task["latitude"])
lon = float(task["longitude"])
year = 1900
cip = None
wdp = None
# Get a crop info provider
try:
t1 = time.time()
cropname, watersupply = select_crop(sa_engine, crop_no)
cip = CropInfoProvider(cropname, watersupply, run_settings.data_dir)
cropdata = cip.getCropData()
msg = "Retrieving data for crop %s, %s took %6.1f seconds"
print msg % (cropname, watersupply, time.time() - t1)
# Everything seems to be set for the task now! Lon-lat combination is
# checked in the call to the getSeasonDates method. We assume that the
# landmask was checked. Also we assume that the data on the growing
# season were checked. These are represented by day-of-year values.
start_doy, end_doy = cip.getSeasonDates(lon, lat)
# Get the weather data
wdp = Hdf5WeatherDataProvider(run_settings.hdf5_meteo_file, lat, lon)
# Loop over the years
t2 = time.time()
tsums = []
for year in get_available_years(wdp):
# Get timer data for the current year
timerdata = cip.getTimerData(start_doy, end_doy, year)
# Check that START_DATE does not fall before the first available
# meteodata and END_DATE not beyond the last data with available
# weather data.
if timerdata['START_DATE'] < wdp.first_date: continue
if timerdata['END_DATE'] > wdp.last_date: continue
sitedata = {}
soildata = {"SMFCF": 0.4}
# Run simulation
pheno = wofostEngine(sitedata,
timerdata,
soildata,
cropdata,
wdp,
config="Wofost71_PhenoOnly.conf")
pheno.run(days=366)
sumresults = pheno.get_summary_output()
if len(sumresults) > 0:
tsums.append(sumresults[0]["TSUM"])
else:
msg = "No summary results for crop/year/lat/lon: %s/%s/%s/%s"
logging.error(msg, crop_no, year, lat, lon)
# end year
# Insert average etc. into database
if len(tsums) > 0:
insert_tsum(sa_engine, 1, crop_no, lat, lon, mean(tsums),
std(tsums, ddof=1), min(tsums), max(tsums), len(tsums))
msg = "Simulating for lat-lon (%s, %s) took %6.1f seconds"
print msg % (str(lat), str(lon), time.time() - t2)
else:
msg = "Failed calculating TSUMs for lat-lon (%s, %s): no TSUMs calculated for 30 years"
logging.error(msg, lat, lon)
finally:
# Clean up
if wdp is not None:
wdp.close()
if cip is not None:
cip.close()
def task_runner(sa_engine, task):
# Get crop_name and mgmt_code
crop_no = task["crop_no"]
lat = float(task["latitude"])
lon = float(task["longitude"])
year = 1900
cip = None
wdp = None
# Get a crop info provider
try:
t1 = time.time()
cropname, watersupply = select_crop(sa_engine, crop_no)
cip = CropInfoProvider(cropname, watersupply, run_settings.data_dir)
cropdata = cip.getCropData()
msg = "Retrieving data for crop %s, %s took %6.1f seconds"
print msg % (cropname, watersupply, time.time()-t1)
# Everything seems to be set for the task now! Lon-lat combination is
# checked in the call to the getSeasonDates method. We assume that the
# landmask was checked. Also we assume that the data on the growing
# season were checked. These are represented by day-of-year values.
start_doy, end_doy = cip.getSeasonDates(lon, lat)
# Get the weather data
wdp = Hdf5WeatherDataProvider(run_settings.hdf5_meteo_file, lat, lon)
# Loop over the years
t2 = time.time()
tsums = []
for year in get_available_years(wdp):
# Get timer data for the current year
timerdata = cip.getTimerData(start_doy, end_doy, year)
# Check that START_DATE does not fall before the first available
# meteodata and END_DATE not beyond the last data with available
# weather data.
if timerdata['START_DATE'] < wdp.first_date: continue
if timerdata['END_DATE'] > wdp.last_date: continue
sitedata = {}
soildata = {"SMFCF":0.4}
# Run simulation
pheno = wofostEngine(sitedata, timerdata, soildata, cropdata, wdp,
config="Wofost71_PhenoOnly.conf")
pheno.run(days=366)
sumresults = pheno.get_summary_output()
if len(sumresults) > 0:
tsums.append(sumresults[0]["TSUM"])
else:
msg = "No summary results for crop/year/lat/lon: %s/%s/%s/%s"
logging.error(msg, crop_no, year, lat, lon)
# end year
# Insert average etc. into database
if len(tsums) > 0:
insert_tsum(sa_engine, 1, crop_no, lat, lon, mean(tsums),
std(tsums, ddof=1), min(tsums), max(tsums), len(tsums))
msg = "Simulating for lat-lon (%s, %s) took %6.1f seconds"
print msg % (str(lat), str(lon), time.time()-t2)
else:
msg = "Failed calculating TSUMs for lat-lon (%s, %s): no TSUMs calculated for 30 years"
logging.error(msg, lat, lon)
finally:
# Clean up
if wdp is not None:
wdp.close()
if cip is not None:
cip.close()
def task_runner(sa_engine, task):
# Get crop_name and mgmt_code
crop_no = task["crop_no"]
lat = float(task["latitude"])
lon = float(task["longitude"])
year = 1900
cip = None
wdp = None
logger = logging.getLogger("GGCMI Task Runner")
logger.info("Starting task runner for task %i" % task["task_id"])
# Get a crop info provider
try:
t1 = time.time()
cropname, watersupply = select_crop(sa_engine, crop_no)
cip = CropInfoProvider(
cropname,
watersupply,
)
cropdata = cip.getCropData()
if (not task.has_key("tsum1")) or (not task.has_key("tsum2")):
msg = "Location specific values for crop parameter(s) missing: TSUM1 and/or TSUM2"
raise PCSEError(msg)
else:
cropdata["TSUM1"] = float(task["tsum1"])
cropdata["TSUM2"] = float(task["tsum2"])
start_doy, end_doy = cip.getSeasonDates(lon, lat)
msg = "Retrieving data for crop %s, %s took %6.1f seconds"
logger.info(msg % (cropname, watersupply, time.time() - t1))
# Everything seems to be set for the task now! Lon-lat combination is
# checked in the call to the getSeasonDates method. We assume that the
# landmask was checked. Also we assume that the data on the growing
# season were checked. These are represented by day-of-year values.
# Get the weather data
t2 = time.time()
wdp = Hdf5WeatherDataProvider(run_settings.hdf5_meteo_file, lat, lon)
available_years = get_available_years(wdp)
msg = "Retrieving weather data for lat-lon %s, %s took %6.1f seconds"
logger.debug(msg % (str(lat), str(lon), time.time() - t2))
# Loop over the years
t3 = time.time()
allresults = []
msg = None
for year in available_years:
# Get timer data for the current year
timerdata = cip.getTimerData(start_doy, end_doy, year)
# Check that START_DATE does not fall before the first available
# meteodata and END_DATE not beyond the last data with available
# weather data.
if timerdata['START_DATE'] < wdp.first_date:
continue
if timerdata['END_DATE'] > wdp.last_date:
continue
# Get soil and site data
soildata = run_settings.get_soil_data(lon, lat)
sitedata = run_settings.get_site_data(soildata)
# Run simulation
if watersupply == 'ir':
configFile = 'GGCMI_PP.conf'
else:
configFile = 'GGCMI_WLP.conf'
if msg is None:
msg = "Starting simulation for %s-%s (%5.2f, %5.2f), planting " \
"at: %s, final harvest at: %s"
msg = msg % (cropname, watersupply, lon, lat,
timerdata['CROP_START_DATE'],
timerdata['CROP_END_DATE'])
logger.info(msg)
wofost = wofostEngine(sitedata,
timerdata,
soildata,
cropdata,
wdp,
config=configFile)
wofost.run_till_terminate()
results = wofost.get_output()
sumresults = wofost.get_summary_output()
if (len(results) > 0) and (len(sumresults) > 0):
allresults.append({
"year": year,
"summary": sumresults,
"results": results
})
else:
msg = "Insufficient results for crop/year/lat/lon: %s/%s/%s/%s"
logger.error(msg, crop_no, year, lat, lon)
# end year
if len(allresults) > 0:
# pickle all results
task_id = task["task_id"]
obj = {
"task_id": task_id,
"crop_no": crop_no,
"longitude": lon,
"latitude": lat
}
obj["allresults"] = allresults
# Write results to pickle file. First to .tmp then rename to .pkl
# to avoid read/write collisions with ggcmi_processsor
pickle_fname = run_settings.output_file_template % task_id
if os.path.splitext(pickle_fname)[1] == ".pkl":
pickle_fname += ".tmp"
else:
pickle_fname += ".pkl.tmp"
pickle_fname_fp = os.path.join(run_settings.output_folder,
pickle_fname)
with open(pickle_fname_fp, 'wb') as f:
cPickle.dump(obj, f, cPickle.HIGHEST_PROTOCOL)
final_fname_fp = os.path.splitext(pickle_fname_fp)[0]
os.rename(pickle_fname_fp, final_fname_fp)
msg = "Simulating for lat-lon (%s, %s) took %6.1f seconds"
logger.info(msg % (str(lat), str(lon), time.time() - t3))
else:
msg = "Failed simulating for lon,lat,crop, watersupply (%s, %s, " \
"%s, %s): no output available."
msg = msg % (lon, lat, cropname, watersupply)
raise PCSEError(msg)
finally:
if wdp is not None:
wdp.close()
if cip is not None:
cip.close()
def task_runner(sa_engine, task):
# Get crop_name and mgmt_code
crop_no = task["crop_no"]
cropname, watersupply = select_crop(sa_engine, crop_no)
msg = "Starting to simulate for %s (%s)" % (cropname, watersupply)
print msg
# Get a crop info provider
t1 = time.time()
cip = CropInfoProvider(cropname, watersupply, run_settings.datadir)
extent = cip.getExtent()
cropdata = cip.getCropData()
msg = ("Retrieving data for crop %s, %s took %6.1f seconds" % (cropname, watersupply, time.time()-t1))
print msg
# Everything seems to be set for the task now!
# Make sure we can loop over the grid cells from UL down to LR corner
nvlp = extent
x_range = arange(nvlp.getMinX() + 0.5*nvlp.dx, nvlp.getMaxX() + 0.5*nvlp.dy, nvlp.dx)
y_range = arange(nvlp.getMinY() + 0.5*nvlp.dy, nvlp.getMaxY() + 0.5*nvlp.dx, nvlp.dy)
if nvlp.xcoords_sort != 'ASC':
x_range = reversed(x_range)
if nvlp.ycoords_sort != 'DESC':
y_range = reversed(y_range)
# Find out whether maybe another process already calculated TSUMs for this crop
minlat, maxlon = get_resumption_point(sa_engine, crop_no)
# Loop but make sure not to waste time on pixels that are not interesting
for lat in y_range:
if (lat > minlat):
continue
eps = 0.0000001
for lon in x_range:
try:
if abs(lat - minlat) < eps and (lon - maxlon) < eps:
continue
# Check the landmask
if not cip.get_landmask(lon, lat):
continue
# Check the data on the growing season represented by day-of-year
# values. If start_doy or end_doy return nodata value (-99), then
# continue with next grid directly.
start_doy, end_doy = cip.getSeasonDates(lon, lat)
if (start_doy == -99) or (end_doy == -99):
continue
wdp = Hdf5WeatherDataProvider(run_settings.hdf5_meteo_file, lat, lon)
# Loop over the years
t2 = time.time()
tsums = []
for year in get_available_years(wdp):
# Get timer data for the current year
timerdata = cip.getTimerData(start_doy, end_doy, year)
# Check that START_DATE does not fall before the first
# available meteodata and END_DATE not beyond the
# last data with available weather data.
if timerdata['START_DATE'] < wdp.first_date or \
timerdata['END_DATE'] > wdp.last_date:
continue
sitedata = {}
soildata = {"SMFCF":0.4}
# Run simulation
pheno = wofostEngine(sitedata, timerdata, soildata, cropdata,
wdp, config="Wofost71_PhenoOnly.conf")
pheno.run(days=366)
sumresults = pheno.get_summary_output()
if len(sumresults) > 0:
tsums.append(sumresults[0]["TSUM"])
else:
msg = "No summary results for crop/year/lat/lon: %s/%s/%s/%s"
logging.error(msg, crop_no, year, lat, lon)
# end year
# Insert average etc. into database
if len(tsums) > 0:
insert_tsum(sa_engine, 1, crop_no, lat, lon, mean(tsums), std(tsums, ddof=1),
min(tsums), max(tsums), len(tsums))
msg = ("Simulating for lat-lon (%s, %s) took %6.1f seconds" % (str(lat), str(lon), time.time()-t2))
print msg
else:
msg = "Failed calculating TSUMs for lat-lon (%s, %s): no TSUMs calculated for 30 years"
logging.error(msg, lat, lon)
if wdp is not None:
wdp.close()
except tables.NoSuchNodeError:
msg = "No weather data found for lat/lon: %s/%s"
logging.error(msg, lat, lon)
except PCSEError:
msg = "Error in PCSE for crop/year/lat/lon: %s/%s/%s/%s"
logging.exception(msg, crop_no, year, lat, lon)
print "Finished simulating for " + cropname + " (" + watersupply + ")"
cip.close()