def main():
# Initialise
db_engine = None;
tasks = []
try:
db_engine = sa_engine.create_engine(run_settings.connstr)
db_metadata = MetaData(db_engine)
task_id = 15051
for crop_no in range(19, 29):
# Report which crop is next
crop_name, mgmt_code = select_crop(db_engine, crop_no)
msg = "About to get TSUM values for " + crop_name + " (" + mgmt_code + ")"
logging.info(msg)
print msg
# Now retrieve how to divide over TSUM1 and TSUM2
cip = CropInfoProvider(crop_name, mgmt_code, run_settings.data_dir)
cropdata = cip.getCropData();
nvlp = cip.getExtent()
# Get the relevant spatial range and loop through
y_range = get_range(db_engine, crop_no, nvlp.dy)
x_range = arange(nvlp.getMinX() + 0.5*nvlp.dx, nvlp.getMaxX() + 0.5*nvlp.dy, nvlp.dx);
taskdict = {"task_id":1, "status":"Pending" , "hostname":"None", "crop_no":1, "longitude":0.0, "latitude": 0.0, "tsum1":0.0, "tsum2":0.0, "process_id":1, "comment":""}
for lat in y_range:
# Retrieve the lat-lon combinations for which there are records in the table TSUM
lons_with_tsums = get_places_with_tsums(db_engine, crop_no, lat)
for lon in x_range:
# Pixel should be on land, without tsum so far and there should be a crop calendar for it
if not cip.get_landmask(lon, lat):
continue;
if lon in lons_with_tsums:
continue;
start_doy, end_doy = cip.getSeasonDates(lon, lat);
if (start_doy == -99) or (end_doy == -99):
continue;
# tsum = get_tsum_from_db(db_engine, crop_no, lon, lat)
tsum1, tsum2 = (0, 0) # split_tsum(cropdata, tsum)
# Create a task and append it
task = Task(task_id, "Pending", "None", crop_no, lon, lat, tsum1, tsum2, 0, "")
tasks.append(task.as_dict())
task_id = task_id + 1
# Once in a while, write them to the database
if (task_id % 1000 == 0):
store_to_database(db_engine, tasks, db_metadata, taskdict)
del tasks[:]
print "Thousand records saved to database. Current latitude is %s" % lat
# end lon
# end lat
store_to_database(db_engine, tasks, db_metadata, taskdict)
print "Another %s records saved to database." % len(tasks)
del tasks[:]
except SQLAlchemyError, inst:
print ("Database error: %s" % inst)
def main():
# Initialise
db_engine = None
tasks = []
try:
db_engine = sa_engine.create_engine(run_settings.connstr)
db_metadata = MetaData(db_engine)
task_id = 15051
for crop_no in range(19, 29):
# Report which crop is next
crop_name, mgmt_code = select_crop(db_engine, crop_no)
msg = "About to get TSUM values for " + crop_name + " (" + mgmt_code + ")"
logging.info(msg)
print msg
# Now retrieve how to divide over TSUM1 and TSUM2
cip = CropInfoProvider(crop_name, mgmt_code, run_settings.data_dir)
cropdata = cip.getCropData()
nvlp = cip.getExtent()
# Get the relevant spatial range and loop through
y_range = get_range(db_engine, crop_no, nvlp.dy)
x_range = arange(nvlp.getMinX() + 0.5 * nvlp.dx,
nvlp.getMaxX() + 0.5 * nvlp.dy, nvlp.dx)
taskdict = {
"task_id": 1,
"status": "Pending",
"hostname": "None",
"crop_no": 1,
"longitude": 0.0,
"latitude": 0.0,
"tsum1": 0.0,
"tsum2": 0.0,
"process_id": 1,
"comment": ""
}
for lat in y_range:
# Retrieve the lat-lon combinations for which there are records in the table TSUM
lons_with_tsums = get_places_with_tsums(
db_engine, crop_no, lat)
for lon in x_range:
# Pixel should be on land, without tsum so far and there should be a crop calendar for it
if not cip.get_landmask(lon, lat):
continue
if lon in lons_with_tsums:
continue
start_doy, end_doy = cip.getSeasonDates(lon, lat)
if (start_doy == -99) or (end_doy == -99):
continue
# tsum = get_tsum_from_db(db_engine, crop_no, lon, lat)
tsum1, tsum2 = (0, 0) # split_tsum(cropdata, tsum)
# Create a task and append it
task = Task(task_id, "Pending", "None", crop_no, lon, lat,
tsum1, tsum2, 0, "")
tasks.append(task.as_dict())
task_id = task_id + 1
# Once in a while, write them to the database
if (task_id % 1000 == 0):
store_to_database(db_engine, tasks, db_metadata,
taskdict)
del tasks[:]
print "Thousand records saved to database. Current latitude is %s" % lat
# end lon
# end lat
store_to_database(db_engine, tasks, db_metadata, taskdict)
print "Another %s records saved to database." % len(tasks)
del tasks[:]
except SQLAlchemyError, inst:
print("Database error: %s" % inst)
def task_runner(sa_engine, task):
# Attributes
cip = None;
extent = None;
cropdata = None;
crop_no = 0;
# Initialise
datadir = r"/home/hoek008/projects/ggcmi/data/"
fn = r"./AgMERRA/AgMERRA_1980-01-01_2010-12-31_final.hf5"
# Get crop_name and mgmt_code
crop_no = task["crop_no"];
cropname, watersupply = select_crop(sa_engine, crop_no);
print "About to simulate for " + cropname + " (" + watersupply + ")";
try:
# Get a crop info provider
t1 = time.time()
cip = CropInfoProvider(cropname, watersupply, datadir);
extent = cip.getExtent();
cropdata = cip.getCropData();
msg = ("Retrieving data for crop %s, %s took %6.1f seconds" % (cropname, watersupply, time.time()-t1))
print msg;
except Exception as e:
print str(e);
# Everything seems to be set for the task now!
try:
# Make sure we can loop over the grid cells from UL down to LR corner
nvlp = extent;
x_range = arange(nvlp.getMinX() + 0.5*nvlp.dx, nvlp.getMaxX() + 0.5*nvlp.dy, nvlp.dx);
y_range = arange(nvlp.getMinY() + 0.5*nvlp.dy, nvlp.getMaxY() + 0.5*nvlp.dx, nvlp.dy);
if (nvlp.xcoords_sort != 'ASC'): x_range = reversed(x_range);
if (nvlp.ycoords_sort != 'DESC'): y_range = reversed(y_range);
# Find out whether maybe another process already calculated TSUMs for this crop
minlat, maxlon = get_resumption_point(sa_engine, crop_no);
# Loop but make sure not to waste time on pixels that are not interesting
for lat in y_range:
if (lat > minlat): continue;
eps = 0.0000001;
for lon in x_range:
if abs(lat - minlat) < eps and (lon - maxlon) < eps: continue;
# Check the landmask
if not cip.get_landmask(lon, lat): continue;
# Check the data on the growing season; start_day eq.to -99 means that area is
# usu. only little above crop_specific base temperature and end_day equal to -99
# means that hot periods need to be avoided -
start_day, end_day = cip.getSeasonDates(lon, lat);
if (start_day == -99) or (end_day == -99):
continue;
# Initialise
wdp = None
try:
# Retrieve the relevant weather data
wdp = Hdf5WeatherDataProvider(fn, lat, lon, datadir);
# Loop over the years
t2 = time.time()
tsums = [];
for year in get_available_years(wdp):
try:
# Get timer data for the current year
timerdata = cip.getTimerData(start_day, end_day, year);
sitedata = {};
soildata = {"SMFCF":0.4};
# Run simulation
pheno = wofostEngine(sitedata, timerdata, soildata, cropdata, wdp, config="Wofost71_PhenoOnly.conf")
pheno.run(days=366)
# Retrieve result
results = pheno.get_output()
#if (results != None) and isinstance(results, list):
# print results[-1];
sumresults = pheno.get_summary_output();
print sumresults;
if isinstance(sumresults[0], dict) and isinstance(sumresults[0]["TSUM"], float):
tsums.append(sumresults[0]["TSUM"]);
except Exception, e:
print str(e);
# end try
# end year
# Insert average etc. into database
if len(tsums) > 0:
insert_tsum(sa_engine, 1, crop_no, lat, lon, mean(tsums), std(tsums, ddof=1), min(tsums), max(tsums), len(tsums));
msg = ("Simulating for lat-lon (%s, %s) took %6.1f seconds" % (str(lat), str(lon), time.time()-t2))
print msg;
except Exception, e:
print str(e);
def task_runner(sa_engine, task):
# Get crop_name and mgmt_code
crop_no = task["crop_no"]
cropname, watersupply = select_crop(sa_engine, crop_no)
msg = "Starting to simulate for %s (%s)" % (cropname, watersupply)
print msg
# Get a crop info provider
t1 = time.time()
cip = CropInfoProvider(cropname, watersupply, run_settings.datadir)
extent = cip.getExtent()
cropdata = cip.getCropData()
msg = ("Retrieving data for crop %s, %s took %6.1f seconds" % (cropname, watersupply, time.time()-t1))
print msg
# Everything seems to be set for the task now!
# Make sure we can loop over the grid cells from UL down to LR corner
nvlp = extent
x_range = arange(nvlp.getMinX() + 0.5*nvlp.dx, nvlp.getMaxX() + 0.5*nvlp.dy, nvlp.dx)
y_range = arange(nvlp.getMinY() + 0.5*nvlp.dy, nvlp.getMaxY() + 0.5*nvlp.dx, nvlp.dy)
if nvlp.xcoords_sort != 'ASC':
x_range = reversed(x_range)
if nvlp.ycoords_sort != 'DESC':
y_range = reversed(y_range)
# Find out whether maybe another process already calculated TSUMs for this crop
minlat, maxlon = get_resumption_point(sa_engine, crop_no)
# Loop but make sure not to waste time on pixels that are not interesting
for lat in y_range:
if (lat > minlat):
continue
eps = 0.0000001
for lon in x_range:
try:
if abs(lat - minlat) < eps and (lon - maxlon) < eps:
continue
# Check the landmask
if not cip.get_landmask(lon, lat):
continue
# Check the data on the growing season represented by day-of-year
# values. If start_doy or end_doy return nodata value (-99), then
# continue with next grid directly.
start_doy, end_doy = cip.getSeasonDates(lon, lat)
if (start_doy == -99) or (end_doy == -99):
continue
wdp = Hdf5WeatherDataProvider(run_settings.hdf5_meteo_file, lat, lon)
# Loop over the years
t2 = time.time()
tsums = []
for year in get_available_years(wdp):
# Get timer data for the current year
timerdata = cip.getTimerData(start_doy, end_doy, year)
# Check that START_DATE does not fall before the first
# available meteodata and END_DATE not beyond the
# last data with available weather data.
if timerdata['START_DATE'] < wdp.first_date or \
timerdata['END_DATE'] > wdp.last_date:
continue
sitedata = {}
soildata = {"SMFCF":0.4}
# Run simulation
pheno = wofostEngine(sitedata, timerdata, soildata, cropdata,
wdp, config="Wofost71_PhenoOnly.conf")
pheno.run(days=366)
sumresults = pheno.get_summary_output()
if len(sumresults) > 0:
tsums.append(sumresults[0]["TSUM"])
else:
msg = "No summary results for crop/year/lat/lon: %s/%s/%s/%s"
logging.error(msg, crop_no, year, lat, lon)
# end year
# Insert average etc. into database
if len(tsums) > 0:
insert_tsum(sa_engine, 1, crop_no, lat, lon, mean(tsums), std(tsums, ddof=1),
min(tsums), max(tsums), len(tsums))
msg = ("Simulating for lat-lon (%s, %s) took %6.1f seconds" % (str(lat), str(lon), time.time()-t2))
print msg
else:
msg = "Failed calculating TSUMs for lat-lon (%s, %s): no TSUMs calculated for 30 years"
logging.error(msg, lat, lon)
if wdp is not None:
wdp.close()
except tables.NoSuchNodeError:
msg = "No weather data found for lat/lon: %s/%s"
logging.error(msg, lat, lon)
except PCSEError:
msg = "Error in PCSE for crop/year/lat/lon: %s/%s/%s/%s"
logging.exception(msg, crop_no, year, lat, lon)
print "Finished simulating for " + cropname + " (" + watersupply + ")"
cip.close()