def main():
# Initialise
recs = []
db_engine = None;
try:
# Initialise further
db_engine = sa_engine.create_engine(run_settings.connstr)
db_metadata = MetaData(db_engine)
# Now loop over the crops
for crop_no in range(1, 29):
# Report which crop is next
crop_name, mgmt_code = select_crop(db_engine, crop_no)
msg = "About to get TSUM values for " + crop_name + " (" + mgmt_code + ")"
print msg
# Now retrieve how to divide over TSUM1 and TSUM2
cip = CropInfoProvider(crop_name, mgmt_code, run_settings.data_dir)
cropdata = cip.getCropData();
fract1, fract2 = split_tsum(cropdata, 1)
rec = {}
rec["crop_no"] = crop_no
rec["fract_tsum1"] = fract1
rec["fract_tsum2"] = fract2
recs.append(rec)
cropinfodict = {"crop_no":1, "fract_tsum1":0.4 , "fract_tsum2":0.6}
store_to_database(db_engine, recs, db_metadata, cropinfodict)
except Exception as e:
print str(e)
finally:
if db_engine != None: db_engine.dispose()
def main():
# Initialise
recs = []
db_engine = None
try:
# Initialise further
db_engine = sa_engine.create_engine(run_settings.connstr)
db_metadata = MetaData(db_engine)
# Now loop over the crops
for crop_no in range(1, 29):
# Report which crop is next
crop_name, mgmt_code = select_crop(db_engine, crop_no)
msg = "About to get TSUM values for " + crop_name + " (" + mgmt_code + ")"
print msg
# Now retrieve how to divide over TSUM1 and TSUM2
cip = CropInfoProvider(crop_name, mgmt_code, run_settings.data_dir)
cropdata = cip.getCropData()
fract1, fract2 = split_tsum(cropdata, 1)
rec = {}
rec["crop_no"] = crop_no
rec["fract_tsum1"] = fract1
rec["fract_tsum2"] = fract2
recs.append(rec)
cropinfodict = {"crop_no": 1, "fract_tsum1": 0.4, "fract_tsum2": 0.6}
store_to_database(db_engine, recs, db_metadata, cropinfodict)
except Exception as e:
print str(e)
finally:
if db_engine != None: db_engine.dispose()
def main():
# Initialise
db_engine = None;
tasks = []
try:
db_engine = sa_engine.create_engine(run_settings.connstr)
db_metadata = MetaData(db_engine)
task_id = 15051
for crop_no in range(19, 29):
# Report which crop is next
crop_name, mgmt_code = select_crop(db_engine, crop_no)
msg = "About to get TSUM values for " + crop_name + " (" + mgmt_code + ")"
logging.info(msg)
print msg
# Now retrieve how to divide over TSUM1 and TSUM2
cip = CropInfoProvider(crop_name, mgmt_code, run_settings.data_dir)
cropdata = cip.getCropData();
nvlp = cip.getExtent()
# Get the relevant spatial range and loop through
y_range = get_range(db_engine, crop_no, nvlp.dy)
x_range = arange(nvlp.getMinX() + 0.5*nvlp.dx, nvlp.getMaxX() + 0.5*nvlp.dy, nvlp.dx);
taskdict = {"task_id":1, "status":"Pending" , "hostname":"None", "crop_no":1, "longitude":0.0, "latitude": 0.0, "tsum1":0.0, "tsum2":0.0, "process_id":1, "comment":""}
for lat in y_range:
# Retrieve the lat-lon combinations for which there are records in the table TSUM
lons_with_tsums = get_places_with_tsums(db_engine, crop_no, lat)
for lon in x_range:
# Pixel should be on land, without tsum so far and there should be a crop calendar for it
if not cip.get_landmask(lon, lat):
continue;
if lon in lons_with_tsums:
continue;
start_doy, end_doy = cip.getSeasonDates(lon, lat);
if (start_doy == -99) or (end_doy == -99):
continue;
# tsum = get_tsum_from_db(db_engine, crop_no, lon, lat)
tsum1, tsum2 = (0, 0) # split_tsum(cropdata, tsum)
# Create a task and append it
task = Task(task_id, "Pending", "None", crop_no, lon, lat, tsum1, tsum2, 0, "")
tasks.append(task.as_dict())
task_id = task_id + 1
# Once in a while, write them to the database
if (task_id % 1000 == 0):
store_to_database(db_engine, tasks, db_metadata, taskdict)
del tasks[:]
print "Thousand records saved to database. Current latitude is %s" % lat
# end lon
# end lat
store_to_database(db_engine, tasks, db_metadata, taskdict)
print "Another %s records saved to database." % len(tasks)
del tasks[:]
except SQLAlchemyError, inst:
print ("Database error: %s" % inst)
def __init__(self, crop_no, model, climate, clim_scenario, sim_scenario, start_year, end_year):
# Initialise
self._crop_no = crop_no
self._model = model
self._climate = climate
self._clim_scenario = clim_scenario
self._sim_scenario = sim_scenario
self._start_year = start_year
self._end_year = end_year
try:
self._db_engine = sa_engine.create_engine(run_settings.connstr)
# Retrieve some info about the crop
self._crop_name, self._crop_label, self._mgmt_code = self._get_crop_info(crop_no)
self._crop_info_provider = CropInfoProvider(self._crop_name, self._mgmt_code)
except Exception as e:
print " Error during initialisation of CropSimOutputWorker instance: \n" + str(e)
raise e
def __init__(self, crop_no, model, climate, clim_scenario, sim_scenario, start_year, end_year):
# Initialise
self._crop_no = crop_no
self._model = model
self._climate = climate
self._clim_scenario = clim_scenario
self._sim_scenario = sim_scenario
self._start_year = start_year
self._end_year = end_year
try:
self._db_engine = sa_engine.create_engine(conv_settings.connstr)
# Retrieve some info about the crop
self._crop_name, self._crop_label, self._mgmt_code = self._get_crop_info(crop_no)
self._crop_info_provider = CropInfoProvider(self._crop_name, self._mgmt_code)
except Exception as e:
print " Error during initialisation of CropSimOutputWorker instance: \n" + str(e)
raise e
def task_runner(sa_engine, task):
# Get crop_name and mgmt_code
crop_no = task["crop_no"]
lat = float(task["latitude"])
lon = float(task["longitude"])
year = 1900
cip = None
wdp = None
# Get a crop info provider
try:
t1 = time.time()
cropname, watersupply = select_crop(sa_engine, crop_no)
cip = CropInfoProvider(cropname, watersupply, run_settings.data_dir)
cropdata = cip.getCropData()
msg = "Retrieving data for crop %s, %s took %6.1f seconds"
print msg % (cropname, watersupply, time.time() - t1)
# Everything seems to be set for the task now! Lon-lat combination is
# checked in the call to the getSeasonDates method. We assume that the
# landmask was checked. Also we assume that the data on the growing
# season were checked. These are represented by day-of-year values.
start_doy, end_doy = cip.getSeasonDates(lon, lat)
# Get the weather data
wdp = Hdf5WeatherDataProvider(run_settings.hdf5_meteo_file, lat, lon)
# Loop over the years
t2 = time.time()
tsums = []
for year in get_available_years(wdp):
# Get timer data for the current year
timerdata = cip.getTimerData(start_doy, end_doy, year)
# Check that START_DATE does not fall before the first available
# meteodata and END_DATE not beyond the last data with available
# weather data.
if timerdata['START_DATE'] < wdp.first_date: continue
if timerdata['END_DATE'] > wdp.last_date: continue
sitedata = {}
soildata = {"SMFCF": 0.4}
# Run simulation
pheno = wofostEngine(sitedata,
timerdata,
soildata,
cropdata,
wdp,
config="Wofost71_PhenoOnly.conf")
pheno.run(days=366)
sumresults = pheno.get_summary_output()
if len(sumresults) > 0:
tsums.append(sumresults[0]["TSUM"])
else:
msg = "No summary results for crop/year/lat/lon: %s/%s/%s/%s"
logging.error(msg, crop_no, year, lat, lon)
# end year
# Insert average etc. into database
if len(tsums) > 0:
insert_tsum(sa_engine, 1, crop_no, lat, lon, mean(tsums),
std(tsums, ddof=1), min(tsums), max(tsums), len(tsums))
msg = "Simulating for lat-lon (%s, %s) took %6.1f seconds"
print msg % (str(lat), str(lon), time.time() - t2)
else:
msg = "Failed calculating TSUMs for lat-lon (%s, %s): no TSUMs calculated for 30 years"
logging.error(msg, lat, lon)
finally:
# Clean up
if wdp is not None:
wdp.close()
if cip is not None:
cip.close()
def task_runner(sa_engine, task):
# Get crop_name and mgmt_code
crop_no = task["crop_no"]
lat = float(task["latitude"])
lon = float(task["longitude"])
year = 1900
cip = None
wdp = None
logger = logging.getLogger("GGCMI Task Runner")
logger.info("Starting task runner for task %i" % task["task_id"])
# Get a crop info provider
try:
t1 = time.time()
cropname, watersupply = select_crop(sa_engine, crop_no)
cip = CropInfoProvider(
cropname,
watersupply,
)
cropdata = cip.getCropData()
if (not task.has_key("tsum1")) or (not task.has_key("tsum2")):
msg = "Location specific values for crop parameter(s) missing: TSUM1 and/or TSUM2"
raise PCSEError(msg)
else:
cropdata["TSUM1"] = float(task["tsum1"])
cropdata["TSUM2"] = float(task["tsum2"])
start_doy, end_doy = cip.getSeasonDates(lon, lat)
msg = "Retrieving data for crop %s, %s took %6.1f seconds"
logger.info(msg % (cropname, watersupply, time.time() - t1))
# Everything seems to be set for the task now! Lon-lat combination is
# checked in the call to the getSeasonDates method. We assume that the
# landmask was checked. Also we assume that the data on the growing
# season were checked. These are represented by day-of-year values.
# Get the weather data
t2 = time.time()
wdp = Hdf5WeatherDataProvider(run_settings.hdf5_meteo_file, lat, lon)
available_years = get_available_years(wdp)
msg = "Retrieving weather data for lat-lon %s, %s took %6.1f seconds"
logger.debug(msg % (str(lat), str(lon), time.time() - t2))
# Loop over the years
t3 = time.time()
allresults = []
msg = None
for year in available_years:
# Get timer data for the current year
timerdata = cip.getTimerData(start_doy, end_doy, year)
# Check that START_DATE does not fall before the first available
# meteodata and END_DATE not beyond the last data with available
# weather data.
if timerdata['START_DATE'] < wdp.first_date:
continue
if timerdata['END_DATE'] > wdp.last_date:
continue
# Get soil and site data
soildata = run_settings.get_soil_data(lon, lat)
sitedata = run_settings.get_site_data(soildata)
# Run simulation
if watersupply == 'ir':
configFile = 'GGCMI_PP.conf'
else:
configFile = 'GGCMI_WLP.conf'
if msg is None:
msg = "Starting simulation for %s-%s (%5.2f, %5.2f), planting " \
"at: %s, final harvest at: %s"
msg = msg % (cropname, watersupply, lon, lat,
timerdata['CROP_START_DATE'],
timerdata['CROP_END_DATE'])
logger.info(msg)
wofost = wofostEngine(sitedata,
timerdata,
soildata,
cropdata,
wdp,
config=configFile)
wofost.run_till_terminate()
results = wofost.get_output()
sumresults = wofost.get_summary_output()
if (len(results) > 0) and (len(sumresults) > 0):
allresults.append({
"year": year,
"summary": sumresults,
"results": results
})
else:
msg = "Insufficient results for crop/year/lat/lon: %s/%s/%s/%s"
logger.error(msg, crop_no, year, lat, lon)
# end year
if len(allresults) > 0:
# pickle all results
task_id = task["task_id"]
obj = {
"task_id": task_id,
"crop_no": crop_no,
"longitude": lon,
"latitude": lat
}
obj["allresults"] = allresults
# Write results to pickle file. First to .tmp then rename to .pkl
# to avoid read/write collisions with ggcmi_processsor
pickle_fname = run_settings.output_file_template % task_id
if os.path.splitext(pickle_fname)[1] == ".pkl":
pickle_fname += ".tmp"
else:
pickle_fname += ".pkl.tmp"
pickle_fname_fp = os.path.join(run_settings.output_folder,
pickle_fname)
with open(pickle_fname_fp, 'wb') as f:
cPickle.dump(obj, f, cPickle.HIGHEST_PROTOCOL)
final_fname_fp = os.path.splitext(pickle_fname_fp)[0]
os.rename(pickle_fname_fp, final_fname_fp)
msg = "Simulating for lat-lon (%s, %s) took %6.1f seconds"
logger.info(msg % (str(lat), str(lon), time.time() - t3))
else:
msg = "Failed simulating for lon,lat,crop, watersupply (%s, %s, " \
"%s, %s): no output available."
msg = msg % (lon, lat, cropname, watersupply)
raise PCSEError(msg)
finally:
if wdp is not None:
wdp.close()
if cip is not None:
cip.close()
def main():
# Initialise
db_engine = None
tasks = []
try:
db_engine = sa_engine.create_engine(run_settings.connstr)
db_metadata = MetaData(db_engine)
task_id = 15051
for crop_no in range(19, 29):
# Report which crop is next
crop_name, mgmt_code = select_crop(db_engine, crop_no)
msg = "About to get TSUM values for " + crop_name + " (" + mgmt_code + ")"
logging.info(msg)
print msg
# Now retrieve how to divide over TSUM1 and TSUM2
cip = CropInfoProvider(crop_name, mgmt_code, run_settings.data_dir)
cropdata = cip.getCropData()
nvlp = cip.getExtent()
# Get the relevant spatial range and loop through
y_range = get_range(db_engine, crop_no, nvlp.dy)
x_range = arange(nvlp.getMinX() + 0.5 * nvlp.dx,
nvlp.getMaxX() + 0.5 * nvlp.dy, nvlp.dx)
taskdict = {
"task_id": 1,
"status": "Pending",
"hostname": "None",
"crop_no": 1,
"longitude": 0.0,
"latitude": 0.0,
"tsum1": 0.0,
"tsum2": 0.0,
"process_id": 1,
"comment": ""
}
for lat in y_range:
# Retrieve the lat-lon combinations for which there are records in the table TSUM
lons_with_tsums = get_places_with_tsums(
db_engine, crop_no, lat)
for lon in x_range:
# Pixel should be on land, without tsum so far and there should be a crop calendar for it
if not cip.get_landmask(lon, lat):
continue
if lon in lons_with_tsums:
continue
start_doy, end_doy = cip.getSeasonDates(lon, lat)
if (start_doy == -99) or (end_doy == -99):
continue
# tsum = get_tsum_from_db(db_engine, crop_no, lon, lat)
tsum1, tsum2 = (0, 0) # split_tsum(cropdata, tsum)
# Create a task and append it
task = Task(task_id, "Pending", "None", crop_no, lon, lat,
tsum1, tsum2, 0, "")
tasks.append(task.as_dict())
task_id = task_id + 1
# Once in a while, write them to the database
if (task_id % 1000 == 0):
store_to_database(db_engine, tasks, db_metadata,
taskdict)
del tasks[:]
print "Thousand records saved to database. Current latitude is %s" % lat
# end lon
# end lat
store_to_database(db_engine, tasks, db_metadata, taskdict)
print "Another %s records saved to database." % len(tasks)
del tasks[:]
except SQLAlchemyError, inst:
print("Database error: %s" % inst)
class CropSimOutputWorker():
_crop_no = 0
_crop_name = ""
_crop_label = ""
_mgmt_code = ""
_model = ""
_climate = ""
_clim_scenario = ""
_sim_scenario = ""
_timestep = "annual"
_start_year = 1945
_end_year = 2045
_db_engine = None
_crop_info_provider = None
_joint_shelves = None
def __init__(self, crop_no, model, climate, clim_scenario, sim_scenario, start_year, end_year):
# Initialise
self._crop_no = crop_no
self._model = model
self._climate = climate
self._clim_scenario = clim_scenario
self._sim_scenario = sim_scenario
self._start_year = start_year
self._end_year = end_year
try:
self._db_engine = sa_engine.create_engine(conv_settings.connstr)
# Retrieve some info about the crop
self._crop_name, self._crop_label, self._mgmt_code = self._get_crop_info(crop_no)
self._crop_info_provider = CropInfoProvider(self._crop_name, self._mgmt_code)
except Exception as e:
print " Error during initialisation of CropSimOutputWorker instance: \n" + str(e)
raise e
def set_joint_shelves(self, obj):
if isinstance(obj, JointShelves):
self._joint_shelves = obj;
def _get_path_to_template(self):
template_fn = "output_template_{clim_lc}_{timestep}_{start_year}_{end_year}.nc4"
template_fn = template_fn.format(clim_lc=self._climate.lower(), timestep=self._timestep,
start_year=self._start_year, end_year=self._end_year)
return os.path.join(conv_settings.results_folder, template_fn)
def _get_output_filename_pattern(self):
ncdfname_templ = "{model}_{climate}_{clim_scenario}_{sim_scenario}_{variable}_{crop}_{timestep}_{start_year}_{end_year}.nc4"
if self._mgmt_code == 'rf': simcode = self._sim_scenario + "_noirr"
else: simcode = self._sim_scenario + "_firr"
result = ncdfname_templ.format(model=self._model, climate=self._climate.lower(), clim_scenario=self._clim_scenario,
sim_scenario=simcode, variable="*", crop=self._crop_label,
timestep=self._timestep, start_year=self._start_year, end_year=self._end_year)
return result
def _get_length_of_season(self, doh, dom, dos, task_id):
if doh or dom:
# doh or dom is specified - assume date objects
f = lambda x, y, z: (x-z).days if x else (y-z).days
return f(doh, dom, dos)
else:
crop_no, lon, lat = self._get_task_info(task_id)
if crop_no != self._crop_no:
raise ValueError("The crop number found for task %s is not as expected" % task_id)
start_doy, end_doy = self._crop_info_provider.getSeasonDates(float(lon), float(lat))
timerdata = self._crop_info_provider.getTimerData(start_doy, end_doy, dos.year)
result = (timerdata['CROP_END_DATE'] - dos).days
if result < 0: result = result + 365
return result
def close(self):
self._db_engine = None
self._joint_shelves.close()
self._joint_shelves = None
del self._joint_shelves
def _get_crop_info(self, crop_no):
crop_name = ""
crop_label = ""
mgmt_code = ""
conn = self._db_engine.connect()
sqlStr = """SELECT m.crop_no, x.name, x.label, m.mgmt_code
FROM crop m INNER JOIN cropinfo x ON m.crop_no = x.crop_no
WHERE m.crop_no=%s"""
rows = conn.execute(sqlStr % crop_no)
for row in rows:
crop_name = row["name"]
crop_label = row["label"]
mgmt_code = row["mgmt_code"]
break
conn.close()
return crop_name, crop_label, mgmt_code
def _get_task_info(self, taskId):
crop_no = 0
lon = 180.0
lat = 90.0
conn = self._db_engine.connect()
sqlStr = """select t.crop_no, t.longitude, t.latitude
from tasklist t where t.task_id =""" + str(taskId)
rows = conn.execute(sqlStr)
for row in rows:
crop_no = row["crop_no"]
lon = row["longitude"]
lat = row["latitude"]
break
conn.close()
return crop_no, lon, lat
def _get_finished_tasks(self, crop_no):
result = []
conn = self._db_engine.connect()
sqlStr = """SELECT task_id FROM tasklist
WHERE crop_no=%s AND status='Finished'"""
rows = conn.execute(sqlStr % crop_no)
conn.close()
for row in rows:
result.append(row["task_id"])
return result
def _get_simresult(self, task_id):
try:
key_fmt = "%010i"
return self._joint_shelves[key_fmt % task_id]
except KeyError:
msg = "No results found for task_id %s " % task_id
print msg
logging.warn(msg)
return None
class CropSimOutputWorker():
_crop_no = 0
_crop_name = ""
_crop_label = ""
_mgmt_code = ""
_model = ""
_climate = ""
_clim_scenario = ""
_sim_scenario = ""
_timestep = "annual"
_start_year = 1945
_end_year = 2045
_db_engine = None
_crop_info_provider = None
_joint_shelves = None
def __init__(self, crop_no, model, climate, clim_scenario, sim_scenario, start_year, end_year):
# Initialise
self._crop_no = crop_no
self._model = model
self._climate = climate
self._clim_scenario = clim_scenario
self._sim_scenario = sim_scenario
self._start_year = start_year
self._end_year = end_year
try:
self._db_engine = sa_engine.create_engine(run_settings.connstr)
# Retrieve some info about the crop
self._crop_name, self._crop_label, self._mgmt_code = self._get_crop_info(crop_no)
self._crop_info_provider = CropInfoProvider(self._crop_name, self._mgmt_code)
except Exception as e:
print " Error during initialisation of CropSimOutputWorker instance: \n" + str(e)
raise e
def set_joint_shelves(self, obj):
if isinstance(obj, JointShelves):
self._joint_shelves = obj;
def _get_path_to_template(self):
template_fn = "output_template_{clim_lc}_{timestep}_{start_year}_{end_year}.nc4"
template_fn = template_fn.format(clim_lc=self._climate.lower(), timestep=self._timestep,
start_year=self._start_year, end_year=self._end_year)
return os.path.join(run_settings.results_folder, template_fn)
def _get_output_filename_pattern(self):
ncdfname_templ = "{model}_{climate}_{clim_scenario}_{sim_scenario}_{variable}_{crop}_{timestep}_{start_year}_{end_year}.nc4"
if self._mgmt_code == 'rf': simcode = self._sim_scenario + "_noirr"
else: simcode = self._sim_scenario + "_firr"
result = ncdfname_templ.format(model=self._model, climate=self._climate.lower(), clim_scenario=self._clim_scenario,
sim_scenario=simcode, variable="*", crop=self._crop_label,
timestep=self._timestep, start_year=self._start_year, end_year=self._end_year)
return result
def _get_length_of_season(self, doh, dom, dos, task_id):
if doh or dom:
# doh or dom is specified - assume date objects
f = lambda x, y, z: (x-z).days if x else (y-z).days
return f(doh, dom, dos)
else:
crop_no, lon, lat = self._get_task_info(task_id)
if crop_no != self._crop_no:
raise ValueError("The crop number found for task %s is not as expected" % task_id)
start_doy, end_doy = self._crop_info_provider.getSeasonDates(float(lon), float(lat))
timerdata = self._crop_info_provider.getTimerData(start_doy, end_doy, dos.year)
result = (timerdata['CROP_END_DATE'] - dos).days
if result < 0: result = result + 365
return result
def close(self):
self._db_engine = None
self._joint_shelves = None
def _get_crop_info(self, crop_no):
crop_name = ""
crop_label = ""
mgmt_code = ""
conn = self._db_engine.connect()
sqlStr = """SELECT m.crop_no, x.name, x.label, m.mgmt_code
FROM crop m INNER JOIN cropinfo x ON m.crop_no = x.crop_no
WHERE m.crop_no=%s"""
rows = conn.execute(sqlStr % crop_no)
for row in rows:
crop_name = row["name"]
crop_label = row["label"]
mgmt_code = row["mgmt_code"]
break
conn.close()
return crop_name, crop_label, mgmt_code
def _get_task_info(self, taskId):
crop_no = 0
lon = 180.0
lat = 90.0
conn = self._db_engine.connect()
sqlStr = """select t.crop_no, t.longitude, t.latitude
from tasklist t where t.task_id =""" + str(taskId)
rows = conn.execute(sqlStr)
for row in rows:
crop_no = row["crop_no"]
lon = row["longitude"]
lat = row["latitude"]
break
conn.close()
return crop_no, lon, lat
def _get_finished_tasks(self, crop_no):
result = []
conn = self._db_engine.connect()
sqlStr = """SELECT task_id FROM tasklist
WHERE crop_no=%s AND status='Finished'"""
rows = conn.execute(sqlStr % crop_no)
conn.close()
for row in rows:
result.append(row["task_id"])
return result
def _get_simresult(self, task_id):
try:
key_fmt = "%010i"
return self._joint_shelves[key_fmt % task_id]
except KeyError:
msg = "No results found for task_id %s " % task_id
print msg
logging.warn(msg)
return None
def task_runner(sa_engine, task):
# Attributes
cip = None;
extent = None;
cropdata = None;
crop_no = 0;
# Initialise
datadir = r"/home/hoek008/projects/ggcmi/data/"
fn = r"./AgMERRA/AgMERRA_1980-01-01_2010-12-31_final.hf5"
# Get crop_name and mgmt_code
crop_no = task["crop_no"];
cropname, watersupply = select_crop(sa_engine, crop_no);
print "About to simulate for " + cropname + " (" + watersupply + ")";
try:
# Get a crop info provider
t1 = time.time()
cip = CropInfoProvider(cropname, watersupply, datadir);
extent = cip.getExtent();
cropdata = cip.getCropData();
msg = ("Retrieving data for crop %s, %s took %6.1f seconds" % (cropname, watersupply, time.time()-t1))
print msg;
except Exception as e:
print str(e);
# Everything seems to be set for the task now!
try:
# Make sure we can loop over the grid cells from UL down to LR corner
nvlp = extent;
x_range = arange(nvlp.getMinX() + 0.5*nvlp.dx, nvlp.getMaxX() + 0.5*nvlp.dy, nvlp.dx);
y_range = arange(nvlp.getMinY() + 0.5*nvlp.dy, nvlp.getMaxY() + 0.5*nvlp.dx, nvlp.dy);
if (nvlp.xcoords_sort != 'ASC'): x_range = reversed(x_range);
if (nvlp.ycoords_sort != 'DESC'): y_range = reversed(y_range);
# Find out whether maybe another process already calculated TSUMs for this crop
minlat, maxlon = get_resumption_point(sa_engine, crop_no);
# Loop but make sure not to waste time on pixels that are not interesting
for lat in y_range:
if (lat > minlat): continue;
eps = 0.0000001;
for lon in x_range:
if abs(lat - minlat) < eps and (lon - maxlon) < eps: continue;
# Check the landmask
if not cip.get_landmask(lon, lat): continue;
# Check the data on the growing season; start_day eq.to -99 means that area is
# usu. only little above crop_specific base temperature and end_day equal to -99
# means that hot periods need to be avoided -
start_day, end_day = cip.getSeasonDates(lon, lat);
if (start_day == -99) or (end_day == -99):
continue;
# Initialise
wdp = None
try:
# Retrieve the relevant weather data
wdp = Hdf5WeatherDataProvider(fn, lat, lon, datadir);
# Loop over the years
t2 = time.time()
tsums = [];
for year in get_available_years(wdp):
try:
# Get timer data for the current year
timerdata = cip.getTimerData(start_day, end_day, year);
sitedata = {};
soildata = {"SMFCF":0.4};
# Run simulation
pheno = wofostEngine(sitedata, timerdata, soildata, cropdata, wdp, config="Wofost71_PhenoOnly.conf")
pheno.run(days=366)
# Retrieve result
results = pheno.get_output()
#if (results != None) and isinstance(results, list):
# print results[-1];
sumresults = pheno.get_summary_output();
print sumresults;
if isinstance(sumresults[0], dict) and isinstance(sumresults[0]["TSUM"], float):
tsums.append(sumresults[0]["TSUM"]);
except Exception, e:
print str(e);
# end try
# end year
# Insert average etc. into database
if len(tsums) > 0:
insert_tsum(sa_engine, 1, crop_no, lat, lon, mean(tsums), std(tsums, ddof=1), min(tsums), max(tsums), len(tsums));
msg = ("Simulating for lat-lon (%s, %s) took %6.1f seconds" % (str(lat), str(lon), time.time()-t2))
print msg;
except Exception, e:
print str(e);
def task_runner(sa_engine, task):
# Get crop_name and mgmt_code
crop_no = task["crop_no"]
lat = float(task["latitude"])
lon = float(task["longitude"])
year = 1900
cip = None
wdp = None
# Get a crop info provider
try:
t1 = time.time()
cropname, watersupply = select_crop(sa_engine, crop_no)
cip = CropInfoProvider(cropname, watersupply, run_settings.data_dir)
cropdata = cip.getCropData()
msg = "Retrieving data for crop %s, %s took %6.1f seconds"
print msg % (cropname, watersupply, time.time()-t1)
# Everything seems to be set for the task now! Lon-lat combination is
# checked in the call to the getSeasonDates method. We assume that the
# landmask was checked. Also we assume that the data on the growing
# season were checked. These are represented by day-of-year values.
start_doy, end_doy = cip.getSeasonDates(lon, lat)
# Get the weather data
wdp = Hdf5WeatherDataProvider(run_settings.hdf5_meteo_file, lat, lon)
# Loop over the years
t2 = time.time()
tsums = []
for year in get_available_years(wdp):
# Get timer data for the current year
timerdata = cip.getTimerData(start_doy, end_doy, year)
# Check that START_DATE does not fall before the first available
# meteodata and END_DATE not beyond the last data with available
# weather data.
if timerdata['START_DATE'] < wdp.first_date: continue
if timerdata['END_DATE'] > wdp.last_date: continue
sitedata = {}
soildata = {"SMFCF":0.4}
# Run simulation
pheno = wofostEngine(sitedata, timerdata, soildata, cropdata, wdp,
config="Wofost71_PhenoOnly.conf")
pheno.run(days=366)
sumresults = pheno.get_summary_output()
if len(sumresults) > 0:
tsums.append(sumresults[0]["TSUM"])
else:
msg = "No summary results for crop/year/lat/lon: %s/%s/%s/%s"
logging.error(msg, crop_no, year, lat, lon)
# end year
# Insert average etc. into database
if len(tsums) > 0:
insert_tsum(sa_engine, 1, crop_no, lat, lon, mean(tsums),
std(tsums, ddof=1), min(tsums), max(tsums), len(tsums))
msg = "Simulating for lat-lon (%s, %s) took %6.1f seconds"
print msg % (str(lat), str(lon), time.time()-t2)
else:
msg = "Failed calculating TSUMs for lat-lon (%s, %s): no TSUMs calculated for 30 years"
logging.error(msg, lat, lon)
finally:
# Clean up
if wdp is not None:
wdp.close()
if cip is not None:
cip.close()
def task_runner(sa_engine, task):
# Get crop_name and mgmt_code
crop_no = task["crop_no"]
cropname, watersupply = select_crop(sa_engine, crop_no)
msg = "Starting to simulate for %s (%s)" % (cropname, watersupply)
print msg
# Get a crop info provider
t1 = time.time()
cip = CropInfoProvider(cropname, watersupply, run_settings.datadir)
extent = cip.getExtent()
cropdata = cip.getCropData()
msg = ("Retrieving data for crop %s, %s took %6.1f seconds" % (cropname, watersupply, time.time()-t1))
print msg
# Everything seems to be set for the task now!
# Make sure we can loop over the grid cells from UL down to LR corner
nvlp = extent
x_range = arange(nvlp.getMinX() + 0.5*nvlp.dx, nvlp.getMaxX() + 0.5*nvlp.dy, nvlp.dx)
y_range = arange(nvlp.getMinY() + 0.5*nvlp.dy, nvlp.getMaxY() + 0.5*nvlp.dx, nvlp.dy)
if nvlp.xcoords_sort != 'ASC':
x_range = reversed(x_range)
if nvlp.ycoords_sort != 'DESC':
y_range = reversed(y_range)
# Find out whether maybe another process already calculated TSUMs for this crop
minlat, maxlon = get_resumption_point(sa_engine, crop_no)
# Loop but make sure not to waste time on pixels that are not interesting
for lat in y_range:
if (lat > minlat):
continue
eps = 0.0000001
for lon in x_range:
try:
if abs(lat - minlat) < eps and (lon - maxlon) < eps:
continue
# Check the landmask
if not cip.get_landmask(lon, lat):
continue
# Check the data on the growing season represented by day-of-year
# values. If start_doy or end_doy return nodata value (-99), then
# continue with next grid directly.
start_doy, end_doy = cip.getSeasonDates(lon, lat)
if (start_doy == -99) or (end_doy == -99):
continue
wdp = Hdf5WeatherDataProvider(run_settings.hdf5_meteo_file, lat, lon)
# Loop over the years
t2 = time.time()
tsums = []
for year in get_available_years(wdp):
# Get timer data for the current year
timerdata = cip.getTimerData(start_doy, end_doy, year)
# Check that START_DATE does not fall before the first
# available meteodata and END_DATE not beyond the
# last data with available weather data.
if timerdata['START_DATE'] < wdp.first_date or \
timerdata['END_DATE'] > wdp.last_date:
continue
sitedata = {}
soildata = {"SMFCF":0.4}
# Run simulation
pheno = wofostEngine(sitedata, timerdata, soildata, cropdata,
wdp, config="Wofost71_PhenoOnly.conf")
pheno.run(days=366)
sumresults = pheno.get_summary_output()
if len(sumresults) > 0:
tsums.append(sumresults[0]["TSUM"])
else:
msg = "No summary results for crop/year/lat/lon: %s/%s/%s/%s"
logging.error(msg, crop_no, year, lat, lon)
# end year
# Insert average etc. into database
if len(tsums) > 0:
insert_tsum(sa_engine, 1, crop_no, lat, lon, mean(tsums), std(tsums, ddof=1),
min(tsums), max(tsums), len(tsums))
msg = ("Simulating for lat-lon (%s, %s) took %6.1f seconds" % (str(lat), str(lon), time.time()-t2))
print msg
else:
msg = "Failed calculating TSUMs for lat-lon (%s, %s): no TSUMs calculated for 30 years"
logging.error(msg, lat, lon)
if wdp is not None:
wdp.close()
except tables.NoSuchNodeError:
msg = "No weather data found for lat/lon: %s/%s"
logging.error(msg, lat, lon)
except PCSEError:
msg = "Error in PCSE for crop/year/lat/lon: %s/%s/%s/%s"
logging.exception(msg, crop_no, year, lat, lon)
print "Finished simulating for " + cropname + " (" + watersupply + ")"
cip.close()