def run(self, forecast, progress_monitor=None, verbose=False):
if not progress_monitor:
progress_monitor = NullMonitor()
logging.getLogger().info('Running pSIMS for forecast "%s" (%s).' %
(forecast.name, forecast.forecast_date))
progress_monitor.end_value = forecast.simulation_count
progress_monitor.job_started()
start_time = datetime.now()
ret_val = self.__run__(forecast.paths.run_script_path, forecast.name,
forecast.simulation_count, progress_monitor,
verbose)
end_time = datetime.now()
logging.getLogger().info(
'Finished running pSIMS for forecast "%s" (%s). Retval = %s. Time: %s.'
% (forecast.name, forecast.forecast_date, ret_val,
end_time - start_time))
if ret_val != 0:
progress_monitor.job_ended(end_status=JOB_STATUS_ERROR)
else:
progress_monitor.job_ended()
return ret_val
def __init__(self, name=None, id=None, progress_monitor=None):
if not name:
name = "Job created at %s" % datetime.now().isoformat()
if not progress_monitor:
progress_monitor = NullMonitor()
self.id = (id if id else uuid4().hex)
self.name = name
self.parent_job = None
self.progress_monitor = progress_monitor
self.progress_monitor.job = self
def __init__(self,
system_config,
parent_task_monitor=None,
max_events=10000):
super(RunImputation, self).__init__(
name='Impute missing data and calculate radiation',
progress_monitor=ProgressMonitor())
self.max_events = max_events
self.station = re.compile('Station: (\d+)')
self.system_config = system_config
if not parent_task_monitor:
parent_task_monitor = NullMonitor()
self.parent_monitor = parent_task_monitor
def reload(self, progress_monitor=None):
if not progress_monitor:
progress_monitor = NullMonitor()
progress_monitor.job_started()
progress_monitor.update_progress(job_status=JOB_STATUS_WAITING)
with self.jobs_lock.blocking_job(priority=LOAD_CONFIGURATION):
progress_monitor.update_progress(job_status=JOB_STATUS_RUNNING)
# Create a temporal backup of the current configuration.
config_clone = copy.copy(self)
try:
new_forecasts_files = SystemConfiguration.load(config_clone)
# If load succeeded, update this class with the new config.
self.update(config_clone)
SystemConfiguration.load_forecasts(
config_object=self, forecast_list=new_forecasts_files)
except Exception as ex:
raise RuntimeError(
'Configuration not updated. An exception was raised: %s' %
str(ex).strip())
def update_rainfall_quantiles(self, omm_ids=None, progress_monitor=None):
if not omm_ids:
omm_ids = self.weather_stations_ids
if isinstance(omm_ids, int):
omm_ids = [omm_ids]
try:
if not progress_monitor:
progress_monitor = NullMonitor()
progress_monitor.start_value = 0
progress_monitor.end_value = len(omm_ids)
for index, omm_id in enumerate(omm_ids):
wth_db = self.system_config.database['weather_db']
cursor = wth_db.cursor()
cursor.execute(
'SELECT campaign, sum FROM pr_campaigns_acum_rainfall(%s,%s)',
(omm_id, self.system_config.campaign_first_month))
np_prcp_sums = WeatherUpdater.parse_rainfalls(cursor)
# daily_cursor = wth_db.cursor()
# daily_cursor.execute('SELECT campaign, sum FROM pr_campaigns_rainfall(%s)', (omm_id,))
#
# np_prcp_values = WeatherUpdater.parse_rainfalls(daily_cursor)
# Update (or insert) weather quantiles.
db = self.system_config.database['yield_db']
db.reference_rainfall.update_one({"omm_id": omm_id}, {
"$set": {
"quantiles":
WeatherUpdater.get_quantiles(
np_prcp_sums, quantiles=[5, 25, 50, 75, 95])
}
},
upsert=True)
# Update progress information.
progress_monitor.update_progress(index)
logging.getLogger().info(
'Updated rainfall quantiles for stations %s.' % list(omm_ids))
except Exception:
logging.getLogger().error(
'Failed to update rainfall quantiles. Reason: %s.',
log_format_exception())
def update_max_dates(self, progress_monitor=None, run_blocking=True):
if len(self.weather_stations_ids) == 0:
return
if not progress_monitor:
progress_monitor = NullMonitor()
if run_blocking:
progress_monitor.update_progress(job_status=JOB_STATUS_WAITING)
# Acquire a blocking job lock with the update weather dates priority.
with self.system_config.jobs_lock.blocking_job(
priority=UPDATE_MAX_WEATHER_DATES):
# Lock acquired, notify observers.
progress_monitor.update_progress(job_status=JOB_STATUS_RUNNING)
self.__update_max_dates__(progress_monitor)
else:
self.__update_max_dates__(progress_monitor)
def refresh_view(self, progress_monitor=None):
if not progress_monitor:
progress_monitor = NullMonitor()
# DROP INDEX IF EXISTS erdi_index;
# REFRESH MATERIALIZED VIEW estacion_registro_diario_completo;
# CREATE INDEX erdi_index ON estacion_registro_diario_completo (omm_id, fecha);
progress_monitor.end_value = 3
progress_monitor.job_started()
cursor = self.system_config.database['weather_db'].cursor()
cursor.execute('DROP INDEX IF EXISTS erdi_index;')
progress_monitor.update_progress(1)
cursor.execute(
'REFRESH MATERIALIZED VIEW estacion_registro_diario_completo;')
progress_monitor.update_progress(2)
cursor.execute(
'CREATE INDEX erdi_index ON estacion_registro_diario_completo (omm_id, fecha);'
)
cursor.execute('COMMIT')
progress_monitor.job_ended()
def reload_file(self,
forecast_file,
scheduler,
forecast_manager,
progress_monitor=None):
if not progress_monitor:
progress_monitor = NullMonitor()
progress_monitor.job_started()
progress_monitor.start_value = 0
progress_monitor.end_value = 100
progress_monitor.update_progress(job_status=JOB_STATUS_WAITING)
with self.jobs_lock.blocking_job(priority=priority.LOAD_FORECAST):
progress_monitor.update_progress(job_status=JOB_STATUS_RUNNING)
file_forecasts = self.system_config.forecasts[forecast_file]
for f in file_forecasts:
if 'job_id' in f:
try:
scheduler.remove_job(f['job_id'])
except JobLookupError:
pass
progress_monitor.update_progress(new_value=50)
if self.__load_file__(forecast_file):
for f in self.system_config.forecasts[forecast_file]:
forecast_manager.schedule_forecast(f)
progress_monitor.update_progress(new_value=90)
def update_rainfall_quantiles(self, omm_ids=None, progress_monitor=None):
logging.getLogger().info('Running rainfall quantiles update.')
if not omm_ids:
omm_ids = self.weather_stations_ids
if isinstance(omm_ids, int):
omm_ids = [omm_ids]
try:
if not progress_monitor:
progress_monitor = NullMonitor()
progress_monitor.start_value = 0
progress_monitor.end_value = len(omm_ids)
progress_monitor.update_progress(job_status=JOB_STATUS_WAITING)
# Acquire a blocking job lock with the update weather dates priority.
with self.system_config.jobs_lock.blocking_job(
priority=UPDATE_RAINFALL_QUANTILES):
# Lock acquired, notify observers.
progress_monitor.update_progress(job_status=JOB_STATUS_RUNNING)
for index, omm_id in enumerate(omm_ids):
wth_db = self.system_config.database['weather_db']
cursor = wth_db.cursor()
cursor.execute(
'SELECT campaign, sum FROM pr_campaigns_acum_rainfall(%s,%s)',
(omm_id, self.system_config.campaign_first_month))
np_prcp_sums = WeatherUpdater.parse_rainfalls(cursor)
# daily_cursor = wth_db.cursor()
# daily_cursor.execute('SELECT campaign, sum FROM pr_campaigns_rainfall(%s)', (omm_id,))
#
# np_prcp_values = WeatherUpdater.parse_rainfalls(daily_cursor)
# Update (or insert) weather quantiles.
db = self.system_config.database['yield_db']
db.reference_rainfall.update_one({"omm_id": omm_id}, {
"$set": {
"quantiles":
WeatherUpdater.get_quantiles(
np_prcp_sums, quantiles=[5, 25, 50, 75, 95])
}
},
upsert=True)
# Update progress information.
progress_monitor.update_progress(index)
logging.getLogger().info(
'Updated rainfall quantiles for stations %s.' % list(omm_ids))
except InvalidRainfallValue as e:
logging.getLogger().error(
'Failed to update rainfall quantiles. Reason: %s.', e.message)
except Exception:
logging.getLogger().error(
'Failed to update rainfall quantiles. Reason: %s.',
log_format_exception())