def delete_old_watershed_geoserver_files(watershed):
"""
Removes old watershed geoserver files from system
"""
#initialize session
session = mainSessionMaker()
main_settings = session.query(MainSettings).order_by(MainSettings.id).first()
#initialize geoserver manager
geoserver_manager = GeoServerDatasetManager(engine_url=watershed.geoserver.url,
username=watershed.geoserver.username,
password=watershed.geoserver.password,
app_instance_id=main_settings.app_instance_id)
session.close()
#delete layers which need to be deleted
if watershed.geoserver_drainage_line_layer:
if watershed.geoserver_drainage_line_layer.uploaded:
geoserver_manager.purge_remove_geoserver_layer(watershed.geoserver_drainage_line_layer.name)
if watershed.geoserver_boundary_layer:
if watershed.geoserver_boundary_layer.uploaded:
geoserver_manager.purge_remove_geoserver_layer(watershed.geoserver_boundary_layer.name)
if watershed.geoserver_gage_layer:
if watershed.geoserver_gage_layer.uploaded:
geoserver_manager.purge_remove_geoserver_layer(watershed.geoserver_gage_layer.name)
if watershed.geoserver_ahps_station_layer:
if watershed.geoserver_ahps_station_layer.uploaded:
geoserver_manager.purge_remove_geoserver_layer(watershed.geoserver_ahps_station_layer.name)
def delete_geoserver_files(self):
"""
Removes old watershed geoserver files from system
"""
# initialize geoserver manager
app_instance_id = app.get_custom_setting('app_instance_id')
geoserver_manager = \
GeoServerDatasetManager(engine_url=self.geoserver.url,
username=self.geoserver.username,
password=self.geoserver.password,
app_instance_id=app_instance_id)
# delete layers which need to be deleted
if self.geoserver_drainage_line_layer:
if self.geoserver_drainage_line_layer.uploaded:
geoserver_manager.purge_remove_geoserver_layer(
self.geoserver_drainage_line_layer.name)
if self.geoserver_boundary_layer:
if self.geoserver_boundary_layer.uploaded:
geoserver_manager.purge_remove_geoserver_layer(
self.geoserver_boundary_layer.name)
if self.geoserver_gage_layer:
if self.geoserver_gage_layer.uploaded:
geoserver_manager.purge_remove_geoserver_layer(
self.geoserver_gage_layer.name)
if self.geoserver_ahps_station_layer:
if self.geoserver_ahps_station_layer.uploaded:
geoserver_manager.purge_remove_geoserver_layer(
self.geoserver_ahps_station_layer.name)
def download_single_watershed_ecmwf_data(watershed,
ecmwf_rapid_prediction_directory,
app_instance_id):
"""
Loads single watersheds ECMWF datasets from data store
"""
if ecmwf_rapid_prediction_directory \
and os.path.exists(ecmwf_rapid_prediction_directory) \
and watershed.ecmwf_data_store_watershed_name \
and watershed.ecmwf_data_store_subbasin_name:
#get data engine
data_store = watershed.data_store
if 'ckan' == data_store.data_store_type.code_name:
#get dataset managers
data_manager = ECMWFRAPIDDatasetManager(data_store.api_endpoint,
data_store.api_key)
#load current datasets
data_manager.download_recent_resource(watershed.ecmwf_data_store_watershed_name,
watershed.ecmwf_data_store_subbasin_name,
ecmwf_rapid_prediction_directory)
path_to_predicitons = os.path.join(ecmwf_rapid_prediction_directory,
"{0}-{1}".format(watershed.ecmwf_data_store_watershed_name,
watershed.ecmwf_data_store_subbasin_name)
)
if os.path.exists(path_to_predicitons):
prediction_directories = sorted(os.listdir(path_to_predicitons),
reverse=True)[14:]
#remove oldest datasets if more than 14 exist
try:
for prediction_directory in prediction_directories:
rmtree(os.path.join(path_to_predicitons, prediction_directory))
except OSError:
pass
if watershed.geoserver_id > 1 and app_instance_id and watershed.geoserver_search_for_flood_map:
try:
#get geoserver engine
geoserver_manager = GeoServerDatasetManager(engine_url=watershed.geoserver.url,
username=watershed.geoserver.username,
password=watershed.geoserver.password,
app_instance_id=app_instance_id)
#remove old geoserver layers
flood_map_layer_name_beginning = "%s-%s-floodmap-" % (watershed.ecmwf_data_store_watershed_name,
watershed.ecmwf_data_store_subbasin_name)
geoserver_directories = sorted([d for d in os.listdir(path_to_predicitons) \
if os.path.isdir(os.path.join(path_to_predicitons, d))],
reverse=True)[7:]
for geoserver_directory in geoserver_directories:
layer_name = geoserver_manager.get_layer_name("%s%s" % (flood_map_layer_name_beginning, geoserver_directory))
print("Deleting geoserver layer group: {0}".format(layer_name))
#TODO: CHECK IF EXISTS BEFORE REMOVING
geoserver_manager.purge_remove_geoserver_layer_group(layer_name)
except Exception as ex:
print(ex)
pass
def delete_old_watershed_geoserver_files(watershed):
"""
Removes old watershed geoserver files from system
"""
#initialize session
session = mainSessionMaker()
main_settings = session.query(MainSettings).order_by(MainSettings.id).first()
#initialize geoserver manager
geoserver_manager = GeoServerDatasetManager(engine_url=watershed.geoserver.url,
username=watershed.geoserver.username,
password=watershed.geoserver.password,
app_instance_id=main_settings.app_instance_id)
session.close()
#delete layers which need to be deleted
if watershed.geoserver_drainage_line_layer:
if watershed.geoserver_drainage_line_layer.uploaded:
geoserver_manager.purge_remove_geoserver_layer(watershed.geoserver_drainage_line_layer.name)
if watershed.geoserver_boundary_layer:
if watershed.geoserver_boundary_layer.uploaded:
geoserver_manager.purge_remove_geoserver_layer(watershed.geoserver_boundary_layer.name)
if watershed.geoserver_gage_layer:
if watershed.geoserver_gage_layer.uploaded:
geoserver_manager.purge_remove_geoserver_layer(watershed.geoserver_gage_layer.name)
if watershed.geoserver_ahps_station_layer:
if watershed.geoserver_ahps_station_layer.uploaded:
geoserver_manager.purge_remove_geoserver_layer(watershed.geoserver_ahps_station_layer.name)
def delete_old_watershed_geoserver_files(watershed):
"""
Removes old watershed geoserver files from system
"""
#initialize geoserver manager
app_instance_id = app.get_custom_setting('app_instance_id')
geoserver_manager = GeoServerDatasetManager(engine_url=watershed.geoserver.url,
username=watershed.geoserver.username,
password=watershed.geoserver.password,
app_instance_id=app_instance_id)
#delete layers which need to be deleted
if watershed.geoserver_drainage_line_layer:
if watershed.geoserver_drainage_line_layer.uploaded:
geoserver_manager.purge_remove_geoserver_layer(watershed.geoserver_drainage_line_layer.name)
if watershed.geoserver_boundary_layer:
if watershed.geoserver_boundary_layer.uploaded:
geoserver_manager.purge_remove_geoserver_layer(watershed.geoserver_boundary_layer.name)
if watershed.geoserver_gage_layer:
if watershed.geoserver_gage_layer.uploaded:
geoserver_manager.purge_remove_geoserver_layer(watershed.geoserver_gage_layer.name)
if watershed.geoserver_ahps_station_layer:
if watershed.geoserver_ahps_station_layer.uploaded:
geoserver_manager.purge_remove_geoserver_layer(watershed.geoserver_ahps_station_layer.name)
def run_autorapid_process(
autoroute_executable_location, #location of AutoRoute executable
autoroute_io_files_location, #path to AutoRoute input/outpuf directory
rapid_io_files_location, #path to AutoRoute input/outpuf directory
forecast_date_timestep,
condor_log_directory,
geoserver_url='',
geoserver_username='',
geoserver_password='',
app_instance_id=''):
"""
This it the main AutoRoute-RAPID process
"""
#initialize HTCondor Directory
condor_init_dir = os.path.join(condor_log_directory,
forecast_date_timestep)
try:
os.makedirs(condor_init_dir)
except OSError:
pass
#run autorapid for each watershed
autoroute_watershed_jobs = {}
#get most recent forecast date/timestep
print "Running AutoRoute process for forecast:", forecast_date_timestep
#loop through input watershed folders
autoroute_input_folder = os.path.join(autoroute_io_files_location, "input")
autoroute_output_folder = os.path.join(autoroute_io_files_location,
"output")
autoroute_input_directories = get_valid_watershed_list(
autoroute_input_folder)
for autoroute_input_directory in autoroute_input_directories:
watershed, subbasin = get_watershed_subbasin_from_folder(
autoroute_input_directory)
#RAPID file paths
master_watershed_rapid_input_directory = os.path.join(
rapid_io_files_location, "input", autoroute_input_directory)
master_watershed_rapid_output_directory = os.path.join(
rapid_io_files_location, 'output', autoroute_input_directory,
forecast_date_timestep)
if not os.path.exists(master_watershed_rapid_input_directory):
print "AutoRoute watershed", autoroute_input_directory, "not in RAPID IO folder. Skipping ..."
continue
if not os.path.exists(master_watershed_rapid_output_directory):
print "AutoRoute watershed", autoroute_input_directory, "missing RAPID forecast folder. Skipping ..."
continue
#setup the output location
master_watershed_autoroute_output_directory = os.path.join(
autoroute_output_folder, autoroute_input_directory,
forecast_date_timestep)
try:
os.makedirs(master_watershed_autoroute_output_directory)
except OSError:
pass
#loop through sub-directories
autoroute_watershed_directory_path = os.path.join(
autoroute_input_folder, autoroute_input_directory)
autoroute_watershed_jobs[
autoroute_input_directory] = run_autoroute_multiprocess(
autoroute_executable_location, #location of AutoRoute executable
autoroute_input_directory=
autoroute_watershed_directory_path, #path to AutoRoute input directory
autoroute_output_directory=
master_watershed_autoroute_output_directory, #path to AutoRoute output directory
log_directory=condor_init_dir,
rapid_output_directory=
master_watershed_rapid_output_directory, #path to ECMWF RAPID input/output directory
mode="htcondor", #multiprocess or htcondor
wait_for_all_processes_to_finish=False)
geoserver_manager = None
if geoserver_url and geoserver_username and geoserver_password and app_instance_id:
try:
geoserver_manager = GeoServerDatasetManager(
geoserver_url, geoserver_username, geoserver_password,
app_instance_id)
except Exception as ex:
print ex
print "Skipping geoserver upload ..."
geoserver_manager = None
pass
#wait for jobs to finish by watershed
for autoroute_watershed_directory, autoroute_watershed_job in autoroute_watershed_jobs.iteritems(
):
master_watershed_autoroute_output_directory = os.path.join(
autoroute_output_folder, autoroute_watershed_directory,
forecast_date_timestep)
#time stamped layer name
geoserver_layer_group_name = "%s-floodmap-%s" % (
autoroute_watershed_directory, forecast_date_timestep)
geoserver_resource_list = []
upload_shapefile_list = []
for job_index, job_handle in enumerate(
autoroute_watershed_job['htcondor_job_list']):
job_handle.wait()
#time stamped layer name
geoserver_resource_name = "%s-%s" % (geoserver_layer_group_name,
job_index)
#upload each shapefile
upload_shapefile = os.path.join(
master_watershed_autoroute_output_directory,
"%s%s" % (geoserver_resource_name, ".shp"))
#rename files
rename_shapefiles(
master_watershed_autoroute_output_directory,
os.path.splitext(upload_shapefile)[0],
autoroute_watershed_job['htcondor_job_info'][job_index]
['output_shapefile_base_name'])
#upload to GeoServer
if geoserver_manager:
if os.path.exists(upload_shapefile):
upload_shapefile_list.append(upload_shapefile)
print "Uploading", upload_shapefile, "to GeoServer as", geoserver_resource_name
shapefile_basename = os.path.splitext(upload_shapefile)[0]
#remove past layer if exists
#geoserver_manager.purge_remove_geoserver_layer(geoserver_manager.get_layer_name(geoserver_resource_name))
#upload updated layer
shapefile_list = glob("%s*" % shapefile_basename)
#Note: Added try, except statement because the request search fails when the app
#deletes the layer after request is made (happens hourly), so the process may throw
#an exception even though it was successful.
"""
...
File "/home/alan/work/scripts/spt_ecmwf_autorapid_process/spt_dataset_manager/dataset_manager.py", line 798, in upload_shapefile
overwrite=True)
File "/usr/lib/tethys/local/lib/python2.7/site-packages/tethys_dataset_services/engines/geoserver_engine.py", line 1288, in create_shapefile_resource
new_resource = catalog.get_resource(name=name, workspace=workspace)
File "/usr/lib/tethys/local/lib/python2.7/site-packages/geoserver/catalog.py", line 616, in get_resource
resource = self.get_resource(name, store)
File "/usr/lib/tethys/local/lib/python2.7/site-packages/geoserver/catalog.py", line 606, in get_resource
candidates = [s for s in self.get_resources(store) if s.name == name]
File "/usr/lib/tethys/local/lib/python2.7/site-packages/geoserver/catalog.py", line 645, in get_resources
return store.get_resources()
File "/usr/lib/tethys/local/lib/python2.7/site-packages/geoserver/store.py", line 58, in get_resources
xml = self.catalog.get_xml(res_url)
File "/usr/lib/tethys/local/lib/python2.7/site-packages/geoserver/catalog.py", line 188, in get_xml
raise FailedRequestError("Tried to make a GET request to %s but got a %d status code: \n%s" % (rest_url, response.status, content))
geoserver.catalog.FailedRequestError: ...
"""
try:
geoserver_manager.upload_shapefile(
geoserver_resource_name, shapefile_list)
except geo_cat_FailedRequestError as ex:
print ex
print "Most likely OK, but always wise to check ..."
pass
geoserver_resource_list.append(
geoserver_manager.get_layer_name(
geoserver_resource_name))
#TODO: Upload to CKAN for history of predicted floodmaps?
else:
print upload_shapefile, "not found. Skipping upload to GeoServer ..."
if geoserver_manager and geoserver_resource_list:
print "Creating Layer Group:", geoserver_layer_group_name
style_list = ['green' for i in range(len(geoserver_resource_list))]
bounds = get_shapefile_layergroup_bounds(upload_shapefile_list)
geoserver_manager.dataset_engine.create_layer_group(
layer_group_id=geoserver_manager.get_layer_name(
geoserver_layer_group_name),
layers=tuple(geoserver_resource_list),
styles=tuple(style_list),
bounds=tuple(bounds))
#remove local shapefile when done
for upload_shapefile in upload_shapefile_list:
shapefile_parts = glob("%s*" %
os.path.splitext(upload_shapefile)[0])
for shapefile_part in shapefile_parts:
try:
os.remove(shapefile_part)
except OSError:
pass
#remove local directories when done
try:
os.rmdir(master_watershed_autoroute_output_directory)
except OSError:
pass
def download_single_watershed_ecmwf_data(watershed,
ecmwf_rapid_prediction_directory,
app_instance_id):
"""
Loads single watersheds ECMWF datasets from data store
"""
if ecmwf_rapid_prediction_directory \
and os.path.exists(ecmwf_rapid_prediction_directory) \
and watershed.ecmwf_data_store_watershed_name \
and watershed.ecmwf_data_store_subbasin_name:
#get data engine
data_store = watershed.data_store
if 'ckan' == data_store.data_store_type.code_name:
#get dataset managers
data_manager = ECMWFRAPIDDatasetManager(data_store.api_endpoint,
data_store.api_key)
#load current datasets
data_manager.download_recent_resource(
watershed.ecmwf_data_store_watershed_name,
watershed.ecmwf_data_store_subbasin_name,
ecmwf_rapid_prediction_directory)
path_to_predicitons = os.path.join(
ecmwf_rapid_prediction_directory,
"{0}-{1}".format(watershed.ecmwf_data_store_watershed_name,
watershed.ecmwf_data_store_subbasin_name))
if os.path.exists(path_to_predicitons):
prediction_directories = sorted(os.listdir(path_to_predicitons),
reverse=True)[14:]
#remove oldest datasets if more than 14 exist
try:
for prediction_directory in prediction_directories:
rmtree(
os.path.join(path_to_predicitons,
prediction_directory))
except OSError:
pass
if watershed.geoserver_id > 1 and app_instance_id and watershed.geoserver_search_for_flood_map:
try:
#get geoserver engine
geoserver_manager = GeoServerDatasetManager(
engine_url=watershed.geoserver.url,
username=watershed.geoserver.username,
password=watershed.geoserver.password,
app_instance_id=app_instance_id)
#remove old geoserver layers
flood_map_layer_name_beginning = "%s-%s-floodmap-" % (
watershed.ecmwf_data_store_watershed_name,
watershed.ecmwf_data_store_subbasin_name)
geoserver_directories = sorted([d for d in os.listdir(path_to_predicitons) \
if os.path.isdir(os.path.join(path_to_predicitons, d))],
reverse=True)[7:]
for geoserver_directory in geoserver_directories:
layer_name = geoserver_manager.get_layer_name(
"%s%s" % (flood_map_layer_name_beginning,
geoserver_directory))
print("Deleting geoserver layer group: {0}".format(
layer_name))
#TODO: CHECK IF EXISTS BEFORE REMOVING
geoserver_manager.purge_remove_geoserver_layer_group(
layer_name)
except Exception as ex:
print(ex)
pass
def run_autorapid_process(autoroute_executable_location, #location of AutoRoute executable
autoroute_io_files_location, #path to AutoRoute input/outpuf directory
rapid_io_files_location, #path to AutoRoute input/outpuf directory
forecast_date_timestep,
condor_log_directory,
geoserver_url='',
geoserver_username='',
geoserver_password='',
app_instance_id=''
):
"""
This it the main AutoRoute-RAPID process
"""
#initialize HTCondor Directory
condor_init_dir = os.path.join(condor_log_directory, forecast_date_timestep)
try:
os.makedirs(condor_init_dir)
except OSError:
pass
#run autorapid for each watershed
autoroute_watershed_jobs = {}
#get most recent forecast date/timestep
print "Running AutoRoute process for forecast:", forecast_date_timestep
#loop through input watershed folders
autoroute_input_folder = os.path.join(autoroute_io_files_location, "input")
autoroute_output_folder = os.path.join(autoroute_io_files_location, "output")
autoroute_input_directories = get_valid_watershed_list(autoroute_input_folder)
for autoroute_input_directory in autoroute_input_directories:
watershed, subbasin = get_watershed_subbasin_from_folder(autoroute_input_directory)
#RAPID file paths
master_watershed_rapid_input_directory = os.path.join(rapid_io_files_location, "input", autoroute_input_directory)
master_watershed_rapid_output_directory = os.path.join(rapid_io_files_location, 'output',
autoroute_input_directory, forecast_date_timestep)
if not os.path.exists(master_watershed_rapid_input_directory):
print "AutoRoute watershed", autoroute_input_directory, "not in RAPID IO folder. Skipping ..."
continue
if not os.path.exists(master_watershed_rapid_output_directory):
print "AutoRoute watershed", autoroute_input_directory, "missing RAPID forecast folder. Skipping ..."
continue
#setup the output location
master_watershed_autoroute_output_directory = os.path.join(autoroute_output_folder,
autoroute_input_directory,
forecast_date_timestep)
try:
os.makedirs(master_watershed_autoroute_output_directory)
except OSError:
pass
#loop through sub-directories
autoroute_watershed_directory_path = os.path.join(autoroute_input_folder, autoroute_input_directory)
autoroute_watershed_jobs[autoroute_input_directory] = run_autoroute_multiprocess(autoroute_executable_location, #location of AutoRoute executable
autoroute_input_directory=autoroute_watershed_directory_path, #path to AutoRoute input directory
autoroute_output_directory=master_watershed_autoroute_output_directory, #path to AutoRoute output directory
log_directory=condor_init_dir,
rapid_output_directory=master_watershed_rapid_output_directory, #path to ECMWF RAPID input/output directory
mode="htcondor", #multiprocess or htcondor
wait_for_all_processes_to_finish=False
)
geoserver_manager = None
if geoserver_url and geoserver_username and geoserver_password and app_instance_id:
try:
geoserver_manager = GeoServerDatasetManager(geoserver_url,
geoserver_username,
geoserver_password,
app_instance_id)
except Exception as ex:
print ex
print "Skipping geoserver upload ..."
geoserver_manager = None
pass
#wait for jobs to finish by watershed
for autoroute_watershed_directory, autoroute_watershed_job in autoroute_watershed_jobs.iteritems():
master_watershed_autoroute_output_directory = os.path.join(autoroute_output_folder,
autoroute_watershed_directory,
forecast_date_timestep)
#time stamped layer name
geoserver_layer_group_name = "%s-floodmap-%s" % (autoroute_watershed_directory,
forecast_date_timestep)
geoserver_resource_list = []
upload_shapefile_list = []
for job_index, job_handle in enumerate(autoroute_watershed_job['htcondor_job_list']):
job_handle.wait()
#time stamped layer name
geoserver_resource_name = "%s-%s" % (geoserver_layer_group_name,
job_index)
#upload each shapefile
upload_shapefile = os.path.join(master_watershed_autoroute_output_directory,
"%s%s" % (geoserver_resource_name, ".shp"))
#rename files
rename_shapefiles(master_watershed_autoroute_output_directory,
os.path.splitext(upload_shapefile)[0],
autoroute_watershed_job['htcondor_job_info'][job_index]['output_shapefile_base_name'])
#upload to GeoServer
if geoserver_manager:
if os.path.exists(upload_shapefile):
upload_shapefile_list.append(upload_shapefile)
print "Uploading", upload_shapefile, "to GeoServer as", geoserver_resource_name
shapefile_basename = os.path.splitext(upload_shapefile)[0]
#remove past layer if exists
#geoserver_manager.purge_remove_geoserver_layer(geoserver_manager.get_layer_name(geoserver_resource_name))
#upload updated layer
shapefile_list = glob("%s*" % shapefile_basename)
#Note: Added try, except statement because the request search fails when the app
#deletes the layer after request is made (happens hourly), so the process may throw
#an exception even though it was successful.
"""
...
File "/home/alan/work/scripts/spt_ecmwf_autorapid_process/spt_dataset_manager/dataset_manager.py", line 798, in upload_shapefile
overwrite=True)
File "/usr/lib/tethys/local/lib/python2.7/site-packages/tethys_dataset_services/engines/geoserver_engine.py", line 1288, in create_shapefile_resource
new_resource = catalog.get_resource(name=name, workspace=workspace)
File "/usr/lib/tethys/local/lib/python2.7/site-packages/geoserver/catalog.py", line 616, in get_resource
resource = self.get_resource(name, store)
File "/usr/lib/tethys/local/lib/python2.7/site-packages/geoserver/catalog.py", line 606, in get_resource
candidates = [s for s in self.get_resources(store) if s.name == name]
File "/usr/lib/tethys/local/lib/python2.7/site-packages/geoserver/catalog.py", line 645, in get_resources
return store.get_resources()
File "/usr/lib/tethys/local/lib/python2.7/site-packages/geoserver/store.py", line 58, in get_resources
xml = self.catalog.get_xml(res_url)
File "/usr/lib/tethys/local/lib/python2.7/site-packages/geoserver/catalog.py", line 188, in get_xml
raise FailedRequestError("Tried to make a GET request to %s but got a %d status code: \n%s" % (rest_url, response.status, content))
geoserver.catalog.FailedRequestError: ...
"""
try:
geoserver_manager.upload_shapefile(geoserver_resource_name,
shapefile_list)
except geo_cat_FailedRequestError as ex:
print ex
print "Most likely OK, but always wise to check ..."
pass
geoserver_resource_list.append(geoserver_manager.get_layer_name(geoserver_resource_name))
#TODO: Upload to CKAN for history of predicted floodmaps?
else:
print upload_shapefile, "not found. Skipping upload to GeoServer ..."
if geoserver_manager and geoserver_resource_list:
print "Creating Layer Group:", geoserver_layer_group_name
style_list = ['green' for i in range(len(geoserver_resource_list))]
bounds = get_shapefile_layergroup_bounds(upload_shapefile_list)
geoserver_manager.dataset_engine.create_layer_group(layer_group_id=geoserver_manager.get_layer_name(geoserver_layer_group_name),
layers=tuple(geoserver_resource_list),
styles=tuple(style_list),
bounds=tuple(bounds))
#remove local shapefile when done
for upload_shapefile in upload_shapefile_list:
shapefile_parts = glob("%s*" % os.path.splitext(upload_shapefile)[0])
for shapefile_part in shapefile_parts:
try:
os.remove(shapefile_part)
except OSError:
pass
#remove local directories when done
try:
os.rmdir(master_watershed_autoroute_output_directory)
except OSError:
pass
