def start(self):
try:
# start the logging
self.setup_logger()
self.logger.info('Processing started.')
except Exception:
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('Unable to initialize logger.')
try:
# create output directory structure
self.create_output_dir()
except Exception:
self.logger.error('Create output directory structures.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('Create output directory structures.')
# convert hrus1 grid to tif
try:
geotools.convert_adf_to_tif(
self.results_dir + '/Raster/hrus1',
self.results_dir + '/Raster/hrus1/hrus1.tif')
except Exception:
self.logger.error('Unable to convert raster from .adf to .tif.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('Unable to convert raster from .adf to .tif.')
# copy hru1 and fields shapefiles to output directory
try:
self.copy_shapefile_to_output_directory()
except Exception:
self.logger.error(
'An error occurred while copying hru1 and fields shapefiles to output directory.'
)
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception(
'An error occurred while copying hru1 and fields shapefiles to output directory.'
)
# merge hru thresholds
try:
self.merge_thresholds()
except Exception:
self.logger.error(
'An error occurred while merging hru thresholds.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('An error occurred while merging hru thresholds.')
try:
# get hrus1 cols, rows, and resolution
hrus1_info = self.get_tif_info()
except Exception:
self.logger.error(
'An error occurred while fetching the hrus1 raster details.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception(
'An error occurred while fetching the hrus1 raster details.')
try:
grid_x_reshape, grid_y_reshape = self.set_gridx_and_gridy_matrices(
hrus1_info)
except Exception:
self.logger.error(
'An error occurred while setting the gridx and gridy matrices.'
)
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception(
'An error occurred while setting the gridx and gridy matrices.'
)
try:
clu = self.create_hru_field_workbook(grid_x_reshape,
grid_y_reshape,
hrus1_info['nodata'])
except Exception:
self.logger.error(
'An error occurred while creating the hru field workbook.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception(
'An error occurred while creating the hru field workbook.')
try:
field_shapefile, output_data, hru_output_data = self.update_field_info(
hrus1_info['nodata'], clu)
except Exception:
self.logger.error('An error occurred while updating field info.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('An error occurred while updating field info.')
try:
self.create_new_field_shapefile(field_shapefile, output_data)
except Exception:
self.logger.error(
'An error occurred while creating the new shapefile.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception(
'An error occurred while creating the new shapefile.')
hru_shapefile = self.results_dir + '/Output/HRU_Response.shp'
try:
self.update_hru_shapefile(hru_shapefile, hru_output_data)
except Exception:
self.logger.error(
'An error occurred while updating the hru shapefile.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception(
'An error occurred while updating the hru shapefile.')
def start(self):
"""
"""
try:
self.setup_logger()
self.logger.info('Processing started.')
except Exception:
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('Unable to initialize logger.')
try:
# create output directory structure
self.create_output_dir()
except Exception:
self.logger.error('Unable to create output directory.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('Unable to create output directory.')
# create lup.dat file and populate with landuse layers' date information
try:
self.create_lupdat_file()
except Exception:
self.logger.error('Unable to create lup.dat file.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('Unable to create lup.dat file.')
# extract lookup information from uploaded lookup file
try:
self.extract_lookup_info()
except Exception:
self.logger.error(
'Unable to extract lookup information from uploaded lookup file.'
)
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception(
'Unable to extract lookup information from uploaded lookup file.'
)
# convert hrus1 grid to tif
try:
geotools.convert_adf_to_tif(
self.results_dir + '/Raster/hrus1',
self.results_dir + '/Raster/hrus1/hrus1.tif')
except Exception:
self.logger.error('Unable to convert raster from .adf to .tif.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('Unable to convert raster from .adf to .tif.')
# merge non-dominant hrus into nearby dominant hrus
try:
self.merge_thresholds()
except Exception:
self.logger.error(
'Unable to merge non-dominant HRUs into nearby dominant hrus.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception(
'Unable to merge non-dominant HRUs into nearby dominant hrus.')
# create fractional area for each dominant hru
try:
self.create_fractional_values()
except Exception:
self.logger.error('Unable to create fractional values.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('Unable to create fractional values.')
# extract hru information from TxtInOut folder
try:
self.extract_hru_files_data()
except Exception:
self.logger.error('Unable to extract HRU files data.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('Unable to extract HRU files data.')
# calculate new fractional areas using the uploaded landuse layers
try:
self.calculate_new_fractional_areas()
except Exception:
self.logger.error('Unable to calculate new fractional areas.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('Unable to caculate new fractional areas.')
def start(self):
"""
Directs the work flow of the LUU Checker task. Essentially these steps
are taken:
1) Create output and temporary output directory structure
2) Convert base landuse raster to geotiff
3) Read base landuse raster into numpy array
4) Convert subbasin shapefile into geotiff
5) Create Emerging LULC (EL) Report
6) Loop through each new landuse raster
L1a) Update EL Report with new entry for current new landuse raster
L1b) Convert new landuse raster to geotiff
L1c) Read new landuse raster into numpy array
L1d) Loop through each subbasin
L2a) Compare (for current subbasin) lulc codes in base landuse
raster and new landuse raster
L2b) Isolate lulc codes that are emerging in new landuse raster
L2c) Update EL Report with emerging lulc codes
L2d) Loop through each emerging lulc code
L3a) Randomize the indicies in a copy of the base landuse
raster that correspond with the current subbasin
L3b) Calculate how many pixels should be reclassified to
the new lulc code by using the subbasin size and the
user provided percentage value
L3c) Use the randomized indicies to inject the new lulc
into the base landuse raster
7) Create composite landuse raster using the updated base landuse raster
8) Close the EL Report
9) Remove temporary output directory
Parameters
----------
None
Returns
-------
None
"""
try:
self.setup_logger()
self.logger.info('Processing started.')
# create output directory structure
self.create_output_dir()
except Exception as e:
print(e)
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('Unable to initialize logger.')
# convert base landuse raster from .adf to .tif
try:
output_filepath = self.temp_output_directory + '/' + os.path.basename(
self.base_landuse_raster_filepath) + '.tif'
geotools.convert_adf_to_tif(self.base_landuse_raster_filepath,
output_filepath)
except Exception:
self.logger.error('Unable to convert raster from .adf to .tif.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('Unable to convert raster from .adf to .tif.')
self.logger.info('Converting base raster geotiff into numpy array.')
try:
# read base landuse raster (.tif) into numpy array
base_raster_array = geotools.read_raster(
self.temp_output_directory + '/' + \
self.base_landuse_raster_filename + '.tif')[0]
# construct shapefile layer information
rows = str(len(base_raster_array))
cols = str(len(base_raster_array[0]))
layer_info = {
"attribute_name": "Subbasin",
"extent": [cols, rows],
"layername": "subs1",
}
except Exception:
self.logger.error('Unable to read the base landuse raster.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('Unable to read the base landuse raster.')
self.logger.info(
'Getting count of subbasins from the subbasin shapefile.')
try:
# get number of subbasins in shapefile
total_subbasin_count = len(
geotools.read_shapefile(self.subbasin_shapefile_filepath))
except Exception:
self.logger.error('Unable to read the subbasin shapefile.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('Unable to read the subbasin shapefile.')
# path and filename for the soon to be created subbasin geotiff
output_tif_filepath = self.temp_output_directory + '/subbasin.tif'
# create geotiff raster of the subbasin shapefile
self.logger.info('Converting subbasin .shp to .tif.')
try:
# convert shapefile to raster
geotools.rasterize_shapefile(layer_info,
self.subbasin_shapefile_filepath,
output_tif_filepath)
except Exception:
self.logger.error(
'Error converting shapefile to raster. Please make ' + \
'sure you uploaded file.shp.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception(
'Error converting shapefile to raster. Please make ' + \
'sure you uploaded file.shp.')
self.logger.info('Converting subbasin geotiff into numpy array.')
try:
# read rasterized shapefile into numpy array
rasterized_shapefile = \
geotools.read_raster(self.temp_output_directory + '/subbasin.tif')[
0]
except Exception:
self.logger.error(
'Unable to read the rasterized subbasin geotiff.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('Unable to read the rasterized subbasin geotiff.')
self.logger.info('Opening Emerging_LULC_Report for writing.')
try:
# remove emerging lulcs report if it already exists
if os.path.isfile(self.output_directory + '/Emerging_LULCs.txt'):
os.remove(self.output_directory + '/Emerging_LULCs.txt')
# create emerging_lulcs text file to store new landuse information
emerging_lulc_report = open(
self.output_directory + '/Emerging_LULC_Report.txt', 'w')
except Exception:
self.logger.error(
'Unable to create emerging_lulcs text file to store new landuse information.'
)
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception(
'Unable to create emerging_lulcs text file to store new landuse information.'
)
self.logger.info('Begin looping through new landuse layers.\n\n')
# Keep track of indices already used for new code injection
# Key: subbasin id, Val: indices
injection_history = {}
try:
# loop through each new landuse layer selected by the user
for landuse_layer in self.new_landuse_raster_filepaths:
self.logger.info('LANDUSE LAYER:' +
os.path.basename(landuse_layer))
# write the landuse layer name to report
emerging_lulc_report.write(landuse_layer + '\n')
# convert the new landuse layer raster to array
geotools.convert_adf_to_tif(
landuse_layer, self.temp_output_directory + '/' +
os.path.basename(landuse_layer) + '.tif')
self.logger.info(
'Converting new landuse geotiff into numpy array.')
self.logger.info(self.temp_output_directory + '/' +
os.path.basename(landuse_layer) + '.tif')
# read new landuse raster (.tif) into numpy array
new_landuse_raster = geotools.read_raster(
self.temp_output_directory + '/' + \
os.path.basename(landuse_layer) + '.tif')[0]
self.logger.info('Begin looping through subbasins.')
# create feature layers based off the FID field & then use as mask
# to extract subbasin landuse information
for i in range(0, total_subbasin_count):
self.logger.info('SUBBASIN #' + str(i + 1) + ':')
# write the subbasin number to report
emerging_lulc_report.write('Subbasin ' + str(i + 1) + '\n')
self.logger.info('Finding indicies in base raster array ' +
'that correspond with current subbasin.')
# find indicies where the value < 255 (remove the NoData)
idx = np.nonzero(rasterized_shapefile == i + 1)
# find lulc codes in the new layer that aren't in the base layer
new_lulc_codes = self.find_unique_lulc_codes(
idx, base_raster_array, new_landuse_raster)
# write the emerging lulc to report
emerging_lulc_report.write(str(new_lulc_codes) + '\n\n')
# inject new lulc codes into the base raster array
base_raster_array, injection_history = self.inject_new_lulc_codes(
i + 1, idx, new_lulc_codes, base_raster_array,
injection_history)
self.logger.info('End looping through subbasins.')
except:
self.logger.error(
'An error occurred while creating the emerging lulc report.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception(
'An error occurred while creating the emerging lulc report.')
self.logger.info('End looping through new landuse layers.\n\n')
try:
# convert the updated base raster array (composite) to geotiff
self.create_composite_raster(
base_raster_array, self.base_landuse_raster_adf_filepath,
self.temp_output_directory + '/base_new1.tif')
except:
self.logger.error(
'An error occurred while creating the composite raster.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception(
'An error occurred while creating the composite raster.')
self.logger.info('Closing Emerging_LULC_Report for writing.')
# close emerging lulc report
emerging_lulc_report.close()
self.logger.info('Removing temporary output directories.')
# remove temporary output directory
if os.path.exists(self.temp_output_directory):
shutil.rmtree(self.temp_output_directory)
self.logger.info('Processing completed.\n\n')
def create_lupdat_file(self):
"""
Creates the lup.dat file. The lup.dat file must be written
in the following format:
id month day year file.dat
Parameters
----------
data: dictionary
Contains file and directory paths for inputs
Examples
--------
1 1 1 1992 file1.dat
2 1 1 1999 file2.dat
3 1 1 2001 file3.dat
Returns
-------
None
"""
self.logger.info('Creating lup.dat file.')
try:
lup_file = open(self.results_dir + '/lup.dat', 'a')
except IOError:
self.logger.error('Unable to open the lup.dat file.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.logger.info(
'Extracting information about landuse layers\' dates.')
# loop through landuse layers and pull out date information provided by user
landuse_layers_data = []
for layer_index in range(0, len(self.landuse_layers_names)):
day = int(self.landuse_days[layer_index])
month = int(self.landuse_months[layer_index])
year = int(self.landuse_years[layer_index])
layer_name = self.landuse_layers_names[layer_index]
landuse_layers_data.append([month, day, year, layer_name])
# convert landuse files to tiff files
layer_path = self.landuse_dir + '/' + layer_name
converted_layer_path = self.results_dir + '/Raster/' + layer_name + '.tif'
try:
geotools.convert_adf_to_tif(layer_path, converted_layer_path)
except Exception:
self.logger.error(
'Unable to convert raster from .adf to .tif.')
UserTask.objects.filter(task_id=self.task_id).update(
task_status=2)
# write the date and file into lup.dat
self.logger.info('Adding landuse layer, ' + layer_name +
', to lup.dat.')
lup_file.write('{0} {1} {2} {3} {4} {5}\n'.format(
''.ljust(4 - len(str(layer_index + 1))),
str(layer_index + 1).ljust(5 - len(str(month)) +
len(str(layer_index + 1)) - 1),
str(month).ljust(5 - len(str(day)) + len(str(month)) - 1),
str(day).ljust(5 - len(str(year)) + len(str(day)) - 1),
str(year).ljust(13 -
len(str('file' + str(layer_index + 1) +
'.dat'))),
('file' + str(layer_index + 1) + '.dat')))
self.logger.info('Closing lup.dat.')
lup_file.close()
self.landuse_layers_data = landuse_layers_data
def start(self):
"""
"""
try:
self.setup_logger()
self.logger.info('Processing started.')
except Exception:
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('Unable to initialize logger.')
# create output directory structure
try:
self.create_output_dir()
except Exception:
self.logger.error('Create output directory structures.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('Create output directory structures.')
# convert hrus1 grid to tif
try:
geotools.convert_adf_to_tif(
self.results_dir + '/Raster/hrus1',
self.results_dir + '/Raster/hrus1/hrus1.tif')
except Exception:
self.logger.error('Unable to convert raster from .adf to .tif.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('Unable to convert raster from .adf to .tif.')
try:
self.create_lupdat_file()
except Exception:
self.logger.error(
'An error occurred while creating the lup dat files.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception(
'An error occurred while creating the lup dat files.')
try:
self.extract_lookup_info()
except Exception:
self.logger.error(
'An error occurred while extracting lookup info.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('An error occurred while extracting lookup info.')
try:
self.merge_thresholds()
except Exception:
self.logger.error('An error occurred while merging thresholds.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('An error occurred while merging thresholds.')
try:
self.create_fractional_values()
except Exception:
self.logger.error(
'An error occurred while creating fractional values.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception(
'An error occurred while creating fractional values.')
try:
self.extract_hru_files_data()
except Exception:
self.logger.error(
'An error occurred while extracting hru files data.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception(
'An error occurred while extracting hru files data.')
try:
self.apply_realization()
except Exception:
self.logger.error(
'An error occurred while applying realization.')
UserTask.objects.filter(task_id=self.task_id).update(task_status=2)
self.email_error_alert_to_user()
raise Exception('An error occurred while applying realization.')