def init_geoprocess_raster(self):
""" Initialize raster geoprocessing """
root_dir = get_directory(self.src, self.dir_startswith)
for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith):
file_name_date = file_name.split('.')[1]
if not os.path.exists(self.dest_dir):
os.makedirs(self.dest_dir)
if self.mosaic_operation or self.max_val_composite:
self._init_stitch_rasters(file_path, file_name_date)
else:
clip_out_ras = self.geoprocess_raster(file_path)
if arcpy.Exists(clip_out_ras):
arcpy.Delete_management(clip_out_ras)
# Perform basic preprocessing
if self.mosaic_operation:
if self.mos_out_ras:
for mos_file_path in self.mos_out_ras:
clip_out_ras = self.geoprocess_raster(mos_file_path)
if arcpy.Exists(clip_out_ras):
arcpy.Delete_management(clip_out_ras)
# Perform cell statistics
if self.mosaic_operation and self.max_val_composite:
if self.clean_ras:
self.mosaic_operation = False
for clean_file_path in self.clean_ras:
file_name_date = ntpath.basename(clean_file_path).split('.')[1]
self._init_stitch_rasters(clean_file_path, file_name_date)
print('RASTER PROCESSING COMPLETED SUCCESSFULLY!!!')
def init_geoprocess_raster(self):
""" Initialize raster geoprocessing """
root_dir = get_directory(self.src, self.dir_startswith)
for source_dir, file_path, file_name in get_file_location(
root_dir, self.file_startswith, self.file_endswith):
file_name_date = file_name.split('.')[1]
if not os.path.exists(self.dest_dir):
os.makedirs(self.dest_dir)
if self.mosaic_operation or self.max_val_composite:
self._init_stitch_rasters(file_path, file_name_date)
else:
clip_out_ras = self.geoprocess_raster(file_path)
if arcpy.Exists(clip_out_ras):
arcpy.Delete_management(clip_out_ras)
# Perform basic preprocessing
if self.mosaic_operation:
if self.mos_out_ras:
for mos_file_path in self.mos_out_ras:
clip_out_ras = self.geoprocess_raster(mos_file_path)
if arcpy.Exists(clip_out_ras):
arcpy.Delete_management(clip_out_ras)
# Perform cell statistics
if self.mosaic_operation and self.max_val_composite:
if self.clean_ras:
self.mosaic_operation = False
for clean_file_path in self.clean_ras:
file_name_date = ntpath.basename(clean_file_path).split(
'.')[1]
self._init_stitch_rasters(clean_file_path, file_name_date)
print('RASTER PROCESSING COMPLETED SUCCESSFULLY!!!')
def _get_stitch_files(self, init_file_name):
""" Get files to be stitched and to calculate maximum values from """
file_paths = []
root_dir = get_directory(self.src, self.dir_startswith)
for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith):
file_name_date = file_name.split('.')[1]
file_name_date_id = file_name_date
if file_name == init_file_name:
masked_out_ras = file_path
if self.mosaic_operation:
masked_out_ras = self._preprocess_stitch_raster(source_dir, file_name)
elif self.max_val_composite and not self.mosaic_operation:
file_name_date_id = file_name_date[:4]
file_paths.append(masked_out_ras)
if file_name_date_id not in set(self.file_date_id):
self.file_date_id.append(file_name_date_id)
else:
init_file_name_date = init_file_name.split('.')[1]
if file_name_date == init_file_name_date:
masked_out_ras = os.path.join(source_dir, file_name).replace('\\', '/')
if self.mosaic_operation:
masked_out_ras = self._preprocess_stitch_raster(source_dir, file_name)
file_paths.append(masked_out_ras)
elif file_name_date[:4] == init_file_name_date[:4]:
if self.max_val_composite and not self.mosaic_operation:
masked_out_ras = os.path.join(source_dir, file_name).replace('\\', '/')
file_paths.append(masked_out_ras)
yield file_paths
def convert_xml_to_csv(self):
"""
Convert XML file into CSV file
:return: None
:rtype: None
"""
id_range = self._get_identity_range()
for root_dir in get_directory(self.base_dir, self.base_dir_startswith):
print('Identified root directory ..... {0}'.format(root_dir))
for src_dir, file_path, file_name in get_file_location(
root_dir, self.file_startswith, self.file_endswith):
print('Processing XML file ..... {0}'.format(file_name))
xml_items = get_xml_items(file_path, id_range)
csv_values = self._get_csv_values(id_range, xml_items)
csv_out_file = src_dir + '/' + file_name[:-3] + 'csv'
with open(csv_out_file, 'w') as csv_file:
print('Converting to CSV file ..... {0}'.format(
csv_out_file))
for k, v in csv_values.items():
values, headers, blank_values = v
writer = csv.DictWriter(csv_file,
headers,
dialect='excel',
lineterminator='\n')
writer.writeheader()
writer.writerow(values)
writer.writerow(blank_values)
def init_geoprocess_raster(self):
""" Initialize raster geoprocessing """
root_dir = get_directory(self.src, self.dir_startswith)
for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith):
# Reproject raster
proj_out_ras = None
if self.target_ref:
current_ref, target_ref = self._get_spatial_ref(file_path, self.target_ref)
try:
if current_ref.name != target_ref.name:
proj_out_ras = self._reproject_raster(current_ref, target_ref, file_path) # Reproject raster
else:
raise ValueError(
'Raster processing FAILED! {0} projection is similar to that of the target reference.'.format(
current_ref.name
)
)
except ValueError as e:
print (e)
# Resample raster
if self.res_method:
if proj_out_ras:
self._resample_processor(proj_out_ras)
else:
self._resample_processor(file_path)
def validate_data(self):
""" First check for invalid/corrupted data """
root_dir = get_directory(self.src, self.dir_startswith)
for source_dir, file_path, file_name in get_file_location(
root_dir, self.file_startswith, self.file_endswith):
self._get_spatial_ref(file_path)
print('Validated..... {0}'.format(file_name))
def init_geoprocess_raster(self):
""" Initialize raster geoprocessing """
root_dir = get_directory(self.src, self.dir_startswith)
for source_dir, file_path, file_name in get_file_location(
root_dir, self.file_startswith, self.file_endswith):
# Reproject raster
proj_out_ras = None
if self.target_ref:
current_ref, target_ref = self._get_spatial_ref(
file_path, self.target_ref)
try:
if current_ref.name != target_ref.name:
proj_out_ras = self._reproject_raster(
current_ref, target_ref,
file_path) # Reproject raster
else:
raise ValueError(
'Raster processing FAILED! {0} projection is similar to that of the target reference.'
.format(current_ref.name))
except ValueError as e:
print(e)
# Resample raster
if self.res_method:
if proj_out_ras:
self._resample_processor(proj_out_ras)
else:
self._resample_processor(file_path)
def _get_file_path(self):
""" Gets files source paths """
file_paths = []
root_dir = get_directory(self.src, self.dir_startswith)
for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith):
file_paths.append(str(file_path))
return file_paths
def _get_stitch_files(self, init_file_name):
""" Get files to be stitched and to calculate maximum values from """
file_paths = []
root_dir = get_directory(self.src, self.dir_startswith)
for source_dir, file_path, file_name in get_file_location(
root_dir, self.file_startswith, self.file_endswith):
file_name_date = file_name.split('.')[1]
file_name_date_id = file_name_date
if file_name == init_file_name:
masked_out_ras = file_path
if self.mosaic_operation:
masked_out_ras = self._preprocess_stitch_raster(
source_dir, file_name)
elif self.max_val_composite and not self.mosaic_operation:
file_name_date_id = file_name_date[:4]
file_paths.append(masked_out_ras)
if file_name_date_id not in set(self.file_date_id):
self.file_date_id.append(file_name_date_id)
else:
init_file_name_date = init_file_name.split('.')[1]
if file_name_date == init_file_name_date:
masked_out_ras = os.path.join(source_dir,
file_name).replace(
'\\', '/')
if self.mosaic_operation:
masked_out_ras = self._preprocess_stitch_raster(
source_dir, file_name)
file_paths.append(masked_out_ras)
elif file_name_date[:4] == init_file_name_date[:4]:
if self.max_val_composite and not self.mosaic_operation:
masked_out_ras = os.path.join(source_dir,
file_name).replace(
'\\', '/')
file_paths.append(masked_out_ras)
yield file_paths
def get_file_path(self):
""" Gets files source paths """
file_names = []
root_dir = get_directory(self.src, self.dir_startswith)
for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith):
file_names.append(file_name)
self._create_directory(self.dest_dir)
self._write_csv(file_names)
def _get_all_mean_rasters(self):
""" Get mean rasters to be used in sxy calculation """
mean_rasters = []
for root_dir, file_startswith, file_endswith, data_id in self._get_source_parameters(self.data_var):
file_startswith = 'MEAN_'
for source_dir, file_path, file_name in get_file_location(root_dir, file_startswith, file_endswith):
mean_rasters.append(file_path)
return mean_rasters
def _uncompress_file(self):
""" Uncompress file """
fname_endswith = '.gz'
root_dir = get_directory(self.src, self.dir_startswith)
for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith):
print('Extracting..... {0}'.format(file_name))
extract_data(source_dir, file_path, file_name, fname_endswith)
self.file_endswith = '.tif'
def _get_raster_year(self):
""" Get year that raster represents """
ras_year = []
root_dir = get_directory(self.src, self.dir_startswith)
for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith):
file_year = file_name[12:16]
if file_year not in set(ras_year):
ras_year.append(file_year)
return ras_year
def _get_sum_rasters(self, year):
""" Get rasters for summation """
ras_file_list = []
root_dir = get_directory(self.src, self.dir_startswith)
for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith):
if file_name[12:16] == year:
if len(file_name) > 20:
ras_file_list.append(file_path)
return ras_file_list
def _get_mean_calculation_rasters(self):
""" Get rasters for mean calculation """
outer_ras_list = []
for root_dir, file_startswith, file_endswith, data_id in self._get_source_parameters(self.data_var):
inner_ras_list = []
for source_dir, file_path, file_name in get_file_location(root_dir, file_startswith, file_endswith):
inner_ras_list.append(file_path)
outer_ras_list.append(inner_ras_list)
return outer_ras_list
def validate_data(self):
""" Check for invalid/corrupted data """
root_dir = get_directory(self.src, self.dir_startswith)
for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith):
self._get_spatial_ref(file_path)
print('Validated..... {0}'.format(file_name))
try:
if len(self.place_name) != 3:
raise ValueError('Input value "{0}" should be made of three characters'.format(self.place_name))
except ValueError as e:
print(e)
def _calculate_syy(self, root_dir, raw_mean_ras_startswith, file_endswith, syy_ras_startswith):
""" Calculate and return Syy raster """
del_rasters = []
syy_out_ras = ''
first_syy_out_ras = ''
for source_dir, file_path, file_name in get_file_location(root_dir, raw_mean_ras_startswith, file_endswith):
syy_out_ras_name = syy_ras_startswith + file_name[len(raw_mean_ras_startswith):]
# Calculate Syy raster
syy_out_ras, first_syy_out_ras, del_raster = self._derive_variable_raster(syy_out_ras_name, syy_out_ras, first_syy_out_ras, source_dir, file_path)
if del_raster:
del_rasters.append(del_raster)
self._delete_raster_file(del_rasters) # Delete collected rasters
return syy_out_ras
def _clip_raster(self, no_data_val):
""" Clip raster to area of interest """
root_dir = get_directory(self.src, self.dir_startswith)
try:
if self.clip_poly and self.clip_poly.endswith('.shp') and arcpy.Exists(self.clip_poly):
for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith):
clipped_ras = os.path.join(self.dest_dir, self.place_name + '_' + file_name).replace('\\', '/')
print('Clipping..... {0} to {1}'.format(file_name, ntpath.basename(clipped_ras)))
arcpy.Clip_management(file_path, '#', clipped_ras, self.clip_poly, no_data_val, 'ClippingGeometry')
else:
raise ValueError('Clipping FAILED! Clipping geometry not provided')
except ValueError as e:
print(e)
def validate_data(self):
""" First check for invalid/corrupted data """
root_dir = get_directory(self.src, self.dir_startswith)
for source_dir, file_path, file_name in get_file_location(
root_dir, self.file_startswith, self.file_endswith):
self._get_spatial_ref(file_path)
print('Validated..... {0}'.format(file_name))
try:
if len(self.place_name) != 3:
raise ValueError(
'Input value "{0}" should be made of three characters'.
format(self.place_name))
except ValueError as e:
print(e)
def _uncompress_file(self):
""" Uncompress file """
fname_endswith = '.gz'
root_dir = get_directory(self.src, self.dir_startswith)
for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith):
if self.months:
for i in tuple(list(set(self.months))):
if int(file_name[17:19]) == i:
print('Extracting..... {0}'.format(file_name))
extract_data(source_dir, file_path, file_name, fname_endswith)
else:
print('Extracting..... {0}'.format(file_name))
extract_data(source_dir, file_path, file_name, fname_endswith)
self.file_endswith = '.tif'
def _resample_raster(self, cell_size):
""" Modify raster resolution """
new_file_startwith = "RES_"
for root_dir, file_startswith, file_endswith, data_id in self._get_source_parameters(self.data_var):
for source_dir, file_path, file_name in get_file_location(root_dir, file_startswith, file_endswith):
fine_cell_size = min(cell_size)
coarse_cell_size = max(cell_size)
in_cell_size = self._get_cell_width(file_path) # For current file
if self.res_fine:
if in_cell_size != fine_cell_size:
self._resample_processor(file_path, fine_cell_size, new_file_startwith, self.res_method) # Resample geoprocessor
self._update_file_startwith(file_startswith, new_file_startwith)
else:
if in_cell_size != coarse_cell_size:
res_method = 'NEAREST'
self._resample_processor(file_path, coarse_cell_size, new_file_startwith, res_method)
self._update_file_startwith(file_startswith, new_file_startwith)
def _calculate_sxy(self, root_dir, file_startswith, file_endswith, mean_ras, median_year, raw_mean_ras_startwith, sxy_ras_startswith):
""" Calculate and return sxy raster """
sxy_out_ras = ''
first_sxy_out_ras = ''
del_rasters = []
for source_dir, file_path, file_name in get_file_location(root_dir, file_startswith, file_endswith):
data_year = self._validate_data_year(file_name) # Get raster year
out_raw_mean_ras = os.path.join(source_dir, raw_mean_ras_startwith + file_name).replace('\\', '/')
print('Calculating..... {0}'.format(out_raw_mean_ras))
temp_raw_mean_ras = arcpy.Raster(file_path) - arcpy.Raster(mean_ras)
print('Saving..... {0}'.format(out_raw_mean_ras))
temp_raw_mean_ras.save(out_raw_mean_ras)
sxy_out_ras_name = sxy_ras_startswith + file_name
# Calculate Sxy raster
sxy_out_ras, first_sxy_out_ras, del_raster = self._derive_variable_raster(sxy_out_ras_name, sxy_out_ras, first_sxy_out_ras, source_dir, file_path, data_year, median_year, temp_raw_mean_ras)
if del_raster:
del_rasters.append(del_raster)
self._delete_raster_file(del_rasters) # Delete collected rasters
self._delete_raster_file(mean_ras) # Delete mean raster
return sxy_out_ras
def _validate_data(self):
""" Check for invalid/corrupted data """
prev_file_path = ''
ras_resolution = []
data_years = {}
for root_dir, file_startswith, file_endswith, data_id in self._get_source_parameters(self.data_var):
data_years_pair = []
for source_dir, file_path, file_name in get_file_location(root_dir, file_startswith, file_endswith):
year = self._validate_file_name(file_name, data_id) # Check file naming convention and return year
if year:
data_years_pair.append(year)
self._validate_spatial_ref(file_path, prev_file_path) # Validate spatial reference
print('Validated..... {0}'.format(file_name))
cell_size = self._get_raster_resolution(file_path, ras_resolution) # Get raster resolution
if cell_size:
ras_resolution.append(cell_size)
prev_file_path = file_path # For spatial reference validation
data_years[str(data_id)] = data_years_pair
self._validate_place_name() # Validated area of interest name as three letter acronym
self._validate_critical_val() # Validate critical values
self._validate_correlation_insig_val()
return ras_resolution, data_years
def convert_xml_to_csv(self):
"""
Convert XML file into CSV file
:return: None
:rtype: None
"""
id_range = self._get_identity_range()
for root_dir in get_directory(self.base_dir, self.base_dir_startswith):
print('Identified root directory ..... {0}'.format(root_dir))
for src_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith):
print('Processing XML file ..... {0}'.format(file_name))
xml_items = get_xml_items(file_path, id_range)
csv_values = self._get_csv_values(id_range, xml_items)
csv_out_file = src_dir + '/' + file_name[:-3] + 'csv'
with open(csv_out_file, 'w') as csv_file:
print('Converting to CSV file ..... {0}'.format(csv_out_file))
for k, v in csv_values.items():
values, headers, blank_values = v
writer = csv.DictWriter(csv_file, headers, dialect='excel', lineterminator='\n')
writer.writeheader()
writer.writerow(values)
writer.writerow(blank_values)
def _uncompress_file(self):
""" Uncompress file """
root_dir = get_directory(self.src, self.dir_startswith)
for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith):
extract_data(self.dest_dir, file_path, file_name, self.file_endswith)
print('Extracted: {0}'.format(file_name))
def validate_data(self):
""" First check for invalid/corrupted data """
root_dir = get_directory(self.src, self.dir_startswith)
for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith):
self._get_spatial_ref(file_path)
print('Validated..... {0}'.format(file_name))
def _get_rxy_numerator_raster(self, root_dir, raw_mean_ras_startswith, file_endswith):
""" Get rasters to be used in the calculation of Pearson's coefficient numerator raster """
for source_dir, file_path, file_name in get_file_location(root_dir, raw_mean_ras_startswith, file_endswith):
yield source_dir, file_path, file_name