def init_geoprocess_raster(self): """ Initialize raster geoprocessing """ root_dir = get_directory(self.src, self.dir_startswith) for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith): file_name_date = file_name.split('.')[1] if not os.path.exists(self.dest_dir): os.makedirs(self.dest_dir) if self.mosaic_operation or self.max_val_composite: self._init_stitch_rasters(file_path, file_name_date) else: clip_out_ras = self.geoprocess_raster(file_path) if arcpy.Exists(clip_out_ras): arcpy.Delete_management(clip_out_ras) # Perform basic preprocessing if self.mosaic_operation: if self.mos_out_ras: for mos_file_path in self.mos_out_ras: clip_out_ras = self.geoprocess_raster(mos_file_path) if arcpy.Exists(clip_out_ras): arcpy.Delete_management(clip_out_ras) # Perform cell statistics if self.mosaic_operation and self.max_val_composite: if self.clean_ras: self.mosaic_operation = False for clean_file_path in self.clean_ras: file_name_date = ntpath.basename(clean_file_path).split('.')[1] self._init_stitch_rasters(clean_file_path, file_name_date) print('RASTER PROCESSING COMPLETED SUCCESSFULLY!!!')
def init_geoprocess_raster(self): """ Initialize raster geoprocessing """ root_dir = get_directory(self.src, self.dir_startswith) for source_dir, file_path, file_name in get_file_location( root_dir, self.file_startswith, self.file_endswith): file_name_date = file_name.split('.')[1] if not os.path.exists(self.dest_dir): os.makedirs(self.dest_dir) if self.mosaic_operation or self.max_val_composite: self._init_stitch_rasters(file_path, file_name_date) else: clip_out_ras = self.geoprocess_raster(file_path) if arcpy.Exists(clip_out_ras): arcpy.Delete_management(clip_out_ras) # Perform basic preprocessing if self.mosaic_operation: if self.mos_out_ras: for mos_file_path in self.mos_out_ras: clip_out_ras = self.geoprocess_raster(mos_file_path) if arcpy.Exists(clip_out_ras): arcpy.Delete_management(clip_out_ras) # Perform cell statistics if self.mosaic_operation and self.max_val_composite: if self.clean_ras: self.mosaic_operation = False for clean_file_path in self.clean_ras: file_name_date = ntpath.basename(clean_file_path).split( '.')[1] self._init_stitch_rasters(clean_file_path, file_name_date) print('RASTER PROCESSING COMPLETED SUCCESSFULLY!!!')
def _get_stitch_files(self, init_file_name): """ Get files to be stitched and to calculate maximum values from """ file_paths = [] root_dir = get_directory(self.src, self.dir_startswith) for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith): file_name_date = file_name.split('.')[1] file_name_date_id = file_name_date if file_name == init_file_name: masked_out_ras = file_path if self.mosaic_operation: masked_out_ras = self._preprocess_stitch_raster(source_dir, file_name) elif self.max_val_composite and not self.mosaic_operation: file_name_date_id = file_name_date[:4] file_paths.append(masked_out_ras) if file_name_date_id not in set(self.file_date_id): self.file_date_id.append(file_name_date_id) else: init_file_name_date = init_file_name.split('.')[1] if file_name_date == init_file_name_date: masked_out_ras = os.path.join(source_dir, file_name).replace('\\', '/') if self.mosaic_operation: masked_out_ras = self._preprocess_stitch_raster(source_dir, file_name) file_paths.append(masked_out_ras) elif file_name_date[:4] == init_file_name_date[:4]: if self.max_val_composite and not self.mosaic_operation: masked_out_ras = os.path.join(source_dir, file_name).replace('\\', '/') file_paths.append(masked_out_ras) yield file_paths
def convert_xml_to_csv(self): """ Convert XML file into CSV file :return: None :rtype: None """ id_range = self._get_identity_range() for root_dir in get_directory(self.base_dir, self.base_dir_startswith): print('Identified root directory ..... {0}'.format(root_dir)) for src_dir, file_path, file_name in get_file_location( root_dir, self.file_startswith, self.file_endswith): print('Processing XML file ..... {0}'.format(file_name)) xml_items = get_xml_items(file_path, id_range) csv_values = self._get_csv_values(id_range, xml_items) csv_out_file = src_dir + '/' + file_name[:-3] + 'csv' with open(csv_out_file, 'w') as csv_file: print('Converting to CSV file ..... {0}'.format( csv_out_file)) for k, v in csv_values.items(): values, headers, blank_values = v writer = csv.DictWriter(csv_file, headers, dialect='excel', lineterminator='\n') writer.writeheader() writer.writerow(values) writer.writerow(blank_values)
def init_geoprocess_raster(self): """ Initialize raster geoprocessing """ root_dir = get_directory(self.src, self.dir_startswith) for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith): # Reproject raster proj_out_ras = None if self.target_ref: current_ref, target_ref = self._get_spatial_ref(file_path, self.target_ref) try: if current_ref.name != target_ref.name: proj_out_ras = self._reproject_raster(current_ref, target_ref, file_path) # Reproject raster else: raise ValueError( 'Raster processing FAILED! {0} projection is similar to that of the target reference.'.format( current_ref.name ) ) except ValueError as e: print (e) # Resample raster if self.res_method: if proj_out_ras: self._resample_processor(proj_out_ras) else: self._resample_processor(file_path)
def validate_data(self): """ First check for invalid/corrupted data """ root_dir = get_directory(self.src, self.dir_startswith) for source_dir, file_path, file_name in get_file_location( root_dir, self.file_startswith, self.file_endswith): self._get_spatial_ref(file_path) print('Validated..... {0}'.format(file_name))
def init_geoprocess_raster(self): """ Initialize raster geoprocessing """ root_dir = get_directory(self.src, self.dir_startswith) for source_dir, file_path, file_name in get_file_location( root_dir, self.file_startswith, self.file_endswith): # Reproject raster proj_out_ras = None if self.target_ref: current_ref, target_ref = self._get_spatial_ref( file_path, self.target_ref) try: if current_ref.name != target_ref.name: proj_out_ras = self._reproject_raster( current_ref, target_ref, file_path) # Reproject raster else: raise ValueError( 'Raster processing FAILED! {0} projection is similar to that of the target reference.' .format(current_ref.name)) except ValueError as e: print(e) # Resample raster if self.res_method: if proj_out_ras: self._resample_processor(proj_out_ras) else: self._resample_processor(file_path)
def _get_file_path(self): """ Gets files source paths """ file_paths = [] root_dir = get_directory(self.src, self.dir_startswith) for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith): file_paths.append(str(file_path)) return file_paths
def _get_stitch_files(self, init_file_name): """ Get files to be stitched and to calculate maximum values from """ file_paths = [] root_dir = get_directory(self.src, self.dir_startswith) for source_dir, file_path, file_name in get_file_location( root_dir, self.file_startswith, self.file_endswith): file_name_date = file_name.split('.')[1] file_name_date_id = file_name_date if file_name == init_file_name: masked_out_ras = file_path if self.mosaic_operation: masked_out_ras = self._preprocess_stitch_raster( source_dir, file_name) elif self.max_val_composite and not self.mosaic_operation: file_name_date_id = file_name_date[:4] file_paths.append(masked_out_ras) if file_name_date_id not in set(self.file_date_id): self.file_date_id.append(file_name_date_id) else: init_file_name_date = init_file_name.split('.')[1] if file_name_date == init_file_name_date: masked_out_ras = os.path.join(source_dir, file_name).replace( '\\', '/') if self.mosaic_operation: masked_out_ras = self._preprocess_stitch_raster( source_dir, file_name) file_paths.append(masked_out_ras) elif file_name_date[:4] == init_file_name_date[:4]: if self.max_val_composite and not self.mosaic_operation: masked_out_ras = os.path.join(source_dir, file_name).replace( '\\', '/') file_paths.append(masked_out_ras) yield file_paths
def get_file_path(self): """ Gets files source paths """ file_names = [] root_dir = get_directory(self.src, self.dir_startswith) for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith): file_names.append(file_name) self._create_directory(self.dest_dir) self._write_csv(file_names)
def _get_all_mean_rasters(self): """ Get mean rasters to be used in sxy calculation """ mean_rasters = [] for root_dir, file_startswith, file_endswith, data_id in self._get_source_parameters(self.data_var): file_startswith = 'MEAN_' for source_dir, file_path, file_name in get_file_location(root_dir, file_startswith, file_endswith): mean_rasters.append(file_path) return mean_rasters
def _uncompress_file(self): """ Uncompress file """ fname_endswith = '.gz' root_dir = get_directory(self.src, self.dir_startswith) for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith): print('Extracting..... {0}'.format(file_name)) extract_data(source_dir, file_path, file_name, fname_endswith) self.file_endswith = '.tif'
def _get_raster_year(self): """ Get year that raster represents """ ras_year = [] root_dir = get_directory(self.src, self.dir_startswith) for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith): file_year = file_name[12:16] if file_year not in set(ras_year): ras_year.append(file_year) return ras_year
def _get_sum_rasters(self, year): """ Get rasters for summation """ ras_file_list = [] root_dir = get_directory(self.src, self.dir_startswith) for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith): if file_name[12:16] == year: if len(file_name) > 20: ras_file_list.append(file_path) return ras_file_list
def _get_mean_calculation_rasters(self): """ Get rasters for mean calculation """ outer_ras_list = [] for root_dir, file_startswith, file_endswith, data_id in self._get_source_parameters(self.data_var): inner_ras_list = [] for source_dir, file_path, file_name in get_file_location(root_dir, file_startswith, file_endswith): inner_ras_list.append(file_path) outer_ras_list.append(inner_ras_list) return outer_ras_list
def validate_data(self): """ Check for invalid/corrupted data """ root_dir = get_directory(self.src, self.dir_startswith) for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith): self._get_spatial_ref(file_path) print('Validated..... {0}'.format(file_name)) try: if len(self.place_name) != 3: raise ValueError('Input value "{0}" should be made of three characters'.format(self.place_name)) except ValueError as e: print(e)
def _calculate_syy(self, root_dir, raw_mean_ras_startswith, file_endswith, syy_ras_startswith): """ Calculate and return Syy raster """ del_rasters = [] syy_out_ras = '' first_syy_out_ras = '' for source_dir, file_path, file_name in get_file_location(root_dir, raw_mean_ras_startswith, file_endswith): syy_out_ras_name = syy_ras_startswith + file_name[len(raw_mean_ras_startswith):] # Calculate Syy raster syy_out_ras, first_syy_out_ras, del_raster = self._derive_variable_raster(syy_out_ras_name, syy_out_ras, first_syy_out_ras, source_dir, file_path) if del_raster: del_rasters.append(del_raster) self._delete_raster_file(del_rasters) # Delete collected rasters return syy_out_ras
def _clip_raster(self, no_data_val): """ Clip raster to area of interest """ root_dir = get_directory(self.src, self.dir_startswith) try: if self.clip_poly and self.clip_poly.endswith('.shp') and arcpy.Exists(self.clip_poly): for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith): clipped_ras = os.path.join(self.dest_dir, self.place_name + '_' + file_name).replace('\\', '/') print('Clipping..... {0} to {1}'.format(file_name, ntpath.basename(clipped_ras))) arcpy.Clip_management(file_path, '#', clipped_ras, self.clip_poly, no_data_val, 'ClippingGeometry') else: raise ValueError('Clipping FAILED! Clipping geometry not provided') except ValueError as e: print(e)
def validate_data(self): """ First check for invalid/corrupted data """ root_dir = get_directory(self.src, self.dir_startswith) for source_dir, file_path, file_name in get_file_location( root_dir, self.file_startswith, self.file_endswith): self._get_spatial_ref(file_path) print('Validated..... {0}'.format(file_name)) try: if len(self.place_name) != 3: raise ValueError( 'Input value "{0}" should be made of three characters'. format(self.place_name)) except ValueError as e: print(e)
def _uncompress_file(self): """ Uncompress file """ fname_endswith = '.gz' root_dir = get_directory(self.src, self.dir_startswith) for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith): if self.months: for i in tuple(list(set(self.months))): if int(file_name[17:19]) == i: print('Extracting..... {0}'.format(file_name)) extract_data(source_dir, file_path, file_name, fname_endswith) else: print('Extracting..... {0}'.format(file_name)) extract_data(source_dir, file_path, file_name, fname_endswith) self.file_endswith = '.tif'
def _resample_raster(self, cell_size): """ Modify raster resolution """ new_file_startwith = "RES_" for root_dir, file_startswith, file_endswith, data_id in self._get_source_parameters(self.data_var): for source_dir, file_path, file_name in get_file_location(root_dir, file_startswith, file_endswith): fine_cell_size = min(cell_size) coarse_cell_size = max(cell_size) in_cell_size = self._get_cell_width(file_path) # For current file if self.res_fine: if in_cell_size != fine_cell_size: self._resample_processor(file_path, fine_cell_size, new_file_startwith, self.res_method) # Resample geoprocessor self._update_file_startwith(file_startswith, new_file_startwith) else: if in_cell_size != coarse_cell_size: res_method = 'NEAREST' self._resample_processor(file_path, coarse_cell_size, new_file_startwith, res_method) self._update_file_startwith(file_startswith, new_file_startwith)
def _calculate_sxy(self, root_dir, file_startswith, file_endswith, mean_ras, median_year, raw_mean_ras_startwith, sxy_ras_startswith): """ Calculate and return sxy raster """ sxy_out_ras = '' first_sxy_out_ras = '' del_rasters = [] for source_dir, file_path, file_name in get_file_location(root_dir, file_startswith, file_endswith): data_year = self._validate_data_year(file_name) # Get raster year out_raw_mean_ras = os.path.join(source_dir, raw_mean_ras_startwith + file_name).replace('\\', '/') print('Calculating..... {0}'.format(out_raw_mean_ras)) temp_raw_mean_ras = arcpy.Raster(file_path) - arcpy.Raster(mean_ras) print('Saving..... {0}'.format(out_raw_mean_ras)) temp_raw_mean_ras.save(out_raw_mean_ras) sxy_out_ras_name = sxy_ras_startswith + file_name # Calculate Sxy raster sxy_out_ras, first_sxy_out_ras, del_raster = self._derive_variable_raster(sxy_out_ras_name, sxy_out_ras, first_sxy_out_ras, source_dir, file_path, data_year, median_year, temp_raw_mean_ras) if del_raster: del_rasters.append(del_raster) self._delete_raster_file(del_rasters) # Delete collected rasters self._delete_raster_file(mean_ras) # Delete mean raster return sxy_out_ras
def _validate_data(self): """ Check for invalid/corrupted data """ prev_file_path = '' ras_resolution = [] data_years = {} for root_dir, file_startswith, file_endswith, data_id in self._get_source_parameters(self.data_var): data_years_pair = [] for source_dir, file_path, file_name in get_file_location(root_dir, file_startswith, file_endswith): year = self._validate_file_name(file_name, data_id) # Check file naming convention and return year if year: data_years_pair.append(year) self._validate_spatial_ref(file_path, prev_file_path) # Validate spatial reference print('Validated..... {0}'.format(file_name)) cell_size = self._get_raster_resolution(file_path, ras_resolution) # Get raster resolution if cell_size: ras_resolution.append(cell_size) prev_file_path = file_path # For spatial reference validation data_years[str(data_id)] = data_years_pair self._validate_place_name() # Validated area of interest name as three letter acronym self._validate_critical_val() # Validate critical values self._validate_correlation_insig_val() return ras_resolution, data_years
def convert_xml_to_csv(self): """ Convert XML file into CSV file :return: None :rtype: None """ id_range = self._get_identity_range() for root_dir in get_directory(self.base_dir, self.base_dir_startswith): print('Identified root directory ..... {0}'.format(root_dir)) for src_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith): print('Processing XML file ..... {0}'.format(file_name)) xml_items = get_xml_items(file_path, id_range) csv_values = self._get_csv_values(id_range, xml_items) csv_out_file = src_dir + '/' + file_name[:-3] + 'csv' with open(csv_out_file, 'w') as csv_file: print('Converting to CSV file ..... {0}'.format(csv_out_file)) for k, v in csv_values.items(): values, headers, blank_values = v writer = csv.DictWriter(csv_file, headers, dialect='excel', lineterminator='\n') writer.writeheader() writer.writerow(values) writer.writerow(blank_values)
def _uncompress_file(self): """ Uncompress file """ root_dir = get_directory(self.src, self.dir_startswith) for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith): extract_data(self.dest_dir, file_path, file_name, self.file_endswith) print('Extracted: {0}'.format(file_name))
def validate_data(self): """ First check for invalid/corrupted data """ root_dir = get_directory(self.src, self.dir_startswith) for source_dir, file_path, file_name in get_file_location(root_dir, self.file_startswith, self.file_endswith): self._get_spatial_ref(file_path) print('Validated..... {0}'.format(file_name))
def _get_rxy_numerator_raster(self, root_dir, raw_mean_ras_startswith, file_endswith): """ Get rasters to be used in the calculation of Pearson's coefficient numerator raster """ for source_dir, file_path, file_name in get_file_location(root_dir, raw_mean_ras_startswith, file_endswith): yield source_dir, file_path, file_name