def __init__(self, quality_control_file, band, with_stats, number_of_processes): QualityControl.list.append(self) self.band = band self.band_name = 'band' + fix_zeros(band, 2) self.qcf = quality_control_file self.with_stats = with_stats self.number_of_processes = number_of_processes self.qc_check_lists = {} self.output_driver = None self.output_bands = [] self.output_filename = "{0}_{1}_band{2}.tif".format( SatelliteData.tile, SatelliteData.shortname, fix_zeros(band, 2)) if self.with_stats: # for save some statistics fields after check the quality control self.quality_control_statistics = {} # initialize quality control bands class for sd in SatelliteData.list: for qc_id_name, qc_checker in sd.qc_bands.items(): qc_checker.init_statistics(quality_control_file)
def save_results(self, output_dir): """Save all processed files in one file per each data band to process, each file to save has the precessed files as bands. :param output_dir: directory to save the output file :type output_dir: path """ print("\nSaving the result for the band {0} in: {1}".format( self.band, self.output_filename)) # get gdal properties of one of data band sd = SatelliteData.list[0] data_band_name = [ x for x in sd.sub_datasets if 'b' + fix_zeros(self.band, 2) in x[1] ][0][0] gdal_data_band = gdal.Open(data_band_name, gdal.GA_ReadOnly) geotransform = gdal_data_band.GetGeoTransform() originX = geotransform[0] originY = geotransform[3] pixelWidth = geotransform[1] pixelHeight = geotransform[5] # create output raster driver = gdal.GetDriverByName('GTiff') nbands = len(self.output_bands) outRaster = driver.Create( os.path.join(output_dir, self.output_filename), sd.get_cols(self.band), sd.get_rows(self.band), nbands, gdal.GDT_Int16, ["COMPRESS=LZW", "PREDICTOR=2", "TILED=YES"]) # write bands for nband, data_band_raster_mmap_file in enumerate(self.output_bands): # load result raster saved in file with memmap (joblib dump) data_band_raster = load(data_band_raster_mmap_file, mmap_mode='r') outband = outRaster.GetRasterBand(nband + 1) outband.WriteArray(data_band_raster) #outband.WriteArray(sd.get_data_band(self.band)) outband.SetNoDataValue(self.nodata_value) #outband.FlushCache() # FlushCache cause WriteEncodedTile/Strip() failed # clean outband = None del data_band_raster shutil.rmtree(os.path.dirname(data_band_raster_mmap_file)) # set projection outRaster.SetGeoTransform( (originX, pixelWidth, 0, originY, 0, pixelHeight)) outRasterSRS = osr.SpatialReference() outRasterSRS.ImportFromWkt(gdal_data_band.GetProjectionRef()) outRaster.SetProjection(outRasterSRS.ExportToWkt()) # clean gdal_data_band = None geotransform = None outRaster = None
def get_data_band(self, band): """Return the raster of the data band for respective band of the file. :param band: band to process :type band: int :return: raster of the data band :rtype: ndarray """ # TODO: optimize/performance the table open/access in memory (pytables?) data_band_name = [ x for x in self.sub_datasets if 'b' + fix_zeros(band, 2) in x[1] ][0][0] gdal_data_band = gdal.Open(data_band_name, gdal.GA_ReadOnly) data_band_raster = gdal_data_band.ReadAsArray() del gdal_data_band return data_band_raster
def __init__(self, file, xml_file): """Initialize the class of MODIS products :param xml_file: path to input xml file :type xml_file: str :param file: path to input file :type file: str """ super().list.append(self) super().__init__(file) # load metadata tree = ET.parse(xml_file) self.satellite = list(tree.iter('PlatformShortName'))[0].text # Terra self.shortname = list(tree.iter('ShortName'))[0].text # MOD09A1 self.tile = list( tree.iter('LocalGranuleID'))[0].text.split('.')[2] # h10v07 # get the beginning date dt_d = [ int(x) for x in list(tree.iter('RangeBeginningDate'))[0].text.split('-') ] self.start_date = date(dt_d[0], dt_d[1], dt_d[2]) # calculate Julian date of the beginning date self.start_jday = self.start_date.timetuple().tm_yday # year and jday (ie 2015034), equal to filename string self.start_year_and_jday = "{0}{1}".format( self.start_date.year, fix_zeros(self.start_jday, 3)) # save in globals vars of class SatelliteData.satellite = self.satellite SatelliteData.shortname = self.shortname SatelliteData.tile = self.tile qc_success_set = self.set_quality_control_bands() self.make_qc = qc_success_set del tree
def get_quality_control_bands(self, band): return [ x for x in self.sub_datasets if 'b' + fix_zeros(band, 2) in x[1] ][0][0]
def get_nodata_value(self, band): data_band_name = [ x for x in self.sub_datasets if 'b' + fix_zeros(band, 2) in x[1] ][0][0] gdal_data_band = gdal.Open(data_band_name, gdal.GA_ReadOnly) return gdal_data_band.GetRasterBand(1).GetNoDataValue()
def get_total_pixels(self, band): data_band_name = [ x for x in self.sub_datasets if 'b' + fix_zeros(band, 2) in x[1] ][0][0] gdal_data_band = gdal.Open(data_band_name, gdal.GA_ReadOnly) return gdal_data_band.RasterXSize * gdal_data_band.RasterYSize
def save_statistics(self, output_dir): """Save statistics of invalid pixels in a image that show the time series of all invalid pixels of all filters as the result after apply the QC4SD """ # force matplotlib to not use any Xwindows backend. import matplotlib matplotlib.use('Agg') import matplotlib.ticker as mtick import matplotlib.pyplot as plt # path to save statistics path_stats = os.path.join(output_dir, self.output_filename.split('.tif')[0]) # if not os.path.isdir(path_stats): # os.makedirs(path_stats) ################################ # graph invalid pixels for the time series img_filename = os.path.join( output_dir, self.output_filename.split('.tif')[0] + "_stats.png") print( "Saving the image of statistics of invalid pixels in: {0}".format( os.path.basename(img_filename))) ################################ # prepare data all_filter_names = set() for sd_invalid_pixels in self.quality_control_statistics.values(): filters = sd_invalid_pixels['invalid_pixels'] # delete elements if the values are empty filters = {k: filters[k] for k in filters if filters[k]} # unpacking the dicts of all filters filters = [x for x in filters.values()] _tmp_dict = {} for filter in filters: _tmp_dict.update(filter) filters = _tmp_dict all_filter_names = all_filter_names | set(filters.keys()) all_filter_names = sorted(list(all_filter_names)) sd_names_sorted = sorted(self.quality_control_statistics.keys()) all_invalid_pixels = [] for sd_name in sd_names_sorted: filters = self.quality_control_statistics[sd_name][ 'invalid_pixels'] # delete elements if the values are empty filters = {k: filters[k] for k in filters if filters[k]} # unpacking the dicts of all filters filters = [x for x in filters.values()] _tmp_dict = {} for filter in filters: _tmp_dict.update(filter) filters = _tmp_dict sd_time_series = [ self.quality_control_statistics[sd_name] ['total_invalid_pixels'] ] sd_time_series += [ self.quality_control_statistics[sd_name]['nodata_pixels'] ] for filter_name in all_filter_names: if filter_name in filters: sd_time_series.append(filters[filter_name]) else: sd_time_series.append(float('nan')) all_invalid_pixels.append(sd_time_series) all_filter_names = ['total_invalid_pixels' ] + ['nodata_pixels'] + list(all_filter_names) #all_filter_names = [name.replace('_', ' ') for name in all_filter_names] ################################ # plot width = 10 + len(sd_names_sorted) * 0.4 if len(sd_names_sorted) == 1: width = 7 fig, ax = plt.subplots(1, 1, figsize=(width, 8), facecolor='white') ax.spines['top'].set_visible(False) ax.spines['bottom'].set_visible(False) ax.spines['right'].set_visible(False) ax.spines['left'].set_visible(False) ax.get_xaxis().tick_bottom() ax.get_yaxis().tick_left() plt.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='on', left='off', right='off', labelleft='on') all_invalid_pixels_T = list(map(list, zip(*all_invalid_pixels))) # delete all group of list that have only zeros, this is delete types of # invalid pixels that not filter any pixel in all image across the time delete_zeros_lists = [ idx for idx, values in enumerate(all_invalid_pixels_T) if [x for x in set(values) if not isnan(x)] == [0] ] delete_zeros_lists.reverse() for del_idx in delete_zeros_lists: del all_invalid_pixels_T[del_idx] del all_filter_names[del_idx] # rewrite list after clean all_invalid_pixels = list(map(list, zip(*all_invalid_pixels_T))) if not all_invalid_pixels: print( "\nWARNING: the invalid pixels is zero! nothing pixels was filtered.\n" ) return max_y = max([max(sub_l) for sub_l in all_invalid_pixels_T ]) # y max over all times # fix position for y label y_pos_label_fixed = deepcopy(all_invalid_pixels[-1]) # if any item in the last position is nan, put the last # valid value for this item for idx, y_pos in enumerate(y_pos_label_fixed): iter_pos = -1 if isnan(y_pos): while isnan(all_invalid_pixels[iter_pos][idx]): iter_pos += -1 y_pos_label_fixed[idx] = all_invalid_pixels[iter_pos][idx] # set initial value for repulsive_items_list repulsive_distance = max_y * 0.035 fix_list = True while fix_list: fix_list, repulsive_distance, y_pos_label_fixed = repulsive_items_list( y_pos_label_fixed, repulsive_distance) # define colors import matplotlib as mpl import matplotlib.cm as cm norm = mpl.colors.Normalize(vmin=0, vmax=len(all_invalid_pixels_T)) cmap = cm.Set1 m = cm.ScalarMappable(norm=norm, cmap=cmap) if len(sd_names_sorted) == 1: # for only one image for idx, line in enumerate(all_invalid_pixels_T): if idx == 0: plt.plot(0, line, 'ro', markersize=9, color=m.to_rgba(idx), linewidth=3.4, alpha=1) # put value of total invalid pixel for each x item (time) for x, y in zip(range(len(SatelliteData.list)), line): ax.text( x, y + max_y * 0.02, "{0}%".format( round( 100 * y / SatelliteData.list[idx].get_total_pixels( self.band), 2)), ha='center', va='bottom', color=m.to_rgba(idx), fontweight='bold', fontsize=12, alpha=1) else: plt.plot(0, line, 'ro', markersize=9, color=m.to_rgba(idx), linewidth=3, alpha=1) # y label of filter name plt.text(0.3, y_pos_label_fixed[idx], all_filter_names[idx], fontsize=12, weight='bold', color=m.to_rgba(idx), alpha=1) plt.xlim(-0.5, 0.5) plt.xticks(range(len(sd_names_sorted)), sd_names_sorted, rotation=90) plt.ylim(-max_y * 0.01, max_y + max_y * 0.07) plt.title( "Invalid pixels for {0} {1} in band {2}\nQC4SD - IDEAM".format( SatelliteData.tile, SatelliteData.shortname, fix_zeros(self.band, 2)), fontsize=18, weight='bold', color="#3A3A3A") plt.xlabel("Date", fontsize=14, weight='bold', color="#3A3A3A") plt.ylabel("Number of invalid pixels", fontsize=14, weight='bold', color="#3A3A3A") plt.tick_params(axis='both', which='major', labelsize=14, color="#3A3A3A") ax.grid(True, color='gray') ax.yaxis.set_major_formatter(mtick.FormatStrFormatter('%g')) fig.tight_layout() fig.subplots_adjust(right=0.6, left=0.4) else: for idx, line in enumerate(all_invalid_pixels_T): if idx == 0: plt.plot(line, color=m.to_rgba(idx), linewidth=3.4, alpha=1) # put value of total invalid pixel for each x item (time) for x, y in zip(range(len(SatelliteData.list)), line): ax.text( x, y + max_y * 0.02, "{0}%".format( round( 100 * y / SatelliteData.list[idx].get_total_pixels( self.band), 2)), ha='center', va='bottom', color=m.to_rgba(idx), fontweight='bold', fontsize=12, alpha=1) else: plt.plot(line, color=m.to_rgba(idx), linewidth=3, alpha=1) # y label of filter name plt.text(len(SatelliteData.list) - 1 + len(SatelliteData.list) * 0.02, y_pos_label_fixed[idx], all_filter_names[idx], fontsize=12, weight='bold', color=m.to_rgba(idx), alpha=1) plt.xlim( -len(SatelliteData.list) * 0.02, len(SatelliteData.list) - 1 + len(SatelliteData.list) * 0.02) plt.xticks(range(len(sd_names_sorted)), sd_names_sorted, rotation=90) plt.ylim(-max_y * 0.01, max_y + max_y * 0.07) plt.title( "Invalid pixels for {0} {1} in band {2}\nQC4SD - IDEAM".format( SatelliteData.tile, SatelliteData.shortname, fix_zeros(self.band, 2)), fontsize=18, weight='bold', color="#3A3A3A") plt.xlabel("Date", fontsize=14, weight='bold', color="#3A3A3A") plt.ylabel("Number of invalid pixels", fontsize=14, weight='bold', color="#3A3A3A") plt.tick_params(axis='both', which='major', labelsize=14, color="#3A3A3A") ax.grid(True, color='gray') ax.yaxis.set_major_formatter(mtick.FormatStrFormatter('%g')) fig.tight_layout() fig.subplots_adjust(right=1.02 - 3.6 / width) plt.savefig(img_filename, dpi=86) plt.close('all') # trim whitespace try: call([ "convert", img_filename, "-trim", "-bordercolor", "white", "-border", "8x8", "+repage", "-alpha", "off", img_filename ]) print('trim image successfully') except: pass