def main(ini_path, show_flag=False, overwrite_flag=True): """Generate Bokeh figures Bokeh issues: Adjust y range based on non-muted data https://stackoverflow.com/questions/43620837/how-to-get-bokeh-to-dynamically-adjust-y-range-when-panning Linked interactive legends so that there is only one legend for the gridplot Maybe hide or mute QA values above max (instead of filtering them in advance) Args: ini_path (str): show_flag (bool): if True, show the figures in the browser. Default is False. overwrite_flag (bool): if True, overwrite existing tables. Default is True (for now) """ logging.info('\nGenerate interactive timeseries figures') # Eventually read from INI plot_var_list = ['NDVI_TOA', 'ALBEDO_SUR', 'TS', 'NDWI_TOA', 'EVI_SUR'] # plot_var_list = [ # 'NDVI_TOA', 'ALBEDO_SUR', 'TS', 'NDWI_TOA', # 'CLOUD_SCORE', 'FMASK_PCT'] output_folder = 'figures' # Read config file ini = inputs.read(ini_path) inputs.parse_section(ini, section='INPUTS') inputs.parse_section(ini, section='ZONAL_STATS') inputs.parse_section(ini, section='SUMMARY') inputs.parse_section(ini, section='FIGURES') inputs.parse_section(ini, section='BEAMER') # Output paths output_ws = os.path.join(ini['SUMMARY']['output_ws'], output_folder) if not os.path.isdir(output_ws): os.makedirs(output_ws) # Start/end year year_list = list( range(ini['INPUTS']['start_year'], ini['INPUTS']['end_year'] + 1)) month_list = list( utils.wrapped_range(ini['INPUTS']['start_month'], ini['INPUTS']['end_month'], 1, 12)) doy_list = list( utils.wrapped_range(ini['INPUTS']['start_doy'], ini['INPUTS']['end_doy'], 1, 366)) # GRIDMET month range (default to water year) gridmet_start_month = ini['SUMMARY']['gridmet_start_month'] gridmet_end_month = ini['SUMMARY']['gridmet_end_month'] gridmet_months = list( utils.month_range(gridmet_start_month, gridmet_end_month)) logging.info('\nGridmet months: {}'.format(', '.join( map(str, gridmet_months)))) # Read in the zonal stats CSV logging.debug(' Reading zonal stats CSV file') input_df = pd.read_csv( os.path.join(ini['ZONAL_STATS']['output_ws'], ini['BEAMER']['output_name'])) logging.debug(input_df.head()) logging.debug(' Filtering Landsat dataframe') input_df = input_df[input_df['PIXEL_COUNT'] > 0] # # This assumes that there are L5/L8 images in the dataframe # if not input_df.empty: # max_pixel_count = max(input_df['PIXEL_COUNT']) # else: # max_pixel_count = 0 if ini['INPUTS']['fid_keep_list']: input_df = input_df[input_df['ZONE_FID'].isin( ini['INPUTS']['fid_keep_list'])] if ini['INPUTS']['fid_skip_list']: input_df = input_df[~input_df['ZONE_FID']. isin(ini['INPUTS']['fid_skip_list'])] if year_list: input_df = input_df[input_df['YEAR'].isin(year_list)] if month_list: input_df = input_df[input_df['MONTH'].isin(month_list)] if doy_list: input_df = input_df[input_df['DOY'].isin(doy_list)] if ini['INPUTS']['path_keep_list']: input_df = input_df[input_df['PATH'].isin( ini['INPUTS']['path_keep_list'])] if (ini['INPUTS']['row_keep_list'] and ini['INPUTS']['row_keep_list'] != ['XXX']): input_df = input_df[input_df['ROW'].isin( ini['INPUTS']['row_keep_list'])] # Assume the default is for these to be True and only filter if False if not ini['INPUTS']['landsat4_flag']: input_df = input_df[input_df['PLATFORM'] != 'LT04'] if not ini['INPUTS']['landsat5_flag']: input_df = input_df[input_df['PLATFORM'] != 'LT05'] if not ini['INPUTS']['landsat7_flag']: input_df = input_df[input_df['PLATFORM'] != 'LE07'] if not ini['INPUTS']['landsat8_flag']: input_df = input_df[input_df['PLATFORM'] != 'LC08'] if ini['INPUTS']['scene_id_keep_list']: # Replace XXX with primary ROW value for checking skip list SCENE_ID scene_id_df = pd.Series([ s.replace('XXX', '{:03d}'.format(int(r))) for s, r in zip(input_df['SCENE_ID'], input_df['ROW']) ]) input_df = input_df[scene_id_df.isin( ini['INPUTS']['scene_id_keep_list']).values] # This won't work: SCENE_ID have XXX but scene_id_skip_list don't # input_df = input_df[input_df['SCENE_ID'].isin( # ini['INPUTS']['scene_id_keep_list'])] if ini['INPUTS']['scene_id_skip_list']: # Replace XXX with primary ROW value for checking skip list SCENE_ID scene_id_df = pd.Series([ s.replace('XXX', '{:03d}'.format(int(r))) for s, r in zip(input_df['SCENE_ID'], input_df['ROW']) ]) input_df = input_df[np.logical_not( scene_id_df.isin(ini['INPUTS']['scene_id_skip_list']).values)] # This won't work: SCENE_ID have XXX but scene_id_skip_list don't # input_df = input_df[~input_df['SCENE_ID'].isin( # ini['INPUTS']['scene_id_skip_list'])] # Filter by QA/QC value if ini['SUMMARY']['max_qa'] >= 0 and not input_df.empty: logging.debug(' Maximum QA: {0}'.format(ini['SUMMARY']['max_qa'])) input_df = input_df[input_df['QA'] <= ini['SUMMARY']['max_qa']] # First filter by average cloud score if ini['SUMMARY']['max_cloud_score'] < 100 and not input_df.empty: logging.debug(' Maximum cloud score: {0}'.format( ini['SUMMARY']['max_cloud_score'])) input_df = input_df[ input_df['CLOUD_SCORE'] <= ini['SUMMARY']['max_cloud_score']] # Filter by Fmask percentage if ini['SUMMARY']['max_fmask_pct'] < 100 and not input_df.empty: input_df['FMASK_PCT'] = 100 * (input_df['FMASK_COUNT'] / input_df['FMASK_TOTAL']) logging.debug(' Max Fmask threshold: {}'.format( ini['SUMMARY']['max_fmask_pct'])) input_df = input_df[ input_df['FMASK_PCT'] <= ini['SUMMARY']['max_fmask_pct']] # Filter low count SLC-off images if ini['SUMMARY']['min_slc_off_pct'] > 0 and not input_df.empty: logging.debug(' Mininum SLC-off threshold: {}%'.format( ini['SUMMARY']['min_slc_off_pct'])) # logging.debug(' Maximum pixel count: {}'.format( # max_pixel_count)) slc_off_mask = ((input_df['PLATFORM'] == 'LE07') & ((input_df['YEAR'] >= 2004) | ((input_df['YEAR'] == 2003) & (input_df['DOY'] > 151)))) slc_off_pct = 100 * (input_df['PIXEL_COUNT'] / input_df['PIXEL_TOTAL']) # slc_off_pct = 100 * (input_df['PIXEL_COUNT'] / max_pixel_count) input_df = input_df[((slc_off_pct >= ini['SUMMARY']['min_slc_off_pct']) & slc_off_mask) | (~slc_off_mask)] if input_df.empty: logging.error(' Empty dataframe after filtering, exiting') return False # Process each zone separately logging.debug(input_df.head()) zone_name_list = sorted(list(set(input_df['ZONE_NAME'].values))) for zone_name in zone_name_list: logging.info('ZONE: {}'.format(zone_name)) # The names are currently stored in the CSV as spaces zone_output_name = zone_name.replace(' ', '_') zone_df = input_df[input_df['ZONE_NAME'] == zone_name] if zone_df.empty: logging.info(' Empty zone dataframe, skipping zone') continue # Output file paths output_doy_path = os.path.join( output_ws, '{}_timeseries_doy.html'.format(zone_output_name)) output_date_path = os.path.join( output_ws, '{}_timeseries_date.html'.format(zone_output_name)) # # Check for QA field # if 'QA' not in zone_df.columns.values: # # logging.warning( # # ' WARNING: QA field not present in CSV\n' # # ' To compute QA/QC values, please run "ee_summary_qaqc.py"\n' # # ' Script will continue with no QA/QC values') # zone_df['QA'] = 0 # # raw_input('ENTER') # # logging.error( # # '\nPlease run the "ee_summary_qaqc.py" script ' # # 'to compute QA/QC values\n') # # sys.exit() # Check that plot variables are present for plot_var in plot_var_list: if plot_var not in zone_df.columns.values: logging.error(' The variable {} does not exist in the ' 'dataframe'.format(plot_var)) sys.exit() # if ini['INPUTS']['scene_id_keep_list']: # # Replace XXX with primary ROW value for checking skip list SCENE_ID # scene_id_df = pd.Series([ # s.replace('XXX', '{:03d}'.format(int(r))) # for s, r in zip(zone_df['SCENE_ID'], zone_df['ROW'])]) # zone_df = zone_df[scene_id_df.isin( # ini['INPUTS']['scene_id_keep_list']).values] # # This won't work: SCENE_ID have XXX but scene_id_skip_list don't # # zone_df = zone_df[zone_df['SCENE_ID'].isin( # # ini['INPUTS']['scene_id_keep_list'])] # if ini['INPUTS']['scene_id_skip_list']: # # Replace XXX with primary ROW value for checking skip list SCENE_ID # scene_id_df = pd.Series([ # s.replace('XXX', '{:03d}'.format(int(r))) # for s, r in zip(zone_df['SCENE_ID'], zone_df['ROW'])]) # zone_df = zone_df[np.logical_not(scene_id_df.isin( # ini['INPUTS']['scene_id_skip_list']).values)] # # This won't work: SCENE_ID have XXX but scene_id_skip_list don't # # zone_df = zone_df[np.logical_not(zone_df['SCENE_ID'].isin( # # ini['INPUTS']['scene_id_skip_list']))] # Compute colors for each QA value logging.debug(' Building column data source') qa_values = sorted(list(set(zone_df['QA'].values))) colors = { qa: "#%02x%02x%02x" % (int(r), int(g), int(b)) for qa, ( r, g, b, _) in zip(qa_values, 255 * cm.viridis(mpl.colors.Normalize()(qa_values))) } logging.debug(' QA values: {}'.format(', '.join(map(str, qa_values)))) # Unpack the data by QA type to support interactive legends sources = dict() for qa_value in qa_values: qa_df = zone_df[zone_df['QA'] == qa_value] qa_data = { 'INDEX': list(range(len(qa_df.index))), 'PLATFORM': qa_df['PLATFORM'], 'DATE': pd.to_datetime(qa_df['DATE']), 'DATE_STR': pd.to_datetime( qa_df['DATE']).map(lambda x: x.strftime('%Y-%m-%d')), 'DOY': qa_df['DOY'].values, 'QA': qa_df['QA'].values, 'COLOR': [colors[qa] for qa in qa_df['QA'].values] } for plot_var in plot_var_list: if plot_var in qa_df.columns.values: qa_data.update({plot_var: qa_df[plot_var].values}) sources[qa_value] = bokeh.models.ColumnDataSource(qa_data) tooltips = [("LANDSAT", "@PLATFORM"), ("DATE", "@TIME"), ("DOY", "@DOY")] # Selection hover_circle = Circle(fill_color='#ff0000', line_color='#ff0000') selected_circle = Circle(fill_color='COLOR', line_color='COLOR') nonselected_circle = Circle(fill_color='#aaaaaa', line_color='#aaaaaa') # Plot the data by DOY logging.debug(' Building DOY timeseries figure') if os.path.isfile(output_doy_path): os.remove(output_doy_path) output_file(output_doy_path, title=zone_name) figure_args = dict( plot_width=750, plot_height=250, title=None, tools="xwheel_zoom,xpan,xbox_zoom,reset,box_select", # tools="xwheel_zoom,xpan,xbox_zoom,reset,tap", active_scroll="xwheel_zoom") plot_args = dict(size=4, alpha=0.9, color='COLOR') if ini['SUMMARY']['max_qa'] > 0: plot_args['legend'] = 'QA' figures = [] for plot_i, plot_var in enumerate(plot_var_list): if plot_i == 0: f = figure( # x_range=Range1d(1, 366, bounds=(1, 366)), y_axis_label=plot_var, **figure_args) else: f = figure(x_range=f.x_range, y_axis_label=plot_var, **figure_args) for qa, source in sorted(sources.items()): r = f.circle('DOY', plot_var, source=source, **plot_args) r.hover_glyph = hover_circle r.selection_glyph = selected_circle r.nonselection_glyph = nonselected_circle r.muted_glyph = nonselected_circle # DEADBEEF - This will display high QA points as muted # if qa > ini['SUMMARY']['max_qa']: # r.muted = True # # r.visible = False f.add_tools(bokeh.models.HoverTool(tooltips=tooltips)) # if ini['SUMMARY']['max_qa'] > 0: f.legend.location = "top_left" f.legend.click_policy = "hide" # f.legend.click_policy = "mute" f.legend.orientation = "horizontal" figures.append(f) # Try to not allow more than 4 plots in a column p = gridplot(figures, ncols=len(plot_var_list) // 3, sizing_mode='stretch_both') if show_flag: show(p) save(p) # Plot the data by DATE logging.debug(' Building date timeseries figure') if os.path.isfile(output_date_path): os.remove(output_date_path) output_file(output_date_path, title=zone_name) figure_args = dict( plot_width=750, plot_height=250, title=None, tools="xwheel_zoom,xpan,xbox_zoom,reset,box_select", # tools="xwheel_zoom,xpan,xbox_zoom,reset,tap", active_scroll="xwheel_zoom", x_axis_type="datetime", ) plot_args = dict(size=4, alpha=0.9, color='COLOR') if ini['SUMMARY']['max_qa'] > 0: plot_args['legend'] = 'QA' figures = [] for plot_i, plot_var in enumerate(plot_var_list): if plot_i == 0: f = figure( # x_range=Range1d(x_limit[0], x_limit[1], bounds=x_limit), y_axis_label=plot_var, **figure_args) else: f = figure(x_range=f.x_range, y_axis_label=plot_var, **figure_args) if plot_var == 'TS': f.y_range.bounds = (270, None) for qa, source in sorted(sources.items()): r = f.circle('DATE', plot_var, source=source, **plot_args) r.hover_glyph = hover_circle r.selection_glyph = selected_circle r.nonselection_glyph = nonselected_circle r.muted_glyph = nonselected_circle # DEADBEEF - This will display high QA points as muted # if qa > ini['SUMMARY']['max_qa']: # r.muted = True # # r.visible = False f.add_tools(bokeh.models.HoverTool(tooltips=tooltips)) # if ini['SUMMARY']['max_qa'] > 0: f.legend.location = "top_left" f.legend.click_policy = "hide" # f.legend.click_policy = "mute" f.legend.orientation = "horizontal" figures.append(f) # Try to not allow more than 4 plots in a column p = gridplot(figures, ncols=len(plot_var_list) // 3, sizing_mode='stretch_both') if show_flag: show(p) save(p) # Pause after each iteration if show is True if show_flag: input('Press ENTER to continue')
def main(ini_path, show_flag=False, overwrite_flag=False): """Generate Beamer ETg summary figures Args: ini_path (str): show_flag (bool): if True, show the figures in the browser. Default is False. overwrite_flag (bool): if True, overwrite existing figures Default is True (for now) """ logging.info('\nGenerate Beamer ETg summary figures') ncolors = [ '#348ABD', '#7A68A6', '#A60628', '#467821', '#CF4457', '#188487', '#E24A33'] xtick_fs = 8 ytick_fs = 8 xlabel_fs = 8 ylabel_fs = 8 ms = 2 figsize = (3.0, 2.5) output_folder = 'figures' # For unit conversion eto_fields = [ 'ETG_MEAN', 'ETG_LPI', 'ETG_UPI', 'ETG_LCI', 'ETG_UCI', 'ET_MEAN', 'ET_LPI', 'ET_UPI', 'ET_LCI', 'ET_UCI', 'WY_ETO'] ppt_fields = ['WY_PPT'] # Read config file ini = inputs.read(ini_path) inputs.parse_section(ini, section='INPUTS') inputs.parse_section(ini, section='ZONAL_STATS') inputs.parse_section(ini, section='SUMMARY') inputs.parse_section(ini, section='FIGURES') inputs.parse_section(ini, section='BEAMER') # Output paths output_ws = os.path.join( ini['SUMMARY']['output_ws'], output_folder) if not os.path.isdir(output_ws): os.makedirs(output_ws) # Start/end year year_list = list(range( ini['INPUTS']['start_year'], ini['INPUTS']['end_year'] + 1)) month_list = list(utils.wrapped_range( ini['INPUTS']['start_month'], ini['INPUTS']['end_month'], 1, 12)) doy_list = list(utils.wrapped_range( ini['INPUTS']['start_doy'], ini['INPUTS']['end_doy'], 1, 366)) # GRIDMET month range (default to water year) gridmet_start_month = ini['SUMMARY']['gridmet_start_month'] gridmet_end_month = ini['SUMMARY']['gridmet_end_month'] gridmet_months = list(utils.month_range( gridmet_start_month, gridmet_end_month)) logging.info('\nGridmet months: {}'.format( ', '.join(map(str, gridmet_months)))) # Read in the zonal stats CSV logging.debug(' Reading zonal stats CSV file') input_df = pd.read_csv(os.path.join( ini['ZONAL_STATS']['output_ws'], ini['BEAMER']['output_name'])) logging.debug(input_df.head()) logging.debug(' Filtering Landsat dataframe') input_df = input_df[input_df['PIXEL_COUNT'] > 0] # # This assumes that there are L5/L8 images in the dataframe # if not input_df.empty: # max_pixel_count = max(input_df['PIXEL_COUNT']) # else: # max_pixel_count = 0 if ini['INPUTS']['fid_keep_list']: input_df = input_df[input_df['ZONE_FID'].isin( ini['INPUTS']['fid_keep_list'])] if ini['INPUTS']['fid_skip_list']: input_df = input_df[~input_df['ZONE_FID'].isin( ini['INPUTS']['fid_skip_list'])] if year_list: input_df = input_df[input_df['YEAR'].isin(year_list)] if month_list: input_df = input_df[input_df['MONTH'].isin(month_list)] if doy_list: input_df = input_df[input_df['DOY'].isin(doy_list)] if ini['INPUTS']['path_keep_list']: input_df = input_df[ input_df['PATH'].isin(ini['INPUTS']['path_keep_list'])] if (ini['INPUTS']['row_keep_list'] and ini['INPUTS']['row_keep_list'] != ['XXX']): input_df = input_df[ input_df['ROW'].isin(ini['INPUTS']['row_keep_list'])] # Assume the default is for these to be True and only filter if False if not ini['INPUTS']['landsat4_flag']: input_df = input_df[input_df['PLATFORM'] != 'LT04'] if not ini['INPUTS']['landsat5_flag']: input_df = input_df[input_df['PLATFORM'] != 'LT05'] if not ini['INPUTS']['landsat7_flag']: input_df = input_df[input_df['PLATFORM'] != 'LE07'] if not ini['INPUTS']['landsat8_flag']: input_df = input_df[input_df['PLATFORM'] != 'LC08'] if ini['INPUTS']['scene_id_keep_list']: # Replace XXX with primary ROW value for checking skip list SCENE_ID scene_id_df = pd.Series([ s.replace('XXX', '{:03d}'.format(int(r))) for s, r in zip(input_df['SCENE_ID'], input_df['ROW'])]) input_df = input_df[scene_id_df.isin( ini['INPUTS']['scene_id_keep_list']).values] # This won't work: SCENE_ID have XXX but scene_id_skip_list don't # input_df = input_df[input_df['SCENE_ID'].isin( # ini['INPUTS']['scene_id_keep_list'])] if ini['INPUTS']['scene_id_skip_list']: # Replace XXX with primary ROW value for checking skip list SCENE_ID scene_id_df = pd.Series([ s.replace('XXX', '{:03d}'.format(int(r))) for s, r in zip(input_df['SCENE_ID'], input_df['ROW'])]) input_df = input_df[np.logical_not(scene_id_df.isin( ini['INPUTS']['scene_id_skip_list']).values)] # This won't work: SCENE_ID have XXX but scene_id_skip_list don't # input_df = input_df[~input_df['SCENE_ID'].isin( # ini['INPUTS']['scene_id_skip_list'])] # Filter by QA/QC value if ini['SUMMARY']['max_qa'] >= 0 and not input_df.empty: logging.debug(' Maximum QA: {0}'.format( ini['SUMMARY']['max_qa'])) input_df = input_df[input_df['QA'] <= ini['SUMMARY']['max_qa']] # First filter by average cloud score if ini['SUMMARY']['max_cloud_score'] < 100 and not input_df.empty: logging.debug(' Maximum cloud score: {0}'.format( ini['SUMMARY']['max_cloud_score'])) input_df = input_df[ input_df['CLOUD_SCORE'] <= ini['SUMMARY']['max_cloud_score']] # Filter by Fmask percentage if ini['SUMMARY']['max_fmask_pct'] < 100 and not input_df.empty: input_df['FMASK_PCT'] = 100 * ( input_df['FMASK_COUNT'] / input_df['FMASK_TOTAL']) logging.debug(' Max Fmask threshold: {}'.format( ini['SUMMARY']['max_fmask_pct'])) input_df = input_df[ input_df['FMASK_PCT'] <= ini['SUMMARY']['max_fmask_pct']] # Filter low count SLC-off images if ini['SUMMARY']['min_slc_off_pct'] > 0 and not input_df.empty: logging.debug(' Mininum SLC-off threshold: {}%'.format( ini['SUMMARY']['min_slc_off_pct'])) # logging.debug(' Maximum pixel count: {}'.format( # max_pixel_count)) slc_off_mask = ( (input_df['PLATFORM'] == 'LE07') & ((input_df['YEAR'] >= 2004) | ((input_df['YEAR'] == 2003) & (input_df['DOY'] > 151)))) slc_off_pct = 100 * (input_df['PIXEL_COUNT'] / input_df['PIXEL_TOTAL']) # slc_off_pct = 100 * (input_df['PIXEL_COUNT'] / max_pixel_count) input_df = input_df[ ((slc_off_pct >= ini['SUMMARY']['min_slc_off_pct']) & slc_off_mask) | (~slc_off_mask)] if input_df.empty: logging.error(' Empty dataframe after filtering, exiting') return False # Process each zone separately logging.debug(input_df.head()) zone_name_list = sorted(list(set(input_df['ZONE_NAME'].values))) for zone_name in zone_name_list: logging.info('ZONE: {}'.format(zone_name)) # The names are currently stored in the CSV with spaces zone_output_name = zone_name.replace(' ', '_') zone_df = input_df[input_df['ZONE_NAME'] == zone_name] if zone_df.empty: logging.info(' Empty zone dataframe, skipping zone') continue logging.debug(' Computing annual summaries') annual_df = zone_df \ .groupby(['ZONE_NAME', 'YEAR']) \ .agg({ 'PIXEL_COUNT': ['count', 'mean'], 'PIXEL_TOTAL': ['mean'], 'FMASK_COUNT': 'mean', 'FMASK_TOTAL': 'mean', 'CLOUD_SCORE': 'mean', 'ETSTAR_COUNT': 'mean', 'NDVI_TOA': 'mean', 'NDWI_TOA': 'mean', 'ALBEDO_SUR': 'mean', 'TS': 'mean', 'EVI_SUR': 'mean', 'ETSTAR_MEAN': 'mean', 'ETG_MEAN': 'mean', 'ETG_LPI': 'mean', 'ETG_UPI': 'mean', 'ETG_LCI': 'mean', 'ETG_UCI': 'mean', 'ET_MEAN': 'mean', 'ET_LPI': 'mean', 'ET_UPI': 'mean', 'ET_LCI': 'mean', 'ET_UCI': 'mean', 'WY_ETO': 'mean', 'WY_PPT': 'mean' }) annual_df.columns = annual_df.columns.map('_'.join) annual_df = annual_df.rename(columns={ 'PIXEL_COUNT_count': 'SCENE_COUNT', 'PIXEL_COUNT_mean': 'PIXEL_COUNT'}) annual_df.rename( columns=lambda x: str(x).replace('_mean', ''), inplace=True) annual_df['SCENE_COUNT'] = annual_df['SCENE_COUNT'].astype(np.int) annual_df['PIXEL_COUNT'] = annual_df['PIXEL_COUNT'].astype(np.int) annual_df['PIXEL_TOTAL'] = annual_df['PIXEL_TOTAL'].astype(np.int) annual_df['FMASK_COUNT'] = annual_df['FMASK_COUNT'].astype(np.int) annual_df['FMASK_TOTAL'] = annual_df['FMASK_TOTAL'].astype(np.int) annual_df['ETSTAR_COUNT'] = annual_df['ETSTAR_COUNT'].astype(np.int) annual_df = annual_df.reset_index() # Convert ETo units if (ini['BEAMER']['eto_units'] == 'mm' and ini['FIGURES']['eto_units'] == 'mm'): pass elif (ini['BEAMER']['eto_units'] == 'mm' and ini['FIGURES']['eto_units'] == 'in'): annual_df[eto_fields] /= (25.4) elif (ini['BEAMER']['eto_units'] == 'mm' and ini['FIGURES']['eto_units'] == 'ft'): annual_df[eto_fields] /= (12 * 25.4) else: logging.error( ('\nERROR: Input units {} and output units {} are not ' + 'currently supported, exiting').format( ini['BEAMER']['eto_units'], ini['FIGURES']['eto_units'])) sys.exit() # Convert PPT units if (ini['BEAMER']['ppt_units'] == 'mm' and ini['FIGURES']['ppt_units'] == 'mm'): pass elif (ini['BEAMER']['ppt_units'] == 'mm' and ini['FIGURES']['ppt_units'] == 'in'): annual_df[ppt_fields] /= (25.4) elif (ini['BEAMER']['ppt_units'] == 'mm' and ini['FIGURES']['ppt_units'] == 'ft'): annual_df[ppt_fields] /= (12 * 25.4) else: logging.error( ('\nERROR: Input units {} and output units {} are not ' 'currently supported, exiting').format( ini['BEAMER']['ppt_units'], ini['FIGURES']['ppt_units'])) sys.exit() logging.debug(' Generating figures') zone_df = annual_df[annual_df['ZONE_NAME'] == zone_name] year_min, year_max = min(zone_df['YEAR']), max(zone_df['YEAR']) # Set default PPT min/max scaling ppt_min = 0 if ini['FIGURES']['ppt_units'] == 'mm': ppt_max = 100 * math.ceil((max(zone_df['WY_PPT']) + 100) / 100) elif ini['FIGURES']['ppt_units'] == 'ft': ppt_max = 0.2 * math.ceil((max(zone_df['WY_PPT']) + 0.1) / 0.2) else: ppt_max = 1.2 * max(zone_df['WY_PPT']) logging.debug(' EVI vs PPT') figure_path = os.path.join( output_ws, '{}_evi.png'.format(zone_output_name)) fig = plt.figure(figsize=figsize) ax1 = fig.add_axes([0.20, 0.21, 0.65, 0.75]) ax1.set_xlabel('Year', fontsize=xlabel_fs) ax2 = ax1.twinx() ax1.plot( zone_df['YEAR'].values, zone_df['WY_PPT'], marker='o', c='0.5', ms=ms, label='WY PPT') ax1.yaxis.tick_right() ax1.yaxis.set_label_position("right") ax1.set_xlim([year_min - 1, year_max + 1]) ax1.set_ylim([ppt_min, ppt_max]) ax1.tick_params(axis='y', labelsize=ytick_fs) ax1.tick_params(axis='x', labelsize=xtick_fs) ax1.tick_params(axis='x', which='both', top='off') ax1.xaxis.set_minor_locator(MultipleLocator(1)) for tick in ax1.get_xticklabels(): tick.set_rotation(45) tick.set_ha('right') ax1.set_ylabel( 'PPT [{}/yr]'.format(ini['FIGURES']['ppt_units']), fontsize=ylabel_fs) ax2.plot( zone_df['YEAR'].values, zone_df['EVI_SUR'].values, marker='o', c=ncolors[0], ms=ms, label='EVI') ax1.plot(0, 0, marker='o', c=ncolors[0], ms=ms, label='EVI') ax2.yaxis.tick_left() ax2.yaxis.set_label_position("left") ax2.set_ylim([ 0.05 * math.floor((min(zone_df['EVI_SUR']) - 0.01) / 0.05), 0.05 * math.ceil((max(zone_df['EVI_SUR']) + 0.01) / 0.05)]) ax2.tick_params(axis='y', labelsize=ytick_fs) ax2.set_ylabel('EVI [dimensionless]', fontsize=ylabel_fs) ax1.legend( loc='upper right', frameon=False, fontsize=6, numpoints=1) if overwrite_flag or not os.path.isfile(figure_path): plt.savefig(figure_path, dpi=300) plt.close() del fig, ax1, ax2 logging.debug(' ETo vs PPT') figure_path = os.path.join( output_ws, '{}_eto.png'.format(zone_output_name)) fig = plt.figure(figsize=figsize) ax1 = fig.add_axes([0.18, 0.21, 0.67, 0.75]) ax1.set_xlabel('Year', fontsize=xlabel_fs) ax2 = ax1.twinx() ax1.plot( zone_df['YEAR'].values, zone_df['WY_PPT'], marker='o', c='0.5', ms=ms, label='WY PPT') ax1.yaxis.tick_right() ax1.yaxis.set_label_position("right") ax1.set_xlim([year_min - 1, year_max + 1]) ax1.set_ylim([ppt_min, ppt_max]) ax1.tick_params(axis='y', labelsize=ytick_fs) ax1.tick_params(axis='x', labelsize=xtick_fs) ax1.tick_params(axis='x', which='both', top='off') ax1.xaxis.set_minor_locator(MultipleLocator(1)) for tick in ax1.get_xticklabels(): tick.set_rotation(45) tick.set_ha('right') ax1.set_ylabel( 'PPT [{}/yr]'.format(ini['FIGURES']['ppt_units']), fontsize=ylabel_fs) ax2.plot( zone_df['YEAR'].values, zone_df['WY_ETO'].values, marker='o', c=ncolors[1], ms=ms, label='ETo') ax1.plot(0, 0, marker='o', c=ncolors[1], ms=ms, label='ETo') ax2.yaxis.tick_left() ax2.yaxis.set_label_position("left") ax2.set_ylim([ max(0, 0.9 * min(zone_df['WY_ETO'])), 1.1 * max(zone_df['WY_ETO'])]) # ax2.set_ylim([ # max(0, 100 * math.floor((min(zone_df['WY_ETO']) - 100) / 100)), # 100 * math.ceil((max(zone_df['WY_ETO']) + 100) / 100)]) ax2.tick_params(axis='y', labelsize=ytick_fs) ax2.set_ylabel( 'ETo [{}/yr]'.format(ini['FIGURES']['eto_units']), fontsize=ylabel_fs) ax1.legend(loc='upper right', frameon=False, fontsize=6, numpoints=1) if overwrite_flag or not os.path.isfile(figure_path): plt.savefig(figure_path, dpi=300) plt.close() del fig, ax1, ax2 logging.debug(' ET vs PPT') figure_path = os.path.join( output_ws, '{}_et.png'.format(zone_output_name)) fig = plt.figure(figsize=figsize) ax1 = fig.add_axes([0.18, 0.21, 0.67, 0.75]) ax1.set_xlabel('Year', fontsize=xlabel_fs) ax2 = ax1.twinx() ax1.plot( zone_df['YEAR'].values, zone_df['WY_PPT'], marker='o', c='0.5', ms=ms, label='WY PPT') ax1.yaxis.tick_right() ax1.yaxis.set_label_position("right") ax1.set_xlim([year_min - 1, year_max + 1]) ax1.set_ylim([ppt_min, ppt_max]) ax1.tick_params(axis='y', labelsize=ytick_fs) ax1.tick_params(axis='x', labelsize=xtick_fs) ax1.tick_params(axis='x', which='both', top='off') ax1.xaxis.set_minor_locator(MultipleLocator(1)) for tick in ax1.get_xticklabels(): tick.set_rotation(45) tick.set_ha('right') ax1.set_ylabel( 'PPT [{}/yr]'.format(ini['FIGURES']['ppt_units']), fontsize=ylabel_fs) ax2.plot( zone_df['YEAR'].values, zone_df['ET_UCI'].values, marker='', c=ncolors[2], alpha=0.5, lw=0.75) ax2.plot( zone_df['YEAR'].values, zone_df['ET_LCI'].values, marker='', c=ncolors[2], alpha=0.5, lw=0.75) ax2.plot( zone_df['YEAR'].values, zone_df['ET_MEAN'].values, marker='o', c=ncolors[2], ms=ms, label='ET') ax1.plot(0, 0, marker='o', c=ncolors[2], ms=ms, label='ET') ax2.yaxis.tick_left() ax2.yaxis.set_label_position("left") ax2.set_ylim([ max(0, 0.9 * min(zone_df['ET_LCI'])), 1.1 * max(zone_df['ET_UCI'])]) # ax2.set_ylim([ # max(0, 100 * math.floor((min(zone_df['ET_MEAN']) - 100) / 100)), # 100 * math.ceil((max(zone_df['ET_MEAN']) + 100) / 100)]) ax2.tick_params(axis='y', labelsize=ytick_fs) ax2.set_ylabel( 'ET [{}/yr]'.format(ini['FIGURES']['eto_units']), fontsize=ylabel_fs) ax1.legend(loc='upper right', frameon=False, fontsize=6, numpoints=1) if overwrite_flag or not os.path.isfile(figure_path): plt.savefig(figure_path, dpi=300) plt.close() del fig, ax1, ax2 logging.debug(' ETg vs PPT') figure_path = os.path.join( output_ws, '{}_etg.png'.format(zone_output_name)) fig = plt.figure(figsize=figsize) ax1 = fig.add_axes([0.18, 0.21, 0.67, 0.75]) ax1.set_xlabel('Year', fontsize=xlabel_fs) ax2 = ax1.twinx() ax1.plot( zone_df['YEAR'].values, zone_df['WY_PPT'], marker='o', c='0.5', ms=ms, label='WY PPT') ax1.yaxis.tick_right() ax1.yaxis.set_label_position("right") ax1.set_xlim([year_min - 1, year_max + 1]) ax1.set_ylim([ppt_min, ppt_max]) ax1.tick_params(axis='y', labelsize=ytick_fs) ax1.tick_params(axis='x', labelsize=xtick_fs) ax1.tick_params(axis='x', which='both', top='off') ax1.xaxis.set_minor_locator(MultipleLocator(1)) for tick in ax1.get_xticklabels(): tick.set_rotation(45) tick.set_ha('right') ax1.set_ylabel( 'PPT [{}/yr]'.format(ini['FIGURES']['ppt_units']), fontsize=ylabel_fs) ax2.plot( zone_df['YEAR'].values, zone_df['ETG_UCI'].values, marker='', c=ncolors[3], alpha=0.5, lw=0.75) ax2.plot( zone_df['YEAR'].values, zone_df['ETG_LCI'].values, marker='', c=ncolors[3], alpha=0.5, lw=0.75) ax2.plot( zone_df['YEAR'].values, zone_df['ETG_MEAN'].values, marker='o', c=ncolors[3], ms=ms, label='ETg') ax1.plot(0, 0, marker='o', c=ncolors[3], ms=ms, label='ETg') ax2.yaxis.tick_left() ax2.yaxis.set_label_position("left") ax2.set_ylim([ max(0, 0.9 * min(zone_df['ETG_LCI'])), 1.1 * max(zone_df['ETG_UCI'])]) # ax2.set_ylim([ # max(0, 100 * math.floor((min(zone_df['ETG_MEAN']) - 100) / 100)), # 100 * math.ceil((max(zone_df['ETG_MEAN']) + 100) / 100)]) ax2.tick_params(axis='y', labelsize=ytick_fs) ax2.set_ylabel( 'ETg [{}/yr]'.format(ini['FIGURES']['eto_units']), fontsize=ylabel_fs) ax1.legend(loc='upper right', frameon=False, fontsize=6, numpoints=1) if overwrite_flag or not os.path.isfile(figure_path): plt.savefig(figure_path, dpi=300) plt.close() del fig, ax1, ax2 logging.debug(' Complimentary') figure_path = os.path.join( output_ws, '{}_complimentary.png'.format(zone_output_name)) fig = plt.figure(figsize=(3, 2.5)) ax = fig.add_axes([0.18, 0.16, 0.78, 0.80]) # ax = fig.add_axes([0.18, 0.21, 0.67, 0.70]) ax.plot( zone_df['WY_PPT'].values, zone_df['WY_ETO'].values, linestyle='', marker='o', c=ncolors[1], ms=3, label='ETo') ax.plot( zone_df['WY_PPT'].values, zone_df['ET_MEAN'].values, linestyle='', marker='o', c=ncolors[2], ms=3, label='ET') # xmax = 100 * math.ceil(max(zone_df['WY_PPT']) / 100) # ymax = 200 * math.ceil((max(zone_df['WY_ETO']) + 200) / 200) ax.set_xlim([ppt_min, ppt_max]) ax.set_ylim([0, 1.2 * max(zone_df['WY_ETO'])]) ax.tick_params(axis='y', labelsize=ytick_fs) ax.tick_params(axis='x', labelsize=xtick_fs) ax.tick_params(axis='x', which='both', top='off') ax.tick_params(axis='y', which='both', right='off') ax.set_xlabel('PPT [{}/yr]'.format( ini['FIGURES']['ppt_units']), fontsize=xlabel_fs) ax.set_ylabel('ET and ETo [{}/yr]'.format( ini['FIGURES']['eto_units']), fontsize=ylabel_fs) ax.legend(loc='upper right', frameon=False, fontsize=6, numpoints=1) if overwrite_flag or not os.path.isfile(figure_path): plt.savefig(figure_path, dpi=300) plt.close() del fig, ax
def main(ini_path, overwrite_flag=True): """Generate Beamer ETg summary tables Args: ini_path (str): overwrite_flag (bool): if True, overwrite existing tables Default is True (for now) """ logging.info('\nGenerate Beamer ETg summary tables') # # Eventually get from INI (like ini['BEAMER']['landsat_products']) # daily_fields = [ # 'ZONE_NAME', 'ZONE_FID', 'DATE', 'SCENE_ID', 'PLATFORM', 'PATH', 'ROW', # 'YEAR', 'MONTH', 'DAY', 'DOY', 'WATER_YEAR', # 'PIXEL_COUNT', 'ETSTAR_COUNT', # 'NDVI_TOA', 'NDWI_TOA', 'ALBEDO_SUR', 'TS', 'EVI_SUR', 'ETSTAR_MEAN', # 'ETG_MEAN', 'ETG_LPI', 'ETG_UPI', 'ETG_LCI', 'ETG_UCI', # 'ET_MEAN', 'ET_LPI', 'ET_UPI', 'ET_LCI', 'ET_UCI', # 'ETO', 'PPT'] # annual_fields = [ # 'SCENE_COUNT', 'PIXEL_COUNT', 'ETSTAR_COUNT', # 'NDVI_TOA', 'NDWI_TOA', 'ALBEDO_SUR', 'TS', # 'EVI_SUR_MEAN', 'EVI_SUR_MEDIAN', 'EVI_SUR_MIN', 'EVI_SUR_MAX', # 'ETSTAR_MEAN', # 'ETG_MEAN', 'ETG_LPI', 'ETG_UPI', 'ETG_LCI', 'ETG_UCI', # 'ET_MEAN', 'ET_LPI', 'ET_UPI', 'ET_LCI', 'ET_UCI', # 'ETO', 'PPT'] # For unit conversion eto_fields = [ 'ETG_MEAN', 'ETG_LPI', 'ETG_UPI', 'ETG_LCI', 'ETG_UCI', 'ET_MEAN', 'ET_LPI', 'ET_UPI', 'ET_LCI', 'ET_UCI', 'ETO' ] ppt_fields = ['PPT'] # Read config file ini = inputs.read(ini_path) inputs.parse_section(ini, section='INPUTS') inputs.parse_section(ini, section='ZONAL_STATS') inputs.parse_section(ini, section='BEAMER') inputs.parse_section(ini, section='SUMMARY') inputs.parse_section(ini, section='TABLES') # Hardcode GRIDMET month range to the water year ini['SUMMARY']['gridmet_start_month'] = 10 ini['SUMMARY']['gridmet_end_month'] = 9 # Output paths output_daily_path = os.path.join( ini['SUMMARY']['output_ws'], ini['BEAMER']['output_name'].replace('.csv', '_daily.xlsx')) output_annual_path = os.path.join( ini['SUMMARY']['output_ws'], ini['BEAMER']['output_name'].replace('.csv', '_annual.xlsx')) # Check if files already exist if overwrite_flag: if os.path.isfile(output_daily_path): os.remove(output_daily_path) if os.path.isfile(output_annual_path): os.remove(output_annual_path) else: if (os.path.isfile(output_daily_path) and os.path.isfile(output_annual_path)): logging.info('\nOutput files already exist and ' 'overwrite is False, exiting') return True # Start/end year year_list = list( range(ini['INPUTS']['start_year'], ini['INPUTS']['end_year'] + 1)) month_list = list( utils.wrapped_range(ini['INPUTS']['start_month'], ini['INPUTS']['end_month'], 1, 12)) doy_list = list( utils.wrapped_range(ini['INPUTS']['start_doy'], ini['INPUTS']['end_doy'], 1, 366)) # GRIDMET month range (default to water year) gridmet_start_month = ini['SUMMARY']['gridmet_start_month'] gridmet_end_month = ini['SUMMARY']['gridmet_end_month'] gridmet_months = list( utils.month_range(gridmet_start_month, gridmet_end_month)) logging.info('\nGridmet months: {}'.format(', '.join( map(str, gridmet_months)))) # Get ee features from shapefile zone_geom_list = gdc.shapefile_2_geom_list_func( ini['INPUTS']['zone_shp_path'], zone_field=ini['INPUTS']['zone_field'], reverse_flag=False) # Filter features by FID before merging geometries if ini['INPUTS']['fid_keep_list']: zone_geom_list = [ zone_obj for zone_obj in zone_geom_list if zone_obj[0] in ini['INPUTS']['fid_keep_list'] ] if ini['INPUTS']['fid_skip_list']: zone_geom_list = [ zone_obj for zone_obj in zone_geom_list if zone_obj[0] not in ini['INPUTS']['fid_skip_list'] ] # # Filter features by FID before merging geometries # if ini['INPUTS']['fid_keep_list']: # landsat_df = landsat_df[landsat_df['ZONE_FID'].isin( # ini['INPUTS']['fid_keep_list'])] # if ini['INPUTS']['fid_skip_list']: # landsat_df = landsat_df[~landsat_df['ZONE_FID'].isin( # ini['INPUTS']['fid_skip_list'])] logging.info('\nProcessing zones') zone_df_dict = {} for zone_fid, zone_name, zone_json in zone_geom_list: zone_name = zone_name.replace(' ', '_') logging.info('ZONE: {} (FID: {})'.format(zone_name, zone_fid)) zone_stats_ws = os.path.join(ini['ZONAL_STATS']['output_ws'], zone_name) if not os.path.isdir(zone_stats_ws): logging.debug( ' Folder {} does not exist, skipping'.format(zone_stats_ws)) continue # Input paths landsat_daily_path = os.path.join( zone_stats_ws, '{}_landsat_daily.csv'.format(zone_name)) gridmet_daily_path = os.path.join( zone_stats_ws, '{}_gridmet_daily.csv'.format(zone_name)) gridmet_monthly_path = os.path.join( zone_stats_ws, '{}_gridmet_monthly.csv'.format(zone_name)) if not os.path.isfile(landsat_daily_path): logging.error(' Landsat daily CSV does not exist, skipping zone') continue elif (not os.path.isfile(gridmet_daily_path) and not os.path.isfile(gridmet_monthly_path)): logging.error( ' GRIDMET daily or monthly CSV does not exist, skipping zone') continue # DEADBEEF - Eventually support generating only Landsat figures # logging.error( # ' GRIDMET daily and/or monthly CSV files do not exist.\n' # ' ETo and PPT will not be processed.') logging.debug(' Reading Landsat CSV') landsat_df = pd.read_csv(landsat_daily_path) logging.debug(' Filtering Landsat dataframe') landsat_df = landsat_df[landsat_df['PIXEL_COUNT'] > 0] # QA field should have been written in zonal stats code # Eventually this block can be removed if 'QA' not in landsat_df.columns.values: landsat_df['QA'] = 0 # # This assumes that there are L5/L8 images in the dataframe # if not landsat_df.empty: # max_pixel_count = max(landsat_df['PIXEL_COUNT']) # else: # max_pixel_count = 0 if year_list: landsat_df = landsat_df[landsat_df['YEAR'].isin(year_list)] if month_list: landsat_df = landsat_df[landsat_df['MONTH'].isin(month_list)] if doy_list: landsat_df = landsat_df[landsat_df['DOY'].isin(doy_list)] # Assume the default is for these to be True and only filter if False if not ini['INPUTS']['landsat4_flag']: landsat_df = landsat_df[landsat_df['PLATFORM'] != 'LT04'] if not ini['INPUTS']['landsat5_flag']: landsat_df = landsat_df[landsat_df['PLATFORM'] != 'LT05'] if not ini['INPUTS']['landsat7_flag']: landsat_df = landsat_df[landsat_df['PLATFORM'] != 'LE07'] if not ini['INPUTS']['landsat8_flag']: landsat_df = landsat_df[landsat_df['PLATFORM'] != 'LC08'] if ini['INPUTS']['path_keep_list']: landsat_df = landsat_df[landsat_df['PATH'].isin( ini['INPUTS']['path_keep_list'])] if (ini['INPUTS']['row_keep_list'] and ini['INPUTS']['row_keep_list'] != ['XXX']): landsat_df = landsat_df[landsat_df['ROW'].isin( ini['INPUTS']['row_keep_list'])] if ini['INPUTS']['scene_id_keep_list']: # Replace XXX with primary ROW value for checking skip list SCENE_ID scene_id_df = pd.Series([ s.replace('XXX', '{:03d}'.format(int(r))) for s, r in zip(landsat_df['SCENE_ID'], landsat_df['ROW']) ]) landsat_df = landsat_df[scene_id_df.isin( ini['INPUTS']['scene_id_keep_list']).values] # This won't work: SCENE_ID have XXX but scene_id_skip_list don't # landsat_df = landsat_df[landsat_df['SCENE_ID'].isin( # ini['INPUTS']['scene_id_keep_list'])] if ini['INPUTS']['scene_id_skip_list']: # Replace XXX with primary ROW value for checking skip list SCENE_ID scene_id_df = pd.Series([ s.replace('XXX', '{:03d}'.format(int(r))) for s, r in zip(landsat_df['SCENE_ID'], landsat_df['ROW']) ]) landsat_df = landsat_df[np.logical_not( scene_id_df.isin(ini['INPUTS']['scene_id_skip_list']).values)] # This won't work: SCENE_ID have XXX but scene_id_skip_list don't # landsat_df = landsat_df[np.logical_not(landsat_df['SCENE_ID'].isin( # ini['INPUTS']['scene_id_skip_list']))] # Filter by QA/QC value if ini['SUMMARY']['max_qa'] >= 0 and not landsat_df.empty: logging.debug(' Maximum QA: {0}'.format( ini['SUMMARY']['max_qa'])) landsat_df = landsat_df[ landsat_df['QA'] <= ini['SUMMARY']['max_qa']] # Filter by average cloud score if ini['SUMMARY']['max_cloud_score'] < 100 and not landsat_df.empty: logging.debug(' Maximum cloud score: {0}'.format( ini['SUMMARY']['max_cloud_score'])) landsat_df = landsat_df[ landsat_df['CLOUD_SCORE'] <= ini['SUMMARY']['max_cloud_score']] # Filter by Fmask percentage if ini['SUMMARY']['max_fmask_pct'] < 100 and not landsat_df.empty: landsat_df['FMASK_PCT'] = 100 * (landsat_df['FMASK_COUNT'] / landsat_df['FMASK_TOTAL']) logging.debug(' Max Fmask threshold: {}'.format( ini['SUMMARY']['max_fmask_pct'])) landsat_df = landsat_df[ landsat_df['FMASK_PCT'] <= ini['SUMMARY']['max_fmask_pct']] # Filter low count SLC-off images if ini['SUMMARY']['min_slc_off_pct'] > 0 and not landsat_df.empty: logging.debug(' Mininum SLC-off threshold: {}%'.format( ini['SUMMARY']['min_slc_off_pct'])) # logging.debug(' Maximum pixel count: {}'.format( # max_pixel_count)) slc_off_mask = ((landsat_df['PLATFORM'] == 'LE07') & ((landsat_df['YEAR'] >= 2004) | ((landsat_df['YEAR'] == 2003) & (landsat_df['DOY'] > 151)))) slc_off_pct = 100 * (landsat_df['PIXEL_COUNT'] / landsat_df['PIXEL_TOTAL']) # slc_off_pct = 100 * (landsat_df['PIXEL_COUNT'] / max_pixel_count) landsat_df = landsat_df[( (slc_off_pct >= ini['SUMMARY']['min_slc_off_pct']) & slc_off_mask) | (~slc_off_mask)] if landsat_df.empty: logging.error( ' Empty Landsat dataframe after filtering, skipping zone') continue # Aggregate GRIDMET (to water year) if os.path.isfile(gridmet_monthly_path): logging.debug(' Reading montly GRIDMET CSV') gridmet_df = pd.read_csv(gridmet_monthly_path) elif os.path.isfile(gridmet_daily_path): logging.debug(' Reading daily GRIDMET CSV') gridmet_df = pd.read_csv(gridmet_daily_path) logging.debug(' Computing GRIDMET summaries') # Summarize GRIDMET for target months year if (gridmet_start_month in [10, 11, 12] and gridmet_end_month in [10, 11, 12]): month_mask = ((gridmet_df['MONTH'] >= gridmet_start_month) & (gridmet_df['MONTH'] <= gridmet_end_month)) gridmet_df.loc[month_mask, 'GROUP_YEAR'] = gridmet_df['YEAR'] + 1 elif (gridmet_start_month in [10, 11, 12] and gridmet_end_month not in [10, 11, 12]): month_mask = gridmet_df['MONTH'] >= gridmet_start_month gridmet_df.loc[month_mask, 'GROUP_YEAR'] = gridmet_df['YEAR'] + 1 month_mask = gridmet_df['MONTH'] <= gridmet_end_month gridmet_df.loc[month_mask, 'GROUP_YEAR'] = gridmet_df['YEAR'] else: month_mask = ((gridmet_df['MONTH'] >= gridmet_start_month) & (gridmet_df['MONTH'] <= gridmet_end_month)) gridmet_df.loc[month_mask, 'GROUP_YEAR'] = gridmet_df['YEAR'] # GROUP_YEAR for rows not in the GRIDMET month range will be NAN gridmet_df = gridmet_df[~pd.isnull(gridmet_df['GROUP_YEAR'])] if year_list: gridmet_df = gridmet_df[gridmet_df['GROUP_YEAR'].isin(year_list)] if gridmet_df.empty: logging.error( ' Empty GRIDMET dataframe after filtering by year') continue # Group GRIDMET data by user specified range (default is water year) gridmet_group_df = gridmet_df \ .groupby(['ZONE_NAME', 'ZONE_FID', 'GROUP_YEAR']) \ .agg({'ETO': np.sum, 'PPT': np.sum}) \ .reset_index() \ .sort_values(by='GROUP_YEAR') # .rename(columns={'ETO': 'ETO', 'PPT': 'PPT'}) \ # Rename wasn't working when chained... gridmet_group_df.rename(columns={'GROUP_YEAR': 'YEAR'}, inplace=True) gridmet_group_df['YEAR'] = gridmet_group_df['YEAR'].astype(int) # # Group GRIDMET data by month # gridmet_month_df = gridmet_df\ # .groupby(['ZONE_NAME', 'ZONE_FID', 'GROUP_YEAR', 'MONTH']) \ # .agg({'ETO': np.sum, 'PPT': np.sum}) \ # .reset_index() \ # .sort_values(by=['GROUP_YEAR', 'MONTH']) # gridmet_month_df.rename(columns={'GROUP_YEAR': 'YEAR'}, inplace=True) # # Rename monthly PPT columns # gridmet_month_df['MONTH'] = 'PPT_M' + gridmet_month_df['MONTH'].astype(str) # # Pivot rows up to separate columns # gridmet_month_df = gridmet_month_df.pivot_table( # 'PPT', ['ZONE_NAME', 'YEAR'], 'MONTH') # gridmet_month_df.reset_index(inplace=True) # columns = ['ZONE_NAME', 'YEAR'] + ['PPT_M{}'.format(m) for m in gridmet_months] # gridmet_month_df = gridmet_month_df[columns] # del gridmet_month_df.index.name # Merge Landsat and GRIDMET collections zone_df = landsat_df.merge(gridmet_group_df, on=['ZONE_NAME', 'ZONE_FID', 'YEAR']) if zone_df is None or zone_df.empty: logging.info(' Empty zone dataframe, not generating figures') continue # Compute ETg zone_df['ETG_MEAN'] = zone_df['ETSTAR_MEAN'] * (zone_df['ETO'] - zone_df['PPT']) zone_df['ETG_LPI'] = zone_df['ETSTAR_LPI'] * (zone_df['ETO'] - zone_df['PPT']) zone_df['ETG_UPI'] = zone_df['ETSTAR_UPI'] * (zone_df['ETO'] - zone_df['PPT']) zone_df['ETG_LCI'] = zone_df['ETSTAR_LCI'] * (zone_df['ETO'] - zone_df['PPT']) zone_df['ETG_UCI'] = zone_df['ETSTAR_UCI'] * (zone_df['ETO'] - zone_df['PPT']) # Compute ET zone_df['ET_MEAN'] = zone_df['ETG_MEAN'] + zone_df['PPT'] zone_df['ET_LPI'] = zone_df['ETG_LPI'] + zone_df['PPT'] zone_df['ET_UPI'] = zone_df['ETG_UPI'] + zone_df['PPT'] zone_df['ET_LCI'] = zone_df['ETG_LCI'] + zone_df['PPT'] zone_df['ET_UCI'] = zone_df['ETG_UCI'] + zone_df['PPT'] # Append zone dataframes zone_df_dict[zone_name] = zone_df # Export each zone to a separate tab if not os.path.isfile(output_daily_path): logging.info('\nWriting daily values to Excel') excel_f = ExcelWriter(output_daily_path) for zone_name, zone_df in sorted(zone_df_dict.items()): logging.info(' {}'.format(zone_name)) zone_df.to_excel(excel_f, zone_name, index=False, float_format='%.4f') # zone_df.to_excel(excel_f, zone_name, index=False) del zone_df excel_f.save() if not os.path.isfile(output_annual_path): logging.info('\nComputing annual summaries') annual_df = pd.concat(list(zone_df_dict.values())) \ .groupby(['ZONE_NAME', 'YEAR']) \ .agg({ 'PIXEL_COUNT': ['count', 'mean'], 'PIXEL_TOTAL': ['mean'], 'FMASK_COUNT': 'mean', 'FMASK_TOTAL': 'mean', 'CLOUD_SCORE': 'mean', 'ETSTAR_COUNT': 'mean', 'NDVI_TOA': 'mean', 'NDWI_TOA': 'mean', 'ALBEDO_SUR': 'mean', 'TS': 'mean', # 'EVI_SUR': 'mean', 'EVI_SUR': ['mean', 'median', 'min', 'max'], 'ETSTAR_MEAN': 'mean', 'ETG_MEAN': 'mean', 'ETG_LPI': 'mean', 'ETG_UPI': 'mean', 'ETG_LCI': 'mean', 'ETG_UCI': 'mean', 'ET_MEAN': 'mean', 'ET_LPI': 'mean', 'ET_UPI': 'mean', 'ET_LCI': 'mean', 'ET_UCI': 'mean', 'ETO': 'mean', 'PPT': 'mean' }) annual_df.columns = annual_df.columns.map('_'.join) annual_df = annual_df.rename(columns={ 'PIXEL_COUNT_count': 'SCENE_COUNT', 'PIXEL_COUNT_mean': 'PIXEL_COUNT' }) annual_df = annual_df.rename( columns={ 'EVI_SUR_mean': 'EVI_SUR_MEAN', 'EVI_SUR_median': 'EVI_SUR_MEDIAN', 'EVI_SUR_min': 'EVI_SUR_MIN', 'EVI_SUR_max': 'EVI_SUR_MAX' }) annual_df.rename(columns=lambda x: str(x).replace('_mean', ''), inplace=True) annual_df['SCENE_COUNT'] = annual_df['SCENE_COUNT'].astype(np.int) annual_df['PIXEL_COUNT'] = annual_df['PIXEL_COUNT'].astype(np.int) annual_df['PIXEL_TOTAL'] = annual_df['PIXEL_TOTAL'].astype(np.int) annual_df['FMASK_COUNT'] = annual_df['FMASK_COUNT'].astype(np.int) annual_df['FMASK_TOTAL'] = annual_df['FMASK_TOTAL'].astype(np.int) annual_df['ETSTAR_COUNT'] = annual_df['ETSTAR_COUNT'].astype(np.int) annual_df = annual_df.reset_index() # Convert ETo units if (ini['BEAMER']['eto_units'] == 'mm' and ini['TABLES']['eto_units'] == 'mm'): pass elif (ini['BEAMER']['eto_units'] == 'mm' and ini['TABLES']['eto_units'] == 'in'): annual_df[eto_fields] /= (25.4) elif (ini['BEAMER']['eto_units'] == 'mm' and ini['TABLES']['eto_units'] == 'ft'): annual_df[eto_fields] /= (12 * 25.4) else: logging.error( ('\nERROR: Input units {} and output units {} are not ' + 'currently supported, exiting').format( ini['BEAMER']['eto_units'], ini['TABLES']['eto_units'])) sys.exit() # Convert PPT units if (ini['BEAMER']['ppt_units'] == 'mm' and ini['TABLES']['ppt_units'] == 'mm'): pass elif (ini['BEAMER']['ppt_units'] == 'mm' and ini['TABLES']['ppt_units'] == 'in'): annual_df[ppt_fields] /= (25.4) elif (ini['BEAMER']['ppt_units'] == 'mm' and ini['TABLES']['ppt_units'] == 'ft'): annual_df[ppt_fields] /= (12 * 25.4) else: logging.error( ('\nERROR: Input units {} and output units {} are not ' + 'currently supported, exiting').format( ini['BEAMER']['ppt_units'], ini['TABLES']['ppt_units'])) sys.exit() logging.info('\nWriting annual values to Excel') excel_f = ExcelWriter(output_annual_path) for zone_name in sorted(zone_df_dict.keys()): logging.info(' {}'.format(zone_name)) zone_df = annual_df[annual_df['ZONE_NAME'] == zone_name] zone_df.to_excel(excel_f, zone_name, index=False, float_format='%.4f') del zone_df excel_f.save()
def main(ini_path=None, overwrite_flag=True, show_flag=False): """Generate summary figures Args: ini_path (str): file path of the control file overwrite_flag (bool): if True, overwrite existing figures show_flag (bool): if True, show figures as they are being built """ logging.info('\nGenerate summary figures') # Read config file ini = inputs.read(ini_path) inputs.parse_section(ini, section='INPUTS') inputs.parse_section(ini, section='ZONAL_STATS') inputs.parse_section(ini, section='SUMMARY') inputs.parse_section(ini, section='FIGURES') # Band options band_list = [ 'albedo_sur', 'cloud_score', 'eto', 'evi_sur', 'fmask_count', 'fmask_total', 'ndvi_sur', 'ndvi_toa', 'ndwi_green_nir_sur', 'ndwi_green_nir_toa', 'ndwi_green_swir1_sur', 'ndwi_green_swir1_toa', 'ndwi_nir_swir1_sur', 'ndwi_nir_swir1_toa', 'ndwi_swir1_green_sur', 'ndwi_swir1_green_toa', # 'ndwi_sur', 'ndwi_toa', 'pixel_count', 'pixel_total', 'ppt', 'tc_bright', 'tc_green', 'tc_wet', 'ts' ] band_name = { 'albedo_sur': 'Albedo', 'cloud_score': 'Cloud Score', 'eto': 'ETo', 'evi_sur': 'EVI', 'fmask_count': 'Fmask Count', 'fmask_total': 'Fmask Total', 'ndvi_sur': 'NDVI', 'ndvi_toa': 'NDVI (TOA)', 'ndwi_green_nir_sur': 'NDWI (Green, NIR)', 'ndwi_green_nir_toa': 'NDWI (Green, NIR) (TOA)', 'ndwi_green_swir1_sur': 'NDWI (Green, SWIR1)', 'ndwi_green_swir1_toa': 'NDWI (Green, SWIR1) (TOA)', 'ndwi_nir_swir1_sur': 'NDWI (NIR, SWIR1)', 'ndwi_nir_swir1_toa': 'NDWI (NIR, SWIR1) (TOA)', 'ndwi_swir1_green_sur': 'NDWI (SWIR1, Green)', 'ndwi_swir1_green_toa': 'NDWI (SWIR1, Green) (TOA)', # 'ndwi_sur': 'NDWI (SWIR1, GREEN)', # 'ndwi_toa': 'NDWI (SWIR1, GREEN) (TOA)', 'pixel_count': 'Pixel Count', 'pixel_total': 'Pixel Total', 'ppt': 'PPT', 'tc_bright': 'Brightness', 'tc_green': 'Greeness', 'tc_wet': 'Wetness', 'ts': 'Ts' } band_unit = { 'albedo_sur': 'dimensionless', 'cloud_score': 'dimensionless', 'evi_sur': 'dimensionless', 'eto': 'mm', 'fmask_count': 'dimensionless', 'fmask_total': 'dimensionless', 'ndvi_sur': 'dimensionless', 'ndvi_toa': 'dimensionless', 'ndwi_green_nir_sur': 'dimensionless', 'ndwi_green_nir_toa': 'dimensionless', 'ndwi_green_swir1_sur': 'dimensionless', 'ndwi_green_swir1_toa': 'dimensionless', 'ndwi_nir_swir1_sur': 'dimensionless', 'ndwi_nir_swir1_toa': 'dimensionless', 'ndwi_swir1_green_sur': 'dimensionless', 'ndwi_swir1_green_toa': 'dimensionless', # 'ndwi_sur': 'dimensionless', # 'ndwi_toa': 'dimensionless', 'pixel_count': 'dimensionless', 'pixel_total': 'dimensionless', 'ppt': 'mm', 'tc_bright': 'dimensionless', 'tc_green': 'dimensionless', 'tc_wet': 'dimensionless', 'ts': 'K', } band_color = { 'albedo_sur': '#CF4457', 'cloud_score': '0.5', 'eto': '#348ABD', 'fmask_count': '0.5', 'fmask_total': '0.5', 'evi_sur': '#FFA500', 'ndvi_sur': '#A60628', 'ndvi_toa': '#A60628', 'ndwi_green_nir_sur': '#4eae4b', 'ndwi_green_nir_toa': '#4eae4b', 'ndwi_green_swir1_sur': '#4eae4b', 'ndwi_green_swir1_toa': '#4eae4b', 'ndwi_nir_swir1_sur': '#4eae4b', 'ndwi_nir_swir1_toa': '#4eae4b', 'ndwi_swir1_green_sur': '#4eae4b', 'ndwi_swir1_green_toa': '#4eae4b', # 'ndwi_sur': '#4eae4b', # 'ndwi_toa': '#4eae4b', 'pixel_count': '0.5', 'pixel_total': '0.5', 'ppt': '0.5', 'tc_bright': '#E24A33', 'tc_green': '#E24A33', 'tc_wet': '#E24A33', 'ts': '#188487' } # A couple of color palettes to sample from # import seaborn as sns # print(sns.color_palette('hls', 20).as_hex()) # print(sns.color_palette('husl', 20).as_hex()) # print(sns.color_palette('hsv', 20).as_hex()) # print(sns.color_palette('Set1', 20).as_hex()) # print(sns.color_palette('Set2', 20).as_hex()) # Hardcoded plot options figures_folder = 'figures' fig_type = 'large' plot_dict = dict() # Center y-labels in figure window (instead of centering on ticks/axes) plot_dict['center_ylabel'] = False # Axes percentages must be 0-1 plot_dict['timeseries_band_ax_pct'] = [0.3, 0.92] plot_dict['timeseries_ppt_ax_pct'] = [0.0, 0.35] plot_dict['complement_band_ax_pct'] = [0.0, 0.5] plot_dict['complement_eto_ax_pct'] = [0.4, 1.0] if fig_type.lower() == 'large': plot_dict['title_fs'] = 12 plot_dict['xtick_fs'] = 10 plot_dict['ytick_fs'] = 10 plot_dict['xlabel_fs'] = 10 plot_dict['ylabel_fs'] = 10 plot_dict['legend_fs'] = 10 plot_dict['ts_ms'] = 3 plot_dict['comp_ms'] = 4 plot_dict['timeseries_ax'] = [0.12, 0.13, 0.78, 0.81] plot_dict['scatter_ax'] = [0.12, 0.10, 0.82, 0.84] plot_dict['complement_ax'] = [0.12, 0.10, 0.78, 0.84] plot_dict['fig_size'] = (6.0, 5.0) elif fig_type.lower() == 'small': plot_dict['title_fs'] = 10 plot_dict['xtick_fs'] = 8 plot_dict['ytick_fs'] = 8 plot_dict['xlabel_fs'] = 8 plot_dict['ylabel_fs'] = 8 plot_dict['legend_fs'] = 8 plot_dict['ts_ms'] = 1.5 plot_dict['comp_ms'] = 2 plot_dict['timeseries_ax'] = [0.18, 0.21, 0.67, 0.70] plot_dict['scatter_ax'] = [0.18, 0.21, 0.67, 0.70] plot_dict['complement_ax'] = [0.18, 0.16, 0.67, 0.75] plot_dict['fig_size'] = (3.0, 2.5) plot_dict['fig_dpi'] = 300 plot_dict['show'] = show_flag plot_dict['overwrite'] = overwrite_flag # CSV parameters landsat_annual_fields = [ 'ZONE_FID', 'ZONE_NAME', 'YEAR', 'SCENE_COUNT', 'CLOUD_SCORE', 'PIXEL_COUNT', 'PIXEL_TOTAL', 'FMASK_COUNT', 'FMASK_TOTAL', 'TS', 'ALBEDO_SUR', 'NDVI_TOA', 'NDVI_SUR', 'EVI_SUR', 'NDWI_GREEN_NIR_SUR', 'NDWI_GREEN_SWIR1_SUR', 'NDWI_NIR_SWIR1_SUR', # 'NDWI_GREEN_NIR_TOA', 'NDWI_GREEN_SWIR1_TOA', 'NDWI_NIR_SWIR1_TOA', # 'NDWI_SWIR1_GREEN_TOA', 'NDWI_SWIR1_GREEN_SUR', # 'NDWI_TOA', 'NDWI_SUR', 'TC_BRIGHT', 'TC_GREEN', 'TC_WET' ] # Add merged row XXX to keep list ini['INPUTS']['row_keep_list'].append('XXX') # Check figure bands timeseries_bands = ini['FIGURES']['timeseries_bands'] scatter_bands = ini['FIGURES']['scatter_bands'] complementary_bands = ini['FIGURES']['complementary_bands'] if timeseries_bands: logging.info('Timeseries Bands:') for band in timeseries_bands: if band not in band_list: logging.info( ' Invalid timeseries band: {}, exiting'.format(band)) return False logging.info(' {}'.format(band)) if scatter_bands: logging.info('Scatter Bands (x:y):') for band_x, band_y in scatter_bands: if band_x not in band_list: logging.info( ' Invalid scatter band: {}, exiting'.format(band_x)) return False elif band_y not in band_list: logging.info(' Invalid band: {}, exiting'.format(band_y)) return False logging.info(' {}:{}'.format(band_x, band_y)) if complementary_bands: logging.info('Complementary Bands:') for band in complementary_bands: if band not in band_list: logging.info( ' Invalid complementary band: {}, exiting'.format(band)) return False logging.info(' {}'.format(band)) # Add input plot options plot_dict['ppt_plot_type'] = ini['FIGURES']['ppt_plot_type'] plot_dict['scatter_best_fit'] = ini['FIGURES']['scatter_best_fit'] # Start/end year year_list = list( range(ini['INPUTS']['start_year'], ini['INPUTS']['end_year'] + 1)) month_list = list( utils.wrapped_range(ini['INPUTS']['start_month'], ini['INPUTS']['end_month'], 1, 12)) doy_list = list( utils.wrapped_range(ini['INPUTS']['start_doy'], ini['INPUTS']['end_doy'], 1, 366)) # GRIDMET month range (default to water year) gridmet_start_month = ini['SUMMARY']['gridmet_start_month'] gridmet_end_month = ini['SUMMARY']['gridmet_end_month'] gridmet_months = list( utils.month_range(gridmet_start_month, gridmet_end_month)) logging.info('\nGridmet months: {}'.format(', '.join( map(str, gridmet_months)))) # Get ee features from shapefile zone_geom_list = gdc.shapefile_2_geom_list_func( ini['INPUTS']['zone_shp_path'], zone_field=ini['INPUTS']['zone_field'], reverse_flag=False) # Filter features by FID before merging geometries if ini['INPUTS']['fid_keep_list']: zone_geom_list = [ zone_obj for zone_obj in zone_geom_list if zone_obj[0] in ini['INPUTS']['fid_keep_list'] ] if ini['INPUTS']['fid_skip_list']: zone_geom_list = [ zone_obj for zone_obj in zone_geom_list if zone_obj[0] not in ini['INPUTS']['fid_skip_list'] ] # # Filter features by FID before merging geometries # if ini['INPUTS']['fid_keep_list']: # landsat_df = landsat_df[landsat_df['ZONE_FID'].isin( # ini['INPUTS']['fid_keep_list'])] # if ini['INPUTS']['fid_skip_list']: # landsat_df = landsat_df[~landsat_df['ZONE_FID'].isin( # ini['INPUTS']['fid_skip_list'])] logging.info('\nProcessing zones') for zone_fid, zone_name, zone_json in zone_geom_list: zone_name = zone_name.replace(' ', '_') logging.info('ZONE: {} (FID: {})'.format(zone_name, zone_fid)) zone_stats_ws = os.path.join(ini['ZONAL_STATS']['output_ws'], zone_name) zone_figures_ws = os.path.join(ini['SUMMARY']['output_ws'], zone_name, figures_folder) if not os.path.isdir(zone_stats_ws): logging.debug( ' Folder {} does not exist, skipping'.format(zone_stats_ws)) continue elif not os.path.isdir(zone_figures_ws): os.makedirs(zone_figures_ws) # Input paths landsat_daily_path = os.path.join( zone_stats_ws, '{}_landsat_daily.csv'.format(zone_name)) gridmet_daily_path = os.path.join( zone_stats_ws, '{}_gridmet_daily.csv'.format(zone_name)) gridmet_monthly_path = os.path.join( zone_stats_ws, '{}_gridmet_monthly.csv'.format(zone_name)) if not os.path.isfile(landsat_daily_path): logging.error(' Landsat daily CSV does not exist, skipping zone') continue elif (not os.path.isfile(gridmet_daily_path) and not os.path.isfile(gridmet_monthly_path)): logging.error( ' GRIDMET daily or monthly CSV does not exist, skipping zone') continue # DEADBEEF - Eventually support generating only Landsat figures # logging.error( # ' GRIDMET daily and/or monthly CSV files do not exist.\n' # ' ETo and PPT will not be processed.') # Output paths landsat_summary_path = os.path.join( zone_figures_ws, '{}_landsat_figures.csv'.format(zone_name)) gridmet_summary_path = os.path.join( zone_figures_ws, '{}_gridmet_figures.csv'.format(zone_name)) zone_summary_path = os.path.join( zone_figures_ws, '{}_zone_figures.csv'.format(zone_name)) logging.debug(' Reading Landsat CSV') landsat_df = pd.read_csv(landsat_daily_path) logging.debug(' Filtering Landsat dataframe') landsat_df = landsat_df[landsat_df['PIXEL_COUNT'] > 0] # QA field should have been written in zonal stats code # Eventually this block can be removed if 'QA' not in landsat_df.columns.values: landsat_df['QA'] = 0 # # This assumes that there are L5/L8 images in the dataframe # if not landsat_df.empty: # max_pixel_count = max(landsat_df['PIXEL_COUNT']) # else: # max_pixel_count = 0 if year_list: landsat_df = landsat_df[landsat_df['YEAR'].isin(year_list)] if month_list: landsat_df = landsat_df[landsat_df['MONTH'].isin(month_list)] if doy_list: landsat_df = landsat_df[landsat_df['DOY'].isin(doy_list)] # Assume the default is for these to be True and only filter if False if not ini['INPUTS']['landsat4_flag']: landsat_df = landsat_df[landsat_df['PLATFORM'] != 'LT04'] if not ini['INPUTS']['landsat5_flag']: landsat_df = landsat_df[landsat_df['PLATFORM'] != 'LT05'] if not ini['INPUTS']['landsat7_flag']: landsat_df = landsat_df[landsat_df['PLATFORM'] != 'LE07'] if not ini['INPUTS']['landsat8_flag']: landsat_df = landsat_df[landsat_df['PLATFORM'] != 'LC08'] if ini['INPUTS']['path_keep_list']: landsat_df = landsat_df[landsat_df['PATH'].isin( ini['INPUTS']['path_keep_list'])] if (ini['INPUTS']['row_keep_list'] and ini['INPUTS']['row_keep_list'] != ['XXX']): landsat_df = landsat_df[landsat_df['ROW'].isin( ini['INPUTS']['row_keep_list'])] if ini['INPUTS']['scene_id_keep_list']: # Replace XXX with primary ROW value for checking skip list SCENE_ID scene_id_df = pd.Series([ s.replace('XXX', '{:03d}'.format(int(r))) for s, r in zip(landsat_df['SCENE_ID'], landsat_df['ROW']) ]) landsat_df = landsat_df[scene_id_df.isin( ini['INPUTS']['scene_id_keep_list']).values] # This won't work: SCENE_ID have XXX but scene_id_skip_list don't # landsat_df = landsat_df[landsat_df['SCENE_ID'].isin( # ini['INPUTS']['scene_id_keep_list'])] if ini['INPUTS']['scene_id_skip_list']: # Replace XXX with primary ROW value for checking skip list SCENE_ID scene_id_df = pd.Series([ s.replace('XXX', '{:03d}'.format(int(r))) for s, r in zip(landsat_df['SCENE_ID'], landsat_df['ROW']) ]) landsat_df = landsat_df[np.logical_not( scene_id_df.isin(ini['INPUTS']['scene_id_skip_list']).values)] # This won't work: SCENE_ID have XXX but scene_id_skip_list don't # landsat_df = landsat_df[np.logical_not(landsat_df['SCENE_ID'].isin( # ini['INPUTS']['scene_id_skip_list']))] # Filter by QA/QC value if ini['SUMMARY']['max_qa'] >= 0 and not landsat_df.empty: logging.debug(' Maximum QA: {0}'.format( ini['SUMMARY']['max_qa'])) landsat_df = landsat_df[ landsat_df['QA'] <= ini['SUMMARY']['max_qa']] # Filter by average cloud score if ini['SUMMARY']['max_cloud_score'] < 100 and not landsat_df.empty: logging.debug(' Maximum cloud score: {0}'.format( ini['SUMMARY']['max_cloud_score'])) landsat_df = landsat_df[ landsat_df['CLOUD_SCORE'] <= ini['SUMMARY']['max_cloud_score']] # Filter by Fmask percentage if ini['SUMMARY']['max_fmask_pct'] < 100 and not landsat_df.empty: landsat_df['FMASK_PCT'] = 100 * (landsat_df['FMASK_COUNT'] / landsat_df['FMASK_TOTAL']) logging.debug(' Max Fmask threshold: {}'.format( ini['SUMMARY']['max_fmask_pct'])) landsat_df = landsat_df[ landsat_df['FMASK_PCT'] <= ini['SUMMARY']['max_fmask_pct']] # Filter low count SLC-off images if ini['SUMMARY']['min_slc_off_pct'] > 0 and not landsat_df.empty: logging.debug(' Mininum SLC-off threshold: {}%'.format( ini['SUMMARY']['min_slc_off_pct'])) # logging.debug(' Maximum pixel count: {}'.format( # max_pixel_count)) slc_off_mask = ((landsat_df['LANDSAT'] == 'LE7') & ((landsat_df['YEAR'] >= 2004) | ((landsat_df['YEAR'] == 2003) & (landsat_df['DOY'] > 151)))) slc_off_pct = 100 * (landsat_df['PIXEL_COUNT'] / landsat_df['PIXEL_TOTAL']) # slc_off_pct = 100 * (landsat_df['PIXEL_COUNT'] / max_pixel_count) landsat_df = landsat_df[( (slc_off_pct >= ini['SUMMARY']['min_slc_off_pct']) & slc_off_mask) | (~slc_off_mask)] if landsat_df.empty: logging.error( ' Empty Landsat dataframe after filtering, skipping zone') continue logging.debug(' Computing Landsat annual summaries') agg_dict = { 'PIXEL_COUNT': { 'PIXEL_COUNT': 'mean', 'SCENE_COUNT': 'count' }, 'PIXEL_TOTAL': { 'PIXEL_TOTAL': 'mean' }, 'FMASK_COUNT': { 'FMASK_COUNT': 'mean' }, 'FMASK_TOTAL': { 'FMASK_TOTAL': 'mean' }, 'CLOUD_SCORE': { 'CLOUD_SCORE': 'mean' } } for field in landsat_df.columns.values: if field in landsat_annual_fields: agg_dict.update({field: {field: 'mean'}}) landsat_df = landsat_df \ .groupby(['ZONE_NAME', 'ZONE_FID', 'YEAR']) \ .agg(agg_dict) landsat_df.columns = landsat_df.columns.droplevel(0) landsat_df.reset_index(inplace=True) # landsat_df = landsat_df[landsat_annual_fields] landsat_df['YEAR'] = landsat_df['YEAR'].astype(np.int) landsat_df['SCENE_COUNT'] = landsat_df['SCENE_COUNT'].astype(np.int) landsat_df['PIXEL_COUNT'] = landsat_df['PIXEL_COUNT'].astype(np.int) landsat_df['PIXEL_TOTAL'] = landsat_df['PIXEL_TOTAL'].astype(np.int) landsat_df['FMASK_COUNT'] = landsat_df['FMASK_COUNT'].astype(np.int) landsat_df['FMASK_TOTAL'] = landsat_df['FMASK_TOTAL'].astype(np.int) landsat_df.sort_values(by='YEAR', inplace=True) # Aggregate GRIDMET (to water year) if os.path.isfile(gridmet_monthly_path): logging.debug(' Reading montly GRIDMET CSV') gridmet_df = pd.read_csv(gridmet_monthly_path) elif os.path.isfile(gridmet_daily_path): logging.debug(' Reading daily GRIDMET CSV') gridmet_df = pd.read_csv(gridmet_daily_path) logging.debug(' Computing GRIDMET summaries') # Summarize GRIDMET for target months year if (gridmet_start_month in [10, 11, 12] and gridmet_end_month in [10, 11, 12]): month_mask = ((gridmet_df['MONTH'] >= gridmet_start_month) & (gridmet_df['MONTH'] <= gridmet_end_month)) gridmet_df.loc[month_mask, 'GROUP_YEAR'] = gridmet_df['YEAR'] + 1 elif (gridmet_start_month in [10, 11, 12] and gridmet_end_month not in [10, 11, 12]): month_mask = gridmet_df['MONTH'] >= gridmet_start_month gridmet_df.loc[month_mask, 'GROUP_YEAR'] = gridmet_df['YEAR'] + 1 month_mask = gridmet_df['MONTH'] <= gridmet_end_month gridmet_df.loc[month_mask, 'GROUP_YEAR'] = gridmet_df['YEAR'] else: month_mask = ((gridmet_df['MONTH'] >= gridmet_start_month) & (gridmet_df['MONTH'] <= gridmet_end_month)) gridmet_df.loc[month_mask, 'GROUP_YEAR'] = gridmet_df['YEAR'] # GROUP_YEAR for rows not in the GRIDMET month range will be NAN gridmet_df = gridmet_df[~pd.isnull(gridmet_df['GROUP_YEAR'])] if year_list: gridmet_df = gridmet_df[gridmet_df['GROUP_YEAR'].isin(year_list)] if gridmet_df.empty: logging.error( ' Empty GRIDMET dataframe after filtering by year') continue # Group GRIDMET data by user specified range (default is water year) gridmet_group_df = gridmet_df \ .groupby(['ZONE_FID', 'ZONE_NAME', 'GROUP_YEAR']) \ .agg({'ETO': {'ETO': 'sum'}, 'PPT': {'PPT': 'sum'}}) gridmet_group_df.columns = gridmet_group_df.columns.droplevel(0) gridmet_group_df.reset_index(inplace=True) gridmet_group_df.rename(columns={'GROUP_YEAR': 'YEAR'}, inplace=True) gridmet_group_df.sort_values(by='YEAR', inplace=True) # # Group GRIDMET data by month # gridmet_month_df = gridmet_df.groupby( # ['ZONE_FID', 'ZONE_NAME', 'GROUP_YEAR', 'MONTH']).agg({ # 'ETO': {'ETO': 'sum'}, 'PPT': {'PPT': 'sum'}}) # gridmet_month_df.columns = gridmet_month_df.columns.droplevel(0) # gridmet_month_df.reset_index(inplace=True) # gridmet_month_df.rename(columns={'GROUP_YEAR': 'YEAR'}, inplace=True) # # gridmet_month_df.sort_values(by=['YEAR', 'MONTH'], inplace=True) # gridmet_month_df.reset_index(inplace=True) # # Rename monthly PPT columns # gridmet_month_df['MONTH'] = 'PPT_M' + gridmet_month_df['MONTH'].astype(str) # # Pivot rows up to separate columns # gridmet_month_df = gridmet_month_df.pivot_table( # 'PPT', ['ZONE_FID', 'YEAR'], 'MONTH') # gridmet_month_df.reset_index(inplace=True) # columns = ['ZONE_FID', 'YEAR'] + ['PPT_M{}'.format(m) for m in gridmet_months] # gridmet_month_df = gridmet_month_df[columns] # del gridmet_month_df.index.name # Merge Landsat and GRIDMET collections zone_df = landsat_df.merge(gridmet_group_df, on=['ZONE_FID', 'ZONE_NAME', 'YEAR']) # gridmet_group_df, on=['ZONE_FID', 'YEAR']) # zone_df = zone_df.merge( # gridmet_month_df, on=['ZONE_FID', 'ZONE_NAME', 'YEAR']) # gridmet_month_df, on=['ZONE_FID', 'YEAR']) if zone_df is None or zone_df.empty: logging.info(' Empty zone dataframe, not generating figures') continue # Save annual Landsat and GRIDMET tables logging.debug(' Saving summary tables') logging.debug(' {}'.format(landsat_summary_path)) landsat_df.sort_values(by=['YEAR'], inplace=True) landsat_df.to_csv(landsat_summary_path, index=False) # columns=export_fields logging.debug(' {}'.format(gridmet_summary_path)) gridmet_group_df.sort_values(by=['YEAR'], inplace=True) gridmet_group_df.to_csv(gridmet_summary_path, index=False) # columns=export_fields logging.debug(' {}'.format(zone_summary_path)) zone_df.sort_values(by=['YEAR'], inplace=True) zone_df.to_csv(zone_summary_path, index=False) # columns=export_fields # Adjust year range based on data availability? # start_year = min(zone_df['YEAR']), # end_year = max(zone_df['YEAR']) logging.debug(' Generating figures') for band in timeseries_bands: timeseries_plot(band, zone_df, zone_name, zone_figures_ws, ini['INPUTS']['start_year'], ini['INPUTS']['end_year'], band_name, band_unit, band_color, plot_dict) for band_x, band_y in scatter_bands: scatter_plot(band_x, band_y, zone_df, zone_name, zone_figures_ws, band_name, band_unit, band_color, plot_dict) for band in complementary_bands: complementary_plot(band, zone_df, zone_name, zone_figures_ws, band_name, band_unit, band_color, plot_dict) del landsat_df, gridmet_df, zone_df
def main(ini_path, overwrite_flag=True): """Generate Beamer ETg summary tables Args: ini_path (str): overwrite_flag (bool): if True, overwrite existing figures Default is True (for now) """ logging.info('\nGenerate Beamer ETg summary tables') # # Eventually get from INI (like ini['BEAMER']['landsat_products']) # daily_fields = [ # 'ZONE_NAME', 'ZONE_FID', 'DATE', 'SCENE_ID', 'PLATFORM', 'PATH', 'ROW', # 'YEAR', 'MONTH', 'DAY', 'DOY', 'WATER_YEAR', # 'PIXEL_COUNT', 'ETSTAR_COUNT', # 'NDVI_TOA', 'NDWI_TOA', 'ALBEDO_SUR', 'TS', 'EVI_SUR', 'ETSTAR_MEAN', # 'ETG_MEAN', 'ETG_LPI', 'ETG_UPI', 'ETG_LCI', 'ETG_UCI', # 'ET_MEAN', 'ET_LPI', 'ET_UPI', 'ET_LCI', 'ET_UCI', # 'WY_ETO', 'WY_PPT'] # annual_fields = [ # 'SCENE_COUNT', 'PIXEL_COUNT', 'ETSTAR_COUNT', # 'NDVI_TOA', 'NDWI_TOA', 'ALBEDO_SUR', 'TS', # 'EVI_SUR_MEAN', 'EVI_SUR_MEDIAN', 'EVI_SUR_MIN', 'EVI_SUR_MAX', # 'ETSTAR_MEAN', # 'ETG_MEAN', 'ETG_LPI', 'ETG_UPI', 'ETG_LCI', 'ETG_UCI', # 'ET_MEAN', 'ET_LPI', 'ET_UPI', 'ET_LCI', 'ET_UCI', # 'WY_ETO', 'WY_PPT'] # For unit conversion eto_fields = [ 'ETG_MEAN', 'ETG_LPI', 'ETG_UPI', 'ETG_LCI', 'ETG_UCI', 'ET_MEAN', 'ET_LPI', 'ET_UPI', 'ET_LCI', 'ET_UCI', 'WY_ETO' ] ppt_fields = ['WY_PPT'] # Read config file ini = inputs.read(ini_path) inputs.parse_section(ini, section='INPUTS') inputs.parse_section(ini, section='ZONAL_STATS') inputs.parse_section(ini, section='BEAMER') inputs.parse_section(ini, section='SUMMARY') inputs.parse_section(ini, section='TABLES') # Output paths output_daily_path = os.path.join( ini['SUMMARY']['output_ws'], ini['BEAMER']['output_name'].replace('.csv', '_daily.xlsx')) output_annual_path = os.path.join( ini['SUMMARY']['output_ws'], ini['BEAMER']['output_name'].replace('.csv', '_annual.xlsx')) # Check if files already exist if overwrite_flag: if os.path.isfile(output_daily_path): os.remove(output_daily_path) if os.path.isfile(output_annual_path): os.remove(output_annual_path) else: if (os.path.isfile(output_daily_path) and os.path.isfile(output_annual_path)): logging.info('\nOutput files already exist and ' 'overwrite is False, exiting') return True # Start/end year year_list = list( range(ini['INPUTS']['start_year'], ini['INPUTS']['end_year'] + 1)) month_list = list( utils.wrapped_range(ini['INPUTS']['start_month'], ini['INPUTS']['end_month'], 1, 12)) doy_list = list( utils.wrapped_range(ini['INPUTS']['start_doy'], ini['INPUTS']['end_doy'], 1, 366)) # GRIDMET month range (default to water year) gridmet_start_month = ini['SUMMARY']['gridmet_start_month'] gridmet_end_month = ini['SUMMARY']['gridmet_end_month'] gridmet_months = list( utils.month_range(gridmet_start_month, gridmet_end_month)) logging.info('\nGridmet months: {}'.format(', '.join( map(str, gridmet_months)))) # Read in the zonal stats CSV logging.debug(' Reading zonal stats CSV file') input_df = pd.read_csv( os.path.join(ini['ZONAL_STATS']['output_ws'], ini['BEAMER']['output_name'])) logging.debug(' Filtering Landsat dataframe') input_df = input_df[input_df['PIXEL_COUNT'] > 0] # # This assumes that there are L5/L8 images in the dataframe # if not input_df.empty: # max_pixel_count = max(input_df['PIXEL_COUNT']) # else: # max_pixel_count = 0 if ini['INPUTS']['fid_keep_list']: input_df = input_df[input_df['ZONE_FID'].isin( ini['INPUTS']['fid_keep_list'])] if ini['INPUTS']['fid_skip_list']: input_df = input_df[~input_df['ZONE_FID']. isin(ini['INPUTS']['fid_skip_list'])] zone_name_list = sorted(list(set(input_df['ZONE_NAME'].values))) if year_list: input_df = input_df[input_df['YEAR'].isin(year_list)] if month_list: input_df = input_df[input_df['MONTH'].isin(month_list)] if doy_list: input_df = input_df[input_df['DOY'].isin(doy_list)] if ini['INPUTS']['path_keep_list']: input_df = input_df[input_df['PATH'].isin( ini['INPUTS']['path_keep_list'])] if (ini['INPUTS']['row_keep_list'] and ini['INPUTS']['row_keep_list'] != ['XXX']): input_df = input_df[input_df['ROW'].isin( ini['INPUTS']['row_keep_list'])] # Assume the default is for these to be True and only filter if False if not ini['INPUTS']['landsat4_flag']: input_df = input_df[input_df['PLATFORM'] != 'LT04'] if not ini['INPUTS']['landsat5_flag']: input_df = input_df[input_df['PLATFORM'] != 'LT05'] if not ini['INPUTS']['landsat7_flag']: input_df = input_df[input_df['PLATFORM'] != 'LE07'] if not ini['INPUTS']['landsat8_flag']: input_df = input_df[input_df['PLATFORM'] != 'LC08'] if ini['INPUTS']['scene_id_keep_list']: # Replace XXX with primary ROW value for checking skip list SCENE_ID scene_id_df = pd.Series([ s.replace('XXX', '{:03d}'.format(int(r))) for s, r in zip(input_df['SCENE_ID'], input_df['ROW']) ]) input_df = input_df[scene_id_df.isin( ini['INPUTS']['scene_id_keep_list']).values] # This won't work: SCENE_ID have XXX but scene_id_skip_list don't # input_df = input_df[input_df['SCENE_ID'].isin( # ini['INPUTS']['scene_id_keep_list'])] if ini['INPUTS']['scene_id_skip_list']: # Replace XXX with primary ROW value for checking skip list SCENE_ID scene_id_df = pd.Series([ s.replace('XXX', '{:03d}'.format(int(r))) for s, r in zip(input_df['SCENE_ID'], input_df['ROW']) ]) input_df = input_df[np.logical_not( scene_id_df.isin(ini['INPUTS']['scene_id_skip_list']).values)] # This won't work: SCENE_ID have XXX but scene_id_skip_list don't # input_df = input_df[~input_df['SCENE_ID'].isin( # ini['INPUTS']['scene_id_skip_list'])] # Filter by QA/QC value if ini['SUMMARY']['max_qa'] >= 0 and not input_df.empty: logging.debug(' Maximum QA: {0}'.format(ini['SUMMARY']['max_qa'])) input_df = input_df[input_df['QA'] <= ini['SUMMARY']['max_qa']] # First filter by average cloud score if ini['SUMMARY']['max_cloud_score'] < 100 and not input_df.empty: logging.debug(' Maximum cloud score: {0}'.format( ini['SUMMARY']['max_cloud_score'])) input_df = input_df[ input_df['CLOUD_SCORE'] <= ini['SUMMARY']['max_cloud_score']] # Filter by Fmask percentage if ini['SUMMARY']['max_fmask_pct'] < 100 and not input_df.empty: input_df['FMASK_PCT'] = 100 * (input_df['FMASK_COUNT'] / input_df['FMASK_TOTAL']) logging.debug(' Max Fmask threshold: {}'.format( ini['SUMMARY']['max_fmask_pct'])) input_df = input_df[ input_df['FMASK_PCT'] <= ini['SUMMARY']['max_fmask_pct']] # Filter low count SLC-off images if ini['SUMMARY']['min_slc_off_pct'] > 0 and not input_df.empty: logging.debug(' Mininum SLC-off threshold: {}%'.format( ini['SUMMARY']['min_slc_off_pct'])) # logging.debug(' Maximum pixel count: {}'.format( # max_pixel_count)) slc_off_mask = ((input_df['PLATFORM'] == 'LE07') & ((input_df['YEAR'] >= 2004) | ((input_df['YEAR'] == 2003) & (input_df['DOY'] > 151)))) slc_off_pct = 100 * (input_df['PIXEL_COUNT'] / input_df['PIXEL_TOTAL']) # slc_off_pct = 100 * (input_df['PIXEL_COUNT'] / max_pixel_count) input_df = input_df[((slc_off_pct >= ini['SUMMARY']['min_slc_off_pct']) & slc_off_mask) | (~slc_off_mask)] if input_df.empty: logging.error(' Empty dataframe after filtering, exiting') return False if not os.path.isfile(output_daily_path): logging.info('\nWriting daily values to Excel') excel_f = ExcelWriter(output_daily_path) for zone_name in sorted(zone_name_list): logging.info(' {}'.format(zone_name)) zone_df = input_df[input_df['ZONE_NAME'] == zone_name] zone_df.to_excel(excel_f, zone_name, index=False, float_format='%.4f') # zone_df.to_excel(excel_f, zone_name, index=False) del zone_df excel_f.save() if not os.path.isfile(output_annual_path): logging.info('\nComputing annual summaries') annual_df = input_df \ .groupby(['ZONE_NAME', 'YEAR']) \ .agg({ 'PIXEL_COUNT': ['count', 'mean'], 'PIXEL_TOTAL': ['mean'], 'FMASK_COUNT': 'mean', 'FMASK_TOTAL': 'mean', 'CLOUD_SCORE': 'mean', 'ETSTAR_COUNT': 'mean', 'NDVI_TOA': 'mean', 'NDWI_TOA': 'mean', 'ALBEDO_SUR': 'mean', 'TS': 'mean', # 'EVI_SUR': 'mean', 'EVI_SUR': ['mean', 'median', 'min', 'max'], 'ETSTAR_MEAN': 'mean', 'ETG_MEAN': 'mean', 'ETG_LPI': 'mean', 'ETG_UPI': 'mean', 'ETG_LCI': 'mean', 'ETG_UCI': 'mean', 'ET_MEAN': 'mean', 'ET_LPI': 'mean', 'ET_UPI': 'mean', 'ET_LCI': 'mean', 'ET_UCI': 'mean', 'WY_ETO': 'mean', 'WY_PPT': 'mean' }) annual_df.columns = annual_df.columns.map('_'.join) annual_df = annual_df.rename(columns={ 'PIXEL_COUNT_count': 'SCENE_COUNT', 'PIXEL_COUNT_mean': 'PIXEL_COUNT' }) annual_df = annual_df.rename( columns={ 'EVI_SUR_mean': 'EVI_SUR_MEAN', 'EVI_SUR_median': 'EVI_SUR_MEDIAN', 'EVI_SUR_min': 'EVI_SUR_MIN', 'EVI_SUR_max': 'EVI_SUR_MAX' }) annual_df.rename(columns=lambda x: str(x).replace('_mean', ''), inplace=True) annual_df['SCENE_COUNT'] = annual_df['SCENE_COUNT'].astype(np.int) annual_df['PIXEL_COUNT'] = annual_df['PIXEL_COUNT'].astype(np.int) annual_df['PIXEL_TOTAL'] = annual_df['PIXEL_TOTAL'].astype(np.int) annual_df['FMASK_COUNT'] = annual_df['FMASK_COUNT'].astype(np.int) annual_df['FMASK_TOTAL'] = annual_df['FMASK_TOTAL'].astype(np.int) annual_df['ETSTAR_COUNT'] = annual_df['ETSTAR_COUNT'].astype(np.int) annual_df = annual_df.reset_index() # Convert ETo units if (ini['BEAMER']['eto_units'] == 'mm' and ini['TABLES']['eto_units'] == 'mm'): pass elif (ini['BEAMER']['eto_units'] == 'mm' and ini['TABLES']['eto_units'] == 'in'): annual_df[eto_fields] /= (25.4) elif (ini['BEAMER']['eto_units'] == 'mm' and ini['TABLES']['eto_units'] == 'ft'): annual_df[eto_fields] /= (12 * 25.4) else: logging.error( ('\nERROR: Input units {} and output units {} are not ' + 'currently supported, exiting').format( ini['BEAMER']['eto_units'], ini['TABLES']['eto_units'])) sys.exit() # Convert PPT units if (ini['BEAMER']['ppt_units'] == 'mm' and ini['TABLES']['ppt_units'] == 'mm'): pass elif (ini['BEAMER']['ppt_units'] == 'mm' and ini['TABLES']['ppt_units'] == 'in'): annual_df[ppt_fields] /= (25.4) elif (ini['BEAMER']['ppt_units'] == 'mm' and ini['TABLES']['ppt_units'] == 'ft'): annual_df[ppt_fields] /= (12 * 25.4) else: logging.error( ('\nERROR: Input units {} and output units {} are not ' + 'currently supported, exiting').format( ini['BEAMER']['ppt_units'], ini['TABLES']['ppt_units'])) sys.exit() logging.info('\nWriting annual values to Excel') excel_f = ExcelWriter(output_annual_path) for zone_name in sorted(zone_name_list): logging.info(' {}'.format(zone_name)) zone_df = annual_df[annual_df['ZONE_NAME'] == zone_name] zone_df.to_excel(excel_f, zone_name, index=False, float_format='%.4f') del zone_df excel_f.save()
def main(ini_path, show_flag=False, overwrite_flag=True): """Generate Bokeh figures Bokeh issues: Adjust y range based on non-muted data https://stackoverflow.com/questions/43620837/how-to-get-bokeh-to-dynamically-adjust-y-range-when-panning Linked interactive legends so that there is only one legend for the gridplot Maybe hide or mute QA values above max (instead of filtering them in advance) Args: ini_path (str): show_flag (bool): if True, show the figures in the browser. Default is False. overwrite_flag (bool): if True, overwrite existing tables. Default is True (for now) """ logging.info('\nGenerate interactive timeseries figures') # Eventually read from INI plot_var_list = ['NDVI_TOA', 'ALBEDO_SUR', 'TS', 'NDWI_TOA', 'EVI_SUR'] # plot_var_list = [ # 'NDVI_TOA', 'ALBEDO_SUR', 'TS', 'NDWI_TOA', # 'CLOUD_SCORE', 'FMASK_PCT'] output_folder = 'figures' # Read config file ini = inputs.read(ini_path) inputs.parse_section(ini, section='INPUTS') inputs.parse_section(ini, section='ZONAL_STATS') inputs.parse_section(ini, section='SUMMARY') inputs.parse_section(ini, section='FIGURES') inputs.parse_section(ini, section='BEAMER') # Hardcode GRIDMET month range to the water year ini['SUMMARY']['gridmet_start_month'] = 10 ini['SUMMARY']['gridmet_end_month'] = 9 # Start/end year year_list = list(range( ini['INPUTS']['start_year'], ini['INPUTS']['end_year'] + 1)) month_list = list(utils.wrapped_range( ini['INPUTS']['start_month'], ini['INPUTS']['end_month'], 1, 12)) doy_list = list(utils.wrapped_range( ini['INPUTS']['start_doy'], ini['INPUTS']['end_doy'], 1, 366)) # GRIDMET month range (default to water year) gridmet_start_month = ini['SUMMARY']['gridmet_start_month'] gridmet_end_month = ini['SUMMARY']['gridmet_end_month'] gridmet_months = list(utils.month_range( gridmet_start_month, gridmet_end_month)) logging.info('\nGridmet months: {}'.format( ', '.join(map(str, gridmet_months)))) # Get ee features from shapefile zone_geom_list = gdc.shapefile_2_geom_list_func( ini['INPUTS']['zone_shp_path'], zone_field=ini['INPUTS']['zone_field'], reverse_flag=False) # Filter features by FID before merging geometries if ini['INPUTS']['fid_keep_list']: zone_geom_list = [ zone_obj for zone_obj in zone_geom_list if zone_obj[0] in ini['INPUTS']['fid_keep_list']] if ini['INPUTS']['fid_skip_list']: zone_geom_list = [ zone_obj for zone_obj in zone_geom_list if zone_obj[0] not in ini['INPUTS']['fid_skip_list']] # # Filter features by FID before merging geometries # if ini['INPUTS']['fid_keep_list']: # landsat_df = landsat_df[landsat_df['ZONE_FID'].isin( # ini['INPUTS']['fid_keep_list'])] # if ini['INPUTS']['fid_skip_list']: # landsat_df = landsat_df[~landsat_df['ZONE_FID'].isin( # ini['INPUTS']['fid_skip_list'])] logging.info('\nProcessing zones') for zone_fid, zone_name, zone_json in zone_geom_list: zone_name = zone_name.replace(' ', '_') logging.info('ZONE: {} (FID: {})'.format(zone_name, zone_fid)) zone_stats_ws = os.path.join( ini['ZONAL_STATS']['output_ws'], zone_name) zone_figures_ws = os.path.join( ini['SUMMARY']['output_ws'], zone_name, output_folder) if not os.path.isdir(zone_stats_ws): logging.debug(' Folder {} does not exist, skipping'.format( zone_stats_ws)) continue elif not os.path.isdir(zone_figures_ws): os.makedirs(zone_figures_ws) # Input paths landsat_daily_path = os.path.join( zone_stats_ws, '{}_landsat_daily.csv'.format(zone_name)) gridmet_daily_path = os.path.join( zone_stats_ws, '{}_gridmet_daily.csv'.format(zone_name)) gridmet_monthly_path = os.path.join( zone_stats_ws, '{}_gridmet_monthly.csv'.format(zone_name)) if not os.path.isfile(landsat_daily_path): logging.error(' Landsat daily CSV does not exist, skipping zone') continue elif (not os.path.isfile(gridmet_daily_path) and not os.path.isfile(gridmet_monthly_path)): logging.error( ' GRIDMET daily or monthly CSV does not exist, skipping zone') continue # DEADBEEF - Eventually support generating only Landsat figures # logging.error( # ' GRIDMET daily and/or monthly CSV files do not exist.\n' # ' ETo and PPT will not be processed.') # Output paths output_doy_path = os.path.join( zone_figures_ws, '{}_timeseries_doy.html'.format(zone_name)) output_date_path = os.path.join( zone_figures_ws, '{}_timeseries_date.html'.format(zone_name)) logging.debug(' Reading Landsat CSV') landsat_df = pd.read_csv(landsat_daily_path) logging.debug(' Filtering Landsat dataframe') landsat_df = landsat_df[landsat_df['PIXEL_COUNT'] > 0] # QA field should have been written in zonal stats code # Eventually this block can be removed if 'QA' not in landsat_df.columns.values: landsat_df['QA'] = 0 # # This assumes that there are L5/L8 images in the dataframe # if not landsat_df.empty: # max_pixel_count = max(landsat_df['PIXEL_COUNT']) # else: # max_pixel_count = 0 if year_list: landsat_df = landsat_df[landsat_df['YEAR'].isin(year_list)] if month_list: landsat_df = landsat_df[landsat_df['MONTH'].isin(month_list)] if doy_list: landsat_df = landsat_df[landsat_df['DOY'].isin(doy_list)] # Assume the default is for these to be True and only filter if False if not ini['INPUTS']['landsat4_flag']: landsat_df = landsat_df[landsat_df['PLATFORM'] != 'LT04'] if not ini['INPUTS']['landsat5_flag']: landsat_df = landsat_df[landsat_df['PLATFORM'] != 'LT05'] if not ini['INPUTS']['landsat7_flag']: landsat_df = landsat_df[landsat_df['PLATFORM'] != 'LE07'] if not ini['INPUTS']['landsat8_flag']: landsat_df = landsat_df[landsat_df['PLATFORM'] != 'LC08'] if ini['INPUTS']['path_keep_list']: landsat_df = landsat_df[ landsat_df['PATH'].isin(ini['INPUTS']['path_keep_list'])] if (ini['INPUTS']['row_keep_list'] and ini['INPUTS']['row_keep_list'] != ['XXX']): landsat_df = landsat_df[ landsat_df['ROW'].isin(ini['INPUTS']['row_keep_list'])] if ini['INPUTS']['scene_id_keep_list']: # Replace XXX with primary ROW value for checking skip list SCENE_ID scene_id_df = pd.Series([ s.replace('XXX', '{:03d}'.format(int(r))) for s, r in zip(landsat_df['SCENE_ID'], landsat_df['ROW'])]) landsat_df = landsat_df[scene_id_df.isin( ini['INPUTS']['scene_id_keep_list']).values] # This won't work: SCENE_ID have XXX but scene_id_skip_list don't # landsat_df = landsat_df[landsat_df['SCENE_ID'].isin( # ini['INPUTS']['scene_id_keep_list'])] if ini['INPUTS']['scene_id_skip_list']: # Replace XXX with primary ROW value for checking skip list SCENE_ID scene_id_df = pd.Series([ s.replace('XXX', '{:03d}'.format(int(r))) for s, r in zip(landsat_df['SCENE_ID'], landsat_df['ROW'])]) landsat_df = landsat_df[np.logical_not(scene_id_df.isin( ini['INPUTS']['scene_id_skip_list']).values)] # This won't work: SCENE_ID have XXX but scene_id_skip_list don't # landsat_df = landsat_df[np.logical_not(landsat_df['SCENE_ID'].isin( # ini['INPUTS']['scene_id_skip_list']))] # Filter by QA/QC value if ini['SUMMARY']['max_qa'] >= 0 and not landsat_df.empty: logging.debug(' Maximum QA: {0}'.format( ini['SUMMARY']['max_qa'])) landsat_df = landsat_df[ landsat_df['QA'] <= ini['SUMMARY']['max_qa']] # Filter by average cloud score if ini['SUMMARY']['max_cloud_score'] < 100 and not landsat_df.empty: logging.debug(' Maximum cloud score: {0}'.format( ini['SUMMARY']['max_cloud_score'])) landsat_df = landsat_df[ landsat_df['CLOUD_SCORE'] <= ini['SUMMARY']['max_cloud_score']] # Filter by Fmask percentage if ini['SUMMARY']['max_fmask_pct'] < 100 and not landsat_df.empty: landsat_df['FMASK_PCT'] = 100 * ( landsat_df['FMASK_COUNT'] / landsat_df['FMASK_TOTAL']) logging.debug(' Max Fmask threshold: {}'.format( ini['SUMMARY']['max_fmask_pct'])) landsat_df = landsat_df[ landsat_df['FMASK_PCT'] <= ini['SUMMARY']['max_fmask_pct']] # Filter low count SLC-off images if ini['SUMMARY']['min_slc_off_pct'] > 0 and not landsat_df.empty: logging.debug(' Mininum SLC-off threshold: {}%'.format( ini['SUMMARY']['min_slc_off_pct'])) # logging.debug(' Maximum pixel count: {}'.format( # max_pixel_count)) slc_off_mask = ( (landsat_df['PLATFORM'] == 'LE07') & ((landsat_df['YEAR'] >= 2004) | ((landsat_df['YEAR'] == 2003) & (landsat_df['DOY'] > 151)))) slc_off_pct = 100 * (landsat_df['PIXEL_COUNT'] / landsat_df['PIXEL_TOTAL']) # slc_off_pct = 100 * (landsat_df['PIXEL_COUNT'] / max_pixel_count) landsat_df = landsat_df[ ((slc_off_pct >= ini['SUMMARY']['min_slc_off_pct']) & slc_off_mask) | (~slc_off_mask)] if landsat_df.empty: logging.error( ' Empty Landsat dataframe after filtering, skipping zone') continue # Aggregate GRIDMET (to water year) if os.path.isfile(gridmet_monthly_path): logging.debug(' Reading montly GRIDMET CSV') gridmet_df = pd.read_csv(gridmet_monthly_path) elif os.path.isfile(gridmet_daily_path): logging.debug(' Reading daily GRIDMET CSV') gridmet_df = pd.read_csv(gridmet_daily_path) logging.debug(' Computing GRIDMET summaries') # Summarize GRIDMET for target months year if (gridmet_start_month in [10, 11, 12] and gridmet_end_month in [10, 11, 12]): month_mask = ( (gridmet_df['MONTH'] >= gridmet_start_month) & (gridmet_df['MONTH'] <= gridmet_end_month)) gridmet_df.loc[month_mask, 'GROUP_YEAR'] = gridmet_df['YEAR'] + 1 elif (gridmet_start_month in [10, 11, 12] and gridmet_end_month not in [10, 11, 12]): month_mask = gridmet_df['MONTH'] >= gridmet_start_month gridmet_df.loc[month_mask, 'GROUP_YEAR'] = gridmet_df['YEAR'] + 1 month_mask = gridmet_df['MONTH'] <= gridmet_end_month gridmet_df.loc[month_mask, 'GROUP_YEAR'] = gridmet_df['YEAR'] else: month_mask = ( (gridmet_df['MONTH'] >= gridmet_start_month) & (gridmet_df['MONTH'] <= gridmet_end_month)) gridmet_df.loc[month_mask, 'GROUP_YEAR'] = gridmet_df['YEAR'] # GROUP_YEAR for rows not in the GRIDMET month range will be NAN gridmet_df = gridmet_df[~pd.isnull(gridmet_df['GROUP_YEAR'])] if year_list: gridmet_df = gridmet_df[gridmet_df['GROUP_YEAR'].isin(year_list)] if gridmet_df.empty: logging.error( ' Empty GRIDMET dataframe after filtering by year') continue # Group GRIDMET data by user specified range (default is water year) gridmet_group_df = gridmet_df \ .groupby(['ZONE_NAME', 'ZONE_FID', 'GROUP_YEAR']) \ .agg({'ETO': np.sum, 'PPT': np.sum}) \ .reset_index() \ .sort_values(by='GROUP_YEAR') # .rename(columns={'ETO': 'ETO', 'PPT': 'ETO'}) \ # Rename wasn't working when chained... gridmet_group_df.rename(columns={'GROUP_YEAR': 'YEAR'}, inplace=True) gridmet_group_df['YEAR'] = gridmet_group_df['YEAR'].astype(int) # # Group GRIDMET data by month # gridmet_month_df = gridmet_df\ # .groupby(['ZONE_NAME', 'ZONE_FID', 'GROUP_YEAR', 'MONTH']) \ # .agg({'ETO': np.sum, 'PPT': np.sum}) \ # .reset_index() \ # .sort_values(by=['GROUP_YEAR', 'MONTH']) # gridmet_month_df.rename(columns={'GROUP_YEAR': 'YEAR'}, inplace=True) # # Rename monthly PPT columns # gridmet_month_df['MONTH'] = 'PPT_M' + gridmet_month_df['MONTH'].astype(str) # # Pivot rows up to separate columns # gridmet_month_df = gridmet_month_df.pivot_table( # 'PPT', ['ZONE_NAME', 'YEAR'], 'MONTH') # gridmet_month_df.reset_index(inplace=True) # columns = ['ZONE_NAME', 'YEAR'] + ['PPT_M{}'.format(m) for m in gridmet_months] # gridmet_month_df = gridmet_month_df[columns] # del gridmet_month_df.index.name # Merge Landsat and GRIDMET collections zone_df = landsat_df.merge( gridmet_group_df, on=['ZONE_NAME', 'ZONE_FID', 'YEAR']) if zone_df is None or zone_df.empty: logging.info(' Empty zone dataframe, not generating figures') continue # Compute ETg zone_df['ETG_MEAN'] = zone_df['ETSTAR_MEAN'] * ( zone_df['ETO'] - zone_df['PPT']) zone_df['ETG_LPI'] = zone_df['ETSTAR_LPI'] * ( zone_df['ETO'] - zone_df['PPT']) zone_df['ETG_UPI'] = zone_df['ETSTAR_UPI'] * ( zone_df['ETO'] - zone_df['PPT']) zone_df['ETG_LCI'] = zone_df['ETSTAR_LCI'] * ( zone_df['ETO'] - zone_df['PPT']) zone_df['ETG_UCI'] = zone_df['ETSTAR_UCI'] * ( zone_df['ETO'] - zone_df['PPT']) # Compute ET zone_df['ET_MEAN'] = zone_df['ETG_MEAN'] + zone_df['PPT'] zone_df['ET_LPI'] = zone_df['ETG_LPI'] + zone_df['PPT'] zone_df['ET_UPI'] = zone_df['ETG_UPI'] + zone_df['PPT'] zone_df['ET_LCI'] = zone_df['ETG_LCI'] + zone_df['PPT'] zone_df['ET_UCI'] = zone_df['ETG_UCI'] + zone_df['PPT'] # ORIGINAL PLOTTING CODE # Check that plot variables are present for plot_var in plot_var_list: if plot_var not in landsat_df.columns.values: logging.error( ' The plotting variable {} does not exist in the ' 'dataframe'.format(plot_var)) sys.exit() # if ini['INPUTS']['scene_id_keep_list']: # # Replace XXX with primary ROW value for checking skip list SCENE_ID # scene_id_df = pd.Series([ # s.replace('XXX', '{:03d}'.format(int(r))) # for s, r in zip(landsat_df['SCENE_ID'], landsat_df['ROW'])]) # landsat_df = landsat_df[scene_id_df.isin( # ini['INPUTS']['scene_id_keep_list']).values] # # This won't work: SCENE_ID have XXX but scene_id_skip_list don't # # landsat_df = landsat_df[landsat_df['SCENE_ID'].isin( # # ini['INPUTS']['scene_id_keep_list'])] # if ini['INPUTS']['scene_id_skip_list']: # # Replace XXX with primary ROW value for checking skip list SCENE_ID # scene_id_df = pd.Series([ # s.replace('XXX', '{:03d}'.format(int(r))) # for s, r in zip(landsat_df['SCENE_ID'], landsat_df['ROW'])]) # landsat_df = landsat_df[np.logical_not(scene_id_df.isin( # ini['INPUTS']['scene_id_skip_list']).values)] # # This won't work: SCENE_ID have XXX but scene_id_skip_list don't # # landsat_df = landsat_df[np.logical_not(landsat_df['SCENE_ID'].isin( # # ini['INPUTS']['scene_id_skip_list']))] # Compute colors for each QA value logging.debug(' Building column data source') qa_values = sorted(list(set(zone_df['QA'].values))) colors = { qa: "#%02x%02x%02x" % (int(r), int(g), int(b)) for qa, (r, g, b, _) in zip( qa_values, 255 * cm.viridis(mpl.colors.Normalize()(qa_values))) } logging.debug(' QA values: {}'.format( ', '.join(map(str, qa_values)))) # Unpack the data by QA type to support interactive legends sources = dict() # sources = defaultdict(dict) # platform_list = ['LT04', 'LT05', 'LE07', 'LC08'] for qa_value in qa_values: # for platform in platform_list: # qa_df = zone_df[ # (zone_df['PLATFORM'] == platform) & # (zone_df['QA'] == qa_value)] qa_df = zone_df[zone_df['QA'] == qa_value] qa_data = { 'INDEX': list(range(len(qa_df.index))), 'PLATFORM': qa_df['PLATFORM'], 'DATE': pd.to_datetime(qa_df['DATE']), 'TIME': pd.to_datetime(qa_df['DATE']).map( lambda x: x.strftime('%Y-%m-%d')), 'DOY': qa_df['DOY'].values, 'QA': qa_df['QA'].values, 'COLOR': [colors[qa] for qa in qa_df['QA'].values] } for plot_var in plot_var_list: if plot_var in qa_df.columns.values: qa_data.update({plot_var: qa_df[plot_var].values}) sources[qa_value] = bokeh.models.ColumnDataSource(qa_data) # sources[qa_value][platform] = bokeh.models.ColumnDataSource( # qa_data) tooltips = [ ("LANDSAT", "@PLATFORM"), ("DATE", "@TIME"), ("DOY", "@DOY")] # hover_tool = bokeh.models.HoverTool(tooltips=tooltips) # tools="xwheel_zoom,xpan,xbox_zoom,reset,box_select" # tools = [ # hover_tool, # bokeh.models.WheelZoomTool(dimensions='width'), # bokeh.models.PanTool(dimensions='width'), # bokeh.models.BoxZoomTool(dimensions='width'), # bokeh.models.ResetTool(), # bokeh.models.BoxSelectTool()] # Selection hover_circle = Circle( fill_color='#ff0000', line_color='#ff0000') selected_circle = Circle( fill_color='COLOR', line_color='COLOR') nonselected_circle = Circle( fill_color='#aaaaaa', line_color='#aaaaaa') # Plot the data by DOY logging.debug(' Building DOY timeseries figure') if os.path.isfile(output_doy_path): os.remove(output_doy_path) output_file(output_doy_path, title=zone_name) figure_args = dict( plot_width=750, plot_height=250, title=None, tools="xwheel_zoom,xpan,xbox_zoom,reset,box_select", # tools="xwheel_zoom,xpan,xbox_zoom,reset,tap", active_scroll="xwheel_zoom") plot_args = dict( size=4, alpha=0.9, color='COLOR') if ini['SUMMARY']['max_qa'] > 0: plot_args['legend'] = 'QA' figures = [] for plot_i, plot_var in enumerate(plot_var_list): if plot_i == 0: f = figure( # x_range=bokeh.models.Range1d(1, 366, bounds=(1, 366)), y_axis_label=plot_var, **figure_args) else: f = figure( x_range=f.x_range, y_axis_label=plot_var, **figure_args) # # Add each QA level as a separate object # for qa, platform_sources in sorted(sources.items()): # for platform, source in platform_sources.items(): # if platform == 'LT05': # r = f.triangle( # 'DOY', plot_var, source=source, **plot_args) # elif platform == 'LE07': # r = f.square( # 'DOY', plot_var, source=source, **plot_args) # elif platform == 'LC08': # r = f.circle( # 'DOY', plot_var, source=source, **plot_args) # else: # r = f.diamond( # 'DOY', plot_var, source=source, **plot_args) # r.hover_glyph = hover_circle # r.selection_glyph = selected_circle # r.nonselection_glyph = nonselected_circle # r.muted_glyph = nonselected_circle # hover_tool.renderers.append(r) # Add each QA level as a separate object for qa, source in sorted(sources.items()): r = f.circle('DOY', plot_var, source=source, **plot_args) r.hover_glyph = hover_circle r.selection_glyph = selected_circle r.nonselection_glyph = nonselected_circle r.muted_glyph = nonselected_circle # # DEADBEEF - This will display high QA points as muted # if qa > ini['SUMMARY']['max_qa']: # r.muted = True # # r.visible = False f.add_tools(bokeh.models.HoverTool(tooltips=tooltips)) # if ini['SUMMARY']['max_qa'] > 0: f.legend.location = "top_left" f.legend.click_policy = "hide" # f.legend.click_policy = "mute" f.legend.orientation = "horizontal" figures.append(f) del f # Try to not allow more than 4 plots in a column p = gridplot( figures, ncols=len(plot_var_list) // 3, sizing_mode='stretch_both') if show_flag: show(p) save(p) # Plot the data by DATE logging.debug(' Building date timeseries figure') if os.path.isfile(output_date_path): os.remove(output_date_path) output_file(output_date_path, title=zone_name) figure_args = dict( plot_width=750, plot_height=250, title=None, tools="xwheel_zoom,xpan,xbox_zoom,reset,box_select", # tools="xwheel_zoom,xpan,xbox_zoom,reset,tap", active_scroll="xwheel_zoom", x_axis_type="datetime",) plot_args = dict( size=4, alpha=0.9, color='COLOR') if ini['SUMMARY']['max_qa'] > 0: plot_args['legend'] = 'QA' figures = [] for plot_i, plot_var in enumerate(plot_var_list): if plot_i == 0: f = figure( # x_range=bokeh.models.Range1d(x_limit[0], x_limit[1], bounds=x_limit), y_axis_label=plot_var, **figure_args) else: f = figure( x_range=f.x_range, y_axis_label=plot_var, **figure_args) if plot_var == 'TS': f.y_range.bounds = (270, None) # # Add each QA level as a separate object # for qa, platform_sources in sorted(sources.items()): # for platform, source in sorted(platform_sources.items()): # if platform == 'LT05': # r = f.triangle( # 'DATE', plot_var, source=source, **plot_args) # elif platform == 'LE07': # r = f.square( # 'DATE', plot_var, source=source, **plot_args) # elif platform == 'LC08': # r = f.circle( # 'DATE', plot_var, source=source, **plot_args) # else: # r = f.diamond( # 'DATE', plot_var, source=source, **plot_args) # r.hover_glyph = hover_circle # r.selection_glyph = selected_circle # r.nonselection_glyph = nonselected_circle # r.muted_glyph = nonselected_circle # hover_tool.renderers.append(r) # Add each QA level as a separate object for qa, source in sorted(sources.items()): r = f.circle('DATE', plot_var, source=source, **plot_args) r.hover_glyph = hover_circle r.selection_glyph = selected_circle r.nonselection_glyph = nonselected_circle r.muted_glyph = nonselected_circle # # DEADBEEF - This will display high QA points as muted # if qa > ini['SUMMARY']['max_qa']: # r.muted = True # # r.visible = False f.add_tools(bokeh.models.HoverTool(tooltips=tooltips)) # if ini['SUMMARY']['max_qa'] > 0: f.legend.location = "top_left" f.legend.click_policy = "hide" # f.legend.click_policy = "mute" f.legend.orientation = "horizontal" figures.append(f) del f # Try to not allow more than 4 plots in a column p = gridplot( figures, ncols=len(plot_var_list) // 3, sizing_mode='stretch_both') if show_flag: show(p) save(p) # Pause after each iteration if show is True if show_flag: input('Press ENTER to continue')
def test_wrapped_range(a, b, x_min, x_max, expected): """Return the values between a range b for a given start/end""" assert list(utils.wrapped_range(a, b, x_min, x_max)) == expected
def main(ini_path=None, overwrite_flag=False): """Generate summary thumbnails Parameters ---------- ini_path : str overwrite_flag : bool, optional If True, overwrite existing files (the default is False). """ logging.info('\nGenerate summary thumbnails') # Inputs (eventually move to INI file?) vis_args = { 'bands': ['red', 'green', 'blue'], # 'bands': ['swir1', 'nir', 'red'], 'min': [0.01, 0.01, 0.01], 'max': [0.4, 0.4, 0.4], 'gamma': [1.8, 1.8, 1.8] } # Buffer zone polygon zone_buffer = 240 # Generate images by DOY doy_flag = True # Generate images by date date_flag = True # Read config file ini = inputs.read(ini_path) inputs.parse_section(ini, section='INPUTS') inputs.parse_section(ini, section='SPATIAL') inputs.parse_section(ini, section='SUMMARY') year_list = range(ini['INPUTS']['start_year'], ini['INPUTS']['end_year'] + 1) month_list = list( utils.wrapped_range(ini['INPUTS']['start_month'], ini['INPUTS']['end_month'], 1, 12)) doy_list = list( utils.wrapped_range(ini['INPUTS']['start_doy'], ini['INPUTS']['end_doy'], 1, 366)) # Add merged row XXX to keep list ini['INPUTS']['row_keep_list'].append('XXX') # Get ee features from shapefile zone_geom_list = gdc.shapefile_2_geom_list_func( ini['INPUTS']['zone_shp_path'], zone_field=ini['INPUTS']['zone_field'], reverse_flag=False) # Filter features by FID before merging geometries if ini['INPUTS']['fid_keep_list']: zone_geom_list = [ zone_obj for zone_obj in zone_geom_list if zone_obj[0] in ini['INPUTS']['fid_keep_list'] ] if ini['INPUTS']['fid_skip_list']: zone_geom_list = [ zone_obj for zone_obj in zone_geom_list if zone_obj[0] not in ini['INPUTS']['fid_skip_list'] ] # Need zone_shp_path projection to build EE geometries zone = {} zone['osr'] = gdc.feature_path_osr(ini['INPUTS']['zone_shp_path']) zone['proj'] = gdc.osr_wkt(zone['osr']) # zone['proj'] = ee.Projection(zone['proj']).wkt().getInfo() # zone['proj'] = zone['proj'].replace('\n', '').replace(' ', '') # logging.debug(' Zone Projection: {}'.format(zone['proj'])) # Initialize Earth Engine API key logging.debug('') ee.Initialize() coll_dict = { 'LT04': 'LANDSAT/LT04/C01/T1_SR', 'LT05': 'LANDSAT/LT05/C01/T1_SR', 'LE07': 'LANDSAT/LE07/C01/T1_SR', 'LC08': 'LANDSAT/LC08/C01/T1_SR' } logging.info('\nProcessing zones') for zone_fid, zone_name, zone_json in zone_geom_list: zone['fid'] = zone_fid zone['name'] = zone_name.replace(' ', '_') zone['json'] = zone_json logging.info('ZONE: {} (FID: {})'.format(zone['name'], zone['fid'])) # Build EE geometry object for zonal stats zone['geom'] = ee.Geometry(geo_json=zone['json'], opt_proj=zone['proj'], opt_geodesic=False) # logging.debug(' Centroid: {}'.format( # zone['geom'].centroid(100).getInfo()['coordinates'])) # Use feature geometry to build extent, transform, and shape zone['extent'] = gdc.Extent( ogr.CreateGeometryFromJson(json.dumps(zone['json'])).GetEnvelope()) # zone['extent'] = gdc.Extent(zone['geom'].GetEnvelope()) zone['extent'] = zone['extent'].ogrenv_swap() zone['extent'] = zone['extent'].buffer(zone_buffer) zone['extent'] = zone['extent'].adjust_to_snap( 'EXPAND', ini['SPATIAL']['snap_x'], ini['SPATIAL']['snap_y'], ini['SPATIAL']['cellsize']) zone['geo'] = zone['extent'].geo(ini['SPATIAL']['cellsize']) zone['transform'] = gdc.geo_2_ee_transform(zone['geo']) # zone['transform'] = '[' + ','.join(map(str, zone['transform'])) + ']' zone['shape'] = zone['extent'].shape(ini['SPATIAL']['cellsize']) logging.debug(' Zone Shape: {}'.format(zone['shape'])) logging.debug(' Zone Transform: {}'.format(zone['transform'])) logging.debug(' Zone Extent: {}'.format(zone['extent'])) # logging.debug(' Zone Geom: {}'.format(zone['geom'].getInfo())) # Build an EE geometry of the extent extent_geom = ee.Geometry.Rectangle(coords=list(zone['extent']), proj=zone['proj'], geodesic=False) if 'SUMMARY' in ini.keys(): zone_output_ws = os.path.join(ini['SUMMARY']['output_ws'], zone['name']) elif 'EXPORT' in ini.keys(): zone_output_ws = os.path.join(ini['EXPORT']['output_ws'], zone['name']) else: logging.error( 'INI file does not contain a SUMMARY or EXPORT section') sys.exit() if not os.path.isdir(zone_output_ws): logging.debug( 'Folder {} does not exist, skipping'.format(zone_output_ws)) continue landsat_daily_path = os.path.join( zone_output_ws, '{}_landsat_daily.csv'.format(zone['name'])) if not os.path.isfile(landsat_daily_path): logging.error(' Landsat daily CSV does not exist, skipping zone') continue output_doy_ws = os.path.join(zone_output_ws, 'thumbnails_doy') output_date_ws = os.path.join(zone_output_ws, 'thumbnails_date') if overwrite_flag and os.path.isdir(output_doy_ws): for file_name in os.listdir(output_doy_ws): os.remove(os.path.join(output_doy_ws, file_name)) if overwrite_flag and os.path.isdir(output_date_ws): for file_name in os.listdir(output_date_ws): os.remove(os.path.join(output_date_ws, file_name)) if doy_flag and not os.path.isdir(output_doy_ws): os.makedirs(output_doy_ws) if date_flag and not os.path.isdir(output_date_ws): os.makedirs(output_date_ws) logging.debug(' Reading Landsat CSV') landsat_df = pd.read_csv(landsat_daily_path) # landsat_df = pd.read_csv( # landsat_daily_path, parse_dates=['DATE'], index_col='DATE') landsat_df = landsat_df[landsat_df['PIXEL_COUNT'] > 0] # Common summary filtering logging.debug(' Filtering using INI SUMMARY parameters') if year_list: landsat_df = landsat_df[landsat_df['YEAR'].isin(year_list)] if month_list: landsat_df = landsat_df[landsat_df['MONTH'].isin(month_list)] if doy_list: landsat_df = landsat_df[landsat_df['DOY'].isin(doy_list)] if ini['INPUTS']['path_keep_list']: landsat_df = landsat_df[landsat_df['PATH'].isin( ini['INPUTS']['path_keep_list'])] if (ini['INPUTS']['row_keep_list'] and ini['INPUTS']['row_keep_list'] != ['XXX']): landsat_df = landsat_df[landsat_df['ROW'].isin( ini['INPUTS']['row_keep_list'])] # Assume the default is for these to be True and only filter if False if not ini['INPUTS']['landsat4_flag']: landsat_df = landsat_df[landsat_df['PLATFORM'] != 'LT04'] if not ini['INPUTS']['landsat5_flag']: landsat_df = landsat_df[landsat_df['PLATFORM'] != 'LT05'] if not ini['INPUTS']['landsat7_flag']: landsat_df = landsat_df[landsat_df['PLATFORM'] != 'LE07'] if not ini['INPUTS']['landsat8_flag']: landsat_df = landsat_df[landsat_df['PLATFORM'] != 'LC08'] if ini['INPUTS']['scene_id_keep_list']: # Replace XXX with primary ROW value for checking skip list SCENE_ID scene_id_df = pd.Series([ s.replace('XXX', '{:03d}'.format(int(r))) for s, r in zip(landsat_df['SCENE_ID'], landsat_df['ROW']) ]) landsat_df = landsat_df[scene_id_df.isin( ini['INPUTS']['scene_id_keep_list']).values] # This won't work: SCENE_ID have XXX but scene_id_skip_list don't # landsat_df = landsat_df[landsat_df['SCENE_ID'].isin( # ini['INPUTS']['scene_id_keep_list'])] if ini['INPUTS']['scene_id_skip_list']: # Replace XXX with primary ROW value for checking skip list SCENE_ID scene_id_df = pd.Series([ s.replace('XXX', '{:03d}'.format(int(r))) for s, r in zip(landsat_df['SCENE_ID'], landsat_df['ROW']) ]) landsat_df = landsat_df[np.logical_not( scene_id_df.isin(ini['INPUTS']['scene_id_skip_list']).values)] # This won't work: SCENE_ID have XXX but scene_id_skip_list don't # landsat_df = landsat_df[np.logical_not(landsat_df['SCENE_ID'].isin( # ini['INPUTS']['scene_id_skip_list']))] # Filter by QA/QC value if ini['SUMMARY']['max_qa'] >= 0 and not landsat_df.empty: logging.debug(' Maximum QA: {0}'.format( ini['SUMMARY']['max_qa'])) landsat_df = landsat_df[ landsat_df['QA'] <= ini['SUMMARY']['max_qa']] # Filter by average cloud score if ini['SUMMARY']['max_cloud_score'] < 100 and not landsat_df.empty: logging.debug(' Maximum cloud score: {0}'.format( ini['SUMMARY']['max_cloud_score'])) landsat_df = landsat_df[ landsat_df['CLOUD_SCORE'] <= ini['SUMMARY']['max_cloud_score']] # Filter by Fmask percentage if ini['SUMMARY']['max_fmask_pct'] < 100 and not landsat_df.empty: landsat_df['FMASK_PCT'] = 100 * (landsat_df['FMASK_COUNT'] / landsat_df['FMASK_TOTAL']) logging.debug(' Max Fmask threshold: {}'.format( ini['SUMMARY']['max_fmask_pct'])) landsat_df = landsat_df[ landsat_df['FMASK_PCT'] <= ini['SUMMARY']['max_fmask_pct']] # Filter low count SLC-off images if ini['SUMMARY']['min_slc_off_pct'] > 0 and not landsat_df.empty: logging.debug(' Mininum SLC-off threshold: {}%'.format( ini['SUMMARY']['min_slc_off_pct'])) # logging.debug(' Maximum pixel count: {}'.format( # max_pixel_count)) slc_off_mask = ((landsat_df['PLATFORM'] == 'LE07') & ((landsat_df['YEAR'] >= 2004) | ((landsat_df['YEAR'] == 2003) & (landsat_df['DOY'] > 151)))) slc_off_pct = 100 * (landsat_df['PIXEL_COUNT'] / landsat_df['PIXEL_TOTAL']) # slc_off_pct = 100 * (landsat_df['PIXEL_COUNT'] / max_pixel_count) landsat_df = landsat_df[( (slc_off_pct >= ini['SUMMARY']['min_slc_off_pct']) & slc_off_mask) | (~slc_off_mask)] if landsat_df.empty: logging.error( ' Empty Landsat dataframe after filtering, skipping zone') continue logging.debug(' Downloading thumbnails') for landsat, start_date in zip(landsat_df['PLATFORM'], landsat_df['DATE']): start_dt = datetime.datetime.strptime(start_date, '%Y-%m-%d') end_dt = start_dt + datetime.timedelta(days=1) end_date = end_dt.strftime('%Y-%m-%d') output_doy_path = os.path.join( output_doy_ws, '{}_{}.png'.format(start_dt.strftime('%j_%Y-%m-%d'), landsat)) output_date_path = os.path.join( output_date_ws, '{}_{}.png'.format(start_dt.strftime('%Y-%m-%d_%j'), landsat)) # DEADBEEF - This seems like a poor approach save_doy_flag = False save_date_flag = False if doy_flag and not os.path.isfile(output_doy_path): save_doy_flag = True if date_flag and not os.path.isfile(output_date_path): save_date_flag = True if not save_doy_flag and not save_date_flag: logging.debug( ' {} - file already exists, skipping'.format(start_date)) continue logging.debug(' {}'.format(start_date)) # logging.debug(' {}'.format(output_path)) if landsat in ['LT04', 'LT05', 'LE07']: ee_coll = ee.ImageCollection(coll_dict[landsat]).select( ['B1', 'B2', 'B3', 'B4', 'B5', 'B7'], ['blue', 'green', 'red', 'nir', 'swir1', 'swir2']) elif landsat in ['LC08']: ee_coll = ee.ImageCollection(coll_dict[landsat]).select( ['B2', 'B3', 'B4', 'B5', 'B6', 'B7'], ['blue', 'green', 'red', 'nir', 'swir1', 'swir2']) ee_coll = ee_coll.filterDate(start_date, end_date) ee_image = ee.Image(ee_coll.median().divide(10000)) \ .visualize(**vis_args) \ .reproject(crs=zone['proj'], crsTransform=zone['transform']) \ .paint(zone['geom'], color=0.5, width=1) \ .clip(extent_geom) # Get the image thumbnail for i in range(10): try: output_url = ee_image.getThumbUrl({'format': 'png'}) break except Exception as e: logging.error(' Exception: {}, retry {}'.format(e, i)) logging.debug('{}'.format(e)) sleep(i**2) for i in range(10): try: # DEADBEEF - This seems like a poor approach if save_doy_flag and save_date_flag: urllib.urlretrieve(output_url, output_doy_path) shutil.copy(output_doy_path, output_date_path) elif save_doy_flag: urllib.urlretrieve(output_url, output_doy_path) elif save_date_flag: urllib.urlretrieve(output_url, output_date_path) break except Exception as e: logging.error(' Exception: {}, retry {}'.format(e, i)) logging.debug('{}'.format(e)) sleep(i**2)