def get_color_params(ds, inx, iny): landsat_dataset = ds myx = inx myy = iny myred = landsat_dataset.red[:, myy, myx].values myblue = landsat_dataset.blue[:, myy, myx].values mygreen = landsat_dataset.green[:, myy, myx].values mynir = landsat_dataset.nir[:, myy, myx].values myswir1 = landsat_dataset.swir1[:, myy, myx].values myswir2 = landsat_dataset.swir2[:, myy, myx].values mypixel_qa = landsat_dataset.pixel_qa[:, myy, myx].values dts = landsat_dataset.time.values scene_count = len(dts) mythermals = np.ones(scene_count) * (273.15) * 10 ordinal_dates = [] for mydate in dts: dval = datetime.utcfromtimestamp(mydate.tolist() / 1e9) ordinal_dates.append(dval.toordinal()) params = (ordinal_dates, myblue, mygreen, myred, mynir, myswir1, myswir2, mythermals, mypixel_qa) mystery_object = ccd.detect(*params) return mystery_object
def _run_ccd_on_pixel(ds, **kwargs): """Performs CCD on a 1x1xn dataset. Returns CCD results. Creates a CCD result from a 1x1xn dimensioned dataset. Flattens all bands to perform analysis. Inputs allows for missing bands. cf_mask is required. Args: ds: xArray dataset with dimensions 1x1xn with any number of SR bands, cf_mask required. params: can be used to define a custom mapping. Returns: The result of ccd.detect """ if 'time' not in ds.dims: raise Exception("You're missing time dims!") available_bands = ds.data_vars scene_count = ds.dims['time'] date = [_n64_to_datetime(t).date().toordinal() for t in ds.time.values] red = np.ones(scene_count) if 'red' not in available_bands else ds.red.values green = np.ones(scene_count) if 'green' not in available_bands else ds.green.values blue = np.ones(scene_count) if 'blue' not in available_bands else ds.blue.values nir = np.ones(scene_count) if 'nir' not in available_bands else ds.nir.values swir1 = np.ones(scene_count) if 'swir1' not in available_bands else ds.swir1.values swir2 = np.ones(scene_count) if 'swir2' not in available_bands else ds.swir2.values thermals = np.ones(scene_count) * (273.15) * 10 if 'thermal' not in available_bands else ds.object.values qa = np.array(ds.pixel_qa.values) bands = (date, blue, green, red, nir, swir1, swir2, thermals, qa) return ccd.detect(*bands, **kwargs)
def test_sample_data_sets(): """ Sanity test to ensure all test data sets run to completion """ samples = [ 'test/resources/sample_1.csv', 'test/resources/sample_2.csv', 'test/resources/sample_WA_grid08_row9_col2267_persistent_snow.csv', 'test/resources/sample_WA_grid08_row12_col2265_fmask_fail.csv', 'test/resources/sample_WA_grid08_row999_col1_normal.csv', 'test/resources/test_3657_3610_observations.csv' ] params = { 'QA_BITPACKED': False, 'QA_FILL': 255, 'QA_CLEAR': 0, 'QA_WATER': 1, 'QA_SHADOW': 2, 'QA_SNOW': 3, 'QA_CLOUD': 4 } for sample in samples: data = read_data(sample) results = ccd.detect(data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7], data[8], params=params)
def CCD_Points(Data): params = {'QA_BITPACKED': False, 'QA_FILL': 255, 'QA_CLEAR': 0, 'QA_WATER': 1, 'QA_SHADOW': 2, 'QA_SNOW': 3, 'QA_CLOUD': 4} dates, blues, greens, reds, nirs, swir1s, swir2s, thermals, qas = Data.T results = ccd.detect(dates, blues, greens, reds, nirs, swir1s, swir2s, thermals, qas, params=params) start_dates = [] end_dates = [] break_dates = [] for num, result in enumerate(results['change_models']): time_length = result['end_day'] - result['start_day'] if time_length > 365: #if result['change_probability'] == 1 and time_length > 730: #if result['change_probability'] == 1 : start_dates.append(result['start_day']) end_dates.append(result['end_day']) break_dates.append(result['break_day']) return {"start_dates": start_dates, "end_dates": end_dates, "break_dates": break_dates, }
def test_endfit(): """ Flex the code the generates start fits. """ sample = 'test/resources/h04v03_-1945125_2844645_pixel_endfit.npy' result = ccd.detect(**np.load(sample)[1]) assert result['change_models'][-1]['curve_qa'] == params.CURVE_QA['END']
def test_insuff_clear(): """ Flex the insufficient clear procedure code to make sure it runs. """ sample = 'test/resources/h04v03_-1947075_2846265_pixel_insuff.npy' result = ccd.detect(**np.load(sample)[1]) assert result['change_models'][0]['curve_qa'] == params.CURVE_QA[ 'INSUF_CLEAR']
def test_perm_snow(): """ Flex the permanent snow procedure code. """ sample = 'test/resources/h04v03_-1947105_2846265_pixel_snow.npy' result = ccd.detect(**np.load(sample)[1]) assert result['change_models'][0]['curve_qa'] == params.CURVE_QA[ 'PERSIST_SNOW']
def detect(timeseries): cx, cy, px, py = first(timeseries) return format(cx=cx, cy=cy, px=px, py=py, dates=get('dates', second(timeseries)), ccdresult=ccd.detect(**second(timeseries)))
def test_npy(): """ Sanity tests for npy test data sets """ samples = ['test/resources/h03v09_-2010765_1964625_pixel.npy' ] # Main loop failure in LF for sample in samples: dat = np.load(sample) results = ccd.detect(**dat[1])
def run_pyccd2(b): # Run pyCCD on current point Plot_interface.pyccd_flag2 = True # Display the legend Plot_interface.lc7.display_legend = True dfPyCCD = Plot_interface.click_df # First two lines no longer required bc we are removing NA's when we load the TS dfPyCCD['pixel_qa'][dfPyCCD['pixel_qa'] > 4] = 0 #TODO: Paramaterize everything params = { 'QA_BITPACKED': False, 'QA_FILL': 255, 'QA_CLEAR': 0, 'QA_WATER': 1, 'QA_SHADOW': 2, 'QA_SNOW': 3, 'QA_CLOUD': 4 } dates = np.array(dfPyCCD['ord_time']) blues = np.array(dfPyCCD['BLUE']) greens = np.array(dfPyCCD['GREEN']) reds = np.array(dfPyCCD['RED']) nirs = np.array(dfPyCCD['NIR']) swir1s = np.array(dfPyCCD['SWIR1']) swir2s = np.array(dfPyCCD['SWIR2']) thermals = np.array(dfPyCCD['THERMAL']) qas = np.array(dfPyCCD['pixel_qa']) results = ccd.detect(dates, blues, greens, reds, nirs, swir1s, swir2s, thermals, qas, params=params) band_names = [ 'Blue SR', 'Green SR', 'Red SR', 'NIR SR', 'SWIR1 SR', 'SWIR2 SR', 'THERMAL' ] plotlabel = band_names[Plot_interface.band_index2] plot_arrays = [blues, greens, reds, nirs, swir1s, swir2s] plotband = plot_arrays[Plot_interface.band_index2] Plot_interface.plot_pyccd(results, Plot_interface.band_index2, plotband, dates, (0, 4000), 'PyCCD Results', 'clicked_ts')
def run_pyccd(b): # Run pyCCD on current point Plot_interface.pyccd_flag = True Plot_interface.lc5.display_legend = True dfPyCCD = Plot_interface.sample_df dfPyCCD['pixel_qa'][dfPyCCD['pixel_qa'] > 4] = 0 #TODO: Paramaterize everything params = { 'QA_BITPACKED': False, 'QA_FILL': 255, 'QA_CLEAR': 0, 'QA_WATER': 1, 'QA_SHADOW': 2, 'QA_SNOW': 3, 'QA_CLOUD': 4 } dates = np.array(dfPyCCD['ord_time']) blues = np.array(dfPyCCD['BLUE']) greens = np.array(dfPyCCD['GREEN']) reds = np.array(dfPyCCD['RED']) nirs = np.array(dfPyCCD['NIR']) swir1s = np.array(dfPyCCD['SWIR1']) swir2s = np.array(dfPyCCD['SWIR2']) thermals = np.array(dfPyCCD['THERMAL']) qas = np.array(dfPyCCD['pixel_qa']) results = ccd.detect(dates, blues, greens, reds, nirs, swir1s, swir2s, thermals, qas, params=params) band_names = [ 'Blue SR', 'Green SR', 'Red SR', 'NIR SR', 'SWIR1 SR', 'SWIR2 SR', 'THERMAL' ] plotlabel = band_names[Plot_interface.band_index1] plot_arrays = [blues, greens, reds, nirs, swir1s, swir2s] plotband = plot_arrays[Plot_interface.band_index1] Plot_interface.plot_pyccd(results, Plot_interface.band_index1, plotband, dates, (0, 4000), 'PyCCD Results', 'sample_ts')
def sample(path, format): """Subcommand for processing sample data.""" logger.debug("Loading data...") samples = np.genfromtxt(path, delimiter=',', dtype=np.int).T logger.debug("Building change model...") results = ccd.detect(*samples) if format == 'table': click.echo(results_to_table(results)) else: click.echo(json.dumps(results, indent=2)) logger.debug("Done...")
def test_sample_data_sets(): """ Sanity test to ensure all test data sets run to completion """ samples = [ 'test/resources/sample_1.csv', 'test/resources/sample_2.csv', 'test/resources/sample_WA_grid08_row9_col2267_persistent_snow.csv', 'test/resources/sample_WA_grid08_row12_col2265_fmask_fail.csv', 'test/resources/sample_WA_grid08_row999_col1_normal.csv', 'test/resources/test_3657_3610_observations.csv' ] for sample in samples: data = read_data(sample) results = ccd.detect(data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7], data[8])
def test_validate_no_preprocessing_sample_2_detection_results(): """ Sample 2 contains two changes and should test the full path through the algorithm including preprocessing """ data = read_data("test/resources/sample_2.csv") results = ccd.detect(data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7], data[8], preprocess=False) assert len(results) != 2, "expected: !{}, actual: {}".format( 2, len(results))
def compute_ccd(coords, year_range=(2000, 2020), doy_range=(1, 365)): # get data from Google Earth Engine # list index order: # 'longitude', 1 # 'latitude',2 # 'time',3 # 'BLUE',4 # 'GREEN',5 # 'RED',6 # 'NIR',7 # 'SWIR1',8 # 'SWIR2',9 # 'THERMAL',10 # 'pixel_qa'11 ### get GEE data from the specific point data_collection = get_full_collection(coords, year_range, doy_range) data_point = get_data_full(data_collection, coords)[1::] # generate a merge/fusion mask layer of nan/none values to filter all data nan_masks = [[0 if dp[i] is None else 1 for dp in data_point] for i in range(3, 12)] # fusion masks nan_mask = [0 if 0 in m else 1 for m in zip(*nan_masks)] # get each features applying the mask dates, blues, greens, reds, nirs, swir1s, swir2s, thermals, qas = \ mask([dp[3] for dp in data_point], nan_mask), mask([dp[4] for dp in data_point], nan_mask), \ mask([dp[5] for dp in data_point], nan_mask), mask([dp[6] for dp in data_point], nan_mask), \ mask([dp[7] for dp in data_point], nan_mask), mask([dp[8] for dp in data_point], nan_mask), \ mask([dp[9] for dp in data_point], nan_mask), mask([dp[10] for dp in data_point], nan_mask), \ mask([dp[11] for dp in data_point], nan_mask) # convert the dates from miliseconds unix time to ordinal dates = np.array([ datetime.fromtimestamp(int(str(int(d))[:-3])).toordinal() for d in dates ]) results = ccd.detect(dates, blues, greens, reds, nirs, swir1s, swir2s, thermals, qas) band_data = np.array(swir1s) return results, dates, band_data
def sample(path, format): """Subcommand for processing sample data.""" logger.debug("Loading data...") samples = np.genfromtxt(path, delimiter=',', dtype=np.int).T logger.debug("Building change model...") start_time = timeit.default_timer() results = ccd.detect(*samples) print("ElapsedTime: ", round((timeit.default_timer() - start_time), 3)) if format == 'table': click.echo(results_to_table(results)) else: click.echo(json.dumps(results, indent=2)) logger.debug("Done...")
def l7_get_color_params(ds, inx, iny): landsat_dataset = ds myx = inx myy = iny myred = landsat_dataset.red[:, myy, myx].values myblue = landsat_dataset.blue[:, myy, myx].values mygreen = landsat_dataset.green[:, myy, myx].values mynir = landsat_dataset.nir[:, myy, myx].values myswir1 = landsat_dataset.swir1[:, myy, myx].values myswir2 = landsat_dataset.swir2[:, myy, myx].values mypixel_qa = landsat_dataset.pixel_qa[:, myy, myx].values # mythermals = landsat_dataset.therm[:, myy, myx].values dts = landsat_dataset.time.values scene_count = len(dts) mythermals = np.ones(scene_count) * (273.15) * 10 ordinal_dates = [] for mydate in dts: dval = datetime.utcfromtimestamp(mydate.tolist() / 1e9) ordinal_dates.append(dval.toordinal()) qa_bit_params = { 'QA_BITPACKED': False, 'QA_FILL': 255, 'QA_CLEAR': 0, 'QA_WATER': 1, 'QA_SHADOW': 2, 'QA_SNOW': 3, 'QA_CLOUD': 4 } bparams = (ordinal_dates, myblue, mygreen, myred, mynir, myswir1, myswir2, mythermals, mypixel_qa) mystery_object = ccd.detect(*bparams, params=qa_bit_params) return mystery_object
def test_validate_sample_2_algorithm_field(): """Tests sample 2 again for two changes and verifies the algorithm field""" data = read_data("test/resources/sample_2.csv") results = ccd.detect(data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7], data[8], preprocess=True) from ccd import __algorithm__ reported = results[0]['algorithm'] actual = __algorithm__ msg = "reported algorithm {0} did not match actual {1}".format( reported, actual) assert reported == actual, msg
def run_pyccd(display_legend, dfPyCCD, band_index): display_legend = True dfPyCCD['pixel_qa'][dfPyCCD['pixel_qa'] > 4] = 0 #TODO: Paramaterize everything params = { 'QA_BITPACKED': False, 'QA_FILL': 255, 'QA_CLEAR': 0, 'QA_WATER': 1, 'QA_SHADOW': 2, 'QA_SNOW': 3, 'QA_CLOUD': 4 } dates = np.array(dfPyCCD['ord_time']) blues = np.array(dfPyCCD['BLUE']) greens = np.array(dfPyCCD['GREEN']) reds = np.array(dfPyCCD['RED']) nirs = np.array(dfPyCCD['NIR']) swir1s = np.array(dfPyCCD['SWIR1']) swir2s = np.array(dfPyCCD['SWIR2']) thermals = np.array(dfPyCCD['THERMAL']) qas = np.array(dfPyCCD['pixel_qa']) results = ccd.detect(dates, blues, greens, reds, nirs, swir1s, swir2s, thermals, qas, params=params) return results
def pixel(args): if args['--path']: path = int(args['--path']) else: print('Calculating path from lon/lat') if args['--row']: row = int(args['--row']) else: print('Calculating row from lon/lat') lon = float(args['--lon']) if np.abs(lon) > 180: print('Invalide longitude value') sys.exit() lat = float(args['--lat']) if np.abs(lat) > 90: print('Invalide latitude value') sys.exit() if args['--band']: band = args['--band'] else: band = 4 print('No band specified, defaulting to 4') yl = None if args['--yl']: yl1 = args['--yl'] yl = [int(i) for i in (yl1.split(' '))] if args['--start']: start = args['--start'] start = '{a}-01-01'.format(a=start) else: start = '1984-01-01' if args['--finish']: finish = args['--finish'] finish = '{a}-01-01'.format(a=finish) else: finish = '2017-01-01' #Location point = {'type':'Point', 'coordinates':[lon, lat]}; #WRS-2 outline fc = ee.FeatureCollection('ft:1_RZgjlcqixp-L9hyS6NYGqLaKOlnhSC35AB5M5Ll'); #Get overlap pgeo = ee.Geometry.Point([lon, lat]); cur_wrs = fc.filterBounds(pgeo); path = cur_wrs.first().get('PATH'); row = cur_wrs.first().get('ROW'); print('Path: {}'.format(int(path.getInfo()))); print('Row: {}'.format(int(row.getInfo()))); # Create image collection #Landsat Collection. TODO: How to reduce line size with API? l8_collection = ee.ImageCollection( 'LANDSAT/LC8_SR').filter( ee.Filter.eq('WRS_PATH', path)).filter( ee.Filter.eq('WRS_ROW', row)).filterDate( start, finish); l7_collection = ee.ImageCollection( 'LANDSAT/LE7_SR').filter( ee.Filter.eq('WRS_PATH', path)).filter( ee.Filter.eq('WRS_ROW', row)).filterDate( start, finish); l5_collection = ee.ImageCollection( 'LANDSAT/LT5_SR').filter( ee.Filter.eq('WRS_PATH', path)).filter( ee.Filter.eq('WRS_ROW', row)).filterDate( start, finish); l8_thermal = ee.ImageCollection( 'LANDSAT/LC08/C01/T1_TOA').filter( ee.Filter.eq('WRS_PATH', path)).filter( ee.Filter.eq('WRS_ROW', row)).filterDate( start, finish).select('B10'); l7_thermal = ee.ImageCollection( 'LANDSAT/LE07/C01/T1_TOA').filter( ee.Filter.eq('WRS_PATH', path)).filter( ee.Filter.eq('WRS_ROW', row)).filterDate( start, finish).select('B6_VCID_1'); l5_thermal = ee.ImageCollection( 'LANDSAT/LT05/C01/T1_TOA').filter( ee.Filter.eq('WRS_PATH', path)).filter( ee.Filter.eq('WRS_ROW', row)).filterDate( start, finish).select('B6'); #LC8 Band names band_list = ['B2','B3','B4','B5','B6','B7','cfmask','cfmask_conf'] #Names to rename LC8 to / L7L5 band names rename_list = ['B1','B2','B3','B4','B5','B7','cfmask','cfmask_conf'] #L8 df_sr = make_db(l8_collection, point, band_list, rename_list) #L7 df_sr2 = make_db(l7_collection, point, rename_list, rename_list) df_sr = update_df(df_sr, df_sr2) #L5 df_sr2 = make_db(l5_collection, point, rename_list, rename_list) df_sr = update_df(df_sr, df_sr2) #thermal band_list = ['B6'] rename_list = ['thermal'] df_thermal = make_db(l5_thermal, point, band_list, rename_list) band_list = ['B6_VCID_1'] df_thermal2 = make_db(l7_thermal, point, band_list, rename_list) df_thermal = update_df(df_thermal, df_thermal2) band_list = ['B10'] df_thermal2 = make_db(l8_thermal, point, band_list, rename_list) df_thermal = update_df(df_thermal, df_thermal2) #Merge the thermal and SR df = pd.merge(df_sr, df_thermal, on='time') df = df.sort_values('time') #Get rid of NaNs # df['cfmask'][df['cfmask'].isnull()] = 4 # df[df.isnull()] = 0 #Scale brightness temperature by 10 for pyccd df['thermal'] = df['thermal'] * 10 #TODO: Paramaterize everything params = {'QA_BITPACKED': False, 'QA_FILL': 255, 'QA_CLEAR': 0, 'QA_WATER': 1, 'QA_SHADOW': 2, 'QA_SNOW': 3, 'QA_CLOUD': 4} dates = np.array(df['time']) blues = np.array(df['B1']) greens = np.array(df['B2']) reds = np.array(df['B3']) nirs = np.array(df['B4']) swir1s = np.array(df['B5']) swir2s = np.array(df['B7']) thermals = np.array(df['thermal']) qas = np.array(df['cfmask']) results = ccd.detect(dates, blues, greens, reds, nirs, swir1s, swir2s, thermals, qas, params=params) band_names = ['Blue SR', 'Green SR', 'Red SR', 'NIR SR', 'SWIR1 SR', 'SWIR2 SR','Thermal'] plotlabel = band_names[band] plot_arrays = [blues, greens, reds, nirs, swir1s, swir2s] plotband = plot_arrays[band] plot_results(results, df, band, plotband, dates, yl, plotlabel)
def pixel(args): calculate = False if args['--path']: path = int(args['--path']) else: print('Calculating path from lon/lat') calculate = True if args['--row']: row = int(args['--row']) else: print('Calculating row from lon/lat') calculate = True lon = float(args['--lon']) if np.abs(lon) > 180: print('Invalide longitude value') sys.exit() lat = float(args['--lat']) if np.abs(lat) > 90: print('Invalide latitude value') sys.exit() if args['--date']: _date = args['--date'] dt = datetime.datetime.strptime(_date, "%Y%j") date = dt.toordinal() else: print('Please specify date') sys.exit() if args['--count']: count = int(args['--count']) else: count = 1 if args['--expand']: expand = int(args['--expand']) else: count = 500 if args['--output']: output = args['--output'] saveout = True else: saveout = False if saveout: if not os.path.isdir(output): os.mkdir(output) if args['--start']: start = args['--start'] start = '{a}-01-01'.format(a=start) else: start = '1984-01-01' if args['--finish']: finish = args['--finish'] finish = '{a}-01-01'.format(a=finish) else: finish = '2017-01-01' #Location point = { 'type': 'Point', 'coordinates': [lon, lat] } if calculate: #WRS-2 outline fc = ee.FeatureCollection( 'ft:1_RZgjlcqixp-L9hyS6NYGqLaKOlnhSC35AB5M5Ll') #Get overlap pgeo = ee.Geometry.Point([lon, lat]) cur_wrs = fc.filterBounds(pgeo) path = cur_wrs.first().get('PATH') row = cur_wrs.first().get('ROW') print('Path: {}'.format(int(path.getInfo()))) print('Row: {}'.format(int(row.getInfo()))) # Create image collection #Landsat Collection. TODO: How to reduce line size with API? l8_collection = ee.ImageCollection('LANDSAT/LC8_SR').filter( ee.Filter.eq('WRS_PATH', path)).filter(ee.Filter.eq('WRS_ROW', row)).filterDate( start, finish) l7_collection = ee.ImageCollection('LANDSAT/LE7_SR').filter( ee.Filter.eq('WRS_PATH', path)).filter(ee.Filter.eq('WRS_ROW', row)).filterDate( start, finish) l5_collection = ee.ImageCollection('LANDSAT/LT5_SR').filter( ee.Filter.eq('WRS_PATH', path)).filter(ee.Filter.eq('WRS_ROW', row)).filterDate( start, finish) l8_thermal = ee.ImageCollection('LANDSAT/LC08/C01/T1_TOA').filter( ee.Filter.eq('WRS_PATH', path)).filter(ee.Filter.eq('WRS_ROW', row)).filterDate( start, finish).select('B10') l7_thermal = ee.ImageCollection('LANDSAT/LE07/C01/T1_TOA').filter( ee.Filter.eq('WRS_PATH', path)).filter(ee.Filter.eq('WRS_ROW', row)).filterDate( start, finish).select('B6_VCID_1') l5_thermal = ee.ImageCollection('LANDSAT/LT05/C01/T1_TOA').filter( ee.Filter.eq('WRS_PATH', path)).filter(ee.Filter.eq('WRS_ROW', row)).filterDate( start, finish).select('B6') #LC8 Band names band_list = ['B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'cfmask', 'cfmask_conf'] #Names to rename LC8 to / L7L5 band names rename_list = ['B1', 'B2', 'B3', 'B4', 'B5', 'B7', 'cfmask', 'cfmask_conf'] #L8 df_sr = make_db(l8_collection, point, band_list, rename_list) #L7 df_sr2 = make_db(l7_collection, point, rename_list, rename_list) df_sr = update_df(df_sr, df_sr2) #L5 df_sr2 = make_db(l5_collection, point, rename_list, rename_list) df_sr = update_df(df_sr, df_sr2) #thermal band_list = ['B6'] rename_list = ['thermal'] df_thermal = make_db(l5_thermal, point, band_list, rename_list) band_list = ['B6_VCID_1'] df_thermal2 = make_db(l7_thermal, point, band_list, rename_list) df_thermal = update_df(df_thermal, df_thermal2) band_list = ['B10'] df_thermal2 = make_db(l8_thermal, point, band_list, rename_list) df_thermal = update_df(df_thermal, df_thermal2) #Merge the thermal and SR df = pd.merge(df_sr, df_thermal, on='time') df = df.sort_values('time') #Get rid of NaNs # df['cfmask'][df['cfmask'].isnull()] = 4 # df[df.isnull()] = 0 #Scale brightness temperature by 10 for pyccd df['thermal'] = df['thermal'] * 10 #TODO: Paramaterize everything params = { 'QA_BITPACKED': False, 'QA_FILL': 255, 'QA_CLEAR': 0, 'QA_WATER': 1, 'QA_SHADOW': 2, 'QA_SNOW': 3, 'QA_CLOUD': 4 } dates = np.array(df['time']) blues = np.array(df['B1']) greens = np.array(df['B2']) reds = np.array(df['B3']) nirs = np.array(df['B4']) swir1s = np.array(df['B5']) swir2s = np.array(df['B7']) thermals = np.array(df['thermal']) qas = np.array(df['cfmask']) results = ccd.detect(dates, blues, greens, reds, nirs, swir1s, swir2s, thermals, qas, params=params) #Get the observed values observed_values, after_indices = get_observed(df, date, count) model, location = get_model(results, date) print('Found model {a} date specified'.format(a=location)) if len(model) == 1: model = model[0] if location == 'during' or 'before': residuals = get_during(model, observed_values) normalized_residuals = norm_res(residuals, model) if saveout: out = output + '/' + output np.save(out, np.array(residuals)) image_count = df['id_x'].iloc[after_indices][0:count] iter = 0 #Save image of absolute residuals outname = output + '/' 'absolute_residuals.png' save_barplot(residuals, outname, 'Absolute Residuals') #Save image of normalized residuals outname = output + '/' 'normalized_residuals.png' save_barplot(normalized_residuals, outname, 'Normalized Residuals') #Save thumbnails for img in image_count.values: save_thumbnail(img, point, output, expand, iter) iter += 1
def _run_ccd_on_pixel(ds): """Performs CCD on a 1x1xn dataset. Returns CCD results. Creates a CCD result from a 1x1xn dimensioned dataset. Flattens all bands to perform analysis. Inputs allows for missing bands. cf_mask is required. Args: ds: xArray dataset with dimensions 1x1xn with any number of SR bands, cf_mask required. Returns: The result of ccd.detect """ if 'time' not in ds.dims: raise Exception("You're missing time dims!") available_bands = ds.data_vars scene_count = ds.dims['time'] date = [_n64_to_datetime(t).date().toordinal() for t in ds.time.values] #qa = np.zeros(scene_count) red = np.ones( scene_count) if 'red' not in available_bands else ds.red.values green = np.ones( scene_count) if 'green' not in available_bands else ds.green.values blue = np.ones( scene_count) if 'blue' not in available_bands else ds.blue.values nir = np.ones( scene_count) if 'nir' not in available_bands else ds.nir.values swir1 = np.ones( scene_count) if 'swir1' not in available_bands else ds.swir1.values swir2 = np.ones( scene_count) if 'swir2' not in available_bands else ds.swir2.values thermals = np.ones(scene_count) * ( 273.15) * 10 if 'thermal' not in available_bands else ds.object.values #Generate CFMask from pixel_qa cloud_mask = utils.create_bit_mask( ds.pixel_qa, [1, 2]) & utils.create_bit_mask(ds.pixel_qa, [1, 2]) cloud_mask2 = cloud_mask.ravel() qa = cloud_mask.astype(int).ravel() params = { 'QA_BITPACKED': False, 'QA_FILL': 255, 'QA_CLEAR': 0, 'QA_CLOUD': 1 } #params = (date, blue, green, red, nir, swir1, swir2, thermals, qa,params = params) return ccd.detect(date, blue, green, red, nir, swir1, swir2, thermals, qa, params=params)