def get_color_params(ds, inx, iny):
landsat_dataset = ds
myx = inx
myy = iny
myred = landsat_dataset.red[:, myy, myx].values
myblue = landsat_dataset.blue[:, myy, myx].values
mygreen = landsat_dataset.green[:, myy, myx].values
mynir = landsat_dataset.nir[:, myy, myx].values
myswir1 = landsat_dataset.swir1[:, myy, myx].values
myswir2 = landsat_dataset.swir2[:, myy, myx].values
mypixel_qa = landsat_dataset.pixel_qa[:, myy, myx].values
dts = landsat_dataset.time.values
scene_count = len(dts)
mythermals = np.ones(scene_count) * (273.15) * 10
ordinal_dates = []
for mydate in dts:
dval = datetime.utcfromtimestamp(mydate.tolist() / 1e9)
ordinal_dates.append(dval.toordinal())
params = (ordinal_dates, myblue, mygreen, myred, mynir, myswir1, myswir2,
mythermals, mypixel_qa)
mystery_object = ccd.detect(*params)
return mystery_object
def _run_ccd_on_pixel(ds, **kwargs):
"""Performs CCD on a 1x1xn dataset. Returns CCD results.
Creates a CCD result from a 1x1xn dimensioned dataset. Flattens all bands to perform analysis. Inputs allows for missing bands. cf_mask is required.
Args:
ds: xArray dataset with dimensions 1x1xn with any number of SR bands, cf_mask required.
params: can be used to define a custom mapping.
Returns:
The result of ccd.detect
"""
if 'time' not in ds.dims:
raise Exception("You're missing time dims!")
available_bands = ds.data_vars
scene_count = ds.dims['time']
date = [_n64_to_datetime(t).date().toordinal() for t in ds.time.values]
red = np.ones(scene_count) if 'red' not in available_bands else ds.red.values
green = np.ones(scene_count) if 'green' not in available_bands else ds.green.values
blue = np.ones(scene_count) if 'blue' not in available_bands else ds.blue.values
nir = np.ones(scene_count) if 'nir' not in available_bands else ds.nir.values
swir1 = np.ones(scene_count) if 'swir1' not in available_bands else ds.swir1.values
swir2 = np.ones(scene_count) if 'swir2' not in available_bands else ds.swir2.values
thermals = np.ones(scene_count) * (273.15) * 10 if 'thermal' not in available_bands else ds.object.values
qa = np.array(ds.pixel_qa.values)
bands = (date, blue, green, red, nir, swir1, swir2, thermals, qa)
return ccd.detect(*bands, **kwargs)
def test_sample_data_sets():
"""
Sanity test to ensure all test data sets run to completion
"""
samples = [
'test/resources/sample_1.csv', 'test/resources/sample_2.csv',
'test/resources/sample_WA_grid08_row9_col2267_persistent_snow.csv',
'test/resources/sample_WA_grid08_row12_col2265_fmask_fail.csv',
'test/resources/sample_WA_grid08_row999_col1_normal.csv',
'test/resources/test_3657_3610_observations.csv'
]
params = {
'QA_BITPACKED': False,
'QA_FILL': 255,
'QA_CLEAR': 0,
'QA_WATER': 1,
'QA_SHADOW': 2,
'QA_SNOW': 3,
'QA_CLOUD': 4
}
for sample in samples:
data = read_data(sample)
results = ccd.detect(data[0],
data[1],
data[2],
data[3],
data[4],
data[5],
data[6],
data[7],
data[8],
params=params)
def CCD_Points(Data):
params = {'QA_BITPACKED': False,
'QA_FILL': 255,
'QA_CLEAR': 0,
'QA_WATER': 1,
'QA_SHADOW': 2,
'QA_SNOW': 3,
'QA_CLOUD': 4}
dates, blues, greens, reds, nirs, swir1s, swir2s, thermals, qas = Data.T
results = ccd.detect(dates, blues, greens, reds, nirs, swir1s, swir2s, thermals, qas, params=params)
start_dates = []
end_dates = []
break_dates = []
for num, result in enumerate(results['change_models']):
time_length = result['end_day'] - result['start_day']
if time_length > 365:
#if result['change_probability'] == 1 and time_length > 730:
#if result['change_probability'] == 1 :
start_dates.append(result['start_day'])
end_dates.append(result['end_day'])
break_dates.append(result['break_day'])
return {"start_dates": start_dates,
"end_dates": end_dates,
"break_dates": break_dates,
}
def test_endfit():
"""
Flex the code the generates start fits.
"""
sample = 'test/resources/h04v03_-1945125_2844645_pixel_endfit.npy'
result = ccd.detect(**np.load(sample)[1])
assert result['change_models'][-1]['curve_qa'] == params.CURVE_QA['END']
def test_insuff_clear():
"""
Flex the insufficient clear procedure code to make sure it runs.
"""
sample = 'test/resources/h04v03_-1947075_2846265_pixel_insuff.npy'
result = ccd.detect(**np.load(sample)[1])
assert result['change_models'][0]['curve_qa'] == params.CURVE_QA[
'INSUF_CLEAR']
def test_perm_snow():
"""
Flex the permanent snow procedure code.
"""
sample = 'test/resources/h04v03_-1947105_2846265_pixel_snow.npy'
result = ccd.detect(**np.load(sample)[1])
assert result['change_models'][0]['curve_qa'] == params.CURVE_QA[
'PERSIST_SNOW']
def detect(timeseries):
cx, cy, px, py = first(timeseries)
return format(cx=cx,
cy=cy,
px=px,
py=py,
dates=get('dates', second(timeseries)),
ccdresult=ccd.detect(**second(timeseries)))
def test_npy():
"""
Sanity tests for npy test data sets
"""
samples = ['test/resources/h03v09_-2010765_1964625_pixel.npy'
] # Main loop failure in LF
for sample in samples:
dat = np.load(sample)
results = ccd.detect(**dat[1])
def run_pyccd2(b):
# Run pyCCD on current point
Plot_interface.pyccd_flag2 = True
# Display the legend
Plot_interface.lc7.display_legend = True
dfPyCCD = Plot_interface.click_df
# First two lines no longer required bc we are removing NA's when we load the TS
dfPyCCD['pixel_qa'][dfPyCCD['pixel_qa'] > 4] = 0
#TODO: Paramaterize everything
params = {
'QA_BITPACKED': False,
'QA_FILL': 255,
'QA_CLEAR': 0,
'QA_WATER': 1,
'QA_SHADOW': 2,
'QA_SNOW': 3,
'QA_CLOUD': 4
}
dates = np.array(dfPyCCD['ord_time'])
blues = np.array(dfPyCCD['BLUE'])
greens = np.array(dfPyCCD['GREEN'])
reds = np.array(dfPyCCD['RED'])
nirs = np.array(dfPyCCD['NIR'])
swir1s = np.array(dfPyCCD['SWIR1'])
swir2s = np.array(dfPyCCD['SWIR2'])
thermals = np.array(dfPyCCD['THERMAL'])
qas = np.array(dfPyCCD['pixel_qa'])
results = ccd.detect(dates,
blues,
greens,
reds,
nirs,
swir1s,
swir2s,
thermals,
qas,
params=params)
band_names = [
'Blue SR', 'Green SR', 'Red SR', 'NIR SR', 'SWIR1 SR', 'SWIR2 SR',
'THERMAL'
]
plotlabel = band_names[Plot_interface.band_index2]
plot_arrays = [blues, greens, reds, nirs, swir1s, swir2s]
plotband = plot_arrays[Plot_interface.band_index2]
Plot_interface.plot_pyccd(results, Plot_interface.band_index2,
plotband, dates, (0, 4000), 'PyCCD Results',
'clicked_ts')
def run_pyccd(b):
# Run pyCCD on current point
Plot_interface.pyccd_flag = True
Plot_interface.lc5.display_legend = True
dfPyCCD = Plot_interface.sample_df
dfPyCCD['pixel_qa'][dfPyCCD['pixel_qa'] > 4] = 0
#TODO: Paramaterize everything
params = {
'QA_BITPACKED': False,
'QA_FILL': 255,
'QA_CLEAR': 0,
'QA_WATER': 1,
'QA_SHADOW': 2,
'QA_SNOW': 3,
'QA_CLOUD': 4
}
dates = np.array(dfPyCCD['ord_time'])
blues = np.array(dfPyCCD['BLUE'])
greens = np.array(dfPyCCD['GREEN'])
reds = np.array(dfPyCCD['RED'])
nirs = np.array(dfPyCCD['NIR'])
swir1s = np.array(dfPyCCD['SWIR1'])
swir2s = np.array(dfPyCCD['SWIR2'])
thermals = np.array(dfPyCCD['THERMAL'])
qas = np.array(dfPyCCD['pixel_qa'])
results = ccd.detect(dates,
blues,
greens,
reds,
nirs,
swir1s,
swir2s,
thermals,
qas,
params=params)
band_names = [
'Blue SR', 'Green SR', 'Red SR', 'NIR SR', 'SWIR1 SR', 'SWIR2 SR',
'THERMAL'
]
plotlabel = band_names[Plot_interface.band_index1]
plot_arrays = [blues, greens, reds, nirs, swir1s, swir2s]
plotband = plot_arrays[Plot_interface.band_index1]
Plot_interface.plot_pyccd(results, Plot_interface.band_index1,
plotband, dates, (0, 4000), 'PyCCD Results',
'sample_ts')
def sample(path, format):
"""Subcommand for processing sample data."""
logger.debug("Loading data...")
samples = np.genfromtxt(path, delimiter=',', dtype=np.int).T
logger.debug("Building change model...")
results = ccd.detect(*samples)
if format == 'table':
click.echo(results_to_table(results))
else:
click.echo(json.dumps(results, indent=2))
logger.debug("Done...")
def test_sample_data_sets():
"""
Sanity test to ensure all test data sets run to completion
"""
samples = [
'test/resources/sample_1.csv', 'test/resources/sample_2.csv',
'test/resources/sample_WA_grid08_row9_col2267_persistent_snow.csv',
'test/resources/sample_WA_grid08_row12_col2265_fmask_fail.csv',
'test/resources/sample_WA_grid08_row999_col1_normal.csv',
'test/resources/test_3657_3610_observations.csv'
]
for sample in samples:
data = read_data(sample)
results = ccd.detect(data[0], data[1], data[2], data[3], data[4],
data[5], data[6], data[7], data[8])
def test_validate_no_preprocessing_sample_2_detection_results():
""" Sample 2 contains two changes and should test the full path through
the algorithm including preprocessing """
data = read_data("test/resources/sample_2.csv")
results = ccd.detect(data[0],
data[1],
data[2],
data[3],
data[4],
data[5],
data[6],
data[7],
data[8],
preprocess=False)
assert len(results) != 2, "expected: !{}, actual: {}".format(
2, len(results))
def compute_ccd(coords, year_range=(2000, 2020), doy_range=(1, 365)):
# get data from Google Earth Engine
# list index order:
# 'longitude', 1
# 'latitude',2
# 'time',3
# 'BLUE',4
# 'GREEN',5
# 'RED',6
# 'NIR',7
# 'SWIR1',8
# 'SWIR2',9
# 'THERMAL',10
# 'pixel_qa'11
### get GEE data from the specific point
data_collection = get_full_collection(coords, year_range, doy_range)
data_point = get_data_full(data_collection, coords)[1::]
# generate a merge/fusion mask layer of nan/none values to filter all data
nan_masks = [[0 if dp[i] is None else 1 for dp in data_point]
for i in range(3, 12)]
# fusion masks
nan_mask = [0 if 0 in m else 1 for m in zip(*nan_masks)]
# get each features applying the mask
dates, blues, greens, reds, nirs, swir1s, swir2s, thermals, qas = \
mask([dp[3] for dp in data_point], nan_mask), mask([dp[4] for dp in data_point], nan_mask), \
mask([dp[5] for dp in data_point], nan_mask), mask([dp[6] for dp in data_point], nan_mask), \
mask([dp[7] for dp in data_point], nan_mask), mask([dp[8] for dp in data_point], nan_mask), \
mask([dp[9] for dp in data_point], nan_mask), mask([dp[10] for dp in data_point], nan_mask), \
mask([dp[11] for dp in data_point], nan_mask)
# convert the dates from miliseconds unix time to ordinal
dates = np.array([
datetime.fromtimestamp(int(str(int(d))[:-3])).toordinal()
for d in dates
])
results = ccd.detect(dates, blues, greens, reds, nirs, swir1s, swir2s,
thermals, qas)
band_data = np.array(swir1s)
return results, dates, band_data
def sample(path, format):
"""Subcommand for processing sample data."""
logger.debug("Loading data...")
samples = np.genfromtxt(path, delimiter=',', dtype=np.int).T
logger.debug("Building change model...")
start_time = timeit.default_timer()
results = ccd.detect(*samples)
print("ElapsedTime: ", round((timeit.default_timer() - start_time), 3))
if format == 'table':
click.echo(results_to_table(results))
else:
click.echo(json.dumps(results, indent=2))
logger.debug("Done...")
def l7_get_color_params(ds, inx, iny):
landsat_dataset = ds
myx = inx
myy = iny
myred = landsat_dataset.red[:, myy, myx].values
myblue = landsat_dataset.blue[:, myy, myx].values
mygreen = landsat_dataset.green[:, myy, myx].values
mynir = landsat_dataset.nir[:, myy, myx].values
myswir1 = landsat_dataset.swir1[:, myy, myx].values
myswir2 = landsat_dataset.swir2[:, myy, myx].values
mypixel_qa = landsat_dataset.pixel_qa[:, myy, myx].values
# mythermals = landsat_dataset.therm[:, myy, myx].values
dts = landsat_dataset.time.values
scene_count = len(dts)
mythermals = np.ones(scene_count) * (273.15) * 10
ordinal_dates = []
for mydate in dts:
dval = datetime.utcfromtimestamp(mydate.tolist() / 1e9)
ordinal_dates.append(dval.toordinal())
qa_bit_params = {
'QA_BITPACKED': False,
'QA_FILL': 255,
'QA_CLEAR': 0,
'QA_WATER': 1,
'QA_SHADOW': 2,
'QA_SNOW': 3,
'QA_CLOUD': 4
}
bparams = (ordinal_dates, myblue, mygreen, myred, mynir, myswir1, myswir2,
mythermals, mypixel_qa)
mystery_object = ccd.detect(*bparams, params=qa_bit_params)
return mystery_object
def test_validate_sample_2_algorithm_field():
"""Tests sample 2 again for two changes and verifies the algorithm field"""
data = read_data("test/resources/sample_2.csv")
results = ccd.detect(data[0],
data[1],
data[2],
data[3],
data[4],
data[5],
data[6],
data[7],
data[8],
preprocess=True)
from ccd import __algorithm__
reported = results[0]['algorithm']
actual = __algorithm__
msg = "reported algorithm {0} did not match actual {1}".format(
reported, actual)
assert reported == actual, msg
def run_pyccd(display_legend, dfPyCCD, band_index):
display_legend = True
dfPyCCD['pixel_qa'][dfPyCCD['pixel_qa'] > 4] = 0
#TODO: Paramaterize everything
params = {
'QA_BITPACKED': False,
'QA_FILL': 255,
'QA_CLEAR': 0,
'QA_WATER': 1,
'QA_SHADOW': 2,
'QA_SNOW': 3,
'QA_CLOUD': 4
}
dates = np.array(dfPyCCD['ord_time'])
blues = np.array(dfPyCCD['BLUE'])
greens = np.array(dfPyCCD['GREEN'])
reds = np.array(dfPyCCD['RED'])
nirs = np.array(dfPyCCD['NIR'])
swir1s = np.array(dfPyCCD['SWIR1'])
swir2s = np.array(dfPyCCD['SWIR2'])
thermals = np.array(dfPyCCD['THERMAL'])
qas = np.array(dfPyCCD['pixel_qa'])
results = ccd.detect(dates,
blues,
greens,
reds,
nirs,
swir1s,
swir2s,
thermals,
qas,
params=params)
return results
def pixel(args):
if args['--path']:
path = int(args['--path'])
else:
print('Calculating path from lon/lat')
if args['--row']:
row = int(args['--row'])
else:
print('Calculating row from lon/lat')
lon = float(args['--lon'])
if np.abs(lon) > 180:
print('Invalide longitude value')
sys.exit()
lat = float(args['--lat'])
if np.abs(lat) > 90:
print('Invalide latitude value')
sys.exit()
if args['--band']:
band = args['--band']
else:
band = 4
print('No band specified, defaulting to 4')
yl = None
if args['--yl']:
yl1 = args['--yl']
yl = [int(i) for i in (yl1.split(' '))]
if args['--start']:
start = args['--start']
start = '{a}-01-01'.format(a=start)
else:
start = '1984-01-01'
if args['--finish']:
finish = args['--finish']
finish = '{a}-01-01'.format(a=finish)
else:
finish = '2017-01-01'
#Location
point = {'type':'Point', 'coordinates':[lon, lat]};
#WRS-2 outline
fc = ee.FeatureCollection('ft:1_RZgjlcqixp-L9hyS6NYGqLaKOlnhSC35AB5M5Ll');
#Get overlap
pgeo = ee.Geometry.Point([lon, lat]);
cur_wrs = fc.filterBounds(pgeo);
path = cur_wrs.first().get('PATH');
row = cur_wrs.first().get('ROW');
print('Path: {}'.format(int(path.getInfo())));
print('Row: {}'.format(int(row.getInfo())));
# Create image collection
#Landsat Collection. TODO: How to reduce line size with API?
l8_collection = ee.ImageCollection(
'LANDSAT/LC8_SR').filter(
ee.Filter.eq('WRS_PATH', path)).filter(
ee.Filter.eq('WRS_ROW', row)).filterDate(
start, finish);
l7_collection = ee.ImageCollection(
'LANDSAT/LE7_SR').filter(
ee.Filter.eq('WRS_PATH', path)).filter(
ee.Filter.eq('WRS_ROW', row)).filterDate(
start, finish);
l5_collection = ee.ImageCollection(
'LANDSAT/LT5_SR').filter(
ee.Filter.eq('WRS_PATH', path)).filter(
ee.Filter.eq('WRS_ROW', row)).filterDate(
start, finish);
l8_thermal = ee.ImageCollection(
'LANDSAT/LC08/C01/T1_TOA').filter(
ee.Filter.eq('WRS_PATH', path)).filter(
ee.Filter.eq('WRS_ROW', row)).filterDate(
start, finish).select('B10');
l7_thermal = ee.ImageCollection(
'LANDSAT/LE07/C01/T1_TOA').filter(
ee.Filter.eq('WRS_PATH', path)).filter(
ee.Filter.eq('WRS_ROW', row)).filterDate(
start, finish).select('B6_VCID_1');
l5_thermal = ee.ImageCollection(
'LANDSAT/LT05/C01/T1_TOA').filter(
ee.Filter.eq('WRS_PATH', path)).filter(
ee.Filter.eq('WRS_ROW', row)).filterDate(
start, finish).select('B6');
#LC8 Band names
band_list = ['B2','B3','B4','B5','B6','B7','cfmask','cfmask_conf']
#Names to rename LC8 to / L7L5 band names
rename_list = ['B1','B2','B3','B4','B5','B7','cfmask','cfmask_conf']
#L8
df_sr = make_db(l8_collection, point, band_list, rename_list)
#L7
df_sr2 = make_db(l7_collection, point, rename_list, rename_list)
df_sr = update_df(df_sr, df_sr2)
#L5
df_sr2 = make_db(l5_collection, point, rename_list, rename_list)
df_sr = update_df(df_sr, df_sr2)
#thermal
band_list = ['B6']
rename_list = ['thermal']
df_thermal = make_db(l5_thermal, point, band_list, rename_list)
band_list = ['B6_VCID_1']
df_thermal2 = make_db(l7_thermal, point, band_list, rename_list)
df_thermal = update_df(df_thermal, df_thermal2)
band_list = ['B10']
df_thermal2 = make_db(l8_thermal, point, band_list, rename_list)
df_thermal = update_df(df_thermal, df_thermal2)
#Merge the thermal and SR
df = pd.merge(df_sr, df_thermal, on='time')
df = df.sort_values('time')
#Get rid of NaNs
# df['cfmask'][df['cfmask'].isnull()] = 4
# df[df.isnull()] = 0
#Scale brightness temperature by 10 for pyccd
df['thermal'] = df['thermal'] * 10
#TODO: Paramaterize everything
params = {'QA_BITPACKED': False,
'QA_FILL': 255,
'QA_CLEAR': 0,
'QA_WATER': 1,
'QA_SHADOW': 2,
'QA_SNOW': 3,
'QA_CLOUD': 4}
dates = np.array(df['time'])
blues = np.array(df['B1'])
greens = np.array(df['B2'])
reds = np.array(df['B3'])
nirs = np.array(df['B4'])
swir1s = np.array(df['B5'])
swir2s = np.array(df['B7'])
thermals = np.array(df['thermal'])
qas = np.array(df['cfmask'])
results = ccd.detect(dates, blues, greens, reds, nirs, swir1s, swir2s, thermals, qas, params=params)
band_names = ['Blue SR', 'Green SR', 'Red SR', 'NIR SR', 'SWIR1 SR', 'SWIR2 SR','Thermal']
plotlabel = band_names[band]
plot_arrays = [blues, greens, reds, nirs, swir1s, swir2s]
plotband = plot_arrays[band]
plot_results(results, df, band, plotband, dates, yl, plotlabel)
def pixel(args):
calculate = False
if args['--path']:
path = int(args['--path'])
else:
print('Calculating path from lon/lat')
calculate = True
if args['--row']:
row = int(args['--row'])
else:
print('Calculating row from lon/lat')
calculate = True
lon = float(args['--lon'])
if np.abs(lon) > 180:
print('Invalide longitude value')
sys.exit()
lat = float(args['--lat'])
if np.abs(lat) > 90:
print('Invalide latitude value')
sys.exit()
if args['--date']:
_date = args['--date']
dt = datetime.datetime.strptime(_date, "%Y%j")
date = dt.toordinal()
else:
print('Please specify date')
sys.exit()
if args['--count']:
count = int(args['--count'])
else:
count = 1
if args['--expand']:
expand = int(args['--expand'])
else:
count = 500
if args['--output']:
output = args['--output']
saveout = True
else:
saveout = False
if saveout:
if not os.path.isdir(output):
os.mkdir(output)
if args['--start']:
start = args['--start']
start = '{a}-01-01'.format(a=start)
else:
start = '1984-01-01'
if args['--finish']:
finish = args['--finish']
finish = '{a}-01-01'.format(a=finish)
else:
finish = '2017-01-01'
#Location
point = {
'type': 'Point',
'coordinates': [lon, lat]
}
if calculate:
#WRS-2 outline
fc = ee.FeatureCollection(
'ft:1_RZgjlcqixp-L9hyS6NYGqLaKOlnhSC35AB5M5Ll')
#Get overlap
pgeo = ee.Geometry.Point([lon, lat])
cur_wrs = fc.filterBounds(pgeo)
path = cur_wrs.first().get('PATH')
row = cur_wrs.first().get('ROW')
print('Path: {}'.format(int(path.getInfo())))
print('Row: {}'.format(int(row.getInfo())))
# Create image collection
#Landsat Collection. TODO: How to reduce line size with API?
l8_collection = ee.ImageCollection('LANDSAT/LC8_SR').filter(
ee.Filter.eq('WRS_PATH', path)).filter(ee.Filter.eq('WRS_ROW',
row)).filterDate(
start, finish)
l7_collection = ee.ImageCollection('LANDSAT/LE7_SR').filter(
ee.Filter.eq('WRS_PATH', path)).filter(ee.Filter.eq('WRS_ROW',
row)).filterDate(
start, finish)
l5_collection = ee.ImageCollection('LANDSAT/LT5_SR').filter(
ee.Filter.eq('WRS_PATH', path)).filter(ee.Filter.eq('WRS_ROW',
row)).filterDate(
start, finish)
l8_thermal = ee.ImageCollection('LANDSAT/LC08/C01/T1_TOA').filter(
ee.Filter.eq('WRS_PATH',
path)).filter(ee.Filter.eq('WRS_ROW', row)).filterDate(
start, finish).select('B10')
l7_thermal = ee.ImageCollection('LANDSAT/LE07/C01/T1_TOA').filter(
ee.Filter.eq('WRS_PATH',
path)).filter(ee.Filter.eq('WRS_ROW', row)).filterDate(
start, finish).select('B6_VCID_1')
l5_thermal = ee.ImageCollection('LANDSAT/LT05/C01/T1_TOA').filter(
ee.Filter.eq('WRS_PATH',
path)).filter(ee.Filter.eq('WRS_ROW', row)).filterDate(
start, finish).select('B6')
#LC8 Band names
band_list = ['B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'cfmask', 'cfmask_conf']
#Names to rename LC8 to / L7L5 band names
rename_list = ['B1', 'B2', 'B3', 'B4', 'B5', 'B7', 'cfmask', 'cfmask_conf']
#L8
df_sr = make_db(l8_collection, point, band_list, rename_list)
#L7
df_sr2 = make_db(l7_collection, point, rename_list, rename_list)
df_sr = update_df(df_sr, df_sr2)
#L5
df_sr2 = make_db(l5_collection, point, rename_list, rename_list)
df_sr = update_df(df_sr, df_sr2)
#thermal
band_list = ['B6']
rename_list = ['thermal']
df_thermal = make_db(l5_thermal, point, band_list, rename_list)
band_list = ['B6_VCID_1']
df_thermal2 = make_db(l7_thermal, point, band_list, rename_list)
df_thermal = update_df(df_thermal, df_thermal2)
band_list = ['B10']
df_thermal2 = make_db(l8_thermal, point, band_list, rename_list)
df_thermal = update_df(df_thermal, df_thermal2)
#Merge the thermal and SR
df = pd.merge(df_sr, df_thermal, on='time')
df = df.sort_values('time')
#Get rid of NaNs
# df['cfmask'][df['cfmask'].isnull()] = 4
# df[df.isnull()] = 0
#Scale brightness temperature by 10 for pyccd
df['thermal'] = df['thermal'] * 10
#TODO: Paramaterize everything
params = {
'QA_BITPACKED': False,
'QA_FILL': 255,
'QA_CLEAR': 0,
'QA_WATER': 1,
'QA_SHADOW': 2,
'QA_SNOW': 3,
'QA_CLOUD': 4
}
dates = np.array(df['time'])
blues = np.array(df['B1'])
greens = np.array(df['B2'])
reds = np.array(df['B3'])
nirs = np.array(df['B4'])
swir1s = np.array(df['B5'])
swir2s = np.array(df['B7'])
thermals = np.array(df['thermal'])
qas = np.array(df['cfmask'])
results = ccd.detect(dates,
blues,
greens,
reds,
nirs,
swir1s,
swir2s,
thermals,
qas,
params=params)
#Get the observed values
observed_values, after_indices = get_observed(df, date, count)
model, location = get_model(results, date)
print('Found model {a} date specified'.format(a=location))
if len(model) == 1:
model = model[0]
if location == 'during' or 'before':
residuals = get_during(model, observed_values)
normalized_residuals = norm_res(residuals, model)
if saveout:
out = output + '/' + output
np.save(out, np.array(residuals))
image_count = df['id_x'].iloc[after_indices][0:count]
iter = 0
#Save image of absolute residuals
outname = output + '/' 'absolute_residuals.png'
save_barplot(residuals, outname, 'Absolute Residuals')
#Save image of normalized residuals
outname = output + '/' 'normalized_residuals.png'
save_barplot(normalized_residuals, outname, 'Normalized Residuals')
#Save thumbnails
for img in image_count.values:
save_thumbnail(img, point, output, expand, iter)
iter += 1
def _run_ccd_on_pixel(ds):
"""Performs CCD on a 1x1xn dataset. Returns CCD results.
Creates a CCD result from a 1x1xn dimensioned dataset. Flattens all bands to perform analysis. Inputs allows for missing bands. cf_mask is required.
Args:
ds: xArray dataset with dimensions 1x1xn with any number of SR bands, cf_mask required.
Returns:
The result of ccd.detect
"""
if 'time' not in ds.dims:
raise Exception("You're missing time dims!")
available_bands = ds.data_vars
scene_count = ds.dims['time']
date = [_n64_to_datetime(t).date().toordinal() for t in ds.time.values]
#qa = np.zeros(scene_count)
red = np.ones(
scene_count) if 'red' not in available_bands else ds.red.values
green = np.ones(
scene_count) if 'green' not in available_bands else ds.green.values
blue = np.ones(
scene_count) if 'blue' not in available_bands else ds.blue.values
nir = np.ones(
scene_count) if 'nir' not in available_bands else ds.nir.values
swir1 = np.ones(
scene_count) if 'swir1' not in available_bands else ds.swir1.values
swir2 = np.ones(
scene_count) if 'swir2' not in available_bands else ds.swir2.values
thermals = np.ones(scene_count) * (
273.15) * 10 if 'thermal' not in available_bands else ds.object.values
#Generate CFMask from pixel_qa
cloud_mask = utils.create_bit_mask(
ds.pixel_qa, [1, 2]) & utils.create_bit_mask(ds.pixel_qa, [1, 2])
cloud_mask2 = cloud_mask.ravel()
qa = cloud_mask.astype(int).ravel()
params = {
'QA_BITPACKED': False,
'QA_FILL': 255,
'QA_CLEAR': 0,
'QA_CLOUD': 1
}
#params = (date, blue, green, red, nir, swir1, swir2, thermals, qa,params = params)
return ccd.detect(date,
blue,
green,
red,
nir,
swir1,
swir2,
thermals,
qa,
params=params)
