def normalize_image_res(image1,
image2,
image2out,
image1out,
out_type='ISIS3',
nodata=-32768.0):
width = max(image1.pixel_width, image2.pixel_width)
f1 = gdal.Warp('/vsimem/temp1.out',
image1.file_name,
targetAlignedPixels=True,
xRes=width,
yRes=width,
format=out_type)
f2 = gdal.Warp('/vsimem/temp2.out',
image2.file_name,
targetAlignedPixels=True,
xRes=width,
yRes=width,
format=out_type)
del (f1, f2)
temp1 = GeoDataset('/vsimem/temp1.out')
temp2 = GeoDataset('/vsimem/temp2.out')
minx = 0
miny = 0
maxx = max(temp1.read_array().shape[1], temp2.read_array().shape[1])
maxy = max(temp1.read_array().shape[0], temp2.read_array().shape[0])
fp1 = gdal.Translate(image1out,
'/vsimem/temp1.out',
srcWin=[minx, miny, maxx - minx, maxy - miny],
noData=nodata)
fp2 = gdal.Translate(image2out,
'/vsimem/temp2.out',
srcWin=[minx, miny, maxx - minx, maxy - miny],
noData=nodata)
del (fp1, fp2)
img2proj,
img2proj,
args=args)
except Exception as e:
print(e)
try:
out1, err1 = run_davinci('thm_tb.dv', img1proj, img1proj)
out2, err2 = run_davinci('thm_tb.dv', img2proj, img2proj)
except Exception as e:
print(e)
img1fh = GeoDataset(img1proj)
img2fh = GeoDataset(img2proj)
img1data = img1fh.read_array(9)
img2data = img2fh.read_array(9)
img1data = np.ma.MaskedArray(img1data, img1data == np.min(img1data))
img2data = np.ma.MaskedArray(img2data, img2data == np.min(img2data))
print(np.min(img1data), np.min(img2data))
print(img2data.shape)
print(img1data.shape)
diff = np.ma.MaskedArray(img1data.data - img2data.data,
img1data.mask | img2data.mask)
hist(diff[~diff.mask], bins=200)
diffavg = np.mean(diff)
diffmin = np.min(diff)
def themis_pairs(root, id1, id2):
def stats(arr, additional_funcs=[]):
return {
'mean': float(np.mean(arr)),
'min': float(np.min(arr)),
'max': float(np.max(arr)),
'stddev': float(np.std(arr))
}
# enforce ID1 < ID2
id1, id2 = sorted([id1, id2])
data_dir = config.data
themis_dir1 = os.path.join(data_dir, "THEMIS", id1[0], id1[1], id1)
themis_dir2 = os.path.join(data_dir, "THEMIS", id2[0], id2[1], id2)
pair_dir = os.path.join(data_dir, "THEMIS_PAIRS", id1, id2)
map_file = config.themis.map_file
if not os.path.isfile(map_file):
raise Exception("{} does not exist.".format(map_file))
pair_original_path = os.path.join(pair_dir, 'original')
pair_images_path = os.path.join(pair_dir, 'imagedata')
bundle_result_path = os.path.join(pair_dir, 'bundle')
plot_path = os.path.join(pair_dir, 'plots')
img1_path = os.path.join(themis_dir1, 'original', 'l1.cub')
img2_path = os.path.join(themis_dir2, 'original', 'l1.cub')
img1_cropped_path = os.path.join(pair_original_path, 'source.l1.cub')
img2_cropped_path = os.path.join(pair_original_path, 'destination.l1.cub')
img1_projected_path = os.path.join(pair_original_path, 'source.l2.cub')
img2_projected_path = os.path.join(pair_original_path, 'destination.l2.cub')
img1_projected_bt_path = os.path.join(pair_original_path, 'source.l2.bt.cub')
img2_projected_bt_path = os.path.join(pair_original_path, 'destination.l2.bt.cub')
img2_matchmapped_path = os.path.join(pair_original_path, 'destination.l2.mm.cub')
img2_matchmapped_bt_path = os.path.join(pair_original_path, 'destination.l2.bt.mm.cub')
cubelis = os.path.join(pair_dir, 'filelist.txt')
cnet_path = os.path.join(bundle_result_path, 'cnet.net')
autocnet_plot_path = os.path.join(plot_path, 'autocnet.tif')
histogram_plot_path = os.path.join(plot_path, 'hist.tif')
overlap_plot_path = os.path.join(plot_path, 'overlap.tif')
img1_b9_path = os.path.join(pair_images_path, 'source.b9.tif')
img2_b9_path = os.path.join(pair_images_path, 'destination.b9.tif')
img1_b9_bt_path = os.path.join(pair_images_path, 'source.b9.bt.tif')
img2_b9_bt_path = os.path.join(pair_images_path, 'destination.b9.bt.tif')
rad_diff_image = os.path.join(pair_images_path, 'rad_diff.tif')
bt_diff_image = os.path.join(pair_images_path, 'bt_diff.tif')
logger.info('Making directories {} and {}'.format(pair_original_path, pair_images_path))
os.makedirs(pair_original_path, exist_ok=True)
os.makedirs(pair_images_path, exist_ok=True)
os.makedirs(bundle_result_path, exist_ok=True)
os.makedirs(plot_path, exist_ok=True)
# write out cubelist
with open(cubelis, 'w') as f:
f.write(img1_cropped_path + '\n')
f.write(img2_cropped_path + '\n')
logger.info('IDs: {} {}'.format(id1, id2))
logger.info('DATA DIR: {}'.format(data_dir))
logger.info('IMAGE 1 PATH: {}'.format(img1_path))
logger.info('IMAGE 2 PATH: {}'.format(img2_path))
logger.info('PAIR OG DIR: {}'.format(pair_original_path))
logger.info('PAIR IMAGE PATH: {}'.format(pair_images_path))
logger.info('PAIR DIR: {}'.format(pair_dir))
img1_smithed = False
img2_smithed = False
img1_smithed = utils.preprocess(id1, themis_dir1, day=True, validate=True, gtiffs=False, projected_images=False)
img2_smithed = utils.preprocess(id2, themis_dir2, day=True, validate=True, gtiffs=False, projected_images=False)
img1_fh = GeoDataset(img1_path)
img2_fh = GeoDataset(img2_path)
# minLat maxLat minLon maxLon
minLat, maxLat, _, _ = img1_fh.footprint.Intersection(img2_fh.footprint).GetEnvelope()
utils.thm_crop(img1_path, img1_cropped_path, minLat, maxLat)
utils.thm_crop(img2_path, img2_cropped_path, minLat, maxLat)
del (img1_fh, img2_fh)
used_smithed = True
if not (img1_smithed and img2_smithed):
logger.info("No smithed kernels found, matching with Autocnet.")
used_smithed = False
cg = utils.match_pair(img1_cropped_path, img2_cropped_path, figpath=autocnet_plot_path)
cg.generate_control_network()
cg.to_isis(os.path.splitext(cnet_path)[0])
bundle_parameters = config.themis.bundle_parameters
bundle_parameters['from_'] = cubelis
bundle_parameters['cnet'] = cnet_path
bundle_parameters['onet'] = cnet_path
bundle_parameters['file_prefix'] = bundle_result_path+'/'
logger.info("Running Jigsaw, parameters:\n")
utils.print_dict(bundle_parameters)
try:
jigsaw(**bundle_parameters)
except ProcessError as e:
logger.error("STDOUT: {}".format(e.stdout.decode('utf-8')))
logger.error("STDERR: {}".format(e.stderr.decode('utf-8')))
raise Exception("Jigsaw Error")
try:
map_pvl = pvl.load(map_file)
except Exception as e:
logger.error("Error loading mapfile {}:\n{}".format(map_file, e))
logger.info('Projecting {} to {} with map file:\n {}'.format(img1_cropped_path, img1_projected_path, map_pvl))
utils.project(img1_cropped_path, img1_projected_path, map_file)
logger.info('Projecting {} to {} with map file:\n {}'.format(img2_cropped_path, img2_projected_path, map_pvl))
utils.project(img2_cropped_path, img2_projected_path, map_file)
img1_footprint = GeoDataset(img1_projected_path).footprint
img2_footprint = GeoDataset(img2_projected_path).footprint
overlap_geom = img2_footprint.Intersection(img1_footprint)
try:
out1, err1 = utils.run_davinci('thm_tb.dv', img1_projected_path, img1_projected_bt_path)
out2, err2 = utils.run_davinci('thm_tb.dv', img2_projected_path, img2_projected_bt_path)
except Exception as e:
logger.error(e)
try:
out1, err1 = utils.run_davinci('thm_post_process.dv', img1_projected_bt_path, img1_projected_bt_path)
out2, err2 = utils.run_davinci('thm_post_process.dv', img2_projected_bt_path, img2_projected_bt_path)
out1, err1 = utils.run_davinci('thm_bandselect.dv', img1_projected_bt_path, img1_projected_bt_path, args=['band=9'])
out2, err2 = utils.run_davinci('thm_bandselect.dv', img2_projected_bt_path, img2_projected_bt_path, args=['band=9'])
except Exception as e:
logger.error(e)
try:
out1, err1 = utils.run_davinci('thm_post_process.dv', img1_projected_path, img1_projected_path)
out2, err2 = utils.run_davinci('thm_post_process.dv', img2_projected_path, img2_projected_path)
out1, err1 = utils.run_davinci('thm_bandselect.dv', img1_projected_path, img1_projected_path, args=['band=9'])
out2, err2 = utils.run_davinci('thm_bandselect.dv', img2_projected_path, img2_projected_path, args=['band=9'])
except Exception as e:
logger.error(e)
footprintinit(from_=img2_projected_bt_path)
footprintinit(from_=img2_projected_path)
logger.info('Creating matchmapped cubes')
utils.project(img2_projected_path, img2_matchmapped_path, img1_projected_path, matchmap=True)
utils.project(img2_projected_bt_path, img2_matchmapped_bt_path, img1_projected_bt_path, matchmap=True)
img1_projected = GeoDataset(img1_projected_path)
img2_projected = GeoDataset(img2_matchmapped_path)
arr1 = img1_projected.read_array()
arr2 = img2_projected.read_array()
arr1[arr1 == pysis.specialpixels.SPECIAL_PIXELS['Real']['Null']] = 0
arr2[arr2 == pysis.specialpixels.SPECIAL_PIXELS['Real']['Null']] = 0
arr1[arr1 == -32768.] = 0
arr2[arr2 == -32768.] = 0
arr1 = np.ma.MaskedArray(arr1, arr1 == 0)
arr2 = np.ma.MaskedArray(arr2, arr2 == 0)
img1_b9_overlap = np.ma.MaskedArray(arr1.data, arr1.mask | arr2.mask)
img2_b9_overlap = np.ma.MaskedArray(arr2.data, arr1.mask | arr2.mask)
rad_diff = np.ma.MaskedArray(img1_b9_overlap.data-img2_b9_overlap.data, arr1.mask | arr2.mask)
img1rads = img1_b9_overlap[~img1_b9_overlap.mask]
img2rads = img2_b9_overlap[~img2_b9_overlap.mask]
img1_b9_overlap.data[img1_b9_overlap.mask] = 0
img2_b9_overlap.data[img2_b9_overlap.mask] = 0
rad_diff.data[rad_diff.mask] = 0
# logger.info('Writing {}'.format(img1_b9_path))
# ds = utils.array2raster(img1_projected_path, img1_b9_overlap, img1_b9_path)
# del ds
#
# logger.info('Writing {}'.format(img2_b9_path))
# ds = utils.array2raster(img2_projected_path, img2_b9_overlap, img2_b9_path)
# del ds
logger.info('Writing {}'.format(rad_diff_image))
ds = utils.array2raster(img1_projected_path, rad_diff, rad_diff_image)
del ds
img1_bt_projected = GeoDataset(img1_projected_bt_path)
img2_bt_projected = GeoDataset(img2_matchmapped_bt_path)
arr1 = img1_bt_projected.read_array()
arr2 = img2_bt_projected.read_array()
arr1[arr1 == pysis.specialpixels.SPECIAL_PIXELS['Real']['Null']] = 0
arr2[arr2 == pysis.specialpixels.SPECIAL_PIXELS['Real']['Null']] = 0
arr1[arr1 == -32768.] = 0
arr2[arr2 == -32768.] = 0
arr1 = np.ma.MaskedArray(arr1, arr1 == 0)
arr2 = np.ma.MaskedArray(arr2, arr2 == 0)
img1_b9_bt_overlap = np.ma.MaskedArray(arr1.data, arr1.mask | arr2.mask)
img2_b9_bt_overlap = np.ma.MaskedArray(arr2.data, arr1.mask | arr2.mask)
bt_diff = np.ma.MaskedArray(img1_b9_bt_overlap.data-img2_b9_bt_overlap.data, arr1.mask | arr2.mask)
img1bt = img1_b9_bt_overlap[~img1_b9_bt_overlap.mask]
img2bt = img2_b9_bt_overlap[~img2_b9_bt_overlap.mask]
img1_b9_bt_overlap.data[img1_b9_bt_overlap.mask] = 0
img2_b9_bt_overlap.data[img2_b9_bt_overlap.mask] = 0
bt_diff.data[bt_diff.mask] = 0
# logger.info('Writing {}'.format(img1_b9_bt_path))
# ds = utils.array2raster(img1_projected_bt_path, img1_b9_bt_overlap, img1_b9_bt_path)
# del ds
#
# logger.info('Writing {}'.format(img2_b9_bt_path))
# ds = utils.array2raster(img2_projected_bt_path, img2_b9_bt_overlap, img2_b9_bt_path)
# del ds
logger.info('Writing {}'.format(bt_diff_image))
ds = utils.array2raster(img1_projected_bt_path, bt_diff, bt_diff_image)
del ds
img1_campt = pvl.loads(campt(from_=img1_path))['GroundPoint']
img2_campt = pvl.loads(campt(from_=img1_path))['GroundPoint']
img1_date = GeoDataset(img1_path).metadata['IsisCube']['Instrument']['StartTime']
img2_date = GeoDataset(img2_path).metadata['IsisCube']['Instrument']['StartTime']
metadata = {}
metadata['img1'] = {}
metadata['img1']['rad'] = stats(img1rads)
metadata['img1']['tb'] = stats(img1bt)
metadata['img1']['emission_angle'] = img1_campt['Emission'].value
metadata['img1']['incidence_angle'] = img1_campt['Incidence'].value
metadata['img1']['solar_lon'] = img1_campt['SolarLongitude'].value
metadata['img1']['date'] = {
'year' : img1_date.year,
'month' : img1_date.month,
'day': img1_date.day
}
metadata['img2'] = {}
metadata['img2']['rad'] = stats(img2rads)
metadata['img2']['tb'] = stats(img1bt)
metadata['img2']['emission_angle'] = img2_campt['Emission'].value
metadata['img2']['incidence_angle'] = img2_campt['Incidence'].value
metadata['img2']['solar_lon'] = img2_campt['SolarLongitude'].value
metadata['img2']['date'] = {
'year' : img2_date.year,
'month' : img2_date.month,
'day': img2_date.day
}
metadata['diff'] = {}
metadata['diff']['rad'] = stats(rad_diff)
metadata['diff']['tb'] = stats(bt_diff)
metadata['diff']['date(days)'] = (img1_date - img2_date).days
metadata['id1'] = id1
metadata['id2'] = id2
metadata['plots'] = {}
metadata['plots']['rad_hist'] = os.path.join(plot_path, 'rad_hist.png')
metadata['plots']['tb_hist'] = os.path.join(plot_path, 'tb_hist.png')
metadata['plots']['diff_hist'] = os.path.join(plot_path, 'diff_hist.png')
metadata['plots']['match_plot'] = autocnet_plot_path
if not used_smithed:
metadata['plots']['matching_plot'] = autocnet_plot_path
metadata['bundle'] = {}
for f in glob(os.path.join(bundle_result_path, '*')):
metadata['bundle'][os.path.basename(os.path.splitext(f)[0])] = f
try:
df = pd.read_csv(metadata['bundle']['residuals'], header=1)
except:
df = pd.read_csv(metadata['bundle']['_residuals'], header=1)
metadata['bundle']['residual_stats'] = stats(np.asarray(df['residual.1'][1:], dtype=float))
utils.print_dict(metadata)
plt.figure(figsize=(25,10))
bins = sns.distplot(img1rads[~img1rads.mask], kde=False, norm_hist=False, label='{} {}'.format(id1, os.path.basename(img1_b9_path)))
bins = sns.distplot(img2rads[~img2rads.mask], kde=False, norm_hist=False, label='{} {}'.format(id2,os.path.basename(img2_b9_path)))
bins.set(xlabel='radiance', ylabel='counts')
plt.legend()
plt.savefig(metadata['plots']['rad_hist'])
plt.close()
plt.figure(figsize=(25,10))
bins = sns.distplot(img1bt[~img1bt.mask], kde=False, norm_hist=False, label='{} {}'.format(id1, os.path.basename(img1_b9_bt_path)))
bins = sns.distplot(img2bt[~img2bt.mask], kde=False, norm_hist=False, label='{} {}'.format(id2, os.path.basename(img2_b9_bt_path)))
bins.set(xlabel='Brightness Temp', ylabel='counts')
plt.legend()
plt.savefig(metadata['plots']['tb_hist'])
plt.close()
plt.figure(figsize=(25,10))
diffplot = sns.distplot(rad_diff[~rad_diff.mask], kde=False)
diffplot.set(xlabel='Delta Radiance', ylabel='counts')
plt.savefig(metadata['plots']['diff_hist'])
plt.close()
metadata_path = os.path.join(pair_dir, 'metadata.json')
json.dump(metadata,open(metadata_path, 'w+'), default=utils.date_converter)
index_path = os.path.join(pair_dir, 'index.json')
index = {}
print(GeoDataset(img1_cropped_path).footprint.ExportToWkt())
print(GeoDataset(img2_cropped_path).footprint.ExportToWkt())
index['overlap_geom'] = overlap_geom.ExportToWkt()
index['img1_geom'] = img1_footprint.ExportToWkt()
index['img2_geom'] = img2_footprint.ExportToWkt()
index['id'] = '{}_{}'.format(id1, id2)
json.dump(index, open(index_path, 'w+'))
utils.print_dict(index)
logger.info("Complete")
def match_pair(img1_path, img2_path, figpath=None):
src_points = point_grid(GeoDataset(img1_path), step=50)
f = open('temp.txt', 'w+')
f.write('\n'.join('{}, {}'.format(int(x), int(y)) for x, y in src_points))
del f
label = pvl.loads(
campt(from_=img1_path, coordlist='temp.txt', coordtype='image'))
points = []
for group in label:
try:
lat = group[1]['PlanetocentricLatitude'].value
lon = group[1]['PositiveEast360Longitude'].value
points.append([lat, lon])
except Exception as e:
continue
logger.info(
"{} points from image1 successfully reprojected to image2, rejected {}"
.format(str(len(points)), str(len(src_points) - len(points))))
if len(points) == 0:
raise Exception("No valid points were found for pair {} {}".format(
img1_path, img2_path))
f = open('temp.txt', 'w+')
f.write('\n'.join('{}, {}'.format(x, y) for x, y in points))
del f
img2label = pvl.loads(
campt(from_=img2_path,
coordlist='temp.txt',
coordtype='ground',
allowoutside=False))
dst_lookup = {}
for i, group in enumerate(img2label):
if not group[1]['Error']:
line = group[1]['Line']
sample = group[1]['Sample']
dst_lookup[i] = [sample, line]
filelist = [img1_path, img2_path]
cg = CandidateGraph.from_filelist(filelist)
edge = cg[0][1]['data']
img1 = GeoDataset(img1_path)
img2 = GeoDataset(img2_path)
src_keypoints = pd.DataFrame(data=src_points, columns=['x', 'y'])
src_keypoints['response'] = 0
src_keypoints['angle'] = 0
src_keypoints['octave'] = 0
src_keypoints['layer'] = 0
src_keypoints
edge.source._keypoints = src_keypoints
results = []
dst_keypoints = []
dst_index = 0
distances = []
arr1 = img1.read_array()
arr2 = img2.read_array()
del img1
del img2
for keypoint in edge.source.keypoints.iterrows():
index, row = keypoint
sx, sy = row['x'], row['y']
try:
dx, dy = dst_lookup[index]
except KeyError:
continue
try:
ret = refine_subpixel(sx,
sy,
dx,
dy,
arr1,
arr2,
size=50,
reduction=10,
convergence_threshold=1)
except Exception as ex:
continue
if ret is not None:
x, y, metrics = ret
else:
continue
dist = np.linalg.norm([x - dx, y - dy])
results.append([0, index, 1, dst_index, dist])
dst_keypoints.append([x, y, 0, 0, 0, 0, 0])
dst_index += 1
matches = pd.DataFrame(data=results,
columns=[
'source_image', 'source_idx',
'destination_image', 'destination_idx',
'distance'
])
if matches.empty:
logger.error(
"After matching points, matches dataframe returned empty.")
dst_keypoints = pd.DataFrame(
data=dst_keypoints,
columns=['x', 'y', 'response', 'size', 'angle', 'octave', 'layer'])
edge.destination._keypoints = dst_keypoints
edge._matches = matches
edge.compute_fundamental_matrix()
distance_check(edge, clean_keys=['fundamental'])
if figpath:
plt.figure(figsize=(10, 25))
cg[0][1]['data'].plot(clean_keys=['fundamental', 'distance'],
nodata=-32768.0)
plt.savefig(figpath)
plt.close()
return cg