def plot_img_csv(folder, csv='log.csv', delim=" ", dispCol="Z"):
"""
Plot image GPS csv on a map with mapbox
Parameters
----------
folder: string
working directory
csv: string
csv name - log.csv is default
delim: string
delimiter of csv " " (space) is defaut;
"""
df = pd.read_csv(csv, sep=" ", index_col=False)
token = 'pk.eyJ1IjoibWljYXNlbnNlIiwiYSI6ImNqYWx5dWNteTJ3cWYzMnBicmZid3g2YzcifQ.Zrq9t7GYocBtBzYyT3P4sw'
color_stops = create_color_stops(np.linspace( df[dispCol].min(),
df[dispCol].max(),
num=8), colors='YlOrRd')
data = df_to_geojson(df, ["#F=N", "Z"],lat='Y',lon='X')
viz = CircleViz(data, access_token=token, color_property=dispCol,
color_stops=color_stops,
center=[df['X'].median(),df['Y'].median()],
zoom=16, height='600px',
style='mapbox://styles/mapbox/satellite-streets-v9')
viz.show()
def test_df_geojson_file_nonsequential_index(df):
df.set_index('Avg Total Payments', inplace=True)
features = df_to_geojson(df, filename='out.geojson')
with open('out.geojson', 'r') as f:
testdata = json.load(f)
assert len(testdata['features']) == 3
def test_df_geojson_file(df):
features = df_to_geojson(df, filename='out.geojson')
with open('out.geojson', 'r') as f:
testdata = json.load(f)
assert len(testdata['features']) == 3
def test_df_no_properties(df_no_properties):
features = df_to_geojson(df_no_properties)['features']
assert tuple(features[0]['properties'].keys()) == ()
def test_df_properties(df):
features = df_to_geojson(df,
properties=['Avg Medicare Payments'])['features']
assert tuple(
features[0]['properties'].keys()) == ('Avg Medicare Payments', )
def test_df_geojson(df):
features = df_to_geojson(df)['features']
assert len(features) == 3
#
# Below we use the `mapboxgl` extension to plot the measured DLS yaw (or heading) angle from each image's meatadata over the whole imageset. We draw circles at each image location, and then color the circle based on the yaw value.
# In[ ]:
import math
import numpy as np
from mapboxgl.viz import *
from mapboxgl.utils import df_to_geojson, create_radius_stops, scale_between
from mapboxgl.utils import create_color_stops
#Insert your mapbox token here
token = 'pk.eyJ1IjoibWljYXNlbnNlIiwiYSI6ImNqYWx5dWNteTJ3cWYzMnBicmZid3g2YzcifQ.Zrq9t7GYocBtBzYyT3P4sw'
color_stops = create_color_stops(np.linspace(-math.pi,math.pi,num=8),colors='YlOrRd')
data = df_to_geojson(df,columns[3:],lat='latitude',lon='longitude')
viz = CircleViz(data, access_token=token, color_property='dls-yaw',
color_stops=color_stops,
center=[df['longitude'].median(),df['latitude'].median()],
zoom=16, height='600px',
style='mapbox://styles/mapbox/satellite-streets-v9')
viz.show()
# ## Plotting capture metadata
# In[ ]:
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')
df = df.head(15000)
df
# In[3]:
acces_token = os.getenv("MAPBOX_ACCESS_TOKEN")
# In[4]:
#data = df_to_geojson(df,
# properties=['Avg Medicare Payments', 'Avg Covered Charges'],
# precision=4)
data = df_to_geojson(df,
properties=["status"],
lat='latitude',
lon='longitude',
precision=None)
# In[187]:
viz = CircleViz(
data,
color_property=None,
# color_stops=None,
color_type="interval",
opacity=0.8,
label_property=None,
div_id='map',
height='500px',
style_url="mapbox://styles/mapbox/light-v9?optimize=true",
'lat': 35.6799683,
'lon': 139.7677253,
'color': 'blue'
},
{
'ts': 3,
'azimuth': 270,
'lat': 35.6699683,
'lon': 139.7677253,
'color': 'green'
},
])
properties = ['ts', 'azimuth', 'color']
geodata = df_to_geojson(data,
properties=properties,
lat='lat',
lon='lon',
precision=10)
viz = ArrowViz(data=geodata, center=(35.6812362, 139.7671248))
viz.export()
print(viz.create_html())
viz_color = ArrowViz(data=geodata,
center=(35.6812362, 139.7671248),
color='marker.properties["color"]')
viz_color.export('color.html')
def write_stacks(self):
if not self.warpMatrices:
print(
'warp_matrices not present, please run BatchProcess.set_warp_matrices before attempting to run BatchProcess.write_stacks'
)
return None
useDLS = True
overwrite = False # can be set to set to False to continue interrupted processing
if self.panelCap is not None:
if self.panelCap.panel_albedo() is not None and not any(
v is None for v in self.panelCap.panel_albedo()):
panel_reflectance_by_band = self.panelCap.panel_albedo()
else:
panel_reflectance_by_band = [0.67, 0.69, 0.68, 0.61,
0.67] #RedEdge band_index order
panel_irradiance = self.panelCap.panel_irradiance(
panel_reflectance_by_band)
img_type = "reflectance"
else:
if useDLS:
img_type = 'reflectance'
else:
img_type = 'radiance'
imgset = imageset.ImageSet.from_directory(self.imagePath)
data, columns = imgset.as_nested_lists()
df = pd.DataFrame.from_records(data,
index='timestamp',
columns=columns)
geojson_data = df_to_geojson(df,
columns[3:],
lat='latitude',
lon='longitude')
if not os.path.exists(self.outputPath):
os.makedirs(self.outputPath)
if self.thumbnails and not os.path.exists(self.thumbnailPath):
os.makedirs(self.thumbnailPath)
# Save out geojson data so we can open the image capture locations in our GIS
with open(os.path.join(self.outputPath, 'imageSet.json'), 'w') as f:
f.write(str(geojson_data))
try:
irradiance = panel_irradiance + [0]
except NameError:
irradiance = None
start = datetime.datetime.now()
for i, capture in enumerate(imgset.captures):
outputFilename = capture.uuid + '.tif'
thumbnailFilename = capture.uuid + '.jpg'
fullOutputPath = os.path.join(self.outputPath, outputFilename)
fullThumbnailPath = os.path.join(self.thumbnailPath,
thumbnailFilename)
if (not os.path.exists(fullOutputPath)) or overwrite:
if (len(capture.images) == len(imgset.captures[0].images)):
capture.create_aligned_capture(
irradiance_list=irradiance,
warp_matrices=self.warpMatrices)
capture.save_capture_as_stack(fullOutputPath)
if self.thumbnails:
# TODO
'''
IMPORTANT - CIR WRITEOUT IMPLEMENTATION COMMENTED OUT BELOW, AWAITING RESPONSE FROM MICASENSE
'''
#capture.save_thermal_over_rgb(fullThumbnailPath)
capture.save_capture_as_rgb(fullThumbnailPath)
capture.clear_image_data()
end = datetime.datetime.now()
print("Saving time: {}".format(end - start))
print("Alignment+Saving rate: {:.2f} images per second".format(
float(len(imgset.captures)) / float(
(end - start).total_seconds())))
'''WRITE METADATA TO ALIGNED STACKS'''
header = "SourceFile,\
GPSDateStamp,GPSTimeStamp,\
GPSLatitude,GpsLatitudeRef,\
GPSLongitude,GPSLongitudeRef,\
GPSAltitude,GPSAltitudeRef,\
FocalLength,\
XResolution,YResolution,ResolutionUnits\n"
lines = [header]
for capture in imgset.captures:
#get lat,lon,alt,time
outputFilename = capture.uuid + '.tif'
fullOutputPath = os.path.join(self.outputPath, outputFilename)
lat, lon, alt = capture.location()
#write to csv in format:
# IMG_0199_1.tif,"33 deg 32' 9.73"" N","111 deg 51' 1.41"" W",526 m Above Sea Level
latdeg, latmin, latsec = self.decdeg2dms(lat)
londeg, lonmin, lonsec = self.decdeg2dms(lon)
latdir = 'North'
if latdeg < 0:
latdeg = -latdeg
latdir = 'South'
londir = 'East'
if londeg < 0:
londeg = -londeg
londir = 'West'
resolution = capture.images[0].focal_plane_resolution_px_per_mm
linestr = '"{}",'.format(fullOutputPath)
linestr += capture.utc_time().strftime("%Y:%m:%d,%H:%M:%S,")
linestr += '"{:d} deg {:d}\' {:.2f}"" {}",{},'.format(
int(latdeg), int(latmin), latsec, latdir[0], latdir)
linestr += '"{:d} deg {:d}\' {:.2f}"" {}",{},{:.1f} m Above Sea Level,Above Sea Level,'.format(
int(londeg), int(lonmin), lonsec, londir[0], londir, alt)
linestr += '{}'.format(capture.images[0].focal_length)
linestr += '{},{},mm'.format(resolution, resolution)
linestr += '\n' # when writing in text mode, the write command will convert to os.linesep
lines.append(linestr)
fullCsvPath = os.path.join(self.outputPath, 'log.csv')
with open(fullCsvPath, 'w') as csvfile: #create CSV
csvfile.writelines(lines)
if os.environ.get('exiftoolpath') is not None:
exiftool_cmd = os.path.normpath(os.environ.get('exiftoolpath'))
else:
exiftool_cmd = 'exiftool'
cmd = '{} -csv="{}" -overwrite_original {}'.format(
exiftool_cmd, fullCsvPath, self.outputPath)
subprocess.check_call(cmd, shell=True)
def pre_processing(alignment_mat_path,
image_path,
flight_alt,
ground_alt=None,
panel_path_before=None,
panel_path_after=None,
reference_panel='micasense',
panel_detection_mode='default',
panel_capture_mode='manual',
reflectance_convert_mode='panel_dls',
save_as_geotiff=False,
generateThumbnails=True,
generateIndividualBands=True,
overwrite=False,
envi_metadata=True,
pix4D_metadata=True,
save_json=True):
# In[]
# ------------- loading the alignment matrices for various altitude -----------
import pickle
# alignment_mat_path = r"C:\Users\BAE User\Box\Digital Ag Lab\Codes\micasense_AliMoghimi\alignment_matrix\alignment_micasense_attempt_4_green_pyramid0.pkl"
pickle_in = open(alignment_mat_path, "rb")
alignment_micasense = pickle.load(pickle_in)
# alt_align_mat_measured_1 = [10, 15, 20, 25, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]
# alt_align_mat_measured = alignment_micasense['altitude']
alt_align_mat_measured = [
key for key in alignment_micasense.keys()
if isinstance(key, (int, float, complex))
]
# In[]
# ------------ setting the folders for saving the outputs ---------------------
outputPath_for_stacked = os.path.join(image_path, '..', 'stacks')
if not os.path.exists(outputPath_for_stacked):
os.makedirs(outputPath_for_stacked)
if generateThumbnails:
thumbnailPath = os.path.join(outputPath_for_stacked, '..',
'thumbnails')
if not os.path.exists(thumbnailPath):
os.makedirs(thumbnailPath)
# In[]
#---------------- correcting the irradiance measured by DLS -------------------
if panel_path_before is not None and reflectance_convert_mode != 'dls':
irr_correction = correction.Irradiance_correction_by_panel(
panel_path_before,
reference_panel=reference_panel,
panel_detection_mode=panel_detection_mode,
panel_capture_mode=panel_capture_mode)
irr_correction.radiance_to_reflectance()
mean_panel_ref = irr_correction.mean_panel_reflectance
print('panel reflectance before flight: {}'.format(mean_panel_ref)
) # it should match with the actual reflectance of the panel
dls_coef = irr_correction.coef_in_situ()
else:
dls_coef = np.array([1, 1, 1, 1, 1])
# some optional parameters:
#panel_corners = irr_correction.panel_corners
#irr_correction.radiance_to_reflectance()
#irr_correction.center_wavelengths
#irr_correction.plot_panel_reflectance()
#irr_correction.plot_coef_in_situ()
#irr_correction.plot_panel_location()
#irr_correction.plot_redi_irradi()
#dls_irr = []
#dls_pos = []
#for b in range(5):
# dls_irr.append(irr_correction.cap.images[b].meta.spectral_irradiance())
# dls_pos.append(irr_correction.cap.images[b].meta.dls_pose())
# In[]:
#---------------- Checking the coef-in-situ by panel images after flight -------------------
''' using the dls_coef calculated based on the panel image taken pre flight to calculate the reflectance
of the panel image taken post flight - I expect the reflectance of the panel after flight to be close
to the actual values when we use the dls_coef'''
if panel_path_after is not None and reflectance_convert_mode != 'dls':
irr_correction_after = correction.Irradiance_correction_by_panel(
panel_path_after,
reference_panel=reference_panel,
panel_detection_mode=panel_detection_mode,
panel_capture_mode=panel_capture_mode)
irr_correction_after.radiance_to_reflectance(dls_coef)
mean_panel_ref_after_with_coef = irr_correction_after.mean_panel_reflectance
print('panel reflectance after flight: {}'.format(
mean_panel_ref_after_with_coef)
) # it should be close to the actual reflectance of the panel
''' if we don't use any in_situ coef (i.e. only use DLS data), then this is the result:'''
no_dls_coef = np.array([1, 1, 1, 1, 1])
irr_correction_after.radiance_to_reflectance(no_dls_coef)
mean_panel_ref_after_no_coef = irr_correction_after.mean_panel_reflectance
print('panel reflectance after flight using only DLS data: {}'.format(
mean_panel_ref_after_no_coef)
) # it should match with the actual reflectance of the panel
''' calculating the in_situ coef based on the panel image taken after flight. I expect these coefficient
to be similar to the dls_coef calculated based on the panel image taken before flight'''
irr_correction_after.radiance_to_reflectance()
mean_panel_ref_after = irr_correction_after.mean_panel_reflectance
print(mean_panel_ref_after
) # it should match with the actual reflectance of the panel
dls_coef_after = irr_correction_after.coef_in_situ()
# In[]:
# -------------- loading the list of images from the given folder -------------
imlist = imageset.ImageSet.from_directory(image_path)
# -------------- convert the imagelist to Panda data frame --------------------
data, columns = imlist.as_nested_lists()
df = pd.DataFrame.from_records(data, index='capture_id', columns=columns)
# df['altitude'] = flight_alt
# print("in total {} set of images were loaded.".format())
# In[]
#------------------- save a report for the code ----------------------------
head_report, tail = os.path.split(image_path)
outputNameReport = head_report + "\\Report.txt"
with open(outputNameReport, "w") as report:
out_string = ""
out_string += 'total number of images: {}'.format(len(df))
out_string += '\n ----------------------------------------------------'
out_string += '\n'
out_string += '\n'
if panel_path_before:
out_string += 'panel reflectance before flight: {}'.format(
[round(item, 2) for item in mean_panel_ref])
out_string += '\n ----------------------------------------------------'
out_string += '\n'
out_string += '\n'
else:
out_string += 'Only DLS data was used for reflectance conversion, no panel was used!'
out_string += '\n ----------------------------------------------------'
out_string += '\n'
out_string += '\n'
if panel_path_after:
out_string += 'panel reflectance after flight: {}'.format(
[round(item, 2) for item in mean_panel_ref_after_with_coef])
out_string += '\n ----------------------------------------------------'
out_string += '\n'
out_string += '\n'
else:
out_string += 'No panel images captured after the flight were provided by user!'
out_string += '\n ----------------------------------------------------'
out_string += '\n'
out_string += '\n'
out_string += 'in_situ coeff used for DLS correction: {}'.format(
[round(item, 3) for item in dls_coef])
out_string += '\n ----------------------------------------------------'
out_string += '\n'
out_string += '\n'
report.write(out_string)
# In[ ]:
# saves df as a geojson so we can use QGIS/ArcGIS to open the captured images as points with the properties defined in 'column' variable
if save_json:
path_for_json = os.path.join(outputPath_for_stacked, '..', 'json')
if not os.path.exists(path_for_json):
os.makedirs(path_for_json)
from mapboxgl.utils import df_to_geojson
geojson_data = df_to_geojson(df, lat='latitude', lon='longitude')
with open(os.path.join(path_for_json, 'imageSet.json'), 'w') as f:
f.write(str(geojson_data))
# In[ ]:
#try:
# irradiance = panel_irradiance+[0]
#except NameError:
# irradiance = None
import datetime
start = datetime.datetime.now()
for i, cap in enumerate(imlist.captures):
if ground_alt:
_, _, alt_above_see = cap.location()
flight_alt = alt_above_see - ground_alt
delta_alt = abs(np.array(alt_align_mat_measured) - flight_alt)
if np.min(delta_alt) < 5:
ind = int(np.where(delta_alt == np.min(delta_alt))[0])
alt_align_close_to_flight_alt = alt_align_mat_measured[ind]
warp_matrices = [
alignment_micasense[alt_align_close_to_flight_alt][band]
for band in alignment_micasense[alt_align_close_to_flight_alt]
]
else:
raise IOError(
'Error: could not find a proper alignment matrix for this altitue!'
)
image_name_blue = cap.images[0].meta.get_item("File:FileName")
image_name = image_name_blue[0:-6] # remove '_1.tif'
outputFilename = image_name + '.tif'
full_outputPath_for_stacked = os.path.join(outputPath_for_stacked,
outputFilename)
if (not os.path.exists(full_outputPath_for_stacked)) or overwrite:
if (len(cap.images) == len(imlist.captures[0].images)):
if reflectance_convert_mode == 'panel':
panel_radiances = irr_correction.dn_to_radiance(
) # this is the irradiance calculated based on the radiance reflected from panel and the known reflectance values of the panel
irradiance = np.pi * panel_radiances / irr_correction.panel_actual_reflectance_by_band # this is actual irradiance to the panel
else: # using panel + DLS OR using DLS only
irradiance = dls_coef * cap.dls_irradiance()
cap.create_aligned_capture(
irradiance_list=irradiance,
warp_matrices=warp_matrices) # convert to reflectance
if save_as_geotiff:
cap.save_capture_as_stack_gtif(
full_outputPath_for_stacked,
flight_alt,
generateIndividualBands=generateIndividualBands)
else:
cap.save_capture_as_stack(
full_outputPath_for_stacked,
generateIndividualBands=generateIndividualBands)
if generateThumbnails:
thumbnailFilename = image_name + '.jpg'
fullThumbnailPath = os.path.join(thumbnailPath,
thumbnailFilename)
cap.save_capture_as_rgb(fullThumbnailPath)
cap.clear_image_data()
# save Metadata
save_meta = saveMetadata(cap, outputPath_for_stacked)
save_meta.save_metadata_pix4D(outputPath_for_stacked, mode='stack')
if envi_metadata:
save_meta.save_metadata_envi()
if pix4D_metadata:
head_csv, tail_csv = os.path.split(image_path)
csv_path = os.path.join(head_csv, 'individual_bands')
if not os.path.exists(csv_path):
os.makedirs(csv_path)
save_meta.save_metadata_pix4D(csv_path, mode='individual')
# update_f2(float(i)/float(len(imgset.captures)))
#update_f2(1.0)
end = datetime.datetime.now()
print("Saving time: {}".format(end - start))
print("Alignment+Saving rate: {:.2f} images per second".format(
float(len(imlist.captures)) / float((end - start).total_seconds())))
