def plot_img_csv(folder, csv='log.csv', delim=" ", dispCol="Z"): """ Plot image GPS csv on a map with mapbox Parameters ---------- folder: string working directory csv: string csv name - log.csv is default delim: string delimiter of csv " " (space) is defaut; """ df = pd.read_csv(csv, sep=" ", index_col=False) token = 'pk.eyJ1IjoibWljYXNlbnNlIiwiYSI6ImNqYWx5dWNteTJ3cWYzMnBicmZid3g2YzcifQ.Zrq9t7GYocBtBzYyT3P4sw' color_stops = create_color_stops(np.linspace( df[dispCol].min(), df[dispCol].max(), num=8), colors='YlOrRd') data = df_to_geojson(df, ["#F=N", "Z"],lat='Y',lon='X') viz = CircleViz(data, access_token=token, color_property=dispCol, color_stops=color_stops, center=[df['X'].median(),df['Y'].median()], zoom=16, height='600px', style='mapbox://styles/mapbox/satellite-streets-v9') viz.show()
def test_df_geojson_file_nonsequential_index(df): df.set_index('Avg Total Payments', inplace=True) features = df_to_geojson(df, filename='out.geojson') with open('out.geojson', 'r') as f: testdata = json.load(f) assert len(testdata['features']) == 3
def test_df_geojson_file(df): features = df_to_geojson(df, filename='out.geojson') with open('out.geojson', 'r') as f: testdata = json.load(f) assert len(testdata['features']) == 3
def test_df_no_properties(df_no_properties): features = df_to_geojson(df_no_properties)['features'] assert tuple(features[0]['properties'].keys()) == ()
def test_df_properties(df): features = df_to_geojson(df, properties=['Avg Medicare Payments'])['features'] assert tuple( features[0]['properties'].keys()) == ('Avg Medicare Payments', )
def test_df_geojson(df): features = df_to_geojson(df)['features'] assert len(features) == 3
# # Below we use the `mapboxgl` extension to plot the measured DLS yaw (or heading) angle from each image's meatadata over the whole imageset. We draw circles at each image location, and then color the circle based on the yaw value. # In[ ]: import math import numpy as np from mapboxgl.viz import * from mapboxgl.utils import df_to_geojson, create_radius_stops, scale_between from mapboxgl.utils import create_color_stops #Insert your mapbox token here token = 'pk.eyJ1IjoibWljYXNlbnNlIiwiYSI6ImNqYWx5dWNteTJ3cWYzMnBicmZid3g2YzcifQ.Zrq9t7GYocBtBzYyT3P4sw' color_stops = create_color_stops(np.linspace(-math.pi,math.pi,num=8),colors='YlOrRd') data = df_to_geojson(df,columns[3:],lat='latitude',lon='longitude') viz = CircleViz(data, access_token=token, color_property='dls-yaw', color_stops=color_stops, center=[df['longitude'].median(),df['latitude'].median()], zoom=16, height='600px', style='mapbox://styles/mapbox/satellite-streets-v9') viz.show() # ## Plotting capture metadata # In[ ]: import matplotlib.pyplot as plt get_ipython().run_line_magic('matplotlib', 'inline')
df = df.head(15000) df # In[3]: acces_token = os.getenv("MAPBOX_ACCESS_TOKEN") # In[4]: #data = df_to_geojson(df, # properties=['Avg Medicare Payments', 'Avg Covered Charges'], # precision=4) data = df_to_geojson(df, properties=["status"], lat='latitude', lon='longitude', precision=None) # In[187]: viz = CircleViz( data, color_property=None, # color_stops=None, color_type="interval", opacity=0.8, label_property=None, div_id='map', height='500px', style_url="mapbox://styles/mapbox/light-v9?optimize=true",
'lat': 35.6799683, 'lon': 139.7677253, 'color': 'blue' }, { 'ts': 3, 'azimuth': 270, 'lat': 35.6699683, 'lon': 139.7677253, 'color': 'green' }, ]) properties = ['ts', 'azimuth', 'color'] geodata = df_to_geojson(data, properties=properties, lat='lat', lon='lon', precision=10) viz = ArrowViz(data=geodata, center=(35.6812362, 139.7671248)) viz.export() print(viz.create_html()) viz_color = ArrowViz(data=geodata, center=(35.6812362, 139.7671248), color='marker.properties["color"]') viz_color.export('color.html')
def write_stacks(self): if not self.warpMatrices: print( 'warp_matrices not present, please run BatchProcess.set_warp_matrices before attempting to run BatchProcess.write_stacks' ) return None useDLS = True overwrite = False # can be set to set to False to continue interrupted processing if self.panelCap is not None: if self.panelCap.panel_albedo() is not None and not any( v is None for v in self.panelCap.panel_albedo()): panel_reflectance_by_band = self.panelCap.panel_albedo() else: panel_reflectance_by_band = [0.67, 0.69, 0.68, 0.61, 0.67] #RedEdge band_index order panel_irradiance = self.panelCap.panel_irradiance( panel_reflectance_by_band) img_type = "reflectance" else: if useDLS: img_type = 'reflectance' else: img_type = 'radiance' imgset = imageset.ImageSet.from_directory(self.imagePath) data, columns = imgset.as_nested_lists() df = pd.DataFrame.from_records(data, index='timestamp', columns=columns) geojson_data = df_to_geojson(df, columns[3:], lat='latitude', lon='longitude') if not os.path.exists(self.outputPath): os.makedirs(self.outputPath) if self.thumbnails and not os.path.exists(self.thumbnailPath): os.makedirs(self.thumbnailPath) # Save out geojson data so we can open the image capture locations in our GIS with open(os.path.join(self.outputPath, 'imageSet.json'), 'w') as f: f.write(str(geojson_data)) try: irradiance = panel_irradiance + [0] except NameError: irradiance = None start = datetime.datetime.now() for i, capture in enumerate(imgset.captures): outputFilename = capture.uuid + '.tif' thumbnailFilename = capture.uuid + '.jpg' fullOutputPath = os.path.join(self.outputPath, outputFilename) fullThumbnailPath = os.path.join(self.thumbnailPath, thumbnailFilename) if (not os.path.exists(fullOutputPath)) or overwrite: if (len(capture.images) == len(imgset.captures[0].images)): capture.create_aligned_capture( irradiance_list=irradiance, warp_matrices=self.warpMatrices) capture.save_capture_as_stack(fullOutputPath) if self.thumbnails: # TODO ''' IMPORTANT - CIR WRITEOUT IMPLEMENTATION COMMENTED OUT BELOW, AWAITING RESPONSE FROM MICASENSE ''' #capture.save_thermal_over_rgb(fullThumbnailPath) capture.save_capture_as_rgb(fullThumbnailPath) capture.clear_image_data() end = datetime.datetime.now() print("Saving time: {}".format(end - start)) print("Alignment+Saving rate: {:.2f} images per second".format( float(len(imgset.captures)) / float( (end - start).total_seconds()))) '''WRITE METADATA TO ALIGNED STACKS''' header = "SourceFile,\ GPSDateStamp,GPSTimeStamp,\ GPSLatitude,GpsLatitudeRef,\ GPSLongitude,GPSLongitudeRef,\ GPSAltitude,GPSAltitudeRef,\ FocalLength,\ XResolution,YResolution,ResolutionUnits\n" lines = [header] for capture in imgset.captures: #get lat,lon,alt,time outputFilename = capture.uuid + '.tif' fullOutputPath = os.path.join(self.outputPath, outputFilename) lat, lon, alt = capture.location() #write to csv in format: # IMG_0199_1.tif,"33 deg 32' 9.73"" N","111 deg 51' 1.41"" W",526 m Above Sea Level latdeg, latmin, latsec = self.decdeg2dms(lat) londeg, lonmin, lonsec = self.decdeg2dms(lon) latdir = 'North' if latdeg < 0: latdeg = -latdeg latdir = 'South' londir = 'East' if londeg < 0: londeg = -londeg londir = 'West' resolution = capture.images[0].focal_plane_resolution_px_per_mm linestr = '"{}",'.format(fullOutputPath) linestr += capture.utc_time().strftime("%Y:%m:%d,%H:%M:%S,") linestr += '"{:d} deg {:d}\' {:.2f}"" {}",{},'.format( int(latdeg), int(latmin), latsec, latdir[0], latdir) linestr += '"{:d} deg {:d}\' {:.2f}"" {}",{},{:.1f} m Above Sea Level,Above Sea Level,'.format( int(londeg), int(lonmin), lonsec, londir[0], londir, alt) linestr += '{}'.format(capture.images[0].focal_length) linestr += '{},{},mm'.format(resolution, resolution) linestr += '\n' # when writing in text mode, the write command will convert to os.linesep lines.append(linestr) fullCsvPath = os.path.join(self.outputPath, 'log.csv') with open(fullCsvPath, 'w') as csvfile: #create CSV csvfile.writelines(lines) if os.environ.get('exiftoolpath') is not None: exiftool_cmd = os.path.normpath(os.environ.get('exiftoolpath')) else: exiftool_cmd = 'exiftool' cmd = '{} -csv="{}" -overwrite_original {}'.format( exiftool_cmd, fullCsvPath, self.outputPath) subprocess.check_call(cmd, shell=True)
def pre_processing(alignment_mat_path, image_path, flight_alt, ground_alt=None, panel_path_before=None, panel_path_after=None, reference_panel='micasense', panel_detection_mode='default', panel_capture_mode='manual', reflectance_convert_mode='panel_dls', save_as_geotiff=False, generateThumbnails=True, generateIndividualBands=True, overwrite=False, envi_metadata=True, pix4D_metadata=True, save_json=True): # In[] # ------------- loading the alignment matrices for various altitude ----------- import pickle # alignment_mat_path = r"C:\Users\BAE User\Box\Digital Ag Lab\Codes\micasense_AliMoghimi\alignment_matrix\alignment_micasense_attempt_4_green_pyramid0.pkl" pickle_in = open(alignment_mat_path, "rb") alignment_micasense = pickle.load(pickle_in) # alt_align_mat_measured_1 = [10, 15, 20, 25, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120] # alt_align_mat_measured = alignment_micasense['altitude'] alt_align_mat_measured = [ key for key in alignment_micasense.keys() if isinstance(key, (int, float, complex)) ] # In[] # ------------ setting the folders for saving the outputs --------------------- outputPath_for_stacked = os.path.join(image_path, '..', 'stacks') if not os.path.exists(outputPath_for_stacked): os.makedirs(outputPath_for_stacked) if generateThumbnails: thumbnailPath = os.path.join(outputPath_for_stacked, '..', 'thumbnails') if not os.path.exists(thumbnailPath): os.makedirs(thumbnailPath) # In[] #---------------- correcting the irradiance measured by DLS ------------------- if panel_path_before is not None and reflectance_convert_mode != 'dls': irr_correction = correction.Irradiance_correction_by_panel( panel_path_before, reference_panel=reference_panel, panel_detection_mode=panel_detection_mode, panel_capture_mode=panel_capture_mode) irr_correction.radiance_to_reflectance() mean_panel_ref = irr_correction.mean_panel_reflectance print('panel reflectance before flight: {}'.format(mean_panel_ref) ) # it should match with the actual reflectance of the panel dls_coef = irr_correction.coef_in_situ() else: dls_coef = np.array([1, 1, 1, 1, 1]) # some optional parameters: #panel_corners = irr_correction.panel_corners #irr_correction.radiance_to_reflectance() #irr_correction.center_wavelengths #irr_correction.plot_panel_reflectance() #irr_correction.plot_coef_in_situ() #irr_correction.plot_panel_location() #irr_correction.plot_redi_irradi() #dls_irr = [] #dls_pos = [] #for b in range(5): # dls_irr.append(irr_correction.cap.images[b].meta.spectral_irradiance()) # dls_pos.append(irr_correction.cap.images[b].meta.dls_pose()) # In[]: #---------------- Checking the coef-in-situ by panel images after flight ------------------- ''' using the dls_coef calculated based on the panel image taken pre flight to calculate the reflectance of the panel image taken post flight - I expect the reflectance of the panel after flight to be close to the actual values when we use the dls_coef''' if panel_path_after is not None and reflectance_convert_mode != 'dls': irr_correction_after = correction.Irradiance_correction_by_panel( panel_path_after, reference_panel=reference_panel, panel_detection_mode=panel_detection_mode, panel_capture_mode=panel_capture_mode) irr_correction_after.radiance_to_reflectance(dls_coef) mean_panel_ref_after_with_coef = irr_correction_after.mean_panel_reflectance print('panel reflectance after flight: {}'.format( mean_panel_ref_after_with_coef) ) # it should be close to the actual reflectance of the panel ''' if we don't use any in_situ coef (i.e. only use DLS data), then this is the result:''' no_dls_coef = np.array([1, 1, 1, 1, 1]) irr_correction_after.radiance_to_reflectance(no_dls_coef) mean_panel_ref_after_no_coef = irr_correction_after.mean_panel_reflectance print('panel reflectance after flight using only DLS data: {}'.format( mean_panel_ref_after_no_coef) ) # it should match with the actual reflectance of the panel ''' calculating the in_situ coef based on the panel image taken after flight. I expect these coefficient to be similar to the dls_coef calculated based on the panel image taken before flight''' irr_correction_after.radiance_to_reflectance() mean_panel_ref_after = irr_correction_after.mean_panel_reflectance print(mean_panel_ref_after ) # it should match with the actual reflectance of the panel dls_coef_after = irr_correction_after.coef_in_situ() # In[]: # -------------- loading the list of images from the given folder ------------- imlist = imageset.ImageSet.from_directory(image_path) # -------------- convert the imagelist to Panda data frame -------------------- data, columns = imlist.as_nested_lists() df = pd.DataFrame.from_records(data, index='capture_id', columns=columns) # df['altitude'] = flight_alt # print("in total {} set of images were loaded.".format()) # In[] #------------------- save a report for the code ---------------------------- head_report, tail = os.path.split(image_path) outputNameReport = head_report + "\\Report.txt" with open(outputNameReport, "w") as report: out_string = "" out_string += 'total number of images: {}'.format(len(df)) out_string += '\n ----------------------------------------------------' out_string += '\n' out_string += '\n' if panel_path_before: out_string += 'panel reflectance before flight: {}'.format( [round(item, 2) for item in mean_panel_ref]) out_string += '\n ----------------------------------------------------' out_string += '\n' out_string += '\n' else: out_string += 'Only DLS data was used for reflectance conversion, no panel was used!' out_string += '\n ----------------------------------------------------' out_string += '\n' out_string += '\n' if panel_path_after: out_string += 'panel reflectance after flight: {}'.format( [round(item, 2) for item in mean_panel_ref_after_with_coef]) out_string += '\n ----------------------------------------------------' out_string += '\n' out_string += '\n' else: out_string += 'No panel images captured after the flight were provided by user!' out_string += '\n ----------------------------------------------------' out_string += '\n' out_string += '\n' out_string += 'in_situ coeff used for DLS correction: {}'.format( [round(item, 3) for item in dls_coef]) out_string += '\n ----------------------------------------------------' out_string += '\n' out_string += '\n' report.write(out_string) # In[ ]: # saves df as a geojson so we can use QGIS/ArcGIS to open the captured images as points with the properties defined in 'column' variable if save_json: path_for_json = os.path.join(outputPath_for_stacked, '..', 'json') if not os.path.exists(path_for_json): os.makedirs(path_for_json) from mapboxgl.utils import df_to_geojson geojson_data = df_to_geojson(df, lat='latitude', lon='longitude') with open(os.path.join(path_for_json, 'imageSet.json'), 'w') as f: f.write(str(geojson_data)) # In[ ]: #try: # irradiance = panel_irradiance+[0] #except NameError: # irradiance = None import datetime start = datetime.datetime.now() for i, cap in enumerate(imlist.captures): if ground_alt: _, _, alt_above_see = cap.location() flight_alt = alt_above_see - ground_alt delta_alt = abs(np.array(alt_align_mat_measured) - flight_alt) if np.min(delta_alt) < 5: ind = int(np.where(delta_alt == np.min(delta_alt))[0]) alt_align_close_to_flight_alt = alt_align_mat_measured[ind] warp_matrices = [ alignment_micasense[alt_align_close_to_flight_alt][band] for band in alignment_micasense[alt_align_close_to_flight_alt] ] else: raise IOError( 'Error: could not find a proper alignment matrix for this altitue!' ) image_name_blue = cap.images[0].meta.get_item("File:FileName") image_name = image_name_blue[0:-6] # remove '_1.tif' outputFilename = image_name + '.tif' full_outputPath_for_stacked = os.path.join(outputPath_for_stacked, outputFilename) if (not os.path.exists(full_outputPath_for_stacked)) or overwrite: if (len(cap.images) == len(imlist.captures[0].images)): if reflectance_convert_mode == 'panel': panel_radiances = irr_correction.dn_to_radiance( ) # this is the irradiance calculated based on the radiance reflected from panel and the known reflectance values of the panel irradiance = np.pi * panel_radiances / irr_correction.panel_actual_reflectance_by_band # this is actual irradiance to the panel else: # using panel + DLS OR using DLS only irradiance = dls_coef * cap.dls_irradiance() cap.create_aligned_capture( irradiance_list=irradiance, warp_matrices=warp_matrices) # convert to reflectance if save_as_geotiff: cap.save_capture_as_stack_gtif( full_outputPath_for_stacked, flight_alt, generateIndividualBands=generateIndividualBands) else: cap.save_capture_as_stack( full_outputPath_for_stacked, generateIndividualBands=generateIndividualBands) if generateThumbnails: thumbnailFilename = image_name + '.jpg' fullThumbnailPath = os.path.join(thumbnailPath, thumbnailFilename) cap.save_capture_as_rgb(fullThumbnailPath) cap.clear_image_data() # save Metadata save_meta = saveMetadata(cap, outputPath_for_stacked) save_meta.save_metadata_pix4D(outputPath_for_stacked, mode='stack') if envi_metadata: save_meta.save_metadata_envi() if pix4D_metadata: head_csv, tail_csv = os.path.split(image_path) csv_path = os.path.join(head_csv, 'individual_bands') if not os.path.exists(csv_path): os.makedirs(csv_path) save_meta.save_metadata_pix4D(csv_path, mode='individual') # update_f2(float(i)/float(len(imgset.captures))) #update_f2(1.0) end = datetime.datetime.now() print("Saving time: {}".format(end - start)) print("Alignment+Saving rate: {:.2f} images per second".format( float(len(imlist.captures)) / float((end - start).total_seconds())))