def process_all_scanner_data(ply_parent, json_parent, out_parent,
save_dir_parent):
list_dirs = os.listdir(ply_parent)
for str_date in list_dirs:
print(str_date)
ply_path = os.path.join(ply_parent, str_date)
json_path = os.path.join(json_parent, str_date)
out_path = os.path.join(out_parent, str_date)
save_path = os.path.join(save_dir_parent, str_date)
if not os.path.isdir(ply_path):
print(ply_path)
continue
if not os.path.isdir(json_path):
print(json_path)
continue
print('start processing ' + str_date)
convt = terra_common.CoordinateConverter()
try:
q_flag = convt.bety_query(str_date, True)
if not q_flag:
print('Bety query failed')
continue
full_day_gen_angle_multi_process(ply_path, json_path, out_path,
convt)
full_day_summary(out_path, save_path, convt)
except Exception as ex:
fail(str(ex))
return
def main():
print("start...")
args = options()
start_date = '2019-04-18'
end_date = '2019-08-31'
convt = terra_common.CoordinateConverter()
qFlag = convt.bety_query(
'2019-06-18'
) # All plot boundaries in one season should be the same, currently 2019-06-18 works best
if not qFlag:
return
full_season_laserHeight_frame(args.ply_dir, args.json_dir, args.out_dir,
start_date, end_date, convt)
full_season_LH_integrate(args.out_dir, args.csv_dir, start_date, end_date,
convt)
bety_dir = os.path.join(args.csv_dir, 'bety_csv')
create_plot_betyCsv(args.csv_dir, bety_dir, start_date, end_date, convt)
return
def modify_param_process(in_dir, out_dir, csv_dir):
spe_date = {'2017-05-17', '2017-05-24'}
for str_date in spe_date:
print(str_date)
in_path = os.path.join(in_dir, str_date)
out_path = os.path.join(out_dir, str_date)
if not os.path.isdir(in_path):
continue
#integrate_cc_results(out_dir, str_date, csv_dir)
convt = terra_common.CoordinateConverter()
try:
#q_flag = convt.bety_query(str_date, False)
#if not q_flag:
# print('Bety query failed')
# continue
#full_day_gen_cc(in_path, out_path, convt)
integrate_cc_results(out_dir, str_date, csv_dir)
except Exception as ex:
fail(str_date + str(ex))
return
def main():
'''
in_dir = os.path.join('/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/raw_data/flirlrCamera', str_date)
out_dir = os.path.join('/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/Level_1/thermalData', str_date)
plot_dir = os.path.join('/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/Level_2/thermalCropToPlot/', str_date)
png_dir = os.path.join('/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/Level_2/thermalCropToPlot_png/', str_date)
crop_thermal_imageToPlot(in_dir, out_dir, plot_dir, png_dir, convt)
'''
print("start...")
start_date = '2019-06-07' # S9 start date
end_date = '2019-06-07' # S9 end date
convt = terra_common.CoordinateConverter()
qFlag = convt.bety_query('2019-06-19') # All plot boundaries in one season should be the same, currently 2019-06-18 works best
if not qFlag:
return
in_dir = os.path.join('/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/raw_data/flirlrCamera')
out_dir = os.path.join('/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/Level_1/thermalData')
plot_dir = os.path.join('/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/Level_2/thermalCropToPlot_new/')
png_dir = os.path.join('/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/Level_2/thermalCropToPlot_png_new/')
stitch_dir = os.path.join('/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/Level_2/StitchedPlotThermal')
full_season_thermalCrop_frame(in_dir, out_dir, plot_dir, png_dir, start_date, end_date, convt)
#full_season_thermal_stitch(png_dir, stitch_dir, start_date, end_date, convt)
return
def process_one_month_data(ply_parent, json_parent, out_parent, str_year,
str_month, str_start_date, str_end_date):
for day in range(int(str_start_date), int(str_end_date) + 1):
target_date = date(int(str_year), int(str_month), day)
str_date = target_date.isoformat()
print(str_date)
ply_path = os.path.join(ply_parent, str_date)
json_path = os.path.join(json_parent, str_date)
out_path = os.path.join(out_parent, str_date)
convt = terra_common.CoordinateConverter()
if not os.path.isdir(ply_path):
print('ply miss' + ply_path)
continue
if not os.path.isdir(json_path):
print('json miss' + json_path)
continue
try:
q_flag = convt.bety_query(str_date, True)
if not q_flag:
print('Bety query failed')
continue
full_day_gen_hist(ply_path, json_path, out_path, convt)
full_day_array_integrate_s6(out_path)
#insert_height_traits_into_betydb(out_path, out_path, str_date, B_TH_PERC, 'e')
except Exception as ex:
fail(str_date + str(ex))
return
def test():
str_date = '2020-02-03'
in_dir = os.path.join(
'/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/Level_2/stereoTop/',
str_date)
out_dir = os.path.join(
'/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/Level_2/StitchedPlotRGB',
str_date)
convt = terra_common.CoordinateConverter()
qFlag = convt.bety_query(str_date)
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
list_dirs = os.listdir(in_dir)
for d in list_dirs:
in_path = os.path.join(in_dir, d)
out_path = os.path.join(out_dir, d)
if not os.path.isdir(in_path):
continue
if not os.path.isdir(out_path):
os.mkdir(out_path)
stitch_plot_rgb_image(in_path, out_path, str_date, convt)
return
def get_plot_num(meta):
center_position = parse_metadata(meta)
convt = terra_common.CoordinateConverter()
plot_row, plot_col = convt.fieldPosition_to_fieldPartition(center_position[0], center_position[1])
plotNum = convt.fieldPartition_to_plotNum(plot_row, plot_col)
return plotNum
def get_plot_num(meta):
center_position, hh = parse_metadata(meta)
if center_position == None:
return 0
convt = terra_common.CoordinateConverter()
plot_row, plot_col = convt.fieldPosition_to_fieldPartition(center_position[0], center_position[1])
plotNum = convt.fieldPartition_to_plotNum(plot_row, plot_col)
if hh < 7 or hh > 18:
plotNum = 0
return plotNum
def laser_main():
in_dir = '/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/Level_2/laser_crop/2019_06__3-6'
out_dir = '/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/Level_2/laser_fusion_data'
convt = terra_common.CoordinateConverter()
qFlag = convt.bety_query(
'2019-06-18'
) # All plot boundaries in one season should be the same, currently 2019-06-18 works best
if not qFlag:
return
roi_list = generate_roi_list_all_plots_center_plant(0.6, convt)
crop_laser_to_roi(roi_list, in_dir, out_dir)
return
def test_single_dir():
in_dir = '/media/zli/Elements/ua-mac/Level_2/RgbCropToPlot/2017-05-05/22-10-343'
out_dir = '/media/zli/Elements/ua-mac/Level_2/StitchedPlotRGB/2017-05-05/22-10-343'
str_date = '2018-05-24'
convt = terra_common.CoordinateConverter()
qFlag = convt.bety_query(str_date)
stitch_plot_rgb_image(in_dir, out_dir, str_date, convt)
stitch_plot_rgb_image(
'/media/zli/Elements/ua-mac/Level_2/RgbCropToPlot/2017-05-05/41-09-649',
'/media/zli/Elements/ua-mac/Level_2/StitchedPlotRGB/2017-05-05/41-09-649',
str_date, convt)
return
def test():
in_dir = '/media/zli/Elements/terra_evaluation/emp/'
out_dir = '/media/zli/Elements/terra_evaluation/emp/outputs/'
bety_dir = '/media/zli/Elements/terra_evaluation/emp/outputs/BETY'
convt = terra_common.CoordinateConverter()
convt.bety_query('2017-04-27', False)
str_date = '2017-04-27'
full_day_aggregate(in_dir, out_dir, bety_dir, str_date, 6, convt)
str_date = '2017-04-28'
full_day_aggregate(in_dir, out_dir, bety_dir, str_date, 6, convt)
str_date = '2017-04-29'
full_day_aggregate(in_dir, out_dir, bety_dir, str_date, 6, convt)
'''
in_1 = '/media/zli/Elements/terra_evaluation/emp/2017-04-27/2017-04-27__09-43-14-707'
in_2 = '/media/zli/Elements/terra_evaluation/emp/2017-04-27/2017-04-27__09-43-16-229'
convt = terra_common.CoordinateConverter()
convt.bety_query('2017-04-27', False)
boxList = []
pl1, emp1 = parse_data_from_emp_csv(in_1, convt)
for emp in emp1:
boxList.append(emp)
pl2, emp2 = parse_data_from_emp_csv(in_2, convt)
for emp in emp2:
saveFlag = True
for savedItem in boxList:
center_dist = math.sqrt((savedItem[0]-emp[0])**2+(savedItem[1]-emp[1])**2)
radiusAdd = savedItem[2]+emp[2]
if center_dist < radiusAdd:
saveFlag = False
break
if saveFlag:
boxList.append(emp)
'''
return
def process_angleData_by_date(ply_parent, json_parent, out_parent, str_year,
str_start_month, str_end_month, str_start_date,
str_end_date, save_dir_parent):
if not os.path.isdir(save_dir_parent):
os.makedirs(save_dir_parent)
d1 = date(int(str_year), int(str_start_month), int(str_start_date))
d2 = date(int(str_year), int(str_end_month), int(str_end_date))
deltaDay = d2 - d1
for i in range(deltaDay.days + 1):
str_date = str(d1 + timedelta(days=i))
print(str_date)
ply_path = os.path.join(ply_parent, str_date)
json_path = os.path.join(json_parent, str_date)
out_path = os.path.join(out_parent, str_date)
save_path = os.path.join(save_dir_parent, str_date)
if not os.path.isdir(ply_path):
continue
if not os.path.isdir(json_path):
continue
convt = terra_common.CoordinateConverter()
try:
q_flag = convt.bety_query(str_date, True)
if not q_flag:
print('Bety query for boundaries failed')
continue
# create one days leaf angle by plot
full_day_gen_angle_data(ply_path, json_path, out_path, convt)
# full days angle hist summary
full_day_summary(out_path, save_path, convt)
# data uploading
insert_leafAngle_traits_into_betydb(save_path, save_path, str_date,
convt)
except Exception as ex:
fail(str_date + str(ex))
return
def gwu_crop_to_roi(in_dir,
out_dir,
sensor_type,
bety_query_date,
temp_dir,
field_size=0.6):
convt = terra_common.CoordinateConverter()
qFlag = convt.bety_query(bety_query_date) # s9, '2019-06-18'
if not qFlag:
return
roi_list = generate_roi_list_all_plots_center_plant(field_size, convt)
crop_to_roi_multisensor_multi_process(roi_list, in_dir, out_dir,
sensor_type, temp_dir)
return
def main():
print("start...")
args = options()
start_date = '2019-04-18'
end_date = '2019-08-31'
convt = terra_common.CoordinateConverter()
qFlag = convt.bety_query('2019-06-18') # All plot boundaries in one season should be the same, currently 2019-06-18 works best
if not qFlag:
return
full_season_cc_frame(args.in_dir, args.out_dir, start_date, end_date, convt)
full_season_cc_integrate(args.out_dir, args.csv_dir, start_date, end_date, convt)
return
def process_by_dates(ply_parent, json_parent, out_parent, str_year,
str_start_month, str_end_month, str_start_date,
str_end_date):
d1 = date(int(str_year), int(str_start_month), int(str_start_date))
d2 = date(int(str_year), int(str_end_month), int(str_end_date))
deltaDay = d2 - d1
for i in range(deltaDay.days + 1):
str_date = str(d1 + timedelta(days=i))
print(str_date)
ply_path = os.path.join(ply_parent, str_date)
json_path = os.path.join(json_parent, str_date)
out_path = os.path.join(out_parent, str_date)
convt = terra_common.CoordinateConverter()
if not os.path.isdir(ply_path):
print('ply miss' + ply_path)
continue
if not os.path.isdir(json_path):
print('json miss' + json_path)
continue
try:
print('bety query')
q_flag = convt.bety_query(str_date, True)
if not q_flag:
print('Bety query failed')
continue
print('multi process')
full_day_gen_hist_multi_process(ply_path, json_path, out_path,
convt)
full_day_array_integrate_s6(out_path, convt)
except Exception as ex:
fail(str_date + str(ex))
return
def main():
start_date = '2020-02-03' # S9 start date
end_date = '2020-02-03' # S9 end date
convt = terra_common.CoordinateConverter()
qFlag = convt.bety_query(
'2020-02-03'
) # All plot boundaries in one season should be the same, currently 2019-06-18 works best
if not qFlag:
return
#args = options()
raw_dir = '/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/raw_data/stereoTop'
out_dir = '/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/Level_1/stereoTop'
plot_dir = '/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/Level_2/rgb_crop'
stitch_dir = '/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/Level_2/StitchedPlotRGB'
full_season_crop_rgb(raw_dir, out_dir, plot_dir, start_date, end_date,
convt)
full_season_rgb_stitch(plot_dir, stitch_dir, start_date, end_date, convt)
def main_local_workstation():
convt = terra_common.CoordinateConverter()
qFlag = convt.bety_query(
'2019-06-18'
) # All plot boundaries in one season should be the same, currently 2019-06-18 works best
if not qFlag:
return
roi_list = generate_roi_list_all_plots_center_plant(0.5, convt)
start_date = '2019-06-07' # S9 start date
end_date = '2019-06-07' # S9 end date
in_rgb_dir = '/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/Level_2/rgb_crop_new'
in_thermal_dir = '/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/Level_2/thermalCropToPlot_png_new'
out_rgb_dir = '/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/Level_2/rgb_fusion_data_0.5_new'
out_thermal_dir = '/media/zli/Seagate Backup Plus Drive/OPEN/ua-mac/Level_2/thermal_fusion_data_0.5_new'
crop_from_both_sensors(roi_list, in_rgb_dir, in_thermal_dir, start_date,
end_date, out_rgb_dir, out_thermal_dir, convt)
return
''' Created on Aug 9, 2016 @author: Zongyang Li ''' import os, sys, json, argparse, shutil, terra_common import numpy as np from glob import glob from plyfile import PlyData from matplotlib import cm import matplotlib.pyplot as plt from datetime import date from terrautils import betydb convt = terra_common.CoordinateConverter() PLOT_RANGE_NUM = 54 PLOT_COL_NUM = 32 HIST_BIN_NUM = 400 # [(35th percentile from E + 35th percentile from W) / 2] cm * 0.97841 + 25.678cm B_TH_PERC = 0.35 B_F_SLOPE = 0.97841 B_F_OFFSET = 25.678 # [37th percentile from E sensor] cm * 0.94323 + 26.41cm E_TH_PERC = 0.37 E_F_SLOPE = 0.94323 E_F_OFFSET = 26.41
