class SteerDataset(Dataset):
"""
Returns img, steering_angle pairs
"""
def __init__(self, datapath, transforms=None):
"""
datapath: source of final processed & augmented data
"""
super(SteerDataset, self).__init__()
self.datapath = datapath
self.dutils = Data_Utils()
self.df = self.dutils.get_df(self.datapath)
self.transforms = transforms
def __len__(self):
return len(self.df)
def __getitem__(self, idx):
"""
Returns dictionary {"img": Tensor, C x H x W, "angle":float 1-Tensor}
"""
cv_img, df_row = self.dutils.df_data_fromidx(self.datapath, self.df,
idx)
if self.transforms:
cv_img = self.transforms(cv_img)
ts_angle = torch.Tensor([df_row[1]]).float()
ts_img = torch.from_numpy(cv_img).permute(2, 0,
1).float() #size (C x H x W)
data_dict = {"img": ts_img, "angle": ts_angle}
return data_dict
def __init__(self, datapath, transforms=None):
"""
datapath: source of final processed & augmented data
"""
super(SteerDataset, self).__init__()
self.datapath = datapath
self.dutils = Data_Utils()
self.df = self.dutils.get_df(self.datapath)
self.transforms = transforms
def __init__(self, sess_path=None, writer=None):
"""
sess_path: current working dir of this session
writer: Tensorboard SummmaryWriter
"""
self.gvis = lambda param: session["visualizer"].get(param)
self.fixangle = lambda angle, units: angle if units == 'rad' else angle * math.pi / 180.0
self.writer = writer
self.data_utils = Data_Utils()
def __init__(self):
jsonfile = steps.session
#The state of the session
self.params_dict = jsonfile["params"]
self.steplist = jsonfile["steps"]
self.curr_step_idx = 0
self.dlist = None #running list of folders to operate on
#Session ID & Logger
self.sess_id, self.sess_path = self._create_session_data(
self.params_dict["abs_path"], self.params_dict["sess_root"])
self.writer = self._create_writer(self.sess_path,
'logs',
comment=self.params_dict["comment"])
#Periphery Stuff for data moving and visualization
self.data_utils = Data_Utils()
self.visualizer = Metric_Visualizer(self.sess_path, self.writer)
import os, json, pdb, cv2, math
import numpy as np
from functools import partial
import pandas as pd
from Data_Utils import Data_Utils
dutils = Data_Utils()
def changeName(dest_dict, prefix):
curr_name = dest_dict.get("row")[0]
dest_dict["row"][0] = prefix + curr_name
return dest_dict
def cannyEdge(args, src_dict):
assert(len(args) == 2), "Incorrect size argument to cannyEdge"
dest_dict = src_dict
src_img = src_dict["img"]
dest_img = cv2.Canny(src_img, args[0], args[1])
dest_dict["img"] = dest_img
return changeName(dest_dict, 'cannyedge')
def filterBadData(args, src_dict):
assert(len(args) == 0), "Incorrect size argument to filterBadData"
dest_dict = src_dict
src_img = src_dict.get("img")
#check to see if picture is largely black
if np.mean(src_img) < 5:
dest_dict["flag"] = False
return changeName(dest_dict, 'fbd')
def opticalFlow(args, src_dict):
class Stepper(object):
"""
Parses steps.session and executes ins. on data folders accordingly to get it into a final state
"""
def __init__(self):
jsonfile = steps.session
#The state of the session
self.params_dict = jsonfile["params"]
self.steplist = jsonfile["steps"]
self.curr_step_idx = 0
self.dlist = None #running list of folders to operate on
#Session ID & Logger
self.sess_id, self.sess_path = self._create_session_data(
self.params_dict["abs_path"], self.params_dict["sess_root"])
self.writer = self._create_writer(self.sess_path,
'logs',
comment=self.params_dict["comment"])
#Periphery Stuff for data moving and visualization
self.data_utils = Data_Utils()
self.visualizer = Metric_Visualizer(self.sess_path, self.writer)
def _create_session_data(self, abs_path, sess_root):
"""
Returns an int representing the unique session ID & makes a folder for the session, along with a string representing the current working directory of the session
sess_root: root folder containing all the sessions
"""
sess_path = os.path.join(abs_path, sess_root)
if not os.path.exists(sess_path):
os.makedirs(sess_path)
sess_id = len(os.listdir(sess_path))
sess_path = os.path.join(sess_path, str(sess_id))
print("SESSION PATH:", sess_path)
print("SESSION ID:", sess_id)
return sess_id, sess_path
def _create_writer(self, sess_path, log_foldername, comment=''):
"""
Retruns a summary writer in the right place
sess_path: current working dir of session
"""
logdir = os.path.join(sess_path, log_foldername)
print("LOGDIR:", logdir)
writer = SummaryWriter(logdir=logdir, comment=comment)
return writer
def B_VER(self, ver_path, dlist):
"""
(B)asic (VER)ification of folder structure in dlist
ver_path is generally sess_path, but it could also be some other folder
similarly, dlist is generally self.dlist, but it could also be someother list of folders
"""
for folder in dlist:
dpath = os.path.join(ver_path, folder)
#Ensure folder exists
assert (
os.path.exists(dpath)
), f"B_VER Error: Folder {folder} cannot be found in {ver_path}"
csvpath = os.path.join(dpath, 'data.csv')
#Check for data.csv
assert (os.path.isfile(csvpath)
), f"B_VER Error: data.csv could not be found in {dpath}"
#Verify 4 columns in csv
df = pd.read_csv(csvpath)
assert (
len(df.columns) == 4
), f"B_VER Error: Need four columns in data.csv in {csvpath}"
#The number of jpg files in this directory should equal num rows
num_jpgs = len(glob.glob1(dpath, "*.jpg"))
assert (
len(df) == num_jpgs
), F"B_VER Error: num_jpgs in {dpath} must = num_rows in data.csv"
#Index the first 20 and last 20, and ensure that those images exist
for i in range(20):
img_name = df.iloc[i, 0]
framepath = os.path.join(dpath, img_name)
frame = cv2.imread(framepath)
assert (
os.path.isfile(framepath)
), F"B_VER Error: frame {framepath} is not a path, but is in the csv"
for i in range(20):
img_name = df.iloc[-i, 0]
framepath = os.path.join(dpath, img_name)
frame = cv2.imread(framepath)
assert (
os.path.isfile(framepath)
), F"B_VER Error: frame {framepath} is not a path, but is in the csv"
def default_vis(self, curr_step):
#visualize data in tensorboard
for i, folder in enumerate(self.dlist):
self.visualizer.standard_log(self.sess_path,
folder,
self.curr_step_idx,
global_step=i,
units=curr_step.get("units", 'rad'))
def exec_init(self, curr_step):
"""
Initializes self.sess_path with filtered data
curr_step: dict, as in steps.session
"""
assert (
self.curr_step_idx == 0 and self.dlist is None
), "Step Error: init instruction can only be called once at the start"
#verify raw data & dlist
self.dlist = curr_step["dlist"]
raw_datadir = os.path.join(self.params_dict["abs_path"],
self.params_dict["raw_data"])
self.B_VER(raw_datadir, self.dlist)
print(f"PASSED B_VER FOR STEP {self.curr_step_idx}")
dest_datadir = self.sess_path
filter_funclist = curr_step["funclist"]
#move & filter each folder
new_dlist = []
for i, folder in enumerate(self.dlist):
new_folder = self.data_utils.MOVE(
raw_datadir,
folder,
dest_datadir,
flist=filter_funclist[i],
preview=self.params_dict["preview"])
new_dlist.append(new_folder)
#visualize data in tensorboard
self.dlist = new_dlist
self.default_vis(curr_step)
def exec_preprocess(self, curr_step):
"""
Preprocesses data as per curr_step in steps.session
"""
assert (self.curr_step_idx > 0 and self.dlist
is not None), "Step Error: Must call init before preprocess"
#verify raw data & dlist
self.B_VER(self.sess_path, self.dlist)
#move & preprocess each folder
funclist = curr_step["funclist"]
raw_datadir = self.sess_path
dest_datadir = self.sess_path
new_dlist = []
for i, folder in enumerate(self.dlist):
flist = funclist[i]
new_folder = self.data_utils.MOVE(raw_datadir,
folder,
dest_datadir,
flist=flist,
preview=False)
new_dlist.append(new_folder)
self.dlist = new_dlist
self.default_vis(curr_step)
def exec_augment(self, curr_step):
"""
Augment data as per curr_step in steps.json
"""
assert (self.curr_step_idx > 0 and self.dlist
is not None), "Step Error: Must call init before augment"
#verify raw data & dlist
self.B_VER(self.sess_path, self.dlist)
#move & preprocess each folder
funclist = curr_step["funclist"]
raw_datadir = self.sess_path
dest_datadir = self.sess_path
for i, folder in enumerate(self.dlist):
flist = funclist[i]
self.data_utils.MOVE(raw_datadir,
folder,
dest_datadir,
flist=flist,
preview=False,
op='aug')
self.default_vis(curr_step)
def exec_combine(self, curr_step):
"""
Combines files currently in dlist into a specified folder
updates dlist to only have this folder
"""
assert (self.curr_step_idx > 0 and self.dlist
is not None), "Step Error: Must call init before combine"
#verify raw data & dlist
self.B_VER(self.sess_path, self.dlist)
#move & preprocess each folder
raw_datadir = self.sess_path
dest_datadir = self.sess_path
foldername = curr_step.get("foldername", "main")
for i, folder in enumerate(self.dlist):
self.data_utils.MOVE(raw_datadir,
folder,
dest_datadir,
flist=[],
preview=False,
op='combine_choosename|' + foldername)
self.dlist = [foldername]
self.default_vis(curr_step)
def step(self):
"""
Executes the instruction for the curr_step
"""
curr_step = self.steplist[self.curr_step_idx] #dictionary
insn_type = curr_step["type"]
#case over insn_types
if insn_type == "init":
self.exec_init(curr_step)
self.curr_step_idx += 1
elif insn_type == "preprocess":
self.exec_preprocess(curr_step)
self.curr_step_idx += 1
elif insn_type == "augment":
self.exec_augment(curr_step)
self.curr_step_idx += 1
elif insn_type == "combine":
self.exec_combine(curr_step)
self.curr_step_idx += 1
class Metric_Visualizer(object):
"""
Visualize metrics in Tensorboard
"""
def __init__(self, sess_path=None, writer=None):
"""
sess_path: current working dir of this session
writer: Tensorboard SummmaryWriter
"""
self.gvis = lambda param: session["visualizer"].get(param)
self.fixangle = lambda angle, units: angle if units == 'rad' else angle * math.pi / 180.0
self.writer = writer
self.data_utils = Data_Utils()
def vis_steer_point(self,
frame,
angle,
cx,
cy,
r,
size=10,
color=(0, 0, 0)):
"""
Tiny point on big steering graphic that shows the steering angle
"""
x = (cx + r * math.cos(-1.0 * angle + math.pi / 2.))
y = (cy - r * math.sin(-1.0 * angle + math.pi / 2.))
cv2.circle(frame, (int(x), int(y)), size, color, -1)
def vis_textdata(self, frame, scalar, label, pos):
"""
Visualize text data. pos = vertical interval on the frame
"""
scalar = float(scalar) if float(scalar) != -0.0 else 0.0
#TEXT PARAMETERS
color = (255, 255, 255)
font = cv2.FONT_HERSHEY_SIMPLEX
font_size = 0.5
font_thickness = 1
v_space = 20
cx, cy = 20, pos * v_space
#draw text on frame
text = label + '%.3f' % (float(scalar))
cv2.putText(frame, text, (cx, cy), font, font_size, color,
font_thickness)
def vis_frame(self,
frame,
angle,
speed,
timestamp,
pred=None,
show_steer=False):
"""
Vis an image w/ text info log + steering_angle_graphic & display it
"""
#log text data in the top left
self.vis_textdata(frame, angle, "angle:", 1)
self.vis_textdata(frame, speed, "speed:", 2)
self.vis_textdata(frame, timestamp, "time:", 3)
if pred is not None:
self.vis_textdata(frame, timestamp, "pred", 4)
if show_steer:
#Big Steering Graphic
h, w, c = frame.shape
cx, cy, r = int(w / 2), h, int((h * 80) / 480)
cv2.circle(frame, (cx, cy), r, (255, 255, 255), 2)
big_steerpoint = int(math.ceil(r * 10.0 / 80.0))
angle_extra = angle * 2
if pred is not None:
pred_extra = pred * 2
#SMALL steering point graphic (angle must be in radians)
self.vis_steer_point(frame,
angle_extra,
cx,
cy,
r,
size=big_steerpoint,
color=(218, 165, 32))
if pred is not None:
self.vis_steer_point(frame,
pred_extra,
cx,
cy,
r,
size=int(math.ceil(big_steerpoint / 2.0)),
color=(0, 0, 0))
def vis_framelist(self,
labelname,
framelist,
angle_list,
global_step=0,
show_steer=False,
vel_list=None,
predangle_list=None,
timestamp_list=None):
"""
Visualize a list of frames (cv2) w/ angles (radians, floats)
"""
vislist = []
tryget = lambda arr, index, default: default if arr is None else arr[
index]
for i, frame in enumerate(framelist):
curr_timestamp = tryget(timestamp_list, i, 0)
curr_predangle = tryget(predangle_list, i, None)
curr_vel = tryget(vel_list, i, 0)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
self.vis_frame(frame,
angle_list[i],
curr_vel,
curr_timestamp,
curr_predangle,
show_steer=True)
frame = cv2.bitwise_not(frame)
vislist.append(frame)
self.writer.add_images(labelname,
vislist,
global_step=global_step,
dataformats='HWC')
def framelist_from_path(self,
dpath,
stepname,
idx,
show_steer=False,
units='rad'):
"""
Send a list of frames to Tensorboard from a path
"""
interesting_idxs = self.data_utils.get_interesting_idxs(dpath, 5)
df = self.data_utils.get_df(dpath)
datalist = list(
map(lambda x: self.data_utils.df_data_fromidx(dpath, df, x),
interesting_idxs)) #list of image, row pairs
framelist = list(map(lambda x: x[0], datalist))
rowlist = list(map(lambda x: x[1], datalist))
splitrow = lambda idx: list(map(lambda x: x[idx], rowlist))
angle_list = splitrow(1)
angle_list = list(map(lambda x: self.fixangle(x, units), angle_list))
vel_list = splitrow(2)
timestamp_list = splitrow(3)
self.vis_framelist(stepname,
framelist,
angle_list,
global_step=idx,
show_steer=show_steer,
vel_list=vel_list,
timestamp_list=timestamp_list)
def vid_from_path(self,
dpath,
stepname,
idx,
show_steer=False,
units='rad'):
"""
Send annotated video to Tensorboard
dpath: abs_path to data folder containing images & csv
labelname: str name for the label
global_step: global_step to record for slider functionality
"""
framebuffer = []
#get dataframe
csvpath = os.path.join(dpath, "data.csv")
df = pd.read_csv(csvpath)
# num_rows = int(0.1 * len(df)) #display about 10% of the frames
num_rows = len(df)
for i in range(num_rows):
if i % 4 == 0:
img_name, angle, speed, timestamp = df.iloc[i, 0], df.iloc[
i, 1], df.iloc[i, 2], df.iloc[i, 3]
angle = self.fixangle(angle, units)
framepath = os.path.join(dpath, img_name)
frame = cv2.imread(framepath)
self.vis_frame(frame,
angle,
speed,
timestamp,
show_steer=show_steer)
framebuffer.append(frame.copy())
self.writer.add_video(stepname,
framebuffer,
fps=10,
global_step=idx,
as_np_framebuffer=True)
def plot_anglehist(self, dpath, tag, idx):
csvpath = os.path.join(dpath, "data.csv")
df = pd.read_csv(csvpath)
angle_column = df.iloc[:, 1].values
num_bins = 100
#save plot w/ matplotlib
fig = plt.figure()
plt.hist(angle_column, num_bins, color='green')
self.writer.add_figure(tag, fig, global_step=idx)
def text_table(self,
dpath,
labelname,
foldername='',
angle_unit='',
global_step=0):
df = self.data_utils.get_df(dpath)
h, w = self.data_utils._get_image_size(dpath)
text = f"Folder | Shape | Units | Num Images\n-----|-----|-----|-----\n{foldername}|({h}, {w})|{angle_unit}|{len(df)}"
self.writer.add_text(labelname, text, global_step=global_step)
def visualize_batch(self,
ts_imgbatch,
ts_anglebatch,
ts_predanglebatch,
global_step=0):
self.writer.add_images("Sample Batch",
ts_imgbatch[:5],
global_step=global_step)
def dict_to_table(self, my_dict):
"""
Converts a python dict into a table
TODO: Consider the numerous edge cases (what if dict is already array)
"""
tabulate_dict = {}
for key in my_dict:
tabulate_dict[key] = [str(my_dict[key])]
text = tabulate(tabulate_dict, headers="keys", tablefmt="github")
return text
def log_training(self, config, train_id, best_train_loss, best_valid_loss):
final_dict = config
final_dict["train_id"] = train_id
final_dict["t_loss"] = best_train_loss
final_dict["v_loss"] = best_valid_loss
text = self.dict_to_table(final_dict)
self.writer.add_text('Train Summary', text, global_step=0)
def standard_log(self,
datadir,
folder,
curr_step,
global_step=0,
units=''):
"""
Log "Standard" things in Tensorboard
datadir: abs_path of directory containing data folders
folder: data folder name
curr_step: progress in steps.json
global_step: The "step" value to log into tensorboard (this allows for the cool slider functionality)
units: 'rad' or 'deg'
"""
labelname = f"STEP-{curr_step}"
dpath = os.path.join(datadir, folder)
self.plot_anglehist(dpath, labelname, global_step)
if self.gvis("vis_type") == 'video':
self.vid_from_path(dpath,
labelname,
global_step,
show_steer=True,
units=units)
elif self.gvis("vis_type") == 'framelist':
self.framelist_from_path(dpath,
labelname,
global_step,
show_steer=True,
units=units)
self.text_table(dpath,
labelname,
foldername=folder,
angle_unit=units,
global_step=global_step)