def runme_p(imgs, params):
# get image info
cars, notcars, image_type, image_shape = imgs
# extract parameters from dictionary
color_space = params['color_space']
spatial_size = params['spatial_size']
hist_bins = params['hist_bins']
orient = params['orient']
pix_per_cell = params['pix_per_cell']
cell_per_block = params['cell_per_block']
hog_channel = params['hog_channel']
spatial_feat = params['spatial_feat']
hist_feat = params['hist_feat']
hog_feat = params['hog_feat']
car_features = ro.extract_features(cars, color_space=color_space,
spatial_size=spatial_size, hist_bins=hist_bins,
orient=orient, pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block,
hog_channel=hog_channel, spatial_feat=spatial_feat,
hist_feat=hist_feat, hog_feat=hog_feat)
notcar_features = ro.extract_features(notcars, color_space=color_space,
spatial_size=spatial_size, hist_bins=hist_bins,
orient=orient, pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block,
hog_channel=hog_channel, spatial_feat=spatial_feat,
hist_feat=hist_feat, hog_feat=hog_feat)
return (car_features, notcar_features)
def process_video(video_file, clf, scaler, params):
'''
create bounding box for each frame in a video file
'''
video_clip = VideoFileClip(video_file)
qsize = 6
heatmap_threshold = 3
old_frames = Queue()
new_frames = Queue()
image_sequence = []
for i, f, in enumerate(video_clip.iter_frames()):
print ('processing frame ', i)
fig = plot_bounding_box(f, clf, scaler, params)
if old_frames.qsize() < qsize:
old_frames.put(fig)
else:
average_frame, old_frames = get_frame_aggregate(old_frames)
_ = old_frames.get()
old_frames.put(fig)
heatmap = ro.apply_threshold(average_frame, heatmap_threshold)
labels = label(heatmap)
print(labels[1], 'cars found')
draw_img = ro.draw_labeled_bboxes(np.copy(f), labels)
image_sequence.append(draw_img)
#plt.figure()
#plt.imshow(draw_img)
#plt.show()
clip = ImageSequenceClip(image_sequence, fps=video_clip.fps)
clip.write_videofile("drive_n_%d_t_%d.mp4" %(qsize, heatmap_threshold), audio=False)
def get_psp(molecule):
"""
Check if the all the atoms of molecule have the nlcc of the type nlss_aw and nlcc_ss.
If it is true the corresponding pseudo is added to the list of psp to be performed, otherwise
only hgh_k is included.
"""
from os import path as p
import Routines as R
possible_psp=['nlcc_aw','nlcc_ss']
to_do=[True,True]
for at in R.get_atoms(molecule):
psp_available = [p.isfile(p.join('psp_'+psp,'psppar.'+at)) for psp in possible_psp]
to_do = [ a and b for a,b in zip(to_do,psp_available)]
return ['hgh_k'] + [psp for yes,psp in zip(to_do,possible_psp) if yes]
def start_routine(self,name):
self.in_routine = True
if name == "seesaw":
self.routine = Routines.Seesaw(self)
if name == "test":
self.routine = Routines.Test(self)
if name == "straight_walls":
self.routine = Routines.StraightWalls(self)
if name == "release_bolt":
self.routine = Routines.ReleaseBolt(self)
if name == "follow_wall_left":
self.routine = Routines.FollowWallLeft(self)
if name == "follow_wall_right":
self.routine = Routines.FollowWallRight(self)
self.routine.start()
def runme():
### TODO: Tweak these parameters and see how the results change.
color_space = 'YCrCb' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 9 # HOG orientations
pix_per_cell = 8 # HOG pixels per cell
cell_per_block = 2 # HOG cells per block
hog_channel = "ALL" # Can be 0, 1, 2, or "ALL"
spatial_size = (12, 12) # Spatial binning dimensions
hist_bins = 20 # Number of histogram bins
spatial_feat = True # Spatial features on or off
hist_feat = True # Histogram features on or off
hog_feat = True # HOG features on or off
y_start_stop = [None, None] # Min and max in y to search in slide_window()
cars, notcars, image_type, image_shape = read_images()
#cars, notcars = sample(cars, notcars)
car_features = ro.extract_features(cars, color_space=color_space,
spatial_size=spatial_size, hist_bins=hist_bins,
orient=orient, pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block,
hog_channel=hog_channel, spatial_feat=spatial_feat,
hist_feat=hist_feat, hog_feat=hog_feat)
notcar_features = ro.extract_features(notcars, color_space=color_space,
spatial_size=spatial_size, hist_bins=hist_bins,
orient=orient, pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block,
hog_channel=hog_channel, spatial_feat=spatial_feat,
hist_feat=hist_feat, hog_feat=hog_feat)
X = np.vstack((car_features, notcar_features)).astype(np.float64)
# Fit a per-column scaler
X_scaler = StandardScaler().fit(X)
# Apply the scaler to X
scaled_X = X_scaler.transform(X)
# Define the labels vector
y = np.hstack((np.ones(len(car_features)), np.zeros(len(notcar_features))))
# Split up data into randomized training and test sets
rand_state = np.random.randint(0, 100)
X_train, X_test, y_train, y_test = train_test_split(
scaled_X, y, test_size=0.2, random_state=rand_state)
print('Using:',orient,'orientations',pix_per_cell,
'pixels per cell and', cell_per_block,'cells per block')
print('Feature vector length:', len(X_train[0]))
# Use a linear SVC
svc = LinearSVC()
# Check the training time for the SVC
t=time.time()
svc.fit(X_train, y_train)
t2 = time.time()
print(round(t2-t, 2), 'Seconds to train SVC...')
# Check the score of the SVC
print('Test Accuracy of SVC = ', round(svc.score(X_test, y_test), 4))
# Check the prediction time for a single sample
t=time.time()
image = mpimg.imread('bbox-example-image.jpg')
draw_image = np.copy(image)
print ('shape of test image: ', image.shape)
imy, imx = image.shape[:2]
# Uncomment the following line if you extracted training
# data from .png images (scaled 0 to 1 by mpimg) and the
# image you are searching is a .jpg (scaled 0 to 255)
#image = image.astype(np.float32)/255
# multi box search
min_fraction = 6
max_fraction = 6
aspect_ratio = 1.0
hot_windows = []
start_scan = int(imy/2)
for f in range(min_fraction, max_fraction+1):
y_start_stop = [start_scan, min(start_scan + (f - 1) * int(imy/f), imy)] # Min and max in y to search in slide_window()
xy_window = (int(imy/f), int(imy/f*aspect_ratio))
windows = ro.slide_window(image, x_start_stop=[None, None], y_start_stop=y_start_stop,
xy_window=xy_window, xy_overlap=(0.5, 0.5))
print (f, len(windows))
new_hot_windows = ro.search_windows(image, windows, svc, X_scaler, image_shape, color_space=color_space,
spatial_size=spatial_size, hist_bins=hist_bins,
orient=orient, pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block,
hog_channel=hog_channel, spatial_feat=spatial_feat,
hist_feat=hist_feat, hog_feat=hog_feat)
print (f, y_start_stop, xy_window, len(new_hot_windows)) #, new_hot_windows)
if len(new_hot_windows) > 0:
hot_windows = hot_windows + new_hot_windows
print ('size of hot_windows before plotting: ', len(hot_windows))
window_img = ro.draw_boxes(draw_image, hot_windows, color=(0, 0, 255), thick=3)
plt.imshow(window_img)
fig1 = plt.gcf()
plt.show()
plt.draw()
fig1.savefig('tessstttyyy.jpg', dpi=100)
def plot_bounding_box_old(clf, scaler, params, index, image_type, image_shape):
'''
given a new image, plot a bounding box
around all the cars detected by the
classification algorithm.
'''
print ('image_shape = ', image_shape)
image = mpimg.imread('bbox-example-image.jpg')
image = mpimg.imread('test/frame300.jpg')
draw_image = np.copy(image)
imy, imx = image.shape[:2]
print ('in old: ', image_type, image.shape)
# Uncomment the following line if you extracted training
# data from .png images (scaled 0 to 1 by mpimg) and the
# image you are searching is a .jpg (scaled 0 to 255)
if image_type == 'png':
image = image.astype(np.float32)/255
# extract parameters from dictionary
color_space = params['color_space']
spatial_size = params['spatial_size']
hist_bins = params['hist_bins']
orient = params['orient']
pix_per_cell = params['pix_per_cell']
cell_per_block = params['cell_per_block']
hog_channel = params['hog_channel']
spatial_feat = params['spatial_feat']
hist_feat = params['hist_feat']
hog_feat = params['hog_feat']
# multi box search
min_fraction = 6
max_fraction = 10
step_fraction = 2
aspect_ratio = 1.0
hot_windows = []
start_scan = int(imy/2)
for f in range(min_fraction, max_fraction+1, step_fraction):
y_start_stop = [start_scan, min(start_scan + (f - 1) * int(imy/f), imy)] # Min and max in y to search in slide_window()
xy_window = (int(imy/f), int(imy/f*aspect_ratio))
windows = ro.slide_window(image, x_start_stop=[None, None], y_start_stop=y_start_stop,
xy_window=xy_window, xy_overlap=(0.5, 0.5))
print (f, len(windows))
new_hot_windows = ro.search_windows(image, windows, clf, scaler, image_shape, color_space=color_space,
spatial_size=spatial_size, hist_bins=hist_bins,
orient=orient, pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block,
hog_channel=hog_channel, spatial_feat=spatial_feat,
hist_feat=hist_feat, hog_feat=hog_feat)
print (f, y_start_stop, xy_window, len(new_hot_windows)) #, new_hot_windows)
if len(new_hot_windows) > 0:
hot_windows = hot_windows + new_hot_windows
print ('size of hot_windows before plotting: ', len(hot_windows))
window_img = ro.draw_boxes(draw_image, hot_windows, color=(0, 0, 255), thick=3)
'''
plt.show()
'''
plt.imshow(window_img)
fig1 = plt.gcf()
plt.draw()
fig1.savefig('new_fig%d.jpg' %index, dpi=100)
heatmap = np.zeros([draw_image.shape[0], draw_image.shape[1]])
heatmap = ro.add_heat(heatmap, hot_windows)
plt.figure()
plt.imshow(heatmap, cmap='gray')
heatmap = ro.apply_threshold(heatmap, 1)
labels = label(heatmap)
print(labels[1], 'cars found')
plt.figure()
plt.imshow(labels[0], cmap='gray')
draw_img = ro.draw_labeled_bboxes(np.copy(draw_image), labels)
# Display the image
plt.figure()
plt.imshow(draw_img)
plt.show()
def plot_bounding_box(image, clf, scaler, params, plot_box=False, plot_heat=False):
'''
given a new image, plot a bounding box
around all the cars detected by the
classification algorithm.
'''
if type(image) is str:
image_type = image.split('.')[-1]
image = mpimg.imread(image)
elif type(image) is np.ndarray:
if image.max() > 1:
image_type = 'jpg'
else:
image_type = 'png'
else:
print ("input not recognized")
return None
draw_image = np.copy(image)
imy, imx = image.shape[:2]
if (image_type=='jpeg' or image_type=='jpg'): # and params['training_image_type']=='png':
image = image.astype(np.float32)/255
# extract parameters from dictionary
color_space = params['color_space']
spatial_size = params['spatial_size']
hist_bins = params['hist_bins']
orient = params['orient']
pix_per_cell = params['pix_per_cell']
cell_per_block = params['cell_per_block']
hog_channel = params['hog_channel']
spatial_feat = params['spatial_feat']
hist_feat = params['hist_feat']
hog_feat = params['hog_feat']
training_image_shape = params['training_image_shape']
# multi box search
min_fraction = 6
max_fraction = 10
step_fraction = 2
aspect_ratio = 1.0
hot_windows = []
start_scan = int(imy/2)
for f in range(min_fraction, max_fraction+1, step_fraction):
y_start_stop = [start_scan, min(start_scan + (f - 1) * int(imy/f), imy)] # Min and max in y to search in slide_window()
xy_window = (int(imy/f), int(imy/f*aspect_ratio))
windows = ro.slide_window(image, x_start_stop=[None, None], y_start_stop=y_start_stop,
xy_window=xy_window, xy_overlap=(0.5, 0.5))
#print (f, len(windows))
new_hot_windows = ro.search_windows(image, windows, clf, scaler, training_image_shape, color_space=color_space,
spatial_size=spatial_size, hist_bins=hist_bins,
orient=orient, pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block,
hog_channel=hog_channel, spatial_feat=spatial_feat,
hist_feat=hist_feat, hog_feat=hog_feat)
#print (f, y_start_stop, xy_window, len(new_hot_windows)) #, new_hot_windows)
if len(new_hot_windows) > 0:
hot_windows = hot_windows + new_hot_windows
window_img = ro.draw_boxes(draw_image, hot_windows, color=(0, 0, 255), thick=3)
heatmap = np.zeros([draw_image.shape[0], draw_image.shape[1]])
heatmap = ro.add_heat(heatmap, hot_windows)
if plot_box:
if plot_heat:
plt.figure()
plt.subplot(121)
plt.imshow(window_img)
plt.subplot(122)
plt.imshow(heatmap)
plt.show()
else:
plt.figure()
plt.imshow(window_img)
return heatmap
def launch_python_post():
curDir = os.path.dirname(os.path.abspath(__file__))
logger = PyPostTools.pyPostLogger()
logger.write("Initializing WRF Python Post-Processing Program")
#Step 1: Load program settings
logger.write(" 1. Application Initalization")
logger.write(" - Loading control file, python_post_control.txt")
_pySet = PyPostSettings.PyPostSettings()
logger.write(" - Success!")
logger.write(" - Testing Environmental Variables")
try:
dask_nodes = os.environ["PYTHON_POST_NODES"]
dask_threads = os.environ["PYTHON_POST_THREADS"]
postDir = os.environ["PYTHON_POST_DIR"]
targetDir = os.environ["PYTHON_POST_TARG_DIR"]
except KeyError:
logger.write("***FAIL*** KeyError encountered while trying to access important environmental variables, abort.")
sys.exit("")
logger.write(" - Success!")
logger.write(" - Initializing Dask (" + str(dask_nodes) + " Nodes Requested), Collecting routines needed")
_routines = Routines.Routines()
logger.write(" - Async IO Loop initialized...")
def f(scheduler_port):
async def g(port):
s = Scheduler(port=scheduler_port)
await s
await s.finished()
asyncio.get_event_loop().run_until_complete(g(scheduler_port))
# Starts the scheduler in its own process - needed as otherwise it will
# occupy the program and make it do an infinite loop
process = Process(target=f, args=(scheduler_port,))
process.start()
logger.write(" - Dask Scheduler initialized (Port " + str(scheduler_port) + ")...")
try:
dask_client = Client("tcp://" + socket.gethostname() + ":" + str(scheduler_port), timeout=30)
except OSError:
logger.write(" <-> Dask Client could not be created, timeout error.")
process.terminate()
sys.exit()
logger.write(" - Dask Client initialized...")
logger.write(" - Writing Dask Worker Job Files...")
with PyPostTools.cd(targetDir):
writeFile = PyPostTools.write_job_file(socket.gethostname(), scheduler_port, project_name="climate_severe", queue="debug-cache-quad", nodes=dask_nodes, wall_time=60, nProcs=1)
if(writeFile == False):
dask_client.close()
logger.write(" - Failed to write job file, are you missing an important parameter?")
sys.exit("")
return
else:
logger.write(" - Dask Worker Job File Written, Submitting to Queue.")
PyPostTools.popen("chmod +x dask-worker.job")
PyPostTools.popen("qsub dask-worker.job")
# Wait here for workers.
logger.write(" -> Worker Job submitted to queue, waiting for workers...")
while len(dask_client.scheduler_info()['workers']) < int(dask_nodes):
time.sleep(2)
logger.write(" -> Workers are now connected.")
logger.write(" - Success!")
logger.write(" 1. Done.")
logger.write(" 2. Start Post-Processing Calculations")
start_calculations(dask_client, _routines, dask_threads, process)
logger.write(" 2. Done.")
logger.write(" 3. Generating Figures")
logger.write(" - Collecting files from target directory (" + targetDir + ").")
fList3 = sorted(glob.glob(targetDir + "WRFPRS_F*"))
logger.write(" - " + str(len(fList3)) + " files have been found.")
logger.write(" -> Pushing run_plotting_routines() to dask.")
fullDict = _pySet.get_full_dict()
plotting_future = start_plotting(dask_client, fullDict, dask_threads, process)
wait(plotting_future)
result_plot = dask_client.gather(plotting_future)[0]
if(result_plot != 0):
logger.write("***FAIL*** An error occured in plotting method, check worker logs for more info.")
logger.close()
sys.exit("")
logger.write(" 3. Done.")
logger.write(" 4. Final Steps")
logger.write(" 4. Done, Closing Dask Client.")
# Close the client object
dask_client.retire_workers(workers=dask_client.scheduler_info()['workers'], close=True)
dask_client.close()
logger.write("All Steps Completed.")
logger.write("***SUCCESS*** Program execution complete.")
logger.close()
del dask_client
process.terminate()
calc = []
for arg in sys.argv[1:]:
calc += subset[int(arg)]
print 'Compute molecules'
print calc
reduced_study_set = [('lda_pt','hgh_k'),('lda_pw','hgh_k'),('pbe','hgh_k'),('pbe','nlcc_aw'),\
('pbe','nlcc_ss'),('pbe0','hgh_k')]
# mpi and omp are set from above exporting the asscoiated variables
code = C.SystemCalculator(skip=True, verbose=True)
for mol in calc:
data = nsp_dataset[mol]
data['results'] = {}
for study in data['study']:
if study in reduced_study_set:
data['results'][study] = {}
data['results'][study] = R.nsp_workflow(alpha_conv=1.0e-3,
term_verb=True,
molecule=mol,
study=study,
code=code,
data_folder='Data')
print ''
# Save the dataset as yaml file
#with open('nsp_results_tol=1em3.yaml', 'w') as outfile:
# yaml.dump(nsp_dataset, outfile, default_flow_style=False)
def launch_python_post():
curDir = os.path.dirname(os.path.abspath(__file__))
logger = PyPostTools.pyPostLogger()
logger.write("Initializing WRF Python Post-Processing Program")
#Step 1: Load program settings
logger.write(" 1. Application Initalization")
logger.write(" - Loading control file, python_post_control.txt")
_pySet = PyPostSettings.PyPostSettings()
logger.write(" - Success!")
logger.write(" - Testing Environmental Variables")
try:
dask_nodes = os.environ["PYTHON_POST_NODES"]
dask_threads = os.environ["PYTHON_POST_THREADS"]
postDir = os.environ["PYTHON_POST_DIR"]
targetDir = os.environ["PYTHON_POST_TARG_DIR"]
except KeyError:
logger.write(
"***FAIL*** KeyError encountered while trying to access important environmental variables, abort."
)
sys.exit("")
logger.write(" - Success!")
logger.write(" - Initializing Dask (" + str(dask_nodes) +
" Nodes Requested), Collecting routines needed")
_routines = Routines.Routines()
# Start Dask Tasks
#cLoop = IOLoop.current()
#t = Thread(target = cLoop.start, daemon = True)
#t.start()
logger.write(" - Async IO Loop initialized...")
async def f(port):
s = Scheduler(port=scheduler_port)
s = await s
await s.finished()
return 1
asyncio.gather(f(scheduler_port))
#asyncio.get_event_loop().run_until_complete(f(scheduler_port))
logger.write(" - Dask Scheduler initialized (Port " +
str(scheduler_port) + ")...")
dask_client = Client("tcp://" + socket.gethostname() + ":" +
str(scheduler_port))
logger.write(" - Dask Client initialized...")
logger.write(" - Writing Dask Worker Job Files...")
with PyPostTools.cd(targetDir):
writeFile1 = PyPostTools.write_job_file(socket.gethostname(),
scheduler_port,
project_name="Nowcast",
queue="default",
nodes=dask_nodes,
wall_time=60,
nProcs=1)
writeFile2 = PyPostTools.write_worker_file(socket.gethostname(),
scheduler_port,
nProcs=1)
if (writeFile1 == False or writeFile2 == False):
dask_client.close()
logger.write(
" - Failed to write job files, are you missing an important parameter?"
)
sys.exit("")
return
else:
logger.write(
" - Dask Worker Job File Written, Submitting to Queue.")
PyPostTools.popen("chmod +x launch-worker.sh")
PyPostTools.popen("chmod +x dask-worker.job")
PyPostTools.popen("qsub dask-worker.job")
# Wait here for workers.
logger.write(" -> Worker Job submitted to queue, waiting for workers...")
while len(dask_client.scheduler_info()['workers']) < int(dask_nodes):
time.sleep(2)
logger.write(" -> Workers are now connected.")
#logger.write(" - Adding local packages to dask workers")
#dask_client.upload_file("PyPostTools.py")
#dask_client.upload_file("ArrayTools.py")
#dask_client.upload_file("Calculation.py")
#dask_client.upload_file("ColorMaps.py")
#dask_client.upload_file("Conversions.py")
#dask_client.upload_file("Plotting.py")
#dask_client.upload_file("PyPostSettings.py")
#dask_client.upload_file("Routines.py")
logger.write(" - Success!")
logger.write(" 1. Done.")
logger.write(" 2. Start Post-Processing Calculations")
calculation_future = start_calculations(dask_client, _routines,
dask_threads)
if (calculation_future != None):
wait(calculation_future)
result_calc = dask_client.gather(calculation_future)[0]
if (result_calc != 0):
logger.write(
"***FAIL*** An error occured in calculations method, check worker logs for more info."
)
logger.close()
sys.exit("")
logger.write(" 2. Done.")
logger.write(" 3. Generating Figures")
logger.write(" - Collecting files from target directory (" + targetDir +
").")
fList3 = sorted(glob.glob(targetDir + "WRFPRS_F*"))
logger.write(" - " + str(len(fList3)) + " files have been found.")
logger.write(" -> Pushing run_plotting_routines() to dask.")
fullDict = _pySet.get_full_dict()
plotting_future = start_plotting(dask_client, fullDict, dask_threads)
wait(plotting_future)
result_plot = dask_client.gather(plotting_future)[0]
if (result_plot != 0):
logger.write(
"***FAIL*** An error occured in plotting method, check worker logs for more info."
)
logger.close()
sys.exit("")
logger.write(" 3. Done.")
logger.write(" 4. Final Steps")
logger.write(" 4. Done, Closing Dask Client.")
dask_client.retire_workers(workers=dask_client.scheduler_info()['workers'],
close=True)
dask_client.close()
logger.write("All Steps Completed.")
logger.write("***SUCCESS*** Program execution complete.")
logger.close()