def runme_p(imgs, params): # get image info cars, notcars, image_type, image_shape = imgs # extract parameters from dictionary color_space = params['color_space'] spatial_size = params['spatial_size'] hist_bins = params['hist_bins'] orient = params['orient'] pix_per_cell = params['pix_per_cell'] cell_per_block = params['cell_per_block'] hog_channel = params['hog_channel'] spatial_feat = params['spatial_feat'] hist_feat = params['hist_feat'] hog_feat = params['hog_feat'] car_features = ro.extract_features(cars, color_space=color_space, spatial_size=spatial_size, hist_bins=hist_bins, orient=orient, pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, hog_channel=hog_channel, spatial_feat=spatial_feat, hist_feat=hist_feat, hog_feat=hog_feat) notcar_features = ro.extract_features(notcars, color_space=color_space, spatial_size=spatial_size, hist_bins=hist_bins, orient=orient, pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, hog_channel=hog_channel, spatial_feat=spatial_feat, hist_feat=hist_feat, hog_feat=hog_feat) return (car_features, notcar_features)
def process_video(video_file, clf, scaler, params): ''' create bounding box for each frame in a video file ''' video_clip = VideoFileClip(video_file) qsize = 6 heatmap_threshold = 3 old_frames = Queue() new_frames = Queue() image_sequence = [] for i, f, in enumerate(video_clip.iter_frames()): print ('processing frame ', i) fig = plot_bounding_box(f, clf, scaler, params) if old_frames.qsize() < qsize: old_frames.put(fig) else: average_frame, old_frames = get_frame_aggregate(old_frames) _ = old_frames.get() old_frames.put(fig) heatmap = ro.apply_threshold(average_frame, heatmap_threshold) labels = label(heatmap) print(labels[1], 'cars found') draw_img = ro.draw_labeled_bboxes(np.copy(f), labels) image_sequence.append(draw_img) #plt.figure() #plt.imshow(draw_img) #plt.show() clip = ImageSequenceClip(image_sequence, fps=video_clip.fps) clip.write_videofile("drive_n_%d_t_%d.mp4" %(qsize, heatmap_threshold), audio=False)
def get_psp(molecule): """ Check if the all the atoms of molecule have the nlcc of the type nlss_aw and nlcc_ss. If it is true the corresponding pseudo is added to the list of psp to be performed, otherwise only hgh_k is included. """ from os import path as p import Routines as R possible_psp=['nlcc_aw','nlcc_ss'] to_do=[True,True] for at in R.get_atoms(molecule): psp_available = [p.isfile(p.join('psp_'+psp,'psppar.'+at)) for psp in possible_psp] to_do = [ a and b for a,b in zip(to_do,psp_available)] return ['hgh_k'] + [psp for yes,psp in zip(to_do,possible_psp) if yes]
def start_routine(self,name): self.in_routine = True if name == "seesaw": self.routine = Routines.Seesaw(self) if name == "test": self.routine = Routines.Test(self) if name == "straight_walls": self.routine = Routines.StraightWalls(self) if name == "release_bolt": self.routine = Routines.ReleaseBolt(self) if name == "follow_wall_left": self.routine = Routines.FollowWallLeft(self) if name == "follow_wall_right": self.routine = Routines.FollowWallRight(self) self.routine.start()
def runme(): ### TODO: Tweak these parameters and see how the results change. color_space = 'YCrCb' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb orient = 9 # HOG orientations pix_per_cell = 8 # HOG pixels per cell cell_per_block = 2 # HOG cells per block hog_channel = "ALL" # Can be 0, 1, 2, or "ALL" spatial_size = (12, 12) # Spatial binning dimensions hist_bins = 20 # Number of histogram bins spatial_feat = True # Spatial features on or off hist_feat = True # Histogram features on or off hog_feat = True # HOG features on or off y_start_stop = [None, None] # Min and max in y to search in slide_window() cars, notcars, image_type, image_shape = read_images() #cars, notcars = sample(cars, notcars) car_features = ro.extract_features(cars, color_space=color_space, spatial_size=spatial_size, hist_bins=hist_bins, orient=orient, pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, hog_channel=hog_channel, spatial_feat=spatial_feat, hist_feat=hist_feat, hog_feat=hog_feat) notcar_features = ro.extract_features(notcars, color_space=color_space, spatial_size=spatial_size, hist_bins=hist_bins, orient=orient, pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, hog_channel=hog_channel, spatial_feat=spatial_feat, hist_feat=hist_feat, hog_feat=hog_feat) X = np.vstack((car_features, notcar_features)).astype(np.float64) # Fit a per-column scaler X_scaler = StandardScaler().fit(X) # Apply the scaler to X scaled_X = X_scaler.transform(X) # Define the labels vector y = np.hstack((np.ones(len(car_features)), np.zeros(len(notcar_features)))) # Split up data into randomized training and test sets rand_state = np.random.randint(0, 100) X_train, X_test, y_train, y_test = train_test_split( scaled_X, y, test_size=0.2, random_state=rand_state) print('Using:',orient,'orientations',pix_per_cell, 'pixels per cell and', cell_per_block,'cells per block') print('Feature vector length:', len(X_train[0])) # Use a linear SVC svc = LinearSVC() # Check the training time for the SVC t=time.time() svc.fit(X_train, y_train) t2 = time.time() print(round(t2-t, 2), 'Seconds to train SVC...') # Check the score of the SVC print('Test Accuracy of SVC = ', round(svc.score(X_test, y_test), 4)) # Check the prediction time for a single sample t=time.time() image = mpimg.imread('bbox-example-image.jpg') draw_image = np.copy(image) print ('shape of test image: ', image.shape) imy, imx = image.shape[:2] # Uncomment the following line if you extracted training # data from .png images (scaled 0 to 1 by mpimg) and the # image you are searching is a .jpg (scaled 0 to 255) #image = image.astype(np.float32)/255 # multi box search min_fraction = 6 max_fraction = 6 aspect_ratio = 1.0 hot_windows = [] start_scan = int(imy/2) for f in range(min_fraction, max_fraction+1): y_start_stop = [start_scan, min(start_scan + (f - 1) * int(imy/f), imy)] # Min and max in y to search in slide_window() xy_window = (int(imy/f), int(imy/f*aspect_ratio)) windows = ro.slide_window(image, x_start_stop=[None, None], y_start_stop=y_start_stop, xy_window=xy_window, xy_overlap=(0.5, 0.5)) print (f, len(windows)) new_hot_windows = ro.search_windows(image, windows, svc, X_scaler, image_shape, color_space=color_space, spatial_size=spatial_size, hist_bins=hist_bins, orient=orient, pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, hog_channel=hog_channel, spatial_feat=spatial_feat, hist_feat=hist_feat, hog_feat=hog_feat) print (f, y_start_stop, xy_window, len(new_hot_windows)) #, new_hot_windows) if len(new_hot_windows) > 0: hot_windows = hot_windows + new_hot_windows print ('size of hot_windows before plotting: ', len(hot_windows)) window_img = ro.draw_boxes(draw_image, hot_windows, color=(0, 0, 255), thick=3) plt.imshow(window_img) fig1 = plt.gcf() plt.show() plt.draw() fig1.savefig('tessstttyyy.jpg', dpi=100)
def plot_bounding_box_old(clf, scaler, params, index, image_type, image_shape): ''' given a new image, plot a bounding box around all the cars detected by the classification algorithm. ''' print ('image_shape = ', image_shape) image = mpimg.imread('bbox-example-image.jpg') image = mpimg.imread('test/frame300.jpg') draw_image = np.copy(image) imy, imx = image.shape[:2] print ('in old: ', image_type, image.shape) # Uncomment the following line if you extracted training # data from .png images (scaled 0 to 1 by mpimg) and the # image you are searching is a .jpg (scaled 0 to 255) if image_type == 'png': image = image.astype(np.float32)/255 # extract parameters from dictionary color_space = params['color_space'] spatial_size = params['spatial_size'] hist_bins = params['hist_bins'] orient = params['orient'] pix_per_cell = params['pix_per_cell'] cell_per_block = params['cell_per_block'] hog_channel = params['hog_channel'] spatial_feat = params['spatial_feat'] hist_feat = params['hist_feat'] hog_feat = params['hog_feat'] # multi box search min_fraction = 6 max_fraction = 10 step_fraction = 2 aspect_ratio = 1.0 hot_windows = [] start_scan = int(imy/2) for f in range(min_fraction, max_fraction+1, step_fraction): y_start_stop = [start_scan, min(start_scan + (f - 1) * int(imy/f), imy)] # Min and max in y to search in slide_window() xy_window = (int(imy/f), int(imy/f*aspect_ratio)) windows = ro.slide_window(image, x_start_stop=[None, None], y_start_stop=y_start_stop, xy_window=xy_window, xy_overlap=(0.5, 0.5)) print (f, len(windows)) new_hot_windows = ro.search_windows(image, windows, clf, scaler, image_shape, color_space=color_space, spatial_size=spatial_size, hist_bins=hist_bins, orient=orient, pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, hog_channel=hog_channel, spatial_feat=spatial_feat, hist_feat=hist_feat, hog_feat=hog_feat) print (f, y_start_stop, xy_window, len(new_hot_windows)) #, new_hot_windows) if len(new_hot_windows) > 0: hot_windows = hot_windows + new_hot_windows print ('size of hot_windows before plotting: ', len(hot_windows)) window_img = ro.draw_boxes(draw_image, hot_windows, color=(0, 0, 255), thick=3) ''' plt.show() ''' plt.imshow(window_img) fig1 = plt.gcf() plt.draw() fig1.savefig('new_fig%d.jpg' %index, dpi=100) heatmap = np.zeros([draw_image.shape[0], draw_image.shape[1]]) heatmap = ro.add_heat(heatmap, hot_windows) plt.figure() plt.imshow(heatmap, cmap='gray') heatmap = ro.apply_threshold(heatmap, 1) labels = label(heatmap) print(labels[1], 'cars found') plt.figure() plt.imshow(labels[0], cmap='gray') draw_img = ro.draw_labeled_bboxes(np.copy(draw_image), labels) # Display the image plt.figure() plt.imshow(draw_img) plt.show()
def plot_bounding_box(image, clf, scaler, params, plot_box=False, plot_heat=False): ''' given a new image, plot a bounding box around all the cars detected by the classification algorithm. ''' if type(image) is str: image_type = image.split('.')[-1] image = mpimg.imread(image) elif type(image) is np.ndarray: if image.max() > 1: image_type = 'jpg' else: image_type = 'png' else: print ("input not recognized") return None draw_image = np.copy(image) imy, imx = image.shape[:2] if (image_type=='jpeg' or image_type=='jpg'): # and params['training_image_type']=='png': image = image.astype(np.float32)/255 # extract parameters from dictionary color_space = params['color_space'] spatial_size = params['spatial_size'] hist_bins = params['hist_bins'] orient = params['orient'] pix_per_cell = params['pix_per_cell'] cell_per_block = params['cell_per_block'] hog_channel = params['hog_channel'] spatial_feat = params['spatial_feat'] hist_feat = params['hist_feat'] hog_feat = params['hog_feat'] training_image_shape = params['training_image_shape'] # multi box search min_fraction = 6 max_fraction = 10 step_fraction = 2 aspect_ratio = 1.0 hot_windows = [] start_scan = int(imy/2) for f in range(min_fraction, max_fraction+1, step_fraction): y_start_stop = [start_scan, min(start_scan + (f - 1) * int(imy/f), imy)] # Min and max in y to search in slide_window() xy_window = (int(imy/f), int(imy/f*aspect_ratio)) windows = ro.slide_window(image, x_start_stop=[None, None], y_start_stop=y_start_stop, xy_window=xy_window, xy_overlap=(0.5, 0.5)) #print (f, len(windows)) new_hot_windows = ro.search_windows(image, windows, clf, scaler, training_image_shape, color_space=color_space, spatial_size=spatial_size, hist_bins=hist_bins, orient=orient, pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, hog_channel=hog_channel, spatial_feat=spatial_feat, hist_feat=hist_feat, hog_feat=hog_feat) #print (f, y_start_stop, xy_window, len(new_hot_windows)) #, new_hot_windows) if len(new_hot_windows) > 0: hot_windows = hot_windows + new_hot_windows window_img = ro.draw_boxes(draw_image, hot_windows, color=(0, 0, 255), thick=3) heatmap = np.zeros([draw_image.shape[0], draw_image.shape[1]]) heatmap = ro.add_heat(heatmap, hot_windows) if plot_box: if plot_heat: plt.figure() plt.subplot(121) plt.imshow(window_img) plt.subplot(122) plt.imshow(heatmap) plt.show() else: plt.figure() plt.imshow(window_img) return heatmap
def launch_python_post(): curDir = os.path.dirname(os.path.abspath(__file__)) logger = PyPostTools.pyPostLogger() logger.write("Initializing WRF Python Post-Processing Program") #Step 1: Load program settings logger.write(" 1. Application Initalization") logger.write(" - Loading control file, python_post_control.txt") _pySet = PyPostSettings.PyPostSettings() logger.write(" - Success!") logger.write(" - Testing Environmental Variables") try: dask_nodes = os.environ["PYTHON_POST_NODES"] dask_threads = os.environ["PYTHON_POST_THREADS"] postDir = os.environ["PYTHON_POST_DIR"] targetDir = os.environ["PYTHON_POST_TARG_DIR"] except KeyError: logger.write("***FAIL*** KeyError encountered while trying to access important environmental variables, abort.") sys.exit("") logger.write(" - Success!") logger.write(" - Initializing Dask (" + str(dask_nodes) + " Nodes Requested), Collecting routines needed") _routines = Routines.Routines() logger.write(" - Async IO Loop initialized...") def f(scheduler_port): async def g(port): s = Scheduler(port=scheduler_port) await s await s.finished() asyncio.get_event_loop().run_until_complete(g(scheduler_port)) # Starts the scheduler in its own process - needed as otherwise it will # occupy the program and make it do an infinite loop process = Process(target=f, args=(scheduler_port,)) process.start() logger.write(" - Dask Scheduler initialized (Port " + str(scheduler_port) + ")...") try: dask_client = Client("tcp://" + socket.gethostname() + ":" + str(scheduler_port), timeout=30) except OSError: logger.write(" <-> Dask Client could not be created, timeout error.") process.terminate() sys.exit() logger.write(" - Dask Client initialized...") logger.write(" - Writing Dask Worker Job Files...") with PyPostTools.cd(targetDir): writeFile = PyPostTools.write_job_file(socket.gethostname(), scheduler_port, project_name="climate_severe", queue="debug-cache-quad", nodes=dask_nodes, wall_time=60, nProcs=1) if(writeFile == False): dask_client.close() logger.write(" - Failed to write job file, are you missing an important parameter?") sys.exit("") return else: logger.write(" - Dask Worker Job File Written, Submitting to Queue.") PyPostTools.popen("chmod +x dask-worker.job") PyPostTools.popen("qsub dask-worker.job") # Wait here for workers. logger.write(" -> Worker Job submitted to queue, waiting for workers...") while len(dask_client.scheduler_info()['workers']) < int(dask_nodes): time.sleep(2) logger.write(" -> Workers are now connected.") logger.write(" - Success!") logger.write(" 1. Done.") logger.write(" 2. Start Post-Processing Calculations") start_calculations(dask_client, _routines, dask_threads, process) logger.write(" 2. Done.") logger.write(" 3. Generating Figures") logger.write(" - Collecting files from target directory (" + targetDir + ").") fList3 = sorted(glob.glob(targetDir + "WRFPRS_F*")) logger.write(" - " + str(len(fList3)) + " files have been found.") logger.write(" -> Pushing run_plotting_routines() to dask.") fullDict = _pySet.get_full_dict() plotting_future = start_plotting(dask_client, fullDict, dask_threads, process) wait(plotting_future) result_plot = dask_client.gather(plotting_future)[0] if(result_plot != 0): logger.write("***FAIL*** An error occured in plotting method, check worker logs for more info.") logger.close() sys.exit("") logger.write(" 3. Done.") logger.write(" 4. Final Steps") logger.write(" 4. Done, Closing Dask Client.") # Close the client object dask_client.retire_workers(workers=dask_client.scheduler_info()['workers'], close=True) dask_client.close() logger.write("All Steps Completed.") logger.write("***SUCCESS*** Program execution complete.") logger.close() del dask_client process.terminate()
calc = [] for arg in sys.argv[1:]: calc += subset[int(arg)] print 'Compute molecules' print calc reduced_study_set = [('lda_pt','hgh_k'),('lda_pw','hgh_k'),('pbe','hgh_k'),('pbe','nlcc_aw'),\ ('pbe','nlcc_ss'),('pbe0','hgh_k')] # mpi and omp are set from above exporting the asscoiated variables code = C.SystemCalculator(skip=True, verbose=True) for mol in calc: data = nsp_dataset[mol] data['results'] = {} for study in data['study']: if study in reduced_study_set: data['results'][study] = {} data['results'][study] = R.nsp_workflow(alpha_conv=1.0e-3, term_verb=True, molecule=mol, study=study, code=code, data_folder='Data') print '' # Save the dataset as yaml file #with open('nsp_results_tol=1em3.yaml', 'w') as outfile: # yaml.dump(nsp_dataset, outfile, default_flow_style=False)
def launch_python_post(): curDir = os.path.dirname(os.path.abspath(__file__)) logger = PyPostTools.pyPostLogger() logger.write("Initializing WRF Python Post-Processing Program") #Step 1: Load program settings logger.write(" 1. Application Initalization") logger.write(" - Loading control file, python_post_control.txt") _pySet = PyPostSettings.PyPostSettings() logger.write(" - Success!") logger.write(" - Testing Environmental Variables") try: dask_nodes = os.environ["PYTHON_POST_NODES"] dask_threads = os.environ["PYTHON_POST_THREADS"] postDir = os.environ["PYTHON_POST_DIR"] targetDir = os.environ["PYTHON_POST_TARG_DIR"] except KeyError: logger.write( "***FAIL*** KeyError encountered while trying to access important environmental variables, abort." ) sys.exit("") logger.write(" - Success!") logger.write(" - Initializing Dask (" + str(dask_nodes) + " Nodes Requested), Collecting routines needed") _routines = Routines.Routines() # Start Dask Tasks #cLoop = IOLoop.current() #t = Thread(target = cLoop.start, daemon = True) #t.start() logger.write(" - Async IO Loop initialized...") async def f(port): s = Scheduler(port=scheduler_port) s = await s await s.finished() return 1 asyncio.gather(f(scheduler_port)) #asyncio.get_event_loop().run_until_complete(f(scheduler_port)) logger.write(" - Dask Scheduler initialized (Port " + str(scheduler_port) + ")...") dask_client = Client("tcp://" + socket.gethostname() + ":" + str(scheduler_port)) logger.write(" - Dask Client initialized...") logger.write(" - Writing Dask Worker Job Files...") with PyPostTools.cd(targetDir): writeFile1 = PyPostTools.write_job_file(socket.gethostname(), scheduler_port, project_name="Nowcast", queue="default", nodes=dask_nodes, wall_time=60, nProcs=1) writeFile2 = PyPostTools.write_worker_file(socket.gethostname(), scheduler_port, nProcs=1) if (writeFile1 == False or writeFile2 == False): dask_client.close() logger.write( " - Failed to write job files, are you missing an important parameter?" ) sys.exit("") return else: logger.write( " - Dask Worker Job File Written, Submitting to Queue.") PyPostTools.popen("chmod +x launch-worker.sh") PyPostTools.popen("chmod +x dask-worker.job") PyPostTools.popen("qsub dask-worker.job") # Wait here for workers. logger.write(" -> Worker Job submitted to queue, waiting for workers...") while len(dask_client.scheduler_info()['workers']) < int(dask_nodes): time.sleep(2) logger.write(" -> Workers are now connected.") #logger.write(" - Adding local packages to dask workers") #dask_client.upload_file("PyPostTools.py") #dask_client.upload_file("ArrayTools.py") #dask_client.upload_file("Calculation.py") #dask_client.upload_file("ColorMaps.py") #dask_client.upload_file("Conversions.py") #dask_client.upload_file("Plotting.py") #dask_client.upload_file("PyPostSettings.py") #dask_client.upload_file("Routines.py") logger.write(" - Success!") logger.write(" 1. Done.") logger.write(" 2. Start Post-Processing Calculations") calculation_future = start_calculations(dask_client, _routines, dask_threads) if (calculation_future != None): wait(calculation_future) result_calc = dask_client.gather(calculation_future)[0] if (result_calc != 0): logger.write( "***FAIL*** An error occured in calculations method, check worker logs for more info." ) logger.close() sys.exit("") logger.write(" 2. Done.") logger.write(" 3. Generating Figures") logger.write(" - Collecting files from target directory (" + targetDir + ").") fList3 = sorted(glob.glob(targetDir + "WRFPRS_F*")) logger.write(" - " + str(len(fList3)) + " files have been found.") logger.write(" -> Pushing run_plotting_routines() to dask.") fullDict = _pySet.get_full_dict() plotting_future = start_plotting(dask_client, fullDict, dask_threads) wait(plotting_future) result_plot = dask_client.gather(plotting_future)[0] if (result_plot != 0): logger.write( "***FAIL*** An error occured in plotting method, check worker logs for more info." ) logger.close() sys.exit("") logger.write(" 3. Done.") logger.write(" 4. Final Steps") logger.write(" 4. Done, Closing Dask Client.") dask_client.retire_workers(workers=dask_client.scheduler_info()['workers'], close=True) dask_client.close() logger.write("All Steps Completed.") logger.write("***SUCCESS*** Program execution complete.") logger.close()