def applyCV2Dir(dirname, infdir_name='sample_imgs/test_output', procdir_name='sample_imgs/cv_output', use_clahe=True, overwrite_file=False, data_reconstruct=False): log_names = os.listdir(dirname) for log_name_short in log_names: print('-' * 20) print(log_name_short) log_name = os.path.join(dirname, log_name_short) infdir = os.path.join(log_name, infdir_name) procdir = os.path.join(log_name, procdir_name) if not os.path.isdir(infdir): continue if (not overwrite_file) and os.path.isdir(procdir): continue print('processing %s...' % infdir) makeDir(procdir) dset_name = None name_split = os.path.basename(log_name).split('_') for cand in dataset_cands: dataset_str = name_split[0] dset_name = cand if (cand in dataset_str) else dset_name for cand in dataset_cands: dataset_str = name_split[-1] dset_name = cand if (cand in dataset_str) else dset_name indir = os.path.join('data/%s_detailed/original' % dset_name) fnames = [os.path.join(indir, fname.replace('_output', '')) for fname in os.listdir(infdir)] for fname in fnames: feature_extractor.main_dirinput( fname, procdir, infdir, use_clahe=use_clahe)
def genDataset(dirname, out_dirname, procfn, **kwargs): for dname in ['original', 'data_split.json']: try: shutil.copytree(os.path.join(dirname, dname), os.path.join(out_dirname, dname)) except: shutil.copy(os.path.join(dirname, dname), os.path.join(out_dirname, dname)) oridir = os.path.join(dirname, 'original') gtdir = os.path.join(dirname, 'gt') gtoutdir = os.path.join(out_dirname, 'gt') makeDir([gtoutdir]) fnames = [fname for fname in os.listdir(oridir)] fnames.sort() for img_name in fnames: gt_fname = convertFnameDomains('original', 'ground_truth', img_name) pred_fname = os.path.join(gtdir, gt_fname) out_fname = os.path.join(gtoutdir, gt_fname) macro_img = cv2.imread(pred_fname) if macro_img is None: continue bin_img = procfn(macro_img, img_name, kwargs) anno_fname = convertFnameDomains('original', 'annotated', img_name) cv2.imwrite(out_fname, bin_img)
def setupFiles(from_dir, to_dir): if os.path.isfile(from_dir): shutil.copyfile(from_dir, to_dir) return makeDir(to_dir) for dname in os.listdir(from_dir): setupFiles(os.path.join(from_dir, dname), os.path.join(to_dir, dname))
def create_images(self, number_images, set_name, computer, config, gen_mode): # Generates images from the OpenAI Gym Cartpole environment specified in self.init root_path = config['paths'][computer]['save_images'] images_path = os.path.join(root_path, self.env_name, gen_mode, set_name) utils.makeDir(images_path) # Init csv file csv_path = os.path.join(images_path, 'data_pairs.csv') with open(csv_path, "w") as csvfile: writer = csv.writer(csvfile) writer.writerow(['X', 'Theta', 'Reward', 'Image', 'Drone angle']) # Create images if gen_mode in [ 'random', 'random_weird', 'random_0', 'random_1', 'random_2', 'random_3', 'random_straight', 'random_0_straight', 'random_1_straight', 'random_2_straight', 'random_3_straight' ]: for i in range(number_images): if i % 100 == 99: print(i) image_name = str(i) image_path = os.path.join(images_path, image_name + '.png') self.one_image(set_name, csv_path, image_path) self.env.close() elif gen_mode == 'incremental' or gen_mode == 'incremental_weird': if self.env_name == 'duckietown': raise ValueError( "It is not possible to run an incremental image generation on the Duckietown environment." ) if set_name == 'train': pass square_side = int(np.floor(np.sqrt(number_images))) x_range = self.env.x_range theta_range = self.env.theta_range i = 0 print('x_range[0]: {}, x_range[1]: {}, num = square_side: {}'. format(x_range[0], x_range[1], square_side)) for x in np.linspace(x_range[0], x_range[1], num=square_side): for theta in np.linspace(theta_range[0], theta_range[1], num=square_side): state = (x, theta) image_name = str(i) image_path = os.path.join(images_path, image_name + '.png') self.one_image(set_name, csv_path, image_path, state) i += 1 self.env.close() elif gen_mode == 'training': raise NotImplementedError( "Generating images during training has not been implemented yet." )
def __init__(self, config, computer, test_incremental): #trained_model_prefix, label_style): # GPU Params use_gpu = torch.cuda.is_available() self.device = torch.device("cuda:0" if use_gpu else "cpu") # Paths and configuration parameters self.environment = config['exp']['env'] self.gen_mode = config['exp']['gen_mode'] if test_incremental: if self.gen_mode == 'random': test_set_gen_mode = 'incremental' elif self.gen_mode == 'random_weird': test_set_gen_mode = 'incremental_weird' else: test_set_gen_mode = self.gen_mode self.label_style = config['exp']['label_type'] model_name = config['exp']['model_name'] self.testing_set_path = os.path.join( config['paths'][computer]['save_images'], self.environment, test_set_gen_mode, 'test') self.root_path = config['paths'][computer]['save_cnn'] self.root_label_path = os.path.join(self.root_path, self.environment, self.label_style, self.gen_mode, model_name) self.rescale_size = tuple(config['cnn']['rescale_size']) self.model_type = config['cnn']['model'] if test_incremental: self.test_results_path = os.path.join(self.root_label_path, 'test_results', 'incremental') else: self.test_results_path = os.path.join(self.root_label_path, 'test_results', 'test_set') print(self.test_results_path) ut.makeDir(self.test_results_path) print(self.test_results_path) self.model_path = os.path.join(self.root_label_path, 'latest_model.pth') self.cnn_params_path = os.path.join(self.root_label_path, 'cnn_params.yaml') # Loading net self.net = self.load_model() self.criterion = nn.MSELoss() # Loading data self.test_set = self.load_data()
def __init__(self, root_dir, height=256, width=256, camera_id=0, fps=30): self.save_dir = utils.makeDir(root_dir, 'video') if root_dir else None self.height = height self.width = width self.camera_id = camera_id self.fps = fps self.frames = []
def copyTarArc( old_tar, new_tar, temp_directory): old_tar = tarfile.open(old_tar, mode='r') members = old_tar.getmembers() members.sort() utils.makeDir(temp_directory) for member in members: fixed_path = utils.correctPath(member.name) if fixed_path : member.name = fixed_path old_tar.extract(member, path=temp_directory) createTarArc(temp_directory, new_tar) old_tar.close()
def save_data(self, data, date, path_to_save): nb_exp_steps = data['nb_exp_steps'] mean_ev_returns = data['mean_ev_returns'] mean_exp_returns = data['mean_exp_returns'] mat_ev_ret = [list (a) for a in zip(mean_ev_returns, nb_exp_steps)] mat_exp_ret = [list (a) for a in zip(mean_exp_returns, nb_exp_steps)] print(tuple(mat_ev_ret)) test_path = os.path.join(path_to_save, 'training_data', 'runs', 'test') train_path = os.path.join(path_to_save, 'training_data', 'runs','train') ut.makeDir(test_path) ut.makeDir(train_path) path_ev_ret = os.path.join(test_path, '{}_avg_returns_test.npy'.format(date.strftime("%Y-%m-%d_%H-%M-%S"))) path_exp_ret = os.path.join(train_path, '{}_avg_returns_train.npy'.format(date.strftime("%Y-%m-%d_%H-%M-%S"))) np.save(path_ev_ret, mat_ev_ret) np.save(path_exp_ret, mat_exp_ret)
def extractAllTarArc(tar_arc, dest=""): if dest: utils.makeDir(dest) tf = tarfile.open(tar_arc, 'r') tf.extractall(dest) tf.close()
def extractAll(zip_archive, dest=""): if dest: utils.makeDir(dest) zf = zipfile.ZipFile(zip_archive) zf.extractall(path=dest ) zf.close()
temp_root = os.environ['SLURM_TMPDIR'] ### CNN saving paths during usage (data training and model) use_cnn_path = os.path.join(temp_root, config['paths'][computer]['cnn']) use_cnn_label_path = os.path.join(use_cnn_path, environment, label_type, gen_mode, model_name) cnn_training_data_path = os.path.join(use_cnn_label_path, 'training_data') cnn_training_plots_path = os.path.join(cnn_training_data_path, 'plots') cnn_latest_model_path = os.path.join(use_cnn_label_path, 'latest_model.pth') cnn_inter_model_path = os.path.join(use_cnn_label_path, 'inter_models') cnn_params_path = os.path.join(use_cnn_label_path, 'cnn_params.yaml') cnn_train_losses_path = os.path.join(use_cnn_label_path, 'train_losses.npy') cnn_test_losses_path = os.path.join(use_cnn_label_path, 'test_losses.npy') ut.makeDir(cnn_training_data_path) ut.makeDir(cnn_training_plots_path) ut.makeDir(cnn_inter_model_path) config_path['use_cnn_path'] = use_cnn_path config_path['training_data_path'] = cnn_training_data_path config_path['training_plots_path'] = cnn_training_plots_path config_path['model_path'] = cnn_latest_model_path config_path['inter_model_path'] = cnn_inter_model_path config_path['cnn_params_path'] = cnn_params_path config_path['cnn_train_losses_path'] = cnn_train_losses_path config_path['cnn_test_losses_path'] = cnn_test_losses_path ### Image dataset path use_images_path = os.path.join(temp_root, config['paths'][computer]['images']) training_set_path = os.path.join(use_images_path, environment, gen_mode,
def __init__(self, env, test_env, config_path, config_rl): self.env = env ### RL parameters self.env_name = config_rl['env_name'] self.test_env_name = config_rl['test_env_name'] self.seed = config_rl['seed'] self.policy = config_rl['policy'] self.eval = config_rl['eval_'] self.gamma = config_rl['gamma'] self.tau = config_rl['tau'] self.lr = config_rl['lr'] self.alpha = config_rl['alpha'] self.automatic_entropy_tuning = config_rl['automatic_entropy_tuning'] self.batch_size = config_rl['batch_size'] self.num_episodes = config_rl['num_episodes'] self.max_steps_episode = config_rl['max_steps_episode'] self.hidden_size = config_rl['hidden_size'] self.updates_per_step = config_rl['updates_per_step'] self.start_steps = config_rl['start_steps'] self.target_update_interval = config_rl['target_update_interval'] self.replay_size = config_rl['replay_size'] self.cuda = config_rl['cuda'] self.gen_est_reward_test = config_rl['gen_est_reward_test'] self.sac_args = SAC_args(self.gamma, self.tau, self.lr, self.alpha, self.automatic_entropy_tuning, self.policy, self.target_update_interval, self.cuda, self.hidden_size) self.env = env self.test_env = test_env #self.env.env.reset() torch.manual_seed(self.seed) np.random.seed(self.seed) self.env.seed(self.seed) if self.env_name not in ['DTC_GT_Reward']: self.agent = SAC(self.env.observation_space.shape[0], self.env.action_space, self.sac_args) else: self.agent = SAC_CNN(self.env.observation_space.shape[0], self.env.action_space, self.sac_args) ### Path parameters self.training_data_path = config_path['training_data_path'] ut.makeDir(self.training_data_path + '/runs') ut.makeDir(self.training_data_path + '/runs/train') ut.makeDir(self.training_data_path + '/runs/test') self.vis_path = self.training_data_path + '/runs/vis/{}_SAC_{}_{}_{}'.format( datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"), self.env_name, self.policy, "autotune" if self.automatic_entropy_tuning else "") self.train_res_path = self.training_data_path + '/runs/train/{}_SAC_{}_{}_{}'.format( datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"), self.env_name, self.policy, "autotune" if self.automatic_entropy_tuning else "") self.train_rew_profit_path = self.training_data_path + '/runs/train/{}_SAC_{}_{}_{}_return_profit'.format( datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"), self.env_name, self.policy, "autotune" if self.automatic_entropy_tuning else "") self.train_rew_error_path = self.training_data_path + '/runs/train/{}_SAC_{}_{}_{}_return_error'.format( datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"), self.env_name, self.policy, "autotune" if self.automatic_entropy_tuning else "") self.test_res_path = self.training_data_path + '/runs/test/{}_SAC_{}_{}_{}_test'.format( datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"), self.env_name, self.policy, "autotune" if self.automatic_entropy_tuning else "") self.test_rew_profit_path = self.training_data_path + '/runs/test/{}_SAC_{}_{}_{}_return_profit_test'.format( datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"), self.env_name, self.policy, "autotune" if self.automatic_entropy_tuning else "") self.test_rew_error_path = self.training_data_path + '/runs/test/{}_SAC_{}_{}_{}_return_error_test'.format( datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"), self.env_name, self.policy, "autotune" if self.automatic_entropy_tuning else "") self.critic_model_path = config_path['critic_model_path'] self.actor_model_path = config_path['actor_model_path'] self.writer = SummaryWriter(logdir=self.vis_path) self.memory = ReplayMemory(self.replay_size) self.reward_mem = [] self.reward_profit_mem = [] self.reward_error_mem = [] self.test_mem = [] self.test_prof_mem = [] self.test_err_mem = []
if computer == 'mila': # Getting local disk info temp_root = os.environ['SLURM_TMPDIR'] + '/' ### RL saving paths during usage (data training and model) use_rl_path = os.path.join(temp_root, config['paths'][computer]['rl']) if model_name: exp_name = env_name + '_' + model_name else: exp_name = env_name use_rl_env_path = os.path.join(use_rl_path, exp_name) print('use_rl_env_path: {}'.format(use_rl_env_path)) rl_training_data_path = os.path.join(use_rl_env_path, 'training_data') ut.makeDir(rl_training_data_path) config_path['training_data_path'] = rl_training_data_path config_path['critic_model_path'] = os.path.join(use_rl_env_path, 'critic_model.pth') config_path['actor_model_path'] = os.path.join(use_rl_env_path, 'actor_model.pth') ### Training environment #### Importing library if test_env_name == 'cartpole': from cartpole_mod_env import * elif test_env_name == 'duckietown' or test_env_name == 'duckietown_cam': from duckietown_mod_env import * #### Loading environment
def set_test_results_path(self, new_path): self.test_results_path = new_path ut.makeDir(self.test_results_path)
def extractAll(zip_archive, dest=""): if dest: utils.makeDir(dest) zf = zipfile.ZipFile(zip_archive) zf.extractall(path=dest) zf.close()
change_list = [] if "dryrun" in sys.argv: dryrun = 1 sys.argv.remove("dryrun") if len(sys.argv) < 2: print "Usage: %s <pisi_packages_directory>" % sys.argv[0] sys.exit(1) if not os.path.exists(sys.argv[1]): print "%s does not exists" % sys.argv[1] sys.exit(1) pisi_directory = sys.argv[1] pisi_list = glob.glob1(pisi_directory, "*6.pisi") temp_directory = "/tmp/oatp" olddir = os.getcwd() for pisi in pisi_list: utils.makeDir(temp_directory) os.chdir(temp_directory) main(os.path.join(pisi_directory, pisi)) if dryrun: inform() os.chdir(olddir)
def convDeepcrack(img_dir, dataset_name): target_sz = {'aigle': (462, 311), 'cfd': None, 'deepcrack': None} deepcrack_dir = os.path.join(img_dir, 'deepcrack') makeDir(deepcrack_dir) gen_deepcrack.main(img_dir, deepcrack_dir, target_sz[dataset_name])
def convDeeplab(img_dir, dataset_name): deeplab_dir = os.path.join(img_dir, 'deeplab') deeplab_tf_dir = os.path.join(img_dir, 'deeplab_tfrecords') makeDir([deeplab_dir, deeplab_tf_dir]) gen_deeplab.main(img_dir, deeplab_dir, deeplab_tf_dir)
import matplotlib as mpl import matplotlib.pyplot as plt import numpy as np from matplotlib.backends.backend_pdf import PdfPages from enum import Enum #----script starts here parser = ambient.ArgumentParser (description="""generate plots from ESO data""") #parse user's command line arguments args = parser.parse_args () #make sure the output directory exists utils.makeDir (args.outdir) #get list of all seeing days allSeeingDays = ambient.Days (os.path.join (args.indir, ambient.parameterBasenames [0] + ".orig"), args.start_date, args.stop_date) #scripts need start/stop dates now #assert args.start_date #if start-date/stop-date is specified on command line, only look at a subset of days if args.start_date: seeingDays = ambient.dayListFromRange (allSeeingDays, args.start_date, args.stop_date) else: seeingDays = allSeeingDays.days def seeingCorrelations (seeingDays, otherDays):