def build_models_for_all_drivers(columns, norm_threshold=0.5, num_falsely_belong=10, num_not_belong=500, store_filename='store.h5'): drivers = utils.get_folders() falsely_belong = utils.get_random_trip_collection(num_falsely_belong, 1000000).fillna(0) not_belong = utils.get_random_trip_collection(num_not_belong, 1000000).fillna(0) random.shuffle(drivers) #print(falsely_belong.dtypes) #print(not_belong.dtypes) store = pd.HDFStore(store_filename) store['falsely_belong'] = falsely_belong store['not_belong'] = not_belong print((len(drivers))) for i, driver in enumerate(drivers): print((i, driver)) pipeline = build_pipeline() predictions, threshold = generate_predictions_for_driver( driver, falsely_belong, not_belong, pipeline, norm_threshold, columns) store.put('predictions_{0}'.format(driver), predictions) store.close()
def set_visuomotor_dataset(self): for i, trial_fold in enumerate( get_folders(DIR, key=lambda name: int(name[5:]))): print(i) IMG = pathname(trial_fold, 'IMG') # scipy.misc.imsave('/data/barry/IMITATION/dataset/THROW/TEST/img_%d.png'%i, np.load(pathname(IMG, 'img_1472.npy'))[:,:,:3]) img_files = get_files(IMG, key=lambda order: int(order[4:-4])) img_array = np.concatenate([[np.load(img_file)[:, :, :3]] for img_file in img_files[1:DATA_NUM + 1]]) joint_conf_array = np.load( pathname(trial_fold, 'joints_configurations.npy'))[0:DATA_NUM] velocities = np.load(pathname(trial_fold, 'velocities.npy')) velocities_array = velocities[1:DATA_NUM + 1] gripper_array = np.zeros(DATA_NUM) first_gripper = velocities_array[:, 6] closing, opening = np.min(np.where(first_gripper > 0.9)), np.min( np.where(first_gripper < -0.9)) gripper_array[closing:opening] = 1.0 assert velocities_array.shape[0] == joint_conf_array.shape[ 0] == img_array.shape[0] == gripper_array.shape[ 0] # == velocities_conf.shape[0] write_tf_records_db(pathname(DIR, '..', 'VISUOMOTOR_GRP', 'demo_%d.tfrecord' % i), img_array, joint_conf_array, velocities_array, gripper_array=gripper_array, step_array=np.array([ 'step_%d_%d' % (i, j) for j in range(DATA_NUM) ]))
def __init__(self, sess, width=256, height=256, channels=3, action_dim=9, learning_rate=0.0001, model_name=None, graph=None, export=False): self.sess = sess self.width = width self.height = height self.channels = channels self.action_dim = action_dim self.learning_rate = learning_rate self.rgb, self.conf, self.vel, = self.create_network('visuonet') if not export: self.velocities = tf.placeholder(tf.float32, [None, 9]) self.loss = tf.reduce_mean( tf.square(self.velocities - self.vel) ) #+ 0.001 * tf.reduce_mean(tf.square(self.gripper - self.vel[:,6])) self.optimizer = tf.train.AdamOptimizer( self.learning_rate).minimize(self.loss) self.model = model(sess, MODELS_DIR, model_name=model_name) self.summary = self.model.summary(graph=graph, **dict(loss=self.loss)) self.eval_indice = len( get_folders(pathname(RES_PATH, **dict(flag=1))))
def preprocess(tiff_path, save_path, land_path, cloud_path, width, height, polys_path, channels, type_filter, filter_by_date, pxl_size_threshold, no_merge, pass_chance): print(f'filter_by_date:{filter_by_date}') if not os.path.exists(save_path): os.makedirs(save_path, exist_ok=True) print("Save directory created.") try: land_src = rasterio.open(land_path, 'r') except IOError: print("Land cover map file not found: {}".format(land_path)) sys.exit() for tiff_name in get_folders(tiff_path): tiff_filepath = os.path.join(tiff_path, tiff_name) if no_merge: tiff_file = join(save_path, f'{tiff_name}.tif') else: tiff_file = merge_bands(tiff_filepath, save_path, channels) data_path = os.path.join(save_path, basename(tiff_file[:-4])) divide_into_pieces(tiff_file, data_path, land_src, width, height) mask_pieces_path = os.path.join(data_path, 'masks') land_pieces_path = os.path.join(data_path, 'landcover') clouds_path = os.path.join(cloud_path, basename(tiff_file[:-4]) + '_clouds.png') if not os.path.exists(clouds_path): clouds_pieces_path = None else: clouds_pieces_path = os.path.join(data_path, 'clouds') if not os.path.exists(clouds_pieces_path): os.mkdir(clouds_pieces_path) pieces_info = os.path.join(data_path, 'image_pieces.csv') mask_path = poly2mask(polys_path, tiff_file, data_path, type_filter, filter_by_date) split_mask(mask_path, mask_pieces_path, clouds_path, clouds_pieces_path, pieces_info) geojson_polygons = os.path.join(data_path, "geojson_polygons") instance_masks_path = os.path.join(data_path, "instance_masks") filter_poly(poly_pieces_path=geojson_polygons, markup_path=polys_path, pieces_info_path=pieces_info, original_image_path=tiff_file, image_pieces_path=os.path.join(data_path, 'images'), mask_pieces_path=mask_pieces_path, land_pieces_path=land_pieces_path, clouds_pieces_path=clouds_pieces_path, pxl_size_threshold=pxl_size_threshold, pass_chance=pass_chance) land_src.close()
def set_base_path(self, base_path): if base_path.exists(): self.base_path = base_path if self.verbose: print(f'Set base_path to {self.base_path}') self.folder_list = get_folders(self.base_path) if self.verbose: print(f'Update folder_list from {self.base_path}') else: print(f'set_base_path {base_path} not found')
def get_status(self): self.folder_list = get_folders(self.base_path) if self.folder_list is not None: fsize = len(self.folder_list) else: fsize = 0 # print(f'MainThread: {self.name} running v:{self.verbose} dr:{self.dry_run} f:{fsize} bp:{self.base_path}') print( f'MainThread: {self.name} running v:{self.verbose} dr:{self.dry_run} f:{fsize} bp:{self.base_path} active threads: {active_count()}' )
def get_data_pathes(data_path=args.data_path, images_folder=args.images_folder, masks_folder=args.masks_folder, instances_folder=args.instances_folder): dataset = get_folders(data_path)[0] images_path = os.path.join(data_path, dataset, images_folder) masks_path = os.path.join(data_path, dataset, masks_folder) instances_path = os.path.join(data_path, dataset, instances_folder) return images_path, masks_path, instances_path
def get_data_info(data_path=args.data_path): _, _, insatnces_path = get_data_pathes(data_path) instances = get_folders(insatnces_path) cols = ['dataset_folder', 'name', 'position'] data_info = pd.DataFrame(columns=cols) for instance in instances: name, position = get_instance_info(instance) data_info = add_record(data_info, dataset_folder=name, name=name, position=position) return data_info
def test_sources(): b_anti = [] b_pro = [] for a,b in s.category_sources.items(): b_anti += b[0] b_pro +=b[1] total_in_settings = set(b_anti+ b_pro) total_in_disk = u.get_folders(s.raw_websources_path) print len(total_in_settings) print len(total_in_disk) print len(list(set(total_in_disk) & set(total_in_settings)))
def write_tjr_dataset(): for i, trial_fold in enumerate( get_folders(DIR, key=lambda name: int(name[5:]))): print(i) IMG = pathname(trial_fold, 'IMG') # scipy.misc.imsave('/data/barry/IMITATION/dataset/THROW/TEST/img_%d.png'%i, np.load(pathname(IMG, 'img_1472.npy'))[:,:,:3]) img_files = get_files(IMG, key=lambda order: int(order[4:-4])) img_array = np.concatenate([[np.load(img_file)[:, :, :3]] for img_file in img_files[0:DATA_NUM + 1]]) prev_img_array = img_array[0:DATA_NUM] next_img_array = img_array[1:DATA_NUM + 1] goal_array = img_array[-1] initial_img_array = img_array[0] joint_conf_array = np.load( pathname(trial_fold, 'joints_configurations.npy'))[0:DATA_NUM + 1] prev_joint_conf_array = joint_conf_array[0:DATA_NUM] next_joint_conf_array = joint_conf_array[1:DATA_NUM + 1] vel_conf = np.load(pathname(trial_fold, 'velocities.npy'))[0:DATA_NUM] assert prev_img_array.shape[0] == next_img_array.shape[0] == \ prev_joint_conf_array.shape[0] == next_joint_conf_array.shape[0] \ == vel_conf.shape[0] # == velocities_conf.shape[0] step_array = np.array(['step_%d_%d' % (i, j) for j in range(DATA_NUM)]) with tf.python_io.TFRecordWriter( pathname(DIR, '..', 'TJR', 'demo_%d.tfrecord' % i)) as writer: for k in range(step_array.shape[0]): example = tf.train.Example(features=tf.train.Features( feature={ 'prev_img': _bytes_feature(prev_img_array[k].tostring()), 'next_img': _bytes_feature(next_img_array[k].tostring()), 'prev_jnt_conf': _bytes_feature(prev_joint_conf_array[k].tostring()), 'next_jnt_conf': _bytes_feature(next_joint_conf_array[k].tostring()), 'goal_img': _bytes_feature(goal_array.tostring()), 'initial_img': _bytes_feature(initial_img_array.tostring()), 'vel_conf': _bytes_feature(vel_conf[k].tostring()), 'step': _bytes_feature(step_array[k].tostring()) })) writer.write(example.SerializeToString())
def main(): folders = get_folders(path=DATA_PATH) p_mode_velocity_file = 'p_mode_velocities.csv' scene_objects = 'scene_objects.npy' success = 0 for i in range(507, 999): folder = DATA_PATH + 'Trial_%d/' % i with Interface() as I: # LOAD ROBOT robot = I.load_model(ROBOT_URDF_PATH) # LOAD TABLE table = I.load_model(TABLE, Pose(Point(0.4, 0.0, 0.05), Euler(1.57, 0.0, 0.0)), fixed_base=True, scaling=1.0) # TODO set camera I.set_camera_pose(0.6, 155, -50, [0.2, 0.0, 0.5]) scene = np.load(os.path.join(folder, scene_objects))[2:, 0] block = I.load_model(CUBE_URDF, Pose(Point(*scene[0])), fixed_base=False) basket = I.load_model(BASKET_URDF, Pose(Point(*scene[1])), fixed_base=False) I.save_body_conf() if os.path.isfile(os.path.join(folder, p_mode_velocity_file)): velocities = Pd(os.path.join( folder, p_mode_velocity_file)).read().values I.replay_velocities(robot, velocities) if inside_basket(I, block, basket): I.restore_body_conf() time.sleep(1.0) I.replay_velocities(robot, velocities, filename=os.path.join(folder, 'img')) success += 1 if success == 270: break print('Trials : ', folder, ' , Success: ', success)
def __init__(self, sess, width=256, height=256, channels=3, learning_rate=0.0001, z_shape=8, j_shape=9, model_name=None, graph=None, coeff=0.002): self.sess = sess self.width = width self.height = height self.channels = channels self.learning_rate = learning_rate self.z_shape = z_shape self.j_shape = j_shape self.img, self.p_t, self.p_t_1 = self.gen_network('generator') # self.p_t = tf.placeholder(tf.float32, shape = (None, self.j_shape)) # self.std = tf.exp(self.logstds) # self.var = self.std ** 2 # self.loss = 0.5 * tf.reduce_sum(tf.square((self.p_t - self.action_mean) / self.var)) \ # + 0.5 * np.log(2.0 * np.pi) * tf.to_float(tf.shape(self.p_t)[0]) \ # + tf.reduce_sum(self.logstds) self.std = tf.exp(self.logstds) # self.var = tf.square(self.std) self.p_t_1_l = tf.placeholder(tf.float32, (None, 9)) self.mse_loss = tf.reduce_mean(tf.square(self.p_t_1 - self.p_t_1_l)) lh_loss = 0.5 * tf.reduce_sum(tf.square((self.p_t_1_l - self.p_t_1) / self.std), axis=-1) \ + 0.5 * np.log(2.0 * np.pi) * tf.to_float(tf.shape(self.p_t_1)[-1]) + tf.reduce_sum(self.logstds, axis=-1) self.lh_loss = tf.reduce_mean(lh_loss) self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize( self.lh_loss) self.model = model(sess, MODELS_DIR, model_name=model_name) self.summary = self.model.summary(graph=graph, **dict(mse_loss=self.mse_loss, lh_loss=self.lh_loss)) self.eval_indice = len(get_folders(pathname(RES_PATH, **dict(flag=1))))
def run(self): if self.verbose: print(f'MainThread: {self.name} starting bp: {self.base_path}') self.folder_list = get_folders(self.base_path) if self.folder_list is not None: self.update() while True: try: # [0] f: new movie # [0] s: scrape # print(f'q items put {self.monitor_q.qsize()}') q_item = self.monitor_q.get_nowait() # new_movie = self.monitor_q.get_nowait() if q_item[0] == 'f': # new_movie.append(q_item[1]) new_movie = q_item[1] self.import_from_path(new_movie) # print(f'new_movie found: {new_movie}') # self.grab_folder(new_movie) # q_item.update self.monitor_q.task_done() # self.update() # print(f'q items task done {self.monitor_q.qsize()}') if q_item[0] == 's': pass except Empty: # with concurrent.futures.ThreadPoolExecutor() as executor: # ttp_res = executor.map(self.import_from_path, new_movie) # new_movie = [] # print(f'q empty') # time.sleep(0.5) pass if self.kill: return if self.verbose: # self.get_status() pass
def geo_split( data_path=args.data_path, markup_path=args.markup_path, mask_type=args.mask_type, masks_folder=args.masks_folder, polygons_folder=args.polygons_folder, test_threshold=0.2, val_bottom_threshold=0.2, val_threshold=0.3, ): datasets = get_folders(data_path) geojson_markup = gp.read_file(markup_path) minY, maxY = get_height_bounds(geojson_markup) height = maxY - minY cols = ['dataset_folder', 'name', 'position'] train_df = pd.DataFrame(columns=cols) val_df = pd.DataFrame(columns=cols) test_df = pd.DataFrame(columns=cols) overall_sizes = {'test': 0, 'train': 0, 'val': 0, 'deleted': 0} for dataset_dir in datasets: polys_path = os.path.join(data_path, dataset_dir, polygons_folder) print(dataset_dir) deleted = 0 train = 0 test = 0 val = 0 for poly_name in os.listdir(polys_path): instance_geojson_path = os.path.join(polys_path, poly_name) instance_geojson = gp.read_file(instance_geojson_path) if geojson_markup.crs != instance_geojson.crs: geojson_markup = geojson_markup.to_crs(instance_geojson.crs) minY, maxY = get_height_bounds(geojson_markup) height = maxY - minY instance_minY, instance_maxY = get_height_bounds(instance_geojson) name, position = get_instance_info(poly_name) masks_path = os.path.join(data_path, dataset_dir, masks_folder) mask_path = get_filepath(masks_path, get_fullname(name, position), file_type=mask_type) mask = Image.open(mask_path) mask_array = np.array(mask) mask_pixels = np.count_nonzero(mask_array) center_pixels = np.count_nonzero(mask_array[10:-10, 10:-10]) border_pixels = mask_pixels - center_pixels if mask_pixels > mask_array.size * 0.001 and center_pixels > border_pixels: if instance_maxY < minY + height * test_threshold: test += 1 test_df = add_record(test_df, dataset_folder=name, name=name, position=position) elif instance_maxY < minY + height * val_threshold \ and instance_minY > minY + height * val_bottom_threshold: val += 1 val_df = add_record(val_df, dataset_folder=name, name=name, position=position) else: train += 1 train_df = add_record(train_df, dataset_folder=name, name=name, position=position) else: deleted += 1 print("Train size", train, "Validation size", val, "Test size", test) print(f"{deleted} images were deleted") overall_sizes = update_overall_sizes(overall_sizes, test, train, val, deleted) print("Overall sizes", overall_sizes) return train_df, val_df, test_df
from utils import get_folders,pathname import os # PATH = '/home/barry/hdd/IMITATION/training_data/STACK' PATH = '/data/barry/IMITATION/dataset/THROW/DEMO' # folders = get_folders(PATH, key=lambda f : int(f[6:]))[1000:] folders = get_folders(PATH, key=lambda f : int(f[5:])) # print(len(folders)) # rg=0 b = range(len(folders)) for fold in folders: # os.rename(fold,pathname(PATH,'demo_%d'%rg)) # rg +=1 a = fold.split('/')[-1][5:] if int(a) not in b: print fold # if rename: # for i,fold in enumerate(sorted(folders,key = lambda file: int(file.split('/')[-1][6:]))): # f = int(fold.split('/')[-1][6:]) # # os.rename(os.path.join(PATH,'Trial_%d'%f ), # os.path.join(PATH, 'Trial_%d' % i))
import utils import save_arrays import sys if __name__ == '__main__': name = "goals" read_folder, save_path = utils.get_folders(sys.argv, name) plans = utils.get_plans(read_folder) goals = [] for p in plans: if p.goals not in goals: goals.append(p.goals) save_arrays.save(goals, save_path) print(len(goals))
) parser.add_argument( '--masks_path', '-mp', dest='masks_path', default='masks', help='Path to masks' ) return parser.parse_args() if __name__ == '__main__': args = parse_args() if not os.path.exists(args.save_path): os.makedirs(args.save_path, exist_ok=True) print("Save directory created.") for tiff_name in get_folders(args.tiff_path): print(tiff_name) tiff_filepath = os.path.join(args.tiff_path, tiff_name) tiff_file = join(args.save_path, f'{tiff_name}.tif') data_path = os.path.join(args.save_path, basename(tiff_file[:-4])) clouds_pieces_path = os.path.join(data_path, 'clouds') masks_pieces_path = os.path.join(data_path, args.masks_path) cloud_path = os.path.join(args.cloud_path, basename(tiff_file[:-4])+'_clouds.tiff') pieces_info = os.path.join(data_path, 'image_pieces.csv') if not os.path.exists(clouds_pieces_path): os.mkdir(clouds_pieces_path) if not os.path.exists(masks_pieces_path): print('No masks_pieces_path directory was found') sys.exit() split_cloud(cloud_path, clouds_pieces_path, pieces_info, masks_pieces_path)
def update_folders(self): # scan base folder for movie folders self.folder_list = get_folders(self.base_path)