def build_models_for_all_drivers(columns,
norm_threshold=0.5,
num_falsely_belong=10,
num_not_belong=500,
store_filename='store.h5'):
drivers = utils.get_folders()
falsely_belong = utils.get_random_trip_collection(num_falsely_belong,
1000000).fillna(0)
not_belong = utils.get_random_trip_collection(num_not_belong,
1000000).fillna(0)
random.shuffle(drivers)
#print(falsely_belong.dtypes)
#print(not_belong.dtypes)
store = pd.HDFStore(store_filename)
store['falsely_belong'] = falsely_belong
store['not_belong'] = not_belong
print((len(drivers)))
for i, driver in enumerate(drivers):
print((i, driver))
pipeline = build_pipeline()
predictions, threshold = generate_predictions_for_driver(
driver, falsely_belong, not_belong, pipeline, norm_threshold,
columns)
store.put('predictions_{0}'.format(driver), predictions)
store.close()
def set_visuomotor_dataset(self):
for i, trial_fold in enumerate(
get_folders(DIR, key=lambda name: int(name[5:]))):
print(i)
IMG = pathname(trial_fold, 'IMG')
# scipy.misc.imsave('/data/barry/IMITATION/dataset/THROW/TEST/img_%d.png'%i, np.load(pathname(IMG, 'img_1472.npy'))[:,:,:3])
img_files = get_files(IMG, key=lambda order: int(order[4:-4]))
img_array = np.concatenate([[np.load(img_file)[:, :, :3]]
for img_file in img_files[1:DATA_NUM +
1]])
joint_conf_array = np.load(
pathname(trial_fold, 'joints_configurations.npy'))[0:DATA_NUM]
velocities = np.load(pathname(trial_fold, 'velocities.npy'))
velocities_array = velocities[1:DATA_NUM + 1]
gripper_array = np.zeros(DATA_NUM)
first_gripper = velocities_array[:, 6]
closing, opening = np.min(np.where(first_gripper > 0.9)), np.min(
np.where(first_gripper < -0.9))
gripper_array[closing:opening] = 1.0
assert velocities_array.shape[0] == joint_conf_array.shape[
0] == img_array.shape[0] == gripper_array.shape[
0] # == velocities_conf.shape[0]
write_tf_records_db(pathname(DIR, '..', 'VISUOMOTOR_GRP',
'demo_%d.tfrecord' % i),
img_array,
joint_conf_array,
velocities_array,
gripper_array=gripper_array,
step_array=np.array([
'step_%d_%d' % (i, j)
for j in range(DATA_NUM)
]))
def __init__(self,
sess,
width=256,
height=256,
channels=3,
action_dim=9,
learning_rate=0.0001,
model_name=None,
graph=None,
export=False):
self.sess = sess
self.width = width
self.height = height
self.channels = channels
self.action_dim = action_dim
self.learning_rate = learning_rate
self.rgb, self.conf, self.vel, = self.create_network('visuonet')
if not export:
self.velocities = tf.placeholder(tf.float32, [None, 9])
self.loss = tf.reduce_mean(
tf.square(self.velocities - self.vel)
) #+ 0.001 * tf.reduce_mean(tf.square(self.gripper - self.vel[:,6]))
self.optimizer = tf.train.AdamOptimizer(
self.learning_rate).minimize(self.loss)
self.model = model(sess, MODELS_DIR, model_name=model_name)
self.summary = self.model.summary(graph=graph,
**dict(loss=self.loss))
self.eval_indice = len(
get_folders(pathname(RES_PATH, **dict(flag=1))))
def preprocess(tiff_path, save_path, land_path, cloud_path, width, height,
polys_path, channels, type_filter, filter_by_date,
pxl_size_threshold, no_merge, pass_chance):
print(f'filter_by_date:{filter_by_date}')
if not os.path.exists(save_path):
os.makedirs(save_path, exist_ok=True)
print("Save directory created.")
try:
land_src = rasterio.open(land_path, 'r')
except IOError:
print("Land cover map file not found: {}".format(land_path))
sys.exit()
for tiff_name in get_folders(tiff_path):
tiff_filepath = os.path.join(tiff_path, tiff_name)
if no_merge:
tiff_file = join(save_path, f'{tiff_name}.tif')
else:
tiff_file = merge_bands(tiff_filepath, save_path, channels)
data_path = os.path.join(save_path, basename(tiff_file[:-4]))
divide_into_pieces(tiff_file, data_path, land_src, width, height)
mask_pieces_path = os.path.join(data_path, 'masks')
land_pieces_path = os.path.join(data_path, 'landcover')
clouds_path = os.path.join(cloud_path,
basename(tiff_file[:-4]) + '_clouds.png')
if not os.path.exists(clouds_path):
clouds_pieces_path = None
else:
clouds_pieces_path = os.path.join(data_path, 'clouds')
if not os.path.exists(clouds_pieces_path):
os.mkdir(clouds_pieces_path)
pieces_info = os.path.join(data_path, 'image_pieces.csv')
mask_path = poly2mask(polys_path, tiff_file, data_path, type_filter,
filter_by_date)
split_mask(mask_path, mask_pieces_path, clouds_path,
clouds_pieces_path, pieces_info)
geojson_polygons = os.path.join(data_path, "geojson_polygons")
instance_masks_path = os.path.join(data_path, "instance_masks")
filter_poly(poly_pieces_path=geojson_polygons,
markup_path=polys_path,
pieces_info_path=pieces_info,
original_image_path=tiff_file,
image_pieces_path=os.path.join(data_path, 'images'),
mask_pieces_path=mask_pieces_path,
land_pieces_path=land_pieces_path,
clouds_pieces_path=clouds_pieces_path,
pxl_size_threshold=pxl_size_threshold,
pass_chance=pass_chance)
land_src.close()
def set_base_path(self, base_path):
if base_path.exists():
self.base_path = base_path
if self.verbose:
print(f'Set base_path to {self.base_path}')
self.folder_list = get_folders(self.base_path)
if self.verbose:
print(f'Update folder_list from {self.base_path}')
else:
print(f'set_base_path {base_path} not found')
def get_status(self):
self.folder_list = get_folders(self.base_path)
if self.folder_list is not None:
fsize = len(self.folder_list)
else:
fsize = 0
# print(f'MainThread: {self.name} running v:{self.verbose} dr:{self.dry_run} f:{fsize} bp:{self.base_path}')
print(
f'MainThread: {self.name} running v:{self.verbose} dr:{self.dry_run} f:{fsize} bp:{self.base_path} active threads: {active_count()}'
)
def get_data_pathes(data_path=args.data_path,
images_folder=args.images_folder,
masks_folder=args.masks_folder,
instances_folder=args.instances_folder):
dataset = get_folders(data_path)[0]
images_path = os.path.join(data_path, dataset, images_folder)
masks_path = os.path.join(data_path, dataset, masks_folder)
instances_path = os.path.join(data_path, dataset, instances_folder)
return images_path, masks_path, instances_path
def get_data_info(data_path=args.data_path):
_, _, insatnces_path = get_data_pathes(data_path)
instances = get_folders(insatnces_path)
cols = ['dataset_folder', 'name', 'position']
data_info = pd.DataFrame(columns=cols)
for instance in instances:
name, position = get_instance_info(instance)
data_info = add_record(data_info, dataset_folder=name, name=name, position=position)
return data_info
def test_sources():
b_anti = []
b_pro = []
for a,b in s.category_sources.items():
b_anti += b[0]
b_pro +=b[1]
total_in_settings = set(b_anti+ b_pro)
total_in_disk = u.get_folders(s.raw_websources_path)
print len(total_in_settings)
print len(total_in_disk)
print len(list(set(total_in_disk) & set(total_in_settings)))
def write_tjr_dataset():
for i, trial_fold in enumerate(
get_folders(DIR, key=lambda name: int(name[5:]))):
print(i)
IMG = pathname(trial_fold, 'IMG')
# scipy.misc.imsave('/data/barry/IMITATION/dataset/THROW/TEST/img_%d.png'%i, np.load(pathname(IMG, 'img_1472.npy'))[:,:,:3])
img_files = get_files(IMG, key=lambda order: int(order[4:-4]))
img_array = np.concatenate([[np.load(img_file)[:, :, :3]]
for img_file in img_files[0:DATA_NUM + 1]])
prev_img_array = img_array[0:DATA_NUM]
next_img_array = img_array[1:DATA_NUM + 1]
goal_array = img_array[-1]
initial_img_array = img_array[0]
joint_conf_array = np.load(
pathname(trial_fold, 'joints_configurations.npy'))[0:DATA_NUM + 1]
prev_joint_conf_array = joint_conf_array[0:DATA_NUM]
next_joint_conf_array = joint_conf_array[1:DATA_NUM + 1]
vel_conf = np.load(pathname(trial_fold, 'velocities.npy'))[0:DATA_NUM]
assert prev_img_array.shape[0] == next_img_array.shape[0] == \
prev_joint_conf_array.shape[0] == next_joint_conf_array.shape[0] \
== vel_conf.shape[0] # == velocities_conf.shape[0]
step_array = np.array(['step_%d_%d' % (i, j) for j in range(DATA_NUM)])
with tf.python_io.TFRecordWriter(
pathname(DIR, '..', 'TJR', 'demo_%d.tfrecord' % i)) as writer:
for k in range(step_array.shape[0]):
example = tf.train.Example(features=tf.train.Features(
feature={
'prev_img':
_bytes_feature(prev_img_array[k].tostring()),
'next_img':
_bytes_feature(next_img_array[k].tostring()),
'prev_jnt_conf':
_bytes_feature(prev_joint_conf_array[k].tostring()),
'next_jnt_conf':
_bytes_feature(next_joint_conf_array[k].tostring()),
'goal_img':
_bytes_feature(goal_array.tostring()),
'initial_img':
_bytes_feature(initial_img_array.tostring()),
'vel_conf':
_bytes_feature(vel_conf[k].tostring()),
'step':
_bytes_feature(step_array[k].tostring())
}))
writer.write(example.SerializeToString())
def main():
folders = get_folders(path=DATA_PATH)
p_mode_velocity_file = 'p_mode_velocities.csv'
scene_objects = 'scene_objects.npy'
success = 0
for i in range(507, 999):
folder = DATA_PATH + 'Trial_%d/' % i
with Interface() as I:
# LOAD ROBOT
robot = I.load_model(ROBOT_URDF_PATH)
# LOAD TABLE
table = I.load_model(TABLE,
Pose(Point(0.4, 0.0, 0.05),
Euler(1.57, 0.0, 0.0)),
fixed_base=True,
scaling=1.0)
# TODO set camera
I.set_camera_pose(0.6, 155, -50, [0.2, 0.0, 0.5])
scene = np.load(os.path.join(folder, scene_objects))[2:, 0]
block = I.load_model(CUBE_URDF,
Pose(Point(*scene[0])),
fixed_base=False)
basket = I.load_model(BASKET_URDF,
Pose(Point(*scene[1])),
fixed_base=False)
I.save_body_conf()
if os.path.isfile(os.path.join(folder, p_mode_velocity_file)):
velocities = Pd(os.path.join(
folder, p_mode_velocity_file)).read().values
I.replay_velocities(robot, velocities)
if inside_basket(I, block, basket):
I.restore_body_conf()
time.sleep(1.0)
I.replay_velocities(robot,
velocities,
filename=os.path.join(folder, 'img'))
success += 1
if success == 270:
break
print('Trials : ', folder, ' , Success: ', success)
def __init__(self,
sess,
width=256,
height=256,
channels=3,
learning_rate=0.0001,
z_shape=8,
j_shape=9,
model_name=None,
graph=None,
coeff=0.002):
self.sess = sess
self.width = width
self.height = height
self.channels = channels
self.learning_rate = learning_rate
self.z_shape = z_shape
self.j_shape = j_shape
self.img, self.p_t, self.p_t_1 = self.gen_network('generator')
# self.p_t = tf.placeholder(tf.float32, shape = (None, self.j_shape))
# self.std = tf.exp(self.logstds)
# self.var = self.std ** 2
# self.loss = 0.5 * tf.reduce_sum(tf.square((self.p_t - self.action_mean) / self.var)) \
# + 0.5 * np.log(2.0 * np.pi) * tf.to_float(tf.shape(self.p_t)[0]) \
# + tf.reduce_sum(self.logstds)
self.std = tf.exp(self.logstds)
# self.var = tf.square(self.std)
self.p_t_1_l = tf.placeholder(tf.float32, (None, 9))
self.mse_loss = tf.reduce_mean(tf.square(self.p_t_1 - self.p_t_1_l))
lh_loss = 0.5 * tf.reduce_sum(tf.square((self.p_t_1_l - self.p_t_1) / self.std), axis=-1) \
+ 0.5 * np.log(2.0 * np.pi) * tf.to_float(tf.shape(self.p_t_1)[-1]) + tf.reduce_sum(self.logstds, axis=-1)
self.lh_loss = tf.reduce_mean(lh_loss)
self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(
self.lh_loss)
self.model = model(sess, MODELS_DIR, model_name=model_name)
self.summary = self.model.summary(graph=graph,
**dict(mse_loss=self.mse_loss,
lh_loss=self.lh_loss))
self.eval_indice = len(get_folders(pathname(RES_PATH, **dict(flag=1))))
def run(self):
if self.verbose:
print(f'MainThread: {self.name} starting bp: {self.base_path}')
self.folder_list = get_folders(self.base_path)
if self.folder_list is not None:
self.update()
while True:
try:
# [0] f: new movie
# [0] s: scrape
# print(f'q items put {self.monitor_q.qsize()}')
q_item = self.monitor_q.get_nowait()
# new_movie = self.monitor_q.get_nowait()
if q_item[0] == 'f':
# new_movie.append(q_item[1])
new_movie = q_item[1]
self.import_from_path(new_movie)
# print(f'new_movie found: {new_movie}')
# self.grab_folder(new_movie)
# q_item.update
self.monitor_q.task_done()
# self.update()
# print(f'q items task done {self.monitor_q.qsize()}')
if q_item[0] == 's':
pass
except Empty:
# with concurrent.futures.ThreadPoolExecutor() as executor:
# ttp_res = executor.map(self.import_from_path, new_movie)
# new_movie = []
# print(f'q empty')
# time.sleep(0.5)
pass
if self.kill:
return
if self.verbose:
# self.get_status()
pass
def geo_split(
data_path=args.data_path,
markup_path=args.markup_path,
mask_type=args.mask_type,
masks_folder=args.masks_folder,
polygons_folder=args.polygons_folder,
test_threshold=0.2,
val_bottom_threshold=0.2,
val_threshold=0.3,
):
datasets = get_folders(data_path)
geojson_markup = gp.read_file(markup_path)
minY, maxY = get_height_bounds(geojson_markup)
height = maxY - minY
cols = ['dataset_folder', 'name', 'position']
train_df = pd.DataFrame(columns=cols)
val_df = pd.DataFrame(columns=cols)
test_df = pd.DataFrame(columns=cols)
overall_sizes = {'test': 0, 'train': 0, 'val': 0, 'deleted': 0}
for dataset_dir in datasets:
polys_path = os.path.join(data_path, dataset_dir, polygons_folder)
print(dataset_dir)
deleted = 0
train = 0
test = 0
val = 0
for poly_name in os.listdir(polys_path):
instance_geojson_path = os.path.join(polys_path, poly_name)
instance_geojson = gp.read_file(instance_geojson_path)
if geojson_markup.crs != instance_geojson.crs:
geojson_markup = geojson_markup.to_crs(instance_geojson.crs)
minY, maxY = get_height_bounds(geojson_markup)
height = maxY - minY
instance_minY, instance_maxY = get_height_bounds(instance_geojson)
name, position = get_instance_info(poly_name)
masks_path = os.path.join(data_path, dataset_dir, masks_folder)
mask_path = get_filepath(masks_path,
get_fullname(name, position),
file_type=mask_type)
mask = Image.open(mask_path)
mask_array = np.array(mask)
mask_pixels = np.count_nonzero(mask_array)
center_pixels = np.count_nonzero(mask_array[10:-10, 10:-10])
border_pixels = mask_pixels - center_pixels
if mask_pixels > mask_array.size * 0.001 and center_pixels > border_pixels:
if instance_maxY < minY + height * test_threshold:
test += 1
test_df = add_record(test_df,
dataset_folder=name,
name=name,
position=position)
elif instance_maxY < minY + height * val_threshold \
and instance_minY > minY + height * val_bottom_threshold:
val += 1
val_df = add_record(val_df,
dataset_folder=name,
name=name,
position=position)
else:
train += 1
train_df = add_record(train_df,
dataset_folder=name,
name=name,
position=position)
else:
deleted += 1
print("Train size", train, "Validation size", val, "Test size", test)
print(f"{deleted} images were deleted")
overall_sizes = update_overall_sizes(overall_sizes, test, train, val,
deleted)
print("Overall sizes", overall_sizes)
return train_df, val_df, test_df
from utils import get_folders,pathname
import os
# PATH = '/home/barry/hdd/IMITATION/training_data/STACK'
PATH = '/data/barry/IMITATION/dataset/THROW/DEMO'
# folders = get_folders(PATH, key=lambda f : int(f[6:]))[1000:]
folders = get_folders(PATH, key=lambda f : int(f[5:]))
# print(len(folders))
# rg=0
b = range(len(folders))
for fold in folders:
# os.rename(fold,pathname(PATH,'demo_%d'%rg))
# rg +=1
a = fold.split('/')[-1][5:]
if int(a) not in b:
print fold
# if rename:
# for i,fold in enumerate(sorted(folders,key = lambda file: int(file.split('/')[-1][6:]))):
# f = int(fold.split('/')[-1][6:])
#
# os.rename(os.path.join(PATH,'Trial_%d'%f ),
# os.path.join(PATH, 'Trial_%d' % i))
import utils
import save_arrays
import sys
if __name__ == '__main__':
name = "goals"
read_folder, save_path = utils.get_folders(sys.argv, name)
plans = utils.get_plans(read_folder)
goals = []
for p in plans:
if p.goals not in goals:
goals.append(p.goals)
save_arrays.save(goals, save_path)
print(len(goals))
)
parser.add_argument(
'--masks_path', '-mp', dest='masks_path',
default='masks',
help='Path to masks'
)
return parser.parse_args()
if __name__ == '__main__':
args = parse_args()
if not os.path.exists(args.save_path):
os.makedirs(args.save_path, exist_ok=True)
print("Save directory created.")
for tiff_name in get_folders(args.tiff_path):
print(tiff_name)
tiff_filepath = os.path.join(args.tiff_path, tiff_name)
tiff_file = join(args.save_path, f'{tiff_name}.tif')
data_path = os.path.join(args.save_path, basename(tiff_file[:-4]))
clouds_pieces_path = os.path.join(data_path, 'clouds')
masks_pieces_path = os.path.join(data_path, args.masks_path)
cloud_path = os.path.join(args.cloud_path, basename(tiff_file[:-4])+'_clouds.tiff')
pieces_info = os.path.join(data_path, 'image_pieces.csv')
if not os.path.exists(clouds_pieces_path):
os.mkdir(clouds_pieces_path)
if not os.path.exists(masks_pieces_path):
print('No masks_pieces_path directory was found')
sys.exit()
split_cloud(cloud_path, clouds_pieces_path, pieces_info, masks_pieces_path)
def update_folders(self):
# scan base folder for movie folders
self.folder_list = get_folders(self.base_path)
