def save(self):
for idx, (name, array) in enumerate(
zip([
'actions', 'rewards', 'screens', 'terminals', 'prestates',
'poststates'
], [
self.actions, self.rewards, self.screens, self.terminals,
self.prestates, self.poststates
])):
save_npy(array, os.path.join(self.model_dir, name))
def main():
csv_path = 'data/all_test_clean.csv'
tweets, targets, labels = load_csv(csv_path)
print('--- LOADED CSV ---')
model = load_bert()
print('--- LOADED MODEL ---')
preds = predict(model, tweets, targets)
save_npy(preds, 'ada_bert', 'preds/')
print('--- SAVED PREDS ---')
print_metrics(preds, labels, 'ada_bert')
def classify_sentiment(model, tweets, neg_label, name):
n_chunks = 10
chunk_size = len(tweets) // n_chunks
chunks = [
tweets[x:x + chunk_size] for x in range(0, len(tweets), chunk_size)
]
preds = list()
for chunk_idx, chunk in enumerate(chunks):
chunk_out = model(chunk)
preds += [0 if out['label'] == neg_label else 1 for out in chunk_out]
print('Chunk', chunk_idx, 'processed')
save_npy(preds, name, 'preds/')
return preds
def train():
"""
Training function
Adapted from https://github.com/jesseniagonzalezv/App_segmentation_water_bodies/
"""
parser = argparse.ArgumentParser()
arg = parser.add_argument
# image-related variables
arg('--image-patches-dir', type=str, default='./data/dataset/split', help='satellite image patches directory')
arg('--masks-dir', type=str, default='./data/dataset/labels', help='numPy masks directory')
arg('--npy-dir', type=str, default='./data/dataset/split_npy', help='numPy preprocessed patches directory')
# preprocessing-related variables
arg('--val-percent', type=float, default=0.25, help='Validation percent')
arg('--test-percent', type=float, default=0.10, help='Test percent')
# training-related variable
arg('--batch-size', type=int, default=16, help='HR:4,VHR:8')
arg('--limit', type=int, default=0, help='number of images in epoch')
arg('--n-epochs', type=int, default=500)
arg('--lr', type=float, default=1e-3)
arg('--step', type=float, default=60)
arg('--model', type=str, help='roof: roof segmentation / income: income determination')
arg('--out-path', type=str, default='./trained_models/', help='model output path')
arg('--pretrained', type=int, default=1, help='0: False; 1: True')
# CUDA devices
arg('--device-ids', type=str, default='0,1', help='For example 0,1 to run on two GPUs')
args = parser.parse_args()
pretrained = True if args.pretrained else False
if args.model == "roof":
model = models.UNet11(pretrained=pretrained)
elif args.model == "income":
model = models.UNet11(pretrained=pretrained, num_classes=4, input_channels=5)
else:
raise ValueError
if torch.cuda.is_available():
if args.device_ids:
device_ids = list(map(int, args.device_ids.split(',')))
else:
device_ids = None
model = torch.nn.DataParallel(model, device_ids=device_ids).cuda()
cudnn.benchmark = True
images_filenames = np.array(sorted(glob.glob(args.image_patches_dir + "/*.tif")))
train_set_indices, val_set_indices, test_set_indices = utils.train_val_test_split(len(images_filenames),
args.val_percent,
args.test_percent)
images_np_filenames = utils.save_npy(images_filenames, args.npy_dir, args.model, args.masks_dir)
channel_num = 4 if args.model == "roof" else 5
max_value, mean_train, std_train = utils.meanstd(np.array(images_np_filenames)[train_set_indices],
channel_num=channel_num)
train_transform = DualCompose([
HorizontalFlip(),
VerticalFlip(),
Rotate(),
ImageOnly(Normalize(mean=mean_train, std=std_train))
])
val_transform = DualCompose([
ImageOnly(Normalize(mean=mean_train, std=std_train))
])
limit = args.limit if args.limit > 0 else None
train_loader = utils.make_loader(filenames=np.array(images_np_filenames)[train_set_indices],
mask_dir=args.masks_dir,
dataset=args.model,
shuffle=False,
transform=train_transform,
mode='train',
batch_size=args.batch_size,
limit=limit)
valid_loader = utils.make_loader(filenames=np.array(images_np_filenames)[val_set_indices],
mask_dir=args.masks_dir,
dataset=args.model,
shuffle=False,
transform=val_transform,
mode='train',
batch_size=args.batch_size,
limit=None)
dataloaders = {
'train': train_loader, 'val': valid_loader
}
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=args.step, gamma=0.1)
name_file = '_' + str(int(args.val_percent * 100)) + '_percent_' + args.model
utils.train_model(name_file=name_file,
model=model,
dataset=args.model,
optimizer=optimizer,
scheduler=scheduler,
dataloaders=dataloaders,
name_model="Unet11",
num_epochs=args.n_epochs)
if not os.path.exists(args.out_path):
os.mkdir(args.out_path)
torch.save(model.module.state_dict(),
(str(args.out_path) + '/model{}_{}_{}epochs').format(name_file, "Unet11", args.n_epochs))
find_metrics(train_file_names=np.array(images_np_filenames)[train_set_indices],
val_file_names=np.array(images_np_filenames)[val_set_indices],
test_file_names=np.array(images_np_filenames)[test_set_indices],
mask_dir=args.masks_dir,
dataset=args.model,
mean_values=mean_train,
std_values=std_train,
model=model,
name_model="Unet11",
epochs=args.n_epochs,
out_file=args.model,
dataset_file=args.model,
name_file=name_file)
def save(self):
for idx, (name, array) in enumerate(
zip(['actions', 'rewards', 'screens', 'terminals', 'prestates', 'poststates'],
[self.actions, self.rewards, self.screens, self.terminals, self.prestates, self.poststates])):
save_npy(array, os.path.join(self.model_dir, name))
def test_test(self,
save_dir,
is_debug=False,
max_degree=10,
interval=1,
resize_factor=0.25):
s2_imgs, s3_imgs, s4_imgs = None, None, None
batch = len(range(-max_degree, max_degree + 1, interval))
for iter_time in range(self.data.numTestImgs):
print('[*] Multi-test processing {} / {}...'.format(
iter_time + 1, self.data.numTestImgs))
s1_imgs, img_name, user_id = self.sess.run([
self.model.imgTests, self.model.img_name_test,
self.model.user_id_test
])
num_imgs, h, w, c = s1_imgs.shape
print('[*] Stage 1: Forwar the network {} times... '.format(
int(num_imgs / batch)))
s1_preds = list()
for i in range(0, num_imgs, batch):
pred_imgs = self.sess.run(
self.model.predTest,
feed_dict={
self.model.ratePh: 0., # keep_ratio = 1 - rate
self.model.trainMode: False,
self.model.inputImgPh: s1_imgs[i:i + batch]
})
s1_preds.extend(pred_imgs)
s1_preds = np.asarray(s1_preds)
if is_debug:
# Save stage 1 results: the rotated, flipped, and cropped images
utils.save_imgs_indiv(s1_imgs,
w_num_imgs=batch,
save_dir=os.path.join(save_dir, 'debug'),
img_name=img_name.astype('U26'),
is_label=False,
name_append='s1_img_',
factor=resize_factor)
utils.save_imgs_indiv(s1_preds,
w_num_imgs=batch,
save_dir=os.path.join(save_dir, 'debug'),
img_name=img_name.astype('U26'),
is_label=True,
name_append='s1_seg_',
factor=resize_factor)
# Save stage 2 results: the flipped and cropped images
print('[*] Stage 2: Processing inverse-rotation...')
s2_preds = utils.inverse_rotate(s1_preds,
batch=batch,
interval=interval,
is_label=True)
if is_debug:
s2_imgs = utils.inverse_rotate(s1_imgs,
batch=batch,
interval=interval,
is_label=False)
utils.save_imgs_indiv(s2_imgs,
w_num_imgs=batch,
save_dir=os.path.join(save_dir, 'debug'),
img_name=img_name.astype('U26'),
is_label=False,
name_append='s2_img_',
factor=resize_factor)
utils.save_imgs_indiv(s2_preds,
w_num_imgs=batch,
save_dir=os.path.join(save_dir, 'debug'),
img_name=img_name.astype('U26'),
is_label=True,
name_append='s2_seg_',
factor=resize_factor)
# Save stage 3 results: the cropped images
print('[*] Stage 3: Processing inverse-flipping...')
s3_preds = utils.inverse_flip(s2_preds)
if is_debug:
s3_imgs = utils.inverse_flip(s2_imgs)
utils.save_imgs_indiv(s3_imgs,
w_num_imgs=batch,
save_dir=os.path.join(save_dir, 'debug'),
img_name=img_name.astype('U26'),
is_label=False,
name_append='s3_img_',
factor=resize_factor)
utils.save_imgs_indiv(s3_preds,
w_num_imgs=batch,
save_dir=os.path.join(save_dir, 'debug'),
img_name=img_name.astype('U26'),
is_label=True,
name_append='s3_seg_',
factor=resize_factor)
# Save stage 4 results
print('[*] Stage 4: Processing inverse-cropping...')
s4_preds = utils.inverse_cropping(s3_preds,
batch=batch,
is_label=True)
if is_debug:
s4_imgs = utils.inverse_cropping(s3_imgs,
batch=batch,
is_label=False)
utils.save_imgs_indiv(s4_imgs,
w_num_imgs=batch,
save_dir=os.path.join(save_dir, 'debug'),
img_name=img_name.astype('U26'),
is_label=False,
name_append='s4_img_',
factor=resize_factor)
utils.save_imgs_indiv(s4_preds,
w_num_imgs=batch,
save_dir=os.path.join(save_dir, 'debug'),
img_name=img_name.astype('U26'),
is_label=True,
name_append='s4_seg_',
factor=resize_factor)
# Save stage 5 results
print('[*] Stage 5: Calculaing the final prediction...')
predCls = np.argmax(np.sum(s4_preds, axis=0, keepdims=True),
axis=3)
utils.save_imgs(img_stores=[
np.expand_dims(s1_imgs[int(batch * 0.5)], axis=0),
np.expand_dims(np.argmax(s4_preds[int(batch * 0.5)], axis=2),
axis=0), predCls
],
saveDir=os.path.join(save_dir, 'debug'),
img_name=img_name.astype('U26'),
is_vertical=False)
# Save images
print('[*] Saving the input and prediction image...')
utils.save_imgs(img_stores=[
np.expand_dims(s1_imgs[int(batch * 0.5)], axis=0), predCls
],
saveDir=save_dir,
img_name=img_name.astype('U26'),
is_vertical=False)
# Write as npy format
print('[*] Saving .npy files for submission...')
utils.save_npy(data=predCls,
save_dir=save_dir,
file_name=img_name.astype('U26'))