class Pushability:
def __init__(self):
(self.mean, self.std, self.norm,
self.norm_inv) = load_models('saved_models/pushability_models.pkl')
# For mahalanobis, the queried robot state needs normalizing
self.y_norm = Normalization(3)
self.y_norm.load_state_dict(self.norm_inv.state_dict())
def mahalanobis(self, request):
XY = features(request)
# XY is:
# [0, 1] - cosine and sine of initial object rotation
# [2, 3] - translation of object x, y
# [4] - change of rotation
# [5, 6, 7, 8, 9] - object properties
X_ = XY[:, 5:]
Y_ = XY[:, 2:5]
X = X_ * 0.0
Y = self.y_norm(Y_)
µ = self.mean(X)
σ = self.std(X)
response = PushabilityResponse()
response.mahalanobis = torch.norm((µ - Y) / σ).data[0]
# These are not relevant here
response.projected_object_x = 0.0
response.projected_object_y = 0.0
response.projected_object_radians = 0.0
return response
def projection(self, request):
XY = features(request)
n_stds = request.num_stds
# XY is:
# [0, 1] - cosine and sine of initial object rotation
# [2, 3] - translation of object x, y
# [4] - change of rotation
# [5, 6, 7, 8, 9] - object properties
X_ = XY[:, 5:]
Y_ = XY[:, 2:5]
X = X_ * 0.0
Y = self.y_norm(Y_)
µ = self.mean(X)
σ = self.std(X)
mahalanobis = torch.norm((µ - Y) / σ)
if mahalanobis.data[0] < 1.0:
y = self.norm_inv(µ + (Y - µ)).data
else:
y = self.norm_inv(µ + (Y - µ) / mahalanobis * n_stds).data
response = PushabilityResponse()
# This is not relevant now
response.mahalanobis = -1
# Actual response
response.projected_object_x = request.object_x + y[0, 0]
response.projected_object_y = request.object_y + y[0, 1]
response.projected_object_radians = request.object_radians + y[0, 2]
return response
def __init__(self):
(self.mean, self.std, self.norm,
self.norm_inv) = load_models('saved_models/pushability_models.pkl')
# For mahalanobis, the queried robot state needs normalizing
self.y_norm = Normalization(3)
self.y_norm.load_state_dict(self.norm_inv.state_dict())
def load_models(path):
with open(path, 'rb') as f:
model_dict = pickle.load(f)
x_size = model_dict['n_features_in']
y_size = model_dict['n_features_out']
n_hidden_units = model_dict['n_hidden_units']
n_residual_units = model_dict['n_residual_units']
norm = Normalization(x_size)
norm.load_state_dict(model_dict['norm'])
norm_inv = NormalizationInverse(y_size)
norm_inv.load_state_dict(model_dict['norm_inv'])
mean = torch.nn.Sequential(
torch.nn.Linear(x_size, n_hidden_units),
*[Residual(n_hidden_units) for _ in range(n_residual_units)],
torch.nn.Linear(n_hidden_units, y_size))
mean.eval()
mean.load_state_dict(model_dict['mean_model'])
std = torch.nn.Sequential(
torch.nn.Linear(x_size, n_hidden_units),
*[Residual(n_hidden_units) for _ in range(n_residual_units)],
torch.nn.Linear(n_hidden_units, y_size), torch.nn.Softplus())
std.eval()
std.load_state_dict(model_dict['std_model'])
return mean, std, norm, norm_inv
def preprocess(self, pil_img):
if len(pil_img.shape) == 3:
pil_img = Normalization(pil_img)
if len(pil_img.shape) == 2:
pil_img = np.expand_dims(pil_img, axis=2)
# HWC to CHW
img_trans = pil_img.transpose((2, 0, 1))
return img_trans.astype(float)
def predict():
net = UNet(n_channels=3, n_classes=2)
# net = ULeNet(n_channels=3, n_classes=6)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
net.to(device=device)
net.load_state_dict(torch.load(model, map_location=device))
evaluator = Evaluator(num_class=2)
evaluator.reset()
for img in os.listdir(dir_img):
image = cv2.imread(os.path.join(dir_img, img))
# mask = cv2.imread(os.path.join(dir_mask, img))
# mask = cv2.cvtColor(mask, cv2.COLOR_RGB2GRAY)
mask = Image.open(os.path.join(dir_mask, img))
mask = mask.convert('L')
mask = np.array(mask)
mask_class = np.unique(mask)
for i in range(len(mask_class)):
mask[mask == mask_class[i]] = i
predict = np.zeros((image.shape[0], image.shape[1]))
p_size = 128
for i in range(0, image.shape[0] - p_size, p_size):
for j in range(0, image.shape[1] - p_size, p_size):
patch = image[i:i + p_size, j:j + p_size, :]
patch = Normalization(patch)
predict[i:i + p_size, j:j + p_size] = getPredict(net=net,
img=patch,
device=device)
evaluator.add_batch(mask, predict)
# mIoU = evaluator.Mean_Intersection_over_Union()
predict = Image.fromarray((predict).astype(np.uint8))
predict.save(
os.path.join(dir_predict,
os.path.splitext(img)[0] + '.tif'))
show_confMat(evaluator.conf_mat, [str(c) for c in range(2)],
re.split('[/.]', model)[1], dir_confmat)
def get_style_model_and_losses(cnn,
normalization_mean,
normalization_std,
style_img,
content_img,
content_layers=content_layers,
style_layers=style_layers):
cnn = copy.deepcopy(cnn)
normalization = Normalization(normalization_mean,
normalization_std).to(device)
content_losses = []
style_losses = []
model = nn.Sequential(normalization)
i = 0 # Control variable to count Conv layers passed
for layer in cnn.children():
if isinstance(layer, nn.Conv2d):
i += 1
name = 'conv_{}'.format(i)
elif isinstance(layer, nn.ReLU):
name = 'relu_{}'.format(i)
layer = nn.ReLU(inplace=False)
elif isinstance(layer, nn.MaxPool2d):
name = 'pool_{}'.format(i)
elif isinstance(layer, nn.BatchNorm2d):
name = 'bn_{}'.format(i)
else:
raise RuntimeError('Unrecognized layer: {}'.format(
layer.__class__.__name__))
model.add_module(name, layer)
# Add Content Loss
if name in content_layers:
target = model(content_img).detach()
content_loss = ContentLoss(target)
model.add_module("content_loss_{}".format(i), content_loss)
content_losses.append(content_loss)
# Add Style Loss
if name in style_layers:
target_feature = model(style_img).detach()
style_loss = StyleLoss(target_feature)
model.add_module("style_loss_{}".format(i), style_loss)
style_losses.append(style_loss)
# now we trim off the layers after the last content and style losses
for i in range(len(model) - 1, -1, -1):
if isinstance(model[i], ContentLoss) or isinstance(
model[i], StyleLoss):
break
model = model[:(i + 1)]
return model, style_losses, content_losses
def __init__(self, args, apply_mask=False, layer=['conv_1']):
super().__init__()
self.perc_wgt = args.perc_wgt
self.apply_mask = apply_mask
self.perc = PerceptualLoss(layer)
vgg = vgg19(pretrained=True).features.to(device).eval()
# normalization
norm_mean = torch.tensor([0.485, 0.456, 0.406]).to(device)
norm_std = torch.tensor([0.229, 0.224, 0.225]).to(device)
normalization = Normalization(norm_mean, norm_std)
self.model = nn.Sequential(normalization)
# load sketch
self.transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
])
# total_sketches = glob.glob("data/sketch/5.png")
# img_loc = total_sketches[0]
# sketch = Image.open(img_loc)
# sketch = sketch.resize((64, 64))
# rgb = sketch.convert("RGB")
# self.sketch = self.transform(rgb).to(device)
i = 0
for vgglayer in vgg.children():
if isinstance(vgglayer, nn.Conv2d):
i += 1
name = 'conv_{}'.format(i)
elif isinstance(vgglayer, nn.ReLU):
name = 'relu_{}'.format(i)
vgglayer = nn.ReLU(inplace=False)
elif isinstance(vgglayer, nn.MaxPool2d):
name = 'pool_{}'.format(i)
elif isinstance(vgglayer, nn.BatchNorm2d):
name = 'bn_{}'.format(i)
else:
raise RuntimeError('Unrecognized layer: {}'.format(
vgglayer.__class__.__name__))
self.model.add_module(name, vgglayer)
if name == layer[0]:
break
print("Criteria Model", self.model)
def get_style_model_and_loss(base_model,
base_mean,
base_std,
style_img,
content_img,
content_layers=content_layers,
style_layers=style_layers):
base_model = copy.deepcopy(
base_model) # Separate Object that doesn't affect each other
norm = Normalization(base_mean, base_std).to(device)
content_losses, style_losses = [], []
model = nn.Sequential(norm) # First Layer = Normalization Layer
i = 0 # Count CNN layers
for layer in base_model.children():
if isinstance(layer, nn.Conv2d):
i += 1
name = "conv_{}".format(i)
elif isinstance(layer, nn.ReLU):
name = "relu_{}".format(i)
layer = nn.ReLU(inplace=False)
elif isinstance(layer, nn.MaxPool2d):
name = "pool_{}".format(i)
elif isinstance(layer, nn.BatchNorm2d):
name = "bn_{}".format(i)
else:
raise RuntimeError("Unrecognized layer: {}".format(
layer.__class__.__name__))
model.add_module(
name, layer
) # Sequentially stack layers of VGG to our new model (Copy most of them, and insert Losses in the right place)
if name in content_layers:
target = model(
content_img).detach() # Feature map of content img so far
content_loss = ContentLoss(target) # input is directly fed
model.add_module(
"content_loss_{}".format(i), content_loss
) # Add a layer that computes loss and returns the original input (Like identity operation in a sense)
content_losses.append(content_loss)
if name in style_layers:
target_feature = model(style_img).detach()
style_loss = StyleLoss(target_feature)
model.add_module(
"style_loss_{}".format(i), style_loss
) # Again, compute the gradient and returns input as is
style_losses.append(style_loss)
# Get rid of unnecessary layers after style and content loss
for i in range(len(model) - 1, -1, -1):
if isinstance(model[i], ContentLoss) or isinstance(
model[i], StyleLoss):
break
model = model[:(i + 1)]
return model, style_losses, content_losses
def __init__(self):
super(GeneratorProduction, self).__init__()
self.generator = FeasibilityGenerator()
self.obj_norm = Normalization(7)
self.rob_norminv = NormalizationInverse(4)
from utils import Dataset, ToTensor, Normalization, RandomFlip, saver, loader
## 트레이닝 파라메터 설정하기
lr = 1e-3
batch_size = 4
num_epoch = 100
data_dir = './data'
ckpt_dir = './checkpoint'
log_dir = './log'
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
## Call Transform
transform = transforms.Compose(
[Normalization(mean=0.5, std=0.5),
RandomFlip(), ToTensor()])
## 데이터셋 생성
dataset_train = Dataset(data_dir=os.path.join(data_dir, 'train'),
transform=transform)
loader_train = DataLoader(dataset_train,
batch_size=batch_size,
shuffle=True,
num_workers=4)
dataset_val = Dataset(data_dir=os.path.join(data_dir, 'val'),
transform=transform)
loader_val = DataLoader(dataset_val,
batch_size=batch_size,
shuffle=False,
def get_style_model_and_losses_lists(self, style_img, content_img, spatial_mask=None):
"""Returns the model, content and style losses with implemented receptive averaging.
Parameters
----------
style_img : list of torch.tensor
The images whose style shall be transfered
content_img : torch.tensor
The image on which the style transfer shall be executed
spatial_mask : list torch.tensor, optional
The masks which determine in which area of the image which masks shall be applied (default is None)
"""
if spatial_mask is None:
raise ValueError("A spatial mask must be provided")
if len(spatial_mask) != len(style_img):
raise ValueError("Number of spatial masks and number of style images must be equal")
style_img = torch.stack(style_img).squeeze(1)
cnn = copy.deepcopy(self.cnn)
# normalization module
normalization = Normalization().to(self.device)
# just in order to have an iterable access to or list of content/syle losses
content_losses = []
style_losses = []
# assuming that cnn is a nn.Sequential, so we make a new nn.Sequential
# to put in modules that are supposed to be activated sequentially
model = nn.Sequential(normalization)
# A list with the spatial masks for the style loss
receptive_masks = []
layer_mask = spatial_mask
receptive_masks.append(layer_mask)
i = 0 # increment every time we see a conv
for layer in cnn.children():
if isinstance(layer, nn.Conv2d):
i += 1
name = 'conv_{}'.format(i)
r_avg = nn.AvgPool2d(layer.kernel_size, layer.stride, layer.padding)
layer_mask = r_avg(layer_mask)
elif isinstance(layer, nn.ReLU):
name = 'relu_{}'.format(i)
layer = nn.ReLU(inplace=False)
elif isinstance(layer, nn.MaxPool2d):
name = 'pool_{}'.format(i)
r_max = nn.MaxPool2d(layer.kernel_size, layer.stride, layer.padding)
layer_mask = r_max(layer_mask)
elif isinstance(layer, nn.BatchNorm2d):
name = 'bn_{}'.format(i)
else:
raise RuntimeError('Unrecognized layer: {}'.format(layer.__class__.__name__))
model.add_module(name, layer)
if name in self.content_layers:
# add content loss:
target = model(content_img).detach()
content_loss = ContentLoss(target)
model.add_module("content_loss_{}".format(i), content_loss)
content_losses.append(content_loss)
if name in self.style_layers:
target_feature = model(style_img).detach()
style_loss = StyleLoss(target_feature, layer_mask)
model.add_module("style_loss_{}".format(i), style_loss)
style_losses.append(style_loss)
# now we trim off the layers after the last content and style losses
for i in range(len(model) - 1, -1, -1):
if isinstance(model[i], ContentLoss) or isinstance(model[i], StyleLoss):
break
model = model[:(i + 1)]
return model, style_losses, content_losses
from vgg import load_vgg16
import sys
import getopt
from tqdm import tqdm
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print("Working on ", device)
imsize = 800 if torch.cuda.is_available() else 128
normalization_mean = torch.tensor([0.485, 0.456, 0.406]).to(device)
normalization_std = torch.tensor([0.229, 0.224, 0.225]).to(device)
normalization = Normalization(normalization_mean, normalization_std)
normalization = normalization.to(device)
content_layers_default = ['conv1', 'conv_4']
style_layers_default = ['conv_1', 'conv_2', 'conv_3', 'conv_4', 'conv_5']
def get_style_model_and_losses(parent_model,
style_img, content_img,
content_layers=content_layers_default,
style_layers=style_layers_default):
parent_model = copy.deepcopy(parent_model)
content_losses = []
style_losses = []