def __call__(self, draw = False):
# Load model
net = Net()
# Trained model
net.load_state_dict(torch.load('./saved_models/model_checkpoint_kpd.pt')['model'])
## print out your net and prepare it for testing (uncomment the line below)
net.eval()
## Data preparation
transformations = transforms.Compose([Rescale(250),
RandomCrop(224),
Normalize(),
ToTensor()])
# create the transformed dataset
transformed_data = KeypointsIterableDataset(self.image, self.keypoints, transform=transformations)
data_loader = DataLoader(transformed_data, num_workers=0)
## if train flag is set, Start training picking up old checkpoint
if self.train:
print("Training...")
## Run each record twice for training.
n_epochs = 2
train_net(n_epochs, data_loader, net)
## Get the prediction
print("Predicting...")
test_images, test_pts, gt_pts, sample = test_net(data_loader, net)
if draw:
visualize_output(test_images, test_pts)
# Rescaled.
return InverseTransform()(sample)
class BasicThymio:
def __init__(self, thymio_name):
"""init"""
self.net = Net()
self.net.load_state_dict(torch.load('./cnn', 'cpu'))
self.net.eval()
self.thymio_name = thymio_name
self.transform = transforms.Compose(
[
transforms.ToPILImage(),
transforms.ToTensor()
])
rospy.init_node('basic_thymio_controller', anonymous=True)
time.sleep(5)
self.velocity_publisher = rospy.Publisher(self.thymio_name + '/cmd_vel',
Twist, queue_size=10)
self.pose_subscriber = rospy.Subscriber(self.thymio_name + '/odom',
Odometry, self.update_state)
self.camera_subscriber = rospy.Subscriber(self.thymio_name + '/camera/image_raw',
Image, self.update_image, queue_size=1)
self.current_pose = Pose()
self.current_twist = Twist()
self.rate = rospy.Rate(10)
def thymio_state_service_request(self, position, orientation):
"""Request the service (set thymio state values) exposed by
the simulated thymio. A teleportation tool, by default in gazebo world frame.
Be aware, this does not mean a reset (e.g. odometry values)."""
rospy.wait_for_service('/gazebo/set_model_state')
try:
model_state = ModelState()
model_state.model_name = self.thymio_name
model_state.reference_frame = '' # the frame for the pose information
model_state.pose.position.x = position[0]
model_state.pose.position.y = position[1]
model_state.pose.position.z = position[2]
qto = quaternion_from_euler(
orientation[0], orientation[1], orientation[2], axes='sxyz')
model_state.pose.orientation.x = qto[0]
model_state.pose.orientation.y = qto[1]
model_state.pose.orientation.z = qto[2]
model_state.pose.orientation.w = qto[3]
# a Twist can also be set but not recomended to do it in a service
gms = rospy.ServiceProxy('/gazebo/set_model_state', SetModelState)
response = gms(model_state)
return response
except rospy.ServiceException, e:
print "Service call failed: %s" % e
class ModelLoader():
# Fill the information for your team
team_name = 'LAG'
team_member = ["Sree Gowri Addepalli"," Amartya prasad", "Sree Lakshmi Addepalli"]
round_number = 1
contact_email = '*****@*****.**'
def __init__(self, model_file="baseline1.pth"):
# You should
# 1. create the model object
# 2. load your state_dict
# 3. call cuda()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
self.model = Net()
self.mdl = NetObj()
self.modelObj = self.mdl.model
if torch.cuda.device_count() > 1:
print("Let's use", torch.cuda.device_count(), "GPUs!")
self.model = nn.DataParallel(self.model)
self.modelObj = nn.DataParallel(self.modelObj)
checkpoint = torch.load("baseline1.pth")
self.state_dict_1 = checkpoint['modelRoadMap_state_dict']
self.state_dict_2 = checkpoint['modelObjectDetection_state_dict']
self.model.load_state_dict(self.state_dict_1)
self.modelObj.load_state_dict(self.state_dict_2)
self.model.eval()
self.modelObj.eval()
self.model.to(device)
self.modelObj.to(device)
def get_bounding_boxes(self,samples):
# samples is a cuda tensor with size [batch_size, 6, 3, 256, 306]
# You need to return a tuple with size 'batch_size' and each element is a cuda tensor [N, 2, 4]
# where N is the number of object
batch_size = list(samples.shape)[0]
# Convert it into [batch_size, 3, 512, 918]
img_tensor = self.combine_images(samples,batch_size)
tup_boxes = []
with torch.no_grad():
for img in img_tensor:
prediction = self.modelObj([img.cuda()])
cbox = self.convertBoundingBoxes(prediction[0]['boxes'])
#print(cbox.shape)
tup_boxes.append(cbox)
return tuple(tup_boxes)
def get_binary_road_map(self,samples):
# samples is a cuda tensor with size [batch_size, 6, 3, 256, 306]
# You need to return a cuda tensor with size [batch_size, 800, 800]
with torch.no_grad():
batch_size = list(samples.shape)[0]
sample = samples.reshape(batch_size,18,256,306)
output = self.model(sample)
#print(output.shape)
output = output.reshape(800,800)
return output
def combine_images(self, samples, batch_size):
# samples is a cuda tensor with size [batch_size, 6, 3, 256, 306]
# You need to return a tuple with size 'batch_size' and each element is a cuda tensor [N, 2, 4]
# where N is the number of object
ss = samples.reshape(batch_size, 2, 3, 3, 256, 306)
t = ss.detach().cpu().clone().numpy().transpose(0, 3, 2, 1, 4, 5)
# MergingImage
tp = np.zeros((batch_size, 3, 3, 512, 306))
for i in range(0, batch_size):
for j in range(0, 3):
for k in range(0, 3):
tp[i][j][k] = np.vstack([t[i][j][k][0], t[i][j][k][1]])
tr = np.zeros((batch_size, 3, 512, 918))
for i in range(0, batch_size):
for j in range(0, 3):
tr[i][j] = np.hstack([tp[i][j][0], tp[i][j][1], tp[i][j][2]])
image_tensor = torch.from_numpy(tr).float()
return image_tensor
def convertBoundingBoxes(self, boxes):
# convert [N,1,4] to [N,2,4]
if len(boxes) == 0:
boxes = [[0,0,0,0]]
convBoxes = []
for box in boxes:
xmin = box[0]
xmin = (xmin - 400)/10
ymin = box[1]
ymin = (-ymin +400)/10
xmax = box[2]
xmax = (xmax - 400)/10
ymax = box[3]
ymax = (-ymax + 400)/10
cbox = [[xmin,xmin,xmax,xmax], [ymin,ymax,ymin,ymax]]
convBoxes.append(cbox)
convBoxes = torch.Tensor(convBoxes)
return convBoxes
class Trainer:
def __init__(self, hidden_dim, buffer_size, gamma, batch_size, device, writer):
self.env = make("connectx", debug=True)
self.device = device
self.policy = Net(self.env.configuration.columns * self.env.configuration.rows, hidden_dim,
self.env.configuration.columns).to(
device)
self.target = Net(self.env.configuration.columns * self.env.configuration.rows, hidden_dim,
self.env.configuration.columns).to(
device)
self.enemyNet = Net(self.env.configuration.columns * self.env.configuration.rows, hidden_dim,
self.env.configuration.columns).to(
device)
self.target.load_state_dict(self.policy.state_dict())
self.target.eval()
self.buffer = ExperienceReplay(buffer_size)
self.enemy = "random"
self.trainingPair = self.env.train([None, self.enemy])
self.loss_function = nn.MSELoss()
self.optimizer = optim.Adam(params=self.policy.parameters(), lr=0.001)
self.gamma = gamma
self.batch_size = batch_size
self.first = True
self.player = 1
self.writer = writer
def agent(self, observation, configuration):
with torch.no_grad():
state = torch.tensor(observation['board'], dtype=torch.float)
reshaped = self.reshape(state)
action = self.takeAction(self.enemyNet(reshaped).view(-1), reshaped, 0, False)
return action
def switch(self):
self.trainingPair = self.env.train([None, "negamax"])
self.enemy = "negamax"
def switchPosition(self):
self.env.reset()
if self.first:
self.trainingPair = self.env.train([self.enemy, None])
self.player = 2
else:
self.trainingPair = self.env.train([None, self.enemy])
self.player = 1
self.first = not self.first
def load(self, path):
self.policy.load_state_dict(torch.load(path))
def synchronize(self):
self.target.load_state_dict(self.policy.state_dict())
def save(self, name):
torch.save(self.policy.state_dict(), name)
def reset(self):
self.env.reset()
return self.trainingPair.reset()
def step(self, action):
return self.trainingPair.step(action)
def addExperience(self, experience):
self.buffer.append(experience)
def epsilon(self, maxE, minE, episode, lastEpisode):
return (maxE - minE) * max((lastEpisode - episode) / lastEpisode, 0) + minE
def change_reward(self, reward, done):
if done and reward == 1:
return 10
if done and reward == -1:
return -10
if reward is None and done:
return -20
if done:
return 1
if reward == 0:
return 1 / 42
else:
return reward
def change_reward_streak(self, reward, done, reshapedBoard, action, useStreak):
if done and reward == 1:
return 20
if done and reward == -1:
return -20
if reward is None and done:
return -40
if done:
return 1
if reward == 0 & useStreak:
return 1 / 42 + self.streakReward(self.player, reshapedBoard, action)
if reward == 0:
return 1 / 42
else:
return reward
def streakReward(self, player, reshapedBoard, action):
verticalReward = 0
horizontalReward = 0
if self.longestVerticalStreak(player, reshapedBoard, action) == 3:
verticalReward = 3
if self.longestHorizontalStreak(player, reshapedBoard, action) == 3:
horizontalReward = 3
return verticalReward + horizontalReward + self.longestDiagonalStreak(player, reshapedBoard, action)
def longestVerticalStreak(self, player, reshapedBoard, action):
count = 0
wasZero = False
for i in range(5, 0, -1):
if reshapedBoard[0][player][i][action] == 0:
wasZero = True
if reshapedBoard[0][player][i][action] == 1 & wasZero:
count = 0
wasZero = False
count += reshapedBoard[0][player][i][action]
if reshapedBoard[0][0][0][action] == 0:
return 0
return count
def longestHorizontalStreak(self, player, reshapedBoard, action):
count = 0
rowOfAction = self.rowOfAction(player, reshapedBoard, action)
wasZero = False
for i in range(7):
if reshapedBoard[0][player][rowOfAction][i] == 0:
wasZero = True
if reshapedBoard[0][player][rowOfAction][i] == 1 & wasZero:
count = 0
wasZero = False
count += reshapedBoard[0][player][rowOfAction][i]
return count
def longestDiagonalStreak(self, player, reshapedBoard, action):
rowOfAction = self.rowOfAction(player, reshapedBoard, action)
for row in range(4):
for col in range(5):
if reshapedBoard[0][player][row][col] == reshapedBoard[0][player][row + 1][col + 1] == \
reshapedBoard[0][player][row + 2][col + 2] == 1 and self.actionInDiagonal1(action, row, col,
rowOfAction):
return 3
for row in range(5, 1, -1):
for col in range(4):
if reshapedBoard[0][player][row][col] == reshapedBoard[0][player][row - 1][col + 1] == \
reshapedBoard[0][player][row - 2][col + 2] == 1 and self.actionInDiagonal2(action, row, col,
rowOfAction):
return 3
return 0
def actionInDiagonal1(self, action, row, col, rowOfAction):
return (rowOfAction == row and action == col or
rowOfAction == row + 1 and action == col + 1 or
rowOfAction == row + 2 and action == col + 2)
def actionInDiagonal2(self, action, row, col, rowOfAction):
return (rowOfAction == row and action == col or
rowOfAction == row - 1 and action == col + 1 or
rowOfAction == row - 2 and action == col + 2)
def rowOfAction(self, player, reshapedBoard, action):
rowOfAction = 10
for i in range(6):
if reshapedBoard[0][player][i][action] == 1:
rowOfAction = min(i, rowOfAction)
return rowOfAction
def policyAction(self, board, episode, lastEpisode, minEp=0.1, maxEp=0.9):
reshaped = self.reshape(torch.tensor(board))
output = self.policy(reshaped).view(-1)
return self.takeAction(output, reshaped, self.epsilon(maxEp, minEp, episode, lastEpisode))
def takeAction(self, actionList: torch.tensor, board, epsilon, train=True):
if (np.random.random() < epsilon) & train:
# invalide actions rein=geht nicht
#return torch.tensor(np.random.choice(len(actionList))).item()
return np.random.choice([i for i in range(len(actionList)) if board[0][0][0][i] == 1])
else:
for i in range(7):
if board[0][0][0][i] == 0:
actionList[i] = float('-inf')
return torch.argmax(actionList).item()
def reshape(self, board: torch.tensor, unsqz=True):
tensor = board.view(-1, 7).long()
# [0] = wo kann er reinwerfen(da wo es geht, steht eine 1), [1] = player1 (da wo es geht steht eine 0), [2] = player2 (da wo es geht steht eine 0)
a = F.one_hot(tensor, 3).permute([2, 0, 1])
b = a[:, :, :]
if unsqz:
return torch.unsqueeze(b, 0).float().to(self.device)
return b.float().to(self.device)
def preprocessState(self, state):
state = self.reshape(torch.tensor(state), True)
return state
def trainActionFromPolicy(self, state, action):
state = self.preprocessState(state)
value = self.policy(state).view(-1).to(self.device)
return value[action].to(self.device)
def trainActionFromTarget(self, next_state, reward, done):
next_state = self.preprocessState(next_state)
target = self.target(next_state)
target = torch.max(target, 1)[0].item()
target = reward + ((self.gamma * target) * (1 - done))
return torch.tensor(target).to(self.device)
def train(self):
if len(self.buffer) > self.batch_size:
self.optimizer.zero_grad()
states, actions, rewards, next_states, dones = self.buffer.sample(self.batch_size, self.device)
meanLoss = 0
for i in range(self.batch_size):
value = self.trainActionFromPolicy(states[i], actions[i])
target = self.trainActionFromTarget(next_states[i], rewards[i], dones[i])
loss = self.loss_function(value, target)
loss.backward()
meanLoss += loss
self.optimizer.step()
return meanLoss / self.batch_size
optimizer.zero_grad()
loss.backward()
optimizer.step()
running_loss += loss.item()
_, pred = torch.max(outputs, dim=1)
correct += (pred == labels).sum().item()
train_acc = correct / total
loss_list.append(running_loss)
train_acc_list.append(train_acc)
print('{}th- epoch: {}, train_loss = {}, train_acc = {}'.format(
i + 1, epoch, running_loss, train_acc))
with torch.no_grad():
net.eval()
correct = 0
total = test_size
pt, tt = [], []
for step_t, images_labels_t in enumerate(test_loader):
inputs_t, labels_t = images_labels_t
inputs_t, labels_t = inputs_t.type(
torch.FloatTensor).to(device), labels_t.type(
torch.LongTensor).to(device)
outputs_t = net(inputs_t)
outputs_t = softmax(outputs_t)
# test accuracy
_, pred_t = torch.max(outputs_t, dim=1)
loss_train = []
running_loss = 0.0
print('Finished training for epoch ' + str(epoch) + ' time taken = ' +
str(time.time() - epoch_start))
file.write('Finished training for epoch ' + str(epoch) + ' time taken = ' +
str(time.time() - epoch_start) + '\n')
file.write(
'##################################evaluation##############################\n'
)
print(
'################################evaluation###########################\n'
)
with torch.no_grad():
val_loss = 0
model.eval()
for i, data in enumerate(test_loader, 0):
step += 1
inputs, labels = data[0].to(device), data[1].to(device)
outputs = model(inputs)
loss = criterion(outputs, labels)
loss_val.append(loss.item())
validation_loss_store.append([epoch, loss.item()])
val_loss += loss
val_loss = val_loss / float(i + 1)
if val_loss < min_loss:
min_loss = val_loss
no_impr_epoch = 0
#=========Setting up==============
#=======Loading weight file to the model========
from Dataset import load_split_train_test
from Model import Net
classes = [
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O',
'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'del', 'nothing',
'space'
]
model = Net(out_fea=len(classes))
PATH = 'weight/epoch_6loss_0.15457870066165924.pt'
model.load_state_dict(torch.load(PATH))
model = model.eval()
img = cv2.imread(
'P:/Hand-Symbol-Recognition/asl_alphabet_test/asl_alphabet_test/J_test.jpg'
)
trans = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
img = trans(img)
img = img.unsqueeze(0)
with torch.no_grad():
output = model(img)
_, predicted = torch.max(output, 1)
