class pytorch_nn():
def __init__(self):
weights_pth = os.path.join('./checkpoints', 'nn_weights.pth')
self.model = MLP(input_size=5, output_size=3)
self.model.load_state_dict(
torch.load(weights_pth, map_location=torch.device('cpu')))
self.model.eval()
def predict(self, in_vector):
in_vector = torch.from_numpy(np.array(in_vector))
in_vector = Variable(in_vector).float()
outputs = self.model.forward(in_vector)
prob, pred = outputs.max(0, keepdim=True)
return outputs, prob, pred
def getNewNN(thisDict):
weights_pth = "checkpoints/nn_weights.pth"
model = MLP(input_size=5, output_size=3)
model.load_state_dict(
torch.load(weights_pth, map_location=torch.device('cpu')))
model.eval()
in_vector = []
for key, value in thisDict.items():
print(key, value)
temp = masterMerged.loc[masterMerged[key + "_x"] == value].head(1)
in_vector.append(temp.iloc[0][key + "_y"])
#in_vector = [1, 589, 9, 1, 0]
in_vector = torch.from_numpy(np.array(in_vector))
in_vector = Variable(in_vector).float()
outputs = model.forward(in_vector)
return outputs.tolist()
class VisualizationDemoMLP(object):
def __init__(self,
cfg_object,
cfg_keypoint,
instance_mode=ColorMode.IMAGE):
"""
Args:
cfg (CfgNode):
instance_mode (ColorMode):
parallel (bool): whether to run the model in different processes from visualization.
Useful since the visualization logic can be slow.
"""
self.metadata_object = MetadataCatalog.get("__unused")
self.metadata_keypoint = MetadataCatalog.get(
cfg_keypoint.DATASETS.TEST[0] if len(cfg_keypoint.DATASETS.TEST
) else "__unused")
self.cpu_device = torch.device("cpu")
self.instance_mode = instance_mode
self.predictor_object = DefaultPredictor(cfg_object)
self.predictor_keypoint = DefaultPredictor(cfg_keypoint)
self.head_pose_module = module_init(cfg_keypoint)
self.mtcnn = MTCNN()
self.transformations = transforms.Compose([transforms.Resize(224), \
transforms.CenterCrop(224), transforms.ToTensor(), \
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])
self.softmax = nn.Softmax(dim=1).cuda()
idx_tensor = [idx for idx in range(66)]
self.idx_tensor = torch.FloatTensor(idx_tensor).cuda()
self.data_json = {}
self.data_json['object_detection'] = {}
self.data_json['keypoint_detection'] = {}
self.data_json['head_pose_estimation'] = {}
self.frame_count = 0
self.mlp_model = MLP(input_size=26, output_size=1).cuda()
self.mlp_model.load_state_dict(torch.load(cfg_keypoint.MLP.PRETRAINED))
self.mlp_model.eval()
def run_on_image(self, image):
"""
Args:
image (np.ndarray): an image of shape (H, W, C) (in BGR order).
This is the format used by OpenCV.
Returns:
predictions (dict): the output of the model.
vis_output (VisImage): the visualized image output.
"""
vis_output = None
predictions = self.predictor(image)
# Convert image from OpenCV BGR format to Matplotlib RGB format.
image = image[:, :, ::-1]
visualizer = Visualizer(image,
self.metadata,
instance_mode=self.instance_mode)
if "instances" in predictions:
instances = predictions["instances"].to(self.cpu_device)
vis_output = visualizer.draw_instance_predictions(
predictions=instances)
return predictions, vis_output
def _frame_from_video(self, video):
while video.isOpened():
success, frame = video.read()
if success:
yield frame
else:
break
def run_on_video(self, video):
"""
Visualizes predictions on frames of the input video.
Args:
video (cv2.VideoCapture): a :class:`VideoCapture` object, whose source can be
either a webcam or a video file.
Yields:
ndarray: BGR visualizations of each video frame.
"""
video_visualizer_object = VideoVisualizer(self.metadata_object,
self.instance_mode)
video_visualizer_keypoint = VideoVisualizer(self.metadata_keypoint,
self.instance_mode)
def get_parameters(annos):
if annos["object_detection"]["pred_boxes"]:
temp = annos["object_detection"]["pred_boxes"][0]
obj_det = [1]
temp = np.asarray(temp)
temp = temp.flatten()
key_det = annos["keypoint_detection"]["pred_keypoints"][0]
key_det = np.asarray(key_det)
key_det = key_det[0:11, 0:2]
key_det = np.subtract(key_det, temp[0:2])
key_det = key_det.flatten()
else:
obj_det = [-1]
obj_det = np.asarray(obj_det)
key_det = annos["keypoint_detection"]["pred_keypoints"][0]
key_det = np.asarray(key_det)
key_det = key_det[0:11, 0:2]
key_det = key_det.flatten()
if annos["head_pose_estimation"]["predictions"]:
hp_est = annos["head_pose_estimation"]["predictions"][0]
hp_est = np.asarray(hp_est)
else:
hp_est = np.asarray([-100, -100, -100])
anno_list = np.concatenate((obj_det, key_det, hp_est))
return anno_list
def process_predictions(frame, predictions_object,
predictions_keypoint):
frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
blank_image = np.zeros((frame.shape[0], frame.shape[1], 3),
np.uint8)
if "instances" in predictions_object:
predictions_object = predictions_object["instances"].to(
self.cpu_device)
self.data_json['object_detection'][
'pred_boxes'] = predictions_object.get(
'pred_boxes').tensor.numpy().tolist()
self.data_json['object_detection'][
'scores'] = predictions_object.get(
'scores').numpy().tolist()
vis_frame = video_visualizer_object.draw_instance_predictions(
frame, predictions_object)
if "instances" in predictions_keypoint:
predictions_keypoint = predictions_keypoint["instances"].to(
self.cpu_device)
self.data_json['keypoint_detection'][
'pred_boxes'] = predictions_keypoint.get(
'pred_boxes').tensor.numpy().tolist()
self.data_json['keypoint_detection'][
'scores'] = predictions_keypoint.get(
'scores').numpy().tolist()
self.data_json['keypoint_detection'][
'pred_keypoints'] = predictions_keypoint.get(
'pred_keypoints').numpy().tolist()
vis_frame = video_visualizer_keypoint.draw_instance_predictions(
vis_frame.get_image(), predictions_keypoint)
# head pose estimation
predictions, bounding_box, face_keypoints, w, face_area = head_pose_estimation(
frame, self.mtcnn, self.head_pose_module, self.transformations,
self.softmax, self.idx_tensor)
self.data_json['head_pose_estimation']['predictions'] = predictions
self.data_json['head_pose_estimation']['pred_boxes'] = bounding_box
# Converts Matplotlib RGB format to OpenCV BGR format
vis_frame = cv2.cvtColor(vis_frame.get_image(), cv2.COLOR_RGB2BGR)
for i in range(len(predictions)):
plot_pose_cube(vis_frame, predictions[i][0], predictions[i][1], predictions[i][2], \
tdx = (face_keypoints[i][0] + face_keypoints[i][2]) / 2, \
tdy= (face_keypoints[i][1] + face_keypoints[i][3]) / 2, \
size = w[i])
# draw_axis(vis_frame, predictions[i][0], predictions[i][1], predictions[i][2], \
# tdx = (face_keypoints[i][0] + face_keypoints[i][2]) / 2, \
# tdy= (face_keypoints[i][1] + face_keypoints[i][3]) / 2, \
# size = w[i])
data_json = self.data_json
self.data_json['frame'] = self.frame_count
self.frame_count += 1
inputs_MLP = get_parameters(self.data_json)
inputs_MLP = Variable(torch.from_numpy(inputs_MLP)).float().cuda()
outputs_MLP = self.mlp_model(inputs_MLP)
predicted_MLP = (outputs_MLP >= 0.5)
cv2.putText(vis_frame,str(predicted_MLP.item()), (10,700), \
cv2.FONT_HERSHEY_SIMPLEX, 3, (0,0,0), 10)
return vis_frame, data_json
frame_gen = self._frame_from_video(video)
for frame in frame_gen:
yield process_predictions(frame, self.predictor_object(frame),
self.predictor_keypoint(frame))
def main():
# get unity environment
env, brain = get_unity_envs()
# get arguments
args = get_arguments()
print(args)
# set gpu environment
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
cudnn.enabled = True
cudnn.benchmark = True
cuda = torch.cuda.is_available()
# set random seed
rn = set_seeds(args.random_seed, cuda)
# make directory
os.makedirs(args.snapshot_dir, exist_ok=True)
# get validation dataset
val_set = get_validation_dataset(args)
print("len of test set: ", len(val_set))
val_loader = data.DataLoader(val_set,
batch_size=args.real_batch_size,
shuffle=False,
num_workers=args.num_workers,
pin_memory=True)
# generate training list
with open(args.syn_list_path, "w") as fp:
for i in range(args.syn_img_num):
if i % 10 != 0:
fp.write(str(i + 1) + '\n')
# get main model
main_model = MLP(args.num_inputs, args.num_outputs, args.hidden_size)
if args.resume != "":
main_model.load_state_dict(torch.load(args.resume))
# get task model
if args.task_model_name == "FCN8s":
task_model = FCN8s_sourceonly(n_class=args.num_classes)
vgg16 = VGG16(pretrained=True)
task_model.copy_params_from_vgg16(vgg16)
else:
raise ValueError("Specified model name: FCN8s")
# save initial task model
torch.save(task_model.state_dict(),
os.path.join(args.snapshot_dir, "task_model_init.pth"))
if cuda:
main_model = main_model.cuda()
task_model = task_model.cuda()
# get optimizer
main_optimizer = optim.Adam(main_model.parameters(), lr=args.main_lr)
task_optimizer = optim.SGD(task_model.parameters(),
lr=args.task_lr,
momentum=0.9,
weight_decay=1e-4)
frame_idx = 0
whole_start_time = time.time()
while frame_idx < args.max_frames:
log_probs = []
rewards = []
start_time = time.time()
for i_step in range(1, args.step_each_frame + 1):
# get initial attribute list
state = np.random.rand(1, args.num_inputs)
state = torch.from_numpy(state).float()
if cuda:
state = state.cuda()
# get modified attribute list
dist = main_model(state)
action = dist.sample()
action_actual = action.float() / 10.0 # [0, 0.9]
# generate images by attribute list
print("action: " + str(action_actual.cpu().numpy()))
get_images_by_attributes(args, i_step, env, brain,
action_actual[0].cpu().numpy())
train_set = get_training_dataset(args, i_step)
train_loader = data.DataLoader(train_set,
batch_size=args.syn_batch_size,
shuffle=True,
num_workers=args.num_workers,
pin_memory=True)
# train the task model using synthetic dataset
task_model.load_state_dict(
torch.load(
os.path.join(args.snapshot_dir, "task_model_init.pth")))
reward = train_task_model(train_loader, val_loader, task_model,
task_optimizer, args, cuda)
log_prob = dist.log_prob(action)[0]
log_probs.append(log_prob)
rewards.append(torch.FloatTensor([reward]))
frame_idx += 1
if frame_idx == 1:
moving_start = torch.FloatTensor([reward])
baseline = compute_returns(rewards, moving_start)
moving_start = baseline[-1]
log_probs = torch.cat(log_probs)
baseline = torch.cat(baseline).detach()
rewards = torch.cat(rewards).detach()
advantage = rewards - baseline
if cuda:
advantage = advantage.cuda()
loss = -(log_probs * advantage.detach()).mean()
with open(os.path.join(args.snapshot_dir, "logs.txt"), 'a') as fp:
fp.write(
"frame idx: {0:4d}, state: {1:s}, action: {2:s}, reward: {3:s}, baseline: {4:s}, loss: {5:.2f} \n"
.format(frame_idx, str(state.cpu()[0].numpy()),
str(action.cpu()[0].numpy()), str(rewards.numpy()),
str(baseline.numpy()), loss.item()))
print("optimize the main model parameters")
main_optimizer.zero_grad()
loss.backward()
main_optimizer.step()
elapsed_time = time.time() - start_time
print("[frame: {0:3d}], [loss: {1:.2f}], [time: {2:.1f}]".format(
frame_idx, loss.item(), elapsed_time))
torch.save(
main_model.state_dict(),
os.path.join(args.snapshot_dir, "main_model_%d.pth" % frame_idx))
elapsed_time = time.time() - whole_start_time
print("whole time: {0:.1f}".format(elapsed_time))
env.close()
from flask import Flask, jsonify, request, send_file, send_from_directory
from model.mlp import MLP
import torch
from torch.autograd import Variable
import numpy as np
from utils.utils import get_rainfall, get_solar_insolation, get_temperature
import os
import requests
from skimage import io
import shutil
import pandas as pd
app = Flask(__name__)
weights_pth = './nn_weight/mlp_weight.pth'
model = MLP(input_size=3, output_size=1)
model.load_state_dict(torch.load(weights_pth,
map_location=torch.device('cpu')))
model.eval()
def get_prediction(in_vector):
in_vector = torch.from_numpy(np.array(in_vector))
in_vector = Variable(in_vector).float()
outputs = model.forward(in_vector)
predicted = (outputs >= 0.755).float()
return predicted.cpu().numpy().tolist()[0]
@app.route('/predict', methods=['POST'])
def predict():
if request.method == 'POST':
data = request.json
trainloader = DataLoader(trainset,
batch_size=batch_size,
shuffle=True,
num_workers=4)
testloader = DataLoader(testset,
batch_size=batch_size,
shuffle=False,
num_workers=4)
# build network
mnist_mlp = MLP(io_bits=io_bits)
mnist_mlp.cuda()
try:
data = torch.load('mlp_mnist_nobias.t7')
mnist_mlp.load_state_dict(data)
print('Loading weight')
except:
print('Initializing model')
#
##
criterian = nn.CrossEntropyLoss(size_average=False)
optimizer = optim.SGD(mnist_mlp.parameters(), lr=learning_rate)
best_acc = 0
for i in range(epoches):
#training
running_acc = 0.
for (img, label) in trainloader:
import io
import json
from torchvision import models
import torchvision.transforms as transforms
from PIL import Image
from flask import Flask, jsonify, request
from model.mlp import MLP
import torch
from torch.autograd import Variable
import numpy as np
app = Flask(__name__)
weights_pth = './nn_weight/mlp_weight.pth'
model = MLP(input_size=3, output_size=1)
model.load_state_dict(torch.load(weights_pth))
model.eval()
def get_prediction(in_vector):
in_vector = torch.from_numpy(np.array(in_vector))
in_vector = Variable(in_vector).float()
outputs = model.forward(in_vector)
predicted = (outputs >= 0.755).float()
return predicted.cpu().numpy().tolist()
@app.route('/predict', methods=['POST'])
def predict():
if request.method == 'POST':
data = request.json