def __init__(self,
in_channels,
out_channels,
A_binary,
num_scales,
window_size,
disentangled_agg=True,
use_Ares=True,
residual=False,
dropout=0,
activation='relu'):
super().__init__()
self.num_scales = num_scales
self.window_size = window_size
self.use_Ares = use_Ares
A = self.build_spatial_temporal_graph(A_binary, window_size)
if disentangled_agg:
A_scales = [k_adjacency(A, k, with_self=True) for k in range(num_scales)]
A_scales = np.concatenate([normalize_adjacency_matrix(g) for g in A_scales])
else:
# Self-loops have already been included in A
A_scales = [normalize_adjacency_matrix(A) for k in range(num_scales)]
A_scales = [np.linalg.matrix_power(g, k) for k, g in enumerate(A_scales)]
A_scales = np.concatenate(A_scales)
self.A_scales = torch.Tensor(A_scales)
self.V = len(A_binary)
if use_Ares:
self.A_res = nn.init.uniform_(nn.Parameter(torch.randn(self.A_scales.shape)), -1e-6, 1e-6)
else:
self.A_res = torch.tensor(0)
self.mlp = MLP(in_channels * num_scales, [out_channels], dropout=dropout, activation='linear')
# Residual connection
if not residual:
self.residual = lambda x: 0
elif (in_channels == out_channels):
self.residual = lambda x: x
else:
self.residual = MLP(in_channels, [out_channels], activation='linear')
self.act = activation_factory(activation)
self.global_pool = nn.AdaptiveAvgPool2d(1)
self.conv_down = nn.Conv2d(
out_channels, out_channels // 4, kernel_size=1, bias=False)
# nn.init.constant_(self.conv_down.weight, 0)
nn.init.normal_(self.conv_down.weight, 0, 0.001)
self.conv_up = nn.Conv2d(
out_channels // 4, out_channels, kernel_size=1, bias=False)
nn.init.constant_(self.conv_up.weight, 0)
self.relu = nn.ReLU()
self.sig = nn.Sigmoid()
def get_classifier(ebd_dim, args):
tprint("Building classifier")
model = MLP(ebd_dim, args)
if args.cuda != -1:
return model.cuda(args.cuda)
else:
return model
def __init__(self):
nn.Module.__init__(self)
self.score_state = MLP(input_size=TRAN_IN_SIZE,
output_size=Transition_num,
num_layers=MLP_LAYERS)
self.score_state_trg = MLP(input_size=TRAN_IN_SIZE,
output_size=Transition_num,
num_layers=MLP_LAYERS)
self.capture_rate = nn.Linear(TRAN_IN_SIZE, Transition_num)
# 提高e-greedy的epsilon值，提高对模型的信任
self.epsilon = 0 if E_GREEDY_INCREMENT is not None else E_GREEDY
class pytorch_nn():
def __init__(self):
weights_pth = os.path.join('./checkpoints', 'nn_weights.pth')
self.model = MLP(input_size=5, output_size=3)
self.model.load_state_dict(
torch.load(weights_pth, map_location=torch.device('cpu')))
self.model.eval()
def predict(self, in_vector):
in_vector = torch.from_numpy(np.array(in_vector))
in_vector = Variable(in_vector).float()
outputs = self.model.forward(in_vector)
prob, pred = outputs.max(0, keepdim=True)
return outputs, prob, pred
def __init__(self,
gnn_module_func,
feature_num_list,
channel_num_list,
max_node_number,
dev,
gnn_hidden_num_list=(8, 8),
num_classes=1):
super().__init__()
self.num_classes = num_classes
gnn_layer_num = len(feature_num_list) - 1
channel_num_list = [2] + channel_num_list
assert len(channel_num_list) == len(feature_num_list)
gnn_hidden_num_list = list(gnn_hidden_num_list)
self.gnn_list = nn.ModuleList()
self.final_read_score = MLP(input_dim=sum(channel_num_list), output_dim=1, activate_func=F.elu,
hidden_dim=sum(channel_num_list) * 2,
num_layers=3,
num_classes=num_classes)
# self.final_read_score = MLP(input_dim=sum(channel_num_list) + sum(feature_num_list)*2,
# output_dim=1, activate_func=torch.tanh,
# hidden_dim=sum(channel_num_list)*2 + sum(feature_num_list)*4,
# num_layers=3)
for i in range(gnn_layer_num):
gnn_feature_list = [feature_num_list[i] + channel_num_list[i]] + gnn_hidden_num_list
gnn = gnn_module_func(feature_nums=gnn_feature_list, out_dim=feature_num_list[i + 1],
channel_num=channel_num_list[i])
self.gnn_list.append(EdgeDensePredictionGNNLayer(gnn_module=gnn, c_in=channel_num_list[i],
c_out=channel_num_list[i + 1],
num_classes=num_classes))
self.mask = torch.ones([max_node_number, max_node_number]) - torch.eye(max_node_number)
self.mask.unsqueeze_(0)
self.mask = self.mask.to(dev)
def __init__(self,
num_scales,
in_channels,
out_channels,
A_binary,
disentangled_agg=True,
use_mask=True,
dropout=0,
activation='relu'):
super().__init__()
self.num_scales = num_scales
if disentangled_agg:
A_powers = [k_adjacency(A_binary, k, with_self=True) for k in range(num_scales)]
A_powers = np.concatenate([normalize_adjacency_matrix(g) for g in A_powers])
else:
A_powers = [A_binary + np.eye(len(A_binary)) for k in range(num_scales)]
A_powers = [normalize_adjacency_matrix(g) for g in A_powers]
A_powers = [np.linalg.matrix_power(g, k) for k, g in enumerate(A_powers)]
A_powers = np.concatenate(A_powers)
self.A_powers = torch.Tensor(A_powers)
self.use_mask = use_mask
if use_mask:
# NOTE: the inclusion of residual mask appears to slow down training noticeably
self.A_res = nn.init.uniform_(nn.Parameter(torch.Tensor(self.A_powers.shape)), -1e-6, 1e-6)
self.mlp = MLP(in_channels * num_scales, [out_channels], dropout=dropout, activation=activation)
def __init__(self, gnn_module, c_in, c_out,
num_classes=1):
super().__init__()
self.multi_channel_gnn_module = gnn_module
self.translate_mlp = MLP(num_layers=3, input_dim=c_in + 2 * gnn_module.get_out_dim(),
hidden_dim=max(c_in, c_out) * 2, output_dim=c_out,
activate_func=F.elu,
use_bn=True,
num_classes=num_classes)
def build_tf_model(sess, n_dim, layers, random_seed):
tf.reset_default_graph()
if random_seed is not None:
# Setting seeds for reproducibility
tf.set_random_seed(random_seed)
np.random.seed(random_seed)
random.seed(random_seed)
model = MLP(ndim=n_dim, layers=layers)
if sess is not None and not sess._closed:
sess.close()
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
model.set_session(sess)
return model, sess
def __init__(self,
cfg_object,
cfg_keypoint,
instance_mode=ColorMode.IMAGE):
"""
Args:
cfg (CfgNode):
instance_mode (ColorMode):
parallel (bool): whether to run the model in different processes from visualization.
Useful since the visualization logic can be slow.
"""
self.metadata_object = MetadataCatalog.get("__unused")
self.metadata_keypoint = MetadataCatalog.get(
cfg_keypoint.DATASETS.TEST[0] if len(cfg_keypoint.DATASETS.TEST
) else "__unused")
self.cpu_device = torch.device("cpu")
self.instance_mode = instance_mode
self.predictor_object = DefaultPredictor(cfg_object)
self.predictor_keypoint = DefaultPredictor(cfg_keypoint)
self.head_pose_module = module_init(cfg_keypoint)
self.mtcnn = MTCNN()
self.transformations = transforms.Compose([transforms.Resize(224), \
transforms.CenterCrop(224), transforms.ToTensor(), \
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])
self.softmax = nn.Softmax(dim=1).cuda()
idx_tensor = [idx for idx in range(66)]
self.idx_tensor = torch.FloatTensor(idx_tensor).cuda()
self.data_json = {}
self.data_json['object_detection'] = {}
self.data_json['keypoint_detection'] = {}
self.data_json['head_pose_estimation'] = {}
self.frame_count = 0
self.mlp_model = MLP(input_size=26, output_size=1).cuda()
self.mlp_model.load_state_dict(torch.load(cfg_keypoint.MLP.PRETRAINED))
self.mlp_model.eval()
def __init__(self, model_name='', model_params={}, random_state=42):
""" Build and evaluate model
Parameters
----------
model_name
model_params
random_state
"""
self.random_state = random_state
if model_name == 'DecisionTreeClassifier':
self.model = DecisionTreeClassifier(random_state=self.random_state)
elif model_name == 'MLP':
self.model = MLP(random_state=self.random_state)
else:
msg = f"{model_name}"
raise NotImplementedError(msg)
self.model.set_params(**model_params)
print(self.model.get_params())
def __init__(self,
num_class,
num_point,
num_person,
num_gcn_scales,
num_g3d_scales,
graph,
in_channels=3):
super(Model, self).__init__()
# self.stgcn_graph = STGCN_Graph(layout='openpose',strategy='spatial')
Graph = import_class(graph)
A_binary = Graph().A_binary
self.data_bn = nn.BatchNorm1d(num_person * in_channels * num_point)
# channels
c1 = 64
c2 = c1 * 1 # 64
# c3 = c2 * 2 # 384
mlp1 = MLP(3 * num_gcn_scales, [c1], dropout=0, activation='relu')
mlp2 = MLP2(c1 * num_gcn_scales, [c2], dropout=0, activation='relu')
# r=3 STGC blocks
# self.gcn3d1 = MultiWindow_MS_G3D(3, c1, A_binary, num_g3d_scales, window_stride=1)
self.sgcn1 = nn.Sequential(
MS_GCN(num_gcn_scales, 3, c1, A_binary, disentangled_agg=True),
mlp1,
MS_TCN(c1, c1, kernel_size=5),
MS_TCN(c1, c1, kernel_size=5))
self.sgcn1[-1].act = nn.Identity()
self.tcn1 = MS_TCN(c1, c1, kernel_size=5)
# self.gcn3d2 = MultiWindow_MS_G3D(c1, c2, A_binary, num_g3d_scales, window_stride=2)
self.sgcn2 = nn.Sequential(
MS_GCN(num_gcn_scales, c1, c1, A_binary, disentangled_agg=True),
mlp2,
MS_TCN(c2, c2, kernel_size=5, stride=2),
MS_TCN(c2, c2, kernel_size=5))
self.sgcn2[-1].act = nn.Identity()
self.tcn2 = MS_TCN(c2, c2, kernel_size=5)
# self.gcn3d3 = MultiWindow_MS_G3D(c2, c3, A_binary, num_g3d_scales, window_stride=2)
# self.sgcn3 = nn.Sequential(
# MS_GCN(num_gcn_scales, c2, c2, A_binary, disentangled_agg=True),
# MS_TCN(c2, c3, stride=2),
# MS_TCN(c3, c3))
# self.sgcn3[-1].act = nn.Identity()
# self.tcn3 = MS_TCN(c3, c3)
self.fc = nn.Linear(c2, num_class)
def main(args):
input_size = 3
output_size = 1
# Make the training and testing set to be less bias
fire_dataset = FireDataset(args.csv_path[0])
fire_train, fire_test = random_split(
fire_dataset,
(round(0.7 * len(fire_dataset)), round(0.3 * len(fire_dataset))))
trainloader = DataLoader(fire_train,
batch_size=4096,
shuffle=True,
num_workers=2)
testloader = DataLoader(fire_test,
batch_size=512,
shuffle=False,
num_workers=2)
save_weights_pth = args.weights_path[0]
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = MLP(input_size=input_size, output_size=output_size)
model.to(device)
criterion = nn.BCELoss()
optimizer = optim.Adam(model.parameters(), lr=5e-7)
epochs = 30
if args.eval_only:
do_test(model, device, testloader, save_weights_pth)
else:
do_train(model, device, trainloader, criterion, optimizer, epochs,
save_weights_pth)
do_test(model, device, testloader, save_weights_pth)
def run_experiment(config):
"""
Run active learning for the 10D rosenbrock function data
It starts from small train dataset and then extends it with points from pool
We compare three sampling methods:
- Random datapoints
- Points with highest uncertainty by MCDUE
- Points with highest uncertainty by NNGP (proposed method)
"""
rmses = {}
for estimator_name in config['estimators']:
print("\nEstimator:", estimator_name)
# load data
rosen = RosenData(config['n_dim'],
config['data_size'],
config['data_split'],
use_cache=config['use_cache'])
x_train, y_train = rosen.dataset('train')
x_val, y_val = rosen.dataset('train')
x_pool, y_pool = rosen.dataset('pool')
# Build neural net and set random seed
set_random(config['random_seed'])
model = MLP(config['layers'])
estimator = build_estimator(estimator_name,
model) # to estimate uncertainties
oracle = IdentityOracle(y_pool) # generate y for X from pool
sampler = EagerSampleSelector(
) # sample X and y from pool by uncertainty estimations
# Active learning training
trainer = ALTrainer(model,
estimator,
sampler,
oracle,
config['al_iterations'],
config['update_size'],
verbose=config['verbose'])
rmses[estimator_name] = trainer.train(x_train, y_train, x_val, y_val,
x_pool)
visualize(rmses)
def get_model(model_name, data, drop):
if model_name == "gcn":
model = GCN(num_features=data.num_features,
num_classes=data.num_classes,
hidden_size=32,
dropout=drop)
elif model_name == "gat":
model = GAT(num_features=data.num_features,
num_classes=data.num_classes,
hidden_size=8,
num_heads=4,
dropout=drop)
else:
model = MLP(num_features=data.num_features,
num_classes=data.num_classes,
hidden_size=32,
dropout=drop)
return model
def __init__(self,
gnn_module_func,
feature_num_list,
channel_num_list,
max_node_number,
dev,
gnn_hidden_num_list=(8, 8),
num_classes=1):
"""
note that `num_classes` means the number of different (gains, biases) in conditional layers
(see the definition of class `MLP` and class `ConditionalLayer1d`)
i.e., num_classes==len(sigma_list)
"""
super().__init__()
self.num_classes = num_classes
gnn_layer_num = len(feature_num_list) - 1
channel_num_list = [2] + channel_num_list
assert len(channel_num_list) == len(feature_num_list)
gnn_hidden_num_list = list(gnn_hidden_num_list)
self.gnn_list = nn.ModuleList()
self.final_read_score = MLP(input_dim=sum(channel_num_list),
output_dim=1,
activate_func=F.elu,
hidden_dim=sum(channel_num_list) * 2,
num_layers=3,
num_classes=num_classes)
for i in range(gnn_layer_num):
gnn_feature_list = [feature_num_list[i] + channel_num_list[i]
] + gnn_hidden_num_list
gnn = gnn_module_func(feature_nums=gnn_feature_list,
out_dim=feature_num_list[i + 1],
channel_num=channel_num_list[i])
self.gnn_list.append(
EdgeDensePredictionGNNLayer(gnn_module=gnn,
c_in=channel_num_list[i],
c_out=channel_num_list[i + 1],
num_classes=num_classes))
self.mask = torch.ones([max_node_number, max_node_number
]) - torch.eye(max_node_number)
self.mask.unsqueeze_(0)
self.mask = self.mask.to(dev)
def __init__(self,
in_channels,
out_channels,
A_binary,
num_scales,
window_size,
window_stride,
window_dilation,
embed_factor=1,
activation='relu'):
super().__init__()
self.window_size = window_size
self.out_channels = out_channels
self.embed_channels_in = self.embed_channels_out = out_channels // embed_factor
if embed_factor == 1:
self.in1x1 = nn.Identity()
self.embed_channels_in = self.embed_channels_out = in_channels
# The first STGC block changes channels right away; others change at collapse
if in_channels == 3:
self.embed_channels_out = out_channels
elif in_channels == 6:
self.embed_channels_out = out_channels
elif in_channels == 9:
self.embed_channels_out = out_channels
else:
self.in1x1 = MLP(in_channels, [self.embed_channels_in])
self.gcn3d = nn.Sequential(
UnfoldTemporalWindows(window_size, window_stride, window_dilation),
SpatialTemporal_MS_GCN(
in_channels=self.embed_channels_in,
out_channels=self.embed_channels_out,
A_binary=A_binary,
num_scales=num_scales,
window_size=window_size,
use_Ares=True,
dropout=0
)
)
self.out_conv = nn.Conv3d(self.embed_channels_out, out_channels, kernel_size=(1, self.window_size, 1))
self.out_bn = nn.BatchNorm2d(out_channels)
def bench_rnn_forward(batch_size=512, num_batch=10, vocab_size=1024, length=30, embed_size=128,\
hidden_size=128, delta=5):
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
rnn = RNNNative(vocab_size, embed_size, hidden_size).to(device)
delta = np.random.randint(-delta, delta, size=num_batch)
input = torch.LongTensor(batch_size).random_(0, vocab_size).to(device)
hx = torch.randn(batch_size, hidden_size).to(device)
start = time.time()
for i in range(num_batch):
for j in range(length + delta[i]):
output, hx = rnn(input, hx)
end = time.time()
print("Elapsed time for RNNNative {:.3f}, avg length: {:.3f}".format(end - start, length + np.mean(delta)))
rnn = RNNTorch(vocab_size, embed_size, hidden_size).to(device)
input = torch.LongTensor(length, batch_size).random_(0, vocab_size).to(device)
start = time.time()
for i in range(num_batch):
output, _ = rnn(input)
end = time.time()
print("Elapsed time for RNNTorch {:.3f}, avg length: {:.3f}".format(end - start, length + np.mean(delta)))
input = torch.LongTensor(1, batch_size).random_(0, vocab_size).to(device)
start = time.time()
hx = torch.randn(1, batch_size, hidden_size).to(device)
for i in range(num_batch):
for j in range(length + delta[i]):
output, hx = rnn(input, hx)
end = time.time()
print("Elapsed time for RNNTorch with variable length {:.3f}, avg length: {:.3f}".format(end - start, length + np.mean(delta)))
mlp = MLP(device, vocab_size, embed_size, hidden_size, length).to(device)
input = []
hx = []
for i in range(length):
input.append(torch.LongTensor(batch_size).random_(0, vocab_size).to(device))
hx.append(torch.randn(batch_size, hidden_size).to(device))
start = time.time()
for i in range(num_batch):
output = mlp(input, hx)
end = time.time()
print("Elapsed time for MLP {:.3f}".format(end - start))
def getNewNN(thisDict):
weights_pth = "checkpoints/nn_weights.pth"
model = MLP(input_size=5, output_size=3)
model.load_state_dict(
torch.load(weights_pth, map_location=torch.device('cpu')))
model.eval()
in_vector = []
for key, value in thisDict.items():
print(key, value)
temp = masterMerged.loc[masterMerged[key + "_x"] == value].head(1)
in_vector.append(temp.iloc[0][key + "_y"])
#in_vector = [1, 589, 9, 1, 0]
in_vector = torch.from_numpy(np.array(in_vector))
in_vector = Variable(in_vector).float()
outputs = model.forward(in_vector)
return outputs.tolist()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if args:
ptr_net = args[0]
else:
ptr_net = kwargs['ptr_net']
assert isinstance(ptr_net, LSTMPointerNet)
self._stop = nn.Parameter(
torch.Tensor(self._lstm_cell.input_size))
init.uniform_(self._stop, -INI, INI)
self.sc = create_sc()
cm = ConfManager()
self.K = cm.K
self.beta = cm.beta
self.mode = cm.mode
self.use_feat = cm.use_feat
print(f'K={self.K}, beta={self.beta}, mode={self.mode}, use_feature={self.use_feat}')
if self.mode == 'alpha':
self.mlp = MLP2()
else:
in_dim = 12 if self.use_feat else 1
to_score = True if self.mode == 'attention' else True
self.mlp = MLP(in_dim=in_dim, to_score=to_score)
i = 1
for n_hidden_nodes in hidden_nodes:
print(f"x-----> {n_hidden_nodes} hidden nodes <-----x")
for eta in eta_values:
for batch_size in [dataset[0].shape[0], 1]:
for _ in range(loops):
name = f"MLP{i:05}_3_n-{n}_b-{batch_size}_h-{n_hidden_nodes}_eta-{eta}_m-{momentum}".replace(
".", ",")
i += 1
print(f"ETA is {eta}")
print(name)
net = MLP(dataset[0],
dataset[1],
n_hidden_nodes,
momentum=momentum)
train_losses, _, train_accuracies, _, pocket_epoch = net.train(
dataset[0],
dataset[1],
dataset[0],
dataset[1],
eta,
epochs,
early_stop_count=early_stop,
shuffle=shuffle,
batch_size=batch_size,
early_stopping_threshold=early_stopping_threshold)
train_accuracies = np.array(train_accuracies) * 100 / n
net.forward(dataset[0])
class VisualizationDemoMLP(object):
def __init__(self,
cfg_object,
cfg_keypoint,
instance_mode=ColorMode.IMAGE):
"""
Args:
cfg (CfgNode):
instance_mode (ColorMode):
parallel (bool): whether to run the model in different processes from visualization.
Useful since the visualization logic can be slow.
"""
self.metadata_object = MetadataCatalog.get("__unused")
self.metadata_keypoint = MetadataCatalog.get(
cfg_keypoint.DATASETS.TEST[0] if len(cfg_keypoint.DATASETS.TEST
) else "__unused")
self.cpu_device = torch.device("cpu")
self.instance_mode = instance_mode
self.predictor_object = DefaultPredictor(cfg_object)
self.predictor_keypoint = DefaultPredictor(cfg_keypoint)
self.head_pose_module = module_init(cfg_keypoint)
self.mtcnn = MTCNN()
self.transformations = transforms.Compose([transforms.Resize(224), \
transforms.CenterCrop(224), transforms.ToTensor(), \
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])
self.softmax = nn.Softmax(dim=1).cuda()
idx_tensor = [idx for idx in range(66)]
self.idx_tensor = torch.FloatTensor(idx_tensor).cuda()
self.data_json = {}
self.data_json['object_detection'] = {}
self.data_json['keypoint_detection'] = {}
self.data_json['head_pose_estimation'] = {}
self.frame_count = 0
self.mlp_model = MLP(input_size=26, output_size=1).cuda()
self.mlp_model.load_state_dict(torch.load(cfg_keypoint.MLP.PRETRAINED))
self.mlp_model.eval()
def run_on_image(self, image):
"""
Args:
image (np.ndarray): an image of shape (H, W, C) (in BGR order).
This is the format used by OpenCV.
Returns:
predictions (dict): the output of the model.
vis_output (VisImage): the visualized image output.
"""
vis_output = None
predictions = self.predictor(image)
# Convert image from OpenCV BGR format to Matplotlib RGB format.
image = image[:, :, ::-1]
visualizer = Visualizer(image,
self.metadata,
instance_mode=self.instance_mode)
if "instances" in predictions:
instances = predictions["instances"].to(self.cpu_device)
vis_output = visualizer.draw_instance_predictions(
predictions=instances)
return predictions, vis_output
def _frame_from_video(self, video):
while video.isOpened():
success, frame = video.read()
if success:
yield frame
else:
break
def run_on_video(self, video):
"""
Visualizes predictions on frames of the input video.
Args:
video (cv2.VideoCapture): a :class:`VideoCapture` object, whose source can be
either a webcam or a video file.
Yields:
ndarray: BGR visualizations of each video frame.
"""
video_visualizer_object = VideoVisualizer(self.metadata_object,
self.instance_mode)
video_visualizer_keypoint = VideoVisualizer(self.metadata_keypoint,
self.instance_mode)
def get_parameters(annos):
if annos["object_detection"]["pred_boxes"]:
temp = annos["object_detection"]["pred_boxes"][0]
obj_det = [1]
temp = np.asarray(temp)
temp = temp.flatten()
key_det = annos["keypoint_detection"]["pred_keypoints"][0]
key_det = np.asarray(key_det)
key_det = key_det[0:11, 0:2]
key_det = np.subtract(key_det, temp[0:2])
key_det = key_det.flatten()
else:
obj_det = [-1]
obj_det = np.asarray(obj_det)
key_det = annos["keypoint_detection"]["pred_keypoints"][0]
key_det = np.asarray(key_det)
key_det = key_det[0:11, 0:2]
key_det = key_det.flatten()
if annos["head_pose_estimation"]["predictions"]:
hp_est = annos["head_pose_estimation"]["predictions"][0]
hp_est = np.asarray(hp_est)
else:
hp_est = np.asarray([-100, -100, -100])
anno_list = np.concatenate((obj_det, key_det, hp_est))
return anno_list
def process_predictions(frame, predictions_object,
predictions_keypoint):
frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
blank_image = np.zeros((frame.shape[0], frame.shape[1], 3),
np.uint8)
if "instances" in predictions_object:
predictions_object = predictions_object["instances"].to(
self.cpu_device)
self.data_json['object_detection'][
'pred_boxes'] = predictions_object.get(
'pred_boxes').tensor.numpy().tolist()
self.data_json['object_detection'][
'scores'] = predictions_object.get(
'scores').numpy().tolist()
vis_frame = video_visualizer_object.draw_instance_predictions(
frame, predictions_object)
if "instances" in predictions_keypoint:
predictions_keypoint = predictions_keypoint["instances"].to(
self.cpu_device)
self.data_json['keypoint_detection'][
'pred_boxes'] = predictions_keypoint.get(
'pred_boxes').tensor.numpy().tolist()
self.data_json['keypoint_detection'][
'scores'] = predictions_keypoint.get(
'scores').numpy().tolist()
self.data_json['keypoint_detection'][
'pred_keypoints'] = predictions_keypoint.get(
'pred_keypoints').numpy().tolist()
vis_frame = video_visualizer_keypoint.draw_instance_predictions(
vis_frame.get_image(), predictions_keypoint)
# head pose estimation
predictions, bounding_box, face_keypoints, w, face_area = head_pose_estimation(
frame, self.mtcnn, self.head_pose_module, self.transformations,
self.softmax, self.idx_tensor)
self.data_json['head_pose_estimation']['predictions'] = predictions
self.data_json['head_pose_estimation']['pred_boxes'] = bounding_box
# Converts Matplotlib RGB format to OpenCV BGR format
vis_frame = cv2.cvtColor(vis_frame.get_image(), cv2.COLOR_RGB2BGR)
for i in range(len(predictions)):
plot_pose_cube(vis_frame, predictions[i][0], predictions[i][1], predictions[i][2], \
tdx = (face_keypoints[i][0] + face_keypoints[i][2]) / 2, \
tdy= (face_keypoints[i][1] + face_keypoints[i][3]) / 2, \
size = w[i])
# draw_axis(vis_frame, predictions[i][0], predictions[i][1], predictions[i][2], \
# tdx = (face_keypoints[i][0] + face_keypoints[i][2]) / 2, \
# tdy= (face_keypoints[i][1] + face_keypoints[i][3]) / 2, \
# size = w[i])
data_json = self.data_json
self.data_json['frame'] = self.frame_count
self.frame_count += 1
inputs_MLP = get_parameters(self.data_json)
inputs_MLP = Variable(torch.from_numpy(inputs_MLP)).float().cuda()
outputs_MLP = self.mlp_model(inputs_MLP)
predicted_MLP = (outputs_MLP >= 0.5)
cv2.putText(vis_frame,str(predicted_MLP.item()), (10,700), \
cv2.FONT_HERSHEY_SIMPLEX, 3, (0,0,0), 10)
return vis_frame, data_json
frame_gen = self._frame_from_video(video)
for frame in frame_gen:
yield process_predictions(frame, self.predictor_object(frame),
self.predictor_keypoint(frame))
class Recommender:
def __init__(self, model_name='', model_params={}, random_state=42):
""" Build and evaluate model
Parameters
----------
model_name
model_params
random_state
"""
self.random_state = random_state
if model_name == 'DecisionTreeClassifier':
self.model = DecisionTreeClassifier(random_state=self.random_state)
elif model_name == 'MLP':
self.model = MLP(random_state=self.random_state)
else:
msg = f"{model_name}"
raise NotImplementedError(msg)
self.model.set_params(**model_params)
print(self.model.get_params())
def fit(self, X, y):
""" fit a model on X, y
Parameters
----------
X
y
Returns
-------
"""
self.model.fit(X, y)
return self
def predict(self, X):
""" predict X
Parameters
----------
X
Returns
-------
"""
return self.model.predict(X)
def predict_proba(self, X):
return self.model.predict_proba(X)
def test(self, X, y):
""" generate model's evaluation report
Parameters
----------
X
y
Returns
-------
"""
y_score = self.predict_proba(X)
fpr, tpr, _ = roc_curve(y, y_score)
auc_score = auc(fpr, tpr)
print(f"auc: {auc_score:.4f}")
y_pred = self.predict(X)
acc = accuracy_score(y, y_pred)
print(f"acc: {acc:.4f}")
cm = confusion_matrix(y, y_pred)
print(cm)
rp = classification_report(y, y_pred)
print(rp)
# best = {"best_acc": 0, "best_loss": np.inf}
for eta in eta_values:
print(f"ETA is {eta}")
results[n_hidden_nodes][eta] = {}
accumulated_metrics = {}
for m in metrics:
accumulated_metrics[m] = []
name = f"MLP{i:05}_d-{dataset_idx}_b-{batch_size}_h-{n_hidden_nodes}_eta-{eta}".replace(".", ",")
i += 1
print(name)
best = {"best_acc": 0, "best_loss": np.inf}
for _ in range(loops):
print(".", end="")
net = MLP(train[0], train[1], n_hidden_nodes, momentum=0)
train_losses, valid_losses, train_accuracies, valid_accuracies, pocket_epoch = net.train(
train[0], train[1], valid[0], valid[1], eta, epochs,
early_stop_count=early_stop, shuffle=shuffle, batch_size=batch_size
)
cm_train = net.confmat(train[0], train[1])
cm_valid = net.confmat(valid[0], valid[1])
print("cm_train", cm_train)
print("cm_valid", cm_valid)
print("pocket epoch", pocket_epoch)
train_loss, valid_loss = train_losses[pocket_epoch], valid_losses[pocket_epoch]
train_acc, train_sens, train_spec, _, _ = two_class_conf_mat_metrics(
net.confmat(train[0], train[1]))
valid_acc, valid_sens, valid_spec, _, _ = two_class_conf_mat_metrics(
net.confmat(valid[0], valid[1]))
def main():
# get unity environment
env, brain = get_unity_envs()
# get arguments
args = get_arguments()
print(args)
# set gpu environment
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
cudnn.enabled = True
cudnn.benchmark = True
cuda = torch.cuda.is_available()
# set random seed
rn = set_seeds(args.random_seed, cuda)
# make directory
os.makedirs(args.snapshot_dir, exist_ok=True)
# get validation dataset
val_set = get_validation_dataset(args)
print("len of test set: ", len(val_set))
val_loader = data.DataLoader(val_set,
batch_size=args.real_batch_size,
shuffle=False,
num_workers=args.num_workers,
pin_memory=True)
# generate training list
with open(args.syn_list_path, "w") as fp:
for i in range(args.syn_img_num):
if i % 10 != 0:
fp.write(str(i + 1) + '\n')
# get main model
main_model = MLP(args.num_inputs, args.num_outputs, args.hidden_size)
if args.resume != "":
main_model.load_state_dict(torch.load(args.resume))
# get task model
if args.task_model_name == "FCN8s":
task_model = FCN8s_sourceonly(n_class=args.num_classes)
vgg16 = VGG16(pretrained=True)
task_model.copy_params_from_vgg16(vgg16)
else:
raise ValueError("Specified model name: FCN8s")
# save initial task model
torch.save(task_model.state_dict(),
os.path.join(args.snapshot_dir, "task_model_init.pth"))
if cuda:
main_model = main_model.cuda()
task_model = task_model.cuda()
# get optimizer
main_optimizer = optim.Adam(main_model.parameters(), lr=args.main_lr)
task_optimizer = optim.SGD(task_model.parameters(),
lr=args.task_lr,
momentum=0.9,
weight_decay=1e-4)
frame_idx = 0
whole_start_time = time.time()
while frame_idx < args.max_frames:
log_probs = []
rewards = []
start_time = time.time()
for i_step in range(1, args.step_each_frame + 1):
# get initial attribute list
state = np.random.rand(1, args.num_inputs)
state = torch.from_numpy(state).float()
if cuda:
state = state.cuda()
# get modified attribute list
dist = main_model(state)
action = dist.sample()
action_actual = action.float() / 10.0 # [0, 0.9]
# generate images by attribute list
print("action: " + str(action_actual.cpu().numpy()))
get_images_by_attributes(args, i_step, env, brain,
action_actual[0].cpu().numpy())
train_set = get_training_dataset(args, i_step)
train_loader = data.DataLoader(train_set,
batch_size=args.syn_batch_size,
shuffle=True,
num_workers=args.num_workers,
pin_memory=True)
# train the task model using synthetic dataset
task_model.load_state_dict(
torch.load(
os.path.join(args.snapshot_dir, "task_model_init.pth")))
reward = train_task_model(train_loader, val_loader, task_model,
task_optimizer, args, cuda)
log_prob = dist.log_prob(action)[0]
log_probs.append(log_prob)
rewards.append(torch.FloatTensor([reward]))
frame_idx += 1
if frame_idx == 1:
moving_start = torch.FloatTensor([reward])
baseline = compute_returns(rewards, moving_start)
moving_start = baseline[-1]
log_probs = torch.cat(log_probs)
baseline = torch.cat(baseline).detach()
rewards = torch.cat(rewards).detach()
advantage = rewards - baseline
if cuda:
advantage = advantage.cuda()
loss = -(log_probs * advantage.detach()).mean()
with open(os.path.join(args.snapshot_dir, "logs.txt"), 'a') as fp:
fp.write(
"frame idx: {0:4d}, state: {1:s}, action: {2:s}, reward: {3:s}, baseline: {4:s}, loss: {5:.2f} \n"
.format(frame_idx, str(state.cpu()[0].numpy()),
str(action.cpu()[0].numpy()), str(rewards.numpy()),
str(baseline.numpy()), loss.item()))
print("optimize the main model parameters")
main_optimizer.zero_grad()
loss.backward()
main_optimizer.step()
elapsed_time = time.time() - start_time
print("[frame: {0:3d}], [loss: {1:.2f}], [time: {2:.1f}]".format(
frame_idx, loss.item(), elapsed_time))
torch.save(
main_model.state_dict(),
os.path.join(args.snapshot_dir, "main_model_%d.pth" % frame_idx))
elapsed_time = time.time() - whole_start_time
print("whole time: {0:.1f}".format(elapsed_time))
env.close()
from PIL import Image
from flask import Flask, jsonify, request, send_file, send_from_directory
from model.mlp import MLP
import torch
from torch.autograd import Variable
import numpy as np
from utils.utils import get_rainfall, get_solar_insolation, get_temperature
import os
import requests
from skimage import io
import shutil
import pandas as pd
app = Flask(__name__)
weights_pth = './nn_weight/mlp_weight.pth'
model = MLP(input_size=3, output_size=1)
model.load_state_dict(torch.load(weights_pth,
map_location=torch.device('cpu')))
model.eval()
def get_prediction(in_vector):
in_vector = torch.from_numpy(np.array(in_vector))
in_vector = Variable(in_vector).float()
outputs = model.forward(in_vector)
predicted = (outputs >= 0.755).float()
return predicted.cpu().numpy().tolist()[0]
@app.route('/predict', methods=['POST'])
def predict():
'/Users/mike/Documents/ml_lab/phd/proposal/code/MCL-GAN/incubating/tictactoe'
)
from settings.run_settings import ExperimentSettings
from model.mlp import MLP
from data.csv_data import Dataset
from results.saver import Saver
from trainer.trainer import BasicTrainer
from loss.loss import *
from evaluate.accuracy import Accuracy
s = ExperimentSettings('validator_settings.txt')
d = Dataset(s)
d.resample()
l = CrossEntropyLoss()
m = MLP(s)
res = Saver(s)
t = BasicTrainer()
e = Accuracy('binary')
t.train(s, m, d, res, l, e)
e.report(d, m, s)
# import gc
# gc.collect()
# y_pred = m.compute(d.X)
# y_pred[y_pred <= .5] = 0
# y_pred[y_pred > .5] = 1
# print y_pred
# y_true = d.Y
# from sklearn.metrics import accuracy_score
# print accuracy_score(y_true, y_pred)
#test script
import sys
# Add the ptdraft folder path to the sys.path list
sys.path.append('/Users/mike/Documents/ml_lab/phd/proposal/code/MCL-GAN/incubating/tictactoe')
from settings.run_settings import ExperimentSettings
from model.mlp import MLP
from data.csv_data import Dataset
from results.saver import Saver
from trainer.trainer import BasicTrainer
from loss.loss import *
from evaluate.accuracy import Accuracy
s = ExperimentSettings('test_settings_ttt.txt')
d = Dataset(s)
l = CrossEntropyLoss()
m = MLP(s)
res = Saver(s)
t = BasicTrainer()
t.train(s,m,d,res,l)
e = Accuracy()
print m.compute(d.X)
print e.evalTTT(d.Y,m.compute(d.X))
# train(self, settings, model, data, saver, loss):
def train_model(data,
model_name="mlp",
log_interval=10,
loss="mse",
optim="adam",
store=False,
visual=False,
verbose=False,
log=False):
if verbose and log:
print("\nTraining model", model_name)
if store:
# Create directory
checkpoint_path = os.path.join("checkpoints/", model_name)
if not os.path.isdir(checkpoint_path):
try:
os.makedirs(checkpoint_path)
except OSError:
sys.exit("Creation of checkpoint directory failed.")
# Model
model = None
if model_name == "mlp":
model = MLP(
num_features=data.X.shape[1],
hidden_size=config("{}.hidden_layer".format(model_name)),
)
elif model_name == "gcn":
model = GCN(
num_features=data.X.shape[1],
hidden_size=config("{}.hidden_layer".format(model_name)),
)
elif model_name == "gat":
model = GAT(
num_features=data.X.shape[1],
hidden_size=config("{}.hidden_layer".format(model_name)),
)
# Criterion and Loss Function
criterion = torch.nn.MSELoss()
loss_fn = None
if loss == "mse":
loss_fn = torch.nn.MSELoss()
elif loss == "nll":
loss_fn = NLLLoss()
# Optimizer
optimizer = None
if optim == "adam":
optimizer = Adam(model.parameters(),
lr=config('{}.learning_rate'.format(model_name)))
elif optim == "adagrad":
optimizer = Adagrad(model.parameters(),
lr=config('{}.learning_rate'.format(model_name)))
# Setup training
fig, axes = None, None
if visual:
fig, axes = make_training_plot(model_name)
start_epoch = 0
stats = []
if store:
# Attempts to restore the latest checkpoint if exists
print('Loading {}...'.format(model_name))
model, start_epoch, stats = restore_checkpoint(
model, config('{}.checkpoint'.format(model_name)))
# Evaluate the randomly initialized model
_evaluate_epoch(axes, data, model, criterion, start_epoch, stats,
log_interval, log)
# Loop over the entire dataset multiple times
patience = config("patience")
best_loss = float('inf')
idx = -1
for epoch in range(start_epoch,
config('{}.num_epochs'.format(model_name))):
# Early stop
if patience < 0:
break
# Train model
_train_epoch(data, model, loss_fn, optimizer)
# Evaluate model
if (epoch + 1) % log_interval == 0:
_evaluate_epoch(axes, data, model, criterion, epoch + 1, stats,
log_interval, log)
if store:
# Save model parameters
save_checkpoint(model, epoch + 1,
config('{}.checkpoint'.format(model_name)),
stats)
valid_loss = stats[-1][0]
if valid_loss < best_loss:
patience = config("patience")
best_loss = valid_loss
idx = epoch
patience -= 1
epoch = idx
idx = min(int((idx + 1) / log_interval), len(stats) - 1)
if verbose:
print("The loss on test dataset is:", stats[idx][2],
"obtained in epoch", epoch)
# Save figure and keep plot open
if visual:
save_training_plot(fig, model_name)
hold_training_plot()
return stats[idx][2]
log_dir = args.log_dir
test_step = args.test_step
early_stop = args.early_stop
use_pretrained = args.use_pretrained
pretrain_path = args.pretrain_path
os.makedirs(log_dir, exist_ok=True)
hyper_str = vars(args)
hyper = open(log_dir + '/hyperparameter', 'w')
hyper.write(json.dumps(hyper_str))
hyper.close()
# Source labeled mlp
network_architecture = dict(hidden=hidden, n_input=_in, n_output=_out)
mlp = MLP(network_architecture,
lr,
optimizer_type,
batch_size,
train_file,
data_index,
label_index,
epoch,
train_n,
log_dir,
use_pretrained=use_pretrained,
pretrain_path=pretrain_path)
test_d, test_l = load_data(test_file, ',', data_index, label_index)
mlp.learn(test_d, test_l, test_step, early_stop)
for dataset_name, (train, valid, test) in datasets:
batch_size = len(train[0])
results[dataset_name] = {}
for eta in ETA_VALUES:
results[dataset_name][eta] = {}
run_save_prefix = os.path.join(save_folder, f"{dataset_name}/e={eta}/")
print(run_save_prefix)
ensure_dir(run_save_prefix)
for number_of_nodes in NODES:
run_name = f'NODE{number_of_nodes:02d}_e={eta}_d={dataset_name}'.replace(".", ",")
mae_array = []
for i in range(LOOPS):
i += 1
print(run_name)
net = MLP(train[0], train[1], number_of_nodes, momentum=0, outtype="linear")
train_losses, valid_losses, _, _, pocket_epoch = net.train(
train[0], train[1], valid[0], valid[1], eta, 300000,
early_stop_count=100000, shuffle=False, batch_size=batch_size
)
net.forward(test[0])
print(mae(net.outputs, test[1]))
mae_array.append(mae(net.outputs, test[1]))
mean, stddev = statistics.mean(mae_array), statistics.stdev(mae_array)
results[dataset_name][eta][number_of_nodes] = mean, stddev
plt.title(f'Performance e={eta} d={dataset_name}')
plt.plot(NODES, np.log([results[dataset_name][eta][node][0] for node in NODES]))
plt.xlabel('number of nodes')
plt.ylabel('log of test MAE')
