def __init__(self, training_dataset_file, model_file=None):
self.__dataset = torch_load(training_dataset_file)
self.__ann = get_ann()
if model_file is not None:
self.__ann.load_state_dict(torch_load(model_file))
self.__ann.eval()
self.__optimizer_batch = SGD(self.__ann.parameters(), lr=learning_rate)
self.__loss_list = []
self.__average_loss_list = []
def validate(model_name, file_name):
ann = get_ann()
ann.load_state_dict(torch_load(model_name))
ann.eval()
difference_list = []
dataset = torch_load(file_name)
for index in range(len(dataset)):
x1, x2 = dataset.points[index]
x = as_tensor([x1, x2])
y = ann(x)
result = dataset.results[index]
difference_list.append(abs(result - y))
graph_validation(difference_list)
def __init__(self, db_path, ext):
self.db_path = db_path
self.ext = ext
if self.ext == ".npy":
self.loader = lambda x: np_load(x)
else:
self.loader = lambda x: np_load(x)["feat"]
if db_path.endswith(".lmdb"):
self.db_type = "lmdb"
self.env = lmdb_open(
db_path,
subdir=isdir(db_path),
readonly=True,
lock=False,
readahead=False,
meminit=False,
)
elif db_path.endswith(".pth"): # Assume a key,value dictionary
self.db_type = "pth"
self.feat_file = torch_load(db_path)
self.loader = lambda x: x
print("HybridLoader: ext is ignored")
elif db_path.endswith("h5"):
self.db_type = "h5"
self.loader = lambda x: np_array(x).astype("float32")
else:
self.db_type = "dir"
def _load_pretrained_base(self, load_pretrained):
if type(load_pretrained) == bool:
return torchvision_vgg16(pretrained=load_pretrained).state_dict()
elif type(load_pretrained) == OrderedDict:
return load_pretrained
else:
return torch_load(load_pretrained)
def run(file_name, x1, x2):
ann = get_ann()
ann.load_state_dict(torch_load(file_name))
ann.eval()
x1 = float(x1)
x2 = float(x2)
x = as_tensor([x1, x2])
print("f({:.3f}, {:.3f}) = {}".format(x1, x2, ann(x).tolist()))
def load(file_path, xtrn=None, method=None):
"""Load object from *file_path*. You should pass the *xtrn* object
that you received as result of the `.save()` method call for this
object. The param *method* can be either 'torch' or 'pickle'. If the
*method* is ``None`` (default), we detect it by trial and error."""
with open(file_path, 'rb') as f:
if method is None:
try:
o = pickle.load(f)
except pickle.UnpicklingError:
o = torch_load(f)
elif method == 'pickle':
o = pickle.load(f)
elif method == 'torch':
o = torch_load(f)
else:
raise ValueError(f'ERROR: Unknown method "{method}"')
if xtrn is not None:
o._push_xtrn(xtrn)
return o
def run(self, file_name, x1=None, x2=None):
if file_name == "config":
self.run_config()
return
ann = get_ann()
ann.load_state_dict(torch_load(file_name))
ann.eval()
x1 = float(x1)
x2 = float(x2)
x = as_tensor([x1, x2])
print("f({:.3f}, {:.3f}) = {}".format(x1, x2, ann(x).tolist()))
def load_model(path, model_name, params_name):
model = None
params_name = op.join(op.dirname(path), params_name)
if model_name == ToNameNet.name() or model_name == DOTNetCNN.name():
to_name = model_name == ToNameNet.name()
with open(params_name) as fd:
init_params = json.load(fd)
model = ToNameNet(**init_params) if to_name else DOTNetCNN(**init_params)
model.load_state_dict(torch_load(path))
model.eval()
return model
def __init__(self, path):
self.config_path = os.path.join(path, 'config.json')
self.model_path = os.path.join(path, 'model_best.pth')
config = json.load(open(self.config_path))
checkpoint = torch_load(self.model_path, map_location='cpu')
m_name, sd, self.classes = _get_model_att(checkpoint)
model = vgg11_bn(self.classes, pretrained=False)
model.load_state_dict(checkpoint['state_dict'])
tsf = _get_transform(config)
# self.device = torch_device("cuda" if torch_cuda.is_available() else "cpu")
self.device = torch_device("cpu")
self.model = model.to(self.device)
self.model.eval()
self.transforms = tsf
def get_answer(sentence):
with open(
os.path.join(os.path.dirname(__file__), '..', 'data',
'intents.json'), 'r') as f:
intents = json.load(f)
FILE = os.path.join(os.path.dirname(__file__), '..', 'data', 'data.pth')
data = torch_load(FILE)
input_size = data['input_size']
hidden_size = data['hidden_size']
output_size = data['output_size']
all_words = data['all_words']
tags = data['tags']
model_state = data['model_state']
device = torch_device('cuda' if torch_cuda.is_available() else 'cpu')
model = NeuralNet(input_size, hidden_size, output_size).to(device)
model.load_state_dict(model_state)
model.eval()
sentence = tokenize(sentence)
X = bag_of_words(sentence, all_words)
X = X.reshape(1, X.shape[0])
X = torch_from_numpy(X)
output = model(X)
_, predicted = torch_max(output, dim=1)
tag = tags[predicted.item()]
answer = ''
probs = torch_softmax(output, dim=1)
prob = probs[0][predicted.item()]
if prob.item() > 0.75:
for intent in intents['intents']:
if tag == intent['tag']:
answer = random.choice(intent['responses'])
else:
answer = "I don't understand..."
return answer, prob
def load_torch_object_bytes(b):
with TemporaryFile() as f:
f.write(b)
f.seek(0)
obj = torch_load(f)
return obj
def show(file_name):
dataset = torch_load(file_name)
print(str(dataset))
def train(
model_id,
sequences_per_img=5,
batch_size=10,
resnet_conv_feature_size=2048,
start_from=None,
input_json_file_name=None,
input_label_h5_file_name=None,
label_smoothing=0,
structure_loss_weight=1,
train_sample_method="sample",
train_beam_size=1,
struc_use_logsoftmax=True,
train_sample_n=5,
structure_loss_type="seqnll",
optimizer_type=NOAM,
noamopt_factor=1,
noamopt_warmup=20000,
core_optimizer="sgd",
learning_rate=0.0005,
optimizer_alpha=0.9,
optimizer_beta=0.999,
optimizer_epsilon=1e-8,
weight_decay=0,
load_best_score=True,
max_epochs=50,
scheduled_sampling_start=-1,
scheduled_sampling_increase_every=5,
scheduled_sampling_increase_prob=0.05,
scheduled_sampling_max_prob=0.25,
self_critical_after=-1,
structure_after=-1,
cached_tokens="coco-train-idxs",
grad_clip_value=0.1,
grad_clip_mode=CLIP_VALUE,
log_loss_iterations=25,
save_every_epoch=True,
save_checkpoint_iterations=3000,
save_history_ckpt=True,
eval_language_model=True,
):
#
# File names
info_file_name = (
join(start_from, "infos_" + model_id + ".pkl") if start_from is not None else ""
)
history_file_name = (
join(start_from, "histories_" + model_id + ".pkl")
if start_from is not None
else ""
)
model_file_name = join(start_from, "model.pth") if start_from is not None else ""
optimizer_file_name = (
join(start_from, "optimizer.pth") if start_from is not None else ""
)
#
# Load data
loader = DataLoader(
sequences_per_img,
batch_size=batch_size,
use_fc=True,
use_att=True,
use_box=0,
norm_att_feat=0,
norm_box_feat=0,
input_json_file_name=input_json_file_name,
input_label_h5_file_name=input_label_h5_file_name,
)
vocab_size = loader.vocab_size
seq_length = loader.seq_length
#
# Initialize training info
infos = {
"iter": 0,
"epoch": 0,
"loader_state_dict": None,
"vocab": loader.get_vocab(),
}
#
# Load existing state training information, if there is any
if start_from is not None and isfile(info_file_name):
#
with open(info_file_name, "rb") as f:
assert True
#
# Create data logger
histories = defaultdict(dict)
if start_from is not None and isfile(history_file_name):
with open(history_file_name, "rb") as f:
histories.update(pickle_load(f))
# tensorboard logger
tb_summary_writer = SummaryWriter(checkpoint_path)
#
# Create our model
vocab = loader.get_vocab()
model = Transformer(
vocab_size, resnet_conv_feature_size=resnet_conv_feature_size
).cuda()
#
# Load pretrained weights:
if start_from is not None and isfile(model_file_name):
model.load_state_dict(torch_load(model_file_name))
#
# Wrap generation model with loss function(used for training)
# This allows loss function computed separately on each machine
lw_model = LossWrapper(
model,
label_smoothing=label_smoothing,
structure_loss_weight=structure_loss_weight,
train_sample_method=train_sample_method,
train_beam_size=train_beam_size,
struc_use_logsoftmax=struc_use_logsoftmax,
train_sample_n=train_sample_n,
structure_loss_type=structure_loss_type,
)
#
# Wrap with dataparallel
dp_model = DataParallel(model)
dp_lw_model = DataParallel(lw_model)
#
# Build optimizer
if optimizer_type == NOAM:
optimizer = get_std_opt(model, factor=noamopt_factor, warmup=noamopt_warmup)
elif optimizer_type == REDUCE_LR:
optimizer = build_optimizer(
model.parameters(),
core_optimizer=core_optimizer,
learning_rate=learning_rate,
optimizer_alpha=optimizer_alpha,
optimizer_beta=optimizer_beta,
optimizer_epsilon=optimizer_epsilon,
weight_decay=weight_decay,
)
optimizer = ReduceLROnPlateau(optimizer, factor=0.5, patience=3)
else:
raise (
Exception("Only supports NoamOpt and ReduceLROnPlateau optimization types")
)
#
# # Load the optimizer
if start_from is not None and isfile(optimizer_file_name):
optimizer.load_state_dict(torch_load(optimizer_file_name))
#
# Prepare for training
iteration = infos["iter"]
epoch = infos["epoch"]
#
# For back compatibility
if "iterators" in infos:
infos["loader_state_dict"] = {
split: {
"index_list": infos["split_ix"][split],
"iter_counter": infos["iterators"][split],
}
for split in ["train", "val", "test"]
}
loader.load_state_dict(infos["loader_state_dict"])
if load_best_score == 1:
best_val_score = infos.get("best_val_score", None)
if optimizer_type == NOAM:
optimizer._step = iteration
#
# Assure in training mode
dp_lw_model.train()
epoch_done = True
#
# Start training
try:
while True:
#
# Check max epochs
if epoch >= max_epochs and max_epochs != -1:
break
#
# Update end of epoch data
if epoch_done:
#
# Assign the scheduled sampling prob
if epoch > scheduled_sampling_start and scheduled_sampling_start >= 0:
frac = (
epoch - scheduled_sampling_start
) // scheduled_sampling_increase_every
ss_prob = min(
scheduled_sampling_increase_prob * frac,
scheduled_sampling_max_prob,
)
model.ss_prob = ss_prob
#
# If start self critical training
if self_critical_after != -1 and epoch >= self_critical_after:
sc_flag = True
init_scorer(cached_tokens)
else:
sc_flag = False
#
# If start structure loss training
if structure_after != -1 and epoch >= structure_after:
struc_flag = True
init_scorer(cached_tokens)
else:
struc_flag = False
#
# End epoch update
epoch_done = False
#
# Compute time to load data
start = time.time()
data = loader.get_batch("train")
load_data_time = time.time() - start
print(f"Time to load data: {load_data_time} seconds")
########################
# SYNC
########################
synchronize()
#
# Compute time to complete epoch
start = time.time()
#
# Make sure data is in GPU memory
tmp = [
data["fc_feats"],
data["att_feats"],
data["labels"],
data["masks"],
data["att_masks"],
]
tmp = [_ if _ is None else _.cuda() for _ in tmp]
fc_feats, att_feats, labels, masks, att_masks = tmp
#
# Reset gradient
optimizer.zero_grad()
#
print("MADE IT TO THE MODEL EVALUATION")
#
# Evaluate model
model_out = dp_lw_model(
fc_feats,
att_feats,
labels,
masks,
att_masks,
data["gts"],
torch_arange(0, len(data["gts"])),
sc_flag,
struc_flag,
)
#
# Average loss over training batch
loss = model_out["loss"].mean()
#
# Compute gradient
loss.backward()
#
# Clip gradient
if grad_clip_value != 0:
gradient_clipping_functions[grad_clip_mode](
model.parameters(), grad_clip_value
)
#
# Update
optimizer.step()
train_loss = loss.item()
end = time.time()
########################
# SYNC
########################
synchronize()
#
# Output status
if struc_flag:
print(
"iter {} (epoch {}), train_loss = {:.3f}, lm_loss = {:.3f}, struc_loss = {:.3f}, time/batch = {:.3f}".format(
iteration,
epoch,
train_loss,
model_out["lm_loss"].mean().item(),
model_out["struc_loss"].mean().item(),
end - start,
)
)
elif not sc_flag:
print(
"iter {} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}".format(
iteration, epoch, train_loss, end - start
)
)
else:
print(
"iter {} (epoch {}), avg_reward = {:.3f}, time/batch = {:.3f}".format(
iteration, epoch, model_out["reward"].mean(), end - start
)
)
#
# Update the iteration and epoch
iteration += 1
if data["bounds"]["wrapped"]:
epoch += 1
epoch_done = True
#
# Write the training loss summary
if iteration % log_loss_iterations == 0:
tb_summary_writer.add_scalar("train_loss", train_loss, iteration)
if optimizer_type == NOAM:
current_lr = optimizer.rate()
elif optimizer_type == REDUCE_LR:
current_lr = optimizer.current_lr
tb_summary_writer.add_scalar("learning_rate", current_lr, iteration)
tb_summary_writer.add_scalar(
"scheduled_sampling_prob", model.ss_prob, iteration
)
if sc_flag:
tb_summary_writer.add_scalar(
"avg_reward", model_out["reward"].mean(), iteration
)
elif struc_flag:
tb_summary_writer.add_scalar(
"lm_loss", model_out["lm_loss"].mean().item(), iteration
)
tb_summary_writer.add_scalar(
"struc_loss", model_out["struc_loss"].mean().item(), iteration
)
tb_summary_writer.add_scalar(
"reward", model_out["reward"].mean().item(), iteration
)
tb_summary_writer.add_scalar(
"reward_var", model_out["reward"].var(1).mean(), iteration
)
histories["loss_history"][iteration] = (
train_loss if not sc_flag else model_out["reward"].mean()
)
histories["lr_history"][iteration] = current_lr
histories["ss_prob_history"][iteration] = model.ss_prob
#
# Update infos
infos["iter"] = iteration
infos["epoch"] = epoch
infos["loader_state_dict"] = loader.state_dict()
#
# Make evaluation on validation set, and save model
if (
iteration % save_checkpoint_iterations == 0 and not save_every_epoch
) or (epoch_done and save_every_epoch):
#
# Evaluate model on Validation set of COCO
eval_kwargs = {"split": "val", "dataset": input_json_file_name}
val_loss, predictions, lang_stats = eval_split(
dp_model,
lw_model.crit,
loader,
verbose=True,
verbose_beam=False,
verbose_loss=True,
num_images=-1,
split="val",
lang_eval=False,
dataset="coco",
beam_size=1,
sample_n=1,
remove_bad_endings=False,
dump_path=False,
dump_images=False,
job_id="FUN_TIME",
)
#
# Reduces learning rate if no improvement in objective
if optimizer_type == REDUCE_LR:
if "CIDEr" in lang_stats:
optimizer.scheduler_step(-lang_stats["CIDEr"])
else:
optimizer.scheduler_step(val_loss)
#
# Write validation result into summary
tb_summary_writer.add_scalar("validation loss", val_loss, iteration)
if lang_stats is not None:
for k, v in lang_stats.items():
tb_summary_writer.add_scalar(k, v, iteration)
histories["val_result_history"][iteration] = {
"loss": val_loss,
"lang_stats": lang_stats,
"predictions": predictions,
}
#
# Save model if is improving on validation result
if eval_language_model:
current_score = lang_stats["CIDEr"]
else:
current_score = -val_loss
best_flag = False
if best_val_score is None or current_score > best_val_score:
best_val_score = current_score
best_flag = True
#
# Dump miscalleous informations
infos["best_val_score"] = best_val_score
#
# Save checkpoints...seems only most recent one keep histories,
# and it's overwritten each time
save_checkpoint(
model,
infos,
optimizer,
checkpoint_dir=checkpoint_path,
histories=histories,
append="RECENT",
)
if save_history_ckpt:
save_checkpoint(
model,
infos,
optimizer,
checkpoint_dir=checkpoint_path,
append=str(epoch) if save_every_epoch else str(iteration),
)
if best_flag:
save_checkpoint(
model,
infos,
optimizer,
checkpoint_dir=checkpoint_path,
append="BEST",
)
except (RuntimeError, KeyboardInterrupt):
print(f'{BAR("=", 20)}Save checkpoint on exception...')
save_checkpoint(
model, infos, optimizer, checkpoint_dir=checkpoint_path, append="EXCEPTION"
)
print(f'...checkpoint saved.{BAR("=", 20)}')
stack_trace = format_exc()
print(stack_trace)
def process_each_frequency(model_dirname, stft, frequency, using_cuda=True):
'''
Setter method on stft.
'''
is_using_cuda = using_cuda and torch_cuda_is_cuda_available()
my_device = torch_device('cuda:0' if is_using_cuda else 'cpu')
# 1. Instantiate Neural Network Model
model_params_fname = os_path_join(
os_path_join(model_dirname, 'k_' + str(frequency)), MODEL_PARAMS_FNAME)
model_save_fpath = os_path_join(model_dirname, 'k_' + str(frequency),
MODEL_SAVE_FNAME)
model = get_which_model_from_params_fname(model_params_fname)
model.load_state_dict(torch_load(os_path_join(
os_path_dirname(model_save_fpath), 'model.dat'),
map_location=my_device),
strict=True)
model.eval()
model = model.to(my_device)
if False:
model.printing = True
from lib.print_layer import PrintLayer
new_model_net = []
for layer in model.net:
new_model_net.append(layer)
new_model_net.append(PrintLayer(layer))
from torch.nn import Sequential
model.net = Sequential(*new_model_net)
# 2. Get X_test
LOGGER.debug('r3.process_each_frequency: stft.shape = {}'.format(
stft.shape))
aperture_data = stft[:, :, frequency] # or stft_frequency
# 2.1. normalize by L1 norm
aperture_data_norm = np_linalg_norm(aperture_data, ord=np_inf, axis=1)
aperture_data /= aperture_data_norm[:, np_newaxis]
# load into torch and onto gpu
aperture_dataset_eval = ApertureDatasetEval(aperture_data)
aperture_dataset_loader = DataLoader(aperture_dataset_eval,
batch_size=EVAL_BATCH_SIZE,
shuffle=False,
num_workers=DATALOADER_NUM_WORKERS,
pin_memory=using_cuda)
# 3. Predict
if is_using_cuda is True:
torch_cuda_empty_cache()
aperture_data_new = predict(model, aperture_dataset_loader, my_device)
del aperture_data, model, aperture_dataset_eval, aperture_dataset_loader, my_device
if is_using_cuda is True:
torch_cuda_empty_cache()
# 4. Postprocess on y_hat
# rescale the data and store new data in stft
stft[:, :,
frequency] = aperture_data_new * aperture_data_norm[:, np_newaxis]
del aperture_data_new, aperture_data_norm
def run(
*,
raw_dir: str,
uasts_dir: str,
instance_file: str,
tensors_dir: str,
checkpoint_file: str,
configs_dir: str,
training_configs_dir: str,
prefix: str,
metadata_dir: Optional[str],
log_level: str,
) -> None:
"""Run the model and output CodRep predictions."""
arguments = locals()
configs_dir_path = Path(configs_dir).expanduser().resolve()
configs_dir_path.mkdir(parents=True, exist_ok=True)
training_configs_dir_path = Path(training_configs_dir).expanduser().resolve()
tensors_dir_path = Path(tensors_dir).expanduser().resolve()
Config.from_arguments(
arguments, ["instance_file", "checkpoint_file"], "configs_dir"
).save(configs_dir_path / "train.json")
logger = setup_logging(__name__, log_level)
training_configs = {}
for step in ["parse", "tensorize", "train"]:
with (training_configs_dir_path / step).with_suffix(".json").open(
"r", encoding="utf8"
) as fh:
training_configs[step] = json_load(fh)
parse(
raw_dir=raw_dir,
uasts_dir=uasts_dir,
configs_dir=configs_dir,
log_level=log_level,
)
tensorize(
uasts_dir=uasts_dir,
instance_file=instance_file,
tensors_dir=tensors_dir,
configs_dir=configs_dir,
n_workers=training_configs["tensorize"]["options"]["n_workers"],
pickle_protocol=training_configs["tensorize"]["options"]["pickle_protocol"],
log_level=log_level,
)
dataset = CodRepDataset(input_dir=tensors_dir_path)
logger.info(f"Dataset of size {len(dataset)}")
with bz2_open(instance_file, "rb") as fh_instance:
instance = pickle_load(fh_instance)
model = build_model(
instance=instance,
model_decoder_type=training_configs["train"]["options"]["model_decoder_type"],
model_encoder_iterations=training_configs["train"]["options"][
"model_encoder_iterations"
],
model_encoder_output_dim=training_configs["train"]["options"][
"model_encoder_output_dim"
],
model_encoder_message_dim=training_configs["train"]["options"][
"model_encoder_message_dim"
],
model_learning_rate=training_configs["train"]["options"]["model_learning_rate"],
model_batch_size=training_configs["train"]["options"]["model_batch_size"],
train_dataset=dataset,
eval_dataset=None,
test_dataset=None,
)
# The model needs a forward to be completely initialized.
model.training_step(instance.collate([dataset[0]]), 0)
logger.info(f"Configured model {model}")
model.load_state_dict(
torch_load(checkpoint_file, map_location="cpu")["model_state_dict"]
)
model.eval()
logger.info(f"Loaded model parameters from %s", checkpoint_file)
metadata = None if metadata_dir is None else model.build_metadata()
metadata_output = (
None if metadata_dir is None else Path(metadata_dir) / "metadata.json"
)
dataloader = model.train_dataloader()
graph_field = instance.get_field_by_type("graph")
label_field = instance.get_field_by_type("label")
indexes_field = instance.get_field_by_type("indexes")
metadata_field = instance.get_field_by_type("metadata")
graph_input_fields = instance.get_fields_by_type("input")
graph_input_dimensions = [48, 48, 32]
feature_names = [field.name for field in graph_input_fields]
with no_grad():
for batch in dataloader:
graph, etypes = batch[graph_field.name]
features = [batch[field_name] for field_name in feature_names]
indexes, offsets = batch[indexes_field.name].indexes
forward = model.forward(graph, etypes, features, indexes)
model.decode(
batched_graph=graph,
indexes=indexes,
offsets=offsets,
forward=forward,
paths=batch[metadata_field.name],
prefix=prefix,
metadata=metadata,
)
if metadata_output is not None:
with metadata_output.open("w", encoding="utf8") as fh:
json_dump(metadata, fh)
def load_model(path):
model = DroneQNet(2, IMG_W, IMG_H, len(actions))
model.load_state_dict(torch_load(path))
model.eval()
model.double()
return model
def load_checkpoint(filename):
if not os.path.isfile(filename):
raise FileNotFoundError('Checkpoint {} not found'.format(filename))
checkpoint = torch_load(filename, map_location='cpu')
return checkpoint
class GazeDetector:
"""
Detect a user's gaze to see if they are engaged
or not
"""
weights_path: str = os.path.abspath("machine_learning/models/weights/gazenet.pth")
coordinate_length: float = 200
device: torch_device = torch_device("cuda:0" if torch_gpu.is_available() else "cpu")
model: gaze_cnn.GazeNet = gaze_cnn.GazeNet(device=device)
model.load_state_dict(torch_load(weights_path, map_location=device))
model.eval()
def __init__(self, logger, base_64_image):
"""
Base 64 image from the student app
:param base_64_image: string from client in request
"""
self.logger = logger
self.opencv_image = self._read_base64(base_64_image)
self.face_detector = FaceDetector(device=self.device)
@staticmethod
def _read_base64(base64_string: str) -> cvtColor:
"""
Read base64 string into CV2
:param base64_string: base 64 string
:return: cv2 image
"""
byte_buffer: BytesIO = BytesIO()
byte_buffer.write(base64_decode(base64_string))
pil_img: Image = Image.open(byte_buffer)
return cvtColor(np.array(pil_img), COLOR_RGB2BGR)
@staticmethod
def _normalize_face(landmarks, frame) -> tuple:
"""
Normalize the face to a standard map
:param landmarks: landmarks on the face
:param frame: image frame
:return: the face, gaze origin and
"""
left_eye_coord: tuple = (0.70, 0.35)
lcenter: tuple = tuple([landmarks[0], landmarks[5]])
rcenter: tuple = tuple([landmarks[1], landmarks[6]])
gaze_origin: tuple = (int((lcenter[0] + rcenter[0]) / 2), int((lcenter[1] + rcenter[1]) / 2))
dY: float = rcenter[1] - lcenter[1]
dX: float = rcenter[0] - lcenter[0]
angle = np.degrees(np.arctan2(dY, dX)) - 180
right_eye_x = 1.0 - left_eye_coord[0]
dist: np.float32 = np.sqrt((dX ** 2) + (dY ** 2))
new_dist = (right_eye_x - left_eye_coord[0])
new_dist *= 112
scale: np.float32 = new_dist / dist
M = getRotationMatrix2D(gaze_origin, angle, scale)
tX = 112 * 0.5
tY = 112 * left_eye_coord[1]
M[0, 2] += (tX - gaze_origin[0])
M[1, 2] += (tY - gaze_origin[1])
face = warpAffine(frame, M, (112, 112), flags=INTER_CUBIC)
return face, gaze_origin, M
@staticmethod
def _get_vector(pitch_yaw: List) -> (np.float32, np.float32):
"""
Get the deltas in the x and y direction to measure gaze
:param pitch_yaw: gaze coords from CNN
:return: dx, dy
"""
dx: np.float32 = -GazeDetector.coordinate_length * np.sin(pitch_yaw[1])
dy: np.float32 = -GazeDetector.coordinate_length * np.sin(pitch_yaw[0]) # coords start in upper left corner
return dx, dy
@staticmethod
def _compute_angle(dx: float, dy: float) -> np.float32:
"""
Computes the angle in degrees of the vector defined
by dx and dy
:param dx: delta x for vector
:param dy: delta y for vector
:return: angle between standard position and the vector in degs
"""
return np.arctan2(dy, dx) * (180 / np.pi) # convert to degrees from radians
@staticmethod
def _compute_magnitude(dx: float, dy: float) -> float:
"""
Compute the magnitude of vector <dy, dx>
:param dx: delta x for vector
:param dy: delta y for vector
:return: magnitude of vector <dx, dy>
"""
return np.sqrt(dx ** 2 + dy ** 2) # find magnitude of vector
def predict_gaze(self) -> (float, float):
"""
Predict the Gaze trajectory as a vector using our GazeNet CNN
and return the angle and magnitude of the vector
:return: angle between delta x and delta y
"""
self.opencv_image = flip_image(self.opencv_image[:, :, ::-1], 1) # helps to rectify rectangular coordinates
faces, landmarks = self.face_detector.detect(Image.fromarray(self.opencv_image))
if len(faces) == 0:
self.logger.info("No faces found!")
return 0, 0, "NOT ENGAGED", 0
for face, landmark in zip(faces, landmarks):
if face[-1] > 0.98: # confidence check
normal_face, gaze_origin, _ = GazeDetector._normalize_face(landmark, self.opencv_image)
# Predict Gaze
with torch_no_grad():
gaze = GazeDetector.model.get_gaze(normal_face)
gaze = gaze[0].data.cpu()
#self.logger("Predicted on CNN: Gaze com puted to be " + str(gaze))
dx, dy = self._get_vector(gaze)
dy *= -1
#self.logger.info("Original Gaze Location: " + str(gaze_origin))
#self.logger.info("Delta X: " + str(dx) + "; Delta Y: " + str(dy))
computed_angle = self._compute_angle(dx, dy).cpu().numpy()
computed_magnitude = self._compute_magnitude(dx, dy).cpu().numpy()
classification, score = self.interpret_vectors(computed_angle, computed_magnitude)
#self.logger.info("Trying to save gaze computation")
# showComputedGaze(Image.fromarray(self.opencv_image), gaze_origin, gaze, computed_angle, computed_magnitude, score, classification)
return computed_angle, computed_magnitude, classification, score
else:
return 0, 0, "NOT ENGAGED", 0
def interpret_vectors(self, angle: np.float32, magnitude: np.float32):
"""
Interpret the Vector's Position and Magnitude for engagement/disengagement
:param angle: the angle of the gaze vector
:param magnitude: the magnitude of the gaze vector
:return: student state and confidence
"""
Rule = namedtuple('Rule', ['trigger_str', 'confidence', 'result', 'score'])
rules: List[Rule] = [ # in order of precedence
Rule('angle == -1.0 and magnitude == -1.0', 0.0, "NOT ENGAGED", 0),
Rule('magnitude <= 30.0 or 0 < angle <= 15', 0.95, "ENGAGED", 10),
Rule('magnitude <= 40.0', 0.95, "ENGAGED", 9),
Rule('magnitude <= 50.0', 0.60, "ENGAGED",8),
Rule('magnitude <= 60.0', 0.65, "ENGAGED", 7),
Rule('15 < angle < 50', 0.70, "SOMEWHAT ENGAGED", 6),
Rule('60 < magnitude <= 70', 0.65, "SOMEWHAT ENGAGED", 5),
Rule('70 < magnitude <= 120', 0.70, "NOT ENGAGED", 4),
Rule('120 < magnitude < 360', 0.70, "NOT ENGAGED", 3),
Rule('magnitude > 60.0', 0.70, "NOT ENGAGED", 2),
Rule('magnitude > 50.0', 0.85, "NOT ENGAGED", 1),
Rule('True', 0.0, "ENGAGED", 10) # default condition (base case-- should never be called)
]
for rule_no, rule in enumerate(rules):
if eval(rule.trigger_str):
self.logger.info("Triggered rule: " + rule.trigger_str + ": " + str(rule))
return rule.result, rule.score
def load_model(self, model_file_full_path):
print("loading parameters from : ", model_file_full_path)
modelParams = torch_load(model_file_full_path)
print("load_state_dict - ")
self.load_state_dict(modelParams)
self.eval()
def load_layer(layer: nn.Module, path: str): return layer.load_state_dict(torch_load(path))
def run(
*,
raw_dir: str,
uasts_dir: str,
instance_file: str,
tensors_dir: str,
checkpoint_file: str,
configs_dir: str,
training_configs_dir: str,
prefix: str,
metadata_dir: Optional[str],
log_level: str,
) -> None:
"""Run the model and output CodRep predictions."""
arguments = locals()
configs_dir_path = Path(configs_dir).expanduser().resolve()
configs_dir_path.mkdir(parents=True, exist_ok=True)
training_configs_dir_path = Path(
training_configs_dir).expanduser().resolve()
tensors_dir_path = Path(tensors_dir).expanduser().resolve()
Config.from_arguments(arguments, ["instance_file", "checkpoint_file"],
"configs_dir").save(configs_dir_path / "train.json")
logger = setup_logging(__name__, log_level)
training_configs = {}
for step in ["parse", "tensorize", "train"]:
with (training_configs_dir_path / step).with_suffix(".json").open(
"r", encoding="utf8") as fh:
training_configs[step] = json_load(fh)
parse(
raw_dir=raw_dir,
uasts_dir=uasts_dir,
configs_dir=configs_dir,
log_level=log_level,
)
tensorize(
uasts_dir=uasts_dir,
instance_file=instance_file,
tensors_dir=tensors_dir,
configs_dir=configs_dir,
n_workers=training_configs["tensorize"]["options"]["n_workers"],
pickle_protocol=training_configs["tensorize"]["options"]
["pickle_protocol"],
log_level=log_level,
)
dataset = CodRepDataset(input_dir=tensors_dir_path)
logger.info(f"Dataset of size {len(dataset)}")
with bz2_open(instance_file, "rb") as fh:
instance = pickle_load(fh)
model = build_model(
instance=instance,
model_decoder_type=training_configs["train"]["options"]
["model_decoder_type"],
model_encoder_iterations=training_configs["train"]["options"]
["model_encoder_iterations"],
model_encoder_output_dim=training_configs["train"]["options"]
["model_encoder_output_dim"],
model_encoder_message_dim=training_configs["train"]["options"]
["model_encoder_message_dim"],
)
# The model needs a forward to be completely initialized.
model(instance.collate([dataset[0]]))
logger.info(f"Configured model {model}")
model.load_state_dict(
torch_load(checkpoint_file, map_location="cpu")["model_state_dict"])
model.eval()
logger.info(f"Loaded model parameters from %s", checkpoint_file)
dataloader = DataLoader(
dataset,
shuffle=False,
collate_fn=instance.collate,
batch_size=10,
num_workers=1,
)
metadata = None if metadata_dir is None else model.build_metadata()
metadata_output = (None if metadata_dir is None else Path(metadata_dir) /
"metadata.json")
with no_grad():
for sample in dataloader:
sample = model(sample)
model.decode(sample=sample, prefix=prefix, metadata=metadata)
if metadata_output is not None:
with metadata_output.open("w", encoding="utf8") as fh:
json_dump(metadata, fh)
def load_checkpoint(path):
if is_checkpoint(path):
return torch_load(path)
else:
raise RuntimeError("Checkpoint at '{}' does not exist!".format(path))
def define_ml_removal_heuristic(number_of_node_features,
number_of_edge_features,
network_params_file,
network=NETWORK_GCN):
device = 'cuda' if is_cuda_available() else 'cpu'
dropout = 0.0
training_state = torch_load(MODEL_PARAMETERS_PATH + network_params_file,
map_location=torch_device(device))
if network == NETWORK_GATEDGCN:
net_params = {
'in_dim': number_of_node_features,
'in_dim_edge': number_of_edge_features,
'hidden_dim': 70,
'out_dim': 70,
'n_classes': OUTPUT_SIZE,
'dropout': dropout,
'L': len(HIDDEN_NODE_DIMENSIONS),
'readout': 'mean',
'graph_norm': False,
'batch_norm': False,
'residual': False,
'edge_feat': True,
'device': device
}
model = GatedGCNNet(net_params)
model.load_state_dict(training_state)
else:
model = GCNNet(input_node_features=number_of_node_features,
hidden_node_dimension_list=HIDDEN_NODE_DIMENSIONS,
input_edge_features=number_of_edge_features,
hidden_edge_dimension_list=HIDDEN_EDGE_DIMENSIONS,
hidden_linear_dimension_list=HIDDEN_LINEAR_DIMENSIONS,
output_feature=OUTPUT_SIZE,
dropout_probability=dropout,
device=device)
model.load_state_dict(
training_state['graph_convolutional_network_state'])
model.eval()
@no_grad()
def ml_removal_heuristic(state, random_state):
nodes_to_destroy = select_random_nodes(state, random_state)
update_features_from_destroyed_nodes(state, nodes_to_destroy)
prediction = argmax(model(state.dgl_graph,
state.dgl_graph.ndata['n_feat'],
state.dgl_graph.edata['e_feat'],
state.node_snorm, state.edge_snorm),
dim=1)
are_nodes_accepted = (prediction == 2)
number_of_tries = state.size
while not are_nodes_accepted and number_of_tries > 0:
nodes_to_destroy = select_random_nodes(state, random_state)
update_features_from_destroyed_nodes(state, nodes_to_destroy)
prediction = argmax(model(state.dgl_graph,
state.dgl_graph.ndata['n_feat'],
state.dgl_graph.edata['e_feat'],
state.node_snorm, state.edge_snorm),
dim=1).item()
random_accept_nodes = True if np.random.random(
) < ACCEPT_NODES_RANDOMLY else False
are_nodes_accepted = True if (prediction == 2
or random_accept_nodes) else False
number_of_tries -= 1
return remove_nodes(state, list(nodes_to_destroy))
return ml_removal_heuristic
