def __post_init__(self):
# Fetching gesture mappings
with open('gestop/data/static_gesture_mapping.json', 'r') as jsonfile:
self.static_gesture_mapping = json.load(jsonfile)
with open('gestop/data/dynamic_gesture_mapping.json', 'r') as jsonfile:
self.dynamic_gesture_mapping = json.load(jsonfile)
with open(self.config_path, 'r') as jsonfile:
self.gesture_action_mapping = json.load(jsonfile)
self.mouse = Controller()
self.user_config = UserConfig()
# Setting up networks
if not self.lite:
logging.info('Loading GestureNet...')
self.gesture_net = GestureNet(self.static_input_dim,
self.static_output_classes,
self.static_gesture_mapping)
self.gesture_net.load_state_dict(
torch.load(self.static_path, map_location=self.map_location))
self.gesture_net.eval()
logging.info('Loading ShrecNet..')
self.shrec_net = ShrecNet(self.dynamic_input_dim,
self.dynamic_output_classes,
self.dynamic_gesture_mapping)
self.shrec_net.load_state_dict(
torch.load(self.dynamic_path, map_location=self.map_location))
self.shrec_net.eval()
class Config:
''' The configuration of the application. '''
if not torch.cuda.is_available():
map_location = torch.device('cpu')
else:
map_location = None
if not os.path.exists('gestop/logs'):
os.mkdir('gestop/logs')
# Set up logger
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[
logging.FileHandler("gestop/logs/debug{}.log".format(
datetime.datetime.now().strftime("%m.%d:%H.%M.%S"))),
logging.StreamHandler(stdout)
])
# Disabled to prevent debug output by matplotlib
logging.getLogger('matplotlib.font_manager').disabled = True
# If lite is true, then the neural networks are not loaded into the config
# This is useful in scripts which do not use the network, or may modify the network.
lite: bool
# Path to action configuration file
config_path: str = 'gestop/data/action_config.json'
# Seed value for reproducibility
seed_val: int = 42
# Refer make_vector() in train_model.py to verify input dimensions
static_input_dim: int = 49
static_output_classes: int = 7
# Refer format_mediapipe() in dynamic_train_model.py to verify input dimensions
dynamic_input_dim: int = 36
dynamic_output_classes: int = 15
shrec_output_classes: int = 14
# Minimum number of epochs
min_epochs: int = 15
static_batch_size: int = 64
dynamic_batch_size: int = 1
pretrained: bool = True
# value for pytorch-lighting trainer attribute accumulate_grad_batches
grad_accum: int = 2
static_gesture_mapping: dict = field(default_factory=dict)
dynamic_gesture_mapping: dict = field(default_factory=dict)
# Screen Resolution
resolution: Tuple = get_screen_resolution()
# Mapping of gestures to actions
gesture_action_mapping: dict = field(default_factory=dict)
static_path: str = 'gestop/models/gesture_net.pth'
shrec_path: str = 'gestop/models/shrec_net.pth'
dynamic_path: str = 'gestop/models/user_net.pth'
gesture_net: GestureNet = field(init=False)
shrec_net: ShrecNet = field(init=False)
# Mouse tracking
mouse: Controller = field(init=False)
# How much a single scroll action should scroll
scroll_unit: int = 10
# Specifying how to map webcam coordinates to the monitor coordinates.
# Format - [x1,y1,x2,y2] where (x1,y1) specifies which coordinate to map to
# the top left of your screen and (x2,y2) specifies which coordinate to map
# to the bottom right of your screen.
map_coord = [0.2, 0.2, 0.8, 0.8]
# User configuration
user_config: UserConfig = field(init=False)
def __post_init__(self):
# Fetching gesture mappings
with open('gestop/data/static_gesture_mapping.json', 'r') as jsonfile:
self.static_gesture_mapping = json.load(jsonfile)
with open('gestop/data/dynamic_gesture_mapping.json', 'r') as jsonfile:
self.dynamic_gesture_mapping = json.load(jsonfile)
with open(self.config_path, 'r') as jsonfile:
self.gesture_action_mapping = json.load(jsonfile)
self.mouse = Controller()
self.user_config = UserConfig()
# Setting up networks
if not self.lite:
logging.info('Loading GestureNet...')
self.gesture_net = GestureNet(self.static_input_dim,
self.static_output_classes,
self.static_gesture_mapping)
self.gesture_net.load_state_dict(
torch.load(self.static_path, map_location=self.map_location))
self.gesture_net.eval()
logging.info('Loading ShrecNet..')
self.shrec_net = ShrecNet(self.dynamic_input_dim,
self.dynamic_output_classes,
self.dynamic_gesture_mapping)
self.shrec_net.load_state_dict(
torch.load(self.dynamic_path, map_location=self.map_location))
self.shrec_net.eval()
def main():
''' Main '''
C = Config(lite=True)
init_seed(C.seed_val)
##################
# INPUT PIPELINE #
##################
# Read and format the csv
df = pd.read_csv("data/static_gestures_data.csv")
train, test = train_test_split(df, test_size=0.1, random_state=C.seed_val)
train_X, train_Y = split_dataframe(train)
test_X, test_Y = split_dataframe(test)
# One Hot Encoding of the target classes
train_Y = np.array(train_Y)
test_Y = np.array(test_Y)
le = LabelEncoder()
le.fit(train_Y)
# Store encoding to disk
le_name_mapping = dict(
zip([int(i) for i in le.transform(le.classes_)], le.classes_))
logging.info(le_name_mapping)
with open('data/static_gesture_mapping.json', 'w') as f:
f.write(json.dumps(le_name_mapping))
train_Y = le.transform(train_Y)
test_Y = le.transform(test_Y)
train_loader = format_and_load(train_X, train_Y, C.static_batch_size)
test_loader = format_and_load(test_X, test_Y, C.static_batch_size)
gesture_net = GestureNet(C.static_input_dim, C.static_output_classes,
C.static_gesture_mapping)
early_stopping = EarlyStopping(
patience=3,
verbose=True,
)
wandb_logger = pl_loggers.WandbLogger(save_dir='logs/',
name='gesture_net',
project='gestop')
trainer = Trainer(gpus=1,
deterministic=True,
logger=wandb_logger,
min_epochs=C.min_epochs,
early_stop_callback=early_stopping)
trainer.fit(gesture_net, train_loader, test_loader)
# gesture_net.load_state_dict(torch.load(PATH))
trainer.test(gesture_net, test_dataloaders=test_loader)
################
# SAVING MODEL #
################
torch.save(gesture_net.state_dict(), C.static_path)