def __convert(self, value, d_type, default, sub_space=None):
"""
AUTHORS:
--------
:author: Samuel Westlake
DESCRIPTION:
------------
Converts a given value to a given data type, and returns the result.
If the value can not be converted, the given default value is returned.
NB: If d_type is given in a list, e.g. [str], it is assume that value should be a list and each item in value
will be converted to the given data type. If any item in the list cannot be converted, None will be returned.
PARAMETERS:
-----------
:param value: the value to be converted
:param d_type: the data type to convert the value to
:param default: the default value to return if the current value
:param sub_space: the list of strings that defines the current sub-space
RETURN:
-------
:return: new_value
"""
sub_space = [] if sub_space is None else sub_space
sub_space = DEEP_CONFIG_DIVIDER.join(sub_space)
# If the value is not set
if value is None and default is not None:
# Notify user that default is being used
if default == {}:
new_value = Namespace()
else:
new_value = default
if not default == []:
Notification(DEEP_NOTIF_WARNING, DEEP_MSG_CONFIG_NOT_SET % (sub_space, default))
elif d_type is dict:
new_value = Namespace(convert_dict(value.get_all()))
else:
new_value = convert(value, d_type)
if new_value is None:
# Notify user that default is being used
Notification(
DEEP_NOTIF_WARNING, DEEP_MSG_CONFIG_NOT_CONVERTED % (
sub_space,
value,
self.__get_dtype_name(d_type),
default
)
)
return new_value
def convert(value, d_type=None):
"""
Convert a value or list of values to data type in order of preference: (float, bool, str)
:param value: value to convert
:param d_type: data type to convert to
:return: converted value
"""
if value is None:
return None
elif d_type is None:
if value is None:
return None
elif isinstance(value, list):
return [convert(item) for item in value]
else:
new_value = convert2float(value)
if new_value is not None:
if round(new_value, 0) == new_value:
return int(new_value)
else:
return new_value
if isinstance(new_value,
str) and new_value.lower() in ["true", "false"]:
new_value = convert2bool(value)
if new_value is not None:
return new_value
return str(value)
elif d_type is str:
return str(value)
elif d_type is int:
return convert2int(value)
elif d_type is float:
return convert2float(value)
elif d_type is bool:
return convert2bool(value)
elif d_type is dict:
try:
return convert_namespace(value)
except AttributeError:
return None
elif isinstance(d_type, dict):
new_value = {}
for key, item in d_type.items():
try:
new_value[key] = convert(value[key], d_type=item)
except KeyError:
new_value[key] = None
except TypeError:
return None
return Namespace(new_value)
elif isinstance(d_type, list):
value = value if isinstance(value, list) else [value]
new_value = []
for item in value:
new_item = convert(item, d_type[0])
if new_item is None:
return None
else:
new_value.append(new_item)
return new_value
def __init__(self, transform_files=None):
super(OutputTransformer, self).__init__()
self.output_transformer = []
if transform_files is not None:
for file in transform_files:
self.output_transformer.append(Namespace(file))
self.load_transform_functions()
def clear_config(self):
"""
Author: SW
Reset the config to an empty Namespace
:return: None
"""
self.config = Namespace()
def load_config(self):
"""
AUTHORS:
--------
:author: Samuel Westlake
DESCRIPTION:
------------
Loads each of the config files in the config directory into self.config
self.check_config() is called penultimately
self.store_config() is called finally
RETURN:
-------
:return: None
"""
try:
Notification(DEEP_NOTIF_INFO, DEEP_MSG_CONFIG_LOADING_DIR % self.config_dir)
# If the config directory exists
if os.path.isdir(self.config_dir):
self.config = Namespace()
# For each expected configuration file
for key, file_name in DEEP_CONFIG_FILES.items():
config_path = "%s/%s" % (self.config_dir, file_name)
if os.path.isfile(config_path):
# Notification(DEEP_NOTIF_INFO, DEEP_MSG_CONFIG_LOADING_FILE % config_path)
try:
self.config.add({key: Namespace(config_path)})
except yaml.scanner.ScannerError as e:
file, line, column = str(e).split(", ")
error, file = file.split("\n")
file = file[5:].replace('"', "")
Notification(
DEEP_NOTIF_FATAL, "Unable to load config file : %s" % error,
solutions="See %s : %s : %s" % (file, line, column)
)
self.check_config(key=key)
else:
self.config = Namespace()
Notification(DEEP_NOTIF_FATAL, DEEP_MSG_FILE_NOT_FOUND % config_path)
Notification(DEEP_NOTIF_SUCCESS, DEEP_MSG_CONFIG_COMPLETE)
else:
Notification(DEEP_NOTIF_ERROR, DEEP_MSG_DIR_NOT_FOUND % self.config_dir)
self.store_config()
except DeepError:
self.sleep()
def __init_config(self):
"""
Author: SW
Writes the config directory and files into the project directory using DEEP_CONFIG
:return: None
"""
config = Namespace(DEEP_CONFIG)
config_dir = "%s/%s/config" % (self.main_path, self.project_name)
os.makedirs(config_dir, exist_ok=True)
for key, namespace in config.get().items():
if isinstance(namespace, Namespace):
namespace.save("%s/%s%s" % (config_dir, key, DEEP_EXT_YAML))
def load_config(self):
"""
AUTHORS:
--------
:author: Samuel Westlake
DESCRIPTION:
------------
Loads each of the config files in the config directory into self.config
self.check_config() is called penultimately
self.store_config() is called finally
RETURN:
-------
:return: None
"""
Notification(DEEP_NOTIF_INFO,
DEEP_MSG_CONFIG_LOADING_DIR % self.config_dir)
# If the config directory exists
if os.path.isdir(self.config_dir):
self.config = Namespace()
# For each expected configuration file
for key, file_name in DEEP_CONFIG_FILES.items():
config_path = "%s/%s" % (self.config_dir, file_name)
if os.path.isfile(config_path):
# Notification(DEEP_NOTIF_INFO, DEEP_MSG_CONFIG_LOADING_FILE % config_path)
self.config.add({key: Namespace(config_path)})
self.check_config(key=key)
else:
self.config = Namespace()
Notification(DEEP_NOTIF_FATAL,
DEEP_MSG_FILE_NOT_FOUND % config_path)
Notification(DEEP_NOTIF_SUCCESS, DEEP_MSG_CONFIG_COMPLETE)
else:
Notification(DEEP_NOTIF_ERROR,
DEEP_MSG_DIR_NOT_FOUND % self.config_dir)
self.store_config()
def load_config(self):
"""
Author: SW
Function: Checks current config path is valid, if not, user is prompted to give another
:return: bool: True if a valid config path is set, otherwise, False
"""
Notification(DEEP_NOTIF_INFO, DEEP_MSG_LOAD_CONFIG_START % self.config_dir)
# If the config directory exists
if os.path.isdir(self.config_dir):
self.clear_config()
# For each expected configuration file
for key, file_name in DEEP_CONFIG_FILES.items():
config_path = "%s/%s" % (self.config_dir, file_name)
if os.path.isfile(config_path):
self.config.add({key: Namespace(config_path)})
Notification(DEEP_NOTIF_SUCCESS, DEEP_MSG_LOAD_CONFIG_FILE % config_path)
else:
Notification(DEEP_NOTIF_ERROR, DEEP_MSG_FILE_NOT_FOUND % config_path)
self.store_config()
else:
Notification(DEEP_NOTIF_ERROR, DEEP_MSG_DIR_NOT_FOUND % self.config_dir)
#self.check_config()
self.config.summary()
def __create_transformer(self, config_entry):
"""
CONTRIBUTORS:
-------------
Creator : Alix Leroy
DESCRIPTION:
------------
Create the adequate transformer
PARAMETERS:
-----------
:param config: The transformer config
:param pointer-> bool : Whether or not the transformer points to another transformer
RETURN:
-------
:return transformer: The created transformer
"""
transformer = None
# NONE
if config_entry is None:
transformer = NoTransformer()
# POINTER
elif self.__is_pointer(config_entry) is True:
transformer = Pointer(
config_entry) # Generic Transformer as a pointer
# TRANSFORMER
else:
config = Namespace(config_entry)
# Check if a method is given by the user
if config.check("method", None) is False:
Notification(
DEEP_NOTIF_FATAL,
"The following transformer does not have any method specified : "
+ str(config_entry))
# Get the corresponding flag
flag = get_corresponding_flag(flag_list=DEEP_LIST_TRANSFORMERS,
info=config.method,
fatal=False)
#
# Create the corresponding Transformer
#
try:
# SEQUENTIAL
if DEEP_TRANSFORMER_SEQUENTIAL.corresponds(flag):
transformer = Sequential(**config.get(ignore="method"))
# ONE OF
elif DEEP_TRANSFORMER_ONE_OF.corresponds(flag):
transformer = OneOf(**config.get(ignore="method"))
# SOME OF
elif DEEP_TRANSFORMER_SOME_OF.corresponds(flag):
transformer = SomeOf(**config.get(ignore="method"))
# If the method does not exist
else:
Notification(
DEEP_NOTIF_FATAL,
"Unknown transformer method specified in %s : %s" %
(config_entry, config.method),
solutions=[
"Ensure a valid transformer method is specified in %s"
% config_entry
])
except TypeError as e:
Notification(
DEEP_NOTIF_FATAL,
"TypeError when loading transformer : %s : %s" %
(config_entry, e),
solutions=["Check the syntax of %s" % config_entry])
return transformer
},
"dataset": {
"train": {
"inputs": {
DEEP_CONFIG_DTYPE: [{
"source": [str],
"join": [str],
"type": str,
"load_method": str
}],
DEEP_CONFIG_DEFAULT:
None,
DEEP_CONFIG_INIT: [
Namespace({
"source": DEF["SOURCE"],
"join": DEF["JOIN"],
"type": DEF["TYPE_IMG"],
"load_method": DEF["LOAD_METHOD"]
})
]
},
"labels": {
DEEP_CONFIG_DTYPE: [{
"source": [str],
"join": [str],
"type": str,
"load_method": str
}],
DEEP_CONFIG_DEFAULT:
None,
DEEP_CONFIG_INIT: [
Namespace({
from deeplodocus.data.transform_manager import TransformManager
from deeplodocus.utils.namespace import Namespace
from deeplodocus.utils.flags import *
# Get the config for the transform managers
config_transforms = Namespace("./transforms")
# Create the transform managers
transform_manager_train = TransformManager(config_transforms.train)
#transform_manager_val = TransformManager(config_transforms.validation, write_logs=False)
#transform_manager_test = TransformManager(config_transforms.test, write_logs=False)
import time
from deeplodocus.data.dataset import Dataset
from deeplodocus.utils.types import *
from PIL import Image
import numpy as np
import cv2
inputs = []
labels = []
additional_data = []
inputs.append([r"input1.txt", r"input2.txt"])
inputs.append([r"./images", r"./images"])
labels.append([r"label1.txt", r"label2.txt"])
#labels.append(r"label3.txt")
#labels = []
additional_data.append(r"label3.txt")
#additional_data.append(r"additional_data.txt")
additional_data = []
def __create_transformer(self, config_entry):
"""
CONTRIBUTORS:
-------------
Creator : Alix Leroy
DESCRIPTION:
------------
Create the adequate transformer
PARAMETERS:
-----------
:param config: The transformer config
:param pointer-> bool : Whether or not the transformer points to another transformer
RETURN:
-------
:return transformer: The created transformer
"""
transformer = None
# NONE
if config_entry is None:
transformer = NoTransformer()
# POINTER
elif self.__is_pointer(config_entry) is True:
transformer = Pointer(
config_entry) # Generic Transformer as a pointer
# TRANSFORMER
else:
config = Namespace(config_entry)
# Check if a method is given by the user
if config.check("method", None) is False:
Notification(
DEEP_NOTIF_FATAL,
"The following transformer does not have any method specified : "
+ str(config_entry))
# Get the corresponding flag
flag = get_corresponding_flag(flag_list=DEEP_LIST_TRANSFORMERS,
info=config.method,
fatal=False)
# Remove the method from the config
delattr(config, 'method')
#
# Create the corresponding Transformer
#
# SEQUENTIAL
if DEEP_TRANSFORMER_SEQUENTIAL.corresponds(flag):
transformer = Sequential(**config.get())
# ONE OF
elif DEEP_TRANSFORMER_ONE_OF.corresponds(flag):
transformer = OneOf(**config.get())
# SOME OF
elif DEEP_TRANSFORMER_SOME_OF.corresponds(flag):
SomeOf(**config.get())
# If the method does not exist
else:
Notification(
DEEP_NOTIF_FATAL,
"The following transformation method does not exist : " +
str(config.method))
return transformer
def forward(self, x):
im_shape = torch.tensor(x.shape[2:4], dtype=torch.float32)
# Check input dims
if x.ndim != 4:
Notification(DEEP_NOTIF_FATAL, MSG_WRONG_DIMS % x.ndim)
# Check input height and width are divisible by 32
if not (x.shape[2] % 32 == 0 and x.shape[3] % 32 == 0):
Notification(DEEP_NOTIF_FATAL, MSG_WRONG_SHAPE % str(x.shape))
# BACKBONE - initial feature detection
x = self.backbone(x) # b x 1024 x h/32 x w/32
# DETECTION ON LARGEST SCALE
x = self.conv_1_1(x) # b x 512 x h/32 x w/32
x = self.conv_1_2(x) # b x 512 x h/32 x w/32
output_1 = self.conv_1_3(x) # b x 1024 x h/32 x w/32
output_1 = self.conv_1_4(
output_1) # b x 3 * (num cls + 5) x h/32 x w/32
output_1 = self.unpack(output_1,
s=2) # Unpack predictions across anchors
# DETECTION ON MID SCALE
x = self.conv_2_1(x) # b x 256 x h/32 x w/32
x = self.upsample(x) # b x 256 x h/16 x w/16
skip = self.backbone.skip[
self.skip_layers[1]] # Get skip layer from backbone
x = torch.cat((x, skip),
1) # Concatenate x with second backbone skip layer
x = self.conv_2_2(x) # b x 256 x h/16 x w/16
x = self.conv_2_3(x) # b x 256 x h/16 x w/16
output_2 = self.conv_2_4(x) # b x 512 x h/16 x w/16
output_2 = self.conv_2_5(
output_2) # b x 3 * (num cls + 5) x h/16 x w/16
output_2 = self.unpack(output_2,
s=1) # Unpack predictions across anchors
# DETECTION ON SMALLEST SCALE
x = self.conv_3_1(x) # b x 128 x h/16 x w/16
x = self.upsample(x) # b x 128 x h/8 x w/8
skip = self.backbone.skip[self.skip_layers[0]]
x = torch.cat((x, skip),
1) # Concatenate x with first backbone skip layer
x = self.conv_3_2(x) # b x 128 x h/8 x w/8
x = self.conv_3_3(x) # b x 128 x h/8 x w/8
output_3 = self.conv_3_4(x) # b x 128 x h/8 x w/8
output_3 = self.conv_3_5(output_3) # b x 3 * (num cls + 5) x h/8 x w/8
output_3 = self.unpack(output_3,
s=0) # Unpack predictions across anchors
return Namespace({
"inference": [output_3, output_2,
output_1], # Scale: [small, medium scale, large]
"anchors":
torch.tensor(self.anchors, dtype=torch.float32,
device=self.device),
"strides":
torch.tensor((
im_shape[0] / output_3.shape[2],
im_shape[0] / output_2.shape[2],
im_shape[0] / output_1.shape[2],
),
dtype=torch.float32,
device=self.device)
})
"inputs": {
DEEP_CONFIG_DTYPE: [{
"source": [str],
"join": [str],
"type": str,
"load_method": str,
"load_as": str,
"move_axes": [int]
}],
DEEP_CONFIG_DEFAULT:
None,
DEEP_CONFIG_INIT: [
Namespace({
"source": DEF["SOURCE"],
"join": DEF["JOIN"],
"type": DEF["TYPE_IMG"],
"load_method": DEF["LOAD_METHOD"],
"load_as": DEF["LOAD_AS"],
"move_axes": DEF["MOVE_AXES"]
})
]
},
"labels": {
DEEP_CONFIG_DTYPE: [{
"source": [str],
"join": [str],
"type": str,
"load_method": str,
"load_as": str,
"move_axes": [int]
}],
DEEP_CONFIG_DEFAULT:
def __convert_dtype(self, value, d_type, default, sub_space=None):
"""
AUTHORS:
--------
:author: Samuel Westlake
DESCRIPTION:
------------
Converts a given value to a given data type, and returns the result.
If the value can not be converted, the given default value is returned.
NB: If d_type is given in a list, e.g. [str], it is assume that value should be a list and each item in value
will be converted to the given data type. If any item in the list cannot be converted, None will be returned.
PARAMETERS:
-----------
:param value: the value to be converted
:param d_type: the data type to convert the value to
:param default: the default value to return if the current value
:param sub_space: the list of strings that defines the current sub-space
RETURN:
-------
:return: new_value
"""
sub_space = [] if sub_space is None else sub_space
if value is None:
return None
else:
try:
len(d_type)
d_type = d_type[0]
is_list = True
except TypeError:
is_list = False
if is_list:
new_values = []
value = value if isinstance(value, list) else [value]
for item in value:
new_item = self.__convert(item, d_type)
if new_item is not None:
new_values.append(new_item)
else:
Notification(DEEP_NOTIF_WARNING,
DEEP_MSG_NOT_CONVERTED
% (DEEP_CONFIG_DIVIDER.join(sub_space), new_values, d_type, default))
return default
return new_values
else:
if d_type == dict:
if isinstance(value, Namespace) or value is None:
return value
else:
Notification(DEEP_NOTIF_WARNING, DEEP_MSG_NOT_CONVERTED % (sub_space, value, d_type, default))
return Namespace(default)
else:
new_value = self.__convert(value, d_type)
if new_value is None:
Notification(DEEP_NOTIF_WARNING, DEEP_MSG_NOT_CONVERTED % (sub_space, value, d_type, default))
return default
else:
return new_value