def __convert(self, value, d_type, default, sub_space=None): """ AUTHORS: -------- :author: Samuel Westlake DESCRIPTION: ------------ Converts a given value to a given data type, and returns the result. If the value can not be converted, the given default value is returned. NB: If d_type is given in a list, e.g. [str], it is assume that value should be a list and each item in value will be converted to the given data type. If any item in the list cannot be converted, None will be returned. PARAMETERS: ----------- :param value: the value to be converted :param d_type: the data type to convert the value to :param default: the default value to return if the current value :param sub_space: the list of strings that defines the current sub-space RETURN: ------- :return: new_value """ sub_space = [] if sub_space is None else sub_space sub_space = DEEP_CONFIG_DIVIDER.join(sub_space) # If the value is not set if value is None and default is not None: # Notify user that default is being used if default == {}: new_value = Namespace() else: new_value = default if not default == []: Notification(DEEP_NOTIF_WARNING, DEEP_MSG_CONFIG_NOT_SET % (sub_space, default)) elif d_type is dict: new_value = Namespace(convert_dict(value.get_all())) else: new_value = convert(value, d_type) if new_value is None: # Notify user that default is being used Notification( DEEP_NOTIF_WARNING, DEEP_MSG_CONFIG_NOT_CONVERTED % ( sub_space, value, self.__get_dtype_name(d_type), default ) ) return new_value
def convert(value, d_type=None): """ Convert a value or list of values to data type in order of preference: (float, bool, str) :param value: value to convert :param d_type: data type to convert to :return: converted value """ if value is None: return None elif d_type is None: if value is None: return None elif isinstance(value, list): return [convert(item) for item in value] else: new_value = convert2float(value) if new_value is not None: if round(new_value, 0) == new_value: return int(new_value) else: return new_value if isinstance(new_value, str) and new_value.lower() in ["true", "false"]: new_value = convert2bool(value) if new_value is not None: return new_value return str(value) elif d_type is str: return str(value) elif d_type is int: return convert2int(value) elif d_type is float: return convert2float(value) elif d_type is bool: return convert2bool(value) elif d_type is dict: try: return convert_namespace(value) except AttributeError: return None elif isinstance(d_type, dict): new_value = {} for key, item in d_type.items(): try: new_value[key] = convert(value[key], d_type=item) except KeyError: new_value[key] = None except TypeError: return None return Namespace(new_value) elif isinstance(d_type, list): value = value if isinstance(value, list) else [value] new_value = [] for item in value: new_item = convert(item, d_type[0]) if new_item is None: return None else: new_value.append(new_item) return new_value
def __init__(self, transform_files=None): super(OutputTransformer, self).__init__() self.output_transformer = [] if transform_files is not None: for file in transform_files: self.output_transformer.append(Namespace(file)) self.load_transform_functions()
def clear_config(self): """ Author: SW Reset the config to an empty Namespace :return: None """ self.config = Namespace()
def load_config(self): """ AUTHORS: -------- :author: Samuel Westlake DESCRIPTION: ------------ Loads each of the config files in the config directory into self.config self.check_config() is called penultimately self.store_config() is called finally RETURN: ------- :return: None """ try: Notification(DEEP_NOTIF_INFO, DEEP_MSG_CONFIG_LOADING_DIR % self.config_dir) # If the config directory exists if os.path.isdir(self.config_dir): self.config = Namespace() # For each expected configuration file for key, file_name in DEEP_CONFIG_FILES.items(): config_path = "%s/%s" % (self.config_dir, file_name) if os.path.isfile(config_path): # Notification(DEEP_NOTIF_INFO, DEEP_MSG_CONFIG_LOADING_FILE % config_path) try: self.config.add({key: Namespace(config_path)}) except yaml.scanner.ScannerError as e: file, line, column = str(e).split(", ") error, file = file.split("\n") file = file[5:].replace('"', "") Notification( DEEP_NOTIF_FATAL, "Unable to load config file : %s" % error, solutions="See %s : %s : %s" % (file, line, column) ) self.check_config(key=key) else: self.config = Namespace() Notification(DEEP_NOTIF_FATAL, DEEP_MSG_FILE_NOT_FOUND % config_path) Notification(DEEP_NOTIF_SUCCESS, DEEP_MSG_CONFIG_COMPLETE) else: Notification(DEEP_NOTIF_ERROR, DEEP_MSG_DIR_NOT_FOUND % self.config_dir) self.store_config() except DeepError: self.sleep()
def __init_config(self): """ Author: SW Writes the config directory and files into the project directory using DEEP_CONFIG :return: None """ config = Namespace(DEEP_CONFIG) config_dir = "%s/%s/config" % (self.main_path, self.project_name) os.makedirs(config_dir, exist_ok=True) for key, namespace in config.get().items(): if isinstance(namespace, Namespace): namespace.save("%s/%s%s" % (config_dir, key, DEEP_EXT_YAML))
def load_config(self): """ AUTHORS: -------- :author: Samuel Westlake DESCRIPTION: ------------ Loads each of the config files in the config directory into self.config self.check_config() is called penultimately self.store_config() is called finally RETURN: ------- :return: None """ Notification(DEEP_NOTIF_INFO, DEEP_MSG_CONFIG_LOADING_DIR % self.config_dir) # If the config directory exists if os.path.isdir(self.config_dir): self.config = Namespace() # For each expected configuration file for key, file_name in DEEP_CONFIG_FILES.items(): config_path = "%s/%s" % (self.config_dir, file_name) if os.path.isfile(config_path): # Notification(DEEP_NOTIF_INFO, DEEP_MSG_CONFIG_LOADING_FILE % config_path) self.config.add({key: Namespace(config_path)}) self.check_config(key=key) else: self.config = Namespace() Notification(DEEP_NOTIF_FATAL, DEEP_MSG_FILE_NOT_FOUND % config_path) Notification(DEEP_NOTIF_SUCCESS, DEEP_MSG_CONFIG_COMPLETE) else: Notification(DEEP_NOTIF_ERROR, DEEP_MSG_DIR_NOT_FOUND % self.config_dir) self.store_config()
def load_config(self): """ Author: SW Function: Checks current config path is valid, if not, user is prompted to give another :return: bool: True if a valid config path is set, otherwise, False """ Notification(DEEP_NOTIF_INFO, DEEP_MSG_LOAD_CONFIG_START % self.config_dir) # If the config directory exists if os.path.isdir(self.config_dir): self.clear_config() # For each expected configuration file for key, file_name in DEEP_CONFIG_FILES.items(): config_path = "%s/%s" % (self.config_dir, file_name) if os.path.isfile(config_path): self.config.add({key: Namespace(config_path)}) Notification(DEEP_NOTIF_SUCCESS, DEEP_MSG_LOAD_CONFIG_FILE % config_path) else: Notification(DEEP_NOTIF_ERROR, DEEP_MSG_FILE_NOT_FOUND % config_path) self.store_config() else: Notification(DEEP_NOTIF_ERROR, DEEP_MSG_DIR_NOT_FOUND % self.config_dir) #self.check_config() self.config.summary()
def __create_transformer(self, config_entry): """ CONTRIBUTORS: ------------- Creator : Alix Leroy DESCRIPTION: ------------ Create the adequate transformer PARAMETERS: ----------- :param config: The transformer config :param pointer-> bool : Whether or not the transformer points to another transformer RETURN: ------- :return transformer: The created transformer """ transformer = None # NONE if config_entry is None: transformer = NoTransformer() # POINTER elif self.__is_pointer(config_entry) is True: transformer = Pointer( config_entry) # Generic Transformer as a pointer # TRANSFORMER else: config = Namespace(config_entry) # Check if a method is given by the user if config.check("method", None) is False: Notification( DEEP_NOTIF_FATAL, "The following transformer does not have any method specified : " + str(config_entry)) # Get the corresponding flag flag = get_corresponding_flag(flag_list=DEEP_LIST_TRANSFORMERS, info=config.method, fatal=False) # # Create the corresponding Transformer # try: # SEQUENTIAL if DEEP_TRANSFORMER_SEQUENTIAL.corresponds(flag): transformer = Sequential(**config.get(ignore="method")) # ONE OF elif DEEP_TRANSFORMER_ONE_OF.corresponds(flag): transformer = OneOf(**config.get(ignore="method")) # SOME OF elif DEEP_TRANSFORMER_SOME_OF.corresponds(flag): transformer = SomeOf(**config.get(ignore="method")) # If the method does not exist else: Notification( DEEP_NOTIF_FATAL, "Unknown transformer method specified in %s : %s" % (config_entry, config.method), solutions=[ "Ensure a valid transformer method is specified in %s" % config_entry ]) except TypeError as e: Notification( DEEP_NOTIF_FATAL, "TypeError when loading transformer : %s : %s" % (config_entry, e), solutions=["Check the syntax of %s" % config_entry]) return transformer
}, "dataset": { "train": { "inputs": { DEEP_CONFIG_DTYPE: [{ "source": [str], "join": [str], "type": str, "load_method": str }], DEEP_CONFIG_DEFAULT: None, DEEP_CONFIG_INIT: [ Namespace({ "source": DEF["SOURCE"], "join": DEF["JOIN"], "type": DEF["TYPE_IMG"], "load_method": DEF["LOAD_METHOD"] }) ] }, "labels": { DEEP_CONFIG_DTYPE: [{ "source": [str], "join": [str], "type": str, "load_method": str }], DEEP_CONFIG_DEFAULT: None, DEEP_CONFIG_INIT: [ Namespace({
from deeplodocus.data.transform_manager import TransformManager from deeplodocus.utils.namespace import Namespace from deeplodocus.utils.flags import * # Get the config for the transform managers config_transforms = Namespace("./transforms") # Create the transform managers transform_manager_train = TransformManager(config_transforms.train) #transform_manager_val = TransformManager(config_transforms.validation, write_logs=False) #transform_manager_test = TransformManager(config_transforms.test, write_logs=False) import time from deeplodocus.data.dataset import Dataset from deeplodocus.utils.types import * from PIL import Image import numpy as np import cv2 inputs = [] labels = [] additional_data = [] inputs.append([r"input1.txt", r"input2.txt"]) inputs.append([r"./images", r"./images"]) labels.append([r"label1.txt", r"label2.txt"]) #labels.append(r"label3.txt") #labels = [] additional_data.append(r"label3.txt") #additional_data.append(r"additional_data.txt") additional_data = []
def __create_transformer(self, config_entry): """ CONTRIBUTORS: ------------- Creator : Alix Leroy DESCRIPTION: ------------ Create the adequate transformer PARAMETERS: ----------- :param config: The transformer config :param pointer-> bool : Whether or not the transformer points to another transformer RETURN: ------- :return transformer: The created transformer """ transformer = None # NONE if config_entry is None: transformer = NoTransformer() # POINTER elif self.__is_pointer(config_entry) is True: transformer = Pointer( config_entry) # Generic Transformer as a pointer # TRANSFORMER else: config = Namespace(config_entry) # Check if a method is given by the user if config.check("method", None) is False: Notification( DEEP_NOTIF_FATAL, "The following transformer does not have any method specified : " + str(config_entry)) # Get the corresponding flag flag = get_corresponding_flag(flag_list=DEEP_LIST_TRANSFORMERS, info=config.method, fatal=False) # Remove the method from the config delattr(config, 'method') # # Create the corresponding Transformer # # SEQUENTIAL if DEEP_TRANSFORMER_SEQUENTIAL.corresponds(flag): transformer = Sequential(**config.get()) # ONE OF elif DEEP_TRANSFORMER_ONE_OF.corresponds(flag): transformer = OneOf(**config.get()) # SOME OF elif DEEP_TRANSFORMER_SOME_OF.corresponds(flag): SomeOf(**config.get()) # If the method does not exist else: Notification( DEEP_NOTIF_FATAL, "The following transformation method does not exist : " + str(config.method)) return transformer
def forward(self, x): im_shape = torch.tensor(x.shape[2:4], dtype=torch.float32) # Check input dims if x.ndim != 4: Notification(DEEP_NOTIF_FATAL, MSG_WRONG_DIMS % x.ndim) # Check input height and width are divisible by 32 if not (x.shape[2] % 32 == 0 and x.shape[3] % 32 == 0): Notification(DEEP_NOTIF_FATAL, MSG_WRONG_SHAPE % str(x.shape)) # BACKBONE - initial feature detection x = self.backbone(x) # b x 1024 x h/32 x w/32 # DETECTION ON LARGEST SCALE x = self.conv_1_1(x) # b x 512 x h/32 x w/32 x = self.conv_1_2(x) # b x 512 x h/32 x w/32 output_1 = self.conv_1_3(x) # b x 1024 x h/32 x w/32 output_1 = self.conv_1_4( output_1) # b x 3 * (num cls + 5) x h/32 x w/32 output_1 = self.unpack(output_1, s=2) # Unpack predictions across anchors # DETECTION ON MID SCALE x = self.conv_2_1(x) # b x 256 x h/32 x w/32 x = self.upsample(x) # b x 256 x h/16 x w/16 skip = self.backbone.skip[ self.skip_layers[1]] # Get skip layer from backbone x = torch.cat((x, skip), 1) # Concatenate x with second backbone skip layer x = self.conv_2_2(x) # b x 256 x h/16 x w/16 x = self.conv_2_3(x) # b x 256 x h/16 x w/16 output_2 = self.conv_2_4(x) # b x 512 x h/16 x w/16 output_2 = self.conv_2_5( output_2) # b x 3 * (num cls + 5) x h/16 x w/16 output_2 = self.unpack(output_2, s=1) # Unpack predictions across anchors # DETECTION ON SMALLEST SCALE x = self.conv_3_1(x) # b x 128 x h/16 x w/16 x = self.upsample(x) # b x 128 x h/8 x w/8 skip = self.backbone.skip[self.skip_layers[0]] x = torch.cat((x, skip), 1) # Concatenate x with first backbone skip layer x = self.conv_3_2(x) # b x 128 x h/8 x w/8 x = self.conv_3_3(x) # b x 128 x h/8 x w/8 output_3 = self.conv_3_4(x) # b x 128 x h/8 x w/8 output_3 = self.conv_3_5(output_3) # b x 3 * (num cls + 5) x h/8 x w/8 output_3 = self.unpack(output_3, s=0) # Unpack predictions across anchors return Namespace({ "inference": [output_3, output_2, output_1], # Scale: [small, medium scale, large] "anchors": torch.tensor(self.anchors, dtype=torch.float32, device=self.device), "strides": torch.tensor(( im_shape[0] / output_3.shape[2], im_shape[0] / output_2.shape[2], im_shape[0] / output_1.shape[2], ), dtype=torch.float32, device=self.device) })
"inputs": { DEEP_CONFIG_DTYPE: [{ "source": [str], "join": [str], "type": str, "load_method": str, "load_as": str, "move_axes": [int] }], DEEP_CONFIG_DEFAULT: None, DEEP_CONFIG_INIT: [ Namespace({ "source": DEF["SOURCE"], "join": DEF["JOIN"], "type": DEF["TYPE_IMG"], "load_method": DEF["LOAD_METHOD"], "load_as": DEF["LOAD_AS"], "move_axes": DEF["MOVE_AXES"] }) ] }, "labels": { DEEP_CONFIG_DTYPE: [{ "source": [str], "join": [str], "type": str, "load_method": str, "load_as": str, "move_axes": [int] }], DEEP_CONFIG_DEFAULT:
def __convert_dtype(self, value, d_type, default, sub_space=None): """ AUTHORS: -------- :author: Samuel Westlake DESCRIPTION: ------------ Converts a given value to a given data type, and returns the result. If the value can not be converted, the given default value is returned. NB: If d_type is given in a list, e.g. [str], it is assume that value should be a list and each item in value will be converted to the given data type. If any item in the list cannot be converted, None will be returned. PARAMETERS: ----------- :param value: the value to be converted :param d_type: the data type to convert the value to :param default: the default value to return if the current value :param sub_space: the list of strings that defines the current sub-space RETURN: ------- :return: new_value """ sub_space = [] if sub_space is None else sub_space if value is None: return None else: try: len(d_type) d_type = d_type[0] is_list = True except TypeError: is_list = False if is_list: new_values = [] value = value if isinstance(value, list) else [value] for item in value: new_item = self.__convert(item, d_type) if new_item is not None: new_values.append(new_item) else: Notification(DEEP_NOTIF_WARNING, DEEP_MSG_NOT_CONVERTED % (DEEP_CONFIG_DIVIDER.join(sub_space), new_values, d_type, default)) return default return new_values else: if d_type == dict: if isinstance(value, Namespace) or value is None: return value else: Notification(DEEP_NOTIF_WARNING, DEEP_MSG_NOT_CONVERTED % (sub_space, value, d_type, default)) return Namespace(default) else: new_value = self.__convert(value, d_type) if new_value is None: Notification(DEEP_NOTIF_WARNING, DEEP_MSG_NOT_CONVERTED % (sub_space, value, d_type, default)) return default else: return new_value