示例#1
0
文件: __init__.py 项目: rgcl/astroNN
def load_folder(folder=None):
    """
    To load astroNN model object from folder

    :param folder: [optional] you should provide folder name if outside folder, do not specific when you are inside the folder
    :type folder: str
    :return: astroNN Neural Network instance
    :rtype: astroNN.nn.NeuralNetMaster.NeuralNetMaster
    :History: 2017-Dec-29 - Written - Henry Leung (University of Toronto)
    """
    currentdir = os.getcwd()

    if folder is not None:
        fullfilepath = os.path.join(currentdir, folder)
    else:
        fullfilepath = currentdir

    astronn_model_obj = None

    if folder is not None and os.path.isfile(
            os.path.join(folder, 'astroNN_model_parameter.json')) is True:
        with open(os.path.join(folder, 'astroNN_model_parameter.json')) as f:
            parameter = json.load(f)
            f.close()
    elif os.path.isfile('astroNN_model_parameter.json') is True:
        with open('astroNN_model_parameter.json') as f:
            parameter = json.load(f)
            f.close()
    elif folder is not None and not os.path.exists(folder):
        raise IOError('Folder not exists: ' + str(currentdir + '/' + folder))
    else:
        raise FileNotFoundError(
            'Are you sure this is an astroNN generated folder? Or is it a folder trained by old '
            'astroNN version?')

    identifier = parameter['id']

    if identifier == 'ApogeeCNN':
        astronn_model_obj = ApogeeCNN()
    elif identifier == 'ApogeeBCNN':
        astronn_model_obj = ApogeeBCNN()
    elif identifier == 'ApogeeBCNNCensored':
        astronn_model_obj = ApogeeBCNNCensored()
    elif identifier == 'ApogeeCVAE':
        astronn_model_obj = ApogeeCVAE()
    elif identifier == 'Cifar10CNN':
        astronn_model_obj = Cifar10CNN()
    elif identifier == 'MNIST_BCNN':
        astronn_model_obj = MNIST_BCNN()
    elif identifier == 'Galaxy10CNN':
        astronn_model_obj = Galaxy10CNN()
    elif identifier == 'StarNet2017':
        astronn_model_obj = StarNet2017()
    elif identifier == 'SimplePolyNN':
        astronn_model_obj = SimplePolyNN()
    else:
        unknown_model_message = f'Unknown model identifier -> {identifier}!'
        # try to load custom model from CUSTOM_MODEL_PATH
        CUSTOM_MODEL_PATH = custom_model_path_reader()
        # try the current folder and see if there is any .py on top of CUSTOM_MODEL_PATH
        list_py_files = [
            os.path.join(fullfilepath, f) for f in os.listdir(fullfilepath)
            if f.endswith(".py")
        ]
        if CUSTOM_MODEL_PATH is None and list_py_files is None:
            print("\n")
            raise TypeError(unknown_model_message)
        else:
            import sys
            from importlib import import_module
            for path_list in (
                    path_list
                    for path_list in [CUSTOM_MODEL_PATH, list_py_files]
                    if path_list is not None):
                for path in path_list:
                    head, tail = os.path.split(path)
                    sys.path.insert(0, head)
                    try:
                        model = getattr(import_module(tail.strip('.py')),
                                        str(identifier))
                        astronn_model_obj = model()
                    except AttributeError:
                        pass

        if astronn_model_obj is None:
            print("\n")
            raise TypeError(unknown_model_message)

    astronn_model_obj.currentdir = currentdir
    astronn_model_obj.fullfilepath = fullfilepath
    astronn_model_obj.folder_name = folder if folder is not None else os.path.basename(
        os.path.normpath(currentdir))

    # Must have parameter
    astronn_model_obj._input_shape = parameter['input']
    astronn_model_obj._labels_shape = parameter['labels']
    astronn_model_obj.num_hidden = parameter['hidden']
    astronn_model_obj.input_norm_mode = parameter['input_norm_mode']
    astronn_model_obj.labels_norm_mode = parameter['labels_norm_mode']
    astronn_model_obj.input_mean = np.array(parameter['input_mean'])
    astronn_model_obj.labels_mean = np.array(parameter['labels_mean'])
    astronn_model_obj.input_std = np.array(parameter['input_std'])
    astronn_model_obj.labels_std = np.array(parameter['labels_std'])
    astronn_model_obj.batch_size = parameter['batch_size']
    astronn_model_obj.targetname = parameter['targetname']
    astronn_model_obj.val_size = parameter['valsize']

    # create normalizer and set correct mean and std
    astronn_model_obj.input_normalizer = Normalizer(
        mode=astronn_model_obj.input_norm_mode)
    astronn_model_obj.labels_normalizer = Normalizer(
        mode=astronn_model_obj.labels_norm_mode)
    astronn_model_obj.input_normalizer.mean_labels = astronn_model_obj.input_mean
    astronn_model_obj.input_normalizer.std_labels = astronn_model_obj.input_std
    astronn_model_obj.labels_normalizer.mean_labels = astronn_model_obj.labels_mean
    astronn_model_obj.labels_normalizer.std_labels = astronn_model_obj.labels_std

    # Conditional parameter depends on neural net architecture
    try:
        astronn_model_obj.num_filters = parameter['filternum']
    except KeyError:
        pass
    try:
        astronn_model_obj.filter_len = parameter['filterlen']
    except KeyError:
        pass
    try:
        pool_length = parameter['pool_length']
        if pool_length is not None:
            if isinstance(pool_length, int):  # multi-dimensional case
                astronn_model_obj.pool_length = parameter['pool_length']
            else:
                astronn_model_obj.pool_length = list(parameter['pool_length'])
    except KeyError or TypeError:
        pass
    try:
        # need to convert to int because of keras do not want array or list
        astronn_model_obj.latent_dim = int(parameter['latent'])
    except KeyError:
        pass
    try:
        astronn_model_obj.task = parameter['task']
    except KeyError:
        pass
    try:
        astronn_model_obj.dropout_rate = parameter['dropout_rate']
    except KeyError:
        pass
    try:
        # if inverse model precision exists, so does length_scale
        astronn_model_obj.inv_model_precision = parameter['inv_tau']
        astronn_model_obj.length_scale = parameter['length_scale']
    except KeyError:
        pass
    try:
        astronn_model_obj.l1 = parameter['l1']
    except KeyError:
        pass
    try:
        astronn_model_obj.l2 = parameter['l2']
    except KeyError:
        pass
    try:
        astronn_model_obj.maxnorm = parameter['maxnorm']
    except KeyError:
        pass

    global _GRAPH_COUTNER
    global _GRAPH_STORAGE
    global _SESSION_STORAGE
    _GRAPH_COUTNER += 1
    _GRAPH_STORAGE.append(Graph())

    with _GRAPH_STORAGE[_GRAPH_COUTNER - 1].as_default():
        _SESSION_STORAGE.append(Session())
        with _SESSION_STORAGE[_GRAPH_COUTNER - 1].as_default():
            with h5py.File(os.path.join(astronn_model_obj.fullfilepath,
                                        'model_weights.h5'),
                           mode='r') as f:
                training_config = f.attrs.get('training_config')
                training_config = json.loads(training_config.decode('utf-8'))
                optimizer_config = training_config['optimizer_config']
                optimizer = optimizers.deserialize(optimizer_config)

                # Recover loss functions and metrics.
                losses = convert_custom_objects(training_config['loss'])
                try:
                    try:
                        [losses_lookup(losses[loss]) for loss in losses]
                    except TypeError:
                        losses_lookup(losses)
                except:
                    pass

                metrics = convert_custom_objects(training_config['metrics'])
                # its weird that keras needs -> metrics[metric][0] instead of metrics[metric] likes losses, need attention
                try:
                    try:
                        [
                            losses_lookup(metrics[metric][0])
                            for metric in metrics
                        ]
                    except TypeError:
                        losses_lookup(metrics[0])
                except:
                    pass

                sample_weight_mode = training_config['sample_weight_mode']
                loss_weights = training_config['loss_weights']

                # compile the model
                astronn_model_obj.compile(optimizer=optimizer)

                # set weights
                astronn_model_obj.keras_model.load_weights(
                    os.path.join(astronn_model_obj.fullfilepath,
                                 'model_weights.h5'))

                # Build train function (to get weight updates), need to consider Sequential model too
                astronn_model_obj.keras_model._make_train_function()
                if isinstance(astronn_model_obj.keras_model, Sequential):
                    astronn_model_obj.keras_model.model._make_train_function()
                else:
                    astronn_model_obj.keras_model._make_train_function()
                optimizer_weights_group = f['optimizer_weights']
                optimizer_weight_names = [
                    n.decode('utf8')
                    for n in optimizer_weights_group.attrs['weight_names']
                ]
                optimizer_weight_values = [
                    optimizer_weights_group[n] for n in optimizer_weight_names
                ]
                astronn_model_obj.keras_model.optimizer.set_weights(
                    optimizer_weight_values)

    astronn_model_obj.graph = _GRAPH_STORAGE[
        _GRAPH_COUTNER - 1]  # the graph associated with the model
    astronn_model_obj.session = _SESSION_STORAGE[
        _GRAPH_COUTNER - 1]  # the model associated with the model
    astronn_model_obj.session.__enter__(
    )  # register the latest model loaded to defualt tensorflow session

    print("========================================================")
    print(
        f"Loaded astroNN model, model type: {astronn_model_obj.name} -> {identifier}"
    )
    print("========================================================")
    return astronn_model_obj
示例#2
0
def load_folder(folder=None):
    """
    To load astroNN model object from folder

    :param folder: [optional] you should provide folder name if outside folder, do not specific when you are inside the folder
    :type folder: str
    :return: astroNN Neural Network instance
    :rtype: astroNN.nn.NeuralNetMaster.NeuralNetMaster
    :History: 2017-Dec-29 - Written - Henry Leung (University of Toronto)
    """
    currentdir = os.getcwd()

    if folder is not None:
        fullfilepath = os.path.join(currentdir, folder)
    else:
        fullfilepath = currentdir

    astronn_model_obj = None

    if folder is not None and os.path.isfile(
            os.path.join(folder, 'astroNN_model_parameter.json')) is True:
        with open(os.path.join(folder, 'astroNN_model_parameter.json')) as f:
            parameter = json.load(f)
            f.close()
    elif os.path.isfile('astroNN_model_parameter.json') is True:
        with open('astroNN_model_parameter.json') as f:
            parameter = json.load(f)
            f.close()
    elif folder is not None and not os.path.exists(folder):
        raise IOError('Folder not exists: ' +
                      str(currentdir + os.sep + folder))
    else:
        raise FileNotFoundError(
            'Are you sure this is an astroNN generated folder?')

    identifier = parameter['id']
    unknown_model_message = f'Unknown model identifier -> {identifier}!'

    # need to point to the actual neural network if non-travial location
    if identifier == 'Galaxy10CNN':
        astronn_model_obj = Galaxy10CNN()
    else:
        # else try to import it from standard way
        try:
            astronn_model_obj = getattr(
                importlib.import_module(f"astroNN.models"), identifier)()
        except ImportError:
            # try to load custom model from CUSTOM_MODEL_PATH if none are working
            CUSTOM_MODEL_PATH = custom_model_path_reader()
            # try the current folder and see if there is any .py on top of CUSTOM_MODEL_PATH
            list_py_files = [
                os.path.join(fullfilepath, f) for f in os.listdir(fullfilepath)
                if f.endswith(".py")
            ]
            if CUSTOM_MODEL_PATH is None and list_py_files is None:
                print("\n")
                raise TypeError(unknown_model_message)
            else:
                for path_list in (
                        path_list
                        for path_list in [CUSTOM_MODEL_PATH, list_py_files]
                        if path_list is not None):
                    for path in path_list:
                        head, tail = os.path.split(path)
                        sys.path.insert(0, head)
                        try:
                            model = getattr(
                                importlib.import_module(tail.strip('.py')),
                                str(identifier))
                            astronn_model_obj = model()
                        except AttributeError:
                            pass

        if astronn_model_obj is None:
            print("\n")
            raise TypeError(unknown_model_message)

    astronn_model_obj.currentdir = currentdir
    astronn_model_obj.fullfilepath = fullfilepath
    astronn_model_obj.folder_name = folder if folder is not None else os.path.basename(
        os.path.normpath(currentdir))

    # Must have parameter
    astronn_model_obj._input_shape = parameter['input']
    astronn_model_obj._labels_shape = parameter['labels']
    if type(astronn_model_obj._input_shape) is not dict:
        astronn_model_obj._input_shape = {
            "input": astronn_model_obj._input_shape
        }
    if type(astronn_model_obj._labels_shape) is not dict:
        astronn_model_obj._labels_shape = {
            "output": astronn_model_obj._labels_shape
        }
    astronn_model_obj.num_hidden = parameter['hidden']
    astronn_model_obj.input_norm_mode = parameter['input_norm_mode']
    astronn_model_obj.labels_norm_mode = parameter['labels_norm_mode']
    astronn_model_obj.batch_size = parameter['batch_size']
    astronn_model_obj.targetname = parameter['targetname']
    astronn_model_obj.val_size = parameter['valsize']

    # Conditional parameter depends on neural net architecture
    try:
        astronn_model_obj.num_filters = parameter['filternum']
    except KeyError:
        pass
    try:
        astronn_model_obj.filter_len = parameter['filterlen']
    except KeyError:
        pass
    try:
        pool_length = parameter['pool_length']
        if pool_length is not None:
            if isinstance(pool_length, int):  # multi-dimensional case
                astronn_model_obj.pool_length = parameter['pool_length']
            else:
                astronn_model_obj.pool_length = list(parameter['pool_length'])
    except KeyError or TypeError:
        pass
    try:
        # need to convert to int because of keras do not want array or list
        astronn_model_obj.latent_dim = int(parameter['latent'])
    except KeyError:
        pass
    try:
        astronn_model_obj.task = parameter['task']
    except KeyError:
        pass
    try:
        astronn_model_obj.dropout_rate = parameter['dropout_rate']
    except KeyError:
        pass
    try:
        # if inverse model precision exists, so does length_scale
        astronn_model_obj.inv_model_precision = parameter['inv_tau']
        astronn_model_obj.length_scale = parameter['length_scale']
    except KeyError:
        pass
    try:
        astronn_model_obj.l1 = parameter['l1']
    except KeyError:
        pass
    try:
        astronn_model_obj.l2 = parameter['l2']
    except KeyError:
        pass
    try:
        astronn_model_obj.maxnorm = parameter['maxnorm']
    except KeyError:
        pass
    try:
        astronn_model_obj.input_names = parameter['input_names']
    except KeyError:
        astronn_model_obj.input_names = ['input']
    try:
        astronn_model_obj.output_names = parameter['output_names']
    except KeyError:
        astronn_model_obj.output_names = ['output']
    try:
        astronn_model_obj._last_layer_activation = parameter[
            'last_layer_activation']
    except KeyError:
        pass
    try:
        astronn_model_obj.activation = parameter['activation']
    except KeyError:
        pass

    with h5py.File(os.path.join(astronn_model_obj.fullfilepath,
                                'model_weights.h5'),
                   mode='r') as f:
        training_config = json.loads(f.attrs['training_config'])
        optimizer_config = training_config['optimizer_config']
        optimizer = optimizers.deserialize(optimizer_config)
        model_config = json.loads(f.attrs['model_config'])

        # input/name names, mean, std
        input_names = []
        output_names = []
        for lay in model_config["config"]["input_layers"]:
            input_names.append(lay[0])
        for lay in model_config["config"]["output_layers"]:
            output_names.append(lay[0])
        astronn_model_obj.input_mean = list_to_dict(
            input_names, dict_list_to_dict_np(parameter['input_mean']))
        astronn_model_obj.labels_mean = list_to_dict(
            output_names, dict_list_to_dict_np(parameter['labels_mean']))
        astronn_model_obj.input_std = list_to_dict(
            input_names, dict_list_to_dict_np(parameter['input_std']))
        astronn_model_obj.labels_std = list_to_dict(
            output_names, dict_list_to_dict_np(parameter['labels_std']))

        # Recover loss functions and metrics.
        losses_raw = convert_custom_objects(training_config['loss'])
        try:
            try:
                loss = [
                    losses_lookup(losses_raw[_loss]) for _loss in losses_raw
                ]
            except TypeError:
                loss = losses_lookup(losses_raw)
        except:
            raise LookupError("Cant lookup loss")

        metrics_raw = convert_custom_objects(training_config['metrics'])
        # its weird that keras needs -> metrics[metric][0] instead of metrics[metric] likes losses
        try:
            try:
                if version.parse(tf.__version__) >= version.parse("2.4.0"):
                    metrics = [
                        losses_lookup(_metric['config']['fn'])
                        for _metric in metrics_raw[0]
                    ]
                else:
                    metrics = [
                        losses_lookup(metrics_raw[_metric][0])
                        for _metric in metrics_raw
                    ]
            except TypeError:
                metrics = [losses_lookup(metrics_raw[0])]
        except:
            metrics = metrics_raw

        sample_weight_mode = training_config['sample_weight_mode'] if \
            hasattr(training_config, 'sample_weight_mode') else None
        loss_weights = training_config['loss_weights']
        weighted_metrics = None

        # compile the model
        astronn_model_obj.compile(optimizer=optimizer,
                                  loss=loss,
                                  metrics=metrics,
                                  weighted_metrics=weighted_metrics,
                                  loss_weights=loss_weights,
                                  sample_weight_mode=sample_weight_mode)

        # set weights
        astronn_model_obj.keras_model.load_weights(
            os.path.join(astronn_model_obj.fullfilepath, 'model_weights.h5'))

        # Build train function (to get weight updates), need to consider Sequential model too
        astronn_model_obj.keras_model.make_train_function()
        try:
            optimizer_weights_group = f['optimizer_weights']
            if version.parse(h5py.__version__) >= version.parse("3.0"):
                optimizer_weight_names = [
                    n for n in optimizer_weights_group.attrs['weight_names']
                ]
            else:
                optimizer_weight_names = [
                    n.decode('utf8')
                    for n in optimizer_weights_group.attrs['weight_names']
                ]
            optimizer_weight_values = [
                optimizer_weights_group[n] for n in optimizer_weight_names
            ]
            astronn_model_obj.keras_model.optimizer._create_all_weights(
                astronn_model_obj.keras_model.trainable_variables)
            astronn_model_obj.keras_model.optimizer.set_weights(
                optimizer_weight_values)
        except KeyError:
            warnings.warn(
                "Error in loading the saved optimizer state. As a result, your model is starting with a freshly initialized optimizer."
            )

    # create normalizer and set correct mean and std
    astronn_model_obj.input_normalizer = Normalizer(
        mode=astronn_model_obj.input_norm_mode)
    astronn_model_obj.labels_normalizer = Normalizer(
        mode=astronn_model_obj.labels_norm_mode)
    astronn_model_obj.input_normalizer.mean_labels = astronn_model_obj.input_mean
    astronn_model_obj.input_normalizer.std_labels = astronn_model_obj.input_std
    astronn_model_obj.labels_normalizer.mean_labels = astronn_model_obj.labels_mean
    astronn_model_obj.labels_normalizer.std_labels = astronn_model_obj.labels_std
    print("========================================================")
    print(
        f"Loaded astroNN model, model type: {astronn_model_obj.name} -> {identifier}"
    )
    print("========================================================")
    return astronn_model_obj