def norm_iterator(iterable): """returns a normalized iterable of tuples""" if isinstance(iterable, list): names = ['list_' + str(i) for i, j in enumerate(iterable)] return szip(names, iterable) else: raise NotImplementedError('Iterables other than lists ' 'cannot be passed to this function')
def _prepare_message(self, model, data, data_val, kwargs, generator=False): """Prepare the elements to be passed to the backend Args: model(supported model): the model to be prepared data(list): the list of dicts or generators used for training data_val(list): the list of dicts or generator used for validation Returns: the transformed data object, the transformed validation data object, the data_hash """ self._check_compile(model, kwargs) kwargs = self._check_serialize(kwargs) gen_setup = [] if generator: nb_data_chunks = [get_nb_chunks(d) for d in data] nb_data_val_chunks = [get_nb_chunks(dv) for dv in data_val] for d_c, dv_c in szip(nb_data_chunks, nb_data_val_chunks): is_val_one = dv_c == 1 is_train_one = d_c == 1 if dv_c is not None: # many to one if d_c > dv_c and is_val_one: gen_setup.append(1) # one to many elif d_c < dv_c and is_train_one: gen_setup.append(2) # equal elif d_c == dv_c: gen_setup.append(3) else: # pragma: no cover Exception('Nb batches in train generator and' 'validation generator not compatible') data_hash = cm.create_gen_hash(data) data, data_val = pickle_gen(data, data_val) else: data_hash = cm.create_data_hash(data) return data, data_val, data_hash, gen_setup
def _prepare_message(self, model, data, data_val, kwargs, generator=False): """Prepare the elements to be passed to the backend Args: model(supported model): the model to be prepared data(list): the list of dicts or generators used for training data_val(list): the list of dicts or generator used for validation Returns: the transformed data object, the transformed validation data object, the data_hash """ self._check_compile(model, kwargs) kwargs = self._check_serialize(kwargs) gen_setup = [] if generator: nb_data_chunks = [get_nb_chunks(d) for d in data] nb_data_val_chunks = [get_nb_chunks(dv) for dv in data_val] for d_c, dv_c in szip(nb_data_chunks, nb_data_val_chunks): is_val_one = dv_c == 1 is_train_one = d_c == 1 if dv_c is not None: # many to one if d_c > dv_c and is_val_one: gen_setup.append(1) # one to many elif d_c < dv_c and is_train_one: gen_setup.append(2) # equal elif d_c == dv_c: gen_setup.append(3) else: # pragma: no cover Exception('Nb batches in train generator and' 'validation generator not compatible') data_hash = cm.create_gen_hash(data) data, data_val = pickle_gen(data, data_val) else: data_hash = cm.create_data_hash(data) return data, data_val, data_hash, gen_setup
def test_experiment_fit_gen_async_nogenval(self, get_model_data_expe): ''' Main case: gen on train, data on val Subcases: 10 chunks on train 1 chunk on train ''' data, data_val, is_classif, model, metric, expe = get_model_data_expe for Nchunks_gen, expected_value in szip([True, False], [10, 1]): gen_train, data_train, data_stream_train = make_gen( Nchunks_gen, is_classif, train=True) for data_val_loc in [None, data_val]: _, thread = expe.fit_gen_async([gen_train], [data_val_loc], model=model, overwrite=True, metrics=metric) thread.join() assert len(expe.full_res['metrics'][ 'score']) == expected_value assert len(expe.full_res['metrics'][ 'val_score']) == expected_value if data_val_loc is not None: assert None not in expe.full_res['metrics'][ 'val_score'] else: assert np.all([np.isnan(v) for v in expe.full_res[ 'metrics']['val_score']]) assert expe.data_id is not None assert expe.mod_id is not None assert expe.params_dump is not None assert expe close_gens(gen_train, data_train, data_stream_train) print(self)
def test_experiment_fit_gen_async_withgenval(self, get_model_data_expe): ''' Main case: gen on train, gen on val Subcases: 10 chunks on train / 10 chunks on val 10 chunks on train / 1 chunk on val 1 chunk on train / 10 chunks on val ''' data, data_val, is_classif, model, metric, expe = get_model_data_expe for Nchunks_gen, Nchunks_val in szip([True, True, False], [True, False, True]): gen_train, data_train, data_stream_train = make_gen( Nchunks_gen, is_classif, train=True) gen_test, data_test, data_stream_test = make_gen( Nchunks_val, is_classif, train=False) _, thread = expe.fit_gen_async( [gen_train], [gen_test], overwrite=True, metrics=metric) thread.join() expected_value_gen = 10 if not Nchunks_gen: expected_value_gen = 1 assert len(expe.full_res['metrics'][ 'score']) == expected_value_gen assert len(expe.full_res['metrics'][ 'val_score']) == 10 assert expe.data_id is not None assert expe.mod_id is not None assert expe.params_dump is not None assert expe close_gens(gen_train, data_train, data_stream_train) print(self)
def train(model, data, data_val, size_gen, generator=False, *args, **kwargs): """Fit a model given hyperparameters and a serialized model Args: model(dict): a serialized keras.Model data(list): a list of dict mapping inputs and outputs to lists or dictionnaries mapping the inputs names to np.arrays data_val(list): same structure than `data` but for validation Returns: the loss (list), the validation loss (list), the number of iterations, and the model """ if generator: from six.moves import reload_module as sreload import theano sreload(theano) results = dict() results['metrics'] = dict() custom_objects = None callbacks = [] fit_gen_val = False suf = 'val_' if 'custom_objects' in kwargs: custom_objects = kwargs.pop('custom_objects') # load model model = model_from_dict_w_opt(model, custom_objects=custom_objects) if 'callbacks' in kwargs: callbacks = kwargs.pop('callbacks') callbacks = [deserialize(**callback) for callback in callbacks] for i, callback in enumerate(callbacks): if inspect.isfunction(callback): callbacks[i] = callback() else: raise TypeError('Your callback is not wrapped in a function') metrics_names = model.metrics_names for metric in metrics_names: results['metrics'][metric] = [] results['metrics'][suf + metric] = [] mod_name = model.__class__.__name__ if generator: data = [pickle.loads(d.encode('raw_unicode_escape')) for d in data] data = [cm.transform_gen(d, mod_name) for d in data] kwargs.pop('batch_size') if all(v is None for v in data_val): val_gen = 0 else: val_gen = check_gen(data_val) if val_gen > 0: if generator: data_val = [pickle.loads(dv.encode('raw_unicode_escape')) for dv in data_val] data_val = [cm.transform_gen(dv, mod_name) for dv in data_val] for i, check in enumerate(size_gen): if check is 1: data_val[i] = next(data_val[i]) fit_gen_val = True else: raise Exception("You should also pass a generator for the training" " data.") # fit the model according to the input/output type if mod_name is "Sequential" or mod_name is "Model": for d, dv in szip(data, data_val): validation = check_validation(dv) if not fit_gen_val: if dv is not None: dv = (dv['X'], dv['y']) if generator: h = model.fit_generator(generator=d, validation_data=dv, callbacks=callbacks, *args, **kwargs) else: X, y = d['X'], d['y'] h = model.fit(x=X, y=y, validation_data=dv, callbacks=callbacks, *args, **kwargs) for metric in metrics_names: results['metrics'][metric] += h.history[metric] if validation: results['metrics'][ suf + metric] += h.history[suf + metric] else: results['metrics'][suf + metric] += [np.nan] * \ len(h.history[metric]) results['metrics']['iter'] = h.epoch[-1] * len(data) else: raise NotImplementedError("This type of model" "is not supported: {}".format(mod_name)) return results, model
def train(model, data, data_val, size_gen, generator=False, *args, **kwargs): """Fit a model given hyperparameters and a serialized model Args: model(dict): a serialized keras.Model data(list): a list of dict mapping inputs and outputs to lists or dictionnaries mapping the inputs names to np.arrays data_val(list): same structure than `data` but for validation Returns: the loss (list), the validation loss (list), the number of iterations, and the model """ if generator: from six.moves import reload_module as sreload import theano sreload(theano) results = dict() results['metrics'] = dict() custom_objects = None callbacks = [] fit_gen_val = False suf = 'val_' if 'custom_objects' in kwargs: custom_objects = kwargs.pop('custom_objects') # load model model = model_from_dict_w_opt(model, custom_objects=custom_objects) if 'callbacks' in kwargs: callbacks = kwargs.pop('callbacks') callbacks = [deserialize(**callback) for callback in callbacks] for i, callback in enumerate(callbacks): if inspect.isfunction(callback): callbacks[i] = callback() else: raise TypeError('Your callback is not wrapped in a function') metrics_names = model.metrics_names for metric in metrics_names: results['metrics'][metric] = [] results['metrics'][suf + metric] = [] mod_name = model.__class__.__name__ if generator: data = [pickle.loads(d.encode('raw_unicode_escape')) for d in data] data = [cm.transform_gen(d, mod_name) for d in data] kwargs.pop('batch_size') if all(v is None for v in data_val): val_gen = 0 else: val_gen = check_gen(data_val) if val_gen > 0: if generator: data_val = [ pickle.loads(dv.encode('raw_unicode_escape')) for dv in data_val ] data_val = [cm.transform_gen(dv, mod_name) for dv in data_val] for i, check in enumerate(size_gen): if check is 1: data_val[i] = next(data_val[i]) fit_gen_val = True else: raise Exception("You should also pass a generator for the training" " data.") # fit the model according to the input/output type if mod_name is "Sequential" or mod_name is "Model": for d, dv in szip(data, data_val): validation = check_validation(dv) if not fit_gen_val: if dv is not None: dv = (dv['X'], dv['y']) if generator: h = model.fit_generator(generator=d, validation_data=dv, callbacks=callbacks, *args, **kwargs) else: X, y = d['X'], d['y'] h = model.fit(x=X, y=y, validation_data=dv, callbacks=callbacks, *args, **kwargs) for metric in metrics_names: results['metrics'][metric] += h.history[metric] if validation: results['metrics'][suf + metric] += h.history[suf + metric] else: results['metrics'][suf + metric] += [np.nan] * \ len(h.history[metric]) results['metrics']['iter'] = h.epoch[-1] * len(data) else: raise NotImplementedError("This type of model" "is not supported: {}".format(mod_name)) return results, model
def train(model, data, data_val, size_gen, generator=False, *args, **kwargs): """Fit a model given parameters and a serialized model Args: model(dict): a serialized sklearn model data(list): a list of dict mapping inputs and outputs to lists or dictionnaries mapping the inputs names to np.arrays XOR -a list of fuel generators data_val(list): same structure than `data` but for validation. it is possible to feed generators for data and plain data for data_val. it is not possible the other way around. Returns: the loss (list), the validation loss (list), the number of iterations, and the model """ # Local variables import sklearn.metrics results = dict() results['metrics'] = dict() custom_objects = None predondata = [] predonval = [] fit_gen_val = False # Load custom_objects if 'custom_objects' in kwargs: # pragma: no cover custom_objects = kwargs.pop('custom_objects') # Load model and get metrics model, metrics = model_from_dict_w_opt(model, custom_objects=custom_objects) # instantiates metrics # there is at least one mandatory metric for sklearn models metrics_names = ["score"] if metrics: for metric in metrics: metrics_names.append(metric) for metric in metrics_names: results['metrics'][metric] = [] results['metrics']["val_" + metric] = [] # pickle data if generator if generator: data = [pickle.loads(d.encode('raw_unicode_escape')) for d in data] # check if data_val is in generator if all(v is None for v in data_val): val_gen = 0 else: val_gen = check_gen(data_val) # if so pickle data_val if val_gen > 0: if generator: data_val = [ pickle.loads(dv.encode('raw_unicode_escape')) for dv in data_val ] fit_gen_val = True else: raise Exception("You should also pass a generator for the training" " data.") # Fit the model # and validates it if len(size_gen) == 0: size_gen = [0] * len(data) # loop over the data/generators for d, dv, s_gen in szip(data, data_val, size_gen): # check if we have a data_val object. # if not, no evaluation of the metrics on data_val. if dv is None: validation = False else: validation = True # not treating the case "not generator and fit_gen_val" # since it is catched above # case A : dict for data and data_val if not generator and not fit_gen_val: X, y = d['X'], d['y'] model.fit(X, y, *args, **kwargs) predondata.append(model.predict(X)) for metric in metrics_names: if metric is not 'score': computed_metric = getattr(sklearn.metrics, metric)(y, predondata[-1]) results['metrics'][metric].append(computed_metric) else: computed_metric = model.score(X, y) results['metrics']['score'].append(computed_metric) # TODO : optimization if validation: X_val, y_val = dv['X'], dv['y'] predonval.append(model.predict(X_val)) for metric in metrics_names: if metric is not 'score': computed_metric = getattr(sklearn.metrics, metric)(y_val, predonval[-1]) else: computed_metric = model.score(X_val, y_val) # TODO : optimization results['metrics']['val_' + metric].append(computed_metric) else: for metric in metrics_names: results['metrics']['val_' + metric].append(np.nan) # case B : generator for data and no generator for data_val # could be dict or None elif generator and not fit_gen_val: if validation: X_val, y_val = dv['X'], dv['y'] for batch_data in d.get_epoch_iterator(): X, y = batch_data model.fit(X, y, *args, **kwargs) predondata.append(model.predict(X)) if validation: predonval.append(model.predict(X_val)) for metric in metrics_names: if metric is not 'score': results['metrics'][metric].append( getattr(sklearn.metrics, metric)(y, predondata[-1])) if validation: results['metrics']['val_' + metric].append( getattr(sklearn.metrics, metric)(y_val, predonval[-1])) else: results['metrics']['val_' + metric].append(np.nan) else: results['metrics']['score'].append(model.score(X, y)) if validation: results['metrics']['val_score'].append( model.score(X_val, y_val)) else: results['metrics']['val_score'].append(np.nan) # case C : generator for data and for data_val else: # case C1: N chunks in gen, 1 chunk in val, many to one if s_gen == 1: X_val, y_val = snext(dv.get_epoch_iterator()) for batch_data in d.get_epoch_iterator(): X, y = batch_data model.fit(X, y, *args, **kwargs) predondata.append(model.predict(X)) predonval.append(model.predict(X_val)) for metric in metrics_names: if metric is not 'score': results['metrics'][metric].append( getattr(sklearn.metrics, metric)(y, predondata[-1])) results['metrics']['val_' + metric].append( getattr(sklearn.metrics, metric)(y_val, predonval[-1])) else: results['metrics']['score'].append( model.score(X, y)) results['metrics']['val_score'].append( model.score(X_val, y_val)) # case C2 : 1 chunk in gen, N chunks in val, one to many elif s_gen == 2: X, y = snext(d.get_epoch_iterator()) model.fit(X, y, *args, **kwargs) predondata.append(model.predict(X)) for metric in metrics_names: if metric is not 'score': results['metrics'][metric].append( getattr(sklearn.metrics, metric)(y, predondata[-1])) else: results['metrics']['score'].append(model.score(X, y)) for batch_val in dv.get_epoch_iterator(): X_val, y_val = batch_val predonval.append(model.predict(X_val)) for metric in metrics_names: if metric is not 'score': results['metrics']['val_' + metric].append( getattr(sklearn.metrics, metric)(y_val, predonval[-1])) else: results['metrics']['val_score'].append( model.score(X_val, y_val)) # case C3 : same numbers of chunks, many to many elif s_gen == 3: for batch_data, batch_val in szip(d.get_epoch_iterator(), dv.get_epoch_iterator()): X, y = batch_data X_val, y_val = batch_val model.fit(X, y, *args, **kwargs) predondata.append(model.predict(X)) predonval.append(model.predict(X_val)) for metric in metrics_names: if metric is not 'score': results['metrics'][metric].append( getattr(sklearn.metrics, metric)(y, predondata[-1])) results['metrics']['val_' + metric].append( getattr(sklearn.metrics, metric)(y_val, predonval[-1])) else: results['metrics']['score'].append( model.score(X, y)) results['metrics']['val_score'].append( model.score(X_val, y_val)) else: # pragma: no cover raise Exception( 'Incoherent generator size for train and validation') # for compatibility with keras backend results['metrics']['iter'] = np.nan return results, model
def train(model, data, data_val, size_gen, generator=False, *args, **kwargs): """Fit a model given parameters and a serialized model Args: model(dict): a serialized sklearn model data(list): a list of dict mapping inputs and outputs to lists or dictionnaries mapping the inputs names to np.arrays XOR -a list of fuel generators data_val(list): same structure than `data` but for validation. it is possible to feed generators for data and plain data for data_val. it is not possible the other way around. Returns: the loss (list), the validation loss (list), the number of iterations, and the model """ # Local variables import sklearn.metrics results = dict() results['metrics'] = dict() custom_objects = None predondata = [] predonval = [] fit_gen_val = False # Load custom_objects if 'custom_objects' in kwargs: # pragma: no cover custom_objects = kwargs.pop('custom_objects') # Load model and get metrics model, metrics = model_from_dict_w_opt(model, custom_objects=custom_objects) # instantiates metrics # there is at least one mandatory metric for sklearn models metrics_names = ["score"] if metrics: for metric in metrics: metrics_names.append(metric) for metric in metrics_names: results['metrics'][metric] = [] results['metrics']["val_" + metric] = [] # pickle data if generator if generator: data = [pickle.loads(d.encode('raw_unicode_escape')) for d in data] # check if data_val is in generator if all(v is None for v in data_val): val_gen = 0 else: val_gen = check_gen(data_val) # if so pickle data_val if val_gen > 0: if generator: data_val = [pickle.loads(dv.encode('raw_unicode_escape')) for dv in data_val] fit_gen_val = True else: raise Exception("You should also pass a generator for the training" " data.") # Fit the model # and validates it if len(size_gen) == 0: size_gen = [0] * len(data) # loop over the data/generators for d, dv, s_gen in szip(data, data_val, size_gen): # check if we have a data_val object. # if not, no evaluation of the metrics on data_val. if dv is None: validation = False else: validation = True # not treating the case "not generator and fit_gen_val" # since it is catched above # case A : dict for data and data_val if not generator and not fit_gen_val: X, y = d['X'], d['y'] model.fit(X, y, *args, **kwargs) predondata.append(model.predict(X)) for metric in metrics_names: if metric is not 'score': computed_metric = getattr( sklearn.metrics, metric)(y, predondata[-1]) results['metrics'][metric].append( computed_metric) else: computed_metric = model.score(X, y) results['metrics']['score'].append( computed_metric) # TODO : optimization if validation: X_val, y_val = dv['X'], dv['y'] predonval.append(model.predict(X_val)) for metric in metrics_names: if metric is not 'score': computed_metric = getattr( sklearn.metrics, metric)(y_val, predonval[-1]) else: computed_metric = model.score(X_val, y_val) # TODO : optimization results['metrics']['val_' + metric].append( computed_metric) else: for metric in metrics_names: results['metrics']['val_' + metric].append(np.nan) # case B : generator for data and no generator for data_val # could be dict or None elif generator and not fit_gen_val: if validation: X_val, y_val = dv['X'], dv['y'] for batch_data in d.get_epoch_iterator(): X, y = batch_data model.fit(X, y, *args, **kwargs) predondata.append(model.predict(X)) if validation: predonval.append(model.predict(X_val)) for metric in metrics_names: if metric is not 'score': results['metrics'][metric].append( getattr(sklearn.metrics, metric)(y, predondata[-1])) if validation: results['metrics']['val_' + metric].append( getattr(sklearn.metrics, metric)(y_val, predonval[-1])) else: results['metrics'][ 'val_' + metric].append(np.nan) else: results['metrics']['score'].append( model.score(X, y)) if validation: results['metrics']['val_score'].append( model.score(X_val, y_val)) else: results['metrics']['val_score'].append(np.nan) # case C : generator for data and for data_val else: # case C1: N chunks in gen, 1 chunk in val, many to one if s_gen == 1: X_val, y_val = snext(dv.get_epoch_iterator()) for batch_data in d.get_epoch_iterator(): X, y = batch_data model.fit(X, y, *args, **kwargs) predondata.append(model.predict(X)) predonval.append(model.predict(X_val)) for metric in metrics_names: if metric is not 'score': results['metrics'][metric].append( getattr(sklearn.metrics, metric)(y, predondata[-1])) results['metrics']['val_' + metric].append( getattr(sklearn.metrics, metric)(y_val, predonval[-1])) else: results['metrics']['score'].append( model.score(X, y)) results['metrics']['val_score'].append( model.score(X_val, y_val)) # case C2 : 1 chunk in gen, N chunks in val, one to many elif s_gen == 2: X, y = snext(d.get_epoch_iterator()) model.fit(X, y, *args, **kwargs) predondata.append(model.predict(X)) for metric in metrics_names: if metric is not 'score': results['metrics'][metric].append( getattr(sklearn.metrics, metric)(y, predondata[-1])) else: results['metrics']['score'].append(model.score(X, y)) for batch_val in dv.get_epoch_iterator(): X_val, y_val = batch_val predonval.append(model.predict(X_val)) for metric in metrics_names: if metric is not 'score': results['metrics']['val_' + metric].append( getattr(sklearn.metrics, metric)(y_val, predonval[-1])) else: results['metrics']['val_score'].append( model.score(X_val, y_val)) # case C3 : same numbers of chunks, many to many elif s_gen == 3: for batch_data, batch_val in szip(d.get_epoch_iterator(), dv.get_epoch_iterator()): X, y = batch_data X_val, y_val = batch_val model.fit(X, y, *args, **kwargs) predondata.append(model.predict(X)) predonval.append(model.predict(X_val)) for metric in metrics_names: if metric is not 'score': results['metrics'][metric].append( getattr(sklearn.metrics, metric)(y, predondata[-1])) results['metrics']['val_' + metric].append( getattr(sklearn.metrics, metric)(y_val, predonval[-1])) else: results['metrics']['score'].append( model.score(X, y)) results['metrics']['val_score'].append( model.score(X_val, y_val)) else: # pragma: no cover raise Exception( 'Incoherent generator size for train and validation') # for compatibility with keras backend results['metrics']['iter'] = np.nan return results, model