def standard_correlation_by_epochs(est,val,modelspecs,epochs_list, rec=None): #Does the same thing as standard_correlation, excpet with subsets of data #defined by epochs_list #To use this, first add epochs to define subsets of data. #Then, pass epochs_list as a list of subsets to test. #For example, ['A', 'B', ['A', 'B']] will measure correlations separately # for all epochs marked 'A', all epochs marked 'B', and all epochs marked # 'A'or 'B' for epochs in epochs_list: # Create a label for this subset. If epochs is a list, join elements with "+" epoch_list_str="+".join([str(x) for x in epochs]) # Make a copy for this subset val_copy=copy.deepcopy(val) for vc in val_copy: vc['resp']=vc['resp'].select_epochs(epochs) est_copy=copy.deepcopy(est) for ec in est_copy: ec['resp']=ec['resp'].select_epochs(epochs) # Compute scores for validation data r_test = [nmet.corrcoef(p, 'pred', 'resp') for p in val_copy] mse_test = [nmet.nmse(p, 'pred', 'resp') for p in val_copy] ll_test = [nmet.likelihood_poisson(p, 'pred', 'resp') for p in val_copy] r_floor = [nmet.r_floor(p, 'pred', 'resp') for p in val] if rec is not None: r_ceiling = [nmet.r_ceiling(p, rec, 'pred', 'resp') for p in val_copy] # Repeat for est data. r_fit = [nmet.corrcoef(p, 'pred', 'resp') for p in est_copy] mse_fit = [nmet.nmse(p, 'pred', 'resp') for p in est_copy] ll_fit = [nmet.likelihood_poisson(p, 'pred', 'resp') for p in est_copy] #Avergage modelspecs[0][0]['meta'][epoch_list_str]={} modelspecs[0][0]['meta'][epoch_list_str]['r_test'] = np.mean(r_test) modelspecs[0][0]['meta'][epoch_list_str]['mse_test'] = np.mean(mse_test) modelspecs[0][0]['meta'][epoch_list_str]['ll_test'] = np.mean(ll_test) modelspecs[0][0]['meta'][epoch_list_str]['r_fit'] = np.mean(r_fit) modelspecs[0][0]['meta'][epoch_list_str]['r_floor'] = np.mean(r_floor) if rec is not None: modelspecs[0][0]['meta'][epoch_list_str]['r_ceiling'] = np.mean(r_ceiling) modelspecs[0][0]['meta'][epoch_list_str]['mse_fit'] = np.mean(mse_fit) modelspecs[0][0]['meta'][epoch_list_str]['ll_fit'] = np.mean(ll_fit) return modelspecs
def fit_cd_nfold_shrinkage(modelspecs, est, ftol=1e-7, maxiter=1000, IsReload=False, **context): ''' fitting n-fold, one from each entry in est, use coordinate descent ''' if not IsReload: fit_kwargs = { 'tolerance': tolerance, 'max_iter': max_iter, 'step_size': 0.05 } metric = lambda d: metrics.nmse(d, 'pred', 'resp') modelspecs = nems.analysis.api.fit_nfold(est, modelspecs, metric=metric, fit_kwargs=fit_kwargs, fitter=coordinate_descent) fit_kwargs = { 'options': { 'tolerance': tolerance, 'max_iter': max_iter } } modelspecs = nems.analysis.api.fit_nfold(est, modelspecs, metric=metric, fitter=scipy_minimize, fit_kwargs=fit_kwargs) return {'modelspecs': modelspecs}
def basic_error(data, modelspec, cost_function=None, segmentor=nems.segmentors.use_all_data, mapper=nems.fitters.mappers.simple_vector, metric=lambda data: nmet.nmse(data, 'pred', 'resp')): ''' Similar to fit_basic except that it just returns the error for the fitting process instead of a modelspec. Intended to be called after a model has already been fit. ''' modelspec = copy.deepcopy(modelspec) if cost_function is None: # Use the cost function defined in this module by default cost_function = basic_cost # apply mask to remove invalid portions of signals and allow fit to # only evaluate the model on the valid portion of the signals if 'mask' in data.signals.keys(): log.info("Data len pre-mask: %d", data['mask'].shape[1]) data = data.apply_mask() log.info("Data len post-mask: %d", data['mask'].shape[1]) packer, unpacker, pack_bounds = mapper(modelspec) evaluator = nems.modelspec.evaluate sigma = packer(modelspec) error = cost_function(sigma, unpacker, modelspec, data, segmentor, evaluator, metric) return error
def correlation_per_model(est, val, modelspecs, rec=None): ''' Expects the lengths of est, val, and modelspecs to match since est[i] should have been evaluated on the fitted modelspecs[i], etc. Similar to standard_correlation, but saves correlation information to every first-module 'meta' entry instead of saving an average to only the first modelspec ''' if not len(est) == len(val) == len(modelspecs): raise ValueError( "est, val, and modelspecs should all be lists" " of equal length. got: %d, %d, %d respectively.", len(est), len(val), len(modelspecs)) modelspecs = copy.deepcopy(modelspecs) r_tests = [nmet.corrcoef(v, 'pred', 'resp') for v in val] #se_tests = [np.std(r)/np.sqrt(len(v)) for r, v in zip(r_tests, val)] mse_tests = [nmet.nmse(v, 'pred', 'resp') for v in val] ll_tests = [nmet.likelihood_poisson(v, 'pred', 'resp') for v in val] r_fits = [nmet.corrcoef(e, 'pred', 'resp') for e in est] #se_fits = [np.std(r)/np.sqrt(len(v)) for r, v in zip(r_fits, val)] mse_fits = [nmet.nmse(e, 'pred', 'resp') for e in est] ll_fits = [nmet.likelihood_poisson(e, 'pred', 'resp') for e in est] r_floors = [nmet.r_floor(v, 'pred', 'resp') for v in val] if rec is None: r_ceilings = [None] * len(r_floors) else: r_ceilings = [nmet.r_ceiling(v, rec, 'pred', 'resp') for v in val] for i, m in enumerate(modelspecs): m[0]['meta'].update({ 'r_test': r_tests[i], #'se_test': se_tests[i], 'mse_test': mse_tests[i], 'll_test': ll_tests[i], 'r_fit': r_fits[i], #'se_fit': se_fits[i], 'mse_fit': mse_fits[i], 'll_fit': ll_fits[i], 'r_floor': r_floors[i], 'r_ceiling': r_ceilings[i], }) return modelspecs
def standard_correlation(est, val, modelspecs): # Compute scores for validation data r_test = [nmet.corrcoef(p, 'pred', 'resp') for p in val] mse_test = [nmet.nmse(p, 'pred', 'resp') for p in val] ll_test = [nmet.likelihood_poisson(p, 'pred', 'resp') for p in val] # Repeat for est data. r_fit = [nmet.corrcoef(p, 'pred', 'resp') for p in est] mse_fit = [nmet.nmse(p, 'pred', 'resp') for p in est] ll_fit = [nmet.likelihood_poisson(p, 'pred', 'resp') for p in est] modelspecs[0][0]['meta']['r_test'] = np.mean(r_test) modelspecs[0][0]['meta']['mse_test'] = np.mean(mse_test) modelspecs[0][0]['meta']['ll_test'] = np.mean(ll_test) modelspecs[0][0]['meta']['r_fit'] = np.mean(r_fit) modelspecs[0][0]['meta']['mse_fit'] = np.mean(mse_fit) modelspecs[0][0]['meta']['ll_fit'] = np.mean(ll_fit) return modelspecs
def standard_correlation_by_set(est, val, modelspecs): # Compute scores for validation data r_test = [nmet.corrcoef(p, 'pred', 'resp') for p in val] mse_test = [nmet.nmse(p, 'pred', 'resp') for p in val] ll_test = [nmet.likelihood_poisson(p, 'pred', 'resp') for p in val] # Repeat for est data. r_fit = [nmet.corrcoef(p, 'pred', 'resp') for p in est] mse_fit = [nmet.nmse(p, 'pred', 'resp') for p in est] ll_fit = [nmet.likelihood_poisson(p, 'pred', 'resp') for p in est] for i in range(len(modelspecs)): modelspecs[i][0]['meta']['r_test'] = r_test[i] modelspecs[i][0]['meta']['mse_test'] = mse_test[i] modelspecs[i][0]['meta']['ll_test'] = ll_test[i] modelspecs[i][0]['meta']['r_fit'] = r_fit[i] modelspecs[i][0]['meta']['mse_fit'] = mse_fit[i] modelspecs[i][0]['meta']['ll_fit'] = ll_fit[i] return modelspecs
def fit_nfold(data_list, modelspecs, generate_psth=False, fitter=scipy_minimize, analysis='fit_basic', metric=None, tolerances=None, module_sets=None, tol_iter=100, fit_iter=20, fit_kwargs={}): ''' Takes njacks jackknifes, where each jackknife has some small fraction of data NaN'd out, and fits modelspec to them. TESTING: if input len(modelspecs) == len(data_list) then use each modelspec as initial condition for corresponding data_list fold if len(modelspecs) == 1, then use the same initial conditions for each fold ''' nfolds = len(data_list) models = [] if metric is None: metric = lambda d: metrics.nmse(d, 'pred', 'resp') for i in range(nfolds): if len(modelspecs) > 1: msidx = i else: msidx = 0 log.info("Fitting fold %d/%d, modelspec %d", i+1, nfolds, msidx) if analysis == 'fit_basic': models += fit_basic(data_list[i], copy.deepcopy(modelspecs[msidx]), fitter=fitter, metric=metric, metaname='fit_nfold', fit_kwargs=fit_kwargs) elif analysis == 'fit_iteratively': models += fit_iteratively( data_list[i], copy.deepcopy(modelspecs[msidx]), fitter=fitter, metric=metric, metaname='fit_nfold', fit_kwargs=fit_kwargs, module_sets=module_sets, invert=False, tolerances=tolerances, tol_iter=tol_iter, fit_iter=fit_iter, ) else: # Unknown analysis # TODO: use getattr / import to make this more general for # use with any analysis function? # Maybe too much of a pain. raise NotImplementedError return models
def fit_basic_cd(modelspecs, est, max_iter=1000, tolerance=1e-8, IsReload=False, shrinkage=0, **context): ''' A basic fit that optimizes every input modelspec. Use coordinate descent for fitting and nmse_shrink for cost function ''' if not IsReload: if shrinkage: metric = lambda d: metrics.nmse_shrink(d, 'pred', 'resp') else: metric = lambda d: metrics.nmse(d, 'pred', 'resp') fit_kwargs = {'tolerance': tolerance, 'max_iter': max_iter} if type(est) is list: # jackknife! modelspecs_out = [] njacks = len(modelspecs) i = 0 for m, d in zip(modelspecs, est): i += 1 log.info("Fitting JK {}/{}".format(i, njacks)) modelspecs_out += nems.analysis.api.fit_basic( d, m, fit_kwargs=fit_kwargs, metric=metric, fitter=coordinate_descent) modelspecs = modelspecs_out else: # standard single shot modelspecs = [ nems.analysis.api.fit_basic(est, modelspec, fit_kwargs=fit_kwargs, metric=metric, fitter=coordinate_descent)[0] for modelspec in modelspecs ] return {'modelspecs': modelspecs}
def fit_nfold(data_list, modelspecs, generate_psth=False, fitter=scipy_minimize, metric=None, fit_kwargs={'options': { 'ftol': 1e-7, 'maxiter': 1000 }}): ''' Takes njacks jackknifes, where each jackknife has some small fraction of data NaN'd out, and fits modelspec to them. ''' nfolds = len(data_list) # if type(modelspec) is not list: # modelspecs=[modelspec]*nfolds # elif len(modelspec)==1: # modelspec=modelspec*nfolds models = [] if not metric: metric = lambda d: metrics.nmse(d, 'pred', 'resp') for i in range(nfolds): log.info("Fitting fold {}/{}".format(i + 1, nfolds)) tms = nems.initializers.prefit_to_target( data_list[i], copy.deepcopy(modelspecs[0]), nems.analysis.api.fit_basic, 'levelshift', fitter=scipy_minimize, fit_kwargs={'options': { 'ftol': 1e-4, 'maxiter': 500 }}) models += fit_basic(data_list[i], tms, fitter=fitter, metric=metric, metaname='fit_nfold', fit_kwargs=fit_kwargs) return models
def init_pop_rand(est, modelspec, IsReload=False, start_count=1, pc_signal='rand_resp', whiten=True, **context): """ initialize population model with random combinations of responses. generates random response combinations and passes them through to init_pop_pca() :param est: recording object with fit data :param modelspec: un-fit modelspec :param IsReload: don't fit if IsReload=True :param pc_signal: name of signal to generate with random combinations of responses :param context: dictionary of other context variables :return: initialized modelspec """ if IsReload: return {} # guess at number of subspace dimensions fit_set_all, fit_set_slice = _figure_out_mod_split(modelspec) dim_count = modelspec[fit_set_slice[0]]['phi']['coefficients'].shape[1] mset = [] E = np.ones(start_count) for i in range(start_count): log.info('Rand init: %d/%d', i, start_count) rec = est.copy() rec[pc_signal] = _random_resp_combos( rec['resp'], dim_count=dim_count, whiten=whiten) log.info('rec signal: %s (%d x %d)', pc_signal, rec[pc_signal].shape[0], rec[pc_signal].shape[1]) mset.append(init_pop_pca(rec, modelspec, pc_signal=pc_signal, **context)) rec = mset[-1]['modelspec'].evaluate(rec) E[i] = metrics.nmse(rec) imax = np.nanargmin(E) for i in range(start_count): ss = "**" if (i == imax) else "" log.info('i=%d E=%.3e %s', i, E[i], ss) return mset[imax]
def fit_state_nfold(data_list, modelspecs, generate_psth=False, fitter=scipy_minimize, metric=None, fit_kwargs={}): ''' Generic state-dependent-stream model fitter Takes njacks jackknifes, where each jackknife has some small fraction of data NaN'd out, and fits modelspec to them. DEPRECATED? REPLACED BY STANDARD nfold? ''' nfolds = len(data_list) models = [] if not metric: metric = lambda d: metrics.nmse(d, 'pred', 'resp') for i in range(nfolds): log.info("Fitting fold {}/{}".format(i + 1, nfolds)) tms = nems.initializers.prefit_to_target( data_list[i], copy.deepcopy(modelspecs[0]), nems.analysis.api.fit_basic, 'merge_channels', fitter=scipy_minimize, fit_kwargs={'options': { 'tolerance': 1e-4, 'max_iter': 500 }}) models += fit_basic(data_list[i], tms, fitter=fitter, metric=metric, metaname='fit_nfold', fit_kwargs=fit_kwargs) return models
def standard_correlation_by_epochs(est, val, modelspec=None, modelspecs=None, epochs_list=None, rec=None): """ Does the same thing as standard_correlation, excpet with subsets of data defined by epochs_list To use this, first add epochs to define subsets of data. Then, pass epochs_list as a list of subsets to test. For example, ['A', 'B', ['A', 'B']] will measure correlations separately for all epochs marked 'A', all epochs marked 'B', and all epochs marked 'A'or 'B' """ # some crazy stuff to maintain backward compatibility # eventually we will only support modelspec and deprecate support for # modelspecs lists if modelspecs is not None: raise Warning('Use of modelspecs list is deprecated') modelspec = modelspecs[0] list_modelspec = True else: list_modelspec = False for epochs in epochs_list: # Create a label for this subset. If epochs is a list, join elements with "+" epoch_list_str = "+".join([str(x) for x in epochs]) # Make a copy for this subset val_copy = copy.deepcopy(val) for vc in val_copy: vc['resp'] = vc['resp'].select_epochs(epochs) est_copy = copy.deepcopy(est) for ec in est_copy: ec['resp'] = ec['resp'].select_epochs(epochs) # Compute scores for validation data r_test = [nmet.corrcoef(p, 'pred', 'resp') for p in val_copy] mse_test = [nmet.nmse(p, 'pred', 'resp') for p in val_copy] ll_test = [ nmet.likelihood_poisson(p, 'pred', 'resp') for p in val_copy ] r_floor = [nmet.r_floor(p, 'pred', 'resp') for p in val] if rec is not None: r_ceiling = [ nmet.r_ceiling(p, rec, 'pred', 'resp') for p in val_copy ] # Repeat for est data. r_fit = [nmet.corrcoef(p, 'pred', 'resp') for p in est_copy] mse_fit = [nmet.nmse(p, 'pred', 'resp') for p in est_copy] ll_fit = [nmet.likelihood_poisson(p, 'pred', 'resp') for p in est_copy] #Avergage modelspec.meta[epoch_list_str] = {} modelspec.meta[epoch_list_str]['r_test'] = np.mean(r_test) modelspec.meta[epoch_list_str]['mse_test'] = np.mean(mse_test) modelspec.meta[epoch_list_str]['ll_test'] = np.mean(ll_test) modelspec.meta[epoch_list_str]['r_fit'] = np.mean(r_fit) modelspec.meta[epoch_list_str]['r_floor'] = np.mean(r_floor) if rec is not None: modelspec.meta[epoch_list_str]['r_ceiling'] = np.mean(r_ceiling) modelspec.meta[epoch_list_str]['mse_fit'] = np.mean(mse_fit) modelspec.meta[epoch_list_str]['ll_fit'] = np.mean(ll_fit) if list_modelspec: # backward compatibility return [modelspec] else: return modelspec
def fit_basic(data, modelspec, fitter=scipy_minimize, cost_function=None, segmentor=nems.segmentors.use_all_data, mapper=nems.fitters.mappers.simple_vector, metric=lambda data: metrics.nmse(data, 'pred', 'resp'), metaname='fit_basic', fit_kwargs={}, require_phi=True): ''' Required Arguments: data A recording object modelspec A modelspec object Optional Arguments: fitter A function of (sigma, costfn) that tests various points, in fitspace (i.e. sigmas) using the cost function costfn, and hopefully returns a better sigma after some time. mapper A class that has two methods, pack and unpack, which define the mapping between modelspecs and a fitter's fitspace. segmentor An function that selects a subset of the data during the fitting process. This is NOT the same as est/val data splits metric A function of a Recording that returns an error value that is to be minimized. Returns A list containing a single modelspec, which has the best parameters found by this fitter. ''' start_time = time.time() if cost_function is None: # Use the cost function defined in this module by default cost_function = basic_cost if require_phi: # Ensure that phi exists for all modules; choose prior mean if not found for i, m in enumerate(modelspec): if not m.get('phi'): log.debug('Phi not found for module, using mean of prior: %s', m) m = nems.priors.set_mean_phi([m])[0] # Inits phi for 1 module modelspec[i] = m ms.fit_mode_on(modelspec) # Create the mapper object that translates to and from modelspecs. # It has two methods that, when defined as mathematical functions, are: # .pack(modelspec) -> fitspace_point # .unpack(fitspace_point) -> modelspec packer, unpacker = mapper(modelspec) # A function to evaluate the modelspec on the data evaluator = nems.modelspec.evaluate my_cost_function = cost_function my_cost_function.counter = 0 # Freeze everything but sigma, since that's all the fitter should be # updating. cost_fn = partial(my_cost_function, unpacker=unpacker, modelspec=modelspec, data=data, segmentor=segmentor, evaluator=evaluator, metric=metric) # get initial sigma value representing some point in the fit space sigma = packer(modelspec) # Results should be a list of modelspecs # (might only be one in list, but still should be packaged as a list) improved_sigma = fitter(sigma, cost_fn, **fit_kwargs) improved_modelspec = unpacker(improved_sigma) elapsed_time = (time.time() - start_time) # TODO: Should this maybe be moved to a higher level # so it applies to ALL the fittters? ms.fit_mode_off(improved_modelspec) ms.set_modelspec_metadata(improved_modelspec, 'fitter', metaname) ms.set_modelspec_metadata(improved_modelspec, 'fit_time', elapsed_time) results = [copy.deepcopy(improved_modelspec)] return results
def standard_correlation(est, val, modelspecs, rec=None, use_mask=True): # use_mask: mask before computing metrics (if mask exists) # Compute scores for validation dat r_ceiling = 0 if type(val) is not list: if ('mask' in val[0].signals.keys()) and use_mask: v = val.apply_mask() e = est.apply_mask() else: v = val e = est r_test, se_test = nmet.j_corrcoef(v, 'pred', 'resp') r_fit, se_fit = nmet.j_corrcoef(e, 'pred', 'resp') r_floor = nmet.r_floor(v, 'pred', 'resp') if rec is not None: # print('running r_ceiling') r_ceiling = nmet.r_ceiling(v, rec, 'pred', 'resp') mse_test = nmet.j_nmse(v, 'pred', 'resp') mse_fit = nmet.j_nmse(e, 'pred', 'resp') elif len(val) == 1: if ('mask' in val[0].signals.keys()) and use_mask: v = val[0].apply_mask() e = est[0].apply_mask() else: v = val[0] e = est[0] r_test, se_test = nmet.j_corrcoef(v, 'pred', 'resp') r_fit, se_fit = nmet.j_corrcoef(e, 'pred', 'resp') r_floor = nmet.r_floor(v, 'pred', 'resp') if rec is not None: try: # print('running r_ceiling') r_ceiling = nmet.r_ceiling(v, rec, 'pred', 'resp') except: r_ceiling = 0 mse_test, se_mse_test = nmet.j_nmse(v, 'pred', 'resp') mse_fit, se_mse_fit = nmet.j_nmse(e, 'pred', 'resp') else: # unclear if this ever excutes since jackknifed val sets are # typically already merged r = [nmet.corrcoef(p, 'pred', 'resp') for p in val] r_test = np.mean(r) se_test = np.std(r) / np.sqrt(len(val)) r = [nmet.corrcoef(p, 'pred', 'resp') for p in est] r_fit = np.mean(r) se_fit = np.std(r) / np.sqrt(len(val)) r_floor = [nmet.r_floor(p, 'pred', 'resp') for p in val] # TODO compute r_ceiling for multiple val sets r_ceiling = 0 mse_test = [nmet.nmse(p, 'pred', 'resp') for p in val] mse_fit = [nmet.nmse(p, 'pred', 'resp') for p in est] se_mse_test = np.std(mse_test) / np.sqrt(len(val)) se_mse_fit = np.std(mse_fit) / np.sqrt(len(est)) mse_test = np.mean(mse_test) mse_fit = np.mean(mse_fit) ll_test = [nmet.likelihood_poisson(p, 'pred', 'resp') for p in val] ll_fit = [nmet.likelihood_poisson(p, 'pred', 'resp') for p in est] modelspecs[0][0]['meta']['r_test'] = r_test modelspecs[0][0]['meta']['se_test'] = se_test modelspecs[0][0]['meta']['r_floor'] = r_floor modelspecs[0][0]['meta']['mse_test'] = mse_test modelspecs[0][0]['meta']['se_mse_test'] = se_mse_test modelspecs[0][0]['meta']['ll_test'] = np.mean(ll_test) modelspecs[0][0]['meta']['r_fit'] = r_fit modelspecs[0][0]['meta']['se_fit'] = se_fit modelspecs[0][0]['meta']['r_ceiling'] = r_ceiling modelspecs[0][0]['meta']['mse_fit'] = mse_fit modelspecs[0][0]['meta']['se_mse_fit'] = se_mse_fit modelspecs[0][0]['meta']['ll_fit'] = np.mean(ll_fit) return modelspecs
def fit_basic(data, modelspec, fitter=scipy_minimize, cost_function=None, segmentor=nems.segmentors.use_all_data, mapper=nems.fitters.mappers.simple_vector, metric=None, metaname='fit_basic', fit_kwargs={}, require_phi=True): ''' Required Arguments: data A recording object modelspec A modelspec object Optional Arguments: fitter A function of (sigma, costfn) that tests various points, in fitspace (i.e. sigmas) using the cost function costfn, and hopefully returns a better sigma after some time. mapper A class that has two methods, pack and unpack, which define the mapping between modelspecs and a fitter's fitspace. segmentor An function that selects a subset of the data during the fitting process. This is NOT the same as est/val data splits metric A function of a Recording that returns an error value that is to be minimized. Returns A list containing a single modelspec, which has the best parameters found by this fitter. ''' start_time = time.time() modelspec = copy.deepcopy(modelspec) output_name = modelspec.meta.get('output_name', 'resp') if metric is None: metric = lambda data: metrics.nmse(data, 'pred', output_name) if cost_function is None: # Use the cost function defined in this module by default cost_function = basic_cost if require_phi: # Ensure that phi exists for all modules; # choose prior mean if not found for i, m in enumerate(modelspec.modules): if ('phi' not in m.keys()) and ('prior' in m.keys()): log.debug('Phi not found for module, using mean of prior: %s', m) m = nems.priors.set_mean_phi([m])[0] # Inits phi for 1 module modelspec[i] = m # apply mask to remove invalid portions of signals and allow fit to # only evaluate the model on the valid portion of the signals if 'mask' in data.signals.keys(): log.info("Data len pre-mask: %d", data['mask'].shape[1]) data = data.apply_mask() log.info("Data len post-mask: %d", data['mask'].shape[1]) # turn on "fit mode". currently this serves one purpose, for normalization # parameters to be re-fit for the output of each module that uses # normalization. does nothing if normalization is not being used. ms.fit_mode_on(modelspec, data) # Create the mapper functions that translates to and from modelspecs. # It has three functions that, when defined as mathematical functions, are: # .pack(modelspec) -> fitspace_point # .unpack(fitspace_point) -> modelspec # .bounds(modelspec) -> fitspace_bounds packer, unpacker, pack_bounds = mapper(modelspec) # A function to evaluate the modelspec on the data evaluator = nems.modelspec.evaluate my_cost_function = cost_function my_cost_function.counter = 0 # Freeze everything but sigma, since that's all the fitter should be # updating. cost_fn = partial(my_cost_function, unpacker=unpacker, modelspec=modelspec, data=data, segmentor=segmentor, evaluator=evaluator, metric=metric) # get initial sigma value representing some point in the fit space, # and corresponding bounds for each value sigma = packer(modelspec) bounds = pack_bounds(modelspec) # Results should be a list of modelspecs # (might only be one in list, but still should be packaged as a list) improved_sigma = fitter(sigma, cost_fn, bounds=bounds, **fit_kwargs) improved_modelspec = unpacker(improved_sigma) elapsed_time = (time.time() - start_time) start_err = cost_fn(sigma) final_err = cost_fn(improved_sigma) log.info("Delta error: %.06f - %.06f = %e", start_err, final_err, final_err-start_err) # TODO: Should this maybe be moved to a higher level # so it applies to ALL the fittters? ms.fit_mode_off(improved_modelspec) ms.set_modelspec_metadata(improved_modelspec, 'fitter', metaname) ms.set_modelspec_metadata(improved_modelspec, 'n_parms', len(improved_sigma)) if modelspec.fit_count == 1: improved_modelspec.meta['fit_time'] = elapsed_time improved_modelspec.meta['loss'] = final_err else: fit_index = modelspec.fit_index if fit_index == 0: improved_modelspec.meta['fit_time'] = np.zeros(improved_modelspec.fit_count) improved_modelspec.meta['loss'] = np.zeros(improved_modelspec.fit_count) improved_modelspec.meta['fit_time'][fit_index] = elapsed_time improved_modelspec.meta['loss'][fit_index] = final_err if type(improved_modelspec) is list: return [copy.deepcopy(improved_modelspec)] else: return improved_modelspec.copy()
def metric(d): metrics.nmse(d, 'pred', 'resp')
def standard_correlation(est, val, modelspecs, rec=None): # Compute scores for validation dat r_ceiling = 0 if type(val) is not list: r_test, se_test = nmet.j_corrcoef(val, 'pred', 'resp') r_fit, se_fit = nmet.j_corrcoef(est, 'pred', 'resp') r_floor = nmet.r_floor(val, 'pred', 'resp') if rec is not None: # print('running r_ceiling') r_ceiling = nmet.r_ceiling(val, rec, 'pred', 'resp') mse_test = nmet.j_nmse(val, 'pred', 'resp') mse_fit = nmet.j_nmse(est, 'pred', 'resp') elif len(val) == 1: r_test, se_test = nmet.j_corrcoef(val[0], 'pred', 'resp') r_fit, se_fit = nmet.j_corrcoef(est[0], 'pred', 'resp') r_floor = nmet.r_floor(val[0], 'pred', 'resp') if rec is not None: # print('running r_ceiling') r_ceiling = nmet.r_ceiling(val[0], rec, 'pred', 'resp') mse_test, se_mse_test = nmet.j_nmse(val[0], 'pred', 'resp') mse_fit, se_mse_fit = nmet.j_nmse(est[0], 'pred', 'resp') else: # unclear if this ever excutes since jackknifed val sets are # typically already merged r = [nmet.corrcoef(p, 'pred', 'resp') for p in val] r_test = np.mean(r) se_test = np.std(r) / np.sqrt(len(val)) r = [nmet.corrcoef(p, 'pred', 'resp') for p in est] r_fit = np.mean(r) se_fit = np.std(r) / np.sqrt(len(val)) r_floor = [nmet.r_floor(p, 'pred', 'resp') for p in val] # TODO compute r_ceiling for multiple val sets r_ceiling = 0 mse_test = [nmet.nmse(p, 'pred', 'resp') for p in val] mse_fit = [nmet.nmse(p, 'pred', 'resp') for p in est] se_mse_test = np.std(mse_test) / np.sqrt(len(val)) se_mse_fit = np.std(mse_fit) / np.sqrt(len(est)) mse_test = np.mean(mse_test) mse_fit = np.mean(mse_fit) ll_test = [nmet.likelihood_poisson(p, 'pred', 'resp') for p in val] ll_fit = [nmet.likelihood_poisson(p, 'pred', 'resp') for p in est] modelspecs[0][0]['meta']['r_test'] = r_test modelspecs[0][0]['meta']['se_test'] = se_test modelspecs[0][0]['meta']['r_floor'] = r_floor modelspecs[0][0]['meta']['mse_test'] = mse_test modelspecs[0][0]['meta']['se_mse_test'] = se_mse_test modelspecs[0][0]['meta']['ll_test'] = np.mean(ll_test) modelspecs[0][0]['meta']['r_fit'] = r_fit modelspecs[0][0]['meta']['se_fit'] = se_fit modelspecs[0][0]['meta']['r_ceiling'] = r_ceiling modelspecs[0][0]['meta']['mse_fit'] = mse_fit modelspecs[0][0]['meta']['se_mse_fit'] = se_mse_fit modelspecs[0][0]['meta']['ll_fit'] = np.mean(ll_fit) return modelspecs