def fit_pop_model_xforms_baphy(cellid, batch, modelname, saveInDB=False): """ Fits a NEMS population model using baphy data DEPRECATED ? Now should work for xhelp.fit_model_xform() """ raise NotImplementedError("Replaced by xhelper function?") log.info("Preparing pop model: ({0},{1},{2})".format( cellid, batch, modelname)) # Segment modelname for meta information kws = modelname.split("_") modelspecname = "-".join(kws[1:-1]) loadkey = kws[0] fitkey = kws[-1] if type(cellid) is list: disp_cellid="_".join(cellid) else: disp_cellid=cellid meta = {'batch': batch, 'cellid': disp_cellid, 'modelname': modelname, 'loader': loadkey, 'fitkey': fitkey, 'modelspecname': modelspecname, 'username': '******', 'labgroup': 'lbhb', 'public': 1, 'githash': os.environ.get('CODEHASH', ''), 'recording': loadkey} uri_key = nems.utils.escaped_split(loadkey, '-')[0] recording_uri = generate_recording_uri(cellid, batch, uri_key) # pass cellid information to xforms so that loader knows which cells # to load from recording_uri xfspec = xhelp.generate_xforms_spec(recording_uri, modelname, meta, xforms_kwargs={'cellid': cellid}) # actually do the fit ctx, log_xf = xforms.evaluate(xfspec) # save some extra metadata modelspec = ctx['modelspec'] destination = '/auto/data/nems_db/results/{0}/{1}/{2}/'.format( batch, disp_cellid, ms.get_modelspec_longname(modelspec)) modelspec.meta['modelpath'] = destination modelspec.meta['figurefile'] = destination+'figure.0000.png' modelspec.meta.update(meta) # extra thing to save for pop model modelspec.meta['cellids'] = ctx['val']['resp'].chans # save results log.info('Saving modelspec(s) to {0} ...'.format(destination)) save_data = xforms.save_analysis(destination, recording=ctx['rec'], modelspec=modelspec, xfspec=xfspec, figures=ctx['figures'], log=log_xf) savepath = save_data['savepath'] if saveInDB: # save in database as well nd.update_results_table(modelspec) return savepath
def fit_model_xforms_baphy(cellid, batch, modelname, autoPlot=True, saveInDB=False): """ DEPRECATED ? Now should work for xhelp.fit_model_xform() Fit a single NEMS model using data from baphy/celldb eg, 'ozgf100ch18_wc18x1_lvl1_fir15x1_dexp1_fit01' generates modelspec with 'wc18x1_lvl1_fir15x1_dexp1' based on this function in nems/scripts/fit_model.py def fit_model(recording_uri, modelstring, destination): xfspec = [ ['nems.xforms.load_recordings', {'recording_uri_list': recordings}], ['nems.xforms.add_average_sig', {'signal_to_average': 'resp', 'new_signalname': 'resp', 'epoch_regex': '^STIM_'}], ['nems.xforms.split_by_occurrence_counts', {'epoch_regex': '^STIM_'}], ['nems.xforms.init_from_keywords', {'keywordstring': modelspecname}], ['nems.xforms.set_random_phi', {}], ['nems.xforms.fit_basic', {}], # ['nems.xforms.add_summary_statistics', {}], ['nems.xforms.plot_summary', {}], # ['nems.xforms.save_recordings', {'recordings': ['est', 'val']}], ['nems.xforms.fill_in_default_metadata', {}], ] """ raise NotImplementedError("Replaced by xhelper function?") raise DeprecationWarning("Replaced by xhelp.fit_model_xforms") log.info('Initializing modelspec(s) for cell/batch %s/%d...', cellid, int(batch)) # Segment modelname for meta information kws = nems.utils.escaped_split(modelname, '_') old = False if (len(kws) > 3) or ((len(kws) == 3) and kws[1].startswith('stategain') and not kws[1].startswith('stategain.')): # Check if modelname uses old format. log.info("Using old modelname format ... ") old = True modelspecname = nems.utils.escaped_join(kws[1:-1], '_') else: modelspecname = nems.utils.escaped_join(kws[1:-1], '-') loadkey = kws[0] fitkey = kws[-1] meta = {'batch': batch, 'cellid': cellid, 'modelname': modelname, 'loader': loadkey, 'fitkey': fitkey, 'modelspecname': modelspecname, 'username': '******', 'labgroup': 'lbhb', 'public': 1, 'githash': os.environ.get('CODEHASH', ''), 'recording': loadkey} if old: recording_uri = ogru(cellid, batch, loadkey) xfspec = oxfh.generate_loader_xfspec(loadkey, recording_uri) xfspec.append(['nems_lbhb.old_xforms.xforms.init_from_keywords', {'keywordstring': modelspecname, 'meta': meta}]) xfspec.extend(oxfh.generate_fitter_xfspec(fitkey)) xfspec.append(['nems.analysis.api.standard_correlation', {}, ['est', 'val', 'modelspec', 'rec'], ['modelspec']]) if autoPlot: log.info('Generating summary plot ...') xfspec.append(['nems.xforms.plot_summary', {}]) else: # uri_key = nems.utils.escaped_split(loadkey, '-')[0] # recording_uri = generate_recording_uri(cellid, batch, uri_key) log.info("DONE? Moved handling of registry_args to xforms_init_context") recording_uri = None # registry_args = {'cellid': cellid, 'batch': int(batch)} registry_args = {} xforms_init_context = {'cellid': cellid, 'batch': int(batch)} xfspec = xhelp.generate_xforms_spec(recording_uri, modelname, meta, xforms_kwargs=registry_args, xforms_init_context=xforms_init_context) log.info(xfspec) # actually do the loading, preprocessing, fit ctx, log_xf = xforms.evaluate(xfspec) # save some extra metadata modelspec = ctx['modelspec'] # this code may not be necessary any more. destination = '/auto/data/nems_db/results/{0}/{1}/{2}/'.format( batch, cellid, ms.get_modelspec_longname(modelspec)) modelspec.meta['modelpath'] = destination modelspec.meta['figurefile'] = destination+'figure.0000.png' modelspec.meta.update(meta) # save results log.info('Saving modelspec(s) to {0} ...'.format(destination)) save_data = xforms.save_analysis(destination, recording=ctx['rec'], modelspec=modelspec, xfspec=xfspec, figures=ctx['figures'], log=log_xf) savepath = save_data['savepath'] # save in database as well if saveInDB: # TODO : db results finalized? nd.update_results_table(modelspec) return savepath
ch.close() rootlogger.removeFilter(ch) log_xf = log_stream.getvalue() # save some extra metadata modelspec = ctx['modelspec'] # destination = '/auto/data/nems_db/results/{0}/{1}/{2}/'.format( batch, cellid, ms.get_modelspec_longname(modelspec)) modelspec['meta']['modelpath'] = destination modelspec['meta']['figurefile'] = destination+'figure.0000.png' # save results if saveToFile: log.info('Saving modelspec(s) to {0} ...'.format(destination)) xforms.save_analysis(destination, recording=ctx['rec'], modelspec=modelspec, xfspec=xfspec, figures=ctx['figures'], log=log_xf) # save in database as well if saveInDB: # TODO : db results finalized? nd.update_results_table(modelspec) if browse_results: aw = browse_context(ctx, signals=['stim', 'pred', 'resp'])
def kamiak_to_database(cellids, batch, modelnames, source_path, executable_path=None, script_path=None): user = '******' linux_user = '******' allowqueuemaster = 1 waitid = 0 parmstring = '' rundataid = 0 priority = 1 reserve_gb = 0 codeHash = 'kamiak' if executable_path in [None, 'None', 'NONE', '']: executable_path = get_setting('DEFAULT_EXEC_PATH') if script_path in [None, 'None', 'NONE', '']: script_path = get_setting('DEFAULT_SCRIPT_PATH') combined = [(c, b, m) for c, b, m in itertools.product(cellids, [batch], modelnames)] notes = ['%s/%s/%s' % (c, b, m) for c, b, m in combined] commandPrompts = [ "%s %s %s %s %s" % (executable_path, script_path, c, b, m) for c, b, m in combined ] engine = nd.Engine() for (c, b, m), note, commandPrompt in zip(combined, notes, commandPrompts): path = os.path.join(source_path, batch, c, m) if not os.path.exists(path): log.warning("missing fit for: \n%s\n%s\n%s\n" "using path: %s\n", batch, c, m, path) continue else: xfspec, ctx = xforms.load_analysis(path, eval_model=False) preview = ctx['modelspec'].meta.get('figurefile', None) if 'log' not in ctx: ctx['log'] = 'missing log' figures_to_load = ctx['figures_to_load'] figures = [xforms.load_resource(f) for f in figures_to_load] ctx['figures'] = figures xforms.save_analysis(None, None, ctx['modelspec'], xfspec, ctx['figures'], ctx['log']) nd.update_results_table(ctx['modelspec'], preview=preview) conn = engine.connect() sql = 'SELECT * FROM tQueue WHERE note="' + note + '"' r = conn.execute(sql) if r.rowcount > 0: # existing job, figure out what to do with it x = r.fetchone() queueid = x['id'] complete = x['complete'] if complete == 1: # Do nothing - the queue already shows a complete job pass elif complete == 2: # Change dead to complete sql = "UPDATE tQueue SET complete=1, killnow=0 WHERE id={}".format( queueid) r = conn.execute(sql) else: # complete in [-1, 0] -- already running or queued # Do nothing pass else: # New job sql = "INSERT INTO tQueue (rundataid,progname,priority," +\ "reserve_gb,parmstring,allowqueuemaster,user," +\ "linux_user,note,waitid,codehash,queuedate,complete) VALUES"+\ " ({},'{}',{}," +\ "{},'{}',{},'{}'," +\ "'{}','{}',{},'{}',NOW(),1)" sql = sql.format(rundataid, commandPrompt, priority, reserve_gb, parmstring, allowqueuemaster, user, linux_user, note, waitid, codeHash) r = conn.execute(sql) conn.close()
def fit_xforms_model(batch, cellid, modelname, save_analysis=False): # parse modelname into loaders, modelspecs, and fit keys load_keywords, model_keywords, fit_keywords = modelname.split("_") # construct the meta data dict meta = { 'batch': batch, 'cellid': cellid, 'modelname': modelname, 'loader': load_keywords, 'fitkey': fit_keywords, 'modelspecname': model_keywords, 'username': '******', 'labgroup': 'lbhb', 'public': 1, 'githash': os.environ.get('CODEHASH', ''), 'recording': load_keywords } xforms_kwargs = {} xforms_init_context = {'cellid': cellid, 'batch': int(batch)} recording_uri = None kw_kwargs = {} xforms_lib = KeywordRegistry(**xforms_kwargs) xforms_lib.register_modules( [default_loaders, default_fitters, default_initializers]) xforms_lib.register_plugins(get_setting('XFORMS_PLUGINS')) keyword_lib = KeywordRegistry() keyword_lib.register_module(default_keywords) keyword_lib.register_plugins(get_setting('KEYWORD_PLUGINS')) # Generate the xfspec, which defines the sequence of events # to run through (like a packaged-up script) xfspec = [] # 0) set up initial context if xforms_init_context is None: xforms_init_context = {} if kw_kwargs is not None: xforms_init_context['kw_kwargs'] = kw_kwargs xforms_init_context['keywordstring'] = model_keywords xforms_init_context['meta'] = meta xfspec.append(['nems.xforms.init_context', xforms_init_context]) # 1) Load the data xfspec.extend(xhelp._parse_kw_string(load_keywords, xforms_lib)) # 2) generate a modelspec xfspec.append( ['nems.xforms.init_from_keywords', { 'registry': keyword_lib }]) # 3) fit the data xfspec.extend(xhelp._parse_kw_string(fit_keywords, xforms_lib)) # Generate a prediction xfspec.append(['nems.xforms.predict', {}]) # 4) add some performance statistics xfspec.append(['nems.xforms.add_summary_statistics', {}]) # 5) plot #xfspec.append(['nems_lbhb.lv_helpers.add_summary_statistics', {}]) # Create a log stream set to the debug level; add it as a root log handler log_stream = io.StringIO() ch = logging.StreamHandler(log_stream) ch.setLevel(logging.DEBUG) fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' formatter = logging.Formatter(fmt) ch.setFormatter(formatter) rootlogger = logging.getLogger() rootlogger.addHandler(ch) ctx = {} for xfa in xfspec: ctx = xforms.evaluate_step(xfa, ctx) # Close the log, remove the handler, and add the 'log' string to context log.info('Done (re-)evaluating xforms.') ch.close() rootlogger.removeFilter(ch) log_xf = log_stream.getvalue() modelspec = ctx['modelspec'] if save_analysis: # save results if get_setting('USE_NEMS_BAPHY_API'): prefix = 'http://' + get_setting( 'NEMS_BAPHY_API_HOST') + ":" + str( get_setting('NEMS_BAPHY_API_PORT')) + '/results/' else: prefix = get_setting('NEMS_RESULTS_DIR') if type(cellid) is list: cell_name = cellid[0].split("-")[0] else: cell_name = cellid destination = os.path.join(prefix, str(batch), cell_name, modelspec.get_longname()) modelspec.meta['modelpath'] = destination modelspec.meta.update(meta) log.info('Saving modelspec(s) to {0} ...'.format(destination)) xforms.save_analysis(destination, recording=ctx['rec'], modelspec=modelspec, xfspec=xfspec, figures=[], log=log_xf) # save performance and some other metadata in database Results table nd.update_results_table(modelspec) return xfspec, ctx
def fit_model_xform(cellid, batch, modelname, autoPlot=True, saveInDB=False, returnModel=False, recording_uri=None, initial_context=None): """ Fit a single NEMS model using data stored in database. First generates an xforms script based on modelname parameter and then evaluates it. :param cellid: cellid and batch specific dataset in database :param batch: :param modelname: string specifying model architecture, preprocessing and fit method :param autoPlot: generate summary plot when complete :param saveInDB: save results to Results table :param returnModel: boolean (default False). If False, return savepath if True return xfspec, ctx tuple :param recording_uri :return: savepath = path to saved results or (xfspec, ctx) tuple """ startime = time.time() log.info('Initializing modelspec(s) for cell/batch %s/%d...', cellid, int(batch)) # Segment modelname for meta information kws = escaped_split(modelname, '_') modelspecname = escaped_join(kws[1:-1], '-') loadkey = kws[0] fitkey = kws[-1] meta = { 'batch': batch, 'cellid': cellid, 'modelname': modelname, 'loader': loadkey, 'fitkey': fitkey, 'modelspecname': modelspecname, 'username': '******', 'labgroup': 'lbhb', 'public': 1, 'githash': os.environ.get('CODEHASH', ''), 'recording': loadkey } if type(cellid) is list: meta['siteid'] = cellid[0][:7] # registry_args = {'cellid': cellid, 'batch': int(batch)} registry_args = {} xforms_init_context = {'cellid': cellid, 'batch': int(batch)} if initial_context is not None: xforms_init_context.update(initial_context) log.info("TODO: simplify generate_xforms_spec parameters") xfspec = generate_xforms_spec(recording_uri=recording_uri, modelname=modelname, meta=meta, xforms_kwargs=registry_args, xforms_init_context=xforms_init_context, autoPlot=autoPlot) log.debug(xfspec) # actually do the loading, preprocessing, fit if initial_context is None: initial_context = {} ctx, log_xf = xforms.evaluate(xfspec) #, context=initial_context) # save some extra metadata modelspec = ctx['modelspec'] if type(cellid) is list: cell_name = cellid[0].split("-")[0] else: cell_name = cellid if 'modelpath' not in modelspec.meta: prefix = get_setting('NEMS_RESULTS_DIR') destination = os.path.join(prefix, str(batch), cell_name, modelspec.get_longname()) log.info(f'Setting modelpath to "{destination}"') modelspec.meta['modelpath'] = destination modelspec.meta['figurefile'] = os.path.join(destination, 'figure.0000.png') else: destination = modelspec.meta['modelpath'] # figure out URI for location to save results (either file or http, depending on USE_NEMS_BAPHY_API) if get_setting('USE_NEMS_BAPHY_API'): prefix = 'http://' + get_setting('NEMS_BAPHY_API_HOST') + ":" + str(get_setting('NEMS_BAPHY_API_PORT')) + \ '/results' save_loc = str( batch) + '/' + cell_name + '/' + modelspec.get_longname() save_destination = prefix + '/' + save_loc # set the modelspec meta save locations to be the filesystem and not baphy modelspec.meta['modelpath'] = get_setting( 'NEMS_RESULTS_DIR') + '/' + save_loc modelspec.meta['figurefile'] = modelspec.meta[ 'modelpath'] + '/' + 'figure.0000.png' else: save_destination = destination modelspec.meta['runtime'] = int(time.time() - startime) modelspec.meta.update(meta) if returnModel: # return fit, skip save! return xfspec, ctx # save results log.info('Saving modelspec(s) to {0} ...'.format(save_destination)) if 'figures' in ctx.keys(): figs = ctx['figures'] else: figs = [] save_data = xforms.save_analysis(save_destination, recording=ctx.get('rec'), modelspec=modelspec, xfspec=xfspec, figures=figs, log=log_xf, update_meta=False) # save in database as well if saveInDB: nd.update_results_table(modelspec) return save_data['savepath']