def test_split_and_join_escaped(): s = r'split_me_on\_underscores' assert escaped_join(escaped_split(s, '_'), '_') == s s = r'split_me_on_\_underscores' assert escaped_join(escaped_split(s, '_'), '_') == s s = r'split_me_on__\underscores' assert escaped_join(escaped_split(s, '_'), '_') == s
def test_join_escaped_delimeter4(): splits = ['join', 'on-', '\\-me', 'test'] delims = ['.', '_', '-'] for d in delims: join = escaped_join(splits, d) assert join == d.join(splits)
def test_join_normal_delimeter(): splits = ['join', 'on', 'me', 'test'] delims = ['.', '_', '-'] for d in delims: join = escaped_join(splits, d) assert join == d.join(splits)
def test_split_and_join(): s = 'my-escaped_modelspec-name_blah.t5\.5' x = escaped_split(s, '_') # escaping the . shouldn't matter here, should only escape # the active delimiter assert len(x) == 3 blah = x[2] # now the . should be ignored during the split, for a length # of 2 instead of 3 assert len(escaped_split(blah, '.')) == 2 # after the reverse operation, original string shouldl be preserved y = escaped_join(x, '_') assert y == s
def test_split_and_join_normal(): s = 'split_me_on_underscores' assert escaped_join(escaped_split(s, '_'), '_') == s
def fit_model_xform(cellid, batch, modelname, autoPlot=True, saveInDB=False, returnModel=False, recording_uri=None, initial_context=None): """ Fit a single NEMS model using data stored in database. First generates an xforms script based on modelname parameter and then evaluates it. :param cellid: cellid and batch specific dataset in database :param batch: :param modelname: string specifying model architecture, preprocessing and fit method :param autoPlot: generate summary plot when complete :param saveInDB: save results to Results table :param returnModel: boolean (default False). If False, return savepath if True return xfspec, ctx tuple :param recording_uri :return: savepath = path to saved results or (xfspec, ctx) tuple """ startime = time.time() log.info('Initializing modelspec(s) for cell/batch %s/%d...', cellid, int(batch)) # Segment modelname for meta information kws = escaped_split(modelname, '_') modelspecname = escaped_join(kws[1:-1], '-') loadkey = kws[0] fitkey = kws[-1] meta = { 'batch': batch, 'cellid': cellid, 'modelname': modelname, 'loader': loadkey, 'fitkey': fitkey, 'modelspecname': modelspecname, 'username': '******', 'labgroup': 'lbhb', 'public': 1, 'githash': os.environ.get('CODEHASH', ''), 'recording': loadkey } if type(cellid) is list: meta['siteid'] = cellid[0][:7] # registry_args = {'cellid': cellid, 'batch': int(batch)} registry_args = {} xforms_init_context = {'cellid': cellid, 'batch': int(batch)} if initial_context is not None: xforms_init_context.update(initial_context) log.info("TODO: simplify generate_xforms_spec parameters") xfspec = generate_xforms_spec(recording_uri=recording_uri, modelname=modelname, meta=meta, xforms_kwargs=registry_args, xforms_init_context=xforms_init_context, autoPlot=autoPlot) log.debug(xfspec) # actually do the loading, preprocessing, fit if initial_context is None: initial_context = {} ctx, log_xf = xforms.evaluate(xfspec) #, context=initial_context) # save some extra metadata modelspec = ctx['modelspec'] if type(cellid) is list: cell_name = cellid[0].split("-")[0] else: cell_name = cellid if 'modelpath' not in modelspec.meta: prefix = get_setting('NEMS_RESULTS_DIR') destination = os.path.join(prefix, str(batch), cell_name, modelspec.get_longname()) log.info(f'Setting modelpath to "{destination}"') modelspec.meta['modelpath'] = destination modelspec.meta['figurefile'] = os.path.join(destination, 'figure.0000.png') else: destination = modelspec.meta['modelpath'] # figure out URI for location to save results (either file or http, depending on USE_NEMS_BAPHY_API) if get_setting('USE_NEMS_BAPHY_API'): prefix = 'http://' + get_setting('NEMS_BAPHY_API_HOST') + ":" + str(get_setting('NEMS_BAPHY_API_PORT')) + \ '/results' save_loc = str( batch) + '/' + cell_name + '/' + modelspec.get_longname() save_destination = prefix + '/' + save_loc # set the modelspec meta save locations to be the filesystem and not baphy modelspec.meta['modelpath'] = get_setting( 'NEMS_RESULTS_DIR') + '/' + save_loc modelspec.meta['figurefile'] = modelspec.meta[ 'modelpath'] + '/' + 'figure.0000.png' else: save_destination = destination modelspec.meta['runtime'] = int(time.time() - startime) modelspec.meta.update(meta) if returnModel: # return fit, skip save! return xfspec, ctx # save results log.info('Saving modelspec(s) to {0} ...'.format(save_destination)) if 'figures' in ctx.keys(): figs = ctx['figures'] else: figs = [] save_data = xforms.save_analysis(save_destination, recording=ctx.get('rec'), modelspec=modelspec, xfspec=xfspec, figures=figs, log=log_xf, update_meta=False) # save in database as well if saveInDB: nd.update_results_table(modelspec) return save_data['savepath']
#lr = 0.01, mi=2500, eval_interval = 75 --- fit=0.686, test=0.684 modelname = "ozgf.fs100.ch18-ld-norm-sev_dlog-wc.18x16-fir.4x18x4-relu.4-wc.4x1_tf.n.rb10" autoPlot = True saveInDB = False save_data = False browse_results = False log.info('Initializing modelspec(s) for cell/batch %s/%d...', cellid, int(batch)) # Segment modelname for meta information, specifically the parts between _s kws = escaped_split(modelname, '_') modelspecname = escaped_join(kws[1:-1], '-') loadkey = kws[0] #name of loader, samplying freq,channels fitkey = kws[-1] #what the fitter will be (in my case now NEMS or tf meta = {'batch': batch, 'cellid': cellid, 'modelname': modelname, 'loader': loadkey, 'fitkey': fitkey, 'modelspecname': modelspecname, 'username': '******', 'labgroup': 'lbhb', 'public': 1, 'githash': os.environ.get('CODEHASH', ''), 'recording': loadkey} if type(cellid) is list: meta['siteid'] = cellid[0][:7] # registry_args = {'cellid': cellid, 'batch': int(batch)} registry_args = {} xforms_init_context = {'cellid': cellid, 'batch': int(batch)}