def test_split_and_join_escaped():
s = r'split_me_on\_underscores'
assert escaped_join(escaped_split(s, '_'), '_') == s
s = r'split_me_on_\_underscores'
assert escaped_join(escaped_split(s, '_'), '_') == s
s = r'split_me_on__\underscores'
assert escaped_join(escaped_split(s, '_'), '_') == s
def test_join_escaped_delimeter4():
splits = ['join', 'on-', '\\-me', 'test']
delims = ['.', '_', '-']
for d in delims:
join = escaped_join(splits, d)
assert join == d.join(splits)
def test_join_normal_delimeter():
splits = ['join', 'on', 'me', 'test']
delims = ['.', '_', '-']
for d in delims:
join = escaped_join(splits, d)
assert join == d.join(splits)
def test_split_and_join():
s = 'my-escaped_modelspec-name_blah.t5\.5'
x = escaped_split(s, '_')
# escaping the . shouldn't matter here, should only escape
# the active delimiter
assert len(x) == 3
blah = x[2]
# now the . should be ignored during the split, for a length
# of 2 instead of 3
assert len(escaped_split(blah, '.')) == 2
# after the reverse operation, original string shouldl be preserved
y = escaped_join(x, '_')
assert y == s
def test_split_and_join_normal():
s = 'split_me_on_underscores'
assert escaped_join(escaped_split(s, '_'), '_') == s
def fit_model_xform(cellid,
batch,
modelname,
autoPlot=True,
saveInDB=False,
returnModel=False,
recording_uri=None,
initial_context=None):
"""
Fit a single NEMS model using data stored in database. First generates an xforms
script based on modelname parameter and then evaluates it.
:param cellid: cellid and batch specific dataset in database
:param batch:
:param modelname: string specifying model architecture, preprocessing
and fit method
:param autoPlot: generate summary plot when complete
:param saveInDB: save results to Results table
:param returnModel: boolean (default False). If False, return savepath
if True return xfspec, ctx tuple
:param recording_uri
:return: savepath = path to saved results or (xfspec, ctx) tuple
"""
startime = time.time()
log.info('Initializing modelspec(s) for cell/batch %s/%d...', cellid,
int(batch))
# Segment modelname for meta information
kws = escaped_split(modelname, '_')
modelspecname = escaped_join(kws[1:-1], '-')
loadkey = kws[0]
fitkey = kws[-1]
meta = {
'batch': batch,
'cellid': cellid,
'modelname': modelname,
'loader': loadkey,
'fitkey': fitkey,
'modelspecname': modelspecname,
'username': '******',
'labgroup': 'lbhb',
'public': 1,
'githash': os.environ.get('CODEHASH', ''),
'recording': loadkey
}
if type(cellid) is list:
meta['siteid'] = cellid[0][:7]
# registry_args = {'cellid': cellid, 'batch': int(batch)}
registry_args = {}
xforms_init_context = {'cellid': cellid, 'batch': int(batch)}
if initial_context is not None:
xforms_init_context.update(initial_context)
log.info("TODO: simplify generate_xforms_spec parameters")
xfspec = generate_xforms_spec(recording_uri=recording_uri,
modelname=modelname,
meta=meta,
xforms_kwargs=registry_args,
xforms_init_context=xforms_init_context,
autoPlot=autoPlot)
log.debug(xfspec)
# actually do the loading, preprocessing, fit
if initial_context is None:
initial_context = {}
ctx, log_xf = xforms.evaluate(xfspec) #, context=initial_context)
# save some extra metadata
modelspec = ctx['modelspec']
if type(cellid) is list:
cell_name = cellid[0].split("-")[0]
else:
cell_name = cellid
if 'modelpath' not in modelspec.meta:
prefix = get_setting('NEMS_RESULTS_DIR')
destination = os.path.join(prefix, str(batch), cell_name,
modelspec.get_longname())
log.info(f'Setting modelpath to "{destination}"')
modelspec.meta['modelpath'] = destination
modelspec.meta['figurefile'] = os.path.join(destination,
'figure.0000.png')
else:
destination = modelspec.meta['modelpath']
# figure out URI for location to save results (either file or http, depending on USE_NEMS_BAPHY_API)
if get_setting('USE_NEMS_BAPHY_API'):
prefix = 'http://' + get_setting('NEMS_BAPHY_API_HOST') + ":" + str(get_setting('NEMS_BAPHY_API_PORT')) + \
'/results'
save_loc = str(
batch) + '/' + cell_name + '/' + modelspec.get_longname()
save_destination = prefix + '/' + save_loc
# set the modelspec meta save locations to be the filesystem and not baphy
modelspec.meta['modelpath'] = get_setting(
'NEMS_RESULTS_DIR') + '/' + save_loc
modelspec.meta['figurefile'] = modelspec.meta[
'modelpath'] + '/' + 'figure.0000.png'
else:
save_destination = destination
modelspec.meta['runtime'] = int(time.time() - startime)
modelspec.meta.update(meta)
if returnModel:
# return fit, skip save!
return xfspec, ctx
# save results
log.info('Saving modelspec(s) to {0} ...'.format(save_destination))
if 'figures' in ctx.keys():
figs = ctx['figures']
else:
figs = []
save_data = xforms.save_analysis(save_destination,
recording=ctx.get('rec'),
modelspec=modelspec,
xfspec=xfspec,
figures=figs,
log=log_xf,
update_meta=False)
# save in database as well
if saveInDB:
nd.update_results_table(modelspec)
return save_data['savepath']
#lr = 0.01, mi=2500, eval_interval = 75 --- fit=0.686, test=0.684
modelname = "ozgf.fs100.ch18-ld-norm-sev_dlog-wc.18x16-fir.4x18x4-relu.4-wc.4x1_tf.n.rb10"
autoPlot = True
saveInDB = False
save_data = False
browse_results = False
log.info('Initializing modelspec(s) for cell/batch %s/%d...',
cellid, int(batch))
# Segment modelname for meta information, specifically the parts between _s
kws = escaped_split(modelname, '_')
modelspecname = escaped_join(kws[1:-1], '-')
loadkey = kws[0] #name of loader, samplying freq,channels
fitkey = kws[-1] #what the fitter will be (in my case now NEMS or tf
meta = {'batch': batch, 'cellid': cellid, 'modelname': modelname,
'loader': loadkey, 'fitkey': fitkey, 'modelspecname': modelspecname,
'username': '******', 'labgroup': 'lbhb', 'public': 1,
'githash': os.environ.get('CODEHASH', ''),
'recording': loadkey}
if type(cellid) is list:
meta['siteid'] = cellid[0][:7]
# registry_args = {'cellid': cellid, 'batch': int(batch)}
registry_args = {}
xforms_init_context = {'cellid': cellid, 'batch': int(batch)}