def evaluate_hypothesis(hypo,sessions,full_output=False,preloaded_curvature=False,debug=False):
"""
Perform a single hypothesis test and return the result to the
callback which will save to the database.
"""
global Field,Hypothesis
#---settings
sn = hypo['sn']
hypo_full = Hypothesis(**hypo)
match = sessions['hypothesis'].query(Hypothesis).filter_by(**hypo_full.base()).one()
#---get the heights and vectors from "memory"
hqs = memory[(sn,'hqs')]
nframes = len(hqs)
#---get the curvature fields
#---! later we will manage the memory externally?
hypo_cf = Field(**hypo)
match_cf = sessions['field'].query(Field).filter_by(**hypo_cf.dict()).one()
fn_cf = namer_cf(match_cf.id)
key_cf = ('curvature',match_cf.id)
if key_cf not in memory and not preloaded_curvature:
memory[key_cf] = load(os.path.basename(fn_cf),rootdir_cf)
curvature_fields = memory[key_cf]['fields']
#---rescale the field to the right curvature
cfs = np.array([hypo['curvature']*c for c in curvature_fields])
#---for the "static" motion, we repeat the same curvature field over all frames
if hypo['motion']=='static':
if not len(cfs)==1:
raise Exception('motion is static but the pre-computed curvature fields have %s'%len(cfs))
cfs = [cfs[0] for fr in range(nframes)]
vecs = memory[(sn,'vecs')]
mn = hqs.shape[1:]
#---prepare the functions
kwargs = dict(cfs=cfs)
if 'fallback' in hypo_full.dict() and hypo_full.dict()['fallback'] != 'none':
kwargs['fallback'] = True
parts = couplecalc(hqs,vecs,**kwargs)
binner = parts['binner']
energy = parts['energy']
q = parts['wavevectors']
#---set the band
#---! this needs added to the database
band = filter_wavevectors(q,low=0.0,high=tweak.get('hicut',1.0))
#---send to the optimizer
init_cond = tweak.get('init',[20,0.0,10.0])
result = optimize_hypothesis(energy,residual,band,init_cond)
#---! on porting to the freshest omnicalc ca 2017.6.6 we find kappa not being saved
#---! ...the following fix makes this whole function read kind of bad
parts.update(**dict([(k,v) for k,v in result.items() if k in 'kappa gamma vibe error'.split()]))
#---we save the parts for later using the record numbers
parts['best_energy'] = np.array(energy(result['kappa'],result['gamma'],result['vibe']))
result = dict([(key,val) for key,val in parts.items() if key not in ['binner','energy']])
store(obj=result,name=os.path.basename(namer_cc(match.id)),path=rootdir_cc,
attrs=hypo_full.base(),verbose=False)
#---update the database
for key,val in result.items(): match.__setattr__(key,val)
sessions['hypothesis'].commit()
if debug:
import ipdb;ipdb.set_trace()
def manual_populate_fields(self):
"""
"""
cctools.data = self.data
#---compute pending fields according to populated rows
fns = [(i.id,self.namer_cf(i.id)) for i in self.sessions['field'].query(self.Field).all()]
pending = [(pk,fn) for pk,fn in fns if not os.path.isfile(fn)]
if pending:
#---loop over absent files
start = time.time()
for ii,(pk,fn) in enumerate(pending):
status('computing curvature field',tag='compute',i=ii,looplen=len(pending),start=start)
hypo = self.sessions['field'].query(self.Field).filter_by(id=pk).one().dict()
sn = hypo['sn']
dat = self.data['undulations'][sn]['data']
vecs = dat['vecs']
mn = np.shape(dat['mesh'])[2:]
fields = construct_curvature_fields_trajectory(vecs=vecs,mn=mn,**hypo)
store({'fields':np.array(fields['fields'])},os.path.basename(fn),self.rootdir_cf,
attrs={key:val for key,val in fields.items()+hypo.items()
if key!='fields'},verbose=False)
def wilderness(self,**kwargs):
"""
Wandering on the error landscape to optimize the curvature coupling hypothesis.
"""
bundle,solutions = {},{}
global Nfeval
spec = self.spec
extents = spec.get('extents',{})
extents_method = extents.get('method')
curvature_sum_method = self.spec['curvature_sum']
#---prepare curvature fields
if not extents_method: raise Exception('set extents method')
elif extents_method=='fixed_isotropic': self.drop_gaussians(**spec)
#---if the output file exists then the optimization is done and we just keep the curvature fields
out_fn_name = 'curvature_%s.dat'%self.signifier
if os.path.isfile(os.path.join(work.postdir,out_fn_name)): return
#---optimize over simulations
for snum,sn in enumerate(work.sns()):
status('starting optimization for %s %d/%d'%(sn,snum+1,len(work.sns())),tag='optimize')
#---load the source data
hqs = self.memory[(sn,'hqs')][:self.nframes]
cfs = self.memory[(sn,'fields_unity')][:self.nframes]
vecs = self.memory[(sn,'vecs')][:self.nframes]
ndrops = cfs.shape[1]
#---formulate the wavevectors
lenscale = 1.0
m,n = mn = np.shape(hqs)[1:]
Lx,Ly = np.mean(vecs,axis=0)[:2]
q2d = lenscale*np.array([[np.sqrt(
((i-m*(i>m/2))/((Lx)/1.)*2*np.pi)**2+
((j-n*(j>n/2))/((Ly)/1.)*2*np.pi)**2)
for j in range(0,n)] for i in range(0,m)])
q_raw = np.reshape(q2d,-1)[1:]
area = (Lx*Ly/lenscale**2)
tweak = {}
#---! remove this eventually
signterm = tweak.get('inner_sign',-1.0)
lowcut = kwargs.get('lowcut',tweak.get('lowcut',0.0))
band = cctools.filter_wavevectors(q_raw,low=lowcut,high=tweak.get('hicut',1.0))
residual_form = kwargs.get('residual_form',tweak.get('residual_form','log'))
if residual_form == 'log':
def residual(values):
return np.sum(np.log10(values.clip(min=machine_eps))**2)/float(len(values))
elif residual_form == 'linear':
def residual(values):
return np.sum((values-1.0)**2)/float(len(values))
else: raise Exception('unclear residual form %s'%residual_form)
def multipliers(x,y):
"""Multiplying complex matrices in the list of terms that contribute to the energy."""
return x*np.conjugate(y)
def callback(args):
"""Watch the optimization."""
global Nfeval
name_groups = ['kappa','gamma','vibe']+['curve(%d)'%i for i in range(ndrops)]
text = ' step = %d '%Nfeval+' '.join([name+' = '+dotplace(val)
for name,val in zip(name_groups,args)+[('error',objective(args))]])
status('searching! '+text,tag='optimize')
Nfeval += 1
def objective(args):
"""
Fit parameters are defined in sequence for the optimizer.
They are: kappa,gamma,vibe,*curvatures-per-dimple.
"""
kappa,gamma,vibe = args[:3]
curvatures = args[3:]
composite = self.curvature_sum(cfs,curvatures,method=curvature_sum_method)
cqs = cctools.fft_field(composite)
termlist = [multipliers(x,y) for x,y in [(hqs,hqs),(hqs,cqs),(cqs,hqs),(cqs,cqs)]]
termlist = [np.reshape(np.mean(k,axis=0),-1)[1:] for k in termlist]
#---skipping assertion and dropping imaginary
termlist = [np.real(k) for k in termlist]
hel = (kappa*area*(termlist[0]*q_raw**4+signterm*termlist[1]*q_raw**2
+signterm*termlist[2]*q_raw**2+termlist[3])
+gamma*area*(termlist[0]*q_raw**2))
ratio = hel/((vibe*q_raw+machine_eps)/(np.exp(vibe*q_raw)-1)+machine_eps)
return residual(ratio[band])
Nfeval = 0
initial_conditions = [25.0*2,0.0,-0.1]+[0.0 for i in range(ndrops)]
fit = scipy.optimize.minimize(objective,
x0=tuple(initial_conditions),method='SLSQP',callback=callback)
#---package the result
bundle[sn] = dict(fit,success=str(fit.success))
solutions['%s_%s'%(sn,'x')] = bundle[sn].pop('x')
solutions['%s_%s'%(sn,'jac')] = bundle[sn].pop('jac')
solutions['%s_%s'%(sn,'cf')] = np.array(self.curvature_sum(cfs,fit.x[3:],
method=curvature_sum_method).mean(axis=0))
try: store(obj=solutions,name=out_fn_name,path=work.postdir,attrs=dict(bundle=bundle,spec=self.spec))
except:
import ipdb;ipdb.set_trace()
i=ii,
looplen=len(pending),
start=start)
hypo = sessions['field'].query(Field).filter_by(id=pk).one().dict()
sn = hypo['sn']
dat = data['undulations'][sn]['data']
vecs = dat['vecs']
mn = np.shape(dat['mesh'])[2:]
fields = construct_curvature_fields_trajectory(vecs=vecs,
mn=mn,
**hypo)
store({'fields': np.array(fields['fields'])},
os.path.basename(fn),
rootdir_cf,
attrs={
key: val
for key, val in fields.items() + hypo.items()
if key != 'fields'
},
verbose=False)
#---solve the hypotheses
#---for memory efficiency we queue up hypotheses according to which curvature field they require
#---note that we had a simpler, memory-hogging loop in a previous iteration of this code
fns = [(i.id, namer_cc(i.id))
for i in sessions['hypothesis'].query(Hypothesis).all()]
pending = [(pk, fn) for pk, fn in fns if not os.path.isfile(fn)]
if pending and not skip_calculation:
hypotheses = [
sessions['hypothesis'].query(Hypothesis).filter_by(id=pk).one().dict()
for pk in zip(*pending)[0]