def plot_data_repeat(dlist,name="",num=None,*args,**kwargs):
""""given a list of Data2D objects dlist, plots them as subplots in maximized window.
returns stats.
"""
res=[]
#fig,axes=plt.subplots(1,len(dlist),num=1)
xs,ys=find_grid_size(len(dlist),3,square=False)
fig,axes=plt.subplots(xs,ys,num=num,clear=True)
plt.close() #prevent from showing inline in notebook with %matplotlib inline
axes=axes.flatten()
maximize()
for i,(ll,ax) in enumerate(zip(dlist,axes)):
plt.subplot(xs,ys,i+1,sharex=axes[0],sharey=axes[0])
ll.plot(stats=True,*args,**kwargs)
res.append(ll.std())
#plt.clim([-3,3])
plt.clim(*(np.nanmean(ll.data)+np.nanstd(ll.data)*np.array([-1,1])))
plt.suptitle(name+' RAW (plane level)')
for ax in axes[:len(dlist)-1:-1]:
fig.delaxes(ax)
#plt.pause(0.1)
commonscale(plt.gcf())
return res
def dcouples_plot(dlist,level=True,dis=False):
"""calculate rotating differences, data are supposed to be already aligned.
Note, differences are not leveled.
if dis is set to True, call display after plots (not found a better way of doing this)."""
dcouples=[c[1]-c[0] for c in list(itertools.combinations(dlist, 2))]
if level:
dcouples=[d.level() for d in dcouples]
plt.clf()
maximize()
xs,ys=find_grid_size(len(dcouples),square=True)[::-1]
fig,axes=plt.subplots(xs,ys)
plt.close() #prevent from showing inline in notebook with %matplotlib inline
if len(np.shape(axes))>1:
axes=axes.flatten()
elif len(np.shape(axes))==0:
axes=[axes]
maximize()
for i,(ll,ax) in enumerate(zip(dcouples,axes)):
plt.subplot(xs,ys,i+1,sharex=axes[0],sharey=axes[0])
ll.plot()
plt.clim(*(ll.std()*np.array([-1,1])))
for ax in axes[:len(dcouples)-1:-1]:
fig.delaxes(ax)
#plt.pause(0.1)
#plt.tight_layout()
#return [d.std() for d in [diff21,diff31,diff32]]
return dcouples
def process_set2(flist,names,outname,crop=None,dis=True):
# from C1S15_4D_DI_calibration_181031.ipynb
# called with:
ypix=None #0.127901442999 #mm
ytox=220./200.
zscale=None #0.6328
scale=(1000,-1000,0.001)
outfolder=os.path.dirname(outname)
outname=os.path.basename(outname)
dlist=[Data2D(*matrix4D_reader(wf2,scale=scale,zscale=zscale,
ypix=ypix,ytox=ytox,center=(0,0),strip=True),name=n,units=['mm','mm','um'])
for wf2,n in zip(flist,names)]
np.mean(dlist).save(os.path.join(outfolder,outname+'_avg.dat'))
plt.close('all')
sx,sy=find_grid_size(len(dlist),3)
f,axes=plt.subplots(sx,sy,sharex=True,sharey=True)
for ax,d in zip(axes.flatten(),dlist):
plt.sca(ax)
d.plot()
plt.clim(*remove_outliers(d.data,nsigma=2,itmax=3,span=1))
for ax in axes.flatten()[:len(dlist)-1:-1]:
f.delaxes(ax)
maximize()
plt.pause(0.1)
plt.tight_layout(rect=[0,0,1,0.9])
plt.suptitle ('Surfaces')
if dis:
display(plt.gcf())
plt.savefig(os.path.join(outfolder,outname+'.png'))
m_repr=[dcouples_plot([dd for dd in dlist])]
if dis:
display(plt.gcf())
plt.savefig(os.path.join(outfolder,'diff_'+outname+'.png'))
if crop is not None:
dc=[d.crop(*crop).level() for d in dlist]
m_repr=[dcouples_plot([dd for dd in dc])]
if dis:
display(plt.gcf())
plt.savefig(os.path.join(outfolder,'diff_'+outname+'_crop.png'))
return dlist
def process_set(flist,m_trans=None,m_arr=None,outfolder=None):
rmslist=[]
#plot surfaces, dlist is list of Data2D objects with raw data
dlist=plot_repeat(flist,
None if outfolder is None else os.path.join(outfolder,k+'.png'),dis=dis)
rms=[d.level(1,1).std() for d in dlist]
if m_trans is None:
tmp2=dlist
else:
tmp2=[d.apply_transform(t) for d,t in zip(dlist,m_trans)]
if m_arr is not None:
#plot transformed data for verification
fig,axes=plt.subplots(xs,ys,sharex='all',sharey='all',squeeze=True)
axes=axes.flatten()
maximize()
for d,ax,m,t in zip(tmp2,axes,m_arr,m_trans):
plt.sca(ax)
ll=d.level(4,byline=True)
ll.plot()
plt.clim(*remove_outliers(ll.data,nsigma=2,itmax=1,span=True))
plt.plot(*((t(m).T)),'xr')
if outfolder is not None:
plt.savefig(None if outfolder is None else os.path.join(outfolder,k+'_alignment.png'))
#plot differences with various levelings
difflist = plot_rep_diff(tmp2,None if outfolder is None else os.path.join(outfolder,k+'.png'),dis=dis)
rmslist.append([[dd.std() for dd in d] for d in difflist])
if columns is None:
columns = pd.RangeIndex(0,np.size(rmslist))
if stats is None: stats = pd.DataFrame(columns=columns)
pstats = pd.DataFrame(columns=stats.columns,data=np.array(rmslist).reshape(1,-1),index=[k])
return pstats
def getdataset(datalist,outfolder=None,fignum=None,figsize=(9.6,6.7),levelingfunc=None,
caldata=None,psdrange=[1e-15,1e-6]):
"""for each file in a list read data and make a preview plot using `plot_surface_analysis`,
return a list of (data,x,y). caldata are optional calibration data of same shape as data."""
"""moved here from newview_plotter. This is a similar function to calibrate samples, """
if levelingfunc==None:
levelingfunc= lambda x: x
result=[]
if outfolder is not None:
os.makedirs(outfolder,exist_ok=True)
fig=fignumber(fignum)
for ff in datalist:
plt.clf()
wdata,x,y=matrixZygo_reader(ff,scale=(1000.,1000,1.),center=(0,0))
if caldata is not None: wdata=wdata-caldata
wdata,x,y=levelingfunc(wdata),x,y #simple way to remove plane
plot_surface_analysis(wdata,x,y,label=os.path.basename(ff),outfolder=outfolder,nsigma_crange=1,
fignum=(fignum+1 if fignum else None),psdrange=psdrange,levelingfunc=levelingfunc,frange=[4,100]) #[1e-11,1e-6]
maximize()
result.append((wdata,x,y))
return result
def psd2d_analysis(wdata,x,y,title=None,wfun=None,vrange=None,
rmsrange=None,prange=None,fignum=5,rmsthr=None,
aspect='auto', ax2f=None, units=None,outname=None,norm=1,rmsnorm=True):
""" Calculates 2D PSD as image obtained combining all profile PSDS calculated along vertical slices of data. Resulting image has size
If title is provided rms slice power is also calculated and plotted on three panels with figure and PSD.
Return PSD as PSD2D object.
uses plot_rms_power(f,p,rmsrange=None) to calculate rms power.
fignum window where to plot, if fignnum is 0 current figure is cleared,
if None new figure is created. Default to figure 5.
rmsrange is one or a list of frequency ranges for plotting integrated rms. Can contain None to use max or min.
If axf2 is set to boolean or array of boolean, plot slice rms for the frequencies
associated to rmsrange on second axis. rmsrange must be set accordingly (same number of elements).
rmsthr sets a threshold for data inclusion. If rms is above the value, the line is considered to contain invalid data and is removed from output PSD.
This makes it easy to average the returned PSDs.
Corresponding data are still visualized in central panel, but are marked with a red cross at top of y axes.
If multiple rms range intervals are provided, line is removed if any of them is over the threshold, but this might change in future, e.g. TODO: make it possible to add vector threshold with as many elements as rms range intervals.
Set outname to empty string to plot without generating output.
typical values:
rmsrange=[[None,0.1],[0.1,0.2],[0.2,1],[1,None]] #list of frequency intervals for rms
ax2f=[0,1,1,0,0] # axis where to plot rms corresponding to intervals in rmsrange: 1 right, 0 left
wfun = np.hanning # type of windows for fourier transform
units = ("mm","mm","$\mu$m") # units of surface data from which PSD is calculated
vrange_surf=([-0.5,0.5]) #color scale of surface map
vrange_leg=([-0.05,0.05]) #color scale of legendre removed map
prange=np.array((1e-8,1.e-1))#np.array((5e-8,1.e-5)) #color scale of 2d psd plot
fs,ps=psd2d_analysis(levellegendre(y,wdata,2),x,y,outname="",wfun=np.hanning,
rmsrange=rmsrange,prange=prange,ax2f=ax2f,units=units)
"""
#2018/04/05
# -critical renaming of plot_psd2d(wdata,x,y..) -> psd2d_analysis
# and _plot_psd2d(f,p,x) -> plot_psd2d consistently with other routines.
# -added title to determine if generating plot, with the intent of replacing outname.
plt.pause(1)
#print('cane',fignum)
if outname is not None: #handle backcompatibility
print ('psd2d_analysis WARNING: `title` replaced `outname` and output figure will be no more generated.'+
'OUTNAME will be removed in next version, use title and save the plot after returning from routine.')
if title is not None:
print('outname should be not used together with title, it will be ignored.')
else:
title=outname
if vrange is not None:
if prange is None:
print("""WARNING: older versions of psd2d use vrange as range of psdvalue.
In updated version prange is range of psd and vrange is range for surface
plots. Update your code.""")
#check compatibility with old interface
if len(wdata.shape)==1:
if len(y.shape)==2:
print("WARNING: your code calling psd2d uses old call")
print("psd2d(x,y,data), automaticly adjusted, but")
print("update IMMEDIATELY your code to call")
print("psd2d(data,x,y)")
x,y,wdata=wdata,x,y
f, p = psd2d(wdata, x, y, wfun=wfun, norm=norm, rmsnorm=rmsnorm) # calculate PSD 2D
# GENERATE OUTPUT
if title is not None:
if np.size(units)==1:
units=np.repeat(units,3)
if prange is None:
if f[0] == 0:
prange=span(p[1:,:])
else:
prange=span(p)
if prange[0]<1e-20: #arbitrary small value
print("WARNING: low limit detected in prange, can cause problems with log color scale.")
if vrange is None:
vrange=span(wdata)
#plot surface map, who knows why on figure 5
fig=fignumber(fignum)
#pdb.set_trace()
#
plt.clf()
maximize()
plt.draw()
plt.suptitle(title)
################
#plot Surface
ax1=plt.subplot(311)
plot_data(wdata,x,y,aspect=aspect,vmin=vrange[0],vmax=vrange[1],units=units,title='Surface',stats=True)
plt.xlabel(None)
plt.grid(1)
################
#plot 2D PSD
ax2=plt.subplot(312,sharex=ax1)
plot_psd2d(f,p,x,prange=prange,units=units)
if rmsrange is not None: #plot horizontal lines to mark frequency ranges for rms extraction,
#rms is tranformed to 2D array
if len(np.array(rmsrange).shape)==1: #if only one interval, make it 2D anyway
rmsrange=[rmsrange]
rmsrange=np.array(rmsrange)
for fr in np.array(rmsrange): #I want fr to be array type, even if I don't care about rmsrange this does the job
ax2.hlines(fr[fr != None],*((span(x)-x.mean())*1.1+x.mean()))
ax2.set_xlabel("")
plt.xlabel(None)
#plt.subplots_adjust(top=0.85)
################
#plot rms slice
ax3=plt.subplot(313,sharex=ax1)
if f[0]==0: #ignore component of 0 frequency in rms calculation if present
ff,pp=f[1:],p[1:]
else:
ff,pp=f,p
rms=plot_rms_power(ff,pp,x,rmsrange=rmsrange,ax2f=ax2f,units=units)
#pdb.set_trace()
mask=np.isfinite(rms) #True if good. it is an array
if rmsthr is not None:
mask = mask & (rms < rmsthr)
ax3.hlines(rmsthr,*ax2.get_xlim()) #plot markers on rms chart
ax2.plot(x[~mask],np.repeat(ax2.get_ylim()[1],len(x[~mask])),'rx') #plot markers on psd2d chart
#pdb.set_trace()
#questa era qui, ma dava errore perche' mask e' lineare
#mask = np.all(mask,axis =0) # if any of mask is False -> False
p[:,~mask]=np.nan
ax3.grid(1)
#plt.tight_layout(rect=(0, 0.03, 1, 0.95) if title else (0, 0, 1, 1))
# replaced with more evolved interactive resize:
# plt.colorbar().remove() #dirty trick to adjust size to other panels
def resize(event):
box1 = ax1.get_position()
box2 = ax2.get_position()
box3 = ax3.get_position()
ax2.set_position([box1.x0, box2.y0, box1.width , box2.height])
ax2.set_adjustable("box",share=True)
ax3.set_position([box1.x0, box3.y0, box1.width , box3.height])
ax3.set_adjustable("box",share=True)
cid = fig.canvas.mpl_connect('draw_event', resize)
cid2 = fig.canvas.mpl_connect('resize_event', resize)
resize(None)
#plt.tight_layout()
if outname: #kept for compatibility, will be removed in next version
plt.savefig(fn_add_subfix(outname,'_2dpsd','.png'))
return f,p
def conf_stats(k,conf,columns=None,dis=True):
"""Analyzes a single configuration (as list of files + processing settings).
returns repeatability statistics in form of pdDataFrame. """
rmslist=[]
#outfile=os.path.join(outfolder,k)
flist=[os.path.join(conf['infolder'],f)for f in conf['files']]
transmode=conf.get('trans',None)
#plot surfaces, dlist is list of Data2D objects with raw data
dlist=plot_repeat(flist,
None if outfolder is None else os.path.join(outfolder,k+'.png'),dis=dis)
rms=[d.level(1,1).std() for d in dlist]
tmp2=[d if t is None else t(d) for d,t in zip(dlist,m_trans)]
#align if necessary:
if transmode=='none' or transmode == "": #do nothing
tmp2=dlist
else:
plt.close('all')
if transmode == 'interactive':
print ("""interactively set markers on each axis with CTRL+LeftMouse.
Remove markers with CTRL+right_mounse, ENTER when ready.""")
#run interactive alignment
m_arr,m_trans=align_interactive(dlist,find_transform=find_affine)
#incorporate markers in config and save backup copy for the configuration.
# This is just temporary ass saver in case of crash, final conf is anyway
# saved outside for all configurations.
conf['trans']='config' # alignment mode, read from config
conf['mref']=m_arr[0].tolist() #reference markers to aligh to
conf['markers']=[m.tolist() for m in m_arr] #markers saved as lists
json.dump({k:conf},open(os.path.join(outfolder,k+'_out.json'),'w'),indent=0)
else:
if transmode == 'config': #gets markers from configfile
pass #conf is already set
elif transmode == 'file':
d=json.load(open(conf['markers'],'r'))
print(list(d.keys()),'\n\n',d[list(d.keys())[0]])
print(len(list(d.keys()))," datasets")
k,conf=list(d.items())[0]
else:
raise ValueError
m_arr=[np.array(m) for m in conf['markers']]
mref=conf['mref']
m_trans=[find_transform(m,mref) for m in m_arr]
#make and plots transformed (aligned) data to verify feature alignment
tmp2=[d.apply_transform(t) for d,t in zip(dlist,m_trans)]
#plot transformed data for verification
fig,axes=plt.subplots(xs,ys,sharex='all',sharey='all',squeeze=True)
axes=axes.flatten()
maximize()
for d,ax,m,t in zip(tmp2,axes,m_arr,m_trans):
plt.sca(ax)
ll=d.level(4,byline=True)
ll.plot()
plt.clim(*remove_outliers(ll.data,nsigma=2,itmax=1,span=True))
plt.plot(*((t(m).T)),'xr')
plt.savefig(None if outfolder is None else os.path.join(outfolder,k+'_alignment.png'))
#yield dlist
#plot differences with various levelings
difflist = plot_rep_diff(tmp2,None if outfolder is None else os.path.join(outfolder,k+'.png'),dis=dis)
rmslist.append([[dd.std() for dd in d] for d in difflist])
if columns is None:
tags=['%02i-%02i'%(i,j) for i,j in itertools.combinations(range(1,nfiles+1),2)]
columns=pd.MultiIndex.from_product([['raw','cyl','con','leg'],tags],names=['terms removed','files#'])
#columns = pd.RangeIndex(0,np.size(rmslist))
#pdb.set_trace()
cstats = pd.DataFrame(columns=columns,data=np.array(rmslist).reshape(1,-1),index=[k])
return cstats
def rep_stats(flist,columns=None,dis=True):
"""Analyzes a list of files with possible transformations.
returns repeatability statistics in form of pdDataFrame. """
rmslist=[]
#plot surfaces, dlist is list of Data2D objects with raw data
dlist=plot_repeat(flist,
None if outfolder is None else os.path.join(outfolder,k+'.png'),dis=dis)
rms=[d.level(1,1).std() for d in dlist]
tmp2=[d if t is None else t(d) for d,t in zip(dlist,m_trans)]
#align if necessary:
if transmode=='none' or transmode == "": #do nothing
tmp2=dlist
else:
plt.close('all')
if transmode == 'interactive':
print ("""interactively set markers on each axis with CTRL+LeftMouse.
Remove markers with CTRL+right_mounse, ENTER when ready.""")
#run interactive alignment
m_arr,m_trans=align_interactive(dlist,find_transform=find_affine)
else:
if transmode == 'file':
#read markers from file that some
raise NotImplementedError
else:
raise ValueError
m_arr=[np.array(m) for m in conf['markers']]
mref=conf['mref']
m_trans=[find_transform(m,mref) for m in m_arr]
#make and plots transformed (aligned) data to verify feature alignment
tmp2=[d.apply_transform(t) for d,t in zip(dlist,m_trans)]
#plot transformed data for verification
fig,axes=plt.subplots(xs,ys,sharex='all',sharey='all',squeeze=True)
axes=axes.flatten()
maximize()
for d,ax,m,t in zip(tmp2,axes,m_arr,m_trans):
plt.sca(ax)
ll=d.level(4,byline=True)
ll.plot()
plt.clim(*remove_outliers(ll.data,nsigma=2,itmax=1,span=True))
plt.plot(*((t(m).T)),'xr')
plt.savefig(None if outfolder is None else os.path.join(outfolder,k+'_alignment.png'))
#yield dlist
#plot differences with various levelings
difflist = plot_rep_diff(tmp2,None if outfolder is None else os.path.join(outfolder,k+'.png'),dis=dis)
rmslist.append([[dd.std() for dd in d] for d in difflist])
if columns is None:
tags=['%02i-%02i'%(i,j) for i,j in itertools.combinations(range(1,nfiles+1),2)]
columns=pd.MultiIndex.from_product([['raw','cyl','con','leg'],tags],names=['terms removed','files#'])
#columns = pd.RangeIndex(0,np.size(rmslist))
#pdb.set_trace()
cstats = pd.DataFrame(columns=columns,data=np.array(rmslist).reshape(1,-1),index=[k])
return cstats
def build_database(confdic,outfolder=None,columns=None,dis=False,
find_transform=find_rototrans):
"""Process all files and return stats database in form of pd.DataFrame from confdic.
plot_repeat and plot_rep_diff are used to read fits files and calculate rmss.
If outfolder is provided, output plots are generated in functions.
Expect an index for columns (can be multiindex.)
If dis is set, display (set to True for single dataset, False for multiple config processing."""
def make_dates(stats):
"""returns a dataframe dates build with indices from stats and value """
dates=[]
for k in stats.index:
try:
date=confdic[k]['date']
except KeyError:
date=k.split('_')[0]
dates.append(parse(date,yearfirst=True).date())
return pd.Series(data=dates,index=stats.index)
#check existence of files before starting processing
for k,conf in confdic.items():
flist=[os.path.join(conf['infolder'],f)for f in conf['files']]
if not np.all([os.path.exists(f) for f in flist]):
print (k," has some files that cannot be found:")
print (conf['infolder'])
print (conf['files'])
for f in flist:
open(f).close() #raise exception if file doesn't exist
stats = None
#real processing
for i,(k,conf) in enumerate(confdic.items()):
print("Process data %i/%i"%(i+1,len(confdic.items())))
#######
rmslist=[]
#outfile=os.path.join(outfolder,k)
flist=[os.path.join(conf['infolder'],f)for f in conf['files']]
#plot surfaces, dlist is list of Data2D objects with raw data
dlist=plot_repeat(flist,
None if outfolder is None else os.path.join(outfolder,k+'.png'),dis=dis)
rms=[d.level(1,1).std() for d in dlist]
#align if necessary:
transmode=conf.get('trans',None)
if transmode=='none' or transmode == "": #do nothing
tmp2=dlist
else:
plt.close('all')
if transmode == 'interactive':
print ("""interactively set markers on each axis with CTRL+LeftMouse.
Remove markers with CTRL+right_mounse, ENTER when ready.""")
#run interactive alignment
m_arr,m_trans=align_interactive(dlist,find_transform=find_transform)
#incorporate markers in config and save backup copy
conf['trans']='config' # alignment mode, read from config
conf['mref']=m_arr[0].tolist() #reference markers to aligh to
conf['markers']=[m.tolist() for m in m_arr] #markers saved as lists
json.dump({k:conf},open(os.path.join(outfolder,k+'_out.json'),'w'),indent=0)
else:
if transmode == 'config': #gets markers from configfile
pass #conf is already set
elif transmode == 'file':
d=json.load(open(conf['markers'],'r'))
print(list(d.keys()),'\n\n',d[list(d.keys())[0]])
print(len(list(d.keys()))," datasets")
k,conf=list(d.items())[0]
else:
raise ValueError
m_arr=[np.array(m) for m in conf['markers']]
mref=conf['mref']
m_trans=[find_transform(m,mref) for m in m_arr]
#make and plots transformed (aligned) data to verify feature alignment
tmp2=[d.apply_transform(t) for d,t in zip(dlist,m_trans)]
#plot transformed data for verification
fig,axes=plt.subplots(xs,ys,sharex='all',sharey='all',squeeze=True)
axes=axes.flatten()
maximize()
for d,ax,m,t in zip(tmp2,axes,m_arr,m_trans):
plt.sca(ax)
ll=d.level(4,byline=True)
ll.plot()
plt.clim(*remove_outliers(ll.data,nsigma=2,itmax=1,span=True))
plt.plot(*((t(m).T)),'xr')
plt.savefig(None if outfolder is None else os.path.join(outfolder,k+'_alignment.png'))
#yield dlist
#plot differences with various levelings
difflist = plot_rep_diff(tmp2,None if outfolder is None else os.path.join(outfolder,k+'.png'),dis=dis)
rmslist.append([[dd.std() for dd in d] for d in difflist])
if columns is None:
columns = pd.RangeIndex(0,np.size(rmslist))
if stats is None: stats = pd.DataFrame(columns=columns)
pstats = pd.DataFrame(columns=stats.columns,data=np.array(rmslist).reshape(1,-1),index=[k])
#### fuori routine
stats=stats.append(pstats)
dates=make_dates(stats)
return stats, dates
def comp2(d1, d2, roi=None):
#adapted (partially) to data2d
"""make 4 panel plot with data, differences and difference slope for two data sets on same x and y.
Return four axis for plot customization.
all data and all stats are leveled.
roi: if passed, plot a corresponding rectangle."""
partial = False #not implemented, if a roi is defined, adds a legend with partial statistics
if roi is not None:
rect = np.array([(roi[0][0], roi[1][0]), (roi[0][0], roi[1][1]),
(roi[0][1], roi[1][1]), (roi[0][1], roi[1][0]),
(roi[0][0], roi[1][0])])
plt.figure(6)
plt.clf()
maximize()
import pdb
#pdb.set_trace()
diff = d1 - d2
ax1 = plt.subplot(141)
d1.level().plot(title='Data')
plt.clim(*remove_outliers(d1.level().data, nsigma=2, itmax=3, span=True))
if roi is not None:
plt.plot(rect[:, 0], rect[:, 1], '--', c='black', lw=2)
legendbox(get_stats(d1.crop(*roi).level()()[0]), loc=4)
ax2 = plt.subplot(142, sharex=ax1, sharey=ax1)
d2.level().plot(title='Simulation')
plt.clim(*remove_outliers(d2.level().data, nsigma=2, itmax=3, span=True))
if roi is not None:
plt.plot(rect[:, 0], rect[:, 1], '--', c='black', lw=2)
legendbox(get_stats(d2.crop(*roi).level()()[0]), loc=4)
ax3 = plt.subplot(143, sharex=ax1, sharey=ax1)
diff.level().plot(title='Difference')
plt.clim(*remove_outliers(diff.level().data, nsigma=2, itmax=3, span=True))
if roi is not None:
plt.plot(rect[:, 0], rect[:, 1], '--', c='black', lw=2)
legendbox(get_stats(diff.crop(*roi).level()()[0]), loc=4)
ax4 = plt.subplot(144, sharex=ax1, sharey=ax1)
#2019/03/28 replaced with object
#grad=np.gradient(diff.level().data)
#dy=np.diff(diff.y)[0]
#slopeax=grad[0]/1000.*206265/dy
#x,y=diff.x,diff.y
#plot_data(slopeax,x,y,units=['mm','mm','arcsec'],title='Axial Slope',stats=True) #,
#vmin=-250,vmax=250)
#plt.clim(*remove_outliers(slopeax,nsigma=2,itmax=3,span=True))
#if roi is not None:
# plt.plot(rect[:,0], rect[:,1], '--',c='black', lw=2)
# legendbox(get_stats(*crop_data(slopeax,x,y,*roi),units=['mm','mm','arcsec']),loc=4)
slopeax = diff.slope(scale=(1., 1., 1000.))[1]
slopeax.name = 'Axial Slope'
slopeax.plot()
if roi is not None:
plt.plot(rect[:, 0], rect[:, 1], '--', c='black', lw=2)
legendbox(get_stats(slopeax.crop(*roi).level()()[0]), loc=4)
axes = [ax1, ax2, ax3, ax4]
'''
if roi is not None: #update legend to include total and partial stats.
for ax in axes:
for art in ax.artists:
if isinstance(art,matplotlib.legend.Legend):
p = art.get_patches()
t = art.get_texts()
newleg=Legend(ax,p,t,loc=2,handletextpad=0, handlelength=0)
ax.add_artist(newleg)
art.remove()
'''
return axes
def plot_surface_analysis(wdata,x,y,label="",outfolder=None,nsigma_crange=1,fignum=None,
figsize=(9.6,6.7),psdrange=None,levelingfunc=None,frange=None):
"""Create two panel preview plots for data.
This is two panels of raw and filtered data (second order removed) + histogram
of height distribution + PSD.
Label is typically the filename and is used to generate title and output filenames.
fignum and figsize are for PSD2D figure (fignum=0 clear and reuse current figure,
None create new).
nsigma_crange and levelingfunc are used to evaluate color range for plot by iteratively
leveling and removing outliers at nsigma until a convergence is reached.
psdrange is used
frange is used for plot of PSD2D
"""
from pySurf.psd2d import psd2d_analysis
units=['mm','mm','$\mu$m']
order_remove=2 #remove sag, this is the order that is removed in leveled panel
if np.size(units)==1: #unneded, put here just in case I decide to take units
units=np.repeat(units,3) #as argument. However ideally should be put in plot psd function.
if levelingfunc is None:
levelingfunc=lambda x: x-legendre2d(x,1,1)[0]
if outfolder is not None:
os.makedirs(os.path.join(outfolder,'PSD2D'),exist_ok=True)
else:
print('OUTFOLDER is none')
wdata=levelingfunc(wdata) #added 20181101
rms=np.nanstd(wdata)
#crange=remove_outliers(wdata,nsigma=nsigma_crange,itmax=1,range=True)
crange=remove_outliers(wdata,nsigma=nsigma_crange,
flattening_func=levelingfunc,itmax=2,span=1)
plt.clf()
#FULL DATA
plt.subplot(221)
#plt.subplot2grid((2,2),(0,0),1,1)
plot_data(wdata,x,y,units=units,title='leveled data',stats=True)
plt.clim(*crange)
#SUBTRACT LEGENDRE
ldata=levellegendre(y,wdata,order_remove)
plt.subplot(223)
lrange=remove_outliers(ldata,nsigma=nsigma_crange,itmax=1,span=True)
#plt.subplot2grid((2,2),(1,0),1,1)
plot_data(ldata,x,y,units=units,title='%i orders legendre removed'%order_remove,stats=True)
plt.clim(*lrange)
#HISTOGRAM
#plt.subplot2grid((2,2),(0,1),2,1)
plt.subplot(222)
plt.hist(wdata.flatten()[np.isfinite(wdata.flatten())],bins=100,label='full data')
plt.hist(ldata.flatten()[np.isfinite(ldata.flatten())],bins=100,color='r',alpha=0.2,label='%i orders filtered'%order_remove)
#plt.vlines(crange,*plt.ylim())
plt.vlines(crange,*plt.ylim(),label='plot range raw',linestyle='-.')
plt.vlines(lrange,*plt.ylim(),label='plot range lev',linestyle=':')
plt.legend(loc=0)
#FREQUENCY AND PSD ANALYSIS
if fignum==0: #this is the figure created by psd2d_analysis
plt.clf()
fig = plt.gcf()
#pdb.set_trace()
stax=plt.gca() #preserve current axis
f,p=psd2d_analysis(ldata,x,y,title=label,
wfun=np.hanning,fignum=fignum,vrange=lrange,
prange=psdrange,units=units,
rmsrange=frange)#,ax2f=[0,1,0,0])
plt.ylabel('slice rms ($\mu$m)')
ps=projection(p,axis=1,span=True) #avgpsd2d(p,span=1)
if outfolder:
maximize()
plt.savefig(os.path.join(outfolder,'PSD2D',
fn_add_subfix(label,"_psd2d",'.png')))
save_data(os.path.join(outfolder,'PSD2D',fn_add_subfix(label,"_psd2d",'.txt')),
p,x,f)
np.savetxt(os.path.join(outfolder,fn_add_subfix(label,"_psd",'.txt')),np.vstack([f,ps[0],ps[1],ps[2]]).T,
header='f(%s^-1)\tPSD(%s^2%s)AVG\tmin\tmax'%(units[1],units[2],units[1]),fmt='%f')
#pdb.set_trace()
plt.sca(stax)
plt.subplot(224)
plt.ylabel('axial PSD ('+units[2]+'$^2$ '+units[1]+')')
plt.xlabel('Freq. ('+units[1]+'$^{-1}$)')
plt.plot(f,ps[0],label='AVG')
plt.plot(f,ps[1],label='point-wise min')
plt.plot(f,ps[2],label='point-wise max')
plt.plot(f,p[:,p.shape[1]//2],label='central profile')
plt.ylim(psdrange)
plt.loglog()
plt.grid(1)
plt.legend(loc=0)
#ideally to be replaced by:
"""
plt.subplot(224)
plot_psd(((f,ps[0]),(f,ps[1]),(f,ps[2),(f,p[:,p.shape[1]//2])),
['AVG','point-wise min','point-wise max','central profile'])
plt.ylabel('axial '+plt.ylabel)
plt.ylim(psdrange)
"""
plt.suptitle('%s: rms=%5.3f 10$^{-3}$'%(label,rms*1000)+units[2])
plt.tight_layout(rect=[0, 0.03, 1, 0.95])
if outfolder:
plt.savefig(os.path.join(outfolder,fn_add_subfix(label,"",'.png')))
return (ldata,x,y),(f,p)
def plot_data_repeat_leveling(dlist,outfile=None,dis=True,name = ""):
"""Create a multiplot on a grid with plots of all data with
same leveling: raw, conical, cyliindrical, sag removed by line
abused function in old notebooks when type of leveling was uncertain or of interest."""
xs,ys=find_grid_size(len(dlist),3,square=False)
res=[]
r=plot_data_repeat(dlist,name=name,num=1)
res.append(r)
if outfile:
os.makedirs(os.path.dirname(outfile)+'\\raw\\',exist_ok=True)
plt.savefig(fn_add_subfix(outfile,'_raw','.png',pre='raw\\'))
if dis: display(plt.gcf())
fig,axes=plt.subplots(xs,ys,num=2)
axes=axes.flatten()
maximize()
res.append([])
for i,(ll,ax) in enumerate(zip(dlist,axes)):
plt.subplot(xs,ys,i+1,sharex=axes[0],sharey=axes[0])
tmp=ll.copy()
tmp.data=ll.data-fit.fitCylMisalign(ll.data)[0]
tmp.plot(stats=True)
res[-1].append(tmp.std())
#plt.clim([-3,3])
plt.clim(*(tmp.std()*np.array([-1,1])))
for ax in axes[:len(dlist)-1:-1]:
fig.delaxes(ax)
#plt.pause(0.1)
plt.suptitle(name+' CYL corrected')
if outfile:
os.makedirs(os.path.dirname(outfile)+'\\cyl\\',exist_ok=True)
plt.savefig(fn_add_subfix(outfile,'_cyl','.png',pre='cyl\\'))
if dis: display(plt.gcf())
fig,axes=plt.subplots(xs,ys,num=3)
axes=axes.flatten()
maximize()
res.append([])
for i,(ll,ax) in enumerate(zip(dlist,axes)):
plt.subplot(xs,ys,i+1,sharex=axes[0],sharey=axes[0])
tmp=ll.copy()
tmp.data=ll.data-fit.fitConeMisalign(ll.data)[0]
tmp.plot(stats=True)
res[-1].append(tmp.std())
#plt.clim([-3,3])
plt.clim(*(tmp.std()*np.array([-1,1])))
for ax in axes[:len(dlist)-1:-1]:
fig.delaxes(ax)
#plt.pause(0.1)
plt.suptitle(name+' CONE corrected')
if outfile:
os.makedirs(os.path.dirname(outfile)+'\\cone\\',exist_ok=True)
plt.savefig(fn_add_subfix(outfile,'_cone','.png',pre='cone\\'))
if dis: display(plt.gcf())
fig,axes=plt.subplots(xs,ys,num=4)
axes=axes.flatten()
maximize()
res.append([])
for i,(ll,ax) in enumerate(zip(dlist,axes)):
plt.subplot(xs,ys,i+1,sharex=axes[0],sharey=axes[0])
tmp=ll.copy()
#tmp.data=level_data(*ll(),2)[0]
tmp=tmp.level(2,axis=1)
tmp.plot(stats=True)
res[-1].append(tmp.std())
#plt.clim([-1,1])
plt.clim(*(tmp.std()*np.array([-1,1])))
for ax in axes[:len(dlist)-1:-1]:
fig.delaxes(ax)
#plt.pause(0.1)
plt.suptitle(name+' SAG removed by line')
plt.tight_layout(rect=[0,0,1,0.95])
if outfile:
os.makedirs(os.path.dirname(outfile)+'\\leg\\',exist_ok=True)
plt.savefig(fn_add_subfix(outfile,'_leg','.png',pre='leg\\'))
if dis: display(plt.gcf())
return res
def display_diff(f1,f2,data=False,dis=True,crd=None):
"""calculates, plots and returns the difference of two data.
It is just a diff-and-plot with a few added options and settings for color scale and reading files,
can probably be replaced by more standard routines.
f1 and f2 are file names, unless data is set, in that case they are interpreted as data.
Data are plotted on common color scale, while range for difference crd can be imposed,
otherwise is calculated on outlier removed difference."""
#set units
units=['mm','mm','nm']
zscale=1000
#load data in d1 and d2 according if arguments are data or filenames
if not(data):
d1=Data2D(file=f1,units=units,scale=(1000,-1000,zscale),center=(0,0))
if f2 is not None:
d2=Data2D(file=f2,units=units,scale=(1000,-1000,zscale),center=(0,0))
else:
d2=cp.deepcopy(d1)*0
#d2.data=d2.data*0
else:
d1=Data2D(f1.data,f1.x,f1.y,units=units)
if f2 is not None:
d2=Data2D(f2.data,f2.x,f2.y,units=units)
else:
d2=cp.deepcopy(d1)*0
#d2.data=d2.data*0
#plane level and diff
d1=d1.level()
d2=d2.level()
diff=(d1-d2).level()
#calculate common color range
cr1=remove_outliers(d1.data,span=True)
cr2=remove_outliers(d2.data,span=True)
cr=[min([cr1[0],cr2[0]]),max([cr1[0],cr2[1]])]
if crd is None: #color range for difference.
crd=remove_outliers(diff.data,span=True)
#plot on three panels, prints stats
plt.clf()
plt.subplot(311)
maximize()
d1.plot()
plt.clim(*cr)
d1.printstats()
plt.subplot(312)
d2.plot()
plt.clim(*cr)
d2.printstats()
plt.subplot(313)
diff.plot()
plt.clim(*crd)
print ("diff 2 - 1 = ")
diff.printstats()
plt.tight_layout()
#plt.clim(*cr)#[-0.0001,0.0001])
if not(data):
if f2 is not None:
plt.title ('Difference %s - %s'%tuple(os.path.basename(ff) for ff in [f1,f2]))
else:
plt.title ('%s '%f2)
plt.show()
#print ("PV: ",span(diff.data,size=True),diff.units[-1])
#print ("rms: ",np.nanstd(diff.data),diff.units[-1])
return d1,d2,diff