def test_holes():
from pySurf.data2D import plot_data
from pySurf.data2D_class import Data2D
nx, ny = 300, 200
a = np.random.random(nx * ny).reshape(ny, nx)
x = np.arange(nx) * 3 - 300
y = np.arange(ny) * 5 - 500
a[:5, :] = np.nan
a[-15:, :] = np.nan
a[:, -3:] = np.nan
a[60:100, :20] = np.nan
a[110:120, 160:165] = 1
a[110:120, 80:95] = np.nan
a[20:40, 105:125] = np.nan
plt.close('all')
#plt.subplot(121)
plt.figure()
plot_data(a, x, y)
plt.title('crop all external nans')
#b=np.where(np.isnan(a),1,a)
#plt.figure(); plot_data(b,x,y)
#plt.title('b')
r = find_internal_rectangle(a, x, y)
plot_poly([[r[0][0], r[1][0]], [r[0][1], r[1][0]], [r[0][1], r[1][1]],
[r[0][0], r[1][1]], [r[0][0], r[1][0]]])
def test_reader(file, reader, outfolder=None, infolder=None, **kwargs):
"""called without `raw` flag, return data,x,y. Infolder is taken from file or can be passed (e.g. to point to local test data)."""
import os
import matplotlib.pyplot as plt
from pySurf.data2D import plot_data
if infolder is None:
infolder = os.path.dirname(file)
df = os.path.join(infolder, os.path.basename(file))
res = reader(df, **kwargs)
header = reader(df, header=True, **kwargs)
print("reading file %s with reader %s" % (df, reader))
print("returned values", [r.shape for r in res], header)
plot_data(res[0], res[1], res[2])
plt.title(
os.path.basename(file) + ' ' +
' '.join(["%s=%s" % (k, v) for k, v in kwargs.items()]))
if outfolder is not None:
if outfolder == "":
display(plt.gcf())
else:
outname = os.path.join(infolder, outfolder, os.path.basename(df))
os.makedirs(os.path.dirname(outname), exist_ok=True)
plt.savefig(fn_add_subfix(outname, '', '.png'))
return res, header
def test_legendbox():
from pySurf.testSurfaces import make_range
from pySurf.data2D import plot_data, get_stats
plt.clf()
nx = 4
ny = 3
data, x, y = make_range(nx, ny)
plot_data(data, x, y)
legendbox(get_stats(data, units=['mm', 'mm', 'um']))
plt.show()
def multimarkers(datalist):
"""add clickable markers to a list of data using `add_clickable_markers2`.
Not sure it is updated, see also Dlist."""
plt.figure()
#difficile condividere gli assi visto che non possono essere passati
s = subplot_grid(len(datalist))
a = [add_clickable_markers2(s[0])]
for ss in s[1:-1]:
plot_data(datalist)
a.append(add_clickable_markers2(ss))
a.append(add_clickable_markers2(s[-1], hold=True))
return a
def test_class_init(wfile=None, *args, **kwargs):
"""test init and plot"""
from pathlib import PureWindowsPath
from dataIO.fn_add_subfix import fn_add_subfix
from pySurf.data2D import load_test_data
# from pySurf.data2D import data, load_test
d1, x1, y1 = load_test_data(wfile, *args, **kwargs)
plt.figure(1)
plt.clf()
# plt.suptitle(relpath)
plt.title("use plot_data function")
plot_data(d1, x1, y1, aspect="equal")
a = Data2D(d1, x1, y1)
plt.figure(2)
plt.clf()
plt.title("From data")
a.plot(aspect="equal")
b = Data2D(file=wfile, ytox=220 / 1000.0, center=(0, 0))
plt.figure(3)
plt.clf()
plt.title("from filename")
b.plot(aspect="equal")
b.save(
os.path.join(
outpath, os.path.basename(fn_add_subfix(relpath.as_posix(), "", ".txt"))
),
makedirs=True,
)
b.remove_nan_frame()
plt.figure()
plt.title("removed nans")
b.plot(aspect="equal")
def plot(self, title=None, *args, **kwargs):
res = plot_data(self.data,
self.x,
self.y,
units=self.units,
*args,
**kwargs)
if title is None:
if self.name is not None:
title = self.name
plt.title(title)
return res
def plot_psd2d(f,p,x,prange=None,includezerofreq=False,units=None,*args,**kwargs):
"""plots a 2d psd as a surface with logaritmic color scale on the current axis. Return axis.
If a zero frequency is present it is excluded by default from plot, unless includezerofreq is set to True.
Units (3-el array) is units of lengths for data (not of PSD), can be None.
2020/07/10 uncommented call to `pySurf.data2D.plot_data` (in "functions" module `data2D`).
The "old" code segment called instead directly `plt.imshow`.
In some sense, replicates code in `plot_data` and was not necessarily in sync
(e.g.: stats flag is not included here).
This seems to fix the issue. """
#2018/04/05 critical renaming of plot_psd2d(wdata,x,y..) -> psd2d_analysis
# and _plot_psd2d(f,p,x) -> plot_psd2d consistently with other routines.
if len(f.shape)==2:
print ("""Wrong size detected in call to plot_psd2d. Functions were renamed,
you probably want to call psd2d_analysis(wdata,x,y..).
I will try to correct automatically for this time, but you need to
update calling code URGENTLY, it will be discontinued.""")
return psd2d_analysis(f,p,x,*args,**kwargs)
if np.size(units)==1:
units=np.repeat(units,3)
#pdb.set_trace()
#plt.yscale('log')
plt.title('2D PSD')
if not includezerofreq:
if f[0]==0:
f=f[1:]
p=p[1:,...]
if prange is None:
prange=[None,None]
# new:
cbunits = psd_units(units)
#pdb.set_trace()
ax = plot_data(p,x,f,norm=LogNorm(vmin=prange[0],vmax=prange[1]),
units=cbunits,aspect='auto',*args,**kwargs)
plt.ylabel('freq. ('+cbunits[1]+')')
#pdb.set_trace()
cb = plt.gca().images[-1].colorbar
#after introducing psd_units, cbunits is always defined,
# conditions can be modified, but leaave like this for generality.
#cb.ax.set_title("PSD"+(" ("+cbunits[2]+")") if cbunits[2] else "")
#plt.ylabel('Frequency ('+cbunits[1]+")")
#plt.xlabel('X'+ (" ("+cbunits[0]+")" if cbunits[0] else ""))
plt.grid(1)
return ax
def test_class_init():
"""test init and plot"""
from pySurf.instrumentReader import matrixZygo_reader
from dataIO.fn_add_subfix import fn_add_subfix
relpath = r'test\input_data\zygo_data\171212_PCO2_Zygo_data.asc'
outpath = r'test\results\data2D_class'
wfile = os.path.join(os.path.dirname(__file__), relpath)
(d1, x1, y1) = matrixZygo_reader(wfile, ytox=220 / 1000., center=(0, 0))
plt.figure(1)
plt.clf()
plt.suptitle(relpath)
plt.title('use plot_data function')
plot_data(d1, x1, y1, aspect='equal')
a = Data2D(d1, x1, y1)
plt.figure(2)
plt.clf()
plt.title('From data')
a.plot(aspect='equal')
b = Data2D(file=wfile, ytox=220 / 1000., center=(0, 0))
plt.figure(3)
plt.clf()
plt.title('from filename')
b.plot(aspect='equal')
b.save(os.path.join(outpath,
os.path.basename(fn_add_subfix(relpath, "", ".txt"))),
makedirs=True)
b.remove_nan_frame()
plt.figure()
plt.title('removed nans')
b.plot(aspect='equal')
def plot(self, title=None, *args, **kwargs):
"""plot using data2d.plot_data and setting automatically labels and colorscales.
by default data are filtered at 3 sigma with 2 iterations for visualization.
Additional arguments are passed to plot."""
units = self.units if self.units is not None else ["","",""]
nsigma0 = 1 # default number of stddev for color scale
# import pdb
# to change the default behavior
stats = kwargs.pop("stats", [[0,1,3],[6],[6]])
# format for legend labels (replace "stdev" with "rms")
fmt = kwargs.pop("fmt", ['mean: %.3g'+units[0],
'rms: %.3g'+units[0],
'PV: %.3g'+units[0],
'%i X','%i'])
# to change the default behavior
nsigma = kwargs.pop("nsigma", nsigma0)
m = self.data
#pdb.set_trace()
res = plot_data(
self.data,
self.x,
self.y,
units=units,
fmt=fmt,
stats=stats,
nsigma=nsigma,
*args,
**kwargs
)
if title is None:
if self.name is not None:
title = self.name
plt.title(title)
return res
def psd2d_analysis(wdata,x,y,title=None,wfun=None,vrange=None,
rmsrange=None,prange=None,fignum=5,rmsthr=None,
aspect='auto', ax2f=None, units=None,outname=None,norm=1,rmsnorm=True):
""" Calculates 2D PSD as image obtained combining all profile PSDS calculated along vertical slices of data. Resulting image has size
If title is provided rms slice power is also calculated and plotted on three panels with figure and PSD.
Return PSD as PSD2D object.
uses plot_rms_power(f,p,rmsrange=None) to calculate rms power.
fignum window where to plot, if fignnum is 0 current figure is cleared,
if None new figure is created. Default to figure 5.
rmsrange is one or a list of frequency ranges for plotting integrated rms. Can contain None to use max or min.
If axf2 is set to boolean or array of boolean, plot slice rms for the frequencies
associated to rmsrange on second axis. rmsrange must be set accordingly (same number of elements).
rmsthr sets a threshold for data inclusion. If rms is above the value, the line is considered to contain invalid data and is removed from output PSD.
This makes it easy to average the returned PSDs.
Corresponding data are still visualized in central panel, but are marked with a red cross at top of y axes.
If multiple rms range intervals are provided, line is removed if any of them is over the threshold, but this might change in future, e.g. TODO: make it possible to add vector threshold with as many elements as rms range intervals.
Set outname to empty string to plot without generating output.
typical values:
rmsrange=[[None,0.1],[0.1,0.2],[0.2,1],[1,None]] #list of frequency intervals for rms
ax2f=[0,1,1,0,0] # axis where to plot rms corresponding to intervals in rmsrange: 1 right, 0 left
wfun = np.hanning # type of windows for fourier transform
units = ("mm","mm","$\mu$m") # units of surface data from which PSD is calculated
vrange_surf=([-0.5,0.5]) #color scale of surface map
vrange_leg=([-0.05,0.05]) #color scale of legendre removed map
prange=np.array((1e-8,1.e-1))#np.array((5e-8,1.e-5)) #color scale of 2d psd plot
fs,ps=psd2d_analysis(levellegendre(y,wdata,2),x,y,outname="",wfun=np.hanning,
rmsrange=rmsrange,prange=prange,ax2f=ax2f,units=units)
"""
#2018/04/05
# -critical renaming of plot_psd2d(wdata,x,y..) -> psd2d_analysis
# and _plot_psd2d(f,p,x) -> plot_psd2d consistently with other routines.
# -added title to determine if generating plot, with the intent of replacing outname.
plt.pause(1)
#print('cane',fignum)
if outname is not None: #handle backcompatibility
print ('psd2d_analysis WARNING: `title` replaced `outname` and output figure will be no more generated.'+
'OUTNAME will be removed in next version, use title and save the plot after returning from routine.')
if title is not None:
print('outname should be not used together with title, it will be ignored.')
else:
title=outname
if vrange is not None:
if prange is None:
print("""WARNING: older versions of psd2d use vrange as range of psdvalue.
In updated version prange is range of psd and vrange is range for surface
plots. Update your code.""")
#check compatibility with old interface
if len(wdata.shape)==1:
if len(y.shape)==2:
print("WARNING: your code calling psd2d uses old call")
print("psd2d(x,y,data), automaticly adjusted, but")
print("update IMMEDIATELY your code to call")
print("psd2d(data,x,y)")
x,y,wdata=wdata,x,y
f, p = psd2d(wdata, x, y, wfun=wfun, norm=norm, rmsnorm=rmsnorm) # calculate PSD 2D
# GENERATE OUTPUT
if title is not None:
if np.size(units)==1:
units=np.repeat(units,3)
if prange is None:
if f[0] == 0:
prange=span(p[1:,:])
else:
prange=span(p)
if prange[0]<1e-20: #arbitrary small value
print("WARNING: low limit detected in prange, can cause problems with log color scale.")
if vrange is None:
vrange=span(wdata)
#plot surface map, who knows why on figure 5
fig=fignumber(fignum)
#pdb.set_trace()
#
plt.clf()
maximize()
plt.draw()
plt.suptitle(title)
################
#plot Surface
ax1=plt.subplot(311)
plot_data(wdata,x,y,aspect=aspect,vmin=vrange[0],vmax=vrange[1],units=units,title='Surface',stats=True)
plt.xlabel(None)
plt.grid(1)
################
#plot 2D PSD
ax2=plt.subplot(312,sharex=ax1)
plot_psd2d(f,p,x,prange=prange,units=units)
if rmsrange is not None: #plot horizontal lines to mark frequency ranges for rms extraction,
#rms is tranformed to 2D array
if len(np.array(rmsrange).shape)==1: #if only one interval, make it 2D anyway
rmsrange=[rmsrange]
rmsrange=np.array(rmsrange)
for fr in np.array(rmsrange): #I want fr to be array type, even if I don't care about rmsrange this does the job
ax2.hlines(fr[fr != None],*((span(x)-x.mean())*1.1+x.mean()))
ax2.set_xlabel("")
plt.xlabel(None)
#plt.subplots_adjust(top=0.85)
################
#plot rms slice
ax3=plt.subplot(313,sharex=ax1)
if f[0]==0: #ignore component of 0 frequency in rms calculation if present
ff,pp=f[1:],p[1:]
else:
ff,pp=f,p
rms=plot_rms_power(ff,pp,x,rmsrange=rmsrange,ax2f=ax2f,units=units)
#pdb.set_trace()
mask=np.isfinite(rms) #True if good. it is an array
if rmsthr is not None:
mask = mask & (rms < rmsthr)
ax3.hlines(rmsthr,*ax2.get_xlim()) #plot markers on rms chart
ax2.plot(x[~mask],np.repeat(ax2.get_ylim()[1],len(x[~mask])),'rx') #plot markers on psd2d chart
#pdb.set_trace()
#questa era qui, ma dava errore perche' mask e' lineare
#mask = np.all(mask,axis =0) # if any of mask is False -> False
p[:,~mask]=np.nan
ax3.grid(1)
#plt.tight_layout(rect=(0, 0.03, 1, 0.95) if title else (0, 0, 1, 1))
# replaced with more evolved interactive resize:
# plt.colorbar().remove() #dirty trick to adjust size to other panels
def resize(event):
box1 = ax1.get_position()
box2 = ax2.get_position()
box3 = ax3.get_position()
ax2.set_position([box1.x0, box2.y0, box1.width , box2.height])
ax2.set_adjustable("box",share=True)
ax3.set_position([box1.x0, box3.y0, box1.width , box3.height])
ax3.set_adjustable("box",share=True)
cid = fig.canvas.mpl_connect('draw_event', resize)
cid2 = fig.canvas.mpl_connect('resize_event', resize)
resize(None)
#plt.tight_layout()
if outname: #kept for compatibility, will be removed in next version
plt.savefig(fn_add_subfix(outname,'_2dpsd','.png'))
return f,p
def diff_images(data1,
data2,
x=None,
y=None,
fignum=None,
titles=None,
vmin=None,
vmax=None,
commonscale=False,
direction=0,
*args,
**kwargs):
"""plots two data sets with common axis and their difference. Return the three axis.
Colorbars are formatted to be same height as plot.
2018/06/19 use data2D routines, allowing to add parameters (e.g. stats legend).
"""
fig = fignumber(fignum)
plt.clf()
aspect = kwargs.pop('aspect', 'auto')
#this is to set scale if not fixed
d1std = [np.nanstd(data) for data in (data1, data2)]
std = min(d1std)
if x is None:
x = np.arange(data1.shape[1])
if y is None:
y = np.arange(data1.shape[0])
ax1 = plt.subplot(131)
s = (std if commonscale else d1std[0])
d1mean = np.nanmean(data1)
plot_data(data1,
x,
y,
aspect=aspect,
vmin=kwargs.get('vmin', d1mean - s),
vmax=kwargs.get('vmax', d1mean + s),
*args,
**kwargs)
ax2 = plt.subplot(132, sharex=ax1, sharey=ax1)
s = (std if commonscale else d1std[1])
d2mean = np.nanmean(data2)
plot_data(data2,
x,
y,
aspect=aspect,
vmin=kwargs.get('vmin', d2mean - s),
vmax=kwargs.get('vmax', d2mean + s),
*args,
**kwargs)
ax3 = plt.subplot(133, sharex=ax1, sharey=ax1)
plt.title('Difference (2-1)')
diff = data2 - data1
plot_data(diff, x, y, aspect=aspect, *args, **kwargs)
axes = [ax1, ax2, ax3]
if titles is not None:
for ax, tit in zip(axes, titles):
if tit is not None:
ax.set_title(tit)
return axes
def plot_surface_analysis(wdata,x,y,label="",outfolder=None,nsigma_crange=1,fignum=None,
figsize=(9.6,6.7),psdrange=None,levelingfunc=None,frange=None):
"""Create two panel preview plots for data.
This is two panels of raw and filtered data (second order removed) + histogram
of height distribution + PSD.
Label is typically the filename and is used to generate title and output filenames.
fignum and figsize are for PSD2D figure (fignum=0 clear and reuse current figure,
None create new).
nsigma_crange and levelingfunc are used to evaluate color range for plot by iteratively
leveling and removing outliers at nsigma until a convergence is reached.
psdrange is used
frange is used for plot of PSD2D
"""
from pySurf.psd2d import psd2d_analysis
units=['mm','mm','$\mu$m']
order_remove=2 #remove sag, this is the order that is removed in leveled panel
if np.size(units)==1: #unneded, put here just in case I decide to take units
units=np.repeat(units,3) #as argument. However ideally should be put in plot psd function.
if levelingfunc is None:
levelingfunc=lambda x: x-legendre2d(x,1,1)[0]
if outfolder is not None:
os.makedirs(os.path.join(outfolder,'PSD2D'),exist_ok=True)
else:
print('OUTFOLDER is none')
wdata=levelingfunc(wdata) #added 20181101
rms=np.nanstd(wdata)
#crange=remove_outliers(wdata,nsigma=nsigma_crange,itmax=1,range=True)
crange=remove_outliers(wdata,nsigma=nsigma_crange,
flattening_func=levelingfunc,itmax=2,span=1)
plt.clf()
#FULL DATA
plt.subplot(221)
#plt.subplot2grid((2,2),(0,0),1,1)
plot_data(wdata,x,y,units=units,title='leveled data',stats=True)
plt.clim(*crange)
#SUBTRACT LEGENDRE
ldata=levellegendre(y,wdata,order_remove)
plt.subplot(223)
lrange=remove_outliers(ldata,nsigma=nsigma_crange,itmax=1,span=True)
#plt.subplot2grid((2,2),(1,0),1,1)
plot_data(ldata,x,y,units=units,title='%i orders legendre removed'%order_remove,stats=True)
plt.clim(*lrange)
#HISTOGRAM
#plt.subplot2grid((2,2),(0,1),2,1)
plt.subplot(222)
plt.hist(wdata.flatten()[np.isfinite(wdata.flatten())],bins=100,label='full data')
plt.hist(ldata.flatten()[np.isfinite(ldata.flatten())],bins=100,color='r',alpha=0.2,label='%i orders filtered'%order_remove)
#plt.vlines(crange,*plt.ylim())
plt.vlines(crange,*plt.ylim(),label='plot range raw',linestyle='-.')
plt.vlines(lrange,*plt.ylim(),label='plot range lev',linestyle=':')
plt.legend(loc=0)
#FREQUENCY AND PSD ANALYSIS
if fignum==0: #this is the figure created by psd2d_analysis
plt.clf()
fig = plt.gcf()
#pdb.set_trace()
stax=plt.gca() #preserve current axis
f,p=psd2d_analysis(ldata,x,y,title=label,
wfun=np.hanning,fignum=fignum,vrange=lrange,
prange=psdrange,units=units,
rmsrange=frange)#,ax2f=[0,1,0,0])
plt.ylabel('slice rms ($\mu$m)')
ps=projection(p,axis=1,span=True) #avgpsd2d(p,span=1)
if outfolder:
maximize()
plt.savefig(os.path.join(outfolder,'PSD2D',
fn_add_subfix(label,"_psd2d",'.png')))
save_data(os.path.join(outfolder,'PSD2D',fn_add_subfix(label,"_psd2d",'.txt')),
p,x,f)
np.savetxt(os.path.join(outfolder,fn_add_subfix(label,"_psd",'.txt')),np.vstack([f,ps[0],ps[1],ps[2]]).T,
header='f(%s^-1)\tPSD(%s^2%s)AVG\tmin\tmax'%(units[1],units[2],units[1]),fmt='%f')
#pdb.set_trace()
plt.sca(stax)
plt.subplot(224)
plt.ylabel('axial PSD ('+units[2]+'$^2$ '+units[1]+')')
plt.xlabel('Freq. ('+units[1]+'$^{-1}$)')
plt.plot(f,ps[0],label='AVG')
plt.plot(f,ps[1],label='point-wise min')
plt.plot(f,ps[2],label='point-wise max')
plt.plot(f,p[:,p.shape[1]//2],label='central profile')
plt.ylim(psdrange)
plt.loglog()
plt.grid(1)
plt.legend(loc=0)
#ideally to be replaced by:
"""
plt.subplot(224)
plot_psd(((f,ps[0]),(f,ps[1]),(f,ps[2),(f,p[:,p.shape[1]//2])),
['AVG','point-wise min','point-wise max','central profile'])
plt.ylabel('axial '+plt.ylabel)
plt.ylim(psdrange)
"""
plt.suptitle('%s: rms=%5.3f 10$^{-3}$'%(label,rms*1000)+units[2])
plt.tight_layout(rect=[0, 0.03, 1, 0.95])
if outfolder:
plt.savefig(os.path.join(outfolder,fn_add_subfix(label,"",'.png')))
return (ldata,x,y),(f,p)
if not (raw):
#reshape datapoints into 2D-Matrix
res.points = np.reshape(
res.points, [res.yPoints, res.xPoints]) #switched x-y 20180503
#it was giving interlaced output with profilometer data (never noticed on square CCI images).
#Scale datapoints without Offset
res.points = (res.points - res.zMin) * res.zSpacing / res.zUnitRatio
#Generate Axis without Offset;
res.xAxis = np.linspace(0, res.xSpacing * res.xPoints / res.xUnitRatio,
res.xPoints)
res.yAxis = np.linspace(0, res.ySpacing * res.yPoints / res.yUnitRatio,
res.yPoints)
fileID.close()
return res
if __name__ == "__main__":
from pySurf.data2D import plot_data
from pySurf.readers.test_readers import testfolder
df = os.path.join(
testfolder,
r'input_data\profilometer\04_test_directions\05_xysurf_pp_Height.sur')
res = readsur(df)
plt.figure()
plot_data(res.points, res.xAxis, res.yAxis)
plt.show()
#from pySurf.readers.instrumentReader import points_reader
#infolder=r'test\input_data\profilometer\04_test_directions'
#infile='07_xysurf_mp_Intensity.txt'
#data,x,y=points_reader(os.path.join(infolder,infile))
img = a
angle = 30 / 180 * np.pi
pivot = (5, 5)
plt.figure()
plt.subplot(121)
plot_data(a)
plt.title("raw; xlim: %.2f:%.2f,ylim: %.2f:%.2f" % (plt.xlim() + plt.ylim()))
plt.clim([0, 2])
plt.subplot(122)
d2 = rot01(a, 15, np.array((5, 5)))
plot_data(d2)
plt.title("raw; xlim: %.2f:%.2f,ylim: %.2f:%.2f" % (plt.xlim() + plt.ylim()))
plt.clim([0, 2])
plt.tight_layout()
display(plt.gcf())
# In[14]:
infolder = r'input_data\profilometer\04_test_directions'
#outfolder=r'test\profilometer_scans\04_test_directions_output'
outfolder = r'results\test_points'
}
],
[
csvZygo_reader,
os.path.join(pwd, testfolder,
r'input_data\zygo_data\171212_PCO2_Zygo_data.asc'), {
'strip': True,
'center': (10, 15),
'intensity': True
}
]
]
plt.ion()
#plt.close('all')
for r, f, o in tests: #reader,file,options
print('reading data %s' % os.path.basename(f))
plt.figure()
plt.subplot(121)
plot_data(*r(f))
plt.title('raw data')
plt.subplot(122)
data, x, y = read_data(f, r, **o)
plot_data(data, x, y)
plt.title('registered')
plt.suptitle(
os.path.basename(f) + ' ' +
' '.join(["%s=%s" % (k, v) for k, v in o.items()]))
plt.tight_layout()
plt.show()
