def main():
from PyMca import EDFStack
from PyMca import EdfFile
import sys
inputfile = "D:\DATA\COTTE\ch09\ch09__mca_0005_0000_0000.edf"
if len(sys.argv) > 1:
inputfile = sys.argv[1]
print(inputfile)
elif os.path.exists(inputfile):
print("Using a default test case")
else:
print("Usage:")
print("python PCAModule.py indexed_edf_stack")
sys.exit(0)
stack = EDFStack.EDFStack(inputfile)
r0, c0, n0 = stack.data.shape
ncomponents = 5
outfile = os.path.basename(inputfile) + "ICA.edf"
e0 = time.time()
images, eigenvalues, eigenvectors = mdpICA(stack.data, ncomponents,
binning=1, svd=True,
dtype='float64')
#images, eigenvalues, eigenvectors = lanczosPCA2(stack.data,
# ncomponents,
# binning=1)
if os.path.exists(outfile):
os.remove(outfile)
f = EdfFile.EdfFile(outfile)
for i in range(ncomponents):
f.WriteImage({}, images[i, :])
stack.data.shape = r0, c0, n0
print("PCA Elapsed = %f" % (time.time() - e0))
print("eigenvectors PCA2 = ", eigenvectors[0, 200:230])
stack = None
stack = EDFStack.EDFStack(inputfile)
e0 = time.time()
images2, eigenvalues, eigenvectors = mdpPCA(stack.data, ncomponents,
binning=1)
stack.data.shape = r0, c0, n0
print("MDP Elapsed = %f" % (time.time() - e0))
print("eigenvectors MDP = ", eigenvectors[0, 200:230])
if os.path.exists(outfile):
os.remove(outfile)
f = EdfFile.EdfFile(outfile)
for i in range(ncomponents):
f.WriteImage({}, images[i, :])
for i in range(ncomponents):
f.WriteImage({}, images2[i, :])
f = None
def getData(self):
if (self.fileName.endswith('.raw') or self.fileName.endswith('.RAW')
or self.fileName.endswith('.img')):
File = open(str(self.fileName), "rb")
size = os.path.getsize(str(self.fileName)) / 4.
self.width = int(size**.5)
self.height = self.width
self.currentSlice = np.fromfile(File, dtype='<f4')
self.currentSlice.resize(self.height, self.width)
elif (self.fileName.endswith('.edf')
or self.fileName.endswith('.EDF')):
fileEdf = EdfFile.EdfFile(str(self.fileName), access='rb')
self.currentSlice = fileEdf.GetData(0)
elif (self.fileName.endswith('.DOWN')
or self.fileName.endswith('.UP')):
fileEdf = EdfFile.EdfFile(str(self.fileName), access='rb')
self.currentSlice = fileEdf.GetData(0)
elif (self.fileName.endswith('.tif') or self.fileName.endswith('.TIF')
or self.fileName.endswith('.TIFF')
or self.fileName.endswith('.tiff')):
fileTif = TiffIO.TiffIO(str(self.fileName), mode='rb')
self.currentSlice = fileTif.getImage(0)
elif (self.fileName.endswith('.png')
or self.fileName.endswith('.PNG')):
filePng = mpimg.imread(self.fileName)
self.currentSlice = filePng
elif (self.fileName.endswith('.dcm')
or self.fileName.endswith('.DCM')):
image = dicom.read_file(str(self.fileName))
self.currentSlice = image.pixel_array
elif (self.fileName.endswith('.mat')
or self.fileName.endswith('.MAT')):
image = sio.loadmat(self.fileName)
self.currentSlice = image[image.keys()[0].keys()[0]]
else:
image = dicom.read_file(str(self.fileName))
self.currentSlice = image
return self.currentSlice
def save2DArrayListAsEDF(datalist, filename, labels=None, dtype=None):
if type(datalist) != type([]):
datalist = [datalist]
ndata = len(datalist)
if os.path.exists(filename):
try:
os.remove(filename)
except OSError:
pass
if labels is None:
labels = []
for i in range(ndata):
labels.append("Array_%d" % i)
if len(labels) != ndata:
raise ValueError("Incorrect number of labels")
edfout = EdfFile.EdfFile(filename, access="ab")
for i in range(ndata):
if dtype is None:
edfout.WriteImage({'Title': labels[i]},
datalist[i], Append=1)
else:
edfout.WriteImage({'Title': labels[i]},
datalist[i].astype(dtype),
Append=1)
del edfout # force file close
def space_to_edf(space, filename):
from PyMca import EdfFile
header = {}
for a in space.axes:
header[str(a.label)] = '{0} {1} {2}'.format(a.min, a.max, a.res)
edf = EdfFile.EdfFile(filename)
edf.WriteImage(header, space.get_masked().filled(0), DataType="Float")
def SetSource(self, source_name=None, source_obj=None):
"""
Sets a new source for data retrieving, an edf file.
If the file exists, self.Source will be the EdfFile
object associated to this file.
Parameters:
source_name: name of the edf file
"""
if source_name == self.SourceName: return 1
if (type(source_name) != type([])): source_name = [source_name]
if (source_name is not None):
if source_obj is not None:
self.Source = source_obj
else:
if (type(source_name) == type([])):
if DEBUG:
print("List of files")
self.Source = []
for name in source_name:
try:
self.Source.append(
EdfFile.EdfFile(name, fastedf=self.fastedf))
except:
#print("EdfFileLayer.SetSource: Error trying to read EDF file %s" % name)
self.Source.append(None)
else:
try:
self.Source = EdfFile.EdfFile(source_name,
fastedf=self.fastedf)
except:
#print("EdfFileLayer.SetSource: Error trying to read EDF file")
self.Source = None
else:
self.Source = None
self.SourceInfo = None
if self.Source is None:
self.SourceName = None
return 0
else:
self.SourceName = ""
for name in source_name:
if self.SourceName != "": self.SourceName += "|"
self.SourceName += name
return 1
def openEdf(filename, read=0, write=0, force=0):
if read:
checkEdfForRead(filename)
if write:
checkEdfForWrite(filename, force)
try:
edf = EdfFile.EdfFile(filename)
except:
raise XiaEdfError("Cannot open EDF file <%s>" % filename)
return edf
def space_to_edf(space, filename):
try:
from PyMca import EdfFile
except ImportError:
from PyMca5.PyMca import EdfFile
header = {}
for a in space.axes:
header[str(a.label)] = '{0} {1} {2}'.format(a.min, a.max, a.res)
edf = EdfFile.EdfFile(filename)
edf.WriteImage(header,
space.get_norm_intensity().filled(0),
DataType="Float")
def get_images(self, scan, first, last, dry_run=False):
imagenos = numpy.array(scan.datacol('img')[slice(first, last + 1)],
dtype=numpy.int)
if self.config.background:
if not os.path.exists(self.config.background):
raise errors.FileError(
'could not find background file {0}'.format(
self.config.background))
if dry_run:
yield
else:
edf = EdfFile.EdfFile(self.config.background)
for i in range(first, last + 1):
self.dbg_pointno = i
yield edf
else:
try:
uccdtagline = scan.header('M')[0].split()[-1]
UCCD = os.path.dirname(uccdtagline).split(os.sep)
except:
print(
'warning: UCCD tag not found, use imagefolder for proper file specification'
)
UCCD = []
pattern = self._get_pattern(UCCD)
matches = self.find_edfs(pattern)
if not set(imagenos).issubset(set(matches.keys())):
raise errors.FileError(
"incorrect number of matches for scan {0} using pattern {1}"
.format(scan.number(), pattern))
if dry_run:
yield
else:
for i in imagenos:
self.dbg_pointno = i
edf = EdfFile.EdfFile(matches[i])
yield edf
def get_images(self, scan, first, last, dry_run=False):
try:
uccdtagline = scan.header('UCCD')[0]
UCCD = os.path.split(os.path.dirname(uccdtagline.split()[-1]))
except:
print('warning: UCCD tag not found, use imagefolder for proper file specification')
UCCD = []
pattern = self._get_pattern(UCCD)
matches = self.find_edfs(pattern, scan.number())
if set(range(first, last + 1)) > set(matches.keys()):
raise errors.FileError("incorrect number of matches for scan {0} using pattern {1}".format(scan.number(), pattern))
if dry_run:
yield
else:
for i in range(first, last+1):
edf = EdfFile.EdfFile(matches[i])
yield edf.GetData(0)
def load_matrix(filename):
if filename == None:
return None
if os.path.exists(filename):
ext = os.path.splitext(filename)[-1]
if ext == '.txt':
return numpy.array(numpy.loadtxt(filename), dtype=numpy.bool)
elif ext == '.npy':
return numpy.array(numpy.load(filename), dtype=numpy.bool)
elif ext == '.edf':
return numpy.array(EdfFile.EdfFile(filename).getData(0),
dtype=numpy.bool)
else:
raise ValueError(
'unknown extension {0}, unable to load matrix!\n'.format(ext))
else:
raise IOError(
'filename: {0} does not exist. Can not load matrix'.format(
filename))
def getObject3DInstance(config=None):
fileTypeList = ['Picture Files (*jpg *jpeg *tif *tiff *png)',
'EDF Files (*edf)',
'EDF Files (*ccd)',
'ADSC Files (*img)',
'EDF Files (*)']
fileList, filterUsed = Object3DFileDialogs.getFileList(None,
filetypelist=fileTypeList,
message="Please select one object data file",
mode="OPEN",
getfilter=True)
if not len(fileList):
return
fname = fileList[0]
if filterUsed.split()[0] == "Picture":
qimage = qt.QImage(fname)
if qimage.isNull():
msg = qt.QMessageBox(self)
msg.setIcon(qt.QMessageBox.Critical)
msg.setText("Cannot read file %s as an image" % fname)
msg.exec_()
return
object3D = Object3DPixmap(os.path.basename(fname))
object3D.setQImage(qimage)
return object3D
if filterUsed.split()[0] in ["EDF", "ADSC"]:
edf = EdfFile.EdfFile(fname)
data = edf.GetData(0)
if True:
object3D = Object3DPixmap(os.path.basename(fname))
object3D.setImageData(data)
else:
(image,size,minmax)= spslut.transform(data, (1,0),
(spslut.LINEAR,3.0),
"RGBX", spslut.TEMP,
1,
(0, 1),
(0, 255), 1)
object3D = Object3DPixmap(os.path.basename(fname))
object3D.setPixmap(image, size[0], size[1], xmirror = False, ymirror = False)
return object3D
return None
def getObject3DInstance(config=None):
#for the time being a former configuration
#for serializing purposes is not implemented
#I do the import here for the case PyMca is not installed
#because the modules could be instanstiated without using
#this method
try:
from PyMca import EdfFile
except ImportError:
import EdfFile
fileTypeList = [
'EDF Files (*edf)', 'EDF Files (*ccd)', 'ADSC Files (*img)',
'EDF Files (*)'
]
old = Object3DFileDialogs.Object3DDirs.nativeFileDialogs * 1
Object3DFileDialogs.Object3DDirs.nativeFileDialogs = False
fileList, filterUsed = Object3DFileDialogs.getFileList(
None,
filetypelist=fileTypeList,
message="Please select one object data file",
mode="OPEN",
getfilter=True)
Object3DFileDialogs.Object3DDirs.nativeFileDialogs = old
if not len(fileList):
return None
if filterUsed == fileTypeList[0]:
fileindex = 2
else:
fileindex = 1
#file index is irrelevant in case of an actual 3D stack.
filename = fileList[0]
legend = os.path.basename(filename)
edf = EdfFile.EdfFile(filename, access='rb')
data = edf.GetData(0).astype(numpy.float32)
object3D = Object3DMesh(os.path.basename(filename))
object3D.setData(data, z=data[:])
return object3D
def _dynamicAction(self, index):
#just support edffiles
fileName = self.imageList[index]
edf = EdfFile.EdfFile(fileName)
self.setImageData(edf.GetData(0))
self.graphWidget.graph.setTitle(os.path.basename(fileName))
def saveImage(self, ffile=None):
self.savedImages = []
if ffile is None:
ffile = os.path.splitext(self._rootname)[0]
ffile = self.os_path_join(self.imgDir, ffile)
if not self.roiFit:
if (self.fileStep > 1) or (self.mcaStep > 1):
trailing = "_filestep_%02d_mcastep_%02d" % (self.fileStep,
self.mcaStep)
else:
trailing = ""
#speclabel = "#L row column"
speclabel = "row column"
if self.chunk is None:
suffix = ".edf"
else:
suffix = "_%06d_partial.edf" % self.chunk
iterationList = self.__peaks * 1
iterationList += ['chisq']
if self._concentrations:
iterationList += self.__concentrationsKeys
for peak in iterationList:
if peak in self.__peaks:
a, b = peak.split()
speclabel += " %s" % (a + "-" + b)
speclabel += " s(%s)" % (a + "-" + b)
edfname = ffile + "_" + a + "_" + b + trailing + suffix
elif peak in self.__concentrationsKeys:
speclabel += " %s" % peak.replace(" ", "-")
edfname = ffile + "_" + peak.replace(
" ", "_") + trailing + suffix
elif peak == 'chisq':
speclabel += " %s" % (peak)
edfname = ffile + "_" + peak + trailing + suffix
else:
print("Unhandled peak name: %s. Not saved." % peak)
continue
dirname = os.path.dirname(edfname)
if not os.path.exists(dirname):
try:
os.mkdir(dirname)
except:
print("I could not create directory %s" % dirname)
Append = 0
if os.path.exists(edfname):
try:
os.remove(edfname)
except:
print("I cannot delete output file")
print("trying to append image to the end")
Append = 1
edfout = EdfFile.EdfFile(edfname, access='ab')
edfout.WriteImage({'Title': peak},
self.__images[peak],
Append=Append)
edfout = None
self.savedImages.append(edfname)
#save specfile format
if self.chunk is None:
specname = ffile + trailing + ".dat"
else:
specname = ffile + trailing + "_%06d_partial.dat" % self.chunk
if os.path.exists(specname):
try:
os.remove(specname)
except:
pass
specfile = open(specname, 'w+')
#specfile.write('\n')
#specfile.write('#S 1 %s\n' % (file+trailing))
#specfile.write('#N %d\n' % (len(self.__peaks)+2))
specfile.write('%s\n' % speclabel)
specline = ""
for row in range(self.__nrows):
for col in range(self.__ncols):
specline += "%d" % row
specline += " %d" % col
for peak in self.__peaks:
#write area
specline += " %g" % self.__images[peak][row][col]
#write sigma area
specline += " %g" % self.__sigmas[peak][row][col]
#write global chisq
specline += " %g" % self.__images['chisq'][row][col]
if self._concentrations:
for peak in self.__concentrationsKeys:
specline += " %g" % self.__images[peak][row][col]
specline += "\n"
specfile.write("%s" % specline)
specline = ""
specfile.write("\n")
specfile.close()
else:
for group in self.__ROIpeaks:
i = 0
grouptext = group.replace(" ", "_")
for roi in self._ROIimages[group].keys():
#roitext = roi.replace(" ","-")
if (self.fileStep > 1) or (self.mcaStep > 1):
edfname = ffile + "_" + grouptext + (
"_%04deVROI_filestep_%02d_mcastep_%02d.edf" %
(self.roiWidth, self.fileStep, self.mcaStep))
else:
edfname = ffile + "_" + grouptext + ("_%04deVROI.edf" %
self.roiWidth)
dirname = os.path.dirname(edfname)
if not os.path.exists(dirname):
try:
os.mkdir(dirname)
except:
print("I could not create directory %s" % dirname)
edfout = EdfFile.EdfFile(edfname)
edfout.WriteImage({'Title': group + " " + roi},
self._ROIimages[group][roi],
Append=i)
if i == 0:
self.savedImages.append(edfname)
i = 1
import sys
import os
from Object3D import SceneGLWindow
try:
from PyMca import EdfFile
except ImportError:
import EdfFile
app = qt.QApplication(sys.argv)
window = SceneGLWindow.SceneGLWindow()
window.show()
if len(sys.argv) > 1:
flist = []
for i in range(1, len(sys.argv)):
flist.append(sys.argv[i])
for f in flist:
edf = EdfFile.EdfFile(f, access='rb')
data = edf.GetData(0)
object3D = Object3DMesh(os.path.basename(f))
else:
data = numpy.arange(200.).astype(numpy.float32)
data.shape = [40, 5]
object3D = Object3DMesh('builtin')
#several options: regular grid, irregular grid
if len(sys.argv) > 1:
#print "IMPOSSING A 1000 OFFSET"
#offset = 1000.0
offset = 0
#irregular grid
xSize, ySize = data.shape[0:2]
zSize = 1
l3.setText(" " + ticks[i])
qt.qApp.processEvents()
time.sleep(2)
msg.close()
result = sthread._result
del sthread
self.raise_()
return result
if __name__ == "__main__":
import os
from PyMca import EdfFile
app = qt.QApplication([])
qt.QObject.connect(app, qt.SIGNAL("lastWindowClosed()"), app,
qt.SLOT("quit()"))
d = NNMADialog()
imageList = []
for t in ["mix1.edf", "mix2.edf", "mix3.edf"]:
fname = os.path.join(os.path.dirname(__file__), "tests", t)
if not os.path.exists(fname):
break
edf = EdfFile.EdfFile(fname)
data = edf.GetData(0)
edf = None
imageList.append(data)
if len(imageList):
d.setData(imageList)
d.show()
app.exec_()
def processList(self):
self.__ncols = None
self.__nrows = self.filestep
counter = 0
ffile = SpecFileLayer.SpecFileLayer()
for fitfile in self._filelist:
self.onNewFile(fitfile, self._filelist)
ffile.SetSource(fitfile)
fileinfo = ffile.GetSourceInfo()
# nscans = len(fileinfo['KeyList'])
for scankey in fileinfo['KeyList']:
scan, order = scankey.split(".")
info, data = ffile.LoadSource(scankey)
scan_obj = ffile.Source.select(scankey)
if info['NbMca'] > 0:
for i in range(info['NbMca']):
point = int(i / info['NbMcaDet']) + 1
mca = (i % info['NbMcaDet']) + 1
key = "%s.%s.%05d.%d" % (scan, order, point, mca)
if i == 0:
mcainfo, mcadata = ffile.LoadSource(key)
mcadata = scan_obj.mca(i + 1)
y0 = numpy.array(mcadata, numpy.float)
if counter == 0:
key0 = "%s key %s" % (os.path.basename(fitfile),
key)
self.__ncols = len(y0)
image = numpy.zeros((self.__nrows,self.__ncols), \
numpy.float)
if self.__ncols != len(y0):
print("spectrum has different number of columns")
print("skipping it")
else:
image[counter, :] = y0[:]
if (counter + 1) == self.filestep:
if self.filestep > 1:
key1 = "%s key %s" % (
os.path.basename(fitfile), key)
title = "%s to %s" % (key0, key1)
else:
title = key0
if 1:
ddict = {}
if 'Channel0' in mcainfo:
ddict['MCA start ch'] =\
int(mcainfo['Channel0'])
if 'McaCalib' in mcainfo:
ddict['MCA a'] = mcainfo['McaCalib'][0]
ddict['MCA b'] = mcainfo['McaCalib'][1]
ddict['MCA c'] = mcainfo['McaCalib'][2]
else:
ddict = mcainfo
ddict['Title'] = title
edfname = os.path.join(
self.outputdir,
title.replace(" ", "_") + ".edf")
edfout = EdfFile.EdfFile(edfname)
edfout.WriteImage(ddict, image, Append=0)
counter = 0
else:
counter += 1
self.onEnd()
def loadFileList(self, filelist, fileindex=0):
if type(filelist) == type(''): filelist = [filelist]
self.__keyList = []
self.sourceName = filelist
self.__indexedStack = True
self.sourceType = SOURCE_TYPE
self.info = {}
self.nbFiles = len(filelist)
#read first edf file
#get information
tempEdf = EdfFileDataSource.EdfFileDataSource(filelist[0])
keylist = tempEdf.getSourceInfo()['KeyList']
nImages = len(keylist)
dataObject = tempEdf.getDataObject(keylist[0])
self.info.update(dataObject.info)
if len(dataObject.data.shape) == 3:
#this is already a stack
self.data = dataObject.data
self.__nFiles = 1
self.__nImagesPerFile = nImages
shape = self.data.shape
for i in range(len(shape)):
key = 'Dim_%d' % (i + 1, )
self.info[key] = shape[i]
self.info["SourceType"] = SOURCE_TYPE
self.info["SourceName"] = filelist[0]
self.info["Size"] = 1
self.info["NumberOfFiles"] = 1
self.info["FileIndex"] = fileindex
return
arrRet = dataObject.data
if self.__dtype is None:
self.__dtype = arrRet.dtype
self.onBegin(self.nbFiles)
singleImageShape = arrRet.shape
if (fileindex == 2) or (self.__imageStack):
self.__imageStack = True
if len(singleImageShape) == 1:
#single line
#be ready for specfile stack?
self.onEnd()
raise IOError("Not implemented yet")
self.data = numpy.zeros(
(arrRet.shape[0], nImages, self.nbFiles), self.__dtype)
self.incrProgressBar = 0
for tempEdfFileName in filelist:
tempEdf = EdfFile.EdfFile(tempEdfFileName, 'rb')
for i in range(nImages):
pieceOfStack = tempEdf.GetData(i)
self.data[:, i, self.incrProgressBar] = pieceOfStack[:]
self.incrProgressBar += 1
self.onProgress(self.incrProgressBar)
self.onEnd()
else:
if nImages > 1:
#this is not the common case
#should I try to convert it to a standard one
#using a 3D matrix or keep as 4D matrix?
if self.nbFiles > 1:
raise IOError(\
"Multiple files with multiple images implemented yet")
self.data = numpy.zeros((arrRet.shape[0], arrRet.shape[1],
nImages * self.nbFiles),
self.__dtype)
self.incrProgressBar = 0
for tempEdfFileName in filelist:
tempEdf = EdfFile.EdfFile(tempEdfFileName, 'rb')
for i in range(nImages):
pieceOfStack = tempEdf.GetData(i)
self.data[:,:,
nImages*self.incrProgressBar+i] = \
pieceOfStack[:,:]
self.incrProgressBar += 1
else:
#this is the common case
try:
# calculate needed megabytes
if self.__dtype == numpy.float:
bytefactor = 8
else:
bytefactor = 4
needed_ = self.nbFiles * \
arrRet.shape[0] *\
arrRet.shape[1] * bytefactor
physicalMemory = PhysicalMemory.getPhysicalMemoryOrNone(
)
if physicalMemory is not None:
# spare 5% or memory
if physicalMemory < (1.05 * needed_):
raise MemoryError(
"Not enough physical memory available")
self.data = numpy.zeros(
(arrRet.shape[0], arrRet.shape[1], self.nbFiles),
self.__dtype)
self.incrProgressBar = 0
for tempEdfFileName in filelist:
tempEdf = EdfFile.EdfFile(tempEdfFileName, 'rb')
pieceOfStack = tempEdf.GetData(0)
self.data[:, :,
self.incrProgressBar] = pieceOfStack
self.incrProgressBar += 1
self.onProgress(self.incrProgressBar)
except (MemoryError, ValueError):
hdf5done = False
if HDF5 and (('PyMcaQt' in sys.modules) or\
('PyMca.PyMcaQt' in sys.modules)):
from PyMca import PyMcaQt as qt
from PyMca import ArraySave
msg = qt.QMessageBox.information(
None, "Memory error\n",
"Do you want to convert your data to HDF5?\n",
qt.QMessageBox.Yes, qt.QMessageBox.No)
if msg != qt.QMessageBox.No:
hdf5file = qt.QFileDialog.getSaveFileName(
None, "Please select output file name",
os.path.dirname(filelist[0]),
"HDF5 files *.h5")
if not len(hdf5file):
raise IOError("Invalid output file")
hdf5file = qt.safe_str(hdf5file)
if not hdf5file.endswith(".h5"):
hdf5file += ".h5"
hdf, self.data = ArraySave.getHDF5FileInstanceAndBuffer(
hdf5file, (self.nbFiles, arrRet.shape[0],
arrRet.shape[1]))
self.incrProgressBar = 0
for tempEdfFileName in filelist:
tempEdf = EdfFile.EdfFile(
tempEdfFileName, 'rb')
pieceOfStack = tempEdf.GetData(0)
self.data[
self.
incrProgressBar, :, :] = pieceOfStack[:, :]
hdf.flush()
self.incrProgressBar += 1
self.onProgress(self.incrProgressBar)
hdf5done = True
if not hdf5done:
for i in range(3):
print("\7")
samplingStep = None
i = 2
while samplingStep is None:
print(
"**************************************************"
)
print(
" Memory error!, attempting %dx%d sampling reduction "
) % (i, i)
print(
"**************************************************"
)
s1, s2 = arrRet[::i, ::i].shape
try:
self.data = numpy.zeros(
(s1, s2, self.nbFiles), self.__dtype)
samplingStep = i
except:
i += 1
self.incrProgressBar = 0
for tempEdfFileName in filelist:
tempEdf = EdfFile.EdfFile(
tempEdfFileName, 'rb')
pieceOfStack = tempEdf.GetData(0)
self.data[:, :, self.
incrProgressBar] = pieceOfStack[::
samplingStep, ::
samplingStep]
self.incrProgressBar += 1
self.onProgress(self.incrProgressBar)
self.onEnd()
else:
self.__imageStack = False
if len(singleImageShape) == 1:
#single line
#be ready for specfile stack?
raise IOError("Not implemented yet")
self.data = numpy.zeros(
(self.nbFiles, arrRet.shape[0], nImages), self.__dtype)
self.incrProgressBar = 0
for tempEdfFileName in filelist:
tempEdf = EdfFile.EdfFile(tempEdfFileName, 'rb')
for i in range(nImages):
pieceOfStack = tempEdf.GetData(i)
self.data[self.incrProgressBar, :, i] = pieceOfStack[:]
self.incrProgressBar += 1
self.onProgress(self.incrProgressBar)
self.onEnd()
else:
if nImages > 1:
#this is not the common case
#should I try to convert it to a standard one
#using a 3D matrix or kepp as 4D matrix?
if self.nbFiles > 1:
if (arrRet.shape[0] > 1) and\
(arrRet.shape[1] > 1):
raise IOError(\
"Multiple files with multiple images not implemented yet")
elif arrRet.shape[0] == 1:
self.data = numpy.zeros(
(self.nbFiles, arrRet.shape[0] * nImages,
arrRet.shape[1]), self.__dtype)
self.incrProgressBar = 0
for tempEdfFileName in filelist:
tempEdf = EdfFile.EdfFile(
tempEdfFileName, 'rb')
for i in range(nImages):
pieceOfStack = tempEdf.GetData(i)
self.data[self.incrProgressBar, i,:] = \
pieceOfStack[:,:]
self.incrProgressBar += 1
self.onProgress(self.incrProgressBar)
elif arrRet.shape[1] == 1:
self.data = numpy.zeros(
(self.nbFiles, arrRet.shape[1] * nImages,
arrRet.shape[0]), self.__dtype)
self.incrProgressBar = 0
for tempEdfFileName in filelist:
tempEdf = EdfFile.EdfFile(
tempEdfFileName, 'rb')
for i in range(nImages):
pieceOfStack = tempEdf.GetData(i)
self.data[self.incrProgressBar, i,:] = \
pieceOfStack[:,:]
self.incrProgressBar += 1
self.onProgress(self.incrProgressBar)
else:
self.data = numpy.zeros(
(nImages * self.nbFiles, arrRet.shape[0],
arrRet.shape[1]), self.__dtype)
self.incrProgressBar = 0
for tempEdfFileName in filelist:
tempEdf = EdfFile.EdfFile(tempEdfFileName, 'rb')
for i in range(nImages):
pieceOfStack = tempEdf.GetData(i)
self.data[nImages * self.incrProgressBar +
i, :, :] = pieceOfStack[:, :]
self.incrProgressBar += 1
self.onProgress(self.incrProgressBar)
self.onEnd()
else:
if fileindex == 1:
try:
self.data = numpy.zeros(
(arrRet.shape[0], self.nbFiles,
arrRet.shape[1]), self.__dtype)
except:
try:
self.data = numpy.zeros(
(arrRet.shape[0], self.nbFiles,
arrRet.shape[1]), numpy.float32)
except:
self.data = numpy.zeros(
(arrRet.shape[0], self.nbFiles,
arrRet.shape[1]), numpy.int16)
else:
try:
# calculate needed megabytes
if self.__dtype == numpy.float:
bytefactor = 8
else:
bytefactor = 4
needed_ = self.nbFiles * \
arrRet.shape[0] *\
arrRet.shape[1] * 4
physicalMemory = PhysicalMemory.getPhysicalMemoryOrNone(
)
if physicalMemory is not None:
# spare 5% of memory
if physicalMemory < (1.05 * needed_):
raise MemoryError(
"Not enough physical memory available")
self.data = numpy.zeros(
(self.nbFiles, arrRet.shape[0],
arrRet.shape[1]), self.__dtype)
except:
try:
needed_ = self.nbFiles * \
arrRet.shape[0] *\
arrRet.shape[1] * 4
physicalMemory = PhysicalMemory.getPhysicalMemoryOrNone(
)
if physicalMemory is not None:
# spare 5 % of memory
if physicalMemory < (1.05 * needed_):
raise MemoryError(
"Not enough physical memory available"
)
self.data = numpy.zeros(
(self.nbFiles, arrRet.shape[0],
arrRet.shape[1]), numpy.float32)
except (MemoryError, ValueError):
text = "Memory Error: Attempt subsampling or convert to HDF5"
if HDF5 and (('PyMcaQt' in sys.modules) or\
('PyMca.PyMcaQt' in sys.modules)):
from PyMca import PyMcaQt as qt
from PyMca import ArraySave
msg = qt.QMessageBox.information(
None, "Memory error\n",
"Do you want to convert your data to HDF5?\n",
qt.QMessageBox.Yes, qt.QMessageBox.No)
if msg == qt.QMessageBox.No:
raise MemoryError(text)
hdf5file = qt.QFileDialog.getSaveFileName(
None, "Please select output file name",
os.path.dirname(filelist[0]),
"HDF5 files *.h5")
if not len(hdf5file):
raise IOError(\
"Invalid output file")
hdf5file = qt.safe_str(hdf5file)
if not hdf5file.endswith(".h5"):
hdf5file += ".h5"
hdf, self.data = ArraySave.getHDF5FileInstanceAndBuffer(
hdf5file,
(self.nbFiles, arrRet.shape[0],
arrRet.shape[1]))
else:
raise MemoryError("Memory Error")
self.incrProgressBar = 0
if fileindex == 1:
for tempEdfFileName in filelist:
tempEdf = EdfFile.EdfFile(tempEdfFileName, 'rb')
pieceOfStack = tempEdf.GetData(0)
self.data[:, self.
incrProgressBar, :] = pieceOfStack[:, :]
self.incrProgressBar += 1
self.onProgress(self.incrProgressBar)
else:
# test for ID24 map
ID24 = False
if "_sample_" in filelist[0]:
i0StartFile = filelist[0].replace(
"_sample_", "_I0start_")
if os.path.exists(i0StartFile):
ID24 = True
id24idx = 0
i0Start = EdfFile.EdfFile(
i0StartFile,
'rb').GetData(0).astype(numpy.float)
i0EndFile = filelist[0].replace(
"_sample_", "_I0end_")
i0Slope = 0.0
if os.path.exists(i0EndFile):
i0End = EdfFile.EdfFile(i0EndFile,
'rb').GetData(0)
i0Slope = (i0End - i0Start) / len(filelist)
for tempEdfFileName in filelist:
tempEdf = EdfFile.EdfFile(tempEdfFileName, 'rb')
if ID24:
pieceOfStack = -numpy.log(
tempEdf.GetData(0) /
(i0Start[0, :] + id24idx * i0Slope))
pieceOfStack[numpy.isfinite(pieceOfStack) ==
False] = 1
id24idx += 1
else:
pieceOfStack = tempEdf.GetData(0)
try:
self.data[
self.
incrProgressBar, :, :] = pieceOfStack[:, :]
except:
if pieceOfStack.shape[1] != arrRet.shape[1]:
print(" ERROR on file %s" %
tempEdfFileName)
print(
" DIM 1 error Assuming missing data were at the end!!!"
)
if pieceOfStack.shape[0] != arrRet.shape[0]:
print(" ERROR on file %s" %
tempEdfFileName)
print(
" DIM 0 error Assuming missing data were at the end!!!"
)
self.data[self.incrProgressBar,\
:pieceOfStack.shape[0],\
:pieceOfStack.shape[1]] = pieceOfStack[:,:]
self.incrProgressBar += 1
self.onProgress(self.incrProgressBar)
self.onEnd()
self.__nFiles = self.incrProgressBar
self.__nImagesPerFile = nImages
shape = self.data.shape
for i in range(len(shape)):
key = 'Dim_%d' % (i + 1, )
self.info[key] = shape[i]
if not isinstance(self.data, numpy.ndarray):
hdf.flush()
self.info["SourceType"] = "HDF5Stack1D"
if self.__imageStack:
self.info["McaIndex"] = 0
self.info["FileIndex"] = 1
else:
self.info["McaIndex"] = 2
self.info["FileIndex"] = 0
self.info["SourceName"] = [hdf5file]
self.info["NumberOfFiles"] = 1
self.info["Size"] = 1
else:
self.info["SourceType"] = SOURCE_TYPE
self.info["FileIndex"] = fileindex
self.info["SourceName"] = self.sourceName
self.info["NumberOfFiles"] = self.__nFiles * 1
self.info["Size"] = self.__nFiles * self.__nImagesPerFile
return gonio
def getDetector(self):
det = {}
for (name, value) in zip(self.detectorHead, self.detectorValue):
det[name] = value
return det
def getFile(self):
ffile = {}
for (head, value) in zip(self.fileHead, self.fileValue):
ffile[head[0]] = value
return ffile
def getDataset(self):
return self.datasetValue
if __name__ == "__main__":
import os
from PyMca import EdfFile
#fd = open('Cu_ZnO_20289.mccd', 'rb')
filename = sys.argv[1]
mccd = MarCCD(filename)
edfFile = filename + ".edf"
if os.path.exists(edfFile):
os.remove(edfFile)
edf = EdfFile.EdfFile(edfFile)
edf.WriteImage(mccd.getInfo(), mccd.getData())
edf = None
def refresh(self):
self._sourceObjectList = []
for name in self.__sourceNameList:
self._sourceObjectList.append(
EdfFile.EdfFile(name, access='rb', fastedf=self._fastedf))
self.__lastKeyInfo = {}
ddict['roi_min'] = ddict0['arguments'][1]
ddict['roi_max'] = ddict0['arguments'][2]
ddict['smoothing'] = ddict0['arguments'][3]
self.parametersWidget.setParameters(ddict)
else:
self.parametersWidget.setParameters(ddict0)
if __name__ == "__main__":
import numpy
app = qt.QApplication([])
if 0:
noise = numpy.random.randn(1000.)
y = numpy.arange(1000.)
w = SNIPDialog(None, y + numpy.sqrt(y) * noise)
elif len(sys.argv) > 1:
from PyMca import EdfFile
edf = EdfFile.EdfFile(sys.argv[1])
data = edf.GetData(0)
w = SNIPDialog(None, data)
else:
x, y = numpy.ogrid[0:200:200j, 0:200:200j]
data = 50 * numpy.exp(-(x-64)*(x-64)/20.) +\
50 * numpy.exp(-(y-128)*(y-128)/20.) +\
100 * numpy.exp(-(1./20) * ((x-64)*(x-64) + (y-128)*(y-128)))
w = SNIPDialog(None, data)
w.show()
ret = w.exec_()
if ret:
print(w.getParameters())
def get_images(self, scan, first, last, dry_run=False):
if self.config.background:
if not os.path.exists(self.config.background):
raise errors.FileError(
'could not find background file {0}'.format(
self.config.background))
if dry_run:
yield
else:
edf = EdfFile.EdfFile(self.config.background)
for i in range(first, last + 1):
self.dbg_pointno = i
yield edf.GetData(0)
else:
if self.is_zap(scan):
scanheaderC = scan.header('C')
zapscanno = int(
scanheaderC[2].split(' ')[-1]
) # is different from scanno should be changed in spec!
try:
uccdtagline = scanheaderC[0]
UCCD = os.path.split(uccdtagline.split()[-1])
except:
print(
'warning: UCCD tag not found, use imagefolder for proper file specification'
)
UCCD = []
pattern = self._get_pattern(UCCD)
matches = self.find_edfs(pattern, zapscanno)
if 0 not in matches:
raise errors.FileError(
'could not find matching edf for zapscannumber {0} using pattern {1}'
.format(zapscanno, pattern))
if dry_run:
yield
else:
edf = EdfFile.EdfFile(matches[0])
for i in range(first, last + 1):
self.dbg_pointno = i
yield edf.GetData(i)
else:
try:
uccdtagline = scan.header('UCCD')[0]
UCCD = os.path.split(
os.path.dirname(uccdtagline.split()[-1]))
except:
print(
'warning: UCCD tag not found, use imagefolder for proper file specification'
)
UCCD = []
pattern = self._get_pattern(UCCD)
matches = self.find_edfs(pattern, scan.number())
if set(range(first, last + 1)) > set(matches.keys()):
raise errors.FileError(
"incorrect number of matches for scan {0} using pattern {1}"
.format(scan.number(), pattern))
if dry_run:
yield
else:
for i in range(first, last + 1):
self.dbg_pointno = i
edf = EdfFile.EdfFile(matches[i])
yield edf.GetData(0)
def buildOutput(self, inputdir=None, outputdir=None, delete=None):
if inputdir is None:inputdir = self.inputDir
if inputdir is None:inputdir = os.getcwd()
if outputdir is None: outputdir = self.outputDir
if outputdir is None: outputdir = inputdir
if delete is None:
if outputdir == inputdir:
delete = True
if DEBUG:
print("delete option = ", delete)
allfiles = os.listdir(inputdir)
partialedflist = []
partialdatlist = []
partialconlist = []
for filename in allfiles:
if filename.endswith('000000_partial.edf'):partialedflist.append(filename)
elif filename.endswith('000000_partial.dat'):partialdatlist.append(filename)
elif filename.endswith('000000_partial_concentrations.txt'):partialconlist.append(filename)
#IMAGES
edfoutlist = []
for filename in partialedflist:
if DEBUG:
print("Dealing with filename %s" % filename)
edflist = self.getIndexedFileList(os.path.join(inputdir, filename))
i = 0
for edfname in edflist:
edf = EdfFile.EdfFile(edfname, access='rb', fastedf = 0)
nImages = edf.GetNumImages()
#get always the last image
data0 = edf.GetData(nImages-1)
data0[data0<0] = 0
if i == 0:
header = edf.GetHeader(0)
data = data0.copy()
else:
data += data0
del edf
i += 1
edfname = filename.replace('_000000_partial.edf',".edf")
edfoutname = os.path.join(outputdir, edfname)
if DEBUG:
print("Dealing with output filename %s" % edfoutname)
if os.path.exists(edfoutname):
if DEBUG:
print("Output file already exists, trying to delete it")
os.remove(edfoutname)
edfout = EdfFile.EdfFile(edfoutname, access="wb")
edfout.WriteImage (header , data, Append=0)
del edfout
edfoutlist.append(edfoutname)
if delete:
for filename in edflist:
try:
os.remove(filename)
except:
print("Cannot delete file %s" % filename)
#DAT IMAGES
datoutlist = []
for filename in partialdatlist:
edflist = self.getIndexedFileList(os.path.join(inputdir, filename))
first = True
for edfname in edflist:
f = open(edfname)
lines = f.readlines()
f.close()
j = 1
while (not len( lines[-j].replace("\n",""))):
j += 1
if first:
first = False
labels = lines[0].replace("\n","").split(" ")
nlabels = len(labels)
nrows = len(lines) - j
data = numpy.zeros((nrows, nlabels), numpy.double)
inputdata = numpy.zeros((nrows, nlabels), numpy.double)
chisqIndex = labels.index('chisq')
for i in range(nrows):
inputdata[i, :] = [float(x) for x in lines[i+1].split()]
if inputdata[i, chisqIndex] < 0.0:
inputdata[i, chisqIndex] = 0.0
data += inputdata
outfilename = os.path.join(outputdir, filename.replace("_000000_partial",""))
if os.path.exists(outfilename):
os.remove(outfilename)
outfile=open(outfilename,'w+')
outfile.write('%s' % lines[0])
line=""
for row in range(nrows):
#line = "%d" % inputdata[row, 0]
for col in range(nlabels):
if col == 0 : line += "%d" % inputdata[row, col]
elif col == 1 : line += " %d" % inputdata[row, col]
else: line += " %g" % data[row, col]
line += "\n"
outfile.write("%s" % line)
line =""
outfile.write("\n")
outfile.close()
datoutlist.append(outfilename)
if delete:
for filename in edflist:
os.remove(filename)
#CONCENTRATIONS
outconlist = []
for filename in partialconlist:
edflist = self.getIndexedFileList(os.path.join(inputdir, filename))
i = 0
for edfname in edflist:
edf = open(edfname, 'rb')
if i == 0:
outfilename = os.path.join(outputdir, filename.replace("_000000_partial",""))
if os.path.exists(outfilename):
os.remove(outfilename)
outfile = open(outfilename,'wb')
lines = edf.readlines()
for line in lines:
outfile.write(line)
edf.close()
i += 1
outfile.close()
outconlist.append(outfilename)
if delete:
for filename in edflist:
os.remove(filename)
return edfoutlist, datoutlist, outconlist
from PyMca import EDFStack
from PyMca import EdfFile
import os
import sys
import time
inputfile = "D:\DATA\COTTE\ch09\ch09__mca_0005_0000_0000.edf"
if len(sys.argv) > 1:
inputfile = sys.argv[1]
print(inputfile)
elif os.path.exists(inputfile):
print("Using a default test case")
else:
print("Usage:")
print("python NNMAModule.py indexed_edf_stack")
sys.exit(0)
stack = EDFStack.EDFStack(inputfile)
r0, c0, n0 = stack.data.shape
ncomponents = 10
outfile = os.path.basename(inputfile) + "ICA.edf"
e0 = time.time()
images, eigenvalues, eigenvectors = nnma(stack.data,
ncomponents,
binning=1)
print("elapsed = %f" % (time.time() - e0))
if os.path.exists(outfile):
os.remove(outfile)
f = EdfFile.EdfFile(outfile)
for i in range(ncomponents):
f.WriteImage({}, images[i, :])
sys.exit(0)
import h5py
f = h5py.File(inputfile, access='r')
stack = DataObject.DataObject()
stack.data = f['/data/NXdata/data']
print("PCA Calculation")
images, eigenvalues, eigenvectors = numpyPCA(stack,
index=0,
binning=1)
for i in range(10):
a = images[i]
#a.shape = r, c
print("Eigenvalue %d = %f" % (i, eigenvalues[i]))
fname = "Image%02d.edf" % i
if os.path.exists(fname):
os.remove(fname)
edf = EdfFile.EdfFile(fname, 'wb')
edf.WriteImage({}, a)
edf = None
inputfile = "D:\DATA\COTTE\CH1777\G4_mca_0012_0000_0000.edf"
stack = EDFStack.EDFStack(inputfile, dtype=numpy.float64)
images, eigenvalues, eigenvectors = numpyPCA(stack,
index=-1,
dtype='float64',
force=True,
binning=1)
for i in range(10):
a = images[i]
#a.shape = r, c
print("Eigenvalue %d = %f" % (i, eigenvalues[i]))
fname = "Image%02d.edf" % (i + 10)
if os.path.exists(fname):