def _checkOutputs(self, outputs, random=False, errorthreshold=0.001):
""" Check that all output files are produced
and are equivalent to the ones in goldStandard folder.
"""
import xmippLib
for out in outputs:
outFile = os.path.join(self._testDir, self.outputDir, out)
fileGoldStd = os.path.join(self.goldDir, out)
# Check the expect output file was produced
msg = "Missing expected output file:\n output: %s" % outFile
self.assertTrue(os.path.exists(outFile), red(msg))
if random:
print(yellow("WARNING: %s was created using a random seed, check skipped..." % outFile))
else:
fnGoldStd = xmippLib.FileName(fileGoldStd)
if fnGoldStd.isImage():
im1 = xmippLib.Image(fileGoldStd)
im2 = xmippLib.Image(outFile)
msg = "Images are not equal (+-%f):\n output: %s\n gold: %s" % \
(errorthreshold, outFile, fileGoldStd)
self.assertTrue(im1.equal(im2, errorthreshold), red(msg))
elif fnGoldStd.isMetaData():
msg = "MetaDatas are not equal:\n output: %s\n gold: %s" % (outFile, fileGoldStd)
self.assertTrue(xmippLib.compareTwoMetadataFiles(outFile, fileGoldStd), red(msg))
else:
msg = "Files are not equal:\n output: %s\n gold: %s" % (outFile, fileGoldStd)
self.assertTrue(xmippLib.compareTwoFiles(outFile, fileGoldStd, 0), red(msg))
def computeCorr(self, vol1, vol2, i, j):
vol = xmippLib.Image(vol1)
defVol = xmippLib.Image(vol2)
corr = vol.correlation(defVol)
corr = 1 - corr
outFile = self._getExtraPath('Pair_%d_%d_correlation.txt' % (i, j))
with open(outFile, 'w') as f:
f.write('%f' % corr)
def convertInputStep(self, volName):
# Read volume and convert to DOUBLE
self.vol = xmippLib.Image(volName)
self.vol.convert2DataType(xmippLib.DT_DOUBLE)
# Mask volume if needed
if self.refMask.get() is not None:
maskName = getImageLocation(self.refMask.get())
self.mask = xmippLib.Image(maskName)
self.mask.convert2DataType(xmippLib.DT_DOUBLE)
self.vol.inplaceMultiply(self.mask)
padding = 2
maxFreq = 0.5
splineDegree = 3
###
self.fourierProjectVol = xmippLib.FourierProjector(self.vol, padding, maxFreq, splineDegree)
###
partSet = self.inputParticles.get()
nPart = len(partSet)
numberOfTasks = min(nPart, max(self.numberOfThreads.get()-1, 1))
taskSize = nPart / numberOfTasks
md = xmippLib.MetaData()
# Convert angles and shifts from volume system of coordinates to
# projection system of coordinates
mdCount = 0
for index, part in enumerate(partSet):
objId = md.addObject()
imgRow = XmippMdRow()
particleToRow(part, imgRow)
shifts, angles = geometryFromMatrix(part.getTransform().getMatrix(), True)
imgRow.setValue(xmippLib.MDL_SHIFT_X,-shifts[0])
imgRow.setValue(xmippLib.MDL_SHIFT_Y,-shifts[1])
imgRow.setValue(xmippLib.MDL_SHIFT_Z,0.)
imgRow.setValue(xmippLib.MDL_ANGLE_ROT,angles[0])
imgRow.setValue(xmippLib.MDL_ANGLE_TILT,angles[1])
imgRow.setValue(xmippLib.MDL_ANGLE_PSI,angles[2])
imgRow.writeToMd(md, objId)
# Write a new metadata every taskSize number of elements
# except in the last chunk where we want to add also the
# remainder and the condition is the last element
if ((index % taskSize == taskSize-1 and
mdCount < numberOfTasks-1) or index == nPart-1):
md.write(self._getInputParticlesSubsetFn(mdCount))
md.clear()
mdCount += 1
x, y, _ = partSet.getDim()
xmippLib.createEmptyFile(self._getProjGalleryFn(), x, y, 1, nPart)
def __init__(self, volfile, **kwargs):
print("on chimera projection client")
ChimeraAngDistClient.__init__(self, volfile, **kwargs)
self.projection = xmippLib.Image()
self.projection.setDataType(md.DT_DOUBLE)
# 0.5 -> Niquiest frequency
# 2 -> bspline interpolation
size = self.kwargs.get('size', None)
defaultSize = self.xdim if self.xdim > 128 else 128
self.size = size if size else defaultSize
paddingFactor = self.kwargs.get('paddingFactor', 1)
maxFreq = self.kwargs.get('maxFreq', 0.5)
splineDegree = self.kwargs.get('splineDegree', 3)
self.fourierprojector = xmippLib.FourierProjector(
self.image, paddingFactor, maxFreq, splineDegree)
self.fourierprojector.projectVolume(self.projection, 0, 0, 0)
self.showjPort = self.kwargs.get('showjPort', None)
self.iw = ImageWindow(filename=os.path.basename(volfile),
image=self.projection,
dim=self.size,
label="Projection")
self.iw.updateData(flipud(self.projection.getData()))
if self.showjPort:
self.showjThread = Thread(name="ChimeraProjClient",
target=self.listenShowJ)
self.showjThread.daemon = True
self.showjThread.start()
self.iw.root.protocol("WM_DELETE_WINDOW", self.exitClient)
self.iw.show()
def run(self):
numberOfThreads = self.getIntParam('-t')
micrographsPath = self.getParam('-i')
coordsPath = self.getParam('-c')
boxSize = self.getIntParam('-s')
numberToPick = self.getIntParam('-n')
outDir = self.getParam('-o')
micsBaseToFullName = {}
for micName in os.listdir(micrographsPath):
if micName.endswith(".mrc") or micName.endswith(".tif"):
baseName = os.path.basename(micName).split(".")[0]
micsBaseToFullName[baseName] = os.path.join(
micrographsPath, micName)
I = xmippLib.Image()
I.read(micsBaseToFullName[baseName])
micShape = I.getData().shape
argsList = []
for posName in os.listdir(coordsPath):
if posName.endswith(".pos"):
baseName = os.path.basename(posName).split(".")[0]
micName = micsBaseToFullName[baseName]
posName = os.path.join(coordsPath, posName)
argsList.append((baseName, micName, posName, micShape, outDir,
numberToPick, boxSize))
Parallel(n_jobs=numberOfThreads, backend="multiprocessing",
verbose=1)(delayed(pickNoiseOneMic, check_pickle=False)(*arg)
for arg in argsList)
def extractTrainData(self, path, label, norm=-1):
metadata = xmippLib.MetaData(path)
Image = []
I = xmippLib.Image()
cont = 0
for itemId in metadata:
fn = metadata.getValue(label, itemId)
I.read(fn)
Imresize = I.getData()
if norm == -1:
Imnormalize = 2 * (Imresize - np.min(Imresize)) / (
np.max(Imresize) - np.min(Imresize)) - 1
elif norm == 0:
Imnormalize = Imresize
elif norm == 1:
Imnormalize = (Imresize - np.min(Imresize)) / (
np.max(Imresize) - np.min(Imresize))
else:
Imnormalize = (Imresize - np.mean(Imresize)) / np.std(Imresize)
Image.append(Imnormalize)
if cont > 3000:
break
cont += 1
Image = np.array(Image).astype('float')
Image = Image.reshape(len(Image), Image.shape[1], Image.shape[2], 1)
return Image
def getDataIfFname(fnameOrNumpy):
if isinstance(fnameOrNumpy, str):
return xmippLib.Image(fnameOrNumpy).getData()
elif isinstance(fnameOrNumpy, np.ndarray):
return fnameOrNumpy
else:
raise ValueError("Error, input must be either file name or numpy array")
def __init__(self):
# Now it will use Xmipp image library
# to read and write most of formats, in the future
# if we want to be independent of Xmipp, we should have
# our own image library
self._img = xmippLib.Image()
self._imgClass = xmippLib.Image
def normalize(self, what):
# Get overall minimum and maximum
V = xmippLib.Image()
minAll = 1e38
maxAll = -1e38
for prot in self.inputRuns:
protId = prot.get().getObjId()
protDir = prot.get()._getPath('')
fnVol = os.path.join(protDir, "extra", "result_%s.vol" % what)
V.read(fnVol)
_, _, minVal, maxVal = V.computeStats()
minAll = min(minAll, minVal)
maxAll = max(maxAll, maxVal)
# Write the Chimera file
for prot in self.inputRuns:
protId = prot.get().getObjId()
protDir = prot.get()._getPath('')
fnRoot = os.path.relpath(os.path.join(protDir, "extra", "result"),
self._getPath(''))
scriptFile = self._getPath('%d_result_%s_chimera.cmd' %
(protId, what))
fhCmd = open(scriptFile, 'w')
fhCmd.write("open %s\n" % (fnRoot + "_final.vol"))
fhCmd.write("open %s\n" % (fnRoot + "_%s.vol" % what))
fhCmd.write("vol #1 hide\n")
fhCmd.write(
"scolor #0 volume #1 cmap rainbow cmapRange %f,%f reverseColors True\n"
% (minAll, maxAll))
fhCmd.close()
def runReconstructionStep(self, iterN, refN, program, method, args, suffix,
**kwargs):
#if input metadata is empty create a Blanck image
reconsXmd = 'reconstructionXmd' + suffix
reconsVol = 'reconstructedFileNamesIters' + suffix
mdFn = self._getFileName(reconsXmd, iter=iterN, ref=refN)
volFn = self._getFileName(reconsVol, iter=iterN, ref=refN)
maskFn = self._getFileName('maskedFileNamesIters', iter=iterN, ref=refN)
if method == "art":
mpi = 1
threads = 1
else:
mpi = self.numberOfMpi.get()
threads = self.numberOfThreads.get()
args += ' --thr %d' % threads
if isMdEmpty(mdFn):
img = xmippLib.Image()
img.read(maskFn)
#(x,y,z,n) = img.getDimensions()
self._log.warning(
"Metadata '%s' is empty. \n Creating a random volume file '%s'" %
(mdFn, volFn))
#createEmptyFile(ReconstructedVolume,x,y,z,n)
img.initRandom()
img.write(volFn)
else:
if method == 'fourier':
if self._fourierMaxFrequencyOfInterest[iterN] == -1:
fourierMaxFrequencyOfInterest = self._getFourierMaxFrequencyOfInterest(
iterN - 1, refN)
fourierMaxFrequencyOfInterest = self.resolSam / fourierMaxFrequencyOfInterest + self._constantToAddToMaxReconstructionFrequency[
iterN]
if fourierMaxFrequencyOfInterest > 0.5:
fourierMaxFrequencyOfInterest = 0.5
elif fourierMaxFrequencyOfInterest < 0.:
fourierMaxFrequencyOfInterest = 0.001
else:
fourierMaxFrequencyOfInterest = self._fourierMaxFrequencyOfInterest[
iterN]
args += ' --max_resolution %s' % fourierMaxFrequencyOfInterest
if mpi > 1:
args += ' --mpi_job_size %s' % self.mpiJobSize.get()
self._log.info(
'*********************************************************************'
)
self._log.info('* Reconstruct volume using %s' % method)
self.runJob(program,
args,
numberOfMpi=mpi,
numberOfThreads=threads,
**kwargs)
def createVol(self, volume):
vol = xmippLib.Image()
vol.setDataType(xmippLib.DT_FLOAT)
vol.resize(projSize, projSize, projSize)
#vol.initRandom(0., .5, xmippLib.XMIPP_RND_UNIFORM)
vol.initConstant(0.)
for coor in volume:
vol.setPixel(coor[0], coor[1], coor[2], coor[3],
coor[4]) # coor4 is the pixel value
vol.write(self.volBaseFn)
def getImageFromPath(imagePath):
""" Read an image using Xmipp, convert to PIL
and then return as expected by Tk.
"""
import xmippLib
img = xmippLib.Image(imagePath)
imgPIL = getPILImage(img)
from PIL import ImageTk
imgTk = ImageTk.PhotoImage(imgPIL)
return imgTk
def createProjection(self, proj, num, baseName):
img = xmippLib.Image()
img.setDataType(xmippLib.DT_FLOAT)
img.resize(projSize, projSize)
#img.initRandom(0., 1., xmippLib.XMIPP_RND_GAUSSIAN)
img.initConstant(0.)
for coor in proj:
value = img.getPixel(coor[0], coor[1], coor[2], coor[3])
img.setPixel(coor[0], coor[1], coor[2], coor[3],
coor[4] + value) # coor4 is the pixel value
img.write("%d@" % num + baseName)
def predictModel(self):
from keras.models import load_model
metadataPart = xmippLib.MetaData(
self._getExtraPath('resizedParticles.xmd'))
if self.model.get() == ITER_TRAIN:
metadataProj = xmippLib.MetaData(
self._getExtraPath('resizedProjections.xmd'))
img = xmippLib.Image()
dimMetadata = getMdSize(self._getExtraPath('resizedParticles.xmd'))
xmippLib.createEmptyFile(self._getExtraPath('particlesDenoised.stk'),
self.newXdim, self.newXdim, 1, dimMetadata)
mdNewParticles = md.MetaData()
self.groupParticles = 5000
if self.model.get() == ITER_PREDICT:
if self.modelPretrain:
model = self.ownModel.get()._getPath('ModelTrained.h5')
model = load_model(model)
else:
myModelfile = xmipp3.Plugin.getModel('deepDenoising',
'PretrainModel.h5')
model = load_model(myModelfile)
for num in range(1, dimMetadata, self.groupParticles):
self.noisyParticles = self.gan.extractInfoMetadata(
metadataPart, xmippLib.MDL_IMAGE, img, num,
self.groupParticles, -1)
if self.model.get() == ITER_TRAIN:
self.predict = self.gan.predict(self.Generator,
self.noisyParticles)
self.projections = self.gan.extractInfoMetadata(
metadataProj, xmippLib.MDL_IMAGE, img, num,
self.groupParticles, 1)
else:
self.predict = self.gan.predict(model, self.noisyParticles)
self.noisyParticles = self.gan.normalization(
self.noisyParticles, 1)
self.prepareMetadata(mdNewParticles, img, num)
mdNewParticles.write(
'particles@' + self._getExtraPath('particlesDenoised.xmd'),
xmippLib.MD_APPEND)
self.runJob(
"xmipp_transform_normalize", "-i %s --method NewXmipp "
"--background circle %d " %
(self._getExtraPath('particlesDenoised.stk'), self.newXdim / 2))
def createMask(self, _maskName):
vol = xmippLib.Image()
vol.setDataType(xmippLib.DT_FLOAT)
vol.resize(projSize, projSize, projSize)
vol.initConstant(0.0) #ROB: not sure this is needed
halfDim = int(projSize / 2)
maskRadius2 = maskRadius * maskRadius
for i in range(-halfDim, halfDim):
for j in range(-halfDim, halfDim):
for k in range(-halfDim, halfDim):
if (i * i + j * j + k * k) < maskRadius2:
vol.setPixel(0, k + halfDim, i + halfDim, j + halfDim,
1.) # coor4 is the pixel value
vol.write(_maskName)
def createRandomMicStep(self, mic):
time.sleep(self.creationInterval.get())
getEnviron = importFromPlugin('xmipp3', 'Plugin').getEnviron
# create image
img = xmippLib.Image()
img.setDataType(xmippLib.DT_FLOAT)
img.resize(self.xDim, self.yDim)
img.initRandom(0., 1., xmippLib.XMIPP_RND_UNIFORM)
baseFn = self._getExtraPath(self._singleImageFn)
img.write(baseFn)
md1 = xmippLib.MetaData()
md1.setColumnFormat(False)
idctf = md1.addObject()
baseFnCtf = self._getTmpPath("ctf_%d.param" % mic)
baseFnImageCTF = self._getExtraPath("imageCTF_%d.xmp" % mic)
md1.setValue(xmippLib.MDL_CTF_SAMPLING_RATE, 1., idctf)
md1.setValue(xmippLib.MDL_CTF_VOLTAGE, 200., idctf)
defocus = 20000 + 10000 * random.random()
udefocus = defocus + 1000 * random.random()
vdefocus = defocus + 1000 * random.random()
if udefocus < vdefocus:
aux = vdefocus
vdefocus = udefocus
udefocus = aux
md1.setValue(xmippLib.MDL_CTF_DEFOCUSU, udefocus, idctf)
md1.setValue(xmippLib.MDL_CTF_DEFOCUSV, vdefocus, idctf)
md1.setValue(xmippLib.MDL_CTF_DEFOCUS_ANGLE, 180.0 * random.random(),
idctf)
md1.setValue(xmippLib.MDL_CTF_CS, 2., idctf)
md1.setValue(xmippLib.MDL_CTF_Q0, 0.07, idctf)
md1.setValue(xmippLib.MDL_CTF_K, 1., idctf)
md1.write(baseFnCtf)
# apply ctf
args = " -i %s" % baseFn
args += " -o %s" % baseFnImageCTF
args += " -f ctf %s" % baseFnCtf
args += " --sampling %f" % self.samplingRate
self.runJob("xmipp_transform_filter", args, env=getEnviron())
self.dictObj[baseFnImageCTF] = True
self._checkProcessedData()
def getIteratorPredictBatch(self):
batchSize = self.batchSize
xdim, ydim, nChann = self.shape
batchStack = np.zeros((self.batchSize, xdim, ydim, nChann))
batchLabels = np.zeros((batchSize, 2))
I = xmippLib.Image()
n = 0
for dataSetNum in range(len(self.mdListTrue)):
mdTrue = self.mdListTrue[dataSetNum]
for objId in mdTrue:
fnImage = mdTrue.getValue(xmippLib.MDL_IMAGE, objId)
I.read(fnImage)
batchStack[n, ...] = np.expand_dims(I.getData(), -1)
batchLabels[n, 1] = 1
n += 1
if n >= batchSize:
# fig=plt.figure()
# ax=fig.add_subplot(1,1,1)
# ax.imshow(np.squeeze(batchStack[np.random.randint(0,n)]), cmap="Greys")
# fig.suptitle('label==1')
# plt.show()
yield batchStack, batchLabels
n = 0
batchLabels = np.zeros((batchSize, 2))
if not self.mdListFalse is None:
for dataSetNum in range(len(self.mdListFalse)):
mdFalse = self.mdListFalse[dataSetNum]
for objId in mdFalse:
fnImage = mdFalse.getValue(xmippLib.MDL_IMAGE, objId)
I.read(fnImage)
batchStack[n, ...] = np.expand_dims(I.getData(), -1)
batchLabels[n, 0] = 1
n += 1
if n >= batchSize:
# fig=plt.figure()
# ax=fig.add_subplot(1,1,1)
# ax.imshow(np.squeeze(batchStack[np.random.randint(0,n)]), cmap="Greys")
# fig.suptitle('label==0')
# plt.show()
yield batchStack, batchLabels
n = 0
batchLabels = np.zeros((batchSize, 2))
if n > 0:
yield batchStack[:n, ...], batchLabels[:n, ...]
class ImageFileHandler(FileHandler):
_image = xmippLib.Image()
_index = ''
def _getImageString(self, filename):
if isStandardImage(filename):
return "Image file."
x, y, z, n = xmippLib.getImageSize(filename)
objType = 'Image'
dimMsg = "*%(objType)s file*\n dimensions: %(x)d x %(y)d"
expMsg = "Columns x Rows "
if z > 1:
dimMsg += " x %(z)d"
expMsg += " x Slices"
objType = 'Volume'
if n > 1:
dimMsg += " x %(n)d"
expMsg += " x Objects"
objType = 'Stack'
return (dimMsg + "\n" + expMsg) % locals()
def _getImagePreview(self, filename):
dim = 128
if isStandardImage(filename):
self.tkImg = gui.getImage(os.path.abspath(filename),
tkImage=True,
maxheight=dim)
else:
fn = self._index + filename
self.tkImg = gui.getTkImage(self._image, fn, dim)
return self.tkImg
def getFilePreview(self, objFile):
fn = objFile.getPath()
return self._getImagePreview(fn), self._getImageString(fn)
def getFileActions(self, objFile):
from pyworkflow.em.viewers import DataView
fn = objFile.getPath()
return [('Open with Xmipp viewer', lambda: DataView(fn).show(),
Icon.ACTION_VISUALIZE)]
def run(self):
# type: () -> object
img_size = self.getParam('--img_size')
weights = self.getParam('--weights')
micrographs = self.getParam('--micrographs')
output_file = self.getParam('--output')
config = get_image_config(int(img_size))
# Default shifts for cropping the images. TODO: add as arguments?
shift = np.max([config["img_rows"], 1])
shifts = np.repeat(shift, 2)
def to_rgb(x):
return np.repeat(x.reshape(*(config["img_rows"],
config["img_cols"], 1)),
3,
axis=-1)
model = particle_size_cnn(config["input_shape"])
load_weights(model, weights)
predictions = []
with open(micrographs, 'r') as micrographs_file:
for micrograph_path in micrographs_file:
micrograph_path = micrograph_path.rstrip('\n')
img = xmipp.Image(micrograph_path).getData()
img = (img - np.min(img)) / (np.max(img) - np.min(img))
img = (img - np.mean(img)) * 255.0 # Vgg16 is not standardized
Xs = crop(img, (config["img_rows"], config["img_cols"]),
shifts)
# reshape to match the number of channels and add an extra dimension
# to account for the batch size
Xs = [np.expand_dims(to_rgb(x), axis=0) for x in Xs]
micrograph_predictions = [float(model.predict(x)) for x in Xs]
predictions.append(np.median(micrograph_predictions))
self.particle_boxsize = int(np.round(np.median(predictions)))
with open(output_file, 'w') as fp:
fp.write(str(self.particle_boxsize) + '\n')
def projectStep(self, start, end, samplingRate, threadNumber):
# Project
md = xmippLib.MetaData(self._getInputParticlesSubsetFn(threadNumber))
##
projection = xmippLib.Image()
projection.setDataType(xmippLib.DT_DOUBLE)
##
for id in md:
rot = md.getValue(xmippLib.MDL_ANGLE_ROT, id)
tilt = md.getValue(xmippLib.MDL_ANGLE_TILT, id)
psi = md.getValue(xmippLib.MDL_ANGLE_PSI, id)
##projection =self.vol.projectVolumeDouble(rot, tilt, psi)
self.fourierProjectVol.projectVolume(projection, rot, tilt, psi)
##
# Apply CTF
if self.projType == self.CORRECT_NONE:
pass
elif self.projType == self.CORRECT_FULL_CTF:
xmippLib.applyCTF(projection, md, samplingRate, id, False)
elif self.projType == self.CORRECT_PHASE_FLIP:
xmippLib.applyCTF(projection, md, samplingRate, id, True)
else:
raise Exception("ERROR: Unknown projection mode: %d" % self.projType)
# Shift image
projection.applyGeo(md,id,True,False)#onlyapplyshist, wrap
ih = ImageHandler()
expProj = ih.read(md.getValue(xmippLib.MDL_IMAGE, id))
expProj.convert2DataType(xmippLib.DT_DOUBLE)
# Subtract from experimental and write result
projection.resetOrigin()
if self.normalize:
expProj = expProj.adjustAndSubtract(projection)
else:
expProj.inplaceSubtract(projection)
expProj.write( self._getProjGalleryIndexFn(id+start-1))
def _processMovie(self, movie):
movieId = movie.getObjId()
fnMovie = movie.getFileName()
gain = self.inputMovies.get().getGain()
args = "-i %s --oroot %s --iter 1 --singleRef --frameStep %d " \
"--gainImage %s" % \
(fnMovie, self._getPath("movie_%06d" % movieId),
self.frameStep, gain)
if self.useExistingGainImage.get():
args += " --applyGain"
self.runJob("xmipp_movie_estimate_gain", args, numberOfMpi=1)
cleanPath(self._getPath("movie_%06d_correction.xmp" % movieId))
fnSummary = self._getPath("summary.txt")
fnMonitorSummary = self._getPath("summaryForMonitor.txt")
if not os.path.exists(fnSummary):
fhSummary = open(fnSummary, "w")
fnMonitorSummary = open(fnMonitorSummary, "w")
else:
fhSummary = open(fnSummary, "a")
fnMonitorSummary = open(fnMonitorSummary, "a")
fnGain = self._getPath("movie_%06d_gain.xmp" % movieId)
if os.path.exists(fnGain):
G = xmippLib.Image()
G.read(fnGain)
mean, dev, min, max = G.computeStats()
Gnp = G.getData()
p = np.percentile(Gnp, [2.5, 25, 50, 75, 97.5])
fhSummary.write("movie_%06d: mean=%f std=%f [min=%f,max=%f]\n" %
(movieId, mean, dev, min, max))
fhSummary.write(
" 2.5%%=%f 25%%=%f 50%%=%f 75%%=%f 97.5%%=%f\n" %
(p[0], p[1], p[2], p[3], p[4]))
fhSummary.close()
fnMonitorSummary.write("movie_%06d: %f %f %f %f\n" %
(movieId, dev, p[0], p[4], max))
fnMonitorSummary.close()
def __init__(self, filename=None, dim=256, dpi=96, image=None, label=None):
pwgui.Window.__init__(self, minsize=None)
if image is None:
image = xmippLib.Image()
if dim is None:
image.read(filename)
dim, _, _, _ = image.getDimensions()
else:
image.readPreview(filename, dim)
dpi = min(dpi, dim)
self.imagePreview = ImagePreview(self.root, dim, dpi, label, 0)
if image is None and filename is None:
raise Exception("ImageWindow: image or filename should be provided.")
if filename is None:
filename = "No filename"
self.updateImage(image)
self.imagePreview.grid(row=0, column=0)
def run(self):
# type: () -> object
img_size = self.getParam('--img_size')
weights = self.getParam('--weights')
feature_scaler = self.getParam('--feature_scaler')
micrographs = self.getParam('--micrographs')
output_file = self.getParam('--output')
config = get_image_config(int(img_size))
# Default shifts for cropping the images. TODO: add as arguments?
shift = np.max([config["img_rows"], 1])
shifts = np.repeat(shift, 2)
feature_scaler = pickle.load(open(feature_scaler, "rb"))
model = ParticleSizeCnn(config["input_shape"])
conv_base = model.conv_base
model.load_weights(weights)
predictions = []
with open(micrographs, 'r') as micrographs_file:
for micrograph_path in micrographs_file:
micrograph_path = micrograph_path.rstrip('\n')
img = xmipp.Image(micrograph_path).getData()
img = feature_scaler.transform(img)
Xs = crop(img, config["input_shape"], shifts)
# reshape to match the number of channels and add an extra dimension
# to account for the batch size
Xs = [
np.expand_dims(x.reshape(*config["input_shape"]), axis=0)
for x in Xs
]
micrograph_predictions = [float(model.predict(x)) for x in Xs]
predictions.append(np.median(micrograph_predictions))
self.particle_boxsize = int(np.round(np.median(predictions)))
with open(output_file, 'w') as fp:
fp.write(str(self.particle_boxsize) + '\n')
def colectMetadata(self, dictData):
mdList = []
fnamesList_merged = []
weightsList_merged = []
nParticles = 0
shapeParticles = (None, None, 1)
for fnameXMDF in sorted(dictData):
weight = float(dictData[fnameXMDF])
mdObject = xmippLib.MetaData(fnameXMDF)
I = xmippLib.Image()
I.read(
mdObject.getValue(xmippLib.MDL_IMAGE, mdObject.firstObject()))
xdim, ydim = I.getData().shape
imgFnames = mdObject.getColumnValues(xmippLib.MDL_IMAGE)
mdList += [mdObject]
fnamesList_merged += imgFnames
tmpShape = (xdim, ydim, 1)
tmpNumParticles = mdObject.size()
if shapeParticles != (None, None, 1):
assert tmpShape == shapeParticles, "Error, particles of different shapes mixed"
else:
shapeParticles = tmpShape
if weight <= 0:
otherParticlesNum = 0
for fnameXMDF_2 in sorted(dictData):
weight_2 = float(dictData[fnameXMDF_2])
if weight_2 > 0:
otherParticlesNum += xmippLib.MetaData(
fnameXMDF_2).size()
weight = max(1, otherParticlesNum // tmpNumParticles)
weightsList_merged += [weight for elem in imgFnames]
nParticles += tmpNumParticles
print(sorted(dictData))
weightsList_merged = np.array(weightsList_merged, dtype=np.float64)
weightsList_merged = weightsList_merged / weightsList_merged.sum()
return mdList, fnamesList_merged, weightsList_merged, nParticles, shapeParticles
def produceOutput(fnVolInOrNumpy, fnMaskOrNumpy, model, sampling, Y, fnVolOut):
V = getDataIfFname(fnVolInOrNumpy)
Orig = V
M = getDataIfFname(fnMaskOrNumpy)
V = V*0
Zdim, Ydim, Xdim = V.shape
idx = 0
if model==1:
if 2*sampling > 2.5:
minValue=2*sampling
else:
minValue=2.5
if model==2:
if 2*sampling > 1.5:
minValue=2*sampling
else:
minValue=1.5
# FIXME: Refactor these loops to increase the readability
boxDim2 = boxDim//2
for z in range(boxDim2,Zdim-boxDim2):
for y in range(boxDim2,Ydim-boxDim2):
for x in range(boxDim2,Xdim-boxDim2):
if M[z,y,x]>0.15 and Orig[z,y,x]>0.00015:
if ((x+y+z)%2)==0:
if model==1:
if Y[idx]>12.9:
Y[idx]=12.9
if Y[idx]<minValue:
Y[idx]=minValue
if model==2:
if Y[idx]>5.9:
Y[idx]=5.9
if Y[idx]<minValue:
Y[idx]=minValue
V[z,y,x]=Y[idx]
idx=idx+1
###mean
for z in range(boxDim2+1,Zdim-boxDim2):
for y in range(boxDim2+1,Ydim-boxDim2):
for x in range(boxDim2+1,Xdim-boxDim2):
if M[z,y,x]>0.15 and Orig[z,y,x]>0.00015:
if ((x+y+z)%2)!=0:
col=0
ct=0
if V[z+1,y,x]>0:
col+=V[z+1,y,x]
ct+=1
if V[z-1,y,x]>0:
col+=V[z-1,y,x]
ct+=1
if V[z,y+1,x]>0:
col+=V[z,y+1,x]
ct+=1
if V[z,y-1,x]>0:
col+=V[z,y-1,x]
ct+=1
if V[z,y,x+1]>0:
col+=V[z,y,x+1]
ct+=1
if V[z,y,x-1]>0:
col+=V[z,y,x-1]
ct+=1
if ct==0:
V[z,y,x]=0
ct=1
else:
meansum=col/ct
V[z,y,x]=meansum
if fnVolOut is not None:
Vxmipp = xmippLib.Image()
Vxmipp.setData(V)
Vxmipp.write(fnVolOut)
return V
def getDataGenerator( imgsMdXmd, masksMdXmd, xmdEmptyParts=None, augmentData=True, nEpochs=-1, isTrain=True, valFraction=0.1,
batchSize= BATCH_SIZE, doTanhNormalize=True, simulateEmptyParts=True, addMismatch=False, path_size_fraction=None):
if nEpochs<1:
nEpochs= 9999999
mdImgs = xmippLib.MetaData(imgsMdXmd)
mdMasks = xmippLib.MetaData(masksMdXmd)
nImages= int(mdImgs.size())
I= xmippLib.Image()
imgFnames = mdImgs.getColumnValues(xmippLib.MDL_IMAGE)
maskFnames= mdMasks.getColumnValues(xmippLib.MDL_IMAGE)
I.read( imgFnames[0] )
shape= I.getData().shape+ (1,)
if not path_size_fraction is None and 0<path_size_fraction<1:
shape= tuple([int(path_size_fraction*elem) for elem in shape[:-1] ])+(1,)
if not xmdEmptyParts is None:
mdEmpty= xmippLib.MetaData(xmdEmptyParts)
nImages+= int(mdEmpty.size())
emptyFnames= mdEmpty.getColumnValues(xmippLib.MDL_IMAGE)
imgFnames+= emptyFnames
maskFnames+= [None]*len(emptyFnames)
stepsPerEpoch= nImages//batchSize
if augmentData:
augmentFuns= [_random_flip_leftright, _random_flip_updown, _random_90degrees_rotation, _random_rotation ]
if simulateEmptyParts:
augmentFuns+= [generateEmptyParticlesFunction(shape, prob=0.2)]
if addMismatch:
augmentFuns+= [_mismatch_projection]
def augmentBatch( batchX, batchY):
for fun in augmentFuns:
if bool(random.getrandbits(1)):
batchX, batchY= fun(batchX, batchY)
return batchX, batchY
else:
def augmentBatch( batchX, batchY): return batchX, batchY
if valFraction>0:
(imgFnames_train, imgFnames_val, maskFnames_train,
maskFnames_val) = train_test_split(imgFnames, maskFnames, test_size=valFraction, random_state=121)
if isTrain:
imgFnames, maskFnames= imgFnames_train, maskFnames_train
else:
imgFnames, maskFnames= imgFnames_val, maskFnames_val
def readOneImage(fname):
I.read(fname)
return I.getData()
def readImgAndMask(fnImageImg, fnImageMask):
img= normalization( np.expand_dims(readOneImage(fnImageImg), -1), sigmoidInsteadTanh= not doTanhNormalize)
if fnImageMask is None:
mask= -1*np.ones_like(img)
else:
mask= normalization(np.expand_dims(readOneImage(fnImageMask), -1), sigmoidInsteadTanh=not doTanhNormalize)
return img, mask
def extractPatch(img, mask):
halfShape0= shape[0]//2
halfShape0Right= halfShape0 + shape[0]%2
halfShape1= shape[1]//2
halfShape1Right= halfShape1 + shape[1]%2
hpos= random.randint(halfShape0, img.shape[0]-halfShape0Right)
wpos= random.randint(halfShape1, img.shape[1]-halfShape1Right)
img= img[hpos-halfShape0: hpos+halfShape0Right, wpos-halfShape1: wpos+halfShape1Right]
mask= mask[hpos-halfShape0: hpos+halfShape0Right, wpos-halfShape1: wpos+halfShape1Right]
return img, mask
def dataIterator(imgFnames, maskFnames, nBatches=None):
batchStack = np.zeros((batchSize,)+shape )
batchLabels = np.zeros((batchSize,)+shape )
currNBatches=0
for epoch in range(nEpochs):
if isTrain:
imgFnames, maskFnames= shuffle(imgFnames, maskFnames)
n=0
for fnImageImg, fnImageMask in zip(imgFnames, maskFnames):
img, mask= readImgAndMask(fnImageImg, fnImageMask)
if not path_size_fraction is None:
img, mask= extractPatch(img, mask)
# print(img.shape, mask.shape)
batchStack[n,...], batchLabels[n,...]= img, mask
n+=1
if n>=batchSize:
yield augmentBatch(batchStack, batchLabels)
n=0
currNBatches+=1
if nBatches and currNBatches>=nBatches:
break
if n>0:
yield augmentBatch(batchStack[:n,...], batchLabels[:n,...])
return dataIterator(imgFnames, maskFnames), stepsPerEpoch
def loadMic(fname): I = xmippLib.Image() I.read(fname) return I.getData()
def _getOneEpochTrainOrValidation(self,
isTrain_or_validation,
nBatches=None):
batchSize = self.batchSize
xdim, ydim, nChann = self.shape
batchStack = np.zeros((self.batchSize, xdim, ydim, nChann))
batchLabels = np.zeros((batchSize, 2))
I = xmippLib.Image()
n = 0
currNBatches = 0
if isTrain_or_validation == "train":
idxListTrue = np.random.choice(
self.trainingIdsPos,
len(self.trainingIdsPos),
True,
p=self.weightListTrue[self.trainingIdsPos] /
np.sum(self.weightListTrue[self.trainingIdsPos]))
idxListFalse = np.random.choice(
self.trainingIdsNeg,
len(self.trainingIdsNeg),
True,
p=self.weightListFalse[self.trainingIdsNeg] /
np.sum(self.weightListFalse[self.trainingIdsNeg]))
augmentBatch = self.augmentBatch
elif isTrain_or_validation == "validation":
idxListTrue = self.validationIdsPos
idxListFalse = self.validationIdsNeg
augmentBatch = lambda x: x
else:
raise ValueError(
"isTrain_or_validation must be either train or validation")
fnMergedListTrue = (self.fnMergedListTrue[i] for i in idxListTrue)
fnMergedListFalse = (self.fnMergedListFalse[i] for i in idxListFalse)
for fnImageTrue, fnImageFalse in zip(fnMergedListTrue,
fnMergedListFalse):
I.read(fnImageTrue)
batchStack[n, ...] = np.expand_dims(I.getData(), -1)
batchLabels[n, 1] = 1
n += 1
if n >= batchSize:
yield augmentBatch(batchStack), batchLabels
n = 0
batchLabels = np.zeros((batchSize, 2))
currNBatches += 1
if nBatches and currNBatches >= nBatches:
break
I.read(fnImageFalse)
batchStack[n, ...] = np.expand_dims(I.getData(), -1)
batchLabels[n, 0] = 1
n += 1
if n >= batchSize:
yield augmentBatch(batchStack), batchLabels
n = 0
batchLabels = np.zeros((batchSize, 2))
currNBatches += 1
if nBatches and currNBatches >= nBatches:
break
if n > 0:
yield augmentBatch(batchStack[:n, ...]), batchLabels[:n, ...]
def initVolumeData(self):
self.image = xmippLib.Image(self.volfile)
self.image.convert2DataType(md.DT_DOUBLE)
self.xdim, self.ydim, self.zdim, self.n = self.image.getDimensions()
self.vol = self.image.getData()
def writeMic(fname, data): I = xmippLib.Image() I.setData(data) I.write(fname)
