def _execEmanProcess(self, numRun, iterN):
clsFn = self._getFileName("cls", run=numRun, iter=iterN)
classesFn = self._getFileName("classes", run=numRun, iter=iterN)
angles = self._getFileName('angles', iter=iterN)
if not exists(angles) and exists(self._getFileName('clsEven',
run=numRun, iter=iterN)):
proc = eman2.Plugin.createEmanProcess(args='read %s %s %s %s 3d'
% (self._getParticlesStack(), clsFn, classesFn,
self._getBaseName('angles', iter=iterN)),
direc=self._getExtraPath())
proc.wait()
def __getRemoteFile(self, sourceFilePath, targetFilePath, gatewayHosts,
sftp):
"""
Send local file to remote machine
sourceFilePath -- Source file path (/file path/...).
targetFilePath -- Target file path (/file path/...).
sftp -- sftp connection.
"""
log.info("Getting " + sourceFilePath + " to " + targetFilePath)
# Check if file already existsFilePath and it is up to date
existsFilePath = False
if (exists(targetFilePath)):
if (self.__getRemoteSHA1(
sourceFilePath,
self.ssh) == self.__getLocalSHA1(targetFilePath)):
existsFilePath = True
log.info(targetFilePath + " already existed")
if (not existsFilePath):
makeFilePath(targetFilePath)
try:
sftp.get(sourceFilePath, targetFilePath)
except IOError as err:
log.error("Fail getting remote file " + sourceFilePath +
" to local file " + targetFilePath + " - " +
str(err))
raise
def _getIterData(self, it):
data_sqlite = self._getFileName('data_scipion', iter=it)
if not exists(data_sqlite):
iterImgSet = em.SetOfParticles(filename=data_sqlite)
iterImgSet.copyInfo(self._getInputParticles())
self._fillDataFromIter(iterImgSet, it)
iterImgSet.write()
iterImgSet.close()
return data_sqlite
def readCoordinates(mic, fileName, coordsSet):
if exists(fileName):
md = MetaData()
md.readPlain(fileName, 'xcoor ycoor')
for objId in md:
x = md.getValue(MDL_XCOOR, objId)
y = md.getValue(MDL_YCOOR, objId)
coord = Coordinate()
coord.setPosition(x, y)
coord.setMicrograph(mic)
coordsSet.append(coord)
def readCoordinates(mic, fileName, coordsSet):
if exists(fileName):
md = MetaData()
md.readPlain(fileName, 'xcoor ycoor')
for objId in md:
x = md.getValue(MDL_XCOOR, objId)
y = md.getValue(MDL_YCOOR, objId)
coord = Coordinate()
coord.setPosition(x, y)
coord.setMicrograph(mic)
coordsSet.append(coord)
def getMethods(self, output):#output is not used but to overwrite getMethods it is used
msg = ""
configfile = join(self._getExtraPath(), 'config.xmd')
existsConfig = exists(configfile)
if existsConfig:
md = xmipp.MetaData('properties@' + configfile)
configobj = md.firstObject()
particleSize = md.getValue(xmipp.MDL_PICKING_PARTICLE_SIZE, configobj)
manualParticlesSize = md.getValue(xmipp.MDL_PICKING_MANUALPARTICLES_SIZE, configobj)
msg = 'User picked %d particles with a particle size of %d.' % (manualParticlesSize, particleSize)
return msg
def getSummary(self, coordsSet):
summary = []
configfile = join(self._getExtraPath(), 'config.xmd')
if exists(configfile):
md = xmipp.MetaData('properties@' + configfile)
configobj = md.firstObject()
activemic = md.getValue(xmipp.MDL_MICROGRAPH, configobj)
manualParticlesSize = md.getValue(xmipp.MDL_PICKING_MANUALPARTICLES_SIZE, configobj)
particleSize = md.getValue(xmipp.MDL_PICKING_PARTICLE_SIZE, configobj)
summary.append("Particles picked: %d"%manualParticlesSize)
summary.append("Particle size: %d"%particleSize)
summary.append("Last micrograph: " + activemic)
return "\n".join(summary)
def __copyLocalFile(self, sourceFilePath, targetFilePath):
"""
Send local file to remote machine
sourceFilePath -- Source file path (/file path/...).
targetFilePath -- Target file path (/file path/...).
"""
log.info("Copying " + sourceFilePath + " to " + targetFilePath)
# Check if file already existsFilePath and it is up to date
existsFilePath = False
if (exists(targetFilePath)):
if ( self.__getLocalSHA1(sourceFilePath) == self.__getLocalSHA1(targetFilePath)):
existsFilePath = True
log.info(targetFilePath + " already existed")
if (not existsFilePath):
makeFilePath(targetFilePath)
shutil.copy2(sourceFilePath, targetFilePath)
def __copyLocalFile(self, sourceFilePath, targetFilePath):
"""
Send local file to remote machine
sourceFilePath -- Source file path (/file path/...).
targetFilePath -- Target file path (/file path/...).
"""
log.info("Copying " + sourceFilePath + " to " + targetFilePath)
# Check if file already existsFilePath and it is up to date
existsFilePath = False
if (exists(targetFilePath)):
if (self.__getLocalSHA1(sourceFilePath) == self.__getLocalSHA1(
targetFilePath)):
existsFilePath = True
log.info(targetFilePath + " already existed")
if (not existsFilePath):
makeFilePath(targetFilePath)
shutil.copy2(sourceFilePath, targetFilePath)
def iterMicrographCoordinates(self, micrograph):
""" Iterates over the set of coordinates belonging to that micrograph. """
self.loadIfEmpty()
pathJsonPos = self.getMicrographCoordFile(micrograph.getObjId())
if pathJsonPos is not None:
if exists(pathJsonPos):
coordJson = loadJson(pathJsonPos)
coordList = coordJson["boxes"]
coorIdList = coordJson["coordId"]
for i, pos in enumerate(coordList):
x = pos[0]
y = pos[1]
coordinate = Coordinate()
coordinate.setPosition(x, y)
coordinate.setMicrograph(micrograph)
coordinate.setBoxSize(self.boxSize.get())
coordinate.setObjId(coorIdList[i])
yield coordinate
def __getRemoteFile(self, sourceFilePath, targetFilePath, gatewayHosts, sftp):
"""
Send local file to remote machine
sourceFilePath -- Source file path (/file path/...).
targetFilePath -- Target file path (/file path/...).
sftp -- sftp connection.
"""
log.info("Getting " + sourceFilePath + " to " + targetFilePath)
# Check if file already existsFilePath and it is up to date
existsFilePath = False
if (exists(targetFilePath)):
if ( self.__getRemoteSHA1(sourceFilePath, self.ssh) == self.__getLocalSHA1(targetFilePath)):
existsFilePath = True
log.info(targetFilePath + " already existed")
if (not existsFilePath):
makeFilePath(targetFilePath)
try:
sftp.get(sourceFilePath, targetFilePath)
except IOError as err:
log.error("Fail getting remote file " + sourceFilePath + " to local file " + targetFilePath + " - " + str(err))
raise
def createOutputStep(self):
# Create the SetOfImages objects on the database and the ImagesTiltPair
mdUntilted = xmipp.MetaData()
mdTilted = xmipp.MetaData()
#for objId in mdPairs:
for uMic, tMic in izip(self.uMics, self.tMics):
umicName = removeBaseExt(uMic.getFileName())
fnMicU = self._getExtraPath(umicName + ".xmd")
fnPosU = self._getExtraPath(umicName + ".pos")
# Check if there are picked particles in this micrographs
if exists(fnMicU):
mdMicU = xmipp.MetaData(fnMicU)
mdPosU = xmipp.MetaData('particles@%s' % fnPosU)
mdPosU.merge(mdMicU)
mdUntilted.unionAll(mdPosU)
tmicName = removeBaseExt(tMic.getFileName())
fnMicT = self._getExtraPath(tmicName + ".xmd")
fnPosT = self._getExtraPath(tmicName + ".pos")
mdMicT = xmipp.MetaData(fnMicT)
mdPosT = xmipp.MetaData('particles@%s' % fnPosT)
mdPosT.merge(mdMicT)
mdTilted.unionAll(mdPosT)
# Write image metadatas (check if it is really necessary)
fnTilted = self._getExtraPath("images_tilted.xmd")
fnUntilted = self._getExtraPath("images_untilted.xmd")
mdUntilted.write(fnUntilted)
mdTilted.write(fnTilted)
# Create outputs SetOfParticles both for tilted and untilted
imgSetU = self._createSetOfParticles(suffix="Untilted")
imgSetU.copyInfo(self.uMics)
imgSetT = self._createSetOfParticles(suffix="Tilted")
imgSetT.copyInfo(self.tMics)
if self.downsampleType == OTHER:
imgSetU.setSamplingRate(self.samplingFinal)
imgSetT.setSamplingRate(self.samplingFinal)
imgSetU.setCoordinates(self.inputCoordinatesTiltedPairs.get().getUntilted())
imgSetT.setCoordinates(self.inputCoordinatesTiltedPairs.get().getTilted())
#Read untilted and tilted particles on a temporary object (also disabled particles)
imgSetAuxU = self._createSetOfParticles('auxU')
imgSetAuxU.copyInfo(imgSetU)
readSetOfParticles(fnUntilted, imgSetAuxU, removeDisabled=False)
imgSetAuxU.write()
imgSetAuxT = self._createSetOfParticles('auxT')
imgSetAuxT.copyInfo(imgSetT)
readSetOfParticles(fnTilted, imgSetAuxT, removeDisabled=False)
imgSetAuxT.write()
coordsT = self.inputCoordinatesTiltedPairs.get().getTilted()
# For each untilted particle retrieve micId from SetOFCoordinates untilted
for imgU, coordU in izip(imgSetAuxU, self.inputCoordinatesTiltedPairs.get().getUntilted()):
#FIXME: REmove this check when sure that objIds are equal
id = imgU.getObjId()
if id != coordU.getObjId():
raise Exception('ObjId in untilted is not equal!!!!')
imgT = imgSetAuxT[id]
coordT = coordsT[id]
#If both particles are enabled append them
if imgU.isEnabled() and imgT.isEnabled():
imgU.setCoordinate(coordU)
imgSetU.append(imgU)
imgT.setCoordinate(coordT)
imgSetT.append(imgT)
# For each untilted particle retrieve micId from SetOFCoordinates tilted
#for img in imgSetAuxU:
# for img, coord in izip(imgSetAuxT, self.inputCoordinatesTiltedPairs.get().getTilted()):
# #FIXME: This can be slow to make a query to grab the coord, maybe use zip(imgSet, coordSet)???
# #FIXME: REmove this check when sure that objIds are equal
# if img.getObjId() != coord.getObjId():
# raise Exception('ObjId is not equal!!!!')
# #coord = self.inputCoordinatesTiltedPairs.get().getTilted()[img.getObjId()]
# img.setCoordinate(coord)
# #img.cleanObjId()
# imgSetT.append(img)
imgSetU.write()
imgSetT.write()
self._storeMethodsInfo(fnUntilted)
# Define output ParticlesTiltPair
outputset = ParticlesTiltPair(filename=self._getPath('particles_pairs.sqlite'))
outputset.setTilted(imgSetT)
outputset.setUntilted(imgSetU)
for imgU, imgT in izip(imgSetU, imgSetT):
outputset.append(TiltPair(imgU, imgT))
outputset.setCoordsPair(self.inputCoordinatesTiltedPairs.get())
self._defineOutputs(outputParticlesTiltPair=outputset)
self._defineSourceRelation(self.inputCoordinatesTiltedPairs, outputset)
def createOutputStep(self):
# Create the SetOfImages objects on the database and the ImagesTiltPair
mdUntilted = xmipp.MetaData()
mdTilted = xmipp.MetaData()
#for objId in mdPairs:
for uMic, tMic in izip(self.uMics, self.tMics):
umicName = removeBaseExt(uMic.getFileName())
fnMicU = self._getExtraPath(umicName + ".xmd")
fnPosU = self._getExtraPath(umicName + ".pos")
# Check if there are picked particles in this micrographs
if exists(fnMicU):
mdMicU = xmipp.MetaData(fnMicU)
mdPosU = xmipp.MetaData('particles@%s' % fnPosU)
mdPosU.merge(mdMicU)
mdUntilted.unionAll(mdPosU)
tmicName = removeBaseExt(tMic.getFileName())
fnMicT = self._getExtraPath(tmicName + ".xmd")
fnPosT = self._getExtraPath(tmicName + ".pos")
mdMicT = xmipp.MetaData(fnMicT)
mdPosT = xmipp.MetaData('particles@%s' % fnPosT)
mdPosT.merge(mdMicT)
mdTilted.unionAll(mdPosT)
# Write image metadatas (check if it is really necessary)
fnTilted = self._getExtraPath("images_tilted.xmd")
fnUntilted = self._getExtraPath("images_untilted.xmd")
mdUntilted.write(fnUntilted)
mdTilted.write(fnTilted)
# Create outputs SetOfParticles both for tilted and untilted
imgSetU = self._createSetOfParticles(suffix="Untilted")
imgSetU.copyInfo(self.uMics)
imgSetT = self._createSetOfParticles(suffix="Tilted")
imgSetT.copyInfo(self.tMics)
if self.downsampleType == OTHER:
imgSetU.setSamplingRate(self.samplingFinal)
imgSetT.setSamplingRate(self.samplingFinal)
imgSetU.setCoordinates(self.inputCoordinatesTiltedPairs.get().getUntilted())
imgSetT.setCoordinates(self.inputCoordinatesTiltedPairs.get().getTilted())
#Read untilted and tilted particles on a temporary object (also disabled particles)
imgSetAuxU = self._createSetOfParticles('auxU')
imgSetAuxU.copyInfo(imgSetU)
readSetOfParticles(fnUntilted, imgSetAuxU, removeDisabled=False)
imgSetAuxU.write()
imgSetAuxT = self._createSetOfParticles('auxT')
imgSetAuxT.copyInfo(imgSetT)
readSetOfParticles(fnTilted, imgSetAuxT, removeDisabled=False)
imgSetAuxT.write()
coordsT = self.inputCoordinatesTiltedPairs.get().getTilted()
# For each untilted particle retrieve micId from SetOFCoordinates untilted
for imgU, coordU in izip(imgSetAuxU, self.inputCoordinatesTiltedPairs.get().getUntilted()):
#FIXME: REmove this check when sure that objIds are equal
id = imgU.getObjId()
if id != coordU.getObjId():
raise Exception('ObjId in untilted is not equal!!!!')
imgT = imgSetAuxT[id]
coordT = coordsT[id]
#If both particles are enabled append them
if imgU.isEnabled() and imgT.isEnabled():
imgU.setCoordinate(coordU)
imgSetU.append(imgU)
imgT.setCoordinate(coordT)
imgSetT.append(imgT)
# For each untilted particle retrieve micId from SetOFCoordinates tilted
#for img in imgSetAuxU:
# for img, coord in izip(imgSetAuxT, self.inputCoordinatesTiltedPairs.get().getTilted()):
# #FIXME: This can be slow to make a query to grab the coord, maybe use zip(imgSet, coordSet)???
# #FIXME: REmove this check when sure that objIds are equal
# if img.getObjId() != coord.getObjId():
# raise Exception('ObjId is not equal!!!!')
# #coord = self.inputCoordinatesTiltedPairs.get().getTilted()[img.getObjId()]
# img.setCoordinate(coord)
# #img.cleanObjId()
# imgSetT.append(img)
imgSetU.write()
imgSetT.write()
self._storeMethodsInfo(fnUntilted)
# Define output ParticlesTiltPair
outputset = ParticlesTiltPair(filename=self._getPath('particles_pairs.sqlite'))
outputset.setTilted(imgSetT)
outputset.setUntilted(imgSetU)
for imgU, imgT in izip(imgSetU, imgSetT):
outputset.append(TiltPair(imgU, imgT))
outputset.setCoordsPair(self.inputCoordinatesTiltedPairs.get())
self._defineOutputs(outputParticlesTiltPair=outputset)
self._defineSourceRelation(self.inputCoordinatesTiltedPairs, outputset)
