def createOutputStep(self):
# Create the SetOfImages object on the database
fnImages = self._getOutputImgMd()
# Create output SetOfParticles
self.imgSet = self._createSetOfParticles()
self.imgSet.copyInfo(self.inputMics)
# set coords from the input, will update later if needed
self.imgSet.setCoordinates(self.inputCoords)
coords2 = SetOfCoordinates()
if self.doFlip:
self.imgSet.setIsPhaseFlipped(not self.inputMics.isPhaseFlipped())
self.imgSet.setSamplingRate(self._getNewSampling())
self.imgSet.setHasCTF(self.ctfRelations.hasValue())
# a SetMdIterator is needed because in some cases, the number of
# items in SetOfCoordinates and in the metadata could be different.
iterator = md.SetMdIterator(fnImages, sortByLabel=md.MDL_ITEM_ID,
updateItemCallback=self.createParticles,
skipDisabled=True)
# THis use case is special, because copyItems method is designed to
# modified the same type of object of the input set. Here, a new type
# (SetOfParticles) is generated, and its needed an auxiliary set of
# coordinates and it not stored.
coordsAux = SetOfCoordinates()
coordsAux.copyItems(self.inputCoords,
updateItemCallback=iterator.updateItem,
copyDisabled=True)
self._storeMethodsInfo(fnImages)
self._defineOutputs(outputParticles=self.imgSet)
self._defineSourceRelation(self.inputCoordinates, self.imgSet)
if self.ctfRelations.hasValue():
self._defineSourceRelation(self.ctfRelations.get(), self.imgSet)
def _loadInputCoords(self, micDict):
""" Load coordinates from the input streaming.
"""
coordsFn = self.getCoords().getFileName()
self.debug("Loading input db: %s" % coordsFn)
coordSet = SetOfCoordinates(filename=coordsFn)
# FIXME: Temporary to avoid loadAllPropertiesFail
coordSet._xmippMd = String()
coordSet.loadAllProperties()
for micKey, mic in micDict.iteritems():
micId = mic.getObjId()
coordList = []
self.debug("Loading coords for mic: %s (%s)" % (micId, micKey))
for coord in coordSet.iterItems(where='_micId=%s' % micId):
# TODO: Check performance penalty of using this clone
coordList.append(coord.clone())
self.debug(" Coords found: %s" % len(coordList))
if coordList:
self.coordDict[micId] = coordList
else:
del micDict[micKey]
self.coordsClosed = coordSet.isStreamClosed()
coordSet.close()
self.debug("Coords are closed? %s" % self.coordsClosed)
self.debug("Closed db.")
return micDict
def main():
parser = argparse.ArgumentParser(prog='Scipion Convert')
parser.add_argument('--coordinates',
help='Convert coordinates',
action="store_true")
parser.add_argument('--fromType', help='Convert from input type')
parser.add_argument('--toType', help='Convert to output type')
parser.add_argument('--input', help='Input file or folder')
parser.add_argument('--output', help='Output file or folder')
parser.add_argument('--extra', help='To add extra parameters')
args = parser.parse_args()
fromType = args.fromType
toType = args.toType
input = args.input
output = args.output
if args.coordinates:
#print 'converting coordinates ...'
micSet = loadSetFromDb(input)
outputDir = output
coordsfn = os.path.join(outputDir, 'coordinates.sqlite')
cleanPath(coordsfn)
coordSet = SetOfCoordinates(filename=coordsfn)
coordSet.setMicrographs(micSet)
if fromType == 'eman2':
if toType == 'xmipp':
#print 'from eman2 to xmipp...'
from pyworkflow.em.packages.eman2.convert import readSetOfCoordinates
readSetOfCoordinates(outputDir, micSet, coordSet)
from pyworkflow.em.packages.xmipp3.convert import writeSetOfCoordinates
writeSetOfCoordinates(outputDir, coordSet, ismanual=False)
if fromType == 'dogpicker':
if toType == 'xmipp':
#print 'from dogpicker to xmipp...'
from pyworkflow.em.packages.appion.convert import readSetOfCoordinates
readSetOfCoordinates(outputDir, micSet, coordSet)
from pyworkflow.em.packages.xmipp3.convert import writeSetOfCoordinates
writeSetOfCoordinates(outputDir, coordSet, ismanual=False)
if fromType == 'relion':
if toType == 'xmipp':
#print 'from relion to xmipp...'
inputCoords = args.extra
starFiles = [
os.path.join(
inputCoords,
pwutils.removeBaseExt(mic.getFileName()) +
'_autopick.star') for mic in micSet
]
from pyworkflow.em.packages.relion.convert import readSetOfCoordinates
readSetOfCoordinates(coordSet, starFiles)
from pyworkflow.em.packages.xmipp3.convert import writeSetOfCoordinates
writeSetOfCoordinates(outputDir, coordSet, ismanual=False)
def main():
parser = argparse.ArgumentParser(prog='Scipion Convert')
parser.add_argument('--coordinates', help='Convert coordinates', action="store_true")
parser.add_argument('--fromType', help='Convert from input type')
parser.add_argument('--toType', help='Convert to output type')
parser.add_argument('--input', help='Input file or folder')
parser.add_argument('--output', help='Output file or folder')
parser.add_argument('--extra', help='To add extra parameters')
args = parser.parse_args()
fromType = args.fromType
toType = args.toType
input = args.input
output = args.output
if args.coordinates:
micSet = loadSetFromDb(input)
outputDir = output
coordsfn = os.path.join(outputDir, 'coordinates.sqlite')
cleanPath(coordsfn)
coordSet = SetOfCoordinates(filename=coordsfn)
coordSet.setMicrographs(micSet)
if fromType == 'eman2':
if toType == 'xmipp':
from pyworkflow.em.packages.eman2.convert import readSetOfCoordinates
elif fromType == 'dogpicker':
if toType == 'xmipp':
from pyworkflow.em.packages.appion.convert import readSetOfCoordinates
elif fromType == 'relion':
if toType == 'xmipp':
def readSetOfCoordinates(outputDir, micSet, coordSet):
from pyworkflow.em.packages.relion.convert import readSetOfCoordinates
inputCoords = args.extra
starFiles = [os.path.join(inputCoords,
pwutils.removeBaseExt(mic.getFileName())
+ '_autopick.star') for mic in micSet]
readSetOfCoordinates(coordSet, starFiles)
elif fromType == 'gautomatch':
if toType == 'xmipp':
from pyworkflow.em.packages.gautomatch.convert import readSetOfCoordinates
elif fromType == 'gempicker':
if toType == 'xmipp':
from pyworkflow.em.packages.igbmc.convert import readSetOfCoordinates
else:
raise Exception('Unknown coordinates type: %s' % fromType)
readSetOfCoordinates(outputDir, micSet, coordSet)
from pyworkflow.em.packages.xmipp3.convert import writeSetOfCoordinatesWithState
writeSetOfCoordinatesWithState(outputDir, coordSet, state='Automatic')
def _loadInputCoords(self, micDict):
""" Load coordinates from the input streaming.
"""
coordsFn = self.getCoords().getFileName()
self.debug("Loading input db: %s" % coordsFn)
coordSet = SetOfCoordinates(filename=coordsFn)
# FIXME: Temporary to avoid loadAllPropertiesFail
coordSet._xmippMd = String()
coordSet.loadAllProperties()
for micKey, mic in micDict.iteritems():
micId = mic.getObjId()
coordList = []
self.debug("Loading coords for mic: %s (%s)" % (micId, micKey))
for coord in coordSet.iterItems(where='_micId=%s' % micId):
# TODO: Check performance penalty of using this clone
coordList.append(coord.clone())
self.debug(" Coords found: %s" % len(coordList))
if coordList:
self.coordDict[micId] = coordList
else:
del micDict[micKey]
self.coordsClosed = coordSet.isStreamClosed()
coordSet.close()
self.debug("Coords are closed? %s" % self.coordsClosed)
self.debug("Closed db.")
return micDict
def _checkNewInput(self):
newMics = []
for idx, coordSet in enumerate(self.inputCoordinates):
coorSet = SetOfCoordinates(filename=coordSet.get().getFileName())
coorSet._xmippMd = String()
coorSet.loadAllProperties()
self.streamClosed = coorSet.isStreamClosed()
coorSet.close()
if self.check[idx] == "":
newMics.append(
[c.clone() for c in coordSet.get().getMicrographs()])
else:
newMics.append([
c.clone()
for c in coordSet.get().getMicrographs().iterItems(
orderBy='creation',
where='creation>"' + str(self.check[idx]) + '"')
])
if len(newMics[idx]) < 1:
continue
else:
for p in coordSet.get().getMicrographs().iterItems(
orderBy='creation', direction='DESC'):
self.check[idx] = p.getObjCreation()
break
newMics[:] = [item for item in newMics if len(item) > 0]
if len(newMics) > 0:
fDeps = self.insertNewCoorsSteps(min(newMics, key=len))
self.inputMics = self.inputMics + len(min(newMics, key=len))
outputStep = self._getFirstJoinStep()
if outputStep is not None:
outputStep.addPrerequisites(*fDeps)
self.updateSteps()
def main():
parser = argparse.ArgumentParser(prog='Scipion Convert')
parser.add_argument('--coordinates', help='Convert coordinates', action="store_true")
parser.add_argument('--fromType', help='Convert from input type')
parser.add_argument('--toType', help='Convert to output type')
parser.add_argument('--input', help='Input file or folder')
parser.add_argument('--output', help='Output file or folder')
parser.add_argument('--extra', help='To add extra parameters')
args = parser.parse_args()
fromType = args.fromType
toType = args.toType
input = args.input
output = args.output
if args.coordinates:
#print 'converting coordinates ...'
micSet = loadSetFromDb(input)
outputDir = output
coordsfn = os.path.join(outputDir, 'coordinates.sqlite')
cleanPath(coordsfn)
coordSet = SetOfCoordinates(filename=coordsfn)
coordSet.setMicrographs(micSet)
if fromType == 'eman2':
if toType == 'xmipp':
#print 'from eman2 to xmipp...'
from pyworkflow.em.packages.eman2.convert import readSetOfCoordinates
readSetOfCoordinates(outputDir, micSet, coordSet)
from pyworkflow.em.packages.xmipp3.convert import writeSetOfCoordinates
writeSetOfCoordinates(outputDir, coordSet, ismanual=False)
if fromType == 'dogpicker':
if toType == 'xmipp':
#print 'from dogpicker to xmipp...'
from pyworkflow.em.packages.appion.convert import readSetOfCoordinates
readSetOfCoordinates(outputDir, micSet, coordSet)
from pyworkflow.em.packages.xmipp3.convert import writeSetOfCoordinates
writeSetOfCoordinates(outputDir, coordSet, ismanual=False)
if fromType == 'relion':
if toType == 'xmipp':
#print 'from relion to xmipp...'
inputCoords = args.extra
starFiles = [os.path.join(inputCoords, pwutils.removeBaseExt(mic.getFileName()) + '_autopick.star')
for mic in micSet]
from pyworkflow.em.packages.relion.convert import readSetOfCoordinates
readSetOfCoordinates(coordSet, starFiles)
from pyworkflow.em.packages.xmipp3.convert import writeSetOfCoordinates
writeSetOfCoordinates(outputDir, coordSet, ismanual=False)
def _loadInputCoords(self, micDict):
""" Load coordinates from the input streaming.
"""
# TODO: this takes for ever if you are NOT
# doing streaming and have several thousands of mics
# so I add a counter to keep the user entertained
import sys
a = datetime.now()
counter = 1
coordsFn = self.getCoords().getFileName()
self.debug("Loading input db: %s" % coordsFn)
coordSet = SetOfCoordinates(filename=coordsFn)
# FIXME: Temporary to avoid loadAllPropertiesFail
coordSet._xmippMd = String()
coordSet.loadAllProperties()
# TODO: horrible code. Rewrite using
# for coord in coordSet.iterItems(orderBy='_micId',
# direction='ASC'):
# micId = coord.getMicId()
# if micId != lastMicId:
# lastMicId = micId
# ...
# ...
for micKey, mic in micDict.iteritems():
if counter % 50 == 0:
b = datetime.now()
print(b - a, 'reading coordinates for mic number',
"%06d" % counter)
sys.stdout.flush() # force buffer to print
counter += 1
micId = mic.getObjId()
coordList = []
self.debug("Loading coords for mic: %s (%s)" % (micId, micKey))
for coord in coordSet.iterItems(where='_micId=%s' % micId):
# TODO: Check performance penalty of using this clone
coordList.append(coord.clone())
self.debug("Coords found: %s" % len(coordList))
if coordList:
self.coordDict[micId] = coordList
else:
del micDict[micKey]
self.coordsClosed = coordSet.isStreamClosed()
coordSet.close()
self.debug("Coords are closed? %s" % self.coordsClosed)
self.debug("Closed db.")
return micDict
def _checkNewOutput(self):
if getattr(self, 'finished', False):
return
self.finished = self.streamClosed and \
self.inputSize == self.outputSize
streamMode = Set.STREAM_CLOSED if self.finished else Set.STREAM_OPEN
# we will read all ready files
files = pwutils.glob(self.getTmpOutputPath('*'))
newData = len(files) > 0
lastToClose = self.finished and hasattr(self, 'outputCoordinates')
if newData or lastToClose:
outSet = self._loadOutputSet()
if newData:
for tmpFile in files:
tmpSet = SetOfCoordinates(filename=tmpFile)
tmpSet.loadAllProperties()
outSet.copyItems(tmpSet)
outSet.setBoxSize(tmpSet.getBoxSize())
tmpSet.close()
pwutils.cleanPath(tmpFile)
self._updateOutputSet('outputCoordinates',
outSet,
state=streamMode)
if self.finished: # Unlock createOutputStep if finished all jobs
outputStep = self._getFirstJoinStep()
if outputStep and outputStep.isWaiting():
outputStep.setStatus(STATUS_NEW)
def test_coodinatesTest1(self):
""" Import an EMX file with just one micrograph
and a few coordinates.
"""
protEmxImport = self.newProtocol(
ProtImportParticles,
objLabel='from emx (coordinatesT1)',
importFrom=ProtImportParticles.IMPORT_FROM_EMX,
emxFile=self.dataset.getFile('coordinatesT1'),
alignType=3,
voltage=100,
magnification=10000,
samplingRate=2.46)
self.launchProtocol(protEmxImport)
# Reference coordinates
coords = SetOfCoordinates(
filename=self.dataset.getFile('coordinatesGoldT1'))
tests.BaseTest.compareSets(self, protEmxImport.outputCoordinates,
coords)
def readSetOfCoordsFromPosFnames(posDir,
setOfInputCoords,
sqliteOutName,
write=True):
'''
posDir: path where there are .pos files with coordinates
setOfInputCoords. Set to find micrographs
sqliteOutName. Path where sqlite map will be created. Warning, it overwrites content
'''
inputMics = setOfInputCoords.getMicrographs()
cleanPath(sqliteOutName)
setOfOutputCoordinates = SetOfCoordinates(filename=sqliteOutName)
setOfOutputCoordinates.setMicrographs(inputMics)
setOfOutputCoordinates.setBoxSize(setOfInputCoords.getBoxSize())
readSetOfCoordinates(posDir,
micSet=inputMics,
coordSet=setOfOutputCoordinates,
readDiscarded=False)
if write:
setOfOutputCoordinates.write()
return setOfOutputCoordinates
def createOutputStep(self):
# PARTICLES
cleanPattern(self._getPath("*.sqlite"))
partSet = self._createSetOfParticles()
readSetOfParticles(self._getPath("particles.xmd"), partSet)
inputSampling = self.inputCoordinates[0].get().getMicrographs(
).getSamplingRate()
partSet.setSamplingRate(self._getDownFactor() * inputSampling)
boxSize = self._getBoxSize()
# COORDINATES
writeSet = False
if self.checkIfPrevRunIsCompatible("coords_"):
writeSet = True
if not "OR" in self.coordinatesDict:
self.loadCoords(self._getExtraPath(
self.CONSENSUS_COOR_PATH_TEMPLATE % 'TRUE'),
'OR',
writeSet=False)
coordSet = SetOfCoordinates(
filename=self._getPath("coordinates.sqlite"))
coordSet.copyInfo(self.coordinatesDict['OR'])
coordSet.setBoxSize(boxSize)
coordSet.setMicrographs(self.coordinatesDict['OR'].getMicrographs())
downFactor = self._getDownFactor()
for part in partSet:
coord = part.getCoordinate().clone()
coord.scale(downFactor)
deepZscoreLabel = '_xmipp_%s' % xmipp.label2Str(
MD.MDL_ZSCORE_DEEPLEARNING1)
setattr(coord, deepZscoreLabel, getattr(part, deepZscoreLabel))
coordSet.append(coord)
coordSet.write()
partSet.write()
self._defineOutputs(outputCoordinates=coordSet)
self._defineOutputs(outputParticles=partSet)
for inSetOfCoords in self.inputCoordinates:
self._defineSourceRelation(inSetOfCoords.get(), coordSet)
self._defineSourceRelation(inSetOfCoords.get(), partSet)
def _loadOutputSet(self):
setFile = self._getPath("coordinates.sqlite")
if os.path.exists(setFile):
outputSet = SetOfCoordinates(filename=setFile)
outputSet.loadAllProperties()
outputSet.enableAppend()
else:
outputSet = SetOfCoordinates(filename=setFile)
outputSet.setStreamState(outputSet.STREAM_OPEN)
self._store(outputSet)
self._defineTransformRelation(self.getInputParticles(), outputSet)
self._defineSourceRelation(self.getInputMicrographs(), outputSet)
outputSet.copyInfo(self.getInputParticles())
outputSet.setMicrographs(self.getInputMicrographs())
return outputSet