def _checkNewOutput(self):
if getattr(self, 'finished', False):
return
self.finished = self.streamClosed and \
self.inputSize == self.outputSize
streamMode = emobj.Set.STREAM_CLOSED if self.finished else emobj.Set.STREAM_OPEN
# we will read all ready files
files = pwutils.glob(self.getTmpOutputPath('*'))
newData = len(files) > 0
lastToClose = self.finished and hasattr(self, 'outputCoordinates')
if newData or lastToClose:
outSet = self._loadOutputSet()
if newData:
for tmpFile in files:
tmpSet = emobj.SetOfCoordinates(filename=tmpFile)
tmpSet.loadAllProperties()
outSet.copyItems(tmpSet)
outSet.setBoxSize(tmpSet.getBoxSize())
tmpSet.close()
pwutils.cleanPath(tmpFile)
self._updateOutputSet('outputCoordinates',
outSet,
state=streamMode)
if self.finished: # Unlock createOutputStep if finished all jobs
outputStep = self._getFirstJoinStep()
if outputStep and outputStep.isWaiting():
outputStep.setStatus(STATUS_NEW)
def testWriteSetOfCoordinatesWithoutFlip(self):
from collections import OrderedDict
# Define a temporary sqlite file for micrographs
fn = self.getOutputPath('convert_mics.sqlite')
mics = emobj.SetOfMicrographs(filename=fn)
# Create SetOfCoordinates data
# Define a temporary sqlite file for coordinates
fn = self.getOutputPath('convert_coordinates.sqlite')
coordSet = emobj.SetOfCoordinates(filename=fn)
coordSet.setBoxSize(60)
coordSet.setMicrographs(mics)
data = OrderedDict()
data['006'] = [(30, 30)]
data['016'] = [(40, 40)]
micList = []
for key, coords in data.items():
mic = emobj.Micrograph(self.ds.getFile('micrographs/%s.mrc' % key))
mics.append(mic)
micList.append(mic)
print("Adding mic: %s, id: %s" % (key, mic.getObjId()))
for x, y in coords:
coord = emobj.Coordinate(x=x, y=y)
coord.setMicrograph(mic)
coordSet.append(coord)
# Get boxDirectory
boxFolder = self.getOutputPath('boxFolder')
os.mkdir(boxFolder)
micFolder = self.getOutputPath('micFolder')
pwutils.makePath(micFolder)
# Invoke the write set of coordinates method
convert.writeSetOfCoordinates(boxFolder, coordSet)
convert.convertMicrographs(micList, micFolder)
# Assert output of writesetofcoordinates
for mic in micList:
boxFile = os.path.join(boxFolder,
convert.getMicIdName(mic, '.box'))
self.assertTrue(os.path.exists(boxFile),
'Missing box file: %s' % boxFile)
micFile = os.path.join(micFolder,
convert.getMicIdName(mic, '.mrc'))
self.assertTrue(os.path.exists(micFile),
'Missing box file: %s' % micFile)
# Assert coordinates in box files
fh = open(os.path.join(boxFolder, 'mic00001.box'))
box1 = fh.readline()
fh.close()
box1 = box1.split('\t')
self.assertEquals(box1[0], '0')
self.assertEquals(box1[1], '964')
def _loadOutputSet(self):
setFile = self._getPath("coordinates.sqlite")
if os.path.exists(setFile):
outputSet = emobj.SetOfCoordinates(filename=setFile)
outputSet.loadAllProperties()
outputSet.enableAppend()
else:
outputSet = emobj.SetOfCoordinates(filename=setFile)
outputSet.setStreamState(outputSet.STREAM_OPEN)
self._store(outputSet)
self._defineTransformRelation(self.getInputParticles(), outputSet)
self._defineSourceRelation(self.getInputMicrographs(), outputSet)
# copyInfo raise an error since particles has no _boxsize
# copyInfo does not do anything else
# outputSet.copyInfo(self.getInputParticles())
outputSet.setMicrographs(self.getInputMicrographs())
return outputSet
def _loadInputCoords(self, micDict):
""" Load coordinates from the input streaming.
"""
# TODO: this takes for ever if you are NOT
# doing streaming and have several thousands of mics
# so I add a counter to keep the user entertained
import sys
a = datetime.now()
counter = 1
coordsFn = self.getCoords().getFileName()
self.debug("Loading input db: %s" % coordsFn)
coordSet = emobj.SetOfCoordinates(filename=coordsFn)
# FIXME: Temporary to avoid loadAllPropertiesFail
coordSet._xmippMd = pwobj.String()
coordSet.loadAllProperties()
micList = dict() # To store a dictionary with mics with coordinates
for micKey, mic in micDict.items():
if counter % 50 == 0:
b = datetime.now()
print(b - a, 'reading coordinates for mic number',
"%06d" % counter)
sys.stdout.flush() # force buffer to print
counter += 1
micId = mic.getObjId()
coordList = []
self.debug("Loading coords for mic: %s (%s)" % (micId, micKey))
for coord in coordSet.iterItems(where='_micId=%s' % micId):
# TODO: Check performance penalty of using this clone
coordList.append(coord.clone())
self.debug("Coords found: %s" % len(coordList))
if coordList:
self.coordDict[micId] = coordList
micList[micKey] = mic
self.coordsClosed = coordSet.isStreamClosed()
coordSet.close()
self.debug("Coords are closed? %s" % self.coordsClosed)
self.debug("Closed db.")
return micList
def importCoordinates(self, inputMics):
""" Import a SetOfCoordinates from a given sqlite file. """
inputSet = emobj.SetOfCoordinates(filename=self._sqliteFile)
# self._findImagesPath(inputSet)
coorSet = self.protocol._createSetOfCoordinates(inputMics)
coorSet.copyInfo(inputSet)
coorSet.setObjComment('Coordinates imported from sqlite file:\n%s' %
self._sqliteFile)
# Update both samplingRate and acquisition with parameters
# selected in the protocol form.
# Acquisition should be updated before, just to ensure that
# scannedPixedSize will be computed properly when calling
# setSamplingRate
coorSet.setBoxSize(self.protocol.boxSize.get())
# Read the micrographs from the 'self._sqliteFile' metadata
# but fixing the filenames with new ones (linked or copy to extraDir)
coorSet.copyItems(inputSet)
self.protocol._defineOutputs(outputCoordinates=coorSet)
self.protocol._defineSourceRelation(inputMics, coorSet)