def _checkNewOutput(self): if getattr(self, 'finished', False): return self.finished = self.streamClosed and \ self.inputSize == self.outputSize streamMode = emobj.Set.STREAM_CLOSED if self.finished else emobj.Set.STREAM_OPEN # we will read all ready files files = pwutils.glob(self.getTmpOutputPath('*')) newData = len(files) > 0 lastToClose = self.finished and hasattr(self, 'outputCoordinates') if newData or lastToClose: outSet = self._loadOutputSet() if newData: for tmpFile in files: tmpSet = emobj.SetOfCoordinates(filename=tmpFile) tmpSet.loadAllProperties() outSet.copyItems(tmpSet) outSet.setBoxSize(tmpSet.getBoxSize()) tmpSet.close() pwutils.cleanPath(tmpFile) self._updateOutputSet('outputCoordinates', outSet, state=streamMode) if self.finished: # Unlock createOutputStep if finished all jobs outputStep = self._getFirstJoinStep() if outputStep and outputStep.isWaiting(): outputStep.setStatus(STATUS_NEW)
def testWriteSetOfCoordinatesWithoutFlip(self): from collections import OrderedDict # Define a temporary sqlite file for micrographs fn = self.getOutputPath('convert_mics.sqlite') mics = emobj.SetOfMicrographs(filename=fn) # Create SetOfCoordinates data # Define a temporary sqlite file for coordinates fn = self.getOutputPath('convert_coordinates.sqlite') coordSet = emobj.SetOfCoordinates(filename=fn) coordSet.setBoxSize(60) coordSet.setMicrographs(mics) data = OrderedDict() data['006'] = [(30, 30)] data['016'] = [(40, 40)] micList = [] for key, coords in data.items(): mic = emobj.Micrograph(self.ds.getFile('micrographs/%s.mrc' % key)) mics.append(mic) micList.append(mic) print("Adding mic: %s, id: %s" % (key, mic.getObjId())) for x, y in coords: coord = emobj.Coordinate(x=x, y=y) coord.setMicrograph(mic) coordSet.append(coord) # Get boxDirectory boxFolder = self.getOutputPath('boxFolder') os.mkdir(boxFolder) micFolder = self.getOutputPath('micFolder') pwutils.makePath(micFolder) # Invoke the write set of coordinates method convert.writeSetOfCoordinates(boxFolder, coordSet) convert.convertMicrographs(micList, micFolder) # Assert output of writesetofcoordinates for mic in micList: boxFile = os.path.join(boxFolder, convert.getMicIdName(mic, '.box')) self.assertTrue(os.path.exists(boxFile), 'Missing box file: %s' % boxFile) micFile = os.path.join(micFolder, convert.getMicIdName(mic, '.mrc')) self.assertTrue(os.path.exists(micFile), 'Missing box file: %s' % micFile) # Assert coordinates in box files fh = open(os.path.join(boxFolder, 'mic00001.box')) box1 = fh.readline() fh.close() box1 = box1.split('\t') self.assertEquals(box1[0], '0') self.assertEquals(box1[1], '964')
def _loadOutputSet(self): setFile = self._getPath("coordinates.sqlite") if os.path.exists(setFile): outputSet = emobj.SetOfCoordinates(filename=setFile) outputSet.loadAllProperties() outputSet.enableAppend() else: outputSet = emobj.SetOfCoordinates(filename=setFile) outputSet.setStreamState(outputSet.STREAM_OPEN) self._store(outputSet) self._defineTransformRelation(self.getInputParticles(), outputSet) self._defineSourceRelation(self.getInputMicrographs(), outputSet) # copyInfo raise an error since particles has no _boxsize # copyInfo does not do anything else # outputSet.copyInfo(self.getInputParticles()) outputSet.setMicrographs(self.getInputMicrographs()) return outputSet
def _loadInputCoords(self, micDict): """ Load coordinates from the input streaming. """ # TODO: this takes for ever if you are NOT # doing streaming and have several thousands of mics # so I add a counter to keep the user entertained import sys a = datetime.now() counter = 1 coordsFn = self.getCoords().getFileName() self.debug("Loading input db: %s" % coordsFn) coordSet = emobj.SetOfCoordinates(filename=coordsFn) # FIXME: Temporary to avoid loadAllPropertiesFail coordSet._xmippMd = pwobj.String() coordSet.loadAllProperties() micList = dict() # To store a dictionary with mics with coordinates for micKey, mic in micDict.items(): if counter % 50 == 0: b = datetime.now() print(b - a, 'reading coordinates for mic number', "%06d" % counter) sys.stdout.flush() # force buffer to print counter += 1 micId = mic.getObjId() coordList = [] self.debug("Loading coords for mic: %s (%s)" % (micId, micKey)) for coord in coordSet.iterItems(where='_micId=%s' % micId): # TODO: Check performance penalty of using this clone coordList.append(coord.clone()) self.debug("Coords found: %s" % len(coordList)) if coordList: self.coordDict[micId] = coordList micList[micKey] = mic self.coordsClosed = coordSet.isStreamClosed() coordSet.close() self.debug("Coords are closed? %s" % self.coordsClosed) self.debug("Closed db.") return micList
def importCoordinates(self, inputMics): """ Import a SetOfCoordinates from a given sqlite file. """ inputSet = emobj.SetOfCoordinates(filename=self._sqliteFile) # self._findImagesPath(inputSet) coorSet = self.protocol._createSetOfCoordinates(inputMics) coorSet.copyInfo(inputSet) coorSet.setObjComment('Coordinates imported from sqlite file:\n%s' % self._sqliteFile) # Update both samplingRate and acquisition with parameters # selected in the protocol form. # Acquisition should be updated before, just to ensure that # scannedPixedSize will be computed properly when calling # setSamplingRate coorSet.setBoxSize(self.protocol.boxSize.get()) # Read the micrographs from the 'self._sqliteFile' metadata # but fixing the filenames with new ones (linked or copy to extraDir) coorSet.copyItems(inputSet) self.protocol._defineOutputs(outputCoordinates=coorSet) self.protocol._defineSourceRelation(inputMics, coorSet)