def _checkProcessedData(self): if self.setof == SET_OF_MOVIES: objSet = emobj.SetOfMovies(filename=self._getPath('movies.sqlite')) elif self.setof == SET_OF_MICROGRAPHS: objSet = emobj.SetOfMicrographs(filename=self._getPath('micrographs.sqlite')) elif self.setof == SET_OF_RANDOM_MICROGRAPHS: objSet = emobj.SetOfMicrographs(filename=self._getPath('micrographs.sqlite')) elif self.setof == SET_OF_PARTICLES: objSet = emobj.SetOfParticles(filename=self._getPath('particles.sqlite')) else: raise Exception('Unknown data type') newObjSet, newObj = self._checkNewItems(objSet) if self.setof == SET_OF_MOVIES: newObjSet.setFramesRange(self.inputMovies.get().getFramesRange()) # check if end .... if self.setof != SET_OF_PARTICLES: self.nDims = self.nDim.get() endObjs = newObjSet.getSize() == self.nDims if newObj: if endObjs: newObjSet.setStreamState(newObjSet.STREAM_CLOSED) self._updateOutput(newObjSet) newObjSet.close()
def _createSubSetFromMicrographsTiltPair(self, micrographsTiltPair): """ Create a subset of Micrographs Tilt Pair. """ output = MicrographsTiltPair( filename=self._getPath('micrographs_pairs.sqlite')) modifiedSet = MicrographsTiltPair(filename=self._dbName, prefix=self._dbPrefix) inputU = micrographsTiltPair.getUntilted() inputT = micrographsTiltPair.getTilted() outputU = emobj.SetOfMicrographs( filename=self._getPath('mics_untilted.sqlite')) outputT = emobj.SetOfMicrographs( filename=self._getPath('mics_tilted.sqlite')) outputU.copyInfo(inputU) outputT.copyInfo(inputT) # noinspection DuplicatedCode for micPair, u, t in izip(modifiedSet, inputU, inputT): if micPair.isEnabled(): output.append(micPair) outputU.append(u) outputT.append(t) output.setUntilted(outputU) output.setTilted(outputT) # Register outputs outputDict = {'outputMicrographsTiltPair': output} self._defineOutputs(**outputDict) self._defineTransformRelation(micrographsTiltPair, output) return output
def loadInputs(self): micsFn = self.getInputMicrographs().getFileName() micsSet = emobj.SetOfMicrographs(filename=micsFn) micsSet.loadAllProperties() availableMics = [] for mic in micsSet: availableMics.append(mic.getObjId()) micsSetClosed = micsSet.isStreamClosed() micsSet.close() partsFn = self.getInputParticles().getFileName() partsSet = emobj.SetOfParticles(filename=partsFn) partsSet.loadAllProperties() newParts = [] newMics = [] for item in partsSet: micKey = item.getCoordinate().getMicId() if micKey not in self.micsDone and micKey in availableMics: newParts.append(item.getObjId()) if micKey not in self.micsDone: newMics.append(micKey) self.micsDone += newMics self.inputSize = partsSet.getSize() partSetClosed = partsSet.isStreamClosed() partsSet.close() return newParts, micsSetClosed and partSetClosed
def testRead(self): """ Read micrographs from a an sqlite file. It should contains Acquisition info. """ micFn = self.dataset.getFile('micsGoldSqlite2') print(">>> Reading gold micrographs from: ", micFn) micSet = emobj.SetOfMicrographs(filename=micFn) self.assertEqual(2, micSet.getSize()) acquisition = emobj.Acquisition() acquisition.setMagnification(10000.) acquisition.setVoltage(200.) acquisition.setSphericalAberration(2.26) acquisition.setAmplitudeContrast(0.1) mic2 = emobj.Micrograph() mic2.setSamplingRate(2.8) mic2.setAcquisition(acquisition) fileNames = [ '/home/roberto/Scipion/Test/Test2/down2_12585', '/home/roberto/Scipion/Test/Test2/down2_12610' ] counter = 0 for mic in micSet: mic2.setFileName(fileNames[counter]) self.assertTrue(mic.equalAttributes(mic2)) counter += 1
def testWriteSetOfCoordinatesWithoutFlip(self): from collections import OrderedDict # Define a temporary sqlite file for micrographs fn = self.getOutputPath('convert_mics.sqlite') mics = emobj.SetOfMicrographs(filename=fn) # Create SetOfCoordinates data # Define a temporary sqlite file for coordinates fn = self.getOutputPath('convert_coordinates.sqlite') coordSet = emobj.SetOfCoordinates(filename=fn) coordSet.setBoxSize(60) coordSet.setMicrographs(mics) data = OrderedDict() data['006'] = [(30, 30)] data['016'] = [(40, 40)] micList = [] for key, coords in data.items(): mic = emobj.Micrograph(self.ds.getFile('micrographs/%s.mrc' % key)) mics.append(mic) micList.append(mic) print("Adding mic: %s, id: %s" % (key, mic.getObjId())) for x, y in coords: coord = emobj.Coordinate(x=x, y=y) coord.setMicrograph(mic) coordSet.append(coord) # Get boxDirectory boxFolder = self.getOutputPath('boxFolder') os.mkdir(boxFolder) micFolder = self.getOutputPath('micFolder') pwutils.makePath(micFolder) # Invoke the write set of coordinates method convert.writeSetOfCoordinates(boxFolder, coordSet) convert.convertMicrographs(micList, micFolder) # Assert output of writesetofcoordinates for mic in micList: boxFile = os.path.join(boxFolder, convert.getMicIdName(mic, '.box')) self.assertTrue(os.path.exists(boxFile), 'Missing box file: %s' % boxFile) micFile = os.path.join(micFolder, convert.getMicIdName(mic, '.mrc')) self.assertTrue(os.path.exists(micFile), 'Missing box file: %s' % micFile) # Assert coordinates in box files fh = open(os.path.join(boxFolder, 'mic00001.box')) box1 = fh.readline() fh.close() box1 = box1.split('\t') self.assertEquals(box1[0], '0') self.assertEquals(box1[1], '964')
def test_mapper(self): """ test that indexes are created when a setOfCoordinates is created """ MICNUMBER = 10 indexesNames = ['_untilted._micId', '_tilted._micId'] prot = createDummyProtocol("dummy_protocol") # create set of untilted and tilted micrographs uMicSet = emobj.SetOfMicrographs(filename=":memory:") tMicSet = emobj.SetOfMicrographs(filename=":memory:") mic = emobj.Micrograph() for i in range(MICNUMBER): mic.setLocation(i, "umic_%06d.mrc" % i) uMicSet.append(mic) mic.cleanObjId() mic.setLocation(i, "tmic_%06d.mrc" % i) tMicSet.append(mic) mic.cleanObjId()
def checkMicSet(self, micSet, goldFn): """ Compare micrographs of micSet with the ones in the goldFn. Maybe except the full path. """ goldSet = emobj.SetOfMicrographs(filename=goldFn) for mic1, mic2 in izip(goldSet, micSet): # Remove the absolute path in the micrographs to # really check that the attributes should be equal mic1.setFileName(os.path.basename(mic1.getFileName())) mic2.setFileName(os.path.basename(mic2.getFileName())) self.assertTrue(mic1.equalAttributes(mic2, verbose=True))
def testCreate(self): cwd = os.getcwd() # Change to test path os.chdir(self.dataset.getPath()) """ Create a SetOfMicrographs from a list of micrographs """ micSet = emobj.SetOfMicrographs(filename=self.dbFn) micSet.setAcquisition(self.acquisition) micSet.setSamplingRate(1.2) for fn in self.mics: mic = emobj.Micrograph() mic.setFileName(fn) micSet.append(mic) micSet.write() self.checkSet(micSet) # Check copy info also copies the samplingRate micSet2 = emobj.SetOfMicrographs(filename=':memory:') micSet2.copyInfo(micSet) self.assertAlmostEqual(micSet.getSamplingRate(), micSet2.getSamplingRate()) os.chdir(cwd)
def importMicrographs(self): """ Import a SetOfMicrographs from a given sqlite file. """ inputSet = emobj.SetOfMicrographs(filename=self._sqliteFile) self._findImagesPath(inputSet) micSet = self.protocol._createSetOfMicrographs() micSet.setObjComment('Micrographs imported from sqlite file:\n%s' % self._sqliteFile) # Update both samplingRate and acquisition with parameters # selected in the protocol form. # Acquisition should be updated before, just to ensure that # scannedPixedSize will be computed properly when calling # setSamplingRate self.protocol.fillAcquisition(micSet.getAcquisition()) self.protocol.setSamplingRate(micSet) micSet.setIsPhaseFlipped(self.protocol.haveDataBeenPhaseFlipped.get()) # Read the micrographs from the 'self._sqliteFile' metadata # but fixing the filenames with new ones (linked or copy to extraDir) micSet.copyItems(inputSet, updateItemCallback=self._updateParticle) self.protocol._defineOutputs(outputMicrographs=micSet)
def test_fromScipion(self): """ Import an EMX file with micrographs and defocus """ micsSqlite = self.dsXmipp.getFile('micrographs/micrographs.sqlite') print("Importing from sqlite: ", micsSqlite) micSet = emobj.SetOfMicrographs(filename=micsSqlite) # Gold values # _samplingRate -> 1.237 # _acquisition._voltage -> 300.0 # _acquisition._magnification -> 50000.0 for k in [ '_samplingRate', '_acquisition._voltage', '_acquisition._magnification' ]: print(k, "->", micSet.getProperty(k)) prot = self.newProtocol( emprot.ProtImportMicrographs, objLabel='from scipion', importFrom=emprot.ProtImportMicrographs.IMPORT_FROM_SCIPION, sqliteFile=micsSqlite, samplingRate=float(micSet.getProperty('_samplingRate')), voltage=float(micSet.getProperty('_acquisition._voltage')), magnification=int( float(micSet.getProperty('_acquisition._magnification')))) self.launchProtocol(prot) micSet = getattr(prot, 'outputMicrographs', None) self.assertIsNotNone(micSet) self.assertAlmostEqual(1.237, micSet.getSamplingRate()) acq = micSet.getAcquisition() self.assertAlmostEqual(300., acq.getVoltage()) self.assertAlmostEqual(50000., acq.getMagnification())
def test_mapper(self): """ test that indexes are created when a setOfCoordinates is created """ PARTNUMBER = 10 MICNUMBER = 600 NUMBERCOORDINATES = PARTNUMBER * MICNUMBER indexesNames = ['_micId'] prot = createDummyProtocol("dummy_protocol") # create set of micrographs micSet = emobj.SetOfMicrographs(filename=":memory:") mic = emobj.Micrograph() for i in range(NUMBERCOORDINATES): mic.setLocation(i, "mic_%06d.mrc" % i) micSet.append(mic) mic.cleanObjId() micSet.write() # create a set of particles coordSet = prot._createSetOfCoordinates(micSet) coord = emobj.Coordinate() for i in range(NUMBERCOORDINATES): coordSet.append(coord) coord.cleanObjId() coordSet.write() # check defined indexes setOfCoordinatesFileName = \ prot._getPath("coordinates.sqlite") print(os.path.abspath(setOfCoordinatesFileName)) indexes = sorted( [index[1] for index in getIndex(setOfCoordinatesFileName)]) for index, indexName in zip(indexes, indexesNames): self.assertEqual(index, 'index_' + indexName) # Test speed: based on loop in file protocol_extractparticles.py # for 600 mic and 100 part the values for the first # second and third case where: # Loop with index: 5 sec # Loop no index: 8:01 min # Loop optimized code: 4 sec # for 6000 mic and 200 part the values for the first # Loop with index: 1:47 min # optimized Loop with index: 1:20 min # Loop no index: after several hours I stopped the process SPEEDTEST = True if SPEEDTEST: # code from protocol_particles. line 415 testTimer = pwutils.Timer() testTimer.tic() for mic in micSet: micId = mic.getObjId() coordList = [] for coord in coordSet.iterItems(where='_micId=%s' % micId): coordList.append(coord.clone()) testTimer.toc("Loop with INDEX took:") lastMicId = None testTimer.tic() for coord in coordSet.iterItems(orderBy='_micId', direction='ASC'): micId = coord.getMicId() if micId != lastMicId: lastMicId = micId coordList = [] coordList.append(coord.clone()) testTimer.toc("Loop with INDEX and proper code, took:") # delete INDEX, this will not work # if database is not sqlite conn = sqlite3.connect(setOfCoordinatesFileName) cur = conn.cursor() for index in indexesNames: cur.execute("DROP INDEX index_%s" % index) cur.close() conn.close() testTimer.tic() for mic in micSet: micId = mic.getObjId() coordList = [] for coord in coordSet.iterItems(where='_micId=%s' % micId): coordList.append(coord.clone()) testTimer.toc("Loop with NO INDEX took:") lastMicId = None testTimer.tic() for coord in coordSet.iterItems(orderBy='_micId', direction='ASC'): micId = coord.getMicId() if micId != lastMicId: lastMicId = micId coordList = [] coordList.append(coord.clone()) testTimer.toc("Loop with NO INDEX but proper code, took:")