def testRead(self): """ Read micrographs from a an sqlite file. It should contains Acquisition info. """ micFn = self.dataset.getFile('micsGoldSqlite2') print(">>> Reading gold micrographs from: ", micFn) micSet = emobj.SetOfMicrographs(filename=micFn) self.assertEqual(2, micSet.getSize()) acquisition = emobj.Acquisition() acquisition.setMagnification(10000.) acquisition.setVoltage(200.) acquisition.setSphericalAberration(2.26) acquisition.setAmplitudeContrast(0.1) mic2 = emobj.Micrograph() mic2.setSamplingRate(2.8) mic2.setAcquisition(acquisition) fileNames = [ '/home/roberto/Scipion/Test/Test2/down2_12585', '/home/roberto/Scipion/Test/Test2/down2_12610' ] counter = 0 for mic in micSet: mic2.setFileName(fileNames[counter]) self.assertTrue(mic.equalAttributes(mic2)) counter += 1
def testWriteSetOfCoordinatesWithoutFlip(self): from collections import OrderedDict # Define a temporary sqlite file for micrographs fn = self.getOutputPath('convert_mics.sqlite') mics = emobj.SetOfMicrographs(filename=fn) # Create SetOfCoordinates data # Define a temporary sqlite file for coordinates fn = self.getOutputPath('convert_coordinates.sqlite') coordSet = emobj.SetOfCoordinates(filename=fn) coordSet.setBoxSize(60) coordSet.setMicrographs(mics) data = OrderedDict() data['006'] = [(30, 30)] data['016'] = [(40, 40)] micList = [] for key, coords in data.items(): mic = emobj.Micrograph(self.ds.getFile('micrographs/%s.mrc' % key)) mics.append(mic) micList.append(mic) print("Adding mic: %s, id: %s" % (key, mic.getObjId())) for x, y in coords: coord = emobj.Coordinate(x=x, y=y) coord.setMicrograph(mic) coordSet.append(coord) # Get boxDirectory boxFolder = self.getOutputPath('boxFolder') os.mkdir(boxFolder) micFolder = self.getOutputPath('micFolder') pwutils.makePath(micFolder) # Invoke the write set of coordinates method convert.writeSetOfCoordinates(boxFolder, coordSet) convert.convertMicrographs(micList, micFolder) # Assert output of writesetofcoordinates for mic in micList: boxFile = os.path.join(boxFolder, convert.getMicIdName(mic, '.box')) self.assertTrue(os.path.exists(boxFile), 'Missing box file: %s' % boxFile) micFile = os.path.join(micFolder, convert.getMicIdName(mic, '.mrc')) self.assertTrue(os.path.exists(micFile), 'Missing box file: %s' % micFile) # Assert coordinates in box files fh = open(os.path.join(boxFolder, 'mic00001.box')) box1 = fh.readline() fh.close() box1 = box1.split('\t') self.assertEquals(box1[0], '0') self.assertEquals(box1[1], '964')
def testLocation(self): fn = self.mic1 mic = emobj.Micrograph() mic.setFileName(fn) # Check that setFileName-getFileName is working properly self.assertEqual(fn, mic.getFileName()) # Check the location is accepted from constructor mic2 = emobj.Micrograph(fn) self.assertEqual(fn, mic2.getFileName()) volStk = '/data/current/volumes/all_volumes.stk' vol1 = emobj.Volume((1, volStk)) self.assertEqual(1, vol1.getIndex()) self.assertEqual(volStk, vol1.getFileName()) self.assertEqual('all_volumes.stk', vol1.getBaseName())
def testOrigin(self): fn = self.mic1 mic = emobj.Micrograph() mic.setFileName(fn) mic.setSamplingRate(1.) referenceT = emobj.Transform() referenceT.setShifts(-4608., -4720., 1) mreferenceT = referenceT.getMatrix() t = mic.getOrigin(True) mt = t.getMatrix() for i in range(4): for j in range(4): self.assertAlmostEquals(mt[i][j], mreferenceT[i][j], delta=0.5)
def test_mapper(self): """ test that indexes are created when a setOfCoordinates is created """ MICNUMBER = 10 indexesNames = ['_untilted._micId', '_tilted._micId'] prot = createDummyProtocol("dummy_protocol") # create set of untilted and tilted micrographs uMicSet = emobj.SetOfMicrographs(filename=":memory:") tMicSet = emobj.SetOfMicrographs(filename=":memory:") mic = emobj.Micrograph() for i in range(MICNUMBER): mic.setLocation(i, "umic_%06d.mrc" % i) uMicSet.append(mic) mic.cleanObjId() mic.setLocation(i, "tmic_%06d.mrc" % i) tMicSet.append(mic) mic.cleanObjId()
def testConvertMic(self): """Check extension of the input micrographs""" micDir = self.getOutputPath('micDir') os.mkdir(micDir) mrcMic = TestSphireConvert.ds.getFile('micrographs/006.mrc') spiMic = os.path.join(micDir, "mic.spi") ImageHandler().convert(mrcMic, spiMic) mic = emobj.Micrograph(objId=1, location=spiMic) # Invoke the createMic function convert.convertMicrographs([mic], micDir) expectedDest = os.path.join(micDir, convert.getMicIdName(mic, '.mrc')) # If ext is not in [".mrc", ".tif", ".jpg"] return .mrc self.assertTrue(os.path.exists(expectedDest), "spi file wasn't converted to mrc.")
def test_mapper(self): """ test that indexes are created when a setOfParticles is created """ MICNUMBER = 10 indexesNames = ['_index'] prot = createDummyProtocol("dummy_protocol") # create set of micrographs micSet = prot._createSetOfMicrographs() mic = emobj.Micrograph() for i in range(MICNUMBER + 1): mic.setLocation(i, "stack.stk") micSet.append(mic) mic.cleanObjId() micSet.write() # check defined indexes setOfMicrographsFileName = prot._getPath("micrographs.sqlite") # print os.path.abspath(setOfPArticleFileName) indexes = sorted( [index[1] for index in getIndex(setOfMicrographsFileName)]) for index, indexName in zip(indexes, indexesNames): self.assertEqual(index, 'index_' + indexName)
def testCreate(self): cwd = os.getcwd() # Change to test path os.chdir(self.dataset.getPath()) """ Create a SetOfMicrographs from a list of micrographs """ micSet = emobj.SetOfMicrographs(filename=self.dbFn) micSet.setAcquisition(self.acquisition) micSet.setSamplingRate(1.2) for fn in self.mics: mic = emobj.Micrograph() mic.setFileName(fn) micSet.append(mic) micSet.write() self.checkSet(micSet) # Check copy info also copies the samplingRate micSet2 = emobj.SetOfMicrographs(filename=':memory:') micSet2.copyInfo(micSet) self.assertAlmostEqual(micSet.getSamplingRate(), micSet2.getSamplingRate()) os.chdir(cwd)
def _checkNewItems(self, objSet): """ Check for already computed micrograph/movie and update the output set. """ objDict = {} newObj = False for obj in objSet: objDict[obj.getFileName()] = True if objDict: if objSet.getSize(): objSet.enableAppend() objSet.loadAllProperties() else: objSet.setStreamState(objSet.STREAM_OPEN) acquisition = emobj.Acquisition() if self.setof == SET_OF_MICROGRAPHS: acquisition.setMagnification( self.inputMics.get().getAcquisition().getMagnification()) acquisition.setVoltage( self.inputMics.get().getAcquisition().getVoltage()) acquisition.setSphericalAberration( self.inputMics.get().getAcquisition().getSphericalAberration()) acquisition.setAmplitudeContrast( self.inputMics.get().getAcquisition().getAmplitudeContrast()) objSet.setAcquisition(acquisition) objSet.setSamplingRate(self.inputMics.get().getSamplingRate()) elif self.setof == SET_OF_MOVIES: acquisition.setMagnification( self.inputMovies.get().getAcquisition().getMagnification()) acquisition.setVoltage( self.inputMovies.get().getAcquisition().getVoltage()) acquisition.setSphericalAberration( self.inputMovies.get().getAcquisition().getSphericalAberration()) acquisition.setAmplitudeContrast( self.inputMovies.get().getAcquisition().getAmplitudeContrast()) objSet.setAcquisition(acquisition) objSet.setSamplingRate( self.inputMovies.get().getSamplingRate()) else: acquisition.setMagnification(self._magnification) acquisition.setVoltage(self._voltage) acquisition.setSphericalAberration(self._sphericalAberration) acquisition.setAmplitudeContrast(self._amplitudeContrast) objSet.setAcquisition(acquisition) if self.setof == SET_OF_PARTICLES: objSet.setSamplingRate( self.inputParticles.get().getSamplingRate()) else: objSet.setSamplingRate(self.samplingRate.get()) if self.setof == SET_OF_MOVIES: obj = emobj.Movie() elif self.setof == SET_OF_MICROGRAPHS: obj = emobj.Micrograph() elif self.setof == SET_OF_RANDOM_MICROGRAPHS: obj = emobj.Micrograph() elif self.setof == SET_OF_PARTICLES: obj = emobj.Particle() else: raise Exception('Unknown data type') for k, v in self.dictObj.items(): if k not in objDict: self.counter += 1 obj.setFileName(k) if self.setof != SET_OF_PARTICLES: obj.setMicName(basename(k)) obj.setObjId(self.counter) objSet.append(obj) newObj = True return objSet, newObj # why a dictionary, a boolean may be enough
def createCoorSetProtocol(self): MICNUMBER = 2 MICSIZE = 1000 # create dummy protocol dummyProt = self.newProtocol(EMProtocol) dummyProt.setObjLabel('dummy protocol') self.launchProtocol(dummyProt) # create set of micrographs # create two tif image matrix = np.zeros((MICSIZE, MICSIZE), np.uint8) R = np.linspace(0, MICSIZE // 2 - 2, 20) # note we have 0 and 2*pi for each radius THETA = np.linspace(0, 2 * np.pi, 30) radii, thetas = np.meshgrid(R, THETA) X = (R * np.cos(thetas)).astype(int) + MICSIZE // 2 Y = (R * np.sin(thetas)).astype(int) + MICSIZE // 2 # TODO: next double loop should be equivalen to # the more elegant solution # pairlist = np.vstack(list(map(np.ravel, (X,Y)))).T # matrix[pairList] = 255 but it is not for xx, yy in zip(X, Y): for x, y in zip(xx, yy): matrix[y][x] = 255 fn = dummyProt._getExtraPath('mic_000001.tif') Image.fromarray(matrix).save(fn) fn = dummyProt._getExtraPath('mic_000002.tif') Image.fromarray(matrix).save(fn) micSet = dummyProt._createSetOfMicrographs() mic = emobj.Micrograph() for i in range(MICNUMBER): mic.setFileName\ (dummyProt._getExtraPath("mic_%06d.tif" % (i%2 + 1 ))) mic.setMicName("mic_%06d.tif" % (i % 2 + 1)) mic.setSamplingRate(1.) micSet.append(mic) mic.cleanObjId() micSet.setSamplingRate(1.) micSet.write() coordSet = dummyProt._createSetOfCoordinates(micSet) coordSet.setBoxSize(10) coord = emobj.Coordinate() for xx, yy in zip(X, Y): for x, y in zip(xx, yy): for mic in range(1, MICNUMBER + 1): coord.setX(x) coord.setY(y) coord.setMicId(mic) coord.setMicName("mic_%06d.tif" % (mic)) coordSet.append(coord) coord.cleanObjId() coordSet.write() outputArgs = {OUTPUT_COORDINATES: coordSet, 'outputMic': micSet} dummyProt._defineOutputs(**outputArgs) dummyProt._store() return dummyProt
def test_mapper(self): """ test that indexes are created when a setOfCoordinates is created """ PARTNUMBER = 10 MICNUMBER = 600 NUMBERCOORDINATES = PARTNUMBER * MICNUMBER indexesNames = ['_micId'] prot = createDummyProtocol("dummy_protocol") # create set of micrographs micSet = emobj.SetOfMicrographs(filename=":memory:") mic = emobj.Micrograph() for i in range(NUMBERCOORDINATES): mic.setLocation(i, "mic_%06d.mrc" % i) micSet.append(mic) mic.cleanObjId() micSet.write() # create a set of particles coordSet = prot._createSetOfCoordinates(micSet) coord = emobj.Coordinate() for i in range(NUMBERCOORDINATES): coordSet.append(coord) coord.cleanObjId() coordSet.write() # check defined indexes setOfCoordinatesFileName = \ prot._getPath("coordinates.sqlite") print(os.path.abspath(setOfCoordinatesFileName)) indexes = sorted( [index[1] for index in getIndex(setOfCoordinatesFileName)]) for index, indexName in zip(indexes, indexesNames): self.assertEqual(index, 'index_' + indexName) # Test speed: based on loop in file protocol_extractparticles.py # for 600 mic and 100 part the values for the first # second and third case where: # Loop with index: 5 sec # Loop no index: 8:01 min # Loop optimized code: 4 sec # for 6000 mic and 200 part the values for the first # Loop with index: 1:47 min # optimized Loop with index: 1:20 min # Loop no index: after several hours I stopped the process SPEEDTEST = True if SPEEDTEST: # code from protocol_particles. line 415 testTimer = pwutils.Timer() testTimer.tic() for mic in micSet: micId = mic.getObjId() coordList = [] for coord in coordSet.iterItems(where='_micId=%s' % micId): coordList.append(coord.clone()) testTimer.toc("Loop with INDEX took:") lastMicId = None testTimer.tic() for coord in coordSet.iterItems(orderBy='_micId', direction='ASC'): micId = coord.getMicId() if micId != lastMicId: lastMicId = micId coordList = [] coordList.append(coord.clone()) testTimer.toc("Loop with INDEX and proper code, took:") # delete INDEX, this will not work # if database is not sqlite conn = sqlite3.connect(setOfCoordinatesFileName) cur = conn.cursor() for index in indexesNames: cur.execute("DROP INDEX index_%s" % index) cur.close() conn.close() testTimer.tic() for mic in micSet: micId = mic.getObjId() coordList = [] for coord in coordSet.iterItems(where='_micId=%s' % micId): coordList.append(coord.clone()) testTimer.toc("Loop with NO INDEX took:") lastMicId = None testTimer.tic() for coord in coordSet.iterItems(orderBy='_micId', direction='ASC'): micId = coord.getMicId() if micId != lastMicId: lastMicId = micId coordList = [] coordList.append(coord.clone()) testTimer.toc("Loop with NO INDEX but proper code, took:")