def testRead(self):
""" Read micrographs from a an sqlite file.
It should contains Acquisition info. """
micFn = self.dataset.getFile('micsGoldSqlite2')
print(">>> Reading gold micrographs from: ", micFn)
micSet = emobj.SetOfMicrographs(filename=micFn)
self.assertEqual(2, micSet.getSize())
acquisition = emobj.Acquisition()
acquisition.setMagnification(10000.)
acquisition.setVoltage(200.)
acquisition.setSphericalAberration(2.26)
acquisition.setAmplitudeContrast(0.1)
mic2 = emobj.Micrograph()
mic2.setSamplingRate(2.8)
mic2.setAcquisition(acquisition)
fileNames = [
'/home/roberto/Scipion/Test/Test2/down2_12585',
'/home/roberto/Scipion/Test/Test2/down2_12610'
]
counter = 0
for mic in micSet:
mic2.setFileName(fileNames[counter])
self.assertTrue(mic.equalAttributes(mic2))
counter += 1
def testWriteSetOfCoordinatesWithoutFlip(self):
from collections import OrderedDict
# Define a temporary sqlite file for micrographs
fn = self.getOutputPath('convert_mics.sqlite')
mics = emobj.SetOfMicrographs(filename=fn)
# Create SetOfCoordinates data
# Define a temporary sqlite file for coordinates
fn = self.getOutputPath('convert_coordinates.sqlite')
coordSet = emobj.SetOfCoordinates(filename=fn)
coordSet.setBoxSize(60)
coordSet.setMicrographs(mics)
data = OrderedDict()
data['006'] = [(30, 30)]
data['016'] = [(40, 40)]
micList = []
for key, coords in data.items():
mic = emobj.Micrograph(self.ds.getFile('micrographs/%s.mrc' % key))
mics.append(mic)
micList.append(mic)
print("Adding mic: %s, id: %s" % (key, mic.getObjId()))
for x, y in coords:
coord = emobj.Coordinate(x=x, y=y)
coord.setMicrograph(mic)
coordSet.append(coord)
# Get boxDirectory
boxFolder = self.getOutputPath('boxFolder')
os.mkdir(boxFolder)
micFolder = self.getOutputPath('micFolder')
pwutils.makePath(micFolder)
# Invoke the write set of coordinates method
convert.writeSetOfCoordinates(boxFolder, coordSet)
convert.convertMicrographs(micList, micFolder)
# Assert output of writesetofcoordinates
for mic in micList:
boxFile = os.path.join(boxFolder,
convert.getMicIdName(mic, '.box'))
self.assertTrue(os.path.exists(boxFile),
'Missing box file: %s' % boxFile)
micFile = os.path.join(micFolder,
convert.getMicIdName(mic, '.mrc'))
self.assertTrue(os.path.exists(micFile),
'Missing box file: %s' % micFile)
# Assert coordinates in box files
fh = open(os.path.join(boxFolder, 'mic00001.box'))
box1 = fh.readline()
fh.close()
box1 = box1.split('\t')
self.assertEquals(box1[0], '0')
self.assertEquals(box1[1], '964')
def testLocation(self):
fn = self.mic1
mic = emobj.Micrograph()
mic.setFileName(fn)
# Check that setFileName-getFileName is working properly
self.assertEqual(fn, mic.getFileName())
# Check the location is accepted from constructor
mic2 = emobj.Micrograph(fn)
self.assertEqual(fn, mic2.getFileName())
volStk = '/data/current/volumes/all_volumes.stk'
vol1 = emobj.Volume((1, volStk))
self.assertEqual(1, vol1.getIndex())
self.assertEqual(volStk, vol1.getFileName())
self.assertEqual('all_volumes.stk', vol1.getBaseName())
def testOrigin(self):
fn = self.mic1
mic = emobj.Micrograph()
mic.setFileName(fn)
mic.setSamplingRate(1.)
referenceT = emobj.Transform()
referenceT.setShifts(-4608., -4720., 1)
mreferenceT = referenceT.getMatrix()
t = mic.getOrigin(True)
mt = t.getMatrix()
for i in range(4):
for j in range(4):
self.assertAlmostEquals(mt[i][j], mreferenceT[i][j], delta=0.5)
def test_mapper(self):
""" test that indexes are created when a
setOfCoordinates is created """
MICNUMBER = 10
indexesNames = ['_untilted._micId', '_tilted._micId']
prot = createDummyProtocol("dummy_protocol")
# create set of untilted and tilted micrographs
uMicSet = emobj.SetOfMicrographs(filename=":memory:")
tMicSet = emobj.SetOfMicrographs(filename=":memory:")
mic = emobj.Micrograph()
for i in range(MICNUMBER):
mic.setLocation(i, "umic_%06d.mrc" % i)
uMicSet.append(mic)
mic.cleanObjId()
mic.setLocation(i, "tmic_%06d.mrc" % i)
tMicSet.append(mic)
mic.cleanObjId()
def testConvertMic(self):
"""Check extension of the input micrographs"""
micDir = self.getOutputPath('micDir')
os.mkdir(micDir)
mrcMic = TestSphireConvert.ds.getFile('micrographs/006.mrc')
spiMic = os.path.join(micDir, "mic.spi")
ImageHandler().convert(mrcMic, spiMic)
mic = emobj.Micrograph(objId=1, location=spiMic)
# Invoke the createMic function
convert.convertMicrographs([mic], micDir)
expectedDest = os.path.join(micDir, convert.getMicIdName(mic, '.mrc'))
# If ext is not in [".mrc", ".tif", ".jpg"] return .mrc
self.assertTrue(os.path.exists(expectedDest),
"spi file wasn't converted to mrc.")
def test_mapper(self):
""" test that indexes are created when a
setOfParticles is created """
MICNUMBER = 10
indexesNames = ['_index']
prot = createDummyProtocol("dummy_protocol")
# create set of micrographs
micSet = prot._createSetOfMicrographs()
mic = emobj.Micrograph()
for i in range(MICNUMBER + 1):
mic.setLocation(i, "stack.stk")
micSet.append(mic)
mic.cleanObjId()
micSet.write()
# check defined indexes
setOfMicrographsFileName = prot._getPath("micrographs.sqlite")
# print os.path.abspath(setOfPArticleFileName)
indexes = sorted(
[index[1] for index in getIndex(setOfMicrographsFileName)])
for index, indexName in zip(indexes, indexesNames):
self.assertEqual(index, 'index_' + indexName)
def testCreate(self):
cwd = os.getcwd()
# Change to test path
os.chdir(self.dataset.getPath())
""" Create a SetOfMicrographs from a list of micrographs """
micSet = emobj.SetOfMicrographs(filename=self.dbFn)
micSet.setAcquisition(self.acquisition)
micSet.setSamplingRate(1.2)
for fn in self.mics:
mic = emobj.Micrograph()
mic.setFileName(fn)
micSet.append(mic)
micSet.write()
self.checkSet(micSet)
# Check copy info also copies the samplingRate
micSet2 = emobj.SetOfMicrographs(filename=':memory:')
micSet2.copyInfo(micSet)
self.assertAlmostEqual(micSet.getSamplingRate(),
micSet2.getSamplingRate())
os.chdir(cwd)
def _checkNewItems(self, objSet):
""" Check for already computed micrograph/movie and
update the output set. """
objDict = {}
newObj = False
for obj in objSet:
objDict[obj.getFileName()] = True
if objDict:
if objSet.getSize():
objSet.enableAppend()
objSet.loadAllProperties()
else:
objSet.setStreamState(objSet.STREAM_OPEN)
acquisition = emobj.Acquisition()
if self.setof == SET_OF_MICROGRAPHS:
acquisition.setMagnification(
self.inputMics.get().getAcquisition().getMagnification())
acquisition.setVoltage(
self.inputMics.get().getAcquisition().getVoltage())
acquisition.setSphericalAberration(
self.inputMics.get().getAcquisition().getSphericalAberration())
acquisition.setAmplitudeContrast(
self.inputMics.get().getAcquisition().getAmplitudeContrast())
objSet.setAcquisition(acquisition)
objSet.setSamplingRate(self.inputMics.get().getSamplingRate())
elif self.setof == SET_OF_MOVIES:
acquisition.setMagnification(
self.inputMovies.get().getAcquisition().getMagnification())
acquisition.setVoltage(
self.inputMovies.get().getAcquisition().getVoltage())
acquisition.setSphericalAberration(
self.inputMovies.get().getAcquisition().getSphericalAberration())
acquisition.setAmplitudeContrast(
self.inputMovies.get().getAcquisition().getAmplitudeContrast())
objSet.setAcquisition(acquisition)
objSet.setSamplingRate(
self.inputMovies.get().getSamplingRate())
else:
acquisition.setMagnification(self._magnification)
acquisition.setVoltage(self._voltage)
acquisition.setSphericalAberration(self._sphericalAberration)
acquisition.setAmplitudeContrast(self._amplitudeContrast)
objSet.setAcquisition(acquisition)
if self.setof == SET_OF_PARTICLES:
objSet.setSamplingRate(
self.inputParticles.get().getSamplingRate())
else:
objSet.setSamplingRate(self.samplingRate.get())
if self.setof == SET_OF_MOVIES:
obj = emobj.Movie()
elif self.setof == SET_OF_MICROGRAPHS:
obj = emobj.Micrograph()
elif self.setof == SET_OF_RANDOM_MICROGRAPHS:
obj = emobj.Micrograph()
elif self.setof == SET_OF_PARTICLES:
obj = emobj.Particle()
else:
raise Exception('Unknown data type')
for k, v in self.dictObj.items():
if k not in objDict:
self.counter += 1
obj.setFileName(k)
if self.setof != SET_OF_PARTICLES:
obj.setMicName(basename(k))
obj.setObjId(self.counter)
objSet.append(obj)
newObj = True
return objSet, newObj # why a dictionary, a boolean may be enough
def createCoorSetProtocol(self):
MICNUMBER = 2
MICSIZE = 1000
# create dummy protocol
dummyProt = self.newProtocol(EMProtocol)
dummyProt.setObjLabel('dummy protocol')
self.launchProtocol(dummyProt)
# create set of micrographs
# create two tif image
matrix = np.zeros((MICSIZE, MICSIZE), np.uint8)
R = np.linspace(0, MICSIZE // 2 - 2, 20)
# note we have 0 and 2*pi for each radius
THETA = np.linspace(0, 2 * np.pi, 30)
radii, thetas = np.meshgrid(R, THETA)
X = (R * np.cos(thetas)).astype(int) + MICSIZE // 2
Y = (R * np.sin(thetas)).astype(int) + MICSIZE // 2
# TODO: next double loop should be equivalen to
# the more elegant solution
# pairlist = np.vstack(list(map(np.ravel, (X,Y)))).T
# matrix[pairList] = 255 but it is not
for xx, yy in zip(X, Y):
for x, y in zip(xx, yy):
matrix[y][x] = 255
fn = dummyProt._getExtraPath('mic_000001.tif')
Image.fromarray(matrix).save(fn)
fn = dummyProt._getExtraPath('mic_000002.tif')
Image.fromarray(matrix).save(fn)
micSet = dummyProt._createSetOfMicrographs()
mic = emobj.Micrograph()
for i in range(MICNUMBER):
mic.setFileName\
(dummyProt._getExtraPath("mic_%06d.tif" % (i%2 + 1 )))
mic.setMicName("mic_%06d.tif" % (i % 2 + 1))
mic.setSamplingRate(1.)
micSet.append(mic)
mic.cleanObjId()
micSet.setSamplingRate(1.)
micSet.write()
coordSet = dummyProt._createSetOfCoordinates(micSet)
coordSet.setBoxSize(10)
coord = emobj.Coordinate()
for xx, yy in zip(X, Y):
for x, y in zip(xx, yy):
for mic in range(1, MICNUMBER + 1):
coord.setX(x)
coord.setY(y)
coord.setMicId(mic)
coord.setMicName("mic_%06d.tif" % (mic))
coordSet.append(coord)
coord.cleanObjId()
coordSet.write()
outputArgs = {OUTPUT_COORDINATES: coordSet, 'outputMic': micSet}
dummyProt._defineOutputs(**outputArgs)
dummyProt._store()
return dummyProt
def test_mapper(self):
""" test that indexes are created when a
setOfCoordinates is created """
PARTNUMBER = 10
MICNUMBER = 600
NUMBERCOORDINATES = PARTNUMBER * MICNUMBER
indexesNames = ['_micId']
prot = createDummyProtocol("dummy_protocol")
# create set of micrographs
micSet = emobj.SetOfMicrographs(filename=":memory:")
mic = emobj.Micrograph()
for i in range(NUMBERCOORDINATES):
mic.setLocation(i, "mic_%06d.mrc" % i)
micSet.append(mic)
mic.cleanObjId()
micSet.write()
# create a set of particles
coordSet = prot._createSetOfCoordinates(micSet)
coord = emobj.Coordinate()
for i in range(NUMBERCOORDINATES):
coordSet.append(coord)
coord.cleanObjId()
coordSet.write()
# check defined indexes
setOfCoordinatesFileName = \
prot._getPath("coordinates.sqlite")
print(os.path.abspath(setOfCoordinatesFileName))
indexes = sorted(
[index[1] for index in getIndex(setOfCoordinatesFileName)])
for index, indexName in zip(indexes, indexesNames):
self.assertEqual(index, 'index_' + indexName)
# Test speed: based on loop in file protocol_extractparticles.py
# for 600 mic and 100 part the values for the first
# second and third case where:
# Loop with index: 5 sec
# Loop no index: 8:01 min
# Loop optimized code: 4 sec
# for 6000 mic and 200 part the values for the first
# Loop with index: 1:47 min
# optimized Loop with index: 1:20 min
# Loop no index: after several hours I stopped the process
SPEEDTEST = True
if SPEEDTEST: # code from protocol_particles. line 415
testTimer = pwutils.Timer()
testTimer.tic()
for mic in micSet:
micId = mic.getObjId()
coordList = []
for coord in coordSet.iterItems(where='_micId=%s' % micId):
coordList.append(coord.clone())
testTimer.toc("Loop with INDEX took:")
lastMicId = None
testTimer.tic()
for coord in coordSet.iterItems(orderBy='_micId', direction='ASC'):
micId = coord.getMicId()
if micId != lastMicId:
lastMicId = micId
coordList = []
coordList.append(coord.clone())
testTimer.toc("Loop with INDEX and proper code, took:")
# delete INDEX, this will not work
# if database is not sqlite
conn = sqlite3.connect(setOfCoordinatesFileName)
cur = conn.cursor()
for index in indexesNames:
cur.execute("DROP INDEX index_%s" % index)
cur.close()
conn.close()
testTimer.tic()
for mic in micSet:
micId = mic.getObjId()
coordList = []
for coord in coordSet.iterItems(where='_micId=%s' % micId):
coordList.append(coord.clone())
testTimer.toc("Loop with NO INDEX took:")
lastMicId = None
testTimer.tic()
for coord in coordSet.iterItems(orderBy='_micId', direction='ASC'):
micId = coord.getMicId()
if micId != lastMicId:
lastMicId = micId
coordList = []
coordList.append(coord.clone())
testTimer.toc("Loop with NO INDEX but proper code, took:")
