def _checkProcessedData(self):
if self.setof == SET_OF_MOVIES:
objSet = emobj.SetOfMovies(filename=self._getPath('movies.sqlite'))
elif self.setof == SET_OF_MICROGRAPHS:
objSet = emobj.SetOfMicrographs(filename=self._getPath('micrographs.sqlite'))
elif self.setof == SET_OF_RANDOM_MICROGRAPHS:
objSet = emobj.SetOfMicrographs(filename=self._getPath('micrographs.sqlite'))
elif self.setof == SET_OF_PARTICLES:
objSet = emobj.SetOfParticles(filename=self._getPath('particles.sqlite'))
else:
raise Exception('Unknown data type')
newObjSet, newObj = self._checkNewItems(objSet)
if self.setof == SET_OF_MOVIES:
newObjSet.setFramesRange(self.inputMovies.get().getFramesRange())
# check if end ....
if self.setof != SET_OF_PARTICLES:
self.nDims = self.nDim.get()
endObjs = newObjSet.getSize() == self.nDims
if newObj:
if endObjs:
newObjSet.setStreamState(newObjSet.STREAM_CLOSED)
self._updateOutput(newObjSet)
newObjSet.close()
def _createSubSetFromMicrographsTiltPair(self, micrographsTiltPair):
""" Create a subset of Micrographs Tilt Pair. """
output = MicrographsTiltPair(
filename=self._getPath('micrographs_pairs.sqlite'))
modifiedSet = MicrographsTiltPair(filename=self._dbName,
prefix=self._dbPrefix)
inputU = micrographsTiltPair.getUntilted()
inputT = micrographsTiltPair.getTilted()
outputU = emobj.SetOfMicrographs(
filename=self._getPath('mics_untilted.sqlite'))
outputT = emobj.SetOfMicrographs(
filename=self._getPath('mics_tilted.sqlite'))
outputU.copyInfo(inputU)
outputT.copyInfo(inputT)
# noinspection DuplicatedCode
for micPair, u, t in izip(modifiedSet, inputU, inputT):
if micPair.isEnabled():
output.append(micPair)
outputU.append(u)
outputT.append(t)
output.setUntilted(outputU)
output.setTilted(outputT)
# Register outputs
outputDict = {'outputMicrographsTiltPair': output}
self._defineOutputs(**outputDict)
self._defineTransformRelation(micrographsTiltPair, output)
return output
def loadInputs(self):
micsFn = self.getInputMicrographs().getFileName()
micsSet = emobj.SetOfMicrographs(filename=micsFn)
micsSet.loadAllProperties()
availableMics = []
for mic in micsSet:
availableMics.append(mic.getObjId())
micsSetClosed = micsSet.isStreamClosed()
micsSet.close()
partsFn = self.getInputParticles().getFileName()
partsSet = emobj.SetOfParticles(filename=partsFn)
partsSet.loadAllProperties()
newParts = []
newMics = []
for item in partsSet:
micKey = item.getCoordinate().getMicId()
if micKey not in self.micsDone and micKey in availableMics:
newParts.append(item.getObjId())
if micKey not in self.micsDone:
newMics.append(micKey)
self.micsDone += newMics
self.inputSize = partsSet.getSize()
partSetClosed = partsSet.isStreamClosed()
partsSet.close()
return newParts, micsSetClosed and partSetClosed
def testRead(self):
""" Read micrographs from a an sqlite file.
It should contains Acquisition info. """
micFn = self.dataset.getFile('micsGoldSqlite2')
print(">>> Reading gold micrographs from: ", micFn)
micSet = emobj.SetOfMicrographs(filename=micFn)
self.assertEqual(2, micSet.getSize())
acquisition = emobj.Acquisition()
acquisition.setMagnification(10000.)
acquisition.setVoltage(200.)
acquisition.setSphericalAberration(2.26)
acquisition.setAmplitudeContrast(0.1)
mic2 = emobj.Micrograph()
mic2.setSamplingRate(2.8)
mic2.setAcquisition(acquisition)
fileNames = [
'/home/roberto/Scipion/Test/Test2/down2_12585',
'/home/roberto/Scipion/Test/Test2/down2_12610'
]
counter = 0
for mic in micSet:
mic2.setFileName(fileNames[counter])
self.assertTrue(mic.equalAttributes(mic2))
counter += 1
def testWriteSetOfCoordinatesWithoutFlip(self):
from collections import OrderedDict
# Define a temporary sqlite file for micrographs
fn = self.getOutputPath('convert_mics.sqlite')
mics = emobj.SetOfMicrographs(filename=fn)
# Create SetOfCoordinates data
# Define a temporary sqlite file for coordinates
fn = self.getOutputPath('convert_coordinates.sqlite')
coordSet = emobj.SetOfCoordinates(filename=fn)
coordSet.setBoxSize(60)
coordSet.setMicrographs(mics)
data = OrderedDict()
data['006'] = [(30, 30)]
data['016'] = [(40, 40)]
micList = []
for key, coords in data.items():
mic = emobj.Micrograph(self.ds.getFile('micrographs/%s.mrc' % key))
mics.append(mic)
micList.append(mic)
print("Adding mic: %s, id: %s" % (key, mic.getObjId()))
for x, y in coords:
coord = emobj.Coordinate(x=x, y=y)
coord.setMicrograph(mic)
coordSet.append(coord)
# Get boxDirectory
boxFolder = self.getOutputPath('boxFolder')
os.mkdir(boxFolder)
micFolder = self.getOutputPath('micFolder')
pwutils.makePath(micFolder)
# Invoke the write set of coordinates method
convert.writeSetOfCoordinates(boxFolder, coordSet)
convert.convertMicrographs(micList, micFolder)
# Assert output of writesetofcoordinates
for mic in micList:
boxFile = os.path.join(boxFolder,
convert.getMicIdName(mic, '.box'))
self.assertTrue(os.path.exists(boxFile),
'Missing box file: %s' % boxFile)
micFile = os.path.join(micFolder,
convert.getMicIdName(mic, '.mrc'))
self.assertTrue(os.path.exists(micFile),
'Missing box file: %s' % micFile)
# Assert coordinates in box files
fh = open(os.path.join(boxFolder, 'mic00001.box'))
box1 = fh.readline()
fh.close()
box1 = box1.split('\t')
self.assertEquals(box1[0], '0')
self.assertEquals(box1[1], '964')
def test_mapper(self):
""" test that indexes are created when a
setOfCoordinates is created """
MICNUMBER = 10
indexesNames = ['_untilted._micId', '_tilted._micId']
prot = createDummyProtocol("dummy_protocol")
# create set of untilted and tilted micrographs
uMicSet = emobj.SetOfMicrographs(filename=":memory:")
tMicSet = emobj.SetOfMicrographs(filename=":memory:")
mic = emobj.Micrograph()
for i in range(MICNUMBER):
mic.setLocation(i, "umic_%06d.mrc" % i)
uMicSet.append(mic)
mic.cleanObjId()
mic.setLocation(i, "tmic_%06d.mrc" % i)
tMicSet.append(mic)
mic.cleanObjId()
def checkMicSet(self, micSet, goldFn):
""" Compare micrographs of micSet with the
ones in the goldFn. Maybe except the full path.
"""
goldSet = emobj.SetOfMicrographs(filename=goldFn)
for mic1, mic2 in izip(goldSet, micSet):
# Remove the absolute path in the micrographs to
# really check that the attributes should be equal
mic1.setFileName(os.path.basename(mic1.getFileName()))
mic2.setFileName(os.path.basename(mic2.getFileName()))
self.assertTrue(mic1.equalAttributes(mic2, verbose=True))
def testCreate(self):
cwd = os.getcwd()
# Change to test path
os.chdir(self.dataset.getPath())
""" Create a SetOfMicrographs from a list of micrographs """
micSet = emobj.SetOfMicrographs(filename=self.dbFn)
micSet.setAcquisition(self.acquisition)
micSet.setSamplingRate(1.2)
for fn in self.mics:
mic = emobj.Micrograph()
mic.setFileName(fn)
micSet.append(mic)
micSet.write()
self.checkSet(micSet)
# Check copy info also copies the samplingRate
micSet2 = emobj.SetOfMicrographs(filename=':memory:')
micSet2.copyInfo(micSet)
self.assertAlmostEqual(micSet.getSamplingRate(),
micSet2.getSamplingRate())
os.chdir(cwd)
def importMicrographs(self):
""" Import a SetOfMicrographs from a given sqlite file. """
inputSet = emobj.SetOfMicrographs(filename=self._sqliteFile)
self._findImagesPath(inputSet)
micSet = self.protocol._createSetOfMicrographs()
micSet.setObjComment('Micrographs imported from sqlite file:\n%s'
% self._sqliteFile)
# Update both samplingRate and acquisition with parameters
# selected in the protocol form.
# Acquisition should be updated before, just to ensure that
# scannedPixedSize will be computed properly when calling
# setSamplingRate
self.protocol.fillAcquisition(micSet.getAcquisition())
self.protocol.setSamplingRate(micSet)
micSet.setIsPhaseFlipped(self.protocol.haveDataBeenPhaseFlipped.get())
# Read the micrographs from the 'self._sqliteFile' metadata
# but fixing the filenames with new ones (linked or copy to extraDir)
micSet.copyItems(inputSet, updateItemCallback=self._updateParticle)
self.protocol._defineOutputs(outputMicrographs=micSet)
def test_fromScipion(self):
""" Import an EMX file with micrographs and defocus
"""
micsSqlite = self.dsXmipp.getFile('micrographs/micrographs.sqlite')
print("Importing from sqlite: ", micsSqlite)
micSet = emobj.SetOfMicrographs(filename=micsSqlite)
# Gold values
# _samplingRate -> 1.237
# _acquisition._voltage -> 300.0
# _acquisition._magnification -> 50000.0
for k in [
'_samplingRate', '_acquisition._voltage',
'_acquisition._magnification'
]:
print(k, "->", micSet.getProperty(k))
prot = self.newProtocol(
emprot.ProtImportMicrographs,
objLabel='from scipion',
importFrom=emprot.ProtImportMicrographs.IMPORT_FROM_SCIPION,
sqliteFile=micsSqlite,
samplingRate=float(micSet.getProperty('_samplingRate')),
voltage=float(micSet.getProperty('_acquisition._voltage')),
magnification=int(
float(micSet.getProperty('_acquisition._magnification'))))
self.launchProtocol(prot)
micSet = getattr(prot, 'outputMicrographs', None)
self.assertIsNotNone(micSet)
self.assertAlmostEqual(1.237, micSet.getSamplingRate())
acq = micSet.getAcquisition()
self.assertAlmostEqual(300., acq.getVoltage())
self.assertAlmostEqual(50000., acq.getMagnification())
def test_mapper(self):
""" test that indexes are created when a
setOfCoordinates is created """
PARTNUMBER = 10
MICNUMBER = 600
NUMBERCOORDINATES = PARTNUMBER * MICNUMBER
indexesNames = ['_micId']
prot = createDummyProtocol("dummy_protocol")
# create set of micrographs
micSet = emobj.SetOfMicrographs(filename=":memory:")
mic = emobj.Micrograph()
for i in range(NUMBERCOORDINATES):
mic.setLocation(i, "mic_%06d.mrc" % i)
micSet.append(mic)
mic.cleanObjId()
micSet.write()
# create a set of particles
coordSet = prot._createSetOfCoordinates(micSet)
coord = emobj.Coordinate()
for i in range(NUMBERCOORDINATES):
coordSet.append(coord)
coord.cleanObjId()
coordSet.write()
# check defined indexes
setOfCoordinatesFileName = \
prot._getPath("coordinates.sqlite")
print(os.path.abspath(setOfCoordinatesFileName))
indexes = sorted(
[index[1] for index in getIndex(setOfCoordinatesFileName)])
for index, indexName in zip(indexes, indexesNames):
self.assertEqual(index, 'index_' + indexName)
# Test speed: based on loop in file protocol_extractparticles.py
# for 600 mic and 100 part the values for the first
# second and third case where:
# Loop with index: 5 sec
# Loop no index: 8:01 min
# Loop optimized code: 4 sec
# for 6000 mic and 200 part the values for the first
# Loop with index: 1:47 min
# optimized Loop with index: 1:20 min
# Loop no index: after several hours I stopped the process
SPEEDTEST = True
if SPEEDTEST: # code from protocol_particles. line 415
testTimer = pwutils.Timer()
testTimer.tic()
for mic in micSet:
micId = mic.getObjId()
coordList = []
for coord in coordSet.iterItems(where='_micId=%s' % micId):
coordList.append(coord.clone())
testTimer.toc("Loop with INDEX took:")
lastMicId = None
testTimer.tic()
for coord in coordSet.iterItems(orderBy='_micId', direction='ASC'):
micId = coord.getMicId()
if micId != lastMicId:
lastMicId = micId
coordList = []
coordList.append(coord.clone())
testTimer.toc("Loop with INDEX and proper code, took:")
# delete INDEX, this will not work
# if database is not sqlite
conn = sqlite3.connect(setOfCoordinatesFileName)
cur = conn.cursor()
for index in indexesNames:
cur.execute("DROP INDEX index_%s" % index)
cur.close()
conn.close()
testTimer.tic()
for mic in micSet:
micId = mic.getObjId()
coordList = []
for coord in coordSet.iterItems(where='_micId=%s' % micId):
coordList.append(coord.clone())
testTimer.toc("Loop with NO INDEX took:")
lastMicId = None
testTimer.tic()
for coord in coordSet.iterItems(orderBy='_micId', direction='ASC'):
micId = coord.getMicId()
if micId != lastMicId:
lastMicId = micId
coordList = []
coordList.append(coord.clone())
testTimer.toc("Loop with NO INDEX but proper code, took:")
