def getCoords(self):
# to support multiple access to db
coordSet = self.inputCoordinates.get()
coordSetCopy = SetOfCoordinates()
coordSetCopy.copy(coordSet)
coordSet.close()
return coordSetCopy
def _convertCoords(self, micSet, tmpDir, coordsType):
""" Link specified coord set to tmpDir folder and convert it to .pos files"""
coordTypes = {
'autopick': 'coordinates.sqlite',
'rejected': 'coordinates_rejected.sqlite'
}
coordsFnIn = self.protocol._getPath(coordTypes[coordsType])
coordsFnOut = pwutils.join(tmpDir, 'coordinates.sqlite')
pwutils.createLink(coordsFnIn, coordsFnOut)
coordSet = SetOfCoordinates(filename=coordsFnOut)
coordSet.setMicrographs(micSet)
from .convert import writeSetOfCoordinatesXmipp
writeSetOfCoordinatesXmipp(tmpDir, coordSet, ismanual=False)
def getReadyMics(coordSet):
coorSet = SetOfCoordinates(filename=coordSet.getFileName())
coorSet._xmippMd = String()
coorSet.loadAllProperties()
setClosed = coorSet.isStreamClosed()
coorSet.close()
currentPickMics = {
micAgg["_micId"]
for micAgg in coordSet.aggregate(["MAX"], "_micId", ["_micId"])
}
return currentPickMics, setClosed
def readSetOfCoordsFromPosFnames(posDir, setOfInputCoords, sqliteOutName,
write=True):
"""
posDir: path where there are .pos files with coordinates
setOfInputCoords. Set to find micrographs
sqliteOutName. Path where sqlite map will be created. Warning, it overwrites
content
"""
inputMics = setOfInputCoords.getMicrographs()
cleanPath(sqliteOutName)
setOfOutputCoordinates= SetOfCoordinates(filename= sqliteOutName)
setOfOutputCoordinates.setMicrographs(inputMics)
setOfOutputCoordinates.setBoxSize(setOfInputCoords.getBoxSize())
readSetOfCoordinates(posDir, micSet=inputMics,
coordSet=setOfOutputCoordinates,
readDiscarded=False)
if write:
setOfOutputCoordinates.write()
return setOfOutputCoordinates
def test_coodinatesTest1(self):
""" Import an EMX file with just one micrograph
and a few coordinates.
"""
protEmxImport = self.newProtocol(
ProtImportParticles,
objLabel='from emx (coordinatesT1)',
importFrom=ProtImportParticles.IMPORT_FROM_EMX,
emxFile=self.dataset.getFile('coordinatesT1'),
alignType=3,
voltage=100,
magnification=10000,
samplingRate=2.46)
self.launchProtocol(protEmxImport)
# Reference coordinates
coords = SetOfCoordinates(
filename=self.dataset.getFile('coordinatesGoldT1'))
tests.BaseTest.compareSets(self, protEmxImport.outputCoordinates,
coords)
def main():
parser = argparse.ArgumentParser(prog=CONVERT_ENTRY_POINT,
usage="pwem command to convert metadata "
"coordinates from several em formats "
"(eman, relion, gautomatch "
"and bsoft) to xmipp metadata")
parser.add_argument('--coordinates',
help='Convert coordinates',
action="store_true")
parser.add_argument('--fromType', help='Convert from input type')
parser.add_argument('--toType', help='Convert to output type')
parser.add_argument('--input', help='Input file or folder', required=True)
parser.add_argument('--output', help='Output file or folder')
parser.add_argument('--extra', help='To add extra parameters')
args = parser.parse_args()
fromType = args.fromType
toType = args.toType
input = args.input
output = args.output
if args.coordinates:
micSet = loadSetFromDb(input)
outputDir = output
coordsfn = os.path.join(outputDir, 'coordinates.sqlite')
cleanPath(coordsfn)
coordSet = SetOfCoordinates(filename=coordsfn)
coordSet.setMicrographs(micSet)
if fromType == 'eman2':
if toType == 'xmipp':
readSetOfCoordinates = pwutils.Config.getDomain(
).importFromPlugin('eman2.convert',
'readSetOfCoordinates',
doRaise=True)
elif fromType == 'dogpicker':
if toType == 'xmipp':
readSetOfCoordinates = pwutils.Config.getDomain(
).importFromPlugin('appion.convert',
'readSetOfCoordinates',
doRaise=True)
elif fromType == 'relion':
if toType == 'xmipp':
def readSetOfCoordinates(outputDir, micSet, coordSet):
readSetOfCoordinates = pwutils.Config.getDomain(
).importFromPlugin('relion.convert',
'readSetOfCoordinates',
doRaise=True)
inputCoords = args.extra
starFiles = [
os.path.join(inputCoords,
"mic_%06d_autopick.star" % mic.getObjId())
for mic in micSet
]
readSetOfCoordinates(coordSet, starFiles)
elif fromType == 'gautomatch':
if toType == 'xmipp':
readSetOfCoordinates = pwutils.Config.getDomain(
).importFromPlugin('gautomatch.convert',
'readSetOfCoordinates',
doRaise=True)
else:
raise Exception('Unknown coordinates type: %s' % fromType)
readSetOfCoordinates(outputDir, micSet, coordSet)
writeSetOfCoordinatesWithState = pwutils.Config.getDomain(
).importFromPlugin('xmipp3.convert',
'writeSetOfCoordinatesWithState',
doRaise=True)
writeSetOfCoordinatesWithState(outputDir, coordSet, state='Automatic')
def convertInputStep(self, inputCoordinates, scale, kfold):
""" Converts a set of coordinates to box files and binaries to mrc
if needed. It generates 2 folders 1 for the box files and another for
the mrc files.
"""
micIds = []
coordSet = self.inputCoordinates.get()
setFn = coordSet.getFileName()
self.debug("Loading input db: %s" % setFn)
# Load set of coordinates with a user determined number of coordinates for the training step
enoughMicrographs = False
while True:
coordSet = SetOfCoordinates(filename=setFn)
coordSet._xmippMd = params.String()
coordSet.loadAllProperties()
for micAgg in coordSet.aggregate(["MAX"], "_micId", ["_micId"]):
micIds.append(micAgg["_micId"])
if len(micIds) == self.micsForTraining.get():
enoughMicrographs = True
break
if enoughMicrographs:
break
else:
if coordSet.isStreamClosed():
raise Exception("We have a problem!!")
self.info("Not yet there: %s" % len(micIds))
import time
time.sleep(10)
# Create input folder and pre-processed micrographs folder
micDir = self._getFileName(TRAINING)
pw.utils.makePath(micDir)
prepDir = self._getFileName(TRAININGPREPROCESS)
pw.utils.makePath(prepDir)
ih = ImageHandler()
# Get a refreshed set of micrographs
micsFn = self.inputCoordinates.get().getMicrographs().getFileName()
# not updating, refresh problem
coordMics = SetOfMicrographs(filename=micsFn)
coordMics.loadAllProperties()
# Create a 0/1 list to mark micrographs for training/testing
n = len(micIds)
indexes = np.zeros(n, dtype='int8')
testSetImages = int((kfold / float(100)) * n)
# Both the training and the test data set should contain at least one micrograph
if testSetImages < 1:
requiredMinimumPercentage = (1 * 100 / n) + 1
testSetImages = int((requiredMinimumPercentage / float(100)) * n)
elif testSetImages == n:
testSetImages = int(0.99 * n)
indexes[:testSetImages] = 1
np.random.shuffle(indexes)
self.info('indexes: %s' % indexes)
# Write micrographs files
csvMics = [
CsvMicrographList(self._getFileName(TRAININGLIST), 'w'),
CsvMicrographList(self._getFileName(TRAININGTEST), 'w')
]
# Store the micId and indexes in micDict
micDict = {}
for i, micId in zip(indexes, micIds):
mic = coordMics[micId]
micFn = mic.getFileName()
baseFn = pw.utils.removeBaseExt(micFn)
inputFn = self._getFileName(TRAINING_MIC, **{"mic": baseFn})
if micFn.endswith('.mrc'):
pwutils.createAbsLink(os.path.abspath(micFn), inputFn)
else:
ih.convert(micFn, inputFn)
prepMicFn = self._getFileName(TRAININGPRE_MIC, **{"mic": baseFn})
csvMics[i].addMic(micId, prepMicFn)
micDict[micId] = i # store if train or test
for csv in csvMics:
csv.close()
# Write particles files
csvParts = [
CsvCoordinateList(self._getFileName(PARTICLES_TRAIN_TXT), 'w'),
CsvCoordinateList(self._getFileName(PARTICLES_TEST_TXT), 'w')
]
for coord in coordSet.iterItems(orderBy='_micId'):
micId = coord.getMicId()
if micId in micDict:
x = int(round(float(coord.getX()) / scale))
y = int(round(float(coord.getY()) / scale))
csvParts[micDict[micId]].addCoord(micId, x, y)
for csv in csvParts:
csv.close()
def createOutputStep(self):
# PARTICLES
cleanPattern(self._getPath("*.sqlite"))
partSet = self._createSetOfParticles()
readSetOfParticles(self._getPath("particles.xmd"), partSet)
inputSampling = self.inputCoordinates[0].get().getMicrographs(
).getSamplingRate()
partSet.setSamplingRate(self._getDownFactor() * inputSampling)
boxSize = self._getBoxSize()
# COORDINATES
writeSet = False
if self.checkIfPrevRunIsCompatible("coords_"):
writeSet = True
if not "OR" in self.coordinatesDict:
self.loadCoords(self._getExtraPath(
self.CONSENSUS_COOR_PATH_TEMPLATE % 'TRUE'),
'OR',
writeSet=False)
coordSet = SetOfCoordinates(
filename=self._getPath("coordinates.sqlite"))
coordSet.copyInfo(self.coordinatesDict['OR'])
coordSet.setBoxSize(boxSize)
coordSet.setMicrographs(self.coordinatesDict['OR'].getMicrographs())
downFactor = self._getDownFactor()
for part in partSet:
coord = part.getCoordinate().clone()
coord.scale(downFactor)
deepZscoreLabel = '_xmipp_%s' % xmipp.label2Str(
MD.MDL_ZSCORE_DEEPLEARNING1)
setattr(coord, deepZscoreLabel, getattr(part, deepZscoreLabel))
coordSet.append(coord)
coordSet.write()
partSet.write()
self._defineOutputs(outputCoordinates=coordSet)
self._defineOutputs(outputParticles=partSet)
for inSetOfCoords in self.inputCoordinates:
self._defineSourceRelation(inSetOfCoords.get(), coordSet)
self._defineSourceRelation(inSetOfCoords.get(), partSet)
def _visualizeCoordinates(self, e=None):
views = []
outCoords = self.protocol.getOutput()
if not outCoords: print(" > Not output found, yet."); return
coordsViewerFn = self.protocol._getExtraPath('coordsViewer.sqlite')
mdLabel = emlib.MDL_GOOD_REGION_SCORE
if not getXmippAttribute(outCoords.getFirstItem(), mdLabel):
print(" > outputCoordinates do NOT have 'MDL_GOOD_REGION_SCORE'!"); return
cleanPath(coordsViewerFn)
newOutput = SetOfCoordinates(filename=coordsViewerFn)
newOutput.copyInfo(outCoords)
newOutput.setMicrographs(outCoords.getMicrographs())
thres = self.visualizeCoordinates.get()
for coord in outCoords:
if getXmippAttribute(coord, mdLabel).get() > thres:
newOutput.append(coord.clone())
newOutput.write()
newOutput.close() #SetOfCoordinates does not implement __exit__, required for with
micSet = newOutput.getMicrographs() # accessing mics to provide metadata file
if micSet is None:
raise Exception('visualize: SetOfCoordinates has no micrographs set.')
fn = self.protocol._getExtraPath("allMics.xmd")
xmipp3.convert.writeSetOfMicrographs(micSet, fn)
tmpDir = self.protocol._getExtraPath('manualThresholding_%03d'%int(thres*100))
cleanPath(tmpDir)
makePath(tmpDir)
xmipp3.convert.writeSetOfCoordinates(tmpDir, newOutput)
views.append(CoordinatesObjectView(self._project, fn, tmpDir,
self.protocol, inTmpFolder=True))
return views