def getCoords(self): # to support multiple access to db coordSet = self.inputCoordinates.get() coordSetCopy = SetOfCoordinates() coordSetCopy.copy(coordSet) coordSet.close() return coordSetCopy
def _convertCoords(self, micSet, tmpDir, coordsType): """ Link specified coord set to tmpDir folder and convert it to .pos files""" coordTypes = { 'autopick': 'coordinates.sqlite', 'rejected': 'coordinates_rejected.sqlite' } coordsFnIn = self.protocol._getPath(coordTypes[coordsType]) coordsFnOut = pwutils.join(tmpDir, 'coordinates.sqlite') pwutils.createLink(coordsFnIn, coordsFnOut) coordSet = SetOfCoordinates(filename=coordsFnOut) coordSet.setMicrographs(micSet) from .convert import writeSetOfCoordinatesXmipp writeSetOfCoordinatesXmipp(tmpDir, coordSet, ismanual=False)
def getReadyMics(coordSet): coorSet = SetOfCoordinates(filename=coordSet.getFileName()) coorSet._xmippMd = String() coorSet.loadAllProperties() setClosed = coorSet.isStreamClosed() coorSet.close() currentPickMics = { micAgg["_micId"] for micAgg in coordSet.aggregate(["MAX"], "_micId", ["_micId"]) } return currentPickMics, setClosed
def readSetOfCoordsFromPosFnames(posDir, setOfInputCoords, sqliteOutName, write=True): """ posDir: path where there are .pos files with coordinates setOfInputCoords. Set to find micrographs sqliteOutName. Path where sqlite map will be created. Warning, it overwrites content """ inputMics = setOfInputCoords.getMicrographs() cleanPath(sqliteOutName) setOfOutputCoordinates= SetOfCoordinates(filename= sqliteOutName) setOfOutputCoordinates.setMicrographs(inputMics) setOfOutputCoordinates.setBoxSize(setOfInputCoords.getBoxSize()) readSetOfCoordinates(posDir, micSet=inputMics, coordSet=setOfOutputCoordinates, readDiscarded=False) if write: setOfOutputCoordinates.write() return setOfOutputCoordinates
def test_coodinatesTest1(self): """ Import an EMX file with just one micrograph and a few coordinates. """ protEmxImport = self.newProtocol( ProtImportParticles, objLabel='from emx (coordinatesT1)', importFrom=ProtImportParticles.IMPORT_FROM_EMX, emxFile=self.dataset.getFile('coordinatesT1'), alignType=3, voltage=100, magnification=10000, samplingRate=2.46) self.launchProtocol(protEmxImport) # Reference coordinates coords = SetOfCoordinates( filename=self.dataset.getFile('coordinatesGoldT1')) tests.BaseTest.compareSets(self, protEmxImport.outputCoordinates, coords)
def main(): parser = argparse.ArgumentParser(prog=CONVERT_ENTRY_POINT, usage="pwem command to convert metadata " "coordinates from several em formats " "(eman, relion, gautomatch " "and bsoft) to xmipp metadata") parser.add_argument('--coordinates', help='Convert coordinates', action="store_true") parser.add_argument('--fromType', help='Convert from input type') parser.add_argument('--toType', help='Convert to output type') parser.add_argument('--input', help='Input file or folder', required=True) parser.add_argument('--output', help='Output file or folder') parser.add_argument('--extra', help='To add extra parameters') args = parser.parse_args() fromType = args.fromType toType = args.toType input = args.input output = args.output if args.coordinates: micSet = loadSetFromDb(input) outputDir = output coordsfn = os.path.join(outputDir, 'coordinates.sqlite') cleanPath(coordsfn) coordSet = SetOfCoordinates(filename=coordsfn) coordSet.setMicrographs(micSet) if fromType == 'eman2': if toType == 'xmipp': readSetOfCoordinates = pwutils.Config.getDomain( ).importFromPlugin('eman2.convert', 'readSetOfCoordinates', doRaise=True) elif fromType == 'dogpicker': if toType == 'xmipp': readSetOfCoordinates = pwutils.Config.getDomain( ).importFromPlugin('appion.convert', 'readSetOfCoordinates', doRaise=True) elif fromType == 'relion': if toType == 'xmipp': def readSetOfCoordinates(outputDir, micSet, coordSet): readSetOfCoordinates = pwutils.Config.getDomain( ).importFromPlugin('relion.convert', 'readSetOfCoordinates', doRaise=True) inputCoords = args.extra starFiles = [ os.path.join(inputCoords, "mic_%06d_autopick.star" % mic.getObjId()) for mic in micSet ] readSetOfCoordinates(coordSet, starFiles) elif fromType == 'gautomatch': if toType == 'xmipp': readSetOfCoordinates = pwutils.Config.getDomain( ).importFromPlugin('gautomatch.convert', 'readSetOfCoordinates', doRaise=True) else: raise Exception('Unknown coordinates type: %s' % fromType) readSetOfCoordinates(outputDir, micSet, coordSet) writeSetOfCoordinatesWithState = pwutils.Config.getDomain( ).importFromPlugin('xmipp3.convert', 'writeSetOfCoordinatesWithState', doRaise=True) writeSetOfCoordinatesWithState(outputDir, coordSet, state='Automatic')
def convertInputStep(self, inputCoordinates, scale, kfold): """ Converts a set of coordinates to box files and binaries to mrc if needed. It generates 2 folders 1 for the box files and another for the mrc files. """ micIds = [] coordSet = self.inputCoordinates.get() setFn = coordSet.getFileName() self.debug("Loading input db: %s" % setFn) # Load set of coordinates with a user determined number of coordinates for the training step enoughMicrographs = False while True: coordSet = SetOfCoordinates(filename=setFn) coordSet._xmippMd = params.String() coordSet.loadAllProperties() for micAgg in coordSet.aggregate(["MAX"], "_micId", ["_micId"]): micIds.append(micAgg["_micId"]) if len(micIds) == self.micsForTraining.get(): enoughMicrographs = True break if enoughMicrographs: break else: if coordSet.isStreamClosed(): raise Exception("We have a problem!!") self.info("Not yet there: %s" % len(micIds)) import time time.sleep(10) # Create input folder and pre-processed micrographs folder micDir = self._getFileName(TRAINING) pw.utils.makePath(micDir) prepDir = self._getFileName(TRAININGPREPROCESS) pw.utils.makePath(prepDir) ih = ImageHandler() # Get a refreshed set of micrographs micsFn = self.inputCoordinates.get().getMicrographs().getFileName() # not updating, refresh problem coordMics = SetOfMicrographs(filename=micsFn) coordMics.loadAllProperties() # Create a 0/1 list to mark micrographs for training/testing n = len(micIds) indexes = np.zeros(n, dtype='int8') testSetImages = int((kfold / float(100)) * n) # Both the training and the test data set should contain at least one micrograph if testSetImages < 1: requiredMinimumPercentage = (1 * 100 / n) + 1 testSetImages = int((requiredMinimumPercentage / float(100)) * n) elif testSetImages == n: testSetImages = int(0.99 * n) indexes[:testSetImages] = 1 np.random.shuffle(indexes) self.info('indexes: %s' % indexes) # Write micrographs files csvMics = [ CsvMicrographList(self._getFileName(TRAININGLIST), 'w'), CsvMicrographList(self._getFileName(TRAININGTEST), 'w') ] # Store the micId and indexes in micDict micDict = {} for i, micId in zip(indexes, micIds): mic = coordMics[micId] micFn = mic.getFileName() baseFn = pw.utils.removeBaseExt(micFn) inputFn = self._getFileName(TRAINING_MIC, **{"mic": baseFn}) if micFn.endswith('.mrc'): pwutils.createAbsLink(os.path.abspath(micFn), inputFn) else: ih.convert(micFn, inputFn) prepMicFn = self._getFileName(TRAININGPRE_MIC, **{"mic": baseFn}) csvMics[i].addMic(micId, prepMicFn) micDict[micId] = i # store if train or test for csv in csvMics: csv.close() # Write particles files csvParts = [ CsvCoordinateList(self._getFileName(PARTICLES_TRAIN_TXT), 'w'), CsvCoordinateList(self._getFileName(PARTICLES_TEST_TXT), 'w') ] for coord in coordSet.iterItems(orderBy='_micId'): micId = coord.getMicId() if micId in micDict: x = int(round(float(coord.getX()) / scale)) y = int(round(float(coord.getY()) / scale)) csvParts[micDict[micId]].addCoord(micId, x, y) for csv in csvParts: csv.close()
def createOutputStep(self): # PARTICLES cleanPattern(self._getPath("*.sqlite")) partSet = self._createSetOfParticles() readSetOfParticles(self._getPath("particles.xmd"), partSet) inputSampling = self.inputCoordinates[0].get().getMicrographs( ).getSamplingRate() partSet.setSamplingRate(self._getDownFactor() * inputSampling) boxSize = self._getBoxSize() # COORDINATES writeSet = False if self.checkIfPrevRunIsCompatible("coords_"): writeSet = True if not "OR" in self.coordinatesDict: self.loadCoords(self._getExtraPath( self.CONSENSUS_COOR_PATH_TEMPLATE % 'TRUE'), 'OR', writeSet=False) coordSet = SetOfCoordinates( filename=self._getPath("coordinates.sqlite")) coordSet.copyInfo(self.coordinatesDict['OR']) coordSet.setBoxSize(boxSize) coordSet.setMicrographs(self.coordinatesDict['OR'].getMicrographs()) downFactor = self._getDownFactor() for part in partSet: coord = part.getCoordinate().clone() coord.scale(downFactor) deepZscoreLabel = '_xmipp_%s' % xmipp.label2Str( MD.MDL_ZSCORE_DEEPLEARNING1) setattr(coord, deepZscoreLabel, getattr(part, deepZscoreLabel)) coordSet.append(coord) coordSet.write() partSet.write() self._defineOutputs(outputCoordinates=coordSet) self._defineOutputs(outputParticles=partSet) for inSetOfCoords in self.inputCoordinates: self._defineSourceRelation(inSetOfCoords.get(), coordSet) self._defineSourceRelation(inSetOfCoords.get(), partSet)
def _visualizeCoordinates(self, e=None): views = [] outCoords = self.protocol.getOutput() if not outCoords: print(" > Not output found, yet."); return coordsViewerFn = self.protocol._getExtraPath('coordsViewer.sqlite') mdLabel = emlib.MDL_GOOD_REGION_SCORE if not getXmippAttribute(outCoords.getFirstItem(), mdLabel): print(" > outputCoordinates do NOT have 'MDL_GOOD_REGION_SCORE'!"); return cleanPath(coordsViewerFn) newOutput = SetOfCoordinates(filename=coordsViewerFn) newOutput.copyInfo(outCoords) newOutput.setMicrographs(outCoords.getMicrographs()) thres = self.visualizeCoordinates.get() for coord in outCoords: if getXmippAttribute(coord, mdLabel).get() > thres: newOutput.append(coord.clone()) newOutput.write() newOutput.close() #SetOfCoordinates does not implement __exit__, required for with micSet = newOutput.getMicrographs() # accessing mics to provide metadata file if micSet is None: raise Exception('visualize: SetOfCoordinates has no micrographs set.') fn = self.protocol._getExtraPath("allMics.xmd") xmipp3.convert.writeSetOfMicrographs(micSet, fn) tmpDir = self.protocol._getExtraPath('manualThresholding_%03d'%int(thres*100)) cleanPath(tmpDir) makePath(tmpDir) xmipp3.convert.writeSetOfCoordinates(tmpDir, newOutput) views.append(CoordinatesObjectView(self._project, fn, tmpDir, self.protocol, inTmpFolder=True)) return views