def __convertCorrectionImage(self, correctionImage):
        """ Will convert a gain or dark file to a compatible one and return
        the final file name. Otherwise, will return same passed parameter"""

        # Get final correction image file
        finalName = self.getFinalCorrectionImagePath(correctionImage)

        # If correctionImage is None, finalName will be None
        if finalName is None:
            return None

        elif not os.path.exists(finalName):
            # Conversion never happened...
            print('converting %s to %s' % (correctionImage, finalName))

            if correctionImage.endswith('.gain'):  # treat as tif file
                fnBase = basename(correctionImage)
                tmpLink = self._getTmpPath(pwutils.replaceExt(fnBase, 'tif'))
                pwutils.createAbsLink(correctionImage, tmpLink)
                correctionImage = tmpLink

            emlib.image.ImageHandler().convert(correctionImage, finalName)

        # return final name
        return os.path.abspath(finalName)
Exemple #2
0
    def fixLinks(self, searchDir):

        runs = self.getRuns()

        for prot in runs:
            broken = False
            if isinstance(prot, em.ProtImport):
                for _, attr in prot.iterOutputEM():
                    fn = attr.getFiles()
                    for f in attr.getFiles():
                        if ':' in f:
                            f = f.split(':')[0]

                        if not os.path.exists(f):
                            if not broken:
                                broken = True
                                print "Found broken links in run: ", pwutils.magenta(prot.getRunName())
                            print "  Missing: ", pwutils.magenta(f)
                            if os.path.islink(f):
                                print "    -> ", pwutils.red(os.path.realpath(f))
                            newFile = pwutils.findFile(os.path.basename(f), searchDir, recursive=True)
                            if newFile:
                                print "  Found file %s, creating link..." % newFile
                                print pwutils.green("   %s -> %s" % (f, newFile))
                                pwutils.createAbsLink(newFile, f)
Exemple #3
0
    def _convertGain(self):
        """ We need to transform gain file for a possible polishing job. """
        rotation = self.gainRot.get()
        flip = self.gainFlip.get()
        gainFn = self.gainFile.get()

        if rotation or flip:
            args = "%s %s " % (gainFn, self._getPath(os.path.basename(gainFn)))

            if flip:
                # flip axis Y - left to right
                args += "--process xform.flip:axis=%s " % ("y" if flip == 2 else "x")

            if rotation:
                args += "--rotate %d " % (rotation * 90)

            from pwem import Domain
            eman2 = Domain.importFromPlugin('eman2')
            pwutils.runJob(self._log, eman2.Plugin.getProgram('e2proc2d.py'), args,
                           env=eman2.Plugin.getEnviron())

            return self._getPath(os.path.basename(gainFn))
        else:
            outputGain = self._getPath(os.path.basename(gainFn))
            pwutils.createAbsLink(gainFn, outputGain)
            return outputGain
Exemple #4
0
 def decompress(program, args, ext, nExt):
     movieFolder = self._getTmpPath()
     movieName = basename(movie.getFileName())
     movieTmpLink = join(movieFolder, movieName)
     pwutils.cleanPath(movieTmpLink)
     pwutils.createAbsLink(os.path.abspath(movieFn), movieTmpLink)
     self.runJob(program, args % movieName, cwd=movieFolder)
     dimMovie.setFileName(movieTmpLink.replace(ext, nExt))
Exemple #5
0
 def decompress(program, args, ext, nExt):
     movieFolder = self._getTmpPath()
     movieName = basename(movie.getFileName())
     movieTmpLink = join(movieFolder, movieName)
     pwutils.cleanPath(movieTmpLink)
     pwutils.createAbsLink(os.path.abspath(movieFn), movieTmpLink)
     self.runJob(program, args % movieName, cwd=movieFolder)
     dimMovie.setFileName(movieTmpLink.replace(ext, nExt))
 def _createInputLinks(cls):
     # Create a test folder path
     pattern = cls.ds.getFile('allMics')
     files = glob(pattern)
     nFiles = len(files)
     
     for i in range(nFiles):
         # Loop over the number of input movies if we want more for testing
         f = files[i % nFiles]
         _, cls.ext = os.path.splitext(f)
         moviePath = cls.proj.getTmpPath('movie%06d%s' % (i + 1, cls.ext))
         pwutils.createAbsLink(f, moviePath)
         time.sleep(10)
    def _createInputLinks(cls):
        # Create a test folder path
        pattern = cls.ds.getFile('allMics')
        files = glob(pattern)
        nFiles = len(files)

        for i in range(nFiles):
            # Loop over the number of input movies if we want more for testing
            f = files[i % nFiles]
            _, cls.ext = os.path.splitext(f)
            moviePath = cls.proj.getTmpPath('movie%06d%s' % (i + 1, cls.ext))
            pwutils.createAbsLink(f, moviePath)
            time.sleep(10)
Exemple #8
0
    def _createInputLinks(cls):
        # Create a test folder path
        pattern = PATTERN if PATTERN else cls.ds.getFile('ribo/Falcon*mrcs')
        files = glob(pattern)
        nFiles = len(files)
        nMovies = MOVS

        for i in range(nMovies):
            # Loop over the number of input movies if we want more for testing
            f = files[i % nFiles]
            _, cls.ext = os.path.splitext(f)
            moviePath = cls.proj.getTmpPath('movie%06d%s' % (i + 1, cls.ext))
            pwutils.createAbsLink(f, moviePath)
            time.sleep(DELAY)
    def _createInputLinks(cls):
        # Create a test folder path
        pattern = PATTERN if PATTERN else cls.ds.getFile('ribo/Falcon*mrcs')
        files = glob(pattern)
        nFiles = len(files)
        nMovies = MOVS

        for i in range(nMovies):
            # Loop over the number of input movies if we want more for testing
            f = files[i % nFiles]
            _, cls.ext = os.path.splitext(f)
            moviePath = cls.proj.getTmpPath('movie%06d%s' % (i+1, cls.ext))
            pwutils.createAbsLink(f, moviePath)
            time.sleep(DELAY)
Exemple #10
0
    def testImportFromCTffind4Conflict(self):
        micsPath = os.path.abspath(self.proj.getTmpPath('micrographs'))
        ctfsPath = os.path.abspath(self.proj.getTmpPath('ctfs'))
        pwutils.makePath(micsPath)
        pwutils.makePath(ctfsPath)

        micDict = {'BPV_1386': 'mic1', 'BPV_1387': 'mic10', 'BPV_1388': 'mic2'}

        pattern = 'micrographs/%s.mrc'

        for k, v in micDict.items():
            # Create a micrograph link with a given naming convention
            pwutils.createAbsLink(self.dsXmipp.getFile(pattern % k),
                                  self.proj.getTmpPath(pattern % v))

            pwutils.createAbsLink(self.dsGrigorieff.getFile('ctffind4/%s' % k),
                                  self.proj.getTmpPath('ctfs/%s' % v))

        protCTF = self.newProtocol(
            emprot.ProtImportCTF,
            importFrom=emprot.ProtImportCTF.IMPORT_FROM_GRIGORIEFF,
            filesPath=ctfsPath,
            filesPattern='mic*/*txt')
        protCTF.inputMicrographs.set(self.protImport.outputMicrographs)
        protCTF.setObjLabel('import from ctffind3 - empty')
        self.launchProtocol(protCTF)

        # FIXME: After the execution of the above case, some empty sets
        # are created, if there is no matching, there should not be any output

        # Let's import now the correct names, but with the problematic matching
        protImport2 = self.newProtocol(emprot.ProtImportMicrographs,
                                       filesPath=micsPath,
                                       filesPattern='mic*mrc',
                                       samplingRate=1.237,
                                       voltage=300)
        self.launchProtocol(protImport2)

        protCTF2 = self.newProtocol(
            emprot.ProtImportCTF,
            importFrom=emprot.ProtImportCTF.IMPORT_FROM_GRIGORIEFF,
            filesPath=ctfsPath,
            filesPattern='mic*/*txt')
        protCTF2.inputMicrographs.set(protImport2.outputMicrographs)
        protCTF2.setObjLabel('import from ctffind3 - match')
        self.launchProtocol(protCTF2)

        # The whole set (3 items) should find its corresponding CTF
        self.assertEqual(protCTF2.outputCTF.getSize(), 3)
    def testImportFromCTffind4Conflict(self):
        micsPath = os.path.abspath(self.proj.getTmpPath('micrographs'))
        ctfsPath = os.path.abspath(self.proj.getTmpPath('ctfs'))
        pwutils.makePath(micsPath)
        pwutils.makePath(ctfsPath)

        micDict = {'BPV_1386': 'mic1',
                   'BPV_1387': 'mic10',
                   'BPV_1388': 'mic2'}

        pattern = 'micrographs/%s.mrc'

        for k, v in micDict.iteritems():
            # Create a micrograph link with a given naming convention
            pwutils.createAbsLink(self.dsXmipp.getFile(pattern % k),
                                  self.proj.getTmpPath(pattern % v))

            pwutils.createAbsLink(self.dsGrigorieff.getFile('ctffind4/%s' % k),
                                  self.proj.getTmpPath('ctfs/%s' % v))

        protCTF = self.newProtocol(ProtImportCTF,
                                   importFrom=ProtImportCTF.IMPORT_FROM_GRIGORIEFF,
                                   filesPath=ctfsPath,
                                   filesPattern='mic*/*txt')
        protCTF.inputMicrographs.set(self.protImport.outputMicrographs)
        protCTF.setObjLabel('import from ctffind3 - empty')
        self.launchProtocol(protCTF)

        # FIXME: After the execution of the above case, some empty sets
        # are created, if there is no matching, there should not be any output

        # Let's import now the correct names, but with the problematic matching
        protImport2 = self.newProtocol(ProtImportMicrographs,
                                       filesPath=micsPath,
                                       filesPattern='mic*mrc',
                                       samplingRate=1.237, voltage=300)
        self.launchProtocol(protImport2)

        protCTF2 = self.newProtocol(ProtImportCTF,
                                   importFrom=ProtImportCTF.IMPORT_FROM_GRIGORIEFF,
                                   filesPath=ctfsPath,
                                   filesPattern='mic*/*txt')
        protCTF2.inputMicrographs.set(protImport2.outputMicrographs)
        protCTF2.setObjLabel('import from ctffind3 - match')
        self.launchProtocol(protCTF2)

        # The whole set (3 items) should find its corresponding CTF
        self.assertEqual(protCTF2.outputCTF.getSize(), 3)
    def _sendMovies(cls, delay=0, port=5000):
        time.sleep(10)  # wait so the server socket is launched
        # Create a test folder path
        pattern = PATTERN if PATTERN else cls.ds.getFile('ribo/Falcon*mrcs')
        files = glob(pattern)
        nFiles = len(files)
        nMovies = MOVS
        clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        host = ''
        print("Connecting to server socket...")
        clientSocket.connect((host, port))
        for i in range(nMovies):
            # Loop over the number of input movies if we want more for testing
            f = files[i % nFiles]
            _, cls.ext = os.path.splitext(f)
            moviePath = cls.proj.getTmpPath('movie%06d%s' % (i + 1, cls.ext))
            print("Creating movie %d link..." % (i + 1))
            pwutils.createAbsLink(f, moviePath)
            absPath = os.path.abspath(moviePath) + '\n'
            try:
                clientSocket.sendall(absPath)
                time.sleep(delay)
            except socket.error as err:
                # Send failed
                print('Failed to send file: %s' % absPath)
                print(err)
                print('Trying to reconnect...')
                clientSocket.shutdown(socket.SHUT_WR)
                clientSocket.close()
                time.sleep(2)
                clientSocket = socket.socket(socket.AF_INET,
                                             socket.SOCK_STREAM)
                clientSocket.connect((host, port))

        serverConnected = True
        print('Last file sent: %s' % absPath)
        print('Waiting for server socket to finish reading...')
        while serverConnected:
            reply = clientSocket.recv(4096)
            if reply:
                if absPath in reply:
                    serverConnected = False
                time.sleep(delay)  # wait a bit to check for reply
            else:
                serverConnected = False
        print('Finished! Disconnecting client...')
        clientSocket.shutdown(socket.SHUT_WR)
        clientSocket.close()
Exemple #13
0
    def importModelStep(self):
        """ Create a link to the provided input model path
        and register the output to be used later for further training
        or picking.
        """
        absPath = os.path.abspath(self.modelPath.get())
        outputPath = self._getExtraPath(os.path.basename(absPath))
        self.info("Creating link:\n" "%s -> %s" % (outputPath, absPath))
        self.info(
            "NOTE: If you move this project to another computer, the symbolic"
            "link to the model will be broken, but you can update the link "
            "and get it working again. ")

        pwutils.createAbsLink(absPath, outputPath)

        self._defineOutputs(outputModel=TopazModel(outputPath))
    def _sendMovies(cls, delay=0, port=5000):
        time.sleep(10)  # wait so the server socket is launched
        # Create a test folder path
        pattern = PATTERN if PATTERN else cls.ds.getFile('ribo/Falcon*mrcs')
        files = glob(pattern)
        nFiles = len(files)
        nMovies = MOVS
        clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        host = ''
        print("Connecting to server socket...")
        clientSocket.connect((host, port))
        for i in range(nMovies):
            # Loop over the number of input movies if we want more for testing
            f = files[i % nFiles]
            _, cls.ext = os.path.splitext(f)
            moviePath = cls.proj.getTmpPath('movie%06d%s' % (i + 1, cls.ext))
            print("Creating movie %d link..." %(i+1))
            pwutils.createAbsLink(f, moviePath)
            absPath = os.path.abspath(moviePath)+'\n'
            try:
                clientSocket.sendall(absPath)
                time.sleep(delay)
            except socket.error as err:
                # Send failed
                print('Failed to send file: %s' % absPath)
                print(err)
                print('Trying to reconnect...')
                clientSocket.shutdown(socket.SHUT_WR)
                clientSocket.close()
                time.sleep(2)
                clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                clientSocket.connect((host, port))

        serverConnected = True
        print('Last file sent: %s' % absPath)
        print('Waiting for server socket to finish reading...')
        while serverConnected:
            reply = clientSocket.recv(4096)
            if reply:
                if absPath in reply:
                    serverConnected = False
                time.sleep(delay)  # wait a bit to check for reply
            else:
                serverConnected = False
        print('Finished! Disconnecting client...')
        clientSocket.shutdown(socket.SHUT_WR)
        clientSocket.close()
    def assignStep(self):
        # Create a Volume object
        self.inVol = self.inVolume.get()
        self.outVol = emobj.Volume()
        self.outVol.copy(self.inVol)

        # Set sampling Rate (or copy from input)
        if self.setSampling.get():
            samplingRate = self.samplingRate.get()
        else:
            samplingRate = self.inVol.getSamplingRate()

        self.outVol.setSamplingRate(samplingRate)

        # set Origin
        if self.setOrigCoord.get():
            origin = emobj.Transform()
            origin.setShifts(self.x.get(), self.y.get(), self.z.get())
        else:
            origin = self.inVol.getOrigin()

        self.outVol.setOrigin(origin)

        # Files system stuff
        fileName = os.path.abspath(self.inVol.getFileName())
        fileName = fileName.replace(':mrc', '')

        # copy or link
        if self.copyFiles:
            imgBase = pwutils.replaceBaseExt(fileName, "mrc")
            imgDst = self._getExtraPath(imgBase)
            Ccp4Header.fixFile(fileName,
                               imgDst,
                               origin.getShifts(),
                               sampling=samplingRate)
        else:
            imgBase = basename(fileName)
            imgDst = self._getExtraPath(imgBase)
            pwutils.createAbsLink(fileName, imgDst)

        self.outVol.setLocation(imgDst)

        # save
        self._defineOutputs(outputVolume=self.outVol)
        self._defineSourceRelation(self.inVol, self.outVol)
Exemple #16
0
    def runGautomatch(cls,
                      micNameList,
                      refStack,
                      workDir,
                      extraArgs,
                      env=None,
                      runJob=None):
        """ Run Gautomatch with the given parameters.
        If micrographs are not .mrc, they will be converted.
        If runJob=None, it will use pwutils.runJob.
        """
        args = ''

        ih = ImageHandler()

        for micName in micNameList:
            # We convert the input micrograph on demand if not in .mrc
            outMic = os.path.join(workDir,
                                  pwutils.replaceBaseExt(micName, 'mrc'))
            if micName.endswith('.mrc'):
                pwutils.createAbsLink(os.path.abspath(micName), outMic)
            else:
                ih.convert(micName, outMic)

        args += ' %s/*.mrc' % workDir

        if refStack is not None:
            args += ' -T %s' % refStack

        args += ' %s' % extraArgs

        environ = env if env is not None else cls.getEnviron()
        if runJob is None:
            pwutils.runJob(None, cls.getProgram(), args, env=environ)
        else:
            runJob(cls.getProgram(), args, env=environ)

        for micName in micNameList:
            outMic = os.path.join(workDir,
                                  pwutils.replaceBaseExt(micName, 'mrc'))
            # After picking we can remove the temporary file.
            pwutils.cleanPath(outMic)
    def convertInputStep(self, micsId, refsId):
        """ Match ctf information against the micrographs. """
        self.ctfDict = {}
        if self.ctfRelations.get() is not None:
            for ctf in self.ctfRelations.get():
                self.ctfDict[ctf.getMicrograph().getMicName()] = ctf.clone()

        ih = emlib.image.ImageHandler()
        for mic in self.getInputMicrographs():
            micName = mic.getFileName()
            # We convert the input micrographs if they are not .mrc
            outMic = os.path.join(self._getTmpPath(),
                                  pwutils.replaceBaseExt(micName, 'mrc'))
            if micName.endswith('.mrc'):
                pwutils.createAbsLink(os.path.abspath(micName), outMic)
            else:
                ih.convert(micName, outMic, emlib.DT_FLOAT)

        if refsId is not None:
            writeReferences(self.getInputReferences(),
                            self._getExtraPath('references.mrc'))
    def _doCtfEstimation(self, mic, **kwargs):
        """ Run ctffind with required parameters.
        :param mic: input mic object
        :param kwargs: dict with arguments
        """
        if self.usePowerSpectra:
            micFn = mic._powerSpectra.getFileName()
            powerSpectraPix = self.psSampling
        else:
            micFn = mic.getFileName()
            powerSpectraPix = None
        micDir = self._getTmpPath('mic_%06d' % mic.getObjId())
        # Create micrograph dir
        pwutils.makePath(micDir)
        micFnMrc = os.path.join(micDir, pwutils.replaceBaseExt(micFn, 'mrc'))

        ih = emlib.image.ImageHandler()

        if not pwutils.exists(micFn):
            raise FileNotFoundError("Missing input micrograph: %s" % micFn)

        if micFn.endswith('.mrc'):
            pwutils.createAbsLink(os.path.abspath(micFn), micFnMrc)
        else:
            ih.convert(micFn, micFnMrc, emlib.DT_FLOAT)

        try:
            program, args = self._ctfProgram.getCommand(
                micFn=micFnMrc,
                powerSpectraPix=powerSpectraPix,
                ctffindOut=self._getCtfOutPath(mic),
                ctffindPSD=self._getPsdPath(mic),
                **kwargs)
            self.runJob(program, args)

            pwutils.cleanPath(micDir)

        except Exception as e:
            self.error("ERROR: Ctffind has failed for %s. %s" %
                       (micFnMrc, self._getErrorFromCtffindTxt(mic, e)))
    def testAssignCTF(self):
        """ Test the particle extraction after importing another
        SetOfMicraphs with a different micName but same ids.
        We will use assign-ctf protocol and extract from the 
        newly imported mics with the assigned CTF.
        For the other mics, we will just create symbolic links.
        """
        # Create the links with a different micrograph name
        micsPath = self.proj.getPath('otherMicrographs')
        pwutils.makePath(micsPath)
        for i in [6, 7, 8]:
            micPath = self.dataset.getFile('micrographs/BPV_138%d.mrc' % i)
            micLink = join(micsPath, basename(micPath).replace('.mrc', '_DW.mrc'))
            pwutils.createAbsLink(micPath, micLink)
            
        protImportDW = self.proj.copyProtocol(self.protImport)
        protImportDW.setObjLabel('import -mics DW')
        protImportDW.filesPath.set(micsPath)
        protImportDW.filesPattern.set('*_DW.mrc')
        self.launchProtocol(protImportDW)
        
        protAssignCTF = self.newProtocol(ProtCTFAssign)
        protAssignCTF.inputSet.set(protImportDW.outputMicrographs)
        protAssignCTF.inputCTF.set(self.protCTF.outputCTF)
        self.launchProtocol(protAssignCTF)
        
        protExtract = self.newProtocol(XmippProtExtractParticles, 
                                       boxSize=183, downsampleType=OTHER, 
                                       downFactor=3.0,doFlip=False)
        protExtract.inputCoordinates.set(self.protPP.outputCoordinates)
        protExtract.inputMicrographs.set(protAssignCTF.outputMicrographs)
        protExtract.ctfRelations.set(self.protCTF.outputCTF)
        protExtract.setObjLabel("extract-other (DW mics)")
        self.launchProtocol(protExtract)

        inputCoords = protExtract.inputCoordinates.get()
        outputParts = protExtract.outputParticles         
     
        
        
    def testAssignCTF(self):
        """ Test the particle extraction after importing another
        SetOfMicraphs with a different micName but same ids.
        We will use assign-ctf protocol and extract from the 
        newly imported mics with the assigned CTF.
        For the other mics, we will just create symbolic links.
        """
        # Create the links with a different micrograph name
        micsPath = self.proj.getPath('otherMicrographs')
        pwutils.makePath(micsPath)
        for i in [6, 7, 8]:
            micPath = self.dataset.getFile('micrographs/BPV_138%d.mrc' % i)
            micLink = join(micsPath,
                           basename(micPath).replace('.mrc', '_DW.mrc'))
            pwutils.createAbsLink(micPath, micLink)

        protImportDW = self.proj.copyProtocol(self.protImport)
        protImportDW.setObjLabel('import -mics DW')
        protImportDW.filesPath.set(micsPath)
        protImportDW.filesPattern.set('*_DW.mrc')
        self.launchProtocol(protImportDW)

        protAssignCTF = self.newProtocol(ProtCTFAssign)
        protAssignCTF.inputSet.set(protImportDW.outputMicrographs)
        protAssignCTF.inputCTF.set(self.protCTF.outputCTF)
        self.launchProtocol(protAssignCTF)

        protExtract = self.newProtocol(XmippProtExtractParticles,
                                       boxSize=183,
                                       downsampleType=OTHER,
                                       downFactor=3.0,
                                       doFlip=False)
        protExtract.inputCoordinates.set(self.protPP.outputCoordinates)
        protExtract.inputMicrographs.set(protAssignCTF.outputMicrographs)
        protExtract.ctfRelations.set(self.protCTF.outputCTF)
        protExtract.setObjLabel("extract-other (DW mics)")
        self.launchProtocol(protExtract)

        inputCoords = protExtract.inputCoordinates.get()
        outputParts = protExtract.outputParticles
    def _runPreviousProtocols(self):
        protImportTomogram = self.newProtocol(tomo.protocols.ProtImportTomograms,
                                              filesPath=self.tomogram,
                                              samplingRate=5)

        self.launchProtocol(protImportTomogram)

        coords = pwutils.removeBaseExt(self.coords3D)
        coords = protImportTomogram._getExtraPath(coords + '.txt')
        pwutils.createAbsLink(self.coords3D_Large, coords)
        protImportCoordinates3d = self.newProtocol(tomo.protocols.ProtImportCoordinates3D,
                                                   auto=IMPORT_FROM_EMAN,
                                                   filesPath=coords,
                                                   importTomograms=protImportTomogram.outputTomograms,
                                                   filesPattern='', boxSize=32,
                                                   samplingRate=5)

        self.launchProtocol(protImportCoordinates3d)
        self.assertIsNotNone(protImportTomogram.outputTomograms,
                             "There was a problem with tomogram output")
        self.assertIsNotNone(protImportCoordinates3d.outputCoordinates,
                             "There was a problem with coordinates 3d output")
        doInvert = False
        doNormalize = False
        boxSize = 32
        protTomoExtraction = self.newProtocol(EmanProtTomoExtraction,
                                              inputTomograms=protImportTomogram.outputTomograms,
                                              inputCoordinates=protImportCoordinates3d.outputCoordinates,
                                              tomoSource=0,
                                              doInvert=doInvert,
                                              doNormalize=doNormalize,
                                              boxSize=boxSize)

        self.launchProtocol(protTomoExtraction)
        self.assertIsNotNone(protTomoExtraction.outputSetOfSubtomogram,
                             "There was a problem with SetOfSubtomogram output")

        return protTomoExtraction
Exemple #22
0
    def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6):
        """ Function to generate thumbnails for the report. Uses data from
        self.thumbPaths.

        ===== Params =====
        - firstThumbIndex: index from which we start generating thumbnails
        - micScaleFactor: how much to reduce in size the micrographs.

        """
        ih = ImageHandler()

        numMics = len(self.thumbPaths[MIC_PATH])

        for i in range(firstThumbIndex, numMics):
            print('Generating images for mic %d' % (i + 1))
            # mic thumbnails
            dstImgPath = join(self.reportDir, self.thumbPaths[MIC_THUMBS][i])
            if not exists(dstImgPath):
                if self.micThumbSymlinks:
                    pwutils.createAbsLink(self.thumbPaths[MIC_PATH][i],
                                          dstImgPath)
                else:
                    ih.computeThumbnail(self.thumbPaths[MIC_PATH][i],
                                        dstImgPath,
                                        scaleFactor=micScaleFactor)

            # shift plots
            if SHIFT_THUMBS in self.thumbPaths:
                dstImgPath = join(self.reportDir,
                                  self.thumbPaths[SHIFT_THUMBS][i])
                if not exists(dstImgPath):
                    pwutils.createAbsLink(self.thumbPaths[SHIFT_PATH][i],
                                          dstImgPath)

            # Psd thumbnails
            if self.ctfProtocol is None:
                srcImgPath = self.thumbPaths[PSD_PATH][i]
                dstImgPath = join(self.reportDir,
                                  self.thumbPaths[PSD_THUMBS][i])
                if not exists(dstImgPath):
                    if srcImgPath.endswith('psd'):
                        psdImg1 = ih.read(srcImgPath)
                        psdImg1.convertPSD()
                        psdImg1.write(dstImgPath)
                        ih.computeThumbnail(dstImgPath,
                                            dstImgPath,
                                            scaleFactor=1)
                    else:
                        pwutils.createAbsLink(srcImgPath, dstImgPath)
            else:
                dstImgPath = join(self.reportDir,
                                  self.thumbPaths[PSD_THUMBS][i])
                if not exists(dstImgPath):
                    ih.computeThumbnail(self.thumbPaths[PSD_PATH][i],
                                        dstImgPath,
                                        scaleFactor=1)

        return
Exemple #23
0
    def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6):
        """ Function to generate thumbnails for the report. Uses data from
        self.thumbPaths.

        ===== Params =====
        - firstThumbIndex: index from which we start generating thumbnails
        - micScaleFactor: how much to reduce in size the micrographs.

        """
        ih = ImageHandler()

        numMics = len(self.thumbPaths[MIC_PATH])

        for i in range(firstThumbIndex, numMics):
            print('Generating images for mic %d' % (i+1))
            # mic thumbnails
            dstImgPath = join(self.reportDir, self.thumbPaths[MIC_THUMBS][i])
            if not exists(dstImgPath):
                if self.micThumbSymlinks:
                    pwutils.createAbsLink(self.thumbPaths[MIC_PATH][i], dstImgPath)
                else:
                    ih.computeThumbnail(self.thumbPaths[MIC_PATH][i],
                                        dstImgPath, scaleFactor=micScaleFactor,
                                        flipOnY=True)

            # shift plots
            if SHIFT_THUMBS in self.thumbPaths:
                dstImgPath = join(self.reportDir, self.thumbPaths[SHIFT_THUMBS][i])
                if not exists(dstImgPath):
                    pwutils.createAbsLink(self.thumbPaths[SHIFT_PATH][i], dstImgPath)

            # Psd thumbnails
            # If there ARE thumbnail for the PSD (no ctf protocol and
            # moviealignment hasn't computed it
            if PSD_THUMBS in self.thumbPaths:
                if self.ctfProtocol is None:
                    srcImgPath = self.thumbPaths[PSD_PATH][i]
                    dstImgPath = join(self.reportDir, self.thumbPaths[PSD_THUMBS][i])
                    if not exists(dstImgPath) and srcImgPath is not None:
                        if srcImgPath.endswith('psd'):
                            psdImg1 = ih.read(srcImgPath)
                            psdImg1.convertPSD()
                            psdImg1.write(dstImgPath)
                            ih.computeThumbnail(dstImgPath, dstImgPath,
                                                scaleFactor=1, flipOnY=True)
                        else:
                            pwutils.createAbsLink(srcImgPath, dstImgPath)
                else:
                    dstImgPath = join(self.reportDir, self.thumbPaths[PSD_THUMBS][i])
                    if not exists(dstImgPath):
                        ih.computeThumbnail(self.thumbPaths[PSD_PATH][i],
                                            dstImgPath, scaleFactor=1, flipOnY=True)

        return
n = len(sys.argv)

if n < 2 or n > 5:
    usage("Incorrect number of input parameters")
fromLocation = sys.argv[1]
projName = fromLocation.rsplit('/', 1)[-1] if n < 3 else sys.argv[2]

jsonFile = None if n < 4 else os.path.abspath(sys.argv[3])
location = None if n < 5 else sys.argv[4]

path = os.path.join(os.environ['SCIPION_HOME'], 'pyworkflow', 'gui',
                    'no-tkinter')
sys.path.insert(1, path)

# Create a new project
manager = Manager()

# if manager.hasProject(projName):
#     usage("There is already a project with this name: %s"
#           % pwutils.red(projName))

project = manager.createProject(projName, location=location)
# copy sqlite and make links of runs
pwutils.copyFile(os.path.join(fromLocation, PROJECT_DBNAME), project.path)
fromRuns = os.path.join(fromLocation, PROJECT_RUNS)
for r in os.listdir(fromRuns):
    print " %s,%s" % (os.path.join(fromRuns,
                                   r), os.path.join(project.runsPath, r))
    pwutils.createAbsLink(os.path.join(fromRuns, r),
                          os.path.join(project.runsPath, r))
if len(sys.argv) == 4:
    delay = sys.argv[3]
else:
    delay = 30

inputFiles = glob(pwutils.expandPattern(inputPattern))
inputFiles.sort()

print "Input pattern: ", inputPattern
print "Input files: ", inputFiles
print "Delay: ", str(delay), " seconds."

print "Cleaning output directory: ", outputDir
pwutils.cleanPath(outputDir)
pwutils.makePath(outputDir)

aTime = int(delay)
n = 5
t = aTime / n
print "t=%s" % t

for f in inputFiles:
    outputPath = os.path.join(outputDir, os.path.basename(f))
    print "Linking %s -> %s" % (outputPath, f)

    for i in range(n):
        open(outputPath, 'w').close()
        time.sleep(t)
    pwutils.cleanPath(outputPath)
    pwutils.createAbsLink(f, outputPath)
Exemple #26
0
    usage("Unexistent project: %s" % pwutils.red(projName))
    
if not os.path.exists(searchDir):
    usage("Unexistent SEARCH_DIR: %s" % pwutils.red(searchDir))
    
project = manager.loadProject(projName)

runs = project.getRuns()

for prot in runs:
    broken = False
    if isinstance(prot, em.ProtImport):
        for _, attr in prot.iterOutputEM():
            fn = attr.getFiles()
            for f in attr.getFiles():
                if ':' in f:
                    f = f.split(':')[0]

                if not os.path.exists(f):                    
                    if not broken:
                        broken = True
                        print "Found broken links in run: ", pwutils.magenta(prot.getRunName())
                    print "  Missing: ", pwutils.magenta(f)
                    if os.path.islink(f):
                        print "    -> ", pwutils.red(os.path.realpath(f))
                    newFile = pwutils.findFile(os.path.basename(f), searchDir, recursive=True)
                    if newFile:
                        print "  Found file %s, creating link..." % newFile
                        print pwutils.green("   %s -> %s" % (f, newFile))
                        pwutils.createAbsLink(newFile, f)
    def processMovieStep(self, movieDict, hasAlignment):
        movie = emobj.Movie()
        movie.setAcquisition(emobj.Acquisition())

        if hasAlignment:
            movie.setAlignment(emobj.MovieAlignment())

        movie.setAttributesFromDict(movieDict,
                                    setBasic=True,
                                    ignoreMissing=True)

        movieFolder = self._getOutputMovieFolder(movie)
        movieFn = movie.getFileName()
        movieName = basename(movieFn)
        movieDoneFn = self._getMovieDone(movie)

        if self.isContinued() and os.path.exists(movieDoneFn):
            self.info("Skipping movie: %s, seems to be done" % movieFn)
            return

        # Clean old finished files
        pwutils.cleanPath(movieDoneFn)

        if self._filterMovie(movie):
            pwutils.makePath(movieFolder)
            pwutils.createAbsLink(os.path.abspath(movieFn),
                                  join(movieFolder, movieName))

            if movieName.endswith('bz2'):
                newMovieName = movieName.replace('.bz2', '')
                # We assume that if compressed the name ends with .mrc.bz2
                if not exists(newMovieName):
                    self.runJob('bzip2',
                                '-d -f %s' % movieName,
                                cwd=movieFolder)

            elif movieName.endswith('tbz'):
                newMovieName = movieName.replace('.tbz', '.mrc')
                # We assume that if compressed the name ends with .tbz
                if not exists(newMovieName):
                    self.runJob('tar', 'jxf %s' % movieName, cwd=movieFolder)

            elif movieName.endswith('.txt'):
                # Support a list of frame as a simple .txt file containing
                # all the frames in a raw list, we could use a xmd as well,
                # but a plain text was choose to simply its generation
                movieTxt = os.path.join(movieFolder, movieName)
                with open(movieTxt) as f:
                    movieOrigin = os.path.basename(os.readlink(movieFn))
                    newMovieName = movieName.replace('.txt', '.mrcs')
                    ih = emlib.image.ImageHandler()
                    for i, line in enumerate(f):
                        if line.strip():
                            inputFrame = os.path.join(movieOrigin,
                                                      line.strip())
                            ih.convert(
                                inputFrame,
                                (i + 1, os.path.join(movieFolder,
                                                     newMovieName)))
            else:
                newMovieName = movieName

            convertExt = self._getConvertExtension(newMovieName)
            correctGain = self._doCorrectGain()

            if convertExt or correctGain:
                inputMovieFn = os.path.join(movieFolder, newMovieName)
                if inputMovieFn.endswith('.em'):
                    inputMovieFn += ":ems"

                if convertExt:
                    newMovieName = pwutils.replaceExt(newMovieName, convertExt)
                else:
                    newMovieName = '%s_corrected.%s' % os.path.splitext(
                        newMovieName)

                outputMovieFn = os.path.join(movieFolder, newMovieName)

                # If the protocols wants Scipion to apply the gain, then
                # there is no reason to convert, since we can produce the
                # output in the format expected by the program. In some cases,
                # the alignment programs can directly deal with gain and dark
                # correction images, so we don't need to apply it
                if self._doCorrectGain():
                    self.info("Correcting gain and dark '%s' -> '%s'" %
                              (inputMovieFn, outputMovieFn))
                    gain, dark = self.getGainAndDark()
                    self.correctGain(inputMovieFn,
                                     outputMovieFn,
                                     gainFn=gain,
                                     darkFn=dark)
                else:
                    self.info("Converting movie '%s' -> '%s'" %
                              (inputMovieFn, outputMovieFn))

                    emlib.image.ImageHandler().convertStack(
                        inputMovieFn, outputMovieFn)

            # Just store the original name in case it is needed in _processMovie
            movie._originalFileName = pwobj.String(objDoStore=False)
            movie._originalFileName.set(movie.getFileName())
            # Now set the new filename (either linked or converted)
            movie.setFileName(os.path.join(movieFolder, newMovieName))
            self.info("Processing movie: %s" % movie.getFileName())

            self._processMovie(movie)

            if self._doMovieFolderCleanUp():
                self._cleanMovieFolder(movieFolder)

        # Mark this movie as finished
        open(movieDoneFn, 'w').close()
 def _link(mic, newName):
     pwutils.createAbsLink(os.path.abspath(mic.getFileName()),
                           os.path.join(micDir, newName))
Exemple #29
0
    def importVolumesStep(self, pattern, samplingRate, setOrigCoord=False):
        """ Copy images matching the filename pattern
        Register other parameters.
        """
        volSet = self._createSetOfVolumes()
        vol = emobj.Volume()

        if self.importFrom == self.IMPORT_FROM_FILES:
            self.info("Using pattern: '%s'" % pattern)

            # Create a Volume template object
            vol.setSamplingRate(samplingRate)

            imgh = emlib.image.ImageHandler()

            volSet.setSamplingRate(samplingRate)

            for fileName, fileId in self.iterFiles():
                x, y, z, n = imgh.getDimensions(fileName)
                if fileName.endswith('.mrc') or fileName.endswith('.map'):
                    fileName += ':mrc'
                    if z == 1 and n != 1:
                        zDim = n
                        n = 1
                    else:
                        zDim = z
                else:
                    zDim = z
                origin = emobj.Transform()
                if setOrigCoord:
                    origin.setShiftsTuple(self._getOrigCoord())
                else:
                    origin.setShifts(x / -2. * samplingRate,
                                     y / -2. * samplingRate,
                                     zDim / -2. * samplingRate)

                vol.setOrigin(origin)  # read origin from form

                if self.copyFiles or setOrigCoord:
                    newFileName = abspath(
                        self._getVolumeFileName(fileName, "mrc"))
                    emconv.Ccp4Header.fixFile(fileName, newFileName,
                                              origin.getShifts(), samplingRate,
                                              emconv.Ccp4Header.ORIGIN)
                    if self.setHalfMaps.get():
                        newFileName1 = abspath(
                            self._getVolumeFileName(self.half1map.get(),
                                                    "mrc"))
                        emconv.Ccp4Header.fixFile(self.half1map.get(),
                                                  newFileName1,
                                                  origin.getShifts(),
                                                  samplingRate,
                                                  emconv.Ccp4Header.ORIGIN)
                        newFileName2 = abspath(
                            self._getVolumeFileName(self.half2map.get(),
                                                    "mrc"))
                        emconv.Ccp4Header.fixFile(self.half2map.get(),
                                                  newFileName2,
                                                  origin.getShifts(),
                                                  samplingRate,
                                                  emconv.Ccp4Header.ORIGIN)

                        vol.setHalfMaps(
                            [relpath(newFileName1),
                             relpath(newFileName2)])
                else:
                    newFileName = abspath(self._getVolumeFileName(fileName))

                    if fileName.endswith(':mrc'):
                        fileName = fileName[:-4]

                    pwutils.createAbsLink(fileName, newFileName)
                    if self.setHalfMaps.get():
                        pwutils.createAbsLink(
                            self.half1map.get(),
                            abspath(
                                self._getVolumeFileName(self.half1map.get())))
                        pwutils.createAbsLink(
                            self.half2map.get(),
                            abspath(
                                self._getVolumeFileName(self.half2map.get())))

                        vol.setHalfMaps([
                            relpath(
                                self._getVolumeFileName(self.half1map.get())),
                            relpath(
                                self._getVolumeFileName(self.half2map.get()))
                        ])

                # Make newFileName relative
                # https://github.com/I2PC/scipion/issues/1935
                newFileName = relpath(newFileName)
                if n == 1:
                    vol.cleanObjId()
                    vol.setFileName(newFileName)
                    volSet.append(vol)
                else:
                    for index in range(1, n + 1):
                        vol.cleanObjId()
                        vol.setLocation(index, newFileName)
                        volSet.append(vol)
        else:  # import from EMDB
            self.info("Downloading map with ID = %s" % self.emdbId)
            try:
                localFileName, sampling, origin = \
                    fetch_emdb_map(self.emdbId,
                                   self._getExtraPath(),
                                   self._getTmpPath())
            except Exception as e:
                print(e)
                return
            # open volume and fill sampling and origin
            vol.setSamplingRate(sampling)
            vol.setFileName(localFileName)
            from pwem.objects.data import Transform
            originMat = Transform()
            originMat.setShifts(origin[0], origin[1], origin[2])
            vol.setOrigin(originMat)

        if volSet.getSize() > 1:
            self._defineOutputs(outputVolumes=volSet)
        else:
            self._defineOutputs(outputVolume=vol)
  def convertInputStep(self, inputCoordinates, scale, kfold):
    """ Converts a set of coordinates to box files and binaries to mrc
    if needed. It generates 2 folders 1 for the box files and another for
    the mrc files.
    """

    micIds = []
    coordSet = self.inputCoordinates.get()
    setFn = coordSet.getFileName()
    self.debug("Loading input db: %s" % setFn)

    # Load set of coordinates with a user determined number of coordinates for the training step
    enoughMicrographs = False
    while True:
      coordSet = SetOfCoordinates(filename=setFn)
      coordSet._xmippMd = params.String()
      coordSet.loadAllProperties()

      for micAgg in coordSet.aggregate(["MAX"], "_micId", ["_micId"]):
        micIds.append(micAgg["_micId"])
        if len(micIds) == self.micsForTraining.get():
          enoughMicrographs = True
          break
      if enoughMicrographs:
        break
      else:
        if coordSet.isStreamClosed():
          raise Exception("We have a problem!!")
        self.info("Not yet there: %s" % len(micIds))
        import time
        time.sleep(10)

    # Create input folder and pre-processed micrographs folder
    micDir = self._getFileName(TRAINING)
    pw.utils.makePath(micDir)
    prepDir = self._getFileName(TRAININGPREPROCESS)
    pw.utils.makePath(prepDir)

    ih = ImageHandler()

    # Get a refreshed set of micrographs
    micsFn = self.inputCoordinates.get().getMicrographs().getFileName()
    # not updating, refresh problem
    coordMics = SetOfMicrographs(filename=micsFn)
    coordMics.loadAllProperties()

    # Create a 0/1 list to mark micrographs for training/testing
    n = len(micIds)
    indexes = np.zeros(n, dtype='int8')
    testSetImages = int((kfold / float(100)) * n)

    # Both the training and the test data set should contain at least one micrograph
    if testSetImages < 1:
      requiredMinimumPercentage = (1 * 100 / n) + 1
      testSetImages = int((requiredMinimumPercentage / float(100)) * n)
    elif testSetImages == n:
      testSetImages = int(0.99 * n)
    indexes[:testSetImages] = 1
    np.random.shuffle(indexes)
    self.info('indexes: %s' % indexes)

    # Write micrographs files
    csvMics = [
      CsvMicrographList(self._getFileName(TRAININGLIST), 'w'),
      CsvMicrographList(self._getFileName(TRAININGTEST), 'w')
    ]

    # Store the micId and indexes in micDict
    micDict = {}
    for i, micId in zip(indexes, micIds):
      mic = coordMics[micId]
      micFn = mic.getFileName()
      baseFn = pw.utils.removeBaseExt(micFn)
      inputFn = self._getFileName(TRAINING_MIC, **{"mic": baseFn})
      if micFn.endswith('.mrc'):
        pwutils.createAbsLink(os.path.abspath(micFn), inputFn)
      else:
        ih.convert(micFn, inputFn)

      prepMicFn = self._getFileName(TRAININGPRE_MIC, **{"mic": baseFn})

      csvMics[i].addMic(micId, prepMicFn)
      micDict[micId] = i  # store if train or test

    for csv in csvMics:
      csv.close()

    # Write particles files
    csvParts = [
      CsvCoordinateList(self._getFileName(PARTICLES_TRAIN_TXT), 'w'),
      CsvCoordinateList(self._getFileName(PARTICLES_TEST_TXT), 'w')
    ]

    for coord in coordSet.iterItems(orderBy='_micId'):
      micId = coord.getMicId()
      if micId in micDict:
        x = int(round(float(coord.getX()) / scale))
        y = int(round(float(coord.getY()) / scale))
        csvParts[micDict[micId]].addCoord(micId, x, y)

    for csv in csvParts:
      csv.close()
Exemple #31
0
manager = Manager()

if not manager.hasProject(projName):
    usage("Unexistent project: %s" % pwutils.red(projName))
    
if not os.path.exists(searchDir):
    usage("Unexistent SEARCH_DIR: %s" % pwutils.red(searchDir))
    
project = manager.loadProject(projName)

runs = project.getRuns()

for prot in runs:
    broken = False
    if isinstance(prot, em.ProtImport):
        for _, attr in prot.iterOutputEM():
            fn = attr.getFiles()
            for f in attr.getFiles():
                if not os.path.exists(f):                    
                    if not broken:
                        broken = True
                        print "Found broken links in run: ", pwutils.magenta(prot.getRunName())
                    print "  Missing: ", pwutils.magenta(f)
                    if os.path.islink(f):
                        print "    -> ", pwutils.red(os.path.realpath(f))
                    newFile = pwutils.findFile(os.path.basename(f), searchDir, recursive=True)
                    if newFile:
                        print "  Found file %s, creating link..." % newFile
                        print pwutils.green("   %s -> %s" % (f, newFile))
                        pwutils.createAbsLink(newFile, f)
Exemple #32
0
def convertBinaryFiles(imgSet,
                       outputDir,
                       extension='mrcs',
                       forceConvert=False):
    """ Convert binary images files to a format read by Relion.
    Or create links if there is no need to convert the binary files.

    Params:
        imgSet: input image set to be converted.
        outputDir: where to put the converted file(s)
        extension: extension accepted by the program
        forceConvert: if True, the files will be converted and no root will be used
    Return:
        A dictionary with old-file as key and new-file as value
        If empty, not conversion was done.
    """
    filesDict = {}
    ih = ImageHandler()
    outputRoot = outputDir if forceConvert else os.path.join(
        outputDir, 'input')
    # Get the extension without the dot
    stackFiles = imgSet.getFiles()
    ext = pwutils.getExt(next(iter(stackFiles)))[1:]
    rootDir = pwutils.commonPath(list(stackFiles))

    def getUniqueFileName(fn, extension):
        """ Get an unique file for either link or convert files.
        It is possible that the base name overlap if they come
        from different runs. (like particles.mrcs after relion preprocess)
        """
        newFn = os.path.join(outputRoot, pwutils.replaceBaseExt(fn, extension))
        newRoot = pwutils.removeExt(newFn)

        values = filesDict.values()
        counter = 1

        while newFn in values:
            counter += 1
            newFn = '%s_%05d.%s' % (newRoot, counter, extension)

        return newFn

    def createBinaryLink(fn):
        """ Just create a link named .mrcs to Relion understand
        that it is a binary stack file and not a volume.
        """
        newFn = getUniqueFileName(fn, extension)
        if not os.path.exists(newFn):
            pwutils.createLink(fn, newFn)
            print("   %s -> %s" % (newFn, fn))
        return newFn

    def convertStack(fn):
        """ Convert from a format that is not read by Relion
        to an spider stack.
        """
        newFn = getUniqueFileName(fn, 'mrcs')
        ih.convertStack(fn, newFn)
        print("   %s -> %s" % (newFn, fn))
        return newFn

    def replaceRoot(fn):
        """ Link create to the root folder, so just replace that
        in the name, no need to do anything else.
        """
        return fn.replace(rootDir, outputRoot)

    if forceConvert:
        print("convertBinaryFiles: forceConvert = True")
        mapFunc = convertStack
    elif ext == extension:
        print("convertBinaryFiles: creating soft links.")
        print("   Root: %s -> %s" % (outputRoot, rootDir))
        mapFunc = replaceRoot
        pwutils.createAbsLink(os.path.abspath(rootDir), outputRoot)
    elif ext == 'mrc' and extension == 'mrcs':
        print("convertBinaryFiles: creating soft links (mrcs -> mrc).")
        mapFunc = createBinaryLink
    elif ext.endswith('hdf'):  # assume eman .hdf format
        print("convertBinaryFiles: converting stacks. (%s -> %s)" %
              (extension, ext))
        mapFunc = convertStack
    else:
        mapFunc = None

    if mapFunc is not None:
        pwutils.makePath(outputRoot)
        for fn in stackFiles:
            newFn = mapFunc(fn)  # convert or link
            filesDict[fn] = newFn  # map new filename

    return filesDict
Exemple #33
0
    def testAssignCTF(self):
        """ Test the particle extraction after importing another
        SetOfMicrographs with a different micName but same ids.
        We will use assign-ctf protocol and extract from the 
        newly imported mics with the assigned CTF.
        For the other mics, we will just create symbolic links.
        """
        # Create the links with a different micrograph name
        micsPath = self.proj.getPath('otherMicrographs')
        pwutils.makePath(micsPath)
        for i in [6, 7, 8]:
            micPath = self.dataset.getFile('micrographs/BPV_138%d.mrc' % i)
            micLink = join(micsPath,
                           basename(micPath).replace('.mrc', '_DW.mrc'))
            pwutils.createAbsLink(micPath, micLink)

        protImportDW = self.proj.copyProtocol(self.protImport)
        protImportDW.setObjLabel('import -mics DW')
        protImportDW.filesPath.set(os.path.abspath(micsPath))
        protImportDW.filesPattern.set('*_DW.mrc')
        self.launchProtocol(protImportDW)

        protAssignCTF = self.newProtocol(ProtCTFAssign)
        protAssignCTF.inputSet.set(protImportDW.outputMicrographs)
        protAssignCTF.inputCTF.set(self.protCTF.outputCTF)
        self.launchProtocol(protAssignCTF)
        downFactor = 3.0

        protExtract = self.newProtocol(XmippProtExtractParticles,
                                       boxSize=183,
                                       downsampleType=OTHER,
                                       doDownsample=True,
                                       downFactor=downFactor,
                                       doInvert=False,
                                       doFlip=False)
        protExtract.inputCoordinates.set(self.protPP.outputCoordinates)
        protExtract.inputMicrographs.set(protAssignCTF.outputMicrographs)
        protExtract.ctfRelations.set(self.protCTF.outputCTF)
        protExtract.setObjLabel("extract-other (DW mics)")
        self.launchProtocol(protExtract)

        inputCoords = protExtract.inputCoordinates.get()
        outputParts = protExtract.outputParticles
        samplingCoords = inputCoords.getMicrographs().getSamplingRate()
        samplingFinal = protImportDW.outputMicrographs.getSamplingRate(
        ) * downFactor
        samplingMics = protExtract.inputMicrographs.get().getSamplingRate()
        factor = samplingFinal / samplingCoords
        self.assertIsNotNone(outputParts,
                             "There was a problem generating the output.")

        def compare(objId, delta=1.0):
            cx, cy = inputCoords[objId].getPosition()
            px, py = outputParts[objId].getCoordinate().getPosition()
            micNameCoord = inputCoords[objId].getMicName()
            micNamePart = outputParts[objId].getCoordinate().getMicName()
            self.assertAlmostEquals(cx / factor, px, delta=delta)
            self.assertAlmostEquals(cy / factor, py, delta=delta)
            self.assertEqual(
                micNameCoord, micNamePart,
                "The micName should be %s and its %s" %
                (micNameCoord, micNamePart))

        compare(45)
        compare(229)

        self.assertAlmostEqual(outputParts.getSamplingRate() / samplingMics,
                               downFactor, 1)
        self._checkSamplingConsistency(outputParts)
Exemple #34
0
    usage("Incorrect number of input parameters")
    
pathToProj = os.path.abspath(sys.argv[1])

now = datetime.now()
tempSpace = "loading-%s" % now.strftime('%Y%m%d-%H%M%S')
customUserData = os.path.join(os.environ['SCIPION_USER_DATA'],
                              'tmp', tempSpace)

projectsDir = os.path.join(customUserData, 'projects')
pwutils.makePath(projectsDir)

print "Loading projects from:\n", projectsDir

projName = os.path.basename(pathToProj)
pwutils.createAbsLink(pathToProj, os.path.join(projectsDir, projName))
 
# Create a new project
manager = Manager(SCIPION_USER_DATA=customUserData)

proj = manager.loadProject(projName)
projPath = manager.getProjectPath(projName)

class EditorProjectWindow(ProjectWindow):
    def close(self, e=None):
        try:
            print "Deleting temporary folder: ", customUserData
            pwutils.cleanPath(customUserData)
        except Exception, ex:
            print "Error: ", ex
        ProjectWindow.close(self, e)
Exemple #35
0
def create_movies_project(request):

    if request.is_ajax():

        import os
        from pyworkflow.object import Pointer
        from pyworkflow.em.protocol import ProtImportMovies
        from pyworkflow.em.packages.xmipp3 import ProtMovieAlignment

        # Create a new project
        projectName = request.GET.get('projectName')

        # Filename to use as test data
        testDataKey = request.GET.get('testData')

        manager = getServiceManager('movies')
        writeCustomMenu(manager.protocols)
        project = manager.createProject(projectName,
                                        runsView=1,
                                        hostsConf=manager.hosts,
                                        protocolsConf=manager.protocols)

        project.getSettings().setLifeTime(336)  # 14 days * 24 hours
        project.saveSettings()

        #         copyFile(customMenu, project.getPath('.config', 'protocols.conf'))

        # Create symbolic link for uploads
        projectPath = manager.getProjectPath(projectName)
        dest = os.path.join(projectPath, 'Uploads')
        os.rmdir(dest)  #in movies uploads is created as a link
        # @todo: this path to uploads dir should be configurable outside the code...
        source = "/services/scipion/data/uploads/" + projectName
        pwutils.path.makePath(source)
        pwutils.createLink(source, dest)

        # 1. Import movies
        if testDataKey:
            attr = getAttrTestFile(testDataKey)
            path_test = attr['path']

            for f in os.listdir(path_test):
                # Create a symbolic link for each file
                file_path = os.path.join(path_test, f)
                source_file = os.path.join(source, f)
                pwutils.createAbsLink(file_path, source_file)

            label_import = "import movies (" + testDataKey + ")"
            protImport = project.newProtocol(ProtImportMovies,
                                             objLabel=label_import)

            protImport.filesPath.set(attr["filesPath"])
            protImport.voltage.set(attr['voltage'])
            protImport.sphericalAberration.set(attr['sphericalAberration'])
            protImport.amplitudeContrast.set(attr['amplitudeContrast'])
            protImport.magnification.set(attr['magnification'])
            protImport.samplingRate.set(attr['samplingRate'])

            project.launchProtocol(protImport, wait=True)
        else:
            protImport = project.newProtocol(ProtImportMovies,
                                             objLabel='import movies')
            project.saveProtocol(protImport)

        # 2. Movie Alignment
        protMovAlign = project.newProtocol(ProtMovieAlignment)
        protMovAlign.setObjLabel('xmipp - movie alignment')
        protMovAlign.inputMovies.set(protImport)
        protMovAlign.inputMovies.setExtended('outputMovies')
        project.saveProtocol(protMovAlign)

    return HttpResponse(mimetype='application/javascript')
Exemple #36
0
if n < 2 or n > 3:
    usage("Incorrect number of input parameters")

pathToProj = os.path.abspath(sys.argv[1])

now = datetime.now()
tempSpace = "loading-%s" % now.strftime('%Y%m%d-%H%M%S')
customUserData = os.path.join(pw.Config.SCIPION_USER_DATA, 'tmp', tempSpace)

projectsDir = os.path.join(customUserData, 'projects')
pwutils.makePath(projectsDir)

print("Loading projects from:\n", projectsDir)

projName = os.path.basename(pathToProj)
pwutils.createAbsLink(pathToProj, os.path.join(projectsDir, projName))

# Create a new project
manager = Manager(workspace=customUserData)

proj = manager.loadProject(projName)
projPath = manager.getProjectPath(projName)


class EditorProjectWindow(ProjectWindow):
    def close(self, e=None):
        try:
            print("Deleting temporary folder: ", customUserData)
            pwutils.cleanPath(customUserData)
        except Exception as ex:
            print("Error: ", ex)
Exemple #37
0
def create_movies_project(request):
    
    if request.is_ajax():
        
        import os
        from pyworkflow.object import Pointer
        from pyworkflow.em.protocol import ProtImportMovies
        from pyworkflow.em.packages.xmipp3 import ProtMovieAlignment
        
        # Create a new project
        projectName = request.GET.get('projectName')
        
        # Filename to use as test data 
        testDataKey = request.GET.get('testData')
        
        manager = getServiceManager('movies')
        writeCustomMenu(manager.protocols)
        project = manager.createProject(projectName, runsView=1, 
                                        hostsConf=manager.hosts,
                                        protocolsConf=manager.protocols
                                        )   
        
        project.getSettings().setLifeTime(14)
        project.saveSettings()
        
#         copyFile(customMenu, project.getPath('.config', 'protocols.conf'))
        
        # Create symbolic link for uploads
        projectPath = manager.getProjectPath(projectName)
        dest = os.path.join(projectPath,'Uploads')
        # @todo: this path to uploads dir should be configurable outside the code...
        source = "/services/scipion/data/uploads/"+ projectName
        pwutils.path.makePath(source)
        pwutils.createLink(source, dest)
        
        # 1. Import movies
        if testDataKey :
            attr = getAttrTestFile(testDataKey)
            path_test = attr['path']

            
            for f in os.listdir(path_test):           
                # Create a symbolic link for each file
                file_path = os.path.join(path_test, f)
                source_file = os.path.join(source, f)
                pwutils.createAbsLink(file_path, source_file)
            
            label_import = "import movies ("+ testDataKey +")" 
            protImport = project.newProtocol(ProtImportMovies, objLabel=label_import)

            protImport.filesPath.set(attr["filesPath"])
            protImport.voltage.set(attr['voltage'])
            protImport.sphericalAberration.set(attr['sphericalAberration'])
            protImport.amplitudeContrast.set(attr['amplitudeContrast'])
            protImport.magnification.set(attr['magnification'])
            protImport.samplingRate.set(attr['samplingRate'])
            
            project.launchProtocol(protImport, wait=True)
        else:
            protImport = project.newProtocol(ProtImportMovies, objLabel='import movies')
            project.saveProtocol(protImport)
        
        # 2. Movie Alignment 
        protMovAlign = project.newProtocol(ProtMovieAlignment)
        protMovAlign.setObjLabel('xmipp - movie alignment')
        protMovAlign.inputMovies.set(protImport)
        protMovAlign.inputMovies.setExtended('outputMovies')
        project.saveProtocol(protMovAlign)
        
    return HttpResponse(mimetype='application/javascript')