예제 #1
0
    def test_readDM4(self):
        """ Check we can read dm4 files (using EMAN)
        """
        micFn = self.dsFormat.getFile('SuperRef_c3-adp-se-xyz-0228_001.dm4')

        ih = ImageHandler()
        # Check that we can read the dimensions of the dm4 file:
        EXPECTED_SIZE = (7676, 7420, 1, 1)
        self.assertEqual(ih.getDimensions(micFn), EXPECTED_SIZE)

        # We could even convert to an mrc file:
        outSuffix = pwutils.replaceBaseExt(micFn, 'mrc')

        outFn = join('/tmp', outSuffix)
        print "Converting: \n%s -> %s" % (micFn, outFn)

        ih.convert(micFn, outFn)

        self.assertTrue(os.path.exists(outFn))
        self.assertTrue(pwutils.getFileSize(outFn) > 0)
        # Check dimensions are still the same:
        self.assertEqual(ih.getDimensions(outFn), EXPECTED_SIZE)

        # Clean up tmp files
        pwutils.cleanPath(outFn)
예제 #2
0
파일: test_data.py 프로젝트: I2PC/scipion
    def test_readDM4(self):
        """ Check we can read dm4 files (using EMAN)
        """
        micFn = self.dsFormat.getFile('SuperRef_c3-adp-se-xyz-0228_001.dm4')

        ih = ImageHandler()
        # Check that we can read the dimensions of the dm4 file:
        EXPECTED_SIZE = (7676, 7420, 1, 1)
        self.assertEqual(ih.getDimensions(micFn), EXPECTED_SIZE)

        # We could even convert to an mrc file:
        outSuffix = pwutils.replaceBaseExt(micFn, 'mrc')

        outFn = join('/tmp', outSuffix)
        print "Converting: \n%s -> %s" % (micFn, outFn)

        ih.convert(micFn, outFn)

        self.assertTrue(os.path.exists(outFn))
        self.assertTrue(pwutils.getFileSize(outFn) > 0)
        # Check dimensions are still the same:
        self.assertEqual(ih.getDimensions(outFn), EXPECTED_SIZE)

        # Clean up tmp files
        pwutils.cleanPath(outFn)
예제 #3
0
    def test_convertMovie(self):
        """Check movie conversion"""
        movFn = self.dsFormat.getFile('qbeta/qbeta.mrc') + ":mrcs"
        
        ih = ImageHandler()
        # Check that we can read the dimensions of the dm4 file:
        EXPECTED_SIZE = (4096, 4096, 1, 7)
        EXPECTED_DT = ImageHandler.DT_USHORT

        self.assertEqual(ih.getDimensions(movFn), EXPECTED_SIZE)
        self.assertEqual(ih.getDataType(movFn), EXPECTED_DT)
        
        outFn = join('/tmp/qbeta_converted.mrcs')

        ih.convertStack(movFn, outFn, 2, 6)
        
        self.assertTrue(os.path.exists(outFn))
        self.assertTrue(pwutils.getFileSize(outFn) > 0)
        self.assertEqual(ih.getDimensions(outFn), (4096, 4096, 1, 5))
        self.assertEqual(ih.getDataType(outFn), EXPECTED_DT)

        if pwutils.envVarOn('SCIPION_DEBUG_NOCLEAN'):
            print "Not cleaning output movie: ", outFn
        else:
            pwutils.cleanPath(outFn)
예제 #4
0
파일: test_data.py 프로젝트: I2PC/scipion
    def test_convertMovie(self):
        """Check movie conversion"""
        movFn = self.dsFormat.getFile('qbeta/qbeta.mrc') + ":mrcs"
        
        ih = ImageHandler()
        # Check that we can read the dimensions of the dm4 file:
        EXPECTED_SIZE = (4096, 4096, 1, 7)
        EXPECTED_DT = ImageHandler.DT_USHORT

        self.assertEqual(ih.getDimensions(movFn), EXPECTED_SIZE)
        self.assertEqual(ih.getDataType(movFn), EXPECTED_DT)
        
        outFn = join('/tmp/qbeta_converted.mrcs')

        ih.convertStack(movFn, outFn, 2, 6)
        
        self.assertTrue(os.path.exists(outFn))
        self.assertTrue(pwutils.getFileSize(outFn) > 0)
        self.assertEqual(ih.getDimensions(outFn), (4096, 4096, 1, 5))
        self.assertEqual(ih.getDataType(outFn), EXPECTED_DT)

        if pwutils.envVarOn('SCIPION_DEBUG_NOCLEAN'):
            print "Not cleaning output movie: ", outFn
        else:
            pwutils.cleanPath(outFn)
예제 #5
0
    def retrieveTrainSets(self):
        """ Retrieve, link and return a setOfParticles
            corresponding to the NegativeTrain DeepConsensus trainning set
            with certain extraction conditions (phaseFlip/invContrast)
        """
        prefixYES = ''
        prefixNO = 'no'
        modelType = "negativeTrain_%sPhaseFlip_%sInvert.mrcs" % (
            prefixYES if self.doInvert.get() else prefixNO,
            prefixYES if self.ignoreCTF.get() else prefixNO)
        modelPath = xmipp3.Plugin.getModel("deepConsensus", modelType)
        print("Precompiled negative particles found at %s" % (modelPath))
        modelFn = self._getTmpPath(modelType)
        pwutils.createLink(modelPath, modelFn)

        tmpSqliteSuff = "AddTrain"
        partSet = self._createSetOfParticles(tmpSqliteSuff)
        img = SetOfParticles.ITEM_TYPE()

        imgh = ImageHandler()
        _, _, _, n = imgh.getDimensions(modelFn)
        if n > 1:
            for index in range(1, n + 1):
                img.cleanObjId()
                img.setMicId(9999)
                img.setFileName(modelFn)
                img.setIndex(index)
                partSet.append(img)
        partSet.setAlignment(ALIGN_NONE)

        cleanPath(self._getPath("particles%s.sqlite" % tmpSqliteSuff))
        return partSet
    def correctGain(self, movieFn, outputFn, gainFn=None, darkFn=None):
        """correct a movie with both gain and dark images"""
        ih = ImageHandler()
        _, _, z, n = ih.getDimensions(movieFn)
        numberOfFrames = max(z, n) # in case of wrong mrc stacks as volumes

        def _readImgFloat(fn):
            img = None
            if fn:
                img = ih.read(fn)
                img.convert2DataType(ih.DT_FLOAT)
            return img

        gainImg = _readImgFloat(gainFn)
        darkImg = _readImgFloat(darkFn)

        img = ih.createImage()

        for i in range(1, numberOfFrames + 1):
            img.read((i, movieFn))
            img.convert2DataType(ih.DT_FLOAT)

            if darkImg:
                img.inplaceSubtract(darkImg)
            if gainImg:
                img.inplaceMultiply(gainImg)

            img.write((i, outputFn))
예제 #7
0
    def importVolumesStep(self, pattern, samplingRate):
        """ Copy images matching the filename pattern
        Register other parameters.
        """
        self.info("Using pattern: '%s'" % pattern)

        # Create a Volume template object
        vol = Volume()
        vol.setSamplingRate(self.samplingRate.get())
        copyOrLink = self.getCopyOrLink()
        imgh = ImageHandler()

        volSet = self._createSetOfVolumes()
        volSet.setSamplingRate(self.samplingRate.get())

        for fileName, fileId in self.iterFiles():
            dst = self._getExtraPath(basename(fileName))
            copyOrLink(fileName, dst)
            x, y, z, n = imgh.getDimensions(dst)
            # First case considers when reading mrc without volume flag
            # Second one considers single volumes (not in stack)
            if (z == 1 and n != 1) or (z !=1 and n == 1):
                vol.setObjId(fileId)
                vol.setLocation(dst)
                volSet.append(vol)
            else:
                for index in range(1, n+1):
                    vol.cleanObjId()
                    vol.setLocation(index, dst)
                    volSet.append(vol)

        if volSet.getSize() > 1:
            self._defineOutputs(outputVolumes=volSet)
        else:
            self._defineOutputs(outputVolume=vol)
예제 #8
0
    def importImagesStep(self, pattern, voltage, sphericalAberration,
                         amplitudeContrast, magnification):
        """ Copy images matching the filename pattern
        Register other parameters.
        """
        self.info("Using pattern: '%s'" % pattern)

        createSetFunc = getattr(self, '_create' + self._outputClassName)
        imgSet = createSetFunc()
        imgSet.setIsPhaseFlipped(self.haveDataBeenPhaseFlipped.get())
        acquisition = imgSet.getAcquisition()

        self.fillAcquisition(acquisition)

        # Call a function that should be implemented by each subclass
        self.setSamplingRate(imgSet)

        outFiles = [imgSet.getFileName()]
        imgh = ImageHandler()
        img = imgSet.ITEM_TYPE()
        img.setAcquisition(acquisition)
        n = 1
        copyOrLink = self.getCopyOrLink()

        for i, (fileName, fileId) in enumerate(self.iterFiles()):
            dst = self._getExtraPath(basename(fileName))
            copyOrLink(fileName, dst)
            # Handle special case of Imagic images, copying also .img or .hed
            self.handleImgHed(copyOrLink, fileName, dst)

            if self._checkStacks:
                _, _, _, n = imgh.getDimensions(dst)

            if n > 1:
                for index in range(1, n + 1):
                    img.cleanObjId()
                    img.setMicId(fileId)
                    img.setFileName(dst)
                    img.setIndex(index)
                    self._addImageToSet(img, imgSet)
            else:
                img.setObjId(fileId)
                img.setFileName(dst)
                # Fill the micName if img is a Micrograph.
                self._fillMicName(img, fileName)
                self._addImageToSet(img, imgSet)

            outFiles.append(dst)

            sys.stdout.write("\rImported %d/%d" % (i + 1, self.numberOfFiles))
            sys.stdout.flush()

        print "\n"

        args = {}
        outputSet = self._getOutputName()
        args[outputSet] = imgSet
        self._defineOutputs(**args)

        return outFiles
예제 #9
0
 def prepareMask(self,maskObject,fnMask,TsMaskOut,XdimOut):
     img=ImageHandler()
     img.convert(maskObject, fnMask)
     self.runJob('xmipp_image_resize',"-i %s --factor %f"%(fnMask,maskObject.getSamplingRate()/TsMaskOut),numberOfMpi=1)
     maskXdim, _, _, _ =img.getDimensions((1,fnMask))
     if XdimOut!=maskXdim:
         self.runJob('xmipp_transform_window',"-i %s --size %d"%(fnMask,XdimOut),numberOfMpi=1)
     self.runJob('xmipp_transform_threshold',"-i %s --select below 0.5 --substitute binarize"%fnMask,numberOfMpi=1)
예제 #10
0
 def importImagesStep(self, pattern, voltage, sphericalAberration, amplitudeContrast, magnification):
     """ Copy images matching the filename pattern
     Register other parameters.
     """
     self.info("Using pattern: '%s'" % pattern)
     
     createSetFunc = getattr(self, '_create' + self._outputClassName)
     imgSet = createSetFunc()
     imgSet.setIsPhaseFlipped(self.haveDataBeenPhaseFlipped.get())
     acquisition = imgSet.getAcquisition()
     
     self.fillAcquisition(acquisition)
     
     # Call a function that should be implemented by each subclass
     self.setSamplingRate(imgSet)
     
     outFiles = [imgSet.getFileName()]
     imgh = ImageHandler()
     img = imgSet.ITEM_TYPE()
     img.setAcquisition(acquisition)
     n = 1
     copyOrLink = self.getCopyOrLink()
     for i, (fileName, fileId) in enumerate(self.iterFiles()):
         dst = self._getExtraPath(basename(fileName))
         copyOrLink(fileName, dst)
         # Handle special case of Imagic images, copying also .img or .hed
         self.handleImgHed(copyOrLink, fileName, dst)
         
         if self._checkStacks:
             _, _, _, n = imgh.getDimensions(dst)
             
         if n > 1:
             for index in range(1, n+1):
                 img.cleanObjId()
                 img.setMicId(fileId)
                 img.setFileName(dst)
                 img.setIndex(index)
                 imgSet.append(img)
         else:
             img.setObjId(fileId)
             img.setFileName(dst)
             self._fillMicName(img, fileName) # fill the micName if img is a Micrograph.
             imgSet.append(img)
         outFiles.append(dst)
         
         sys.stdout.write("\rImported %d/%d" % (i+1, self.numberOfFiles))
         sys.stdout.flush()
         
     print "\n"
     
     args = {}
     outputSet = self._getOutputName()
     args[outputSet] = imgSet
     self._defineOutputs(**args)
     
     return outFiles
예제 #11
0
    def test_truncateMask(self):
        ih = ImageHandler()

        maskFn = self.dataset.getFile('masks/mask.vol')
        outFn = join('/tmp', 'mask.vol')

        ih.truncateMask(maskFn, outFn, newDim=128)
        EXPECTED_SIZE = (128, 128, 128, 1)

        self.assertTrue(os.path.exists(outFn))
        self.assertTrue(pwutils.getFileSize(outFn) > 0)
        self.assertEqual(ih.getDimensions(outFn), EXPECTED_SIZE)

        pwutils.cleanPath(outFn)
예제 #12
0
    def test_readCompressedTIF(self):
        """ Check we can read tif files
        """
        micFn = self.dsFormat.getFile('c3-adp-se-xyz-0228_200.tif')

        ih = ImageHandler()
        # Check that we can read the dimensions of the dm4 file:
        EXPECTED_SIZE = (7676, 7420, 1, 38)
        self.assertEqual(ih.getDimensions(micFn), EXPECTED_SIZE)

        # We could even convert to an mrc file:
        outSuffix = pwutils.replaceBaseExt(micFn, 'mrc')

        outFn = join('/tmp', outSuffix)
        print "Converting: \n%s -> %s" % ((1, micFn), outFn)

        ih.convert((1, micFn), outFn)

        self.assertTrue(os.path.exists(outFn))
        self.assertTrue(pwutils.getFileSize(outFn) > 0)
        self.assertEqual(ih.getDimensions(outFn), (7676, 7420, 1, 1))

        # Clean up tmp files
        pwutils.cleanPath(outFn)
예제 #13
0
 def prepareMask(self, maskObject, fnMask, TsMaskOut, XdimOut):
     img = ImageHandler()
     img.convert(maskObject, fnMask)
     self.runJob('xmipp_image_resize',
                 "-i %s --factor %f" %
                 (fnMask, maskObject.getSamplingRate() / TsMaskOut),
                 numberOfMpi=1)
     maskXdim, _, _, _ = img.getDimensions((1, fnMask))
     if XdimOut != maskXdim:
         self.runJob('xmipp_transform_window',
                     "-i %s --size %d" % (fnMask, XdimOut),
                     numberOfMpi=1)
     self.runJob('xmipp_transform_threshold',
                 "-i %s --select below 0.5 --substitute binarize" % fnMask,
                 numberOfMpi=1)
예제 #14
0
파일: test_data.py 프로젝트: I2PC/scipion
    def test_readCompressedTIF(self):
        """ Check we can read tif files
        """
        micFn = self.dsFormat.getFile('c3-adp-se-xyz-0228_200.tif')

        ih = ImageHandler()
        # Check that we can read the dimensions of the dm4 file:
        EXPECTED_SIZE = (7676, 7420, 1, 38)
        self.assertEqual(ih.getDimensions(micFn), EXPECTED_SIZE)

        # We could even convert to an mrc file:
        outSuffix = pwutils.replaceBaseExt(micFn, 'mrc')

        outFn = join('/tmp', outSuffix)
        print "Converting: \n%s -> %s" % ((1, micFn), outFn)

        ih.convert((1, micFn), outFn)

        self.assertTrue(os.path.exists(outFn))
        self.assertTrue(pwutils.getFileSize(outFn) > 0)
        self.assertEqual(ih.getDimensions(outFn), (7676, 7420, 1, 1))

        # Clean up tmp files
        pwutils.cleanPath(outFn)
예제 #15
0
 def _validate(self):
     from pyworkflow.em.convert import ImageHandler
     ci = self.getImportClass()
     if ci is None:
         errors = ProtImportMicBase._validate(self)
         for micFn, _ in self.iterFiles():
             imgh = ImageHandler()
             if imgh.isImageFile(micFn):
                 _, _, z, n = imgh.getDimensions(micFn)
                 if n > 1 or z > 1:
                     errors.append("The protocol not support micrographs stored in stacks. "
                                   "If you want to obtain your micrographs individually, "
                                   "you can run the following command:\n"
                                   "scipion run scipion_directory/scripts/split_stacks.py --files *your files* --ext *extension*")
             # JMRT: only check the first image, for large dataset
             # even reading the header can take a while
             break 
         return errors
         
     else:
         return ci.validateMicrographs()
    def _insertMovieStep(self, movie):
        # Redefine this function to add the shifts and factor
        # to the processMovieStep function and run properly in parallel with threads

        #retrive shifts here so there is no conflict
        #if the object is accessed inside at the same time by multiple threads
        if self.applyAlignment and movie.hasAlignment():
            shifts = movie.getAlignment().getShifts()
        else:
            #TODO: I do not think this option is ever used
            # Read movie dimensions to iterate through each frame
            movieName =  movie.getFileName()
            imgh = ImageHandler()
            _, _, _, n = imgh.getDimensions(movieName)
            shifts = [0] * (2*n)
                 
        movieStepId = self._insertFunctionStep('processMovieStep',
                                               movie.getObjId(), movie.getFileName(),
                                               shifts, prerequisites=[])  
        
        return movieStepId 
예제 #17
0
    def _validate(self):
        from pyworkflow.em.convert import ImageHandler
        ci = self.getImportClass()
        if ci is None:
            errors = ProtImportMicBase._validate(self)
            for micFn, _ in self.iterFiles():
                imgh = ImageHandler()
                if imgh.isImageFile(micFn):
                    _, _, z, n = imgh.getDimensions(micFn)
                    if n > 1 or z > 1:
                        errors.append(
                            "The protocol not support micrographs stored in stacks. "
                            "If you want to obtain your micrographs individually, "
                            "you can run the following command:\n"
                            "scipion run scipion_directory/scripts/split_stacks.py --files *your files* --ext *extension*"
                        )
                # JMRT: only check the first image, for large dataset
                # even reading the header can take a while
                break
            return errors

        else:
            return ci.validateMicrographs()
예제 #18
0
파일: masks.py 프로젝트: coocoky/scipion
    def importMaskStep(self, path, samplingRate):
        """ Copy mask from maskPath.
        """
        self.info("Using mask path: '%s'" % path)

        # Copy the image file into the project
        dst = self._getExtraPath(basename(path))
        pwutils.copyFile(path, dst)

        # Retrive image dimensions
        imgh = ImageHandler()
        _, _, z, n = imgh.getDimensions(dst)

        # Create a 2D or 3D Mask, consider the case of n>1
        # as the case of some volume maps in mrc format
        if z > 1 or n > 1:
            mask = VolumeMask()
        else:
            mask = Mask()

        mask.setFileName(dst)
        mask.setSamplingRate(samplingRate)

        self._defineOutputs(outputMask=mask)
예제 #19
0
    def importMaskStep(self, path, samplingRate):
        """ Copy mask from maskPath.
        """
        self.info("Using mask path: '%s'" % path)

        # Copy the image file into the project
        dst = self._getExtraPath(basename(path))
        pwutils.copyFile(path, dst)

        # Retrive image dimensions
        imgh = ImageHandler()
        _, _, z, n = imgh.getDimensions(dst)

        # Create a 2D or 3D Mask, consider the case of n>1
        # as the case of some volume maps in mrc format
        if z > 1 or n > 1:
            mask = VolumeMask()
        else:
            mask = Mask()

        mask.setFileName(dst)
        mask.setSamplingRate(samplingRate)

        self._defineOutputs(outputMask=mask)
예제 #20
0
파일: images.py 프로젝트: I2PC/scipion
    def importImagesStreamStep(self, pattern, voltage, sphericalAberration,
                         amplitudeContrast, magnification):
        """ Copy images matching the filename pattern
        Register other parameters.
        """
        self.info("Using pattern: '%s'" % pattern)

        imgSet = self._getOutputSet() if self.isContinued() else None
        
        self.importedFiles = set()
        if imgSet is None:
            createSetFunc = getattr(self, '_create' + self._outputClassName)
            imgSet = createSetFunc()
        elif imgSet.getSize() > 0: # in case of continue
            imgSet.loadAllProperties()
            self._fillImportedFiles(imgSet)
            imgSet.enableAppend()
        
        pointerExcludedMovs = getattr(self, 'moviesToExclude', None)
        if pointerExcludedMovs is not None:
            excludedMovs = pointerExcludedMovs.get()
            self._fillImportedFiles(excludedMovs)

        imgSet.setIsPhaseFlipped(self.haveDataBeenPhaseFlipped.get())
        acquisition = imgSet.getAcquisition()
        self.fillAcquisition(acquisition)
        # Call a function that should be implemented by each subclass
        self.setSamplingRate(imgSet)
        outFiles = [imgSet.getFileName()]
        imgh = ImageHandler()
        img = imgSet.ITEM_TYPE()
        img.setAcquisition(acquisition)
        n = 1
        copyOrLink = self.getCopyOrLink()
        outputName = self._getOutputName()
        alreadyWarned = False  # Use this flag to warn only once

        finished = False
        # this is only used when creating stacks from frame files
        self.createdStacks = set()

        i = 0
        lastDetectedChange = datetime.now()

        # Ignore the timeout variables if we are not really in streaming mode
        if self.dataStreaming:
            timeout = timedelta(seconds=self.timeout.get())
            fileTimeout = timedelta(seconds=self.fileTimeout.get())
        else:
            timeout = timedelta(seconds=5)
            fileTimeout = timedelta(seconds=5)

        while not finished:
            time.sleep(3) # wait 3 seconds before check for new files
            someNew = False
            someAdded = False

            for fileName, uniqueFn, fileId in self.iterNewInputFiles():
                someNew = True
                if self.fileModified(fileName, fileTimeout):
                    continue
                
                dst = self._getExtraPath(uniqueFn)
                self.importedFiles.add(uniqueFn)
                if ' ' in dst:
                    if not alreadyWarned:
                        self.warning('Warning: your file names have white spaces!')
                        self.warning('Removing white spaces from copies/symlinks.')
                        alreadyWarned = True
                    dst = dst.replace(' ', '')
                copyOrLink(fileName, dst)

                self.debug('Importing file: %s' % fileName)
                self.debug("uniqueFn: %s" % uniqueFn)
                self.debug("dst Fn: %s" % dst)

                if self._checkStacks:
                    _, _, _, n = imgh.getDimensions(dst)

                someAdded = True
                self.debug('Appending file to DB...')
                if self.importedFiles: # enable append after first append
                    imgSet.enableAppend()

                if n > 1:
                    for index in range(1, n+1):
                        img.cleanObjId()
                        img.setMicId(fileId)
                        img.setFileName(dst)
                        img.setIndex(index)
                        self._addImageToSet(img, imgSet)
                else:
                    img.setObjId(fileId)
                    img.setFileName(dst)
                    # Fill the micName if img is either a Micrograph or a Movie
                    uniqueFn = uniqueFn.replace(' ', '')
                    self.debug("FILENAME TO fillMicName: %s" % uniqueFn)
                    self._fillMicName(img, uniqueFn)
                    self._addImageToSet(img, imgSet)

                outFiles.append(dst)
                self.debug('After append. Files: %d' % len(outFiles))

            if someAdded:
                self.debug('Updating output...')
                self._updateOutputSet(outputName, imgSet,
                                      state=imgSet.STREAM_OPEN)
                self.debug('Update Done.')

            self.debug('Checking if finished...someNew: %s' % someNew)

            now = datetime.now()

            if not someNew:
                # If there are no new detected files, we should check the
                # inactivity time elapsed (from last event to now) and
                # if it is greater than the defined timeout, we conclude
                # the import and close the output set
                # Another option is to check if the protocol have some
                # special stop condition, this can be used to manually stop
                # some protocols such as import movies
                finished = (now - lastDetectedChange > timeout or
                            self.streamingHasFinished())
                self.debug("Checking if finished:")
                self.debug("   Now - Last Change: %s"
                           % pwutils.prettyDelta(now - lastDetectedChange))
                self.debug("Finished: %s" % finished)
            else:
                # If we have detected some files, we should update
                # the timestamp of the last event
                lastDetectedChange = now

        self._updateOutputSet(outputName, imgSet,
                              state=imgSet.STREAM_CLOSED)

        self._cleanUp()

        return outFiles
예제 #21
0
    def importImagesStep(self, pattern, voltage, sphericalAberration,
                         amplitudeContrast, magnification):
        """ Copy images matching the filename pattern
        Register other parameters.
        """
        self.info("Using pattern: '%s'" % pattern)

        createSetFunc = getattr(self, '_create' + self._outputClassName)
        imgSet = createSetFunc()
        imgSet.setIsPhaseFlipped(self.haveDataBeenPhaseFlipped.get())
        acquisition = imgSet.getAcquisition()

        self.fillAcquisition(acquisition)

        # Call a function that should be implemented by each subclass
        self.setSamplingRate(imgSet)

        outFiles = [imgSet.getFileName()]
        imgh = ImageHandler()
        img = imgSet.ITEM_TYPE()
        img.setAcquisition(acquisition)
        n = 1
        copyOrLink = self.getCopyOrLink()
        alreadyWarned = False  # Use this flag to warn only once

        for i, (fileName, fileId) in enumerate(self.iterFiles()):
            uniqueFn = self._getUniqueFileName(fileName)
            dst = self._getExtraPath(uniqueFn)
            if ' ' in dst:
                if not alreadyWarned:
                    self.warning('Warning: your file names have white spaces!')
                    self.warning('Removing white spaces from copies/symlinks.')
                    alreadyWarned = True
                dst = dst.replace(' ', '')
            copyOrLink(fileName, dst)
            # Handle special case of Imagic images, copying also .img or .hed
            self.handleImgHed(copyOrLink, fileName, dst)

            if self._checkStacks:
                _, _, _, n = imgh.getDimensions(dst)

            if n > 1:
                for index in range(1, n + 1):
                    img.cleanObjId()
                    img.setMicId(fileId)
                    img.setFileName(dst)
                    img.setIndex(index)
                    self._addImageToSet(img, imgSet)
            else:
                img.setObjId(fileId)
                img.setFileName(dst)
                # Fill the micName if img is either Micrograph or Movie
                self._fillMicName(img, uniqueFn)
                self._addImageToSet(img, imgSet)

            outFiles.append(dst)

            sys.stdout.write("\rImported %d/%d" % (i + 1, self.numberOfFiles))
            sys.stdout.flush()

        print "\n"

        args = {}
        outputSet = self._getOutputName()
        args[outputSet] = imgSet
        self._defineOutputs(**args)

        return outFiles
예제 #22
0
파일: images.py 프로젝트: I2PC/scipion
    def importImagesStep(self, pattern, voltage, sphericalAberration, 
                         amplitudeContrast, magnification):
        """ Copy images matching the filename pattern
        Register other parameters.
        """
        self.info("Using pattern: '%s'" % pattern)
        
        createSetFunc = getattr(self, '_create' + self._outputClassName)
        imgSet = createSetFunc()
        imgSet.setIsPhaseFlipped(self.haveDataBeenPhaseFlipped.get())
        acquisition = imgSet.getAcquisition()
        
        self.fillAcquisition(acquisition)
        
        # Call a function that should be implemented by each subclass
        self.setSamplingRate(imgSet)
        
        outFiles = [imgSet.getFileName()]
        imgh = ImageHandler()
        img = imgSet.ITEM_TYPE()
        img.setAcquisition(acquisition)
        n = 1
        copyOrLink = self.getCopyOrLink()
        alreadyWarned = False # Use this flag to warn only once

        for i, (fileName, fileId) in enumerate(self.iterFiles()):
            uniqueFn = self._getUniqueFileName(fileName)
            dst = self._getExtraPath(uniqueFn)
            if ' ' in dst:
                if not alreadyWarned:
                    self.warning('Warning: your file names have white spaces!')
                    self.warning('Removing white spaces from copies/symlinks.')
                    alreadyWarned = True
                dst = dst.replace(' ', '')
            copyOrLink(fileName, dst)
            # Handle special case of Imagic images, copying also .img or .hed
            self.handleImgHed(copyOrLink, fileName, dst)
            
            if self._checkStacks:
                _, _, _, n = imgh.getDimensions(dst)
                
            if n > 1:
                for index in range(1, n+1):
                    img.cleanObjId()
                    img.setMicId(fileId)
                    img.setFileName(dst)
                    img.setIndex(index)
                    self._addImageToSet(img, imgSet)
            else:
                img.setObjId(fileId)
                img.setFileName(dst)
                # Fill the micName if img is either Micrograph or Movie
                uniqueFn = uniqueFn.replace(' ', '')
                self._fillMicName(img, uniqueFn)
                self._addImageToSet(img, imgSet)

            outFiles.append(dst)
            
            sys.stdout.write("\rImported %d/%d" % (i+1, self.numberOfFiles))
            sys.stdout.flush()
            
        print "\n"
        
        args = {}
        outputSet = self._getOutputName()
        args[outputSet] = imgSet
        self._defineOutputs(**args)
        
        return outFiles
    def extractunitCell(self, sym, offset=0, cropZ=False):
        """ extract unit cell from icosahedral phantom
            using xmipp_i2 symmetry
        """
        # create phantom (3D map)
        _samplingRate = 1.34
        _, outputFile1 = mkstemp(suffix=".mrc")
        command = "xmipp_phantom_create "
        args = " -i %s" % self.filename[sym]
        args += " -o %s" % outputFile1
        runJob(None, command, args, env=Plugin.getEnviron())
        ccp4header = Ccp4Header(outputFile1, readHeader=True)
        x, y, z = ccp4header.getDims()
        t = Transform()

        if cropZ:
            _, outputFile2 = mkstemp(suffix=".mrc")
            args = "-i %s -o %s" % (outputFile1, outputFile2)
            args += " --corners "
            args += " %d " % (-x / 2.)
            args += " %d " % (-y / 2.)
            args += " %d " % (0.)
            args += " %d " % (+x / 2.)
            args += " %d " % (+y / 2.)
            args += " %d " % (+z / 2.)
            runJob(None,
                   "xmipp_transform_window",
                   args,
                   env=Plugin.getEnviron())
            t.setShifts(0, 0, 0)
            outputFile = outputFile2
            ccp4header = Ccp4Header(outputFile2, readHeader=True)

        else:
            t.setShifts(0, 0, 0)
            outputFile = outputFile1

        ccp4header.setSampling(_samplingRate)
        ccp4header.setOrigin(t.getShifts())
        ccp4header.writeHeader()

        # import volume
        if cropZ:
            args = {
                'filesPath': outputFile,
                'filesPattern': '',
                'samplingRate': _samplingRate,
                'copyFiles': True,
                'setOrigCoord': True,
                'x': 90. * _samplingRate,
                'y': 90. * _samplingRate,
                'z': 0.
                # x, y, z in Angstroms
            }
        else:
            args = {
                'filesPath': outputFile,
                'filesPattern': '',
                'samplingRate': _samplingRate,
                'copyFiles': True,
                'setDefaultOrigin': False,
            }
        prot = self.newProtocol(ProtImportVolumes, **args)
        prot.setObjLabel('import volume(%s)' % XMIPP_SYM_NAME[sym])
        self.launchProtocol(prot)
        # execute protocol extract unitCell
        args = {
            'inputVolumes': prot.outputVolume,
            'symmetryGroup': sym,
            'symmetryOrder': self.symOrder,
            'innerRadius': self.innerRadius,
            'outerRadius': self.outerRadius,
            'expandFactor': .2,
            'offset': offset
        }
        prot = self.newProtocol(XmippProtExtractUnit, **args)
        prot.setObjLabel('extract unit cell')
        self.launchProtocol(prot)

        # check results
        ih = ImageHandler()
        xdim, ydim, zdim, ndim = \
            ih.getDimensions(prot.outputVolume.getFileName())
        self.assertTrue(abs(xdim - self.box[sym][0]) < 2)
        self.assertTrue(abs(ydim - self.box[sym][1]) < 2)
        self.assertTrue(abs(zdim - self.box[sym][2]) < 2)

        # create pdb fileoutput
        args = {
            'inputStructure': prot.outputVolume,
            'maskMode': NMA_MASK_THRE,
            'maskThreshold': 0.5,
            'pseudoAtomRadius': 1.5
        }
        prot = self.newProtocol(XmippProtConvertToPseudoAtoms, **args)
        prot.setObjLabel('get pdb')
        self.launchProtocol(prot)

        # check results
        filenamePdb = prot._getPath('pseudoatoms.pdb')
        self.assertTrue(os.path.isfile(filenamePdb))
        # delete temporary files
        os.remove(self.filename[sym])
        os.remove(outputFile)
예제 #24
0
    def _loadTable(self, tableName):
        """ Load information from tables PREFIX_Classes, PREFIX_Objects. """

        tableName = self.tablePrefixes[tableName]

        BASIC_COLUMNS = [
            Column('id', int, renderType=COL_RENDER_ID),
            Column('enabled', bool, renderType=COL_RENDER_CHECKBOX),
            Column('label', str),
            Column('comment', str),
            Column('creation', str)
        ]
        # Load columns from PREFIX_Classes table
        columns = list(BASIC_COLUMNS)
        db = SqliteDb()
        db._createConnection(self._dbName, 1000)
        db.executeCommand("SELECT * FROM %sClasses;" % tableName)
        # This will store the images columsn to join
        # the _index and the _filename
        imgCols = {}
        for row in db._iterResults():
            renderType = COL_RENDER_NONE
            colName = row['column_name']
            colLabel = row['label_property']

            if colLabel != 'self':
                # Keep track of _index and _filename pairs to mark as renderable images
                if colLabel.endswith('_index'):
                    imgCols[colLabel.replace('_index', '')] = colName

                elif colLabel.endswith('_filename'):

                    # TODO: Maybe not all the labels endswith "_filename"
                    # have to be rendered.
                    # for example in the RotSpectra with '_representative._filename'

                    prefix = colLabel.replace('_filename', '')
                    if prefix in imgCols:
                        renderType = COL_RENDER_IMAGE
                        imgCols[colName] = imgCols[prefix]

                #CTF FIX
                elif (colLabel.endswith('_psdFile')
                      or colLabel.endswith('_enhanced_psd')
                      or colLabel.endswith('_ctfmodel_quadrant')
                      or colLabel.endswith('_ctfmodel_halfplane')):

                    renderType = COL_RENDER_IMAGE

                if row['class_name'] == 'Boolean':
                    renderType = COL_RENDER_CHECKBOX
                columns.append(
                    Column(colName, str, label=colLabel,
                           renderType=renderType))
        table = Table(*columns)

        checkedImgCols = {}  # Check if the image columns are volumes
        ih = ImageHandler()

        # Populate the table in the DataSet
        db.executeCommand("SELECT * FROM %sObjects;" % tableName)
        for row in db._iterResults():
            rowDict = dict(row)
            for k, v in rowDict.iteritems():
                if v is None:
                    rowDict[k] = ''
                # Set the index@filename for images columns values
                if k in imgCols:
                    colName = imgCols[k]
                    index = rowDict[colName]

                    filename = os.path.join(self.projectPath, rowDict[k])
                    filepath = filename.replace(":mrc", "")
                    if not checkedImgCols.get(colName, False):
                        if os.path.exists(filepath):
                            #print "Fn to get dims: %s@%s" % (index,filename)
                            x, y, z, n = ih.getDimensions((index, filename))
                            if z > 1:
                                table.getColumn(k).setRenderType(
                                    COL_RENDER_VOLUME)
                        checkedImgCols[colName] = True
                    if index:
                        rowDict[k] = '%06d@%s' % (index, filename)
            table.addRow(row['id'], **rowDict)

        return table
예제 #25
0
    def _processMovie(self, movieId, movieName, movieFolder,
                      shifts):  ###pasar shifts

        movieName = os.path.join(movieFolder, movieName)

        boxSize = self.boxSize.get()
        # Read movie dimensions to iterate through each frame
        imgh = ImageHandler()
        x, y, z, n = imgh.getDimensions(movieName)

        first = self.firstFrame.get()
        if first <= 1:
            first = 1
        last = self.lastFrame.get()
        if last <= 0 or last >= n:
            last = n
        numberOfFrames = last - first + 1

        if shifts is None:
            frames = max(z, n)
            shifts = [0] * (2 * frames)

        stkIndex = 0
        movieStk = self._getMovieName(movieId, '.stk')
        movieMdFile = self._getMovieName(movieId, '.xmd')
        movieMd = md.MetaData()
        frameMd = md.MetaData()
        frameMdImages = md.MetaData()
        frameRow = md.Row()

        for frame in range(first, last + 1):
            # Get the frame shifts
            index = frame - first
            shiftX = shifts[2 * index]
            shiftY = shifts[2 * index + 1]

            frameRoot = os.path.join(movieFolder, 'frame_%02d' % frame)
            frameName = frameRoot + '.mrc'
            frameMdFile = frameRoot + '.xmd'
            framePosFile = frameRoot + '_coordinates.xmd'
            coordinatesName = frameRoot + '_coordinates.xmd'

            hasCoordinates = self._writeXmippPosFile(movieId, movieName,
                                                     coordinatesName, shiftX,
                                                     shiftY)
            if hasCoordinates:
                self.info("Writing frame: %s" % frameName)
                #TODO: there is no need to write the frame and then operate
                #the input of the first operation should be the movie
                imgh.convert(tuple([frame, movieName]), frameName)

                if self.doRemoveDust:
                    self.info("Removing Dust")
                    self._runNoDust(frameName)

                self.info("Extracting particles")
                frameImages = frameRoot + '_images'
                args = '-i %(frameName)s --pos %(coordinatesName)s ' \
                       '-o %(frameRoot)s --Xdim %(boxSize)d' % locals()

                if self.doInvert:
                    args += " --invert"

                args += " --downsampling %f " % self.factor
                self.runJob('xmipp_micrograph_scissor', args)
                cleanPath(frameName)

                frameStk = frameRoot + '.stk'

                self.info("Combining particles into one stack.")

                frameMdImages.read(frameMdFile)
                frameMd.read('particles@%s' % framePosFile)
                frameMd.merge(frameMdImages)

                for objId in frameMd:
                    stkIndex += 1
                    frameRow.readFromMd(frameMd, objId)
                    location = xmippToLocation(frameRow.getValue(md.MDL_IMAGE))
                    newLocation = (stkIndex, movieStk)
                    imgh.convert(location, newLocation)

                    # Fix the name to be accesible from the Project directory
                    # so we know that the movie stack file will be moved
                    # to final particles folder
                    newImageName = '%d@%s' % newLocation
                    frameRow.setValue(md.MDL_IMAGE, newImageName)
                    frameRow.setValue(md.MDL_MICROGRAPH_ID, long(movieId))
                    frameRow.setValue(md.MDL_MICROGRAPH, str(movieId))
                    frameRow.setValue(md.MDL_FRAME_ID, long(frame))
                    frameRow.setValue(md.MDL_PARTICLE_ID,
                                      frameRow.getValue(md.MDL_ITEM_ID))
                    frameRow.writeToMd(movieMd, movieMd.addObject())
                movieMd.addItemId()
                movieMd.write(movieMdFile)
                cleanPath(frameStk)

        if self.doNormalize:
            self._runNormalize(movieStk, numberOfFrames)
예제 #26
0
파일: dataset.py 프로젝트: coocoky/scipion
    def _loadTable(self, tableName):
        """ Load information from tables PREFIX_Classes, PREFIX_Objects. """
        
        tableName = self.tablePrefixes[tableName]
        
        BASIC_COLUMNS = [Column('id', int, renderType=COL_RENDER_ID), 
                         Column('enabled', bool ,renderType=COL_RENDER_CHECKBOX),
                         Column('label', str), 
                         Column('comment', str),
                         Column('creation', str)]
        # Load columns from PREFIX_Classes table
        columns = list(BASIC_COLUMNS)
        db = SqliteDb()
        db._createConnection(self._dbName, 1000)
        db.executeCommand("SELECT * FROM %sClasses;" % tableName)
        # This will store the images columsn to join
        # the _index and the _filename
        imgCols = {}
        for row in db._iterResults():
            renderType = COL_RENDER_NONE
            colName = row['column_name']
            colLabel = row['label_property']
            
            if colLabel != 'self':
                # Keep track of _index and _filename pairs to mark as renderable images
                if colLabel.endswith('_index'):
                    imgCols[colLabel.replace('_index', '')] = colName
                
                elif colLabel.endswith('_filename'):
                    
                    # TODO: Maybe not all the labels endswith "_filename" 
                    # have to be rendered. 
                    # for example in the RotSpectra with '_representative._filename'

                    prefix = colLabel.replace('_filename', '')
                    if prefix in imgCols:
                        renderType = COL_RENDER_IMAGE
                        imgCols[colName] = imgCols[prefix]
                
                #CTF FIX
                elif (colLabel.endswith('_psdFile') or 
                      colLabel.endswith('_enhanced_psd') or 
                      colLabel.endswith('_ctfmodel_quadrant') or 
                      colLabel.endswith('_ctfmodel_halfplane')):
                    
                    renderType = COL_RENDER_IMAGE
                
                if row['class_name'] == 'Boolean':
                    renderType = COL_RENDER_CHECKBOX   
                columns.append(Column(colName, str, label=colLabel, renderType=renderType))
        table = Table(*columns)
        
        checkedImgCols = {} # Check if the image columns are volumes
        ih = ImageHandler() 
        
        # Populate the table in the DataSet
        db.executeCommand("SELECT * FROM %sObjects;" % tableName)
        for row in db._iterResults():
            rowDict = dict(row)
            for k, v in rowDict.iteritems():
                if v is None:
                    rowDict[k] = ''
                # Set the index@filename for images columns values
                if k in imgCols:
                    colName = imgCols[k]
                    index = rowDict[colName]
                    
                    filename = os.path.join(self.projectPath, rowDict[k])
                    filepath = filename.replace(":mrc", "")
                    if not checkedImgCols.get(colName, False):
                        if os.path.exists(filepath):
                            #print "Fn to get dims: %s@%s" % (index,filename)
                            x, y, z, n = ih.getDimensions((index, filename))
                            if z > 1:
                                table.getColumn(k).setRenderType(COL_RENDER_VOLUME)
                        checkedImgCols[colName] = True
                    if index:
                        rowDict[k] = '%06d@%s' % (index, filename)
            table.addRow(row['id'], **rowDict)
            
        return table
예제 #27
0
        def __applyTransform(suffix, pdbFileName, shift, angles, sampling):
            """ auxiliary function, transform PDB and 3dmap files"""
            # create a Scipion transformation matrix
            from numpy import deg2rad
            rotation_matrix = euler_matrix(deg2rad(angles[0]),
                                           deg2rad(angles[1]),
                                           deg2rad(angles[2]), 'szyz')
            translation = translation_matrix(shift)
            M = concatenate_matrices(rotation_matrix, translation)

            # apply it to the pdb file
            # if rotation move to center
            aSH = AtomicStructHandler(pdbFileName)
            if (angles[0] != 0. or angles[1] != 0. or angles[2] != 0.):
                from pyworkflow.em.convert import ImageHandler
                ih = ImageHandler()
                x, y, z, n = ih.getDimensions("emd_%s.map" % EMDBID)
                x /= 2.
                y /= 2.
                z /= 2.
                localShift = [-x, -y, -z]
                rotation_matrix = euler_matrix(0., 0., 0., 'szyz')
                translation = translation_matrix(localShift)
                localM = concatenate_matrices(rotation_matrix, translation)
                aSH.transform(localM, sampling=sampling)

            aSH.transform(M, sampling=sampling)

            if (angles[0] != 0. or angles[1] != 0. or angles[2] != 0.):
                localShift = [x, y, z]
                rotation_matrix = euler_matrix(0., 0., 0., 'szyz')
                translation = translation_matrix(localShift)
                localM = concatenate_matrices(rotation_matrix, translation)
                aSH.transform(localM, sampling=sampling)

            aSH.write("%s_%s_transformed.ent" % (suffix, PDBID.lower()))

            # get equivalent xmipp transformation
            shift, angles = __getXmippEulerAngles(M)
            # shift 3map and set sampling
            __runXmippProgram("xmipp_transform_geometry",
                              '-i emd_%s.map '
                              '-o %s_emd_%s_transform.map '
                              '--interp linear '
                              '--shift %f %f %f '
                              '--rotate_volume euler %f %f %f ' % (
                                  EMDBID,
                                  suffix,
                                  EMDBID,
                                  shift[0], shift[1], shift[2],
                                  angles[0], angles[1], angles[2]
                              )
                              )
            header = Ccp4Header("%s_emd_%s_transform.map" % (suffix, EMDBID),
                                readHeader=True)
            header.setSampling(sampling)
            # put the sampling back, xmipp_transform_geometry erased it
            header.writeHeader()

            # view the results with chimera
            from pyworkflow.em.viewers import Chimera
            args = "%s %s %s %s" % (
                   pdbFileName,
                   "emd_%s.map" % EMDBID,
                   "%s_%s_transformed.ent" % (suffix, PDBID.lower()),
                   "%s_emd_%s_transform.map" % (suffix, EMDBID)
            )
            Chimera.runProgram(args)
예제 #28
0
    def importImagesStreamStep(self, pattern, voltage, sphericalAberration,
                               amplitudeContrast, magnification):
        """ Copy images matching the filename pattern
        Register other parameters.
        """
        self.info("Using pattern: '%s'" % pattern)

        imgSet = self._getOutputSet() if self.isContinued() else None

        self.importedFiles = set()
        if imgSet is None:
            createSetFunc = getattr(self, '_create' + self._outputClassName)
            imgSet = createSetFunc()
        elif imgSet.getSize() > 0:  # in case of continue
            imgSet.loadAllProperties()
            self._fillImportedFiles(imgSet)
            imgSet.enableAppend()

        pointerExcludedMovs = getattr(self, 'moviesToExclude', None)
        if pointerExcludedMovs is not None:
            excludedMovs = pointerExcludedMovs.get()
            self._fillImportedFiles(excludedMovs)

        imgSet.setIsPhaseFlipped(self.haveDataBeenPhaseFlipped.get())
        acquisition = imgSet.getAcquisition()
        self.fillAcquisition(acquisition)
        # Call a function that should be implemented by each subclass
        self.setSamplingRate(imgSet)
        outFiles = [imgSet.getFileName()]
        imgh = ImageHandler()
        img = imgSet.ITEM_TYPE()
        img.setAcquisition(acquisition)
        n = 1
        copyOrLink = self.getCopyOrLink()
        outputName = self._getOutputName()

        finished = False
        # this is only used when creating stacks from frame files
        self.createdStacks = set()

        i = 0
        lastDetectedChange = datetime.now()

        # Ignore the timeout variables if we are not really in streaming mode
        if self.dataStreaming:
            timeout = timedelta(seconds=self.timeout.get())
            fileTimeout = timedelta(seconds=self.fileTimeout.get())
        else:
            timeout = timedelta(seconds=5)
            fileTimeout = timedelta(seconds=5)

        while not finished:
            time.sleep(3)  # wait 3 seconds before check for new files
            someNew = False
            someAdded = False

            for fileName, uniqueFn, fileId in self.iterNewInputFiles():
                someNew = True
                if self.fileModified(fileName, fileTimeout):
                    continue

                dst = self._getExtraPath(uniqueFn)
                self.importedFiles.add(uniqueFn)
                copyOrLink(fileName, dst)

                self.debug('Importing file: %s' % fileName)
                self.debug("uniqueFn: %s" % uniqueFn)
                self.debug("dst Fn: %s" % dst)

                if self._checkStacks:
                    _, _, _, n = imgh.getDimensions(dst)

                someAdded = True
                self.debug('Appending file to DB...')
                if self.importedFiles:  # enable append after first append
                    imgSet.enableAppend()

                if n > 1:
                    for index in range(1, n + 1):
                        img.cleanObjId()
                        img.setMicId(fileId)
                        img.setFileName(dst)
                        img.setIndex(index)
                        self._addImageToSet(img, imgSet)
                else:
                    img.setObjId(fileId)
                    img.setFileName(dst)
                    # Fill the micName if img is either a Micrograph or a Movie
                    self.debug("FILENAME TO fillMicName: %s" % uniqueFn)
                    self._fillMicName(img, uniqueFn)
                    self._addImageToSet(img, imgSet)

                outFiles.append(dst)
                self.debug('After append. Files: %d' % len(outFiles))

            if someAdded:
                self.debug('Updating output...')
                self._updateOutputSet(outputName,
                                      imgSet,
                                      state=imgSet.STREAM_OPEN)
                self.debug('Update Done.')

            self.debug('Checking if finished...someNew: %s' % someNew)

            now = datetime.now()

            if not someNew:
                # If there are no new detected files, we should check the
                # inactivity time elapsed (from last event to now) and
                # if it is greater than the defined timeout, we conclude
                # the import and close the output set
                # Another option is to check if the protocol have some
                # special stop condition, this can be used to manually stop
                # some protocols such as import movies
                finished = (now - lastDetectedChange > timeout
                            or self.streamingHasFinished())
                self.debug("Checking if finished:")
                self.debug("   Now - Last Change: %s" %
                           pwutils.prettyDelta(now - lastDetectedChange))
                self.debug("Finished: %s" % finished)
            else:
                # If we have detected some files, we should update
                # the timestamp of the last event
                lastDetectedChange = now

        self._updateOutputSet(outputName, imgSet, state=imgSet.STREAM_CLOSED)

        self._cleanUp()

        return outFiles
    def _processMovie(self, movieId, movieName, movieFolder, shifts):###pasar shifts
        
        movieName = os.path.join(movieFolder, movieName)

        boxSize = self.boxSize.get()
        # Read movie dimensions to iterate through each frame
        imgh = ImageHandler()
        x, y, z, n = imgh.getDimensions(movieName)
        
        first = self.firstFrame.get()
        if first <= 1:
            first = 1
        last = self.lastFrame.get()
        if last <= 0 or last >= n:
            last = n
        numberOfFrames = last - first + 1
        
        stkIndex = 0
        movieStk = self._getMovieName(movieId, '.stk')
        movieMdFile = self._getMovieName(movieId, '.xmd')
        movieMd = md.MetaData()
        frameMd = md.MetaData()
        frameMdImages = md.MetaData()
        frameRow = md.Row()
         
        for frame in range(first, last+1):
            # Get the frame shifts
            index = frame - first
            shiftX = shifts[2*index]
            shiftY = shifts[2*index+1]
            
            frameRoot = os.path.join(movieFolder, 'frame_%02d' % frame)
            frameName = frameRoot + '.mrc'
            frameMdFile = frameRoot + '.xmd'
            framePosFile = frameRoot + '_coordinates.xmd'
            coordinatesName = frameRoot + '_coordinates.xmd'
            
            hasCoordinates = self._writeXmippPosFile(movieId, movieName, coordinatesName, 
                                                     shiftX, shiftY)
            if hasCoordinates:
                self.info("Writing frame: %s" % frameName)
                #TODO: there is no need to write the frame and then operate
                #the input of the first operation should be the movie
                imgh.convert(tuple([frame, movieName]), frameName)
                
                if self.doRemoveDust:
                    self.info("Removing Dust")
                    self._runNoDust(frameName)

                self.info("Extracting particles")
                frameImages = frameRoot + '_images'
                args = '-i %(frameName)s --pos %(coordinatesName)s ' \
                       '-o %(frameRoot)s --Xdim %(boxSize)d' % locals()
                
                if self.doInvert:
                    args += " --invert"

                args += " --downsampling %f " % self.factor
                self.runJob('xmipp_micrograph_scissor', args)
                cleanPath(frameName)
                
                frameStk = frameRoot + '.stk'
                
                self.info("Combining particles into one stack.")
                 
                frameMdImages.read(frameMdFile)
                frameMd.read('particles@%s' % framePosFile)
                frameMd.merge(frameMdImages)
                 
                for objId in frameMd:
                    stkIndex += 1
                    frameRow.readFromMd(frameMd, objId)
                    location = xmippToLocation(frameRow.getValue(md.MDL_IMAGE))
                    newLocation = (stkIndex, movieStk)
                    imgh.convert(location, newLocation)
                    
                    # Fix the name to be accesible from the Project directory
                    # so we know that the movie stack file will be moved
                    # to final particles folder
                    newImageName = '%d@%s' % newLocation
                    frameRow.setValue(md.MDL_IMAGE, newImageName)
                    frameRow.setValue(md.MDL_MICROGRAPH_ID, long(movieId))
                    frameRow.setValue(md.MDL_MICROGRAPH, str(movieId))
                    frameRow.setValue(md.MDL_FRAME_ID, long(frame))
                    frameRow.setValue(md.MDL_PARTICLE_ID, frameRow.getValue(md.MDL_ITEM_ID))
                    frameRow.writeToMd(movieMd, movieMd.addObject())
                movieMd.addItemId()
                movieMd.write(movieMdFile)
                cleanPath(frameStk)
        
        if self.doNormalize:
            self._runNormalize(movieStk, numberOfFrames)