예제 #1
0
 def doReport(self):
     secs = time.time() - self.startTimeAll
     sys.stderr.write(
         "\n%s run %d tests (%0.3f secs)\n" %
         (pwutils.greenStr("[==========]"), self.numberTests, secs))
     if self.testFailed:
         sys.stderr.write("%s %d tests\n" %
                          (pwutils.redStr("[  FAILED  ]"), self.testFailed))
     sys.stdout.write("%s %d tests\n" %
                      (pwutils.greenStr("[  PASSED  ]"),
                       self.numberTests - self.testFailed))
예제 #2
0
    def testMerge(self):
        """Test that the union operation works as expected."""

        print "\n", greenStr(" Test Merge ".center(75, '-'))

        def check(set0):
            "Simple checks on merge, coming from many split sets of set0."
            print magentaStr("\n==> Check merge of %s" % type(set0).__name__)
            p_union = self.proj.newProtocol(ProtUnionSet)

            setsIds = []
            for i in range(random.randint(1, 5)):
                n = random.randint(1, len(set0) // 2)
                p_split = self.split(set0, n=n, randomize=True)
                setRandom = random.choice(list(self.outputs(p_split)))
                setsIds.append([x.strId() for x in setRandom])
                p_union.inputSets.append(setRandom)
            self.proj.launchProtocol(p_union, wait=True)

            output = self.outputs(p_union).next()  # first (and only!) output
            self.assertEqual(len(output), sum(len(x) for x in setsIds))
            # We might be able to do more interesting tests, using the
            # collected setsIds.

        check(self.micros)
        check(self.vols)
        check(self.movies)
        check(self.particles)
예제 #3
0
    def testMerge(self):
        """Test that the union operation works as expected."""

        print("\n", pwutils.greenStr(" Test Merge ".center(75, '-')))

        def check(set0):
            # Simple checks on merge, coming from many split sets of set0.
            print(
                pwutils.magentaStr("\n==> Check merge of %s" %
                                   type(set0).__name__))
            p_union = self.proj.newProtocol(emprot.ProtUnionSet)

            setsIds = []
            for i in range(random.randint(1, 5)):
                n = random.randint(1, len(set0) // 2)
                p_split = self.split(set0, n=n, randomize=True)
                setRandom = random.choice(list(self.outputs(p_split)))
                setsIds.append([x.strId() for x in setRandom])
                p_union.inputSets.append(setRandom)
            self.proj.launchProtocol(p_union, wait=True)

            output = next(self.outputs(p_union))  # first (and only!) output
            self.assertEqual(len(output), sum(len(x) for x in setsIds))
            # We might be able to do more interesting tests, using the
            # collected setsIds.

        check(self.micros)
        check(self.vols)
        check(self.movies)
        check(self.particles)
예제 #4
0
    def testSplit(self):
        """Test that the split operation works as expected."""

        print("\n", pwutils.greenStr(" Test Split ".center(75, '-')))

        def check(set0, n=2, randomize=False):
            # Simple checks on split sets from set0.
            print(
                pwutils.magentaStr("\n==> Check split of %s" %
                                   type(set0).__name__))
            unsplit_set = [x.strId() for x in set0]
            p_split = self.split(set0, n=n, randomize=randomize)
            # Are all output elements of the protocol in the original set?
            for em_set in self.outputs(p_split):
                for elem in em_set:
                    self.assertTrue(elem.strId() in unsplit_set)
            # Number of elements of all split sets equal to original number?
            self.assertEqual(sum(len(x) for x in self.outputs(p_split)),
                             len(set0))

        check(self.micros)
        check(self.micros, randomize=True)
        check(self.vols)
        check(self.movies)
        check(self.particles)
        check(self.particles, n=4)
예제 #5
0
def _submit(hostConfig, submitDict, cwd=None, env=None):
    """ Submit a protocol to a queue system. Return its job id.
    """
    # Create first the submission script to be launched
    # formatting using the template
    template = hostConfig.getSubmitTemplate() % submitDict
    # FIXME: CREATE THE PATH FIRST
    scripPath = submitDict['JOB_SCRIPT']
    f = open(scripPath, 'w')
    # Ensure the path exists
    makeFilePath(scripPath)
    # Add some line ends because in some clusters it fails
    # to submit jobs if the submit script does not have end of line
    f.write(template + '\n\n')
    f.close()
    # This should format the command using a template like:
    # "qsub %(JOB_SCRIPT)s"
    command = hostConfig.getSubmitCommand() % submitDict
    gcmd = greenStr(command)
    print("** Submitting to queue: '%s'" % gcmd)

    p = Popen(command, shell=True, stdout=PIPE, cwd=cwd, env=env)
    out = p.communicate()[0]
    # Try to parse the result of qsub, searching for a number (jobId)
    # Review this, seems to exclusive to torque batch system
    s = re.search('(\d+)', str(out))
    if p.returncode == 0 and s:
        job = int(s.group(0))
        print("Launched job with id %s" % job)
        return job
    else:
        print("Couldn't submit to queue for reason: %s " %
              redStr(out.decode()))
        return UNKNOWN_JOBID
예제 #6
0
def _runRemote(protocol, mode):
    """ Launch remotely 'pw_protocol_remote.py' script to run or stop a protocol. 
    Params:
        protocol: the protocol to be ran or stopped.
        mode: should be either 'run' or 'stop'
    """
    host = protocol.getHostConfig()
    tpl = "ssh %(address)s '%(scipion)s/scipion "

    if host.getScipionConfig() is not None:
        tpl += "--config %(config)s "

    tpl += "runprotocol pw_protocol_remote.py %(mode)s "
    tpl += "%(project)s %(protDb)s %(protId)s' "

    # Use project base name,
    # in remote SCIPION_USER_DATA/projects should be prepended
    projectPath = os.path.basename(protocol.getProject().path)

    args = {'address': host.getAddress(),
            'mode': mode,
            'scipion': host.getScipionHome(),
            'config': host.getScipionConfig(),
            'project': projectPath,
            'protDb': protocol.getDbPath(),
            'protId': protocol.getObjId()
            }
    cmd = tpl % args
    print "** Running remote: %s" % greenStr(cmd)
    p = Popen(cmd, shell=True, stdout=PIPE)

    return p
예제 #7
0
def _submit(hostConfig, submitDict):
    """ Submit a protocol to a queue system. Return its job id.
    """
    # Create forst the submission script to be launched
    # formatting using the template
    template = hostConfig.getSubmitTemplate() % submitDict
    #FIXME: CREATE THE PATH FIRST
    scripPath = submitDict['JOB_SCRIPT']
    f = open(scripPath, 'w')
    #Ensure the path exists
    makeFilePath(scripPath)
    # Add some line ends because in some clusters it fails
    # to submit jobs if the submit script does not have end of line
    f.write(template+'\n\n')
    f.close()
    # This should format the command using a template like: 
    # "qsub %(JOB_SCRIPT)s"
    command = hostConfig.getSubmitCommand() % submitDict
    gcmd = greenStr(command)
    print "** Submiting to queue: '%s'" % gcmd
    p = Popen(command, shell=True, stdout=PIPE)
    out = p.communicate()[0]
    # Try to parse the result of qsub, searching for a number (jobId)
    s = re.search('(\d+)', out)
    if s:
        return int(s.group(0))
    else:
        print "** Couldn't parse %s ouput: %s" % (gcmd, redStr(out)) 
        return UNKNOWN_JOBID
    def test_filterParticles(self):
        print "\n", greenStr(" Filter Particles ".center(75, '-'))

        def test(parts=self.protImport.outputParticles, **kwargs):
            "Launch XmippProtFilterParticles on parts and check results."
            print magentaStr("\n==> Input params: %s" % kwargs)
            prot = self.newProtocol(XmippProtFilterParticles, **kwargs)
            prot.inputParticles.set(parts)
            self.launchProtocol(prot)
            self.assertIsNotNone(prot.outputParticles,
                                 "There was a problem with filter particles")
            self.assertTrue(prot.outputParticles.equalAttributes(
                parts, ignore=['_mapperPath'], verbose=True))
            # Compare the individual particles too.
            self.assertTrue(prot.outputParticles.equalItemAttributes(
                parts, ignore=['_filename', '_index'], verbose=True))

        # Check a few different cases.
        test(filterSpace=FILTER_SPACE_FOURIER, lowFreq=0.1, highFreq=0.25)
        test(filterSpace=FILTER_SPACE_REAL, filterModeReal=xfh.FM_MEDIAN)
        # For wavelets, we need the input's size to be a power of 2
        print magentaStr("\n==> Resizing particles to 256 pixels")
        protResize = self.newProtocol(XmippProtCropResizeParticles,
                                      doResize=True,
                                      resizeOption=xrh.RESIZE_DIMENSIONS,
                                      resizeDim=256)
        protResize.inputParticles.set(self.protImport.outputParticles)
        self.launchProtocol(protResize)
        test(parts=protResize.outputParticles, filterSpace=FILTER_SPACE_WAVELET,
             filterModeWavelets=xfh.FM_DAUB12, waveletMode=xfh.FM_REMOVE_SCALE)
 def setUpClass(cls):
     print "\n", greenStr(
         " Crop/Resize Volumes Set Up - Collect data ".center(75, '-'))
     setupTestProject(cls)
     TestXmippBase.setData()
     cls.protImport1 = cls.runImportVolumes(cls.volumes, 9.896)
     cls.protImport2 = cls.runImportVolumes(cls.vol1, 9.896)
예제 #10
0
def _run(command, wait, stdin=None, stdout=None, stderr=None):
    """ Execute a command in a subprocess and return the pid. """
    gcmd = greenStr(command)
    print "** Running command: '%s'" % gcmd
    p = Popen(command, shell=True, stdout=stdout, stderr=stderr)
    jobId = p.pid
    if wait:
        p.wait()

    return jobId
예제 #11
0
    def setUpClass(cls):
        """Prepare the data that we will use later on."""

        print "\n", greenStr(" Set Up - Collect data ".center(75, '-'))

        setupTestProject(cls)  # defined in BaseTest, creates cls.proj

        cls.dataset_xmipp = DataSet.getDataSet('xmipp_tutorial')
        cls.dataset_mda = DataSet.getDataSet('mda')
        cls.dataset_ribo = DataSet.getDataSet('ribo_movies')

        #
        # Imports
        #
        new = cls.proj.newProtocol  # short notation
        launch = cls.proj.launchProtocol
        # Micrographs
        print magentaStr("\n==> Importing data - micrographs")
        p_imp_micros = new(ProtImportMicrographs,
                           filesPath=cls.dataset_xmipp.getFile('allMics'),
                           samplingRate=1.237,
                           voltage=300)
        launch(p_imp_micros, wait=True)
        cls.micros = p_imp_micros.outputMicrographs

        # Volumes
        print magentaStr("\n==> Importing data - volumes")
        p_imp_volumes = new(ProtImportVolumes,
                            filesPath=cls.dataset_xmipp.getFile('volumes'),
                            samplingRate=9.896)
        launch(p_imp_volumes, wait=True)
        cls.vols = p_imp_volumes.outputVolumes

        # Movies
        print magentaStr("\n==> Importing data - movies")
        p_imp_movies = new(ProtImportMovies,
                           filesPath=cls.dataset_ribo.getFile('movies'),
                           samplingRate=2.37,
                           magnification=59000,
                           voltage=300,
                           sphericalAberration=2.0)
        launch(p_imp_movies, wait=True)
        cls.movies = p_imp_movies.outputMovies

        # Particles
        print magentaStr("\n==> Importing data - particles")
        p_imp_particles = new(ProtImportParticles,
                              filesPath=cls.dataset_mda.getFile('particles'),
                              samplingRate=3.5)
        launch(p_imp_particles, wait=True)
        cls.particles = p_imp_particles.outputParticles
예제 #12
0
    def testAlignAndReconstruct(self):
        print("\n", greenStr(" Test Ts alignment and tomogram reconstruction with Jjsof".center(75, '-')))

        # preparing and launching the protocol
        palireco = self.newProtocol(ProtJjsoftAlignReconstructTomogram,
                                    inputSetOfTiltSeries=self.setOfFiducialTs,
                                    inputSetOfLandmarkModels=self.setOfFiducials)
        self.launchProtocol(palireco, wait=True)
        setOfTomograms = palireco.outputTomograms

        # some general assertions
        self.assertIsNotNone(setOfTomograms,
                             "There was some problem with the output")
        self.assertEqual(setOfTomograms.getSize(), self.setOfFiducialTs.getSize(),
                         "The number of tomograms is wrong")
예제 #13
0
    def testReconstructionWBP(self):
        print ("\n", greenStr(" Test tomo3D reconstruction with WBP".center(75, '-')))

        # preparing and launching the protocol
        ptomo3D = self.newProtocol(ProtJjsoftReconstructTomogram,
                                   inputSetOfTiltSeries=self.setOfTs,
                                   method=0)
        self.launchProtocol(ptomo3D, wait=True)
        setOfReconstructedTomograms = ptomo3D.outputTomograms

        # some general assertions           
        self.assertIsNotNone(setOfReconstructedTomograms,           
                             "There was some problem with the output")
        self.assertEqual(setOfReconstructedTomograms.getSize(), self.setOfTs.getSize(),
                         "The number of the denoised tomograms is wrong")
예제 #14
0
    def testWarpAlign(self):
        print ("\n", greenStr(" Test Ts alignment with warpalign ".center(75, '-')))

        # preparing and launching the protocol
        pwarpalign = self.newProtocol(ProtJjsoftAlignTs,
                                      inputSetOfTiltSeries=self.setOfFiducialTs,
                                      inputSetOfLandmarkModels=self.setOfFiducials)
        self.launchProtocol(pwarpalign, wait=True)
        setOfAlignedTs = pwarpalign.outputInterpolatedSetOfTiltSeries

        # some general assertions
        self.assertIsNotNone(setOfAlignedTs,
                             "There was some problem with the output")
        self.assertEqual(setOfAlignedTs.getSize(), self.setOfFiducialTs.getSize(),
                         "The number of the aligned Ts is wrong")
예제 #15
0
    def setUpClass(cls):
        """Prepare the data that we will use later on."""

        print "\n", greenStr(" Set Up - Collect data ".center(75, '-'))

        setupTestProject(cls)  # defined in BaseTest, creates cls.proj

        cls.dataset_xmipp = DataSet.getDataSet('xmipp_tutorial')
        cls.dataset_mda = DataSet.getDataSet('mda')
        cls.dataset_ribo = DataSet.getDataSet('ribo_movies')

        #
        # Imports
        #
        new = cls.proj.newProtocol  # short notation
        launch = cls.proj.launchProtocol
        # Micrographs
        print magentaStr("\n==> Importing data - micrographs")
        p_imp_micros = new(ProtImportMicrographs,
                           filesPath=cls.dataset_xmipp.getFile('allMics'),
                           samplingRate=1.237, voltage=300)
        launch(p_imp_micros, wait=True)
        cls.micros = p_imp_micros.outputMicrographs

        # Volumes
        print magentaStr("\n==> Importing data - volumes")
        p_imp_volumes = new(ProtImportVolumes,
                            filesPath=cls.dataset_xmipp.getFile('volumes'),
                            samplingRate=9.896)
        launch(p_imp_volumes, wait=True)
        cls.vols = p_imp_volumes.outputVolumes

        # Movies
        print magentaStr("\n==> Importing data - movies")
        p_imp_movies = new(ProtImportMovies,
                           filesPath=cls.dataset_ribo.getFile('movies'),
                           samplingRate=2.37, magnification=59000,
                           voltage=300, sphericalAberration=2.0)
        launch(p_imp_movies, wait=True)
        cls.movies = p_imp_movies.outputMovies

        # Particles
        print magentaStr("\n==> Importing data - particles")
        p_imp_particles = new(ProtImportParticles,
                              filesPath=cls.dataset_mda.getFile('particles'),
                              samplingRate=3.5)
        launch(p_imp_particles, wait=True)
        cls.particles = p_imp_particles.outputParticles
    def testConsensus6(self):
        print("\n", greenStr(" Test Consensus with four sets".center(75, '-')))

        # preparing and launching the protocol
        pConsClass = self.proj.newProtocol(
            XmippProtConsensusClasses3D,
            inputMultiClasses=[self.set1, self.set2, self.set5, self.set6])
        self.proj.launchProtocol(pConsClass, wait=True)
        setOfIntersections = pConsClass.outputClasses

        # some general assertions
        self.assertIsNotNone(setOfIntersections,
                             "There was some problem with the output")
        self.assertEqual(setOfIntersections.getSize(), 225,
                         "The number of the outputClasses is wrong")
        self.checkPopulation(setOfIntersections, 75)
예제 #17
0
    def testSubsetByMic(self):
        """Test that the subset by Mic operation works as expected."""
        print("\n", pwutils.greenStr(" Test Subset by Mic".center(75, '-')))
        "Simple checks on subsets, coming from split sets of setMics."
        print(
            pwutils.magentaStr("\n==> Check subset of %s by %s" % (type(
                self.partMicId).__name__, type(self.micsMicId).__name__)))

        # launch the protocol for a certain mics input
        def launchSubsetByMic(micsSubset):
            pSubsetbyMic = self.newProtocol(emprot.ProtSubSetByMic)
            pSubsetbyMic.inputParticles.set(self.partMicId)
            pSubsetbyMic.inputMicrographs.set(micsSubset)
            self.launchProtocol(pSubsetbyMic)
            return pSubsetbyMic.outputParticles

        # Check if the Output is generated, the subset size is correct and
        #  the micId of the particle with certain partId is correct.
        def checkAsserts(setParts, size, partId, micId):
            self.assertIsNotNone(setParts, "Output SetOfParticles"
                                 " were not created.")
            self.assertEqual(setParts.getSize(), size,
                             "The number of created particles is incorrect.")
            p = setParts[partId]
            self.assertEqual(p.getMicId(), micId)

        # Whole set of micrographs
        setMics = self.micsMicId
        # Create a subsets of Mics to apply the protocol
        pSplit = self.split(setMics, n=2, randomize=False)
        setMics2 = pSplit.outputMicrographs02
        # Create a subset of a single micrograph to apply the protocol
        pSplit = self.split(setMics, n=20, randomize=False)
        setMics3 = pSplit.outputMicrographs03

        # Launch subset by mics protocol with the whole set of Mics
        partByMic1 = launchSubsetByMic(setMics)
        # Launch subset by mics protocol with a subset of Mics
        partByMic2 = launchSubsetByMic(setMics2)
        # Launch subset by mics protocol with a single SetOfMics
        partByMic3 = launchSubsetByMic(setMics3)

        # Assertions for the three sets
        checkAsserts(partByMic1, self.partMicId.getSize(), 1885, 7)
        checkAsserts(partByMic2, 2638, 4330, 16)
        checkAsserts(partByMic3, 270, 725, 3)
예제 #18
0
    def testDenoisingBFlow(self):
        print ("\n", greenStr(" Test BFlow denoising ".center(75, '-')))

        # preparing and launching the protocol
        pDenoiseBFlow = self.newProtocol(ProtJjsoftProtDenoiseTomogram,
                                         inputSetTomograms=self.setOfTomograms,
                                         method=1,
                                         SigmaGaussian=0.5,
                                         nIter=1,
                                         TimeStep=0.1)
        self.launchProtocol(pDenoiseBFlow, wait=True)
        setOfBFlowDenoisedTomograms = pDenoiseBFlow.outputTomograms

        # some general assertions
        self.assertIsNotNone(setOfBFlowDenoisedTomograms,
                             "There was some problem with the output")
        self.assertEqual(setOfBFlowDenoisedTomograms.getSize(), self.setOfTomograms.getSize(),
                         "The number of the denoised tomograms is wrong")
예제 #19
0
    def testSubsetByCoord(self):
        """Test that the subset by Coord operation works as expected."""
        print("\n", pwutils.greenStr(" Test Subset by Coord".center(75, '-')))

        p_extract_coordinates = self.newProtocol(emprot.ProtExtractCoords)
        p_extract_coordinates.inputParticles.set(self.partMicId)
        p_extract_coordinates.inputMicrographs.set(self.micsMicId)
        self.launchProtocol(p_extract_coordinates)

        p_subset_by_coords = self.newProtocol(emprot.ProtSubSetByCoord)
        p_subset_by_coords.inputParticles.set(self.partMicId)
        p_subset_by_coords.inputCoordinates.set(
            p_extract_coordinates.outputCoordinates)
        self.launchProtocol(p_subset_by_coords)
        self.assertIsNotNone(p_subset_by_coords.outputParticles,
                             "Output SetOfParticles were not created.")
        self.assertEqual(p_subset_by_coords.outputParticles.getSize(), 5236,
                         "The number of created particles is incorrect.")
예제 #20
0
 def setParamValue(self, alias, newValue):
     paramsSetted = 0
     for field in self.params.values():
         if field.getAlias() == alias:
             oldValue = field.getValue()
             field.setValue(newValue)
             if field.validate():
                 paramsSetted += 1
                 print(
                     greenStr("%s set to %s") %
                     (field.getTitle(), str(newValue)))
             else:
                 field.setValue(oldValue)
                 raise Exception(
                     "%s is not compatible with %s(%s) parameter." %
                     (newValue, field.getTitle(), alias))
     if not paramsSetted:
         raise Exception("Alias %s not recognized." % alias)
     return paramsSetted
    def testConsensus5(self):
        print "\n", greenStr(" Test Consensus with two sets".center(75, '-'))

        # preparing and launching the protocol
        pConsClass = self.proj.newProtocol(
            XmippProtConsensusClasses3D,
            inputMultiClasses=[self.set5, self.set6])
        self.proj.launchProtocol(pConsClass, wait=True)
        setOfIntersections = pConsClass.outputClasses

        # some general assertions
        self.assertIsNotNone(setOfIntersections,
                             "There was some problem with the output")
        self.assertEqual(setOfIntersections.getSize(), 25,
                         "The number of the outputClasses is wrong")
        self.checkPopulation(setOfIntersections, 75)

        # some specific assetions
        self.checkIntersections(setOfIntersections, 1, [48, 74, 44, 53])
        self.checkIntersections(setOfIntersections, 7, [51, 68, 14])
        self.checkIntersections(setOfIntersections, 20, [36, 5])
예제 #22
0
    def getPackagesStatus(cls, printAll=True):
        """
        Check for scipion-app, scipion-pyworkflow and scipion-em updates
        return: a list of modules to be updated
        """
        outdatedPackages = []
        for package in cls.packageNames:

            needToUpdate, version = cls.getPackageState(package[0], package[1])
            if needToUpdate:
                outdatedPackages.append((package[0], version))
                print(
                    redStr(
                        'The package %s is out of date. Your version is %s, '
                        'the latest is %s.' %
                        (package[0], package[1], version)))
            elif printAll:
                print(
                    greenStr('The package %s is up to date.  Your version '
                             'is %s' % (package[0], version)))

        return outdatedPackages
    def testConsensus3(self):
        print "\n", greenStr(
            " Test Consensus with different set sizes 2".center(75, '-'))

        # preparing and launching the protocol
        pConsClass = self.proj.newProtocol(
            XmippProtConsensusClasses3D,
            inputMultiClasses=[self.set4, self.set2, self.set1])
        self.proj.launchProtocol(pConsClass, wait=True)
        setOfIntersections = pConsClass.outputClasses

        # some general assertions
        self.assertIsNotNone(setOfIntersections,
                             "There was some problem with the output")
        self.assertEqual(setOfIntersections.getSize(), 27,
                         "The number of the outputClasses is wrong")
        self.checkPopulation(setOfIntersections, 69)

        # some specific assetions
        self.checkIntersections(setOfIntersections, 1, [74, 53])
        self.checkIntersections(setOfIntersections, 7, [25, 43, 68, 54, 57])
        self.checkIntersections(setOfIntersections, 20, [46])
예제 #24
0
    def testSplit(self):
        """Test that the split operation works as expected."""

        print "\n", greenStr(" Test Split ".center(75, '-'))

        def check(set0, n=2, randomize=False):
            "Simple checks on split sets from set0."
            print magentaStr("\n==> Check split of %s" % type(set0).__name__)
            unsplit_set = [x.strId() for x in set0]
            p_split = self.split(set0, n=n, randomize=randomize)
            # Are all output elements of the protocol in the original set?
            for em_set in self.outputs(p_split):
                for elem in em_set:
                    self.assertTrue(elem.strId() in unsplit_set)
            # Number of elements of all splitted sets equal to original number?
            self.assertEqual(sum(len(x) for x in self.outputs(p_split)),
                             len(set0))

        check(self.micros)
        check(self.micros, randomize=True)
        check(self.vols)
        check(self.movies)
        check(self.particles)
        check(self.particles, n=4)
 def setUpClass(cls):
     print "\n", greenStr(" Crop/Resize Set Up - Collect data ".center(75, '-'))
     setupTestProject(cls)
     TestXmippBase.setData('relion_tutorial')
 def setUpClass(cls):
     print "\n", greenStr(" Set Up - Collect data ".center(75, '-'))
     setupTestProject(cls)
     TestXmippBase.setData('xmipp_tutorial')
     cls.protImport = cls.runImportParticles(cls.particlesFn, 1.237, True, True)
예제 #27
0
    def setUpClass(cls):
        """Prepare the data that we will use later on."""

        print("\n", pwutils.greenStr(" Set Up - Collect data ".center(75,
                                                                      '-')))

        pwtests.setupTestProject(cls)  # defined in BaseTest, creates cls.proj

        cls.dataset_xmipp = pwtests.DataSet.getDataSet('xmipp_tutorial')
        cls.dataset_mda = pwtests.DataSet.getDataSet('mda')
        cls.dataset_ribo = pwtests.DataSet.getDataSet('ribo_movies')
        cls.datasetRelion = pwtests.DataSet.getDataSet('relion_tutorial')

        #
        # Imports
        #
        new = cls.proj.newProtocol  # short notation
        launch = cls.proj.launchProtocol
        # Micrographs
        # NOTE: This dataset has 3 mic with heterogeneous dimensions!! But so
        # far is not failing, should it?
        print(pwutils.magentaStr("\n==> Importing data - micrographs"))
        p_imp_micros = new(emprot.ProtImportMicrographs,
                           filesPath=cls.dataset_xmipp.getFile('allMics'),
                           samplingRate=1.237,
                           voltage=300)
        launch(p_imp_micros, wait=True)
        cls.micros = p_imp_micros.outputMicrographs

        # Micrographs SMALL - This is a mic with different dimensions
        print(pwutils.magentaStr("\n==> Importing data - micrographs SMALL"))
        p_imp_micros = new(emprot.ProtImportMicrographs,
                           filesPath=cls.dataset_xmipp.getFile('mic3'),
                           samplingRate=1.237,
                           voltage=300)
        launch(p_imp_micros, wait=True)
        cls.microsSmall = p_imp_micros.outputMicrographs

        # Volumes
        print(pwutils.magentaStr("\n==> Importing data - volumes"))
        p_imp_volumes = new(emprot.ProtImportVolumes,
                            filesPath=cls.dataset_xmipp.getFile('volumes'),
                            samplingRate=9.896)
        launch(p_imp_volumes, wait=True)
        cls.vols = p_imp_volumes.outputVolumes

        # Movies
        print(pwutils.magentaStr("\n==> Importing data - movies"))
        p_imp_movies = new(emprot.ProtImportMovies,
                           filesPath=cls.dataset_ribo.getFile('movies'),
                           samplingRate=2.37,
                           magnification=59000,
                           voltage=300,
                           sphericalAberration=2.0,
                           gainFile=cls.dataset_ribo.getFile('volume'),
                           darkFile=cls.dataset_ribo.getFile('volume'))
        launch(p_imp_movies, wait=True)
        cls.movies = p_imp_movies.outputMovies

        # Particles
        print(pwutils.magentaStr("\n==> Importing data - particles"))
        p_imp_particles = new(emprot.ProtImportParticles,
                              filesPath=cls.dataset_mda.getFile('particles'),
                              samplingRate=3.5)
        launch(p_imp_particles, wait=True)
        cls.particles = p_imp_particles.outputParticles

        # Particles with micId
        print(
            pwutils.magentaStr("\n==> Importing data - particles with micId"))
        relionFile = 'import/case2/relion_it015_data.star'
        pImpPartMicId = new(
            emprot.ProtImportParticles,
            objLabel='from relion (auto-refine 3d)',
            importFrom=emprot.ProtImportParticles.IMPORT_FROM_RELION,
            starFile=cls.datasetRelion.getFile(relionFile),
            magnification=10000,
            samplingRate=7.08,
            haveDataBeenPhaseFlipped=True)
        launch(pImpPartMicId, wait=True)
        cls.partMicId = pImpPartMicId.outputParticles
        cls.micsMicId = pImpPartMicId.outputMicrographs
예제 #28
0
    def testSubset(self):
        """Test that the subset operation works as expected."""

        print("\n", pwutils.greenStr(" Test Subset ".center(75, '-')))

        def check(set0, n1=2, n2=2):
            """Simple checks on subsets, coming from split sets of set0."""
            print(
                pwutils.magentaStr("\n==> Check subset of %s" %
                                   type(set0).__name__))
            p_split1 = self.split(set0, n=n1, randomize=True)
            p_split2 = self.split(set0, n=n2, randomize=True)

            setFull = random.choice(list(self.outputs(p_split1)))
            setSub = random.choice(list(self.outputs(p_split2)))

            label = '%s - %s,%s ' % (set0.getClassName(), n1, n2)
            # Launch intersection subset
            p_subset = self.newProtocol(emprot.ProtSubSet)
            p_subset.setObjLabel(label + 'intersection')
            p_subset.inputFullSet.set(setFull)
            p_subset.inputSubSet.set(setSub)
            self.launchProtocol(p_subset)

            # Launch difference subset
            p_subset_diff = self.proj.copyProtocol(p_subset)
            p_subset_diff.setOperation.set(p_subset_diff.SET_DIFFERENCE)
            p_subset_diff.setObjLabel(label + 'difference')
            self.launchProtocol(p_subset_diff)

            setFullIds = setFull.getIdSet()
            setSubIds = setSub.getIdSet()
            n = len(setFull)

            # Check intersection
            outputs = [o for o in self.outputs(p_subset)]
            n1 = 0
            if outputs:
                output = outputs[0]

                # Check properties
                self.assertTrue(
                    set0.equalAttributes(output,
                                         ignore=['_mapperPath', '_size'],
                                         verbose=True),
                    "Intersection subset attributes are wrong")

                n1 = len(output)
                for elem in output:
                    self.assertTrue(elem.getObjId() in setFullIds)
                    self.assertTrue(
                        elem.getObjId() in setSubIds,
                        'object id %s not in set: %s' %
                        (elem.getObjId(), setSubIds))

            # Check difference
            outputs = [o for o in self.outputs(p_subset_diff)]
            n2 = 0
            if outputs:
                output_diff = outputs[0]
                # Check properties
                self.assertTrue(
                    set0.equalAttributes(output_diff,
                                         ignore=['_mapperPath', '_size'],
                                         verbose=True),
                    "In subset attributes are wrong")

                n2 = len(output_diff)
                for elem in output_diff:
                    self.assertTrue(elem.getObjId() in setFullIds)
                    self.assertTrue(elem.getObjId() not in setSubIds)

            self.assertTrue(n >= n1)
            self.assertTrue(n >= n2)
            self.assertEqual(n, n1 + n2)

        # We won't do these first two, there are too few elements.
        #   check(self.micros)
        #   check(self.vols)
        check(self.movies)
        check(self.particles)
        check(self.particles, n1=3, n2=5)
 def setUpClass(cls):
     print "\n", greenStr(" Crop/Resize Set Up - Collect data ".center(75, '-'))
     setupTestProject(cls)
     TestXmippBase.setData('xmipp_tutorial')
     cls.protImport = cls.runImportParticles(cls.particlesFn, 1.237, True)
     cls.acquisition = cls.protImport.outputParticles.getAcquisition()
예제 #30
0
def _submit(hostConfig, submitDict):
    """ Submit a protocol to a queue system. Return its job id.
    """
    # Create forst the submission script to be launched
    # formatting using the template
    template = hostConfig.getSubmitTemplate() % submitDict
    #FIXME: CREATE THE PATH FIRST
    scripPath = submitDict['JOB_SCRIPT']
    f = open(scripPath, 'w')
    #Ensure the path exists
    makeFilePath(scripPath)
    # Add some line ends because in some clusters it fails
    # to submit jobs if the submit script does not have end of line
    f.write(template + '\n\n')
    f.close()
    # This should format the command using a template like:
    # "qsub %(JOB_SCRIPT)s"
    command = hostConfig.getSubmitCommand() % submitDict
    gcmd = greenStr(command)
    print "** Submiting to queue: '%s'" % gcmd

    # zf = open('/home/jtq89441/Desktop/scipion.log','w+')
    # zf.write('It works!%s'%submitDict)
    # zf.close()
    #  ----------------------------------
    DLS_SCIPION = '/dls_sw/apps/scipion/release-1.2.1-zo'

    # command_for_recipe = 'module load %s &&'%DLS_SCIPION +'; '+ command

    projpath = submitDict['JOB_COMMAND'].split()[4]

    command_for_queue = '%s %s' % (command.split()[0], '/'.join(
        [projpath, command.split()[1]]))

    print 'command_for_queue: %s' % command_for_queue

    zocolo_cmd = 'module load dials; dials.python /dls_sw/apps/scipion/scipion_1_2_1_dials/scipion/pyworkflow/protocol/generic_template.py %s' % command_for_queue

    print zocolo_cmd

    print '****Before Zocolo****'
    msg_p = Popen(zocolo_cmd, shell=True)
    print '****After Zocolo****'

    #  ------------------------------------

    #Generating the recipe for ActiveMQ
    # default_configuration = '/dls_sw/apps/zocalo/secrets/credentials-live.cfg'
    # # override default stomp host
    # try:
    #     StompTransport.load_configuration_file(default_configuration)
    # except workflows.Error as e:
    #     print "Error: %s\n" % str(e)
    #
    # # StompTransport.add_command_line_options(parser)
    # # (options, args) = parser.parse_args(sys.argv[1:])
    # stomp = StompTransport()
    #
    # message = {'recipes': [],
    #            'parameters': {},
    #            }
    # # Build a custom recipe
    # command_for_recipe = 'module load scipion/release-1.2.1-headless &&' + command
    #
    # recipe = {}
    # recipe['1'] = {}
    # recipe['1']['service'] = "motioncor2_runner"
    # recipe['1']['queue'] = "motioncor2_runner"
    # recipe['1']['parameters'] = {}
    # recipe['1']['parameters']['arguments'] = command_for_recipe
    # recipe['start'] = [[1, []]]
    #
    # message['custom_recipe'] = recipe
    # print "******************************** THIS IS THE SUBMITTED RECIPE**********************************************"
    #
    # stomp.connect()
    # test_valid_recipe = workflows.recipe.Recipe(recipe)
    # test_valid_recipe.validate()
    # print message
    #
    # stomp.send('processing_recipe',message)
    # print("\nMotioncor2 job submitted")
    ## end of recipe generation

    # Npn zocalo scipion send command

    p = Popen(command, shell=True, stdout=PIPE)
    out = p.communicate()[0]
    # Try to parse the result of qsub, searching for a number (jobId)
    s = re.search('(\d+)', out)
    if s:
        return int(s.group(0))
    else:
        print "** Couldn't parse %s ouput: %s" % (gcmd, redStr(out))
        return UNKNOWN_JOBID
예제 #31
0
 def setUpClass(cls):
     print "\n", greenStr(" Crop/Resize Volumes Set Up - Collect data ".center(75, '-'))
     setupTestProject(cls)
     TestXmippBase.setData()
     cls.protImport1 = cls.runImportVolumes(cls.volumes, 9.896)
     cls.protImport2 = cls.runImportVolumes(cls.vol1, 9.896)
예제 #32
0
    def testSubset(self):
        """Test that the subset operation works as expected."""

        print "\n", greenStr(" Test Subset ".center(75, '-'))

        def check(set0, n1=2, n2=2):
            "Simple checks on subsets, coming from split sets of set0."
            print magentaStr("\n==> Check subset of %s" % type(set0).__name__)
            p_split1 = self.split(set0, n=n1, randomize=True)
            p_split2 = self.split(set0, n=n2, randomize=True)

            setFull = random.choice(list(self.outputs(p_split1)))
            setSub = random.choice(list(self.outputs(p_split2)))
            
            label = '%s - %s,%s ' % (set0.getClassName(), n1, n2)
            # Launch intersection subset
            p_subset = self.newProtocol(ProtSubSet)
            p_subset.setObjLabel(label + 'intersection')
            p_subset.inputFullSet.set(setFull)
            p_subset.inputSubSet.set(setSub)
            self.launchProtocol(p_subset)
            
            # Launch difference subset
            p_subset_diff = self.proj.copyProtocol(p_subset)
            p_subset_diff.setOperation.set(p_subset_diff.SET_DIFFERENCE)
            p_subset_diff.setObjLabel(label + 'difference')
            self.launchProtocol(p_subset_diff)

            setFullIds = setFull.getIdSet()
            setSubIds = setSub.getIdSet()
            n = len(setFull)
            
            # Check intersection
            outputs = [o for o in self.outputs(p_subset)]
            n1 = 0
            if outputs:
                output = outputs[0]
                n1 = len(output)
                for elem in output:
                    self.assertTrue(elem.getObjId() in setFullIds)
                    self.assertTrue(elem.getObjId() in setSubIds,
                                    'object id %s not in set: %s' % (elem.getObjId(), setSubIds))
                
            # Check difference
            outputs = [o for o in self.outputs(p_subset_diff)]
            n2 = 0
            if outputs:
                output_diff = outputs[0]
                n2 = len(output_diff)
                for elem in output_diff:
                    self.assertTrue(elem.getObjId() in setFullIds)
                    self.assertTrue(elem.getObjId() not in setSubIds)
                
                
            self.assertTrue(n >= n1)
            self.assertTrue(n >= n2)            
            self.assertEqual(n, n1+n2)

        # We won't do these first two, there are too few elements.
        #   check(self.micros)
        #   check(self.vols)
        check(self.movies)
        check(self.particles)
        check(self.particles, n1=3, n2=5)
예제 #33
0
 def addSuccess(self, test):
     secs = self.toc()
     sys.stderr.write(
         "%s %s (%0.3f secs)\n" %
         (pwutils.greenStr('[ RUN   OK ]'), self.getTestName(test), secs))
    def setUpClass(cls):
        """Prepare the data that we will use later on."""

        print("\n", greenStr(" Set Up - Collect data ".center(75, '-')))

        setupTestProject(cls)  # defined in BaseTest, creates cls.proj

        cls.xmippDataTest = DataSet.getDataSet('xmipp_tutorial')

        def _importClasses(label, numClasses, randomSeed, numPart=None):
            """ We import a set of classes with an alterate import particles
                because there is not a import classes protocol
            """
            pImpClasses = cls.proj.newProtocol(
                ProtImportParticles,
                filesPath=cls.xmippDataTest.getFile('particles'),
                samplingRate=3.5)

            pImpClasses.setObjLabel('Import %s' % label)

            # we launch the protocol to obtain the particles
            cls.proj.launchProtocol(pImpClasses, wait=True)

            # fake importing classes handmade
            partSet = pImpClasses.outputParticles

            setOfClasses = pImpClasses._createSetOfClasses3D(partSet)

            numOfPart = partSet.getSize() if numPart is None else numPart
            partIds = list(partSet.getIdSet())
            m = int(numOfPart / numClasses)

            # random shuffle with a certain seed to get always the same classes
            random.seed(randomSeed)
            random.shuffle(partIds)
            for clInx in list(range(numClasses)):
                currIds = partIds[clInx * m:(clInx + 1) * m]

                newClass = Class3D()
                newClass.copyInfo(partSet)
                newClass.setAcquisition(partSet.getAcquisition())
                # newClass.setRepresentative(clRep.getRepresentative())

                setOfClasses.append(newClass)

                enabledClass = setOfClasses[newClass.getObjId()]
                enabledClass.enableAppend()
                for itemId in currIds:
                    item = partSet[itemId]
                    enabledClass.append(item)

                setOfClasses.update(enabledClass)

            # including the outputClasses as protocol output by hand
            pImpClasses._defineOutputs(outputClasses=setOfClasses)

            # relaunching the protocol to get the new output
            cls.proj.launchProtocol(pImpClasses, wait=True)

            return pImpClasses.outputClasses

        cls.set1 = _importClasses('3 Classes I', 3, 65)
        cls.set2 = _importClasses('3 Classes II', 3, 123)
        cls.set3 = _importClasses('3 Classes III', 3, 256)
        cls.set4 = _importClasses('3 Classes IV (not all part)', 3, 568, 70)
        cls.set5 = _importClasses('5 Classes I', 5, 745)
        cls.set6 = _importClasses('5 Classes II', 5, 1025)