def doReport(self):
secs = time.time() - self.startTimeAll
sys.stderr.write(
"\n%s run %d tests (%0.3f secs)\n" %
(pwutils.greenStr("[==========]"), self.numberTests, secs))
if self.testFailed:
sys.stderr.write("%s %d tests\n" %
(pwutils.redStr("[ FAILED ]"), self.testFailed))
sys.stdout.write("%s %d tests\n" %
(pwutils.greenStr("[ PASSED ]"),
self.numberTests - self.testFailed))
def testMerge(self):
"""Test that the union operation works as expected."""
print "\n", greenStr(" Test Merge ".center(75, '-'))
def check(set0):
"Simple checks on merge, coming from many split sets of set0."
print magentaStr("\n==> Check merge of %s" % type(set0).__name__)
p_union = self.proj.newProtocol(ProtUnionSet)
setsIds = []
for i in range(random.randint(1, 5)):
n = random.randint(1, len(set0) // 2)
p_split = self.split(set0, n=n, randomize=True)
setRandom = random.choice(list(self.outputs(p_split)))
setsIds.append([x.strId() for x in setRandom])
p_union.inputSets.append(setRandom)
self.proj.launchProtocol(p_union, wait=True)
output = self.outputs(p_union).next() # first (and only!) output
self.assertEqual(len(output), sum(len(x) for x in setsIds))
# We might be able to do more interesting tests, using the
# collected setsIds.
check(self.micros)
check(self.vols)
check(self.movies)
check(self.particles)
def testMerge(self):
"""Test that the union operation works as expected."""
print("\n", pwutils.greenStr(" Test Merge ".center(75, '-')))
def check(set0):
# Simple checks on merge, coming from many split sets of set0.
print(
pwutils.magentaStr("\n==> Check merge of %s" %
type(set0).__name__))
p_union = self.proj.newProtocol(emprot.ProtUnionSet)
setsIds = []
for i in range(random.randint(1, 5)):
n = random.randint(1, len(set0) // 2)
p_split = self.split(set0, n=n, randomize=True)
setRandom = random.choice(list(self.outputs(p_split)))
setsIds.append([x.strId() for x in setRandom])
p_union.inputSets.append(setRandom)
self.proj.launchProtocol(p_union, wait=True)
output = next(self.outputs(p_union)) # first (and only!) output
self.assertEqual(len(output), sum(len(x) for x in setsIds))
# We might be able to do more interesting tests, using the
# collected setsIds.
check(self.micros)
check(self.vols)
check(self.movies)
check(self.particles)
def testSplit(self):
"""Test that the split operation works as expected."""
print("\n", pwutils.greenStr(" Test Split ".center(75, '-')))
def check(set0, n=2, randomize=False):
# Simple checks on split sets from set0.
print(
pwutils.magentaStr("\n==> Check split of %s" %
type(set0).__name__))
unsplit_set = [x.strId() for x in set0]
p_split = self.split(set0, n=n, randomize=randomize)
# Are all output elements of the protocol in the original set?
for em_set in self.outputs(p_split):
for elem in em_set:
self.assertTrue(elem.strId() in unsplit_set)
# Number of elements of all split sets equal to original number?
self.assertEqual(sum(len(x) for x in self.outputs(p_split)),
len(set0))
check(self.micros)
check(self.micros, randomize=True)
check(self.vols)
check(self.movies)
check(self.particles)
check(self.particles, n=4)
def _submit(hostConfig, submitDict, cwd=None, env=None):
""" Submit a protocol to a queue system. Return its job id.
"""
# Create first the submission script to be launched
# formatting using the template
template = hostConfig.getSubmitTemplate() % submitDict
# FIXME: CREATE THE PATH FIRST
scripPath = submitDict['JOB_SCRIPT']
f = open(scripPath, 'w')
# Ensure the path exists
makeFilePath(scripPath)
# Add some line ends because in some clusters it fails
# to submit jobs if the submit script does not have end of line
f.write(template + '\n\n')
f.close()
# This should format the command using a template like:
# "qsub %(JOB_SCRIPT)s"
command = hostConfig.getSubmitCommand() % submitDict
gcmd = greenStr(command)
print("** Submitting to queue: '%s'" % gcmd)
p = Popen(command, shell=True, stdout=PIPE, cwd=cwd, env=env)
out = p.communicate()[0]
# Try to parse the result of qsub, searching for a number (jobId)
# Review this, seems to exclusive to torque batch system
s = re.search('(\d+)', str(out))
if p.returncode == 0 and s:
job = int(s.group(0))
print("Launched job with id %s" % job)
return job
else:
print("Couldn't submit to queue for reason: %s " %
redStr(out.decode()))
return UNKNOWN_JOBID
def _runRemote(protocol, mode):
""" Launch remotely 'pw_protocol_remote.py' script to run or stop a protocol.
Params:
protocol: the protocol to be ran or stopped.
mode: should be either 'run' or 'stop'
"""
host = protocol.getHostConfig()
tpl = "ssh %(address)s '%(scipion)s/scipion "
if host.getScipionConfig() is not None:
tpl += "--config %(config)s "
tpl += "runprotocol pw_protocol_remote.py %(mode)s "
tpl += "%(project)s %(protDb)s %(protId)s' "
# Use project base name,
# in remote SCIPION_USER_DATA/projects should be prepended
projectPath = os.path.basename(protocol.getProject().path)
args = {'address': host.getAddress(),
'mode': mode,
'scipion': host.getScipionHome(),
'config': host.getScipionConfig(),
'project': projectPath,
'protDb': protocol.getDbPath(),
'protId': protocol.getObjId()
}
cmd = tpl % args
print "** Running remote: %s" % greenStr(cmd)
p = Popen(cmd, shell=True, stdout=PIPE)
return p
def _submit(hostConfig, submitDict):
""" Submit a protocol to a queue system. Return its job id.
"""
# Create forst the submission script to be launched
# formatting using the template
template = hostConfig.getSubmitTemplate() % submitDict
#FIXME: CREATE THE PATH FIRST
scripPath = submitDict['JOB_SCRIPT']
f = open(scripPath, 'w')
#Ensure the path exists
makeFilePath(scripPath)
# Add some line ends because in some clusters it fails
# to submit jobs if the submit script does not have end of line
f.write(template+'\n\n')
f.close()
# This should format the command using a template like:
# "qsub %(JOB_SCRIPT)s"
command = hostConfig.getSubmitCommand() % submitDict
gcmd = greenStr(command)
print "** Submiting to queue: '%s'" % gcmd
p = Popen(command, shell=True, stdout=PIPE)
out = p.communicate()[0]
# Try to parse the result of qsub, searching for a number (jobId)
s = re.search('(\d+)', out)
if s:
return int(s.group(0))
else:
print "** Couldn't parse %s ouput: %s" % (gcmd, redStr(out))
return UNKNOWN_JOBID
def test_filterParticles(self):
print "\n", greenStr(" Filter Particles ".center(75, '-'))
def test(parts=self.protImport.outputParticles, **kwargs):
"Launch XmippProtFilterParticles on parts and check results."
print magentaStr("\n==> Input params: %s" % kwargs)
prot = self.newProtocol(XmippProtFilterParticles, **kwargs)
prot.inputParticles.set(parts)
self.launchProtocol(prot)
self.assertIsNotNone(prot.outputParticles,
"There was a problem with filter particles")
self.assertTrue(prot.outputParticles.equalAttributes(
parts, ignore=['_mapperPath'], verbose=True))
# Compare the individual particles too.
self.assertTrue(prot.outputParticles.equalItemAttributes(
parts, ignore=['_filename', '_index'], verbose=True))
# Check a few different cases.
test(filterSpace=FILTER_SPACE_FOURIER, lowFreq=0.1, highFreq=0.25)
test(filterSpace=FILTER_SPACE_REAL, filterModeReal=xfh.FM_MEDIAN)
# For wavelets, we need the input's size to be a power of 2
print magentaStr("\n==> Resizing particles to 256 pixels")
protResize = self.newProtocol(XmippProtCropResizeParticles,
doResize=True,
resizeOption=xrh.RESIZE_DIMENSIONS,
resizeDim=256)
protResize.inputParticles.set(self.protImport.outputParticles)
self.launchProtocol(protResize)
test(parts=protResize.outputParticles, filterSpace=FILTER_SPACE_WAVELET,
filterModeWavelets=xfh.FM_DAUB12, waveletMode=xfh.FM_REMOVE_SCALE)
def setUpClass(cls):
print "\n", greenStr(
" Crop/Resize Volumes Set Up - Collect data ".center(75, '-'))
setupTestProject(cls)
TestXmippBase.setData()
cls.protImport1 = cls.runImportVolumes(cls.volumes, 9.896)
cls.protImport2 = cls.runImportVolumes(cls.vol1, 9.896)
def _run(command, wait, stdin=None, stdout=None, stderr=None):
""" Execute a command in a subprocess and return the pid. """
gcmd = greenStr(command)
print "** Running command: '%s'" % gcmd
p = Popen(command, shell=True, stdout=stdout, stderr=stderr)
jobId = p.pid
if wait:
p.wait()
return jobId
def setUpClass(cls):
"""Prepare the data that we will use later on."""
print "\n", greenStr(" Set Up - Collect data ".center(75, '-'))
setupTestProject(cls) # defined in BaseTest, creates cls.proj
cls.dataset_xmipp = DataSet.getDataSet('xmipp_tutorial')
cls.dataset_mda = DataSet.getDataSet('mda')
cls.dataset_ribo = DataSet.getDataSet('ribo_movies')
#
# Imports
#
new = cls.proj.newProtocol # short notation
launch = cls.proj.launchProtocol
# Micrographs
print magentaStr("\n==> Importing data - micrographs")
p_imp_micros = new(ProtImportMicrographs,
filesPath=cls.dataset_xmipp.getFile('allMics'),
samplingRate=1.237,
voltage=300)
launch(p_imp_micros, wait=True)
cls.micros = p_imp_micros.outputMicrographs
# Volumes
print magentaStr("\n==> Importing data - volumes")
p_imp_volumes = new(ProtImportVolumes,
filesPath=cls.dataset_xmipp.getFile('volumes'),
samplingRate=9.896)
launch(p_imp_volumes, wait=True)
cls.vols = p_imp_volumes.outputVolumes
# Movies
print magentaStr("\n==> Importing data - movies")
p_imp_movies = new(ProtImportMovies,
filesPath=cls.dataset_ribo.getFile('movies'),
samplingRate=2.37,
magnification=59000,
voltage=300,
sphericalAberration=2.0)
launch(p_imp_movies, wait=True)
cls.movies = p_imp_movies.outputMovies
# Particles
print magentaStr("\n==> Importing data - particles")
p_imp_particles = new(ProtImportParticles,
filesPath=cls.dataset_mda.getFile('particles'),
samplingRate=3.5)
launch(p_imp_particles, wait=True)
cls.particles = p_imp_particles.outputParticles
def testAlignAndReconstruct(self):
print("\n", greenStr(" Test Ts alignment and tomogram reconstruction with Jjsof".center(75, '-')))
# preparing and launching the protocol
palireco = self.newProtocol(ProtJjsoftAlignReconstructTomogram,
inputSetOfTiltSeries=self.setOfFiducialTs,
inputSetOfLandmarkModels=self.setOfFiducials)
self.launchProtocol(palireco, wait=True)
setOfTomograms = palireco.outputTomograms
# some general assertions
self.assertIsNotNone(setOfTomograms,
"There was some problem with the output")
self.assertEqual(setOfTomograms.getSize(), self.setOfFiducialTs.getSize(),
"The number of tomograms is wrong")
def testReconstructionWBP(self):
print ("\n", greenStr(" Test tomo3D reconstruction with WBP".center(75, '-')))
# preparing and launching the protocol
ptomo3D = self.newProtocol(ProtJjsoftReconstructTomogram,
inputSetOfTiltSeries=self.setOfTs,
method=0)
self.launchProtocol(ptomo3D, wait=True)
setOfReconstructedTomograms = ptomo3D.outputTomograms
# some general assertions
self.assertIsNotNone(setOfReconstructedTomograms,
"There was some problem with the output")
self.assertEqual(setOfReconstructedTomograms.getSize(), self.setOfTs.getSize(),
"The number of the denoised tomograms is wrong")
def testWarpAlign(self):
print ("\n", greenStr(" Test Ts alignment with warpalign ".center(75, '-')))
# preparing and launching the protocol
pwarpalign = self.newProtocol(ProtJjsoftAlignTs,
inputSetOfTiltSeries=self.setOfFiducialTs,
inputSetOfLandmarkModels=self.setOfFiducials)
self.launchProtocol(pwarpalign, wait=True)
setOfAlignedTs = pwarpalign.outputInterpolatedSetOfTiltSeries
# some general assertions
self.assertIsNotNone(setOfAlignedTs,
"There was some problem with the output")
self.assertEqual(setOfAlignedTs.getSize(), self.setOfFiducialTs.getSize(),
"The number of the aligned Ts is wrong")
def setUpClass(cls):
"""Prepare the data that we will use later on."""
print "\n", greenStr(" Set Up - Collect data ".center(75, '-'))
setupTestProject(cls) # defined in BaseTest, creates cls.proj
cls.dataset_xmipp = DataSet.getDataSet('xmipp_tutorial')
cls.dataset_mda = DataSet.getDataSet('mda')
cls.dataset_ribo = DataSet.getDataSet('ribo_movies')
#
# Imports
#
new = cls.proj.newProtocol # short notation
launch = cls.proj.launchProtocol
# Micrographs
print magentaStr("\n==> Importing data - micrographs")
p_imp_micros = new(ProtImportMicrographs,
filesPath=cls.dataset_xmipp.getFile('allMics'),
samplingRate=1.237, voltage=300)
launch(p_imp_micros, wait=True)
cls.micros = p_imp_micros.outputMicrographs
# Volumes
print magentaStr("\n==> Importing data - volumes")
p_imp_volumes = new(ProtImportVolumes,
filesPath=cls.dataset_xmipp.getFile('volumes'),
samplingRate=9.896)
launch(p_imp_volumes, wait=True)
cls.vols = p_imp_volumes.outputVolumes
# Movies
print magentaStr("\n==> Importing data - movies")
p_imp_movies = new(ProtImportMovies,
filesPath=cls.dataset_ribo.getFile('movies'),
samplingRate=2.37, magnification=59000,
voltage=300, sphericalAberration=2.0)
launch(p_imp_movies, wait=True)
cls.movies = p_imp_movies.outputMovies
# Particles
print magentaStr("\n==> Importing data - particles")
p_imp_particles = new(ProtImportParticles,
filesPath=cls.dataset_mda.getFile('particles'),
samplingRate=3.5)
launch(p_imp_particles, wait=True)
cls.particles = p_imp_particles.outputParticles
def testConsensus6(self):
print("\n", greenStr(" Test Consensus with four sets".center(75, '-')))
# preparing and launching the protocol
pConsClass = self.proj.newProtocol(
XmippProtConsensusClasses3D,
inputMultiClasses=[self.set1, self.set2, self.set5, self.set6])
self.proj.launchProtocol(pConsClass, wait=True)
setOfIntersections = pConsClass.outputClasses
# some general assertions
self.assertIsNotNone(setOfIntersections,
"There was some problem with the output")
self.assertEqual(setOfIntersections.getSize(), 225,
"The number of the outputClasses is wrong")
self.checkPopulation(setOfIntersections, 75)
def testSubsetByMic(self):
"""Test that the subset by Mic operation works as expected."""
print("\n", pwutils.greenStr(" Test Subset by Mic".center(75, '-')))
"Simple checks on subsets, coming from split sets of setMics."
print(
pwutils.magentaStr("\n==> Check subset of %s by %s" % (type(
self.partMicId).__name__, type(self.micsMicId).__name__)))
# launch the protocol for a certain mics input
def launchSubsetByMic(micsSubset):
pSubsetbyMic = self.newProtocol(emprot.ProtSubSetByMic)
pSubsetbyMic.inputParticles.set(self.partMicId)
pSubsetbyMic.inputMicrographs.set(micsSubset)
self.launchProtocol(pSubsetbyMic)
return pSubsetbyMic.outputParticles
# Check if the Output is generated, the subset size is correct and
# the micId of the particle with certain partId is correct.
def checkAsserts(setParts, size, partId, micId):
self.assertIsNotNone(setParts, "Output SetOfParticles"
" were not created.")
self.assertEqual(setParts.getSize(), size,
"The number of created particles is incorrect.")
p = setParts[partId]
self.assertEqual(p.getMicId(), micId)
# Whole set of micrographs
setMics = self.micsMicId
# Create a subsets of Mics to apply the protocol
pSplit = self.split(setMics, n=2, randomize=False)
setMics2 = pSplit.outputMicrographs02
# Create a subset of a single micrograph to apply the protocol
pSplit = self.split(setMics, n=20, randomize=False)
setMics3 = pSplit.outputMicrographs03
# Launch subset by mics protocol with the whole set of Mics
partByMic1 = launchSubsetByMic(setMics)
# Launch subset by mics protocol with a subset of Mics
partByMic2 = launchSubsetByMic(setMics2)
# Launch subset by mics protocol with a single SetOfMics
partByMic3 = launchSubsetByMic(setMics3)
# Assertions for the three sets
checkAsserts(partByMic1, self.partMicId.getSize(), 1885, 7)
checkAsserts(partByMic2, 2638, 4330, 16)
checkAsserts(partByMic3, 270, 725, 3)
def testDenoisingBFlow(self):
print ("\n", greenStr(" Test BFlow denoising ".center(75, '-')))
# preparing and launching the protocol
pDenoiseBFlow = self.newProtocol(ProtJjsoftProtDenoiseTomogram,
inputSetTomograms=self.setOfTomograms,
method=1,
SigmaGaussian=0.5,
nIter=1,
TimeStep=0.1)
self.launchProtocol(pDenoiseBFlow, wait=True)
setOfBFlowDenoisedTomograms = pDenoiseBFlow.outputTomograms
# some general assertions
self.assertIsNotNone(setOfBFlowDenoisedTomograms,
"There was some problem with the output")
self.assertEqual(setOfBFlowDenoisedTomograms.getSize(), self.setOfTomograms.getSize(),
"The number of the denoised tomograms is wrong")
def testSubsetByCoord(self):
"""Test that the subset by Coord operation works as expected."""
print("\n", pwutils.greenStr(" Test Subset by Coord".center(75, '-')))
p_extract_coordinates = self.newProtocol(emprot.ProtExtractCoords)
p_extract_coordinates.inputParticles.set(self.partMicId)
p_extract_coordinates.inputMicrographs.set(self.micsMicId)
self.launchProtocol(p_extract_coordinates)
p_subset_by_coords = self.newProtocol(emprot.ProtSubSetByCoord)
p_subset_by_coords.inputParticles.set(self.partMicId)
p_subset_by_coords.inputCoordinates.set(
p_extract_coordinates.outputCoordinates)
self.launchProtocol(p_subset_by_coords)
self.assertIsNotNone(p_subset_by_coords.outputParticles,
"Output SetOfParticles were not created.")
self.assertEqual(p_subset_by_coords.outputParticles.getSize(), 5236,
"The number of created particles is incorrect.")
def setParamValue(self, alias, newValue):
paramsSetted = 0
for field in self.params.values():
if field.getAlias() == alias:
oldValue = field.getValue()
field.setValue(newValue)
if field.validate():
paramsSetted += 1
print(
greenStr("%s set to %s") %
(field.getTitle(), str(newValue)))
else:
field.setValue(oldValue)
raise Exception(
"%s is not compatible with %s(%s) parameter." %
(newValue, field.getTitle(), alias))
if not paramsSetted:
raise Exception("Alias %s not recognized." % alias)
return paramsSetted
def testConsensus5(self):
print "\n", greenStr(" Test Consensus with two sets".center(75, '-'))
# preparing and launching the protocol
pConsClass = self.proj.newProtocol(
XmippProtConsensusClasses3D,
inputMultiClasses=[self.set5, self.set6])
self.proj.launchProtocol(pConsClass, wait=True)
setOfIntersections = pConsClass.outputClasses
# some general assertions
self.assertIsNotNone(setOfIntersections,
"There was some problem with the output")
self.assertEqual(setOfIntersections.getSize(), 25,
"The number of the outputClasses is wrong")
self.checkPopulation(setOfIntersections, 75)
# some specific assetions
self.checkIntersections(setOfIntersections, 1, [48, 74, 44, 53])
self.checkIntersections(setOfIntersections, 7, [51, 68, 14])
self.checkIntersections(setOfIntersections, 20, [36, 5])
def getPackagesStatus(cls, printAll=True):
"""
Check for scipion-app, scipion-pyworkflow and scipion-em updates
return: a list of modules to be updated
"""
outdatedPackages = []
for package in cls.packageNames:
needToUpdate, version = cls.getPackageState(package[0], package[1])
if needToUpdate:
outdatedPackages.append((package[0], version))
print(
redStr(
'The package %s is out of date. Your version is %s, '
'the latest is %s.' %
(package[0], package[1], version)))
elif printAll:
print(
greenStr('The package %s is up to date. Your version '
'is %s' % (package[0], version)))
return outdatedPackages
def testConsensus3(self):
print "\n", greenStr(
" Test Consensus with different set sizes 2".center(75, '-'))
# preparing and launching the protocol
pConsClass = self.proj.newProtocol(
XmippProtConsensusClasses3D,
inputMultiClasses=[self.set4, self.set2, self.set1])
self.proj.launchProtocol(pConsClass, wait=True)
setOfIntersections = pConsClass.outputClasses
# some general assertions
self.assertIsNotNone(setOfIntersections,
"There was some problem with the output")
self.assertEqual(setOfIntersections.getSize(), 27,
"The number of the outputClasses is wrong")
self.checkPopulation(setOfIntersections, 69)
# some specific assetions
self.checkIntersections(setOfIntersections, 1, [74, 53])
self.checkIntersections(setOfIntersections, 7, [25, 43, 68, 54, 57])
self.checkIntersections(setOfIntersections, 20, [46])
def testSplit(self):
"""Test that the split operation works as expected."""
print "\n", greenStr(" Test Split ".center(75, '-'))
def check(set0, n=2, randomize=False):
"Simple checks on split sets from set0."
print magentaStr("\n==> Check split of %s" % type(set0).__name__)
unsplit_set = [x.strId() for x in set0]
p_split = self.split(set0, n=n, randomize=randomize)
# Are all output elements of the protocol in the original set?
for em_set in self.outputs(p_split):
for elem in em_set:
self.assertTrue(elem.strId() in unsplit_set)
# Number of elements of all splitted sets equal to original number?
self.assertEqual(sum(len(x) for x in self.outputs(p_split)),
len(set0))
check(self.micros)
check(self.micros, randomize=True)
check(self.vols)
check(self.movies)
check(self.particles)
check(self.particles, n=4)
def setUpClass(cls):
print "\n", greenStr(" Crop/Resize Set Up - Collect data ".center(75, '-'))
setupTestProject(cls)
TestXmippBase.setData('relion_tutorial')
def setUpClass(cls):
print "\n", greenStr(" Set Up - Collect data ".center(75, '-'))
setupTestProject(cls)
TestXmippBase.setData('xmipp_tutorial')
cls.protImport = cls.runImportParticles(cls.particlesFn, 1.237, True, True)
def setUpClass(cls):
"""Prepare the data that we will use later on."""
print("\n", pwutils.greenStr(" Set Up - Collect data ".center(75,
'-')))
pwtests.setupTestProject(cls) # defined in BaseTest, creates cls.proj
cls.dataset_xmipp = pwtests.DataSet.getDataSet('xmipp_tutorial')
cls.dataset_mda = pwtests.DataSet.getDataSet('mda')
cls.dataset_ribo = pwtests.DataSet.getDataSet('ribo_movies')
cls.datasetRelion = pwtests.DataSet.getDataSet('relion_tutorial')
#
# Imports
#
new = cls.proj.newProtocol # short notation
launch = cls.proj.launchProtocol
# Micrographs
# NOTE: This dataset has 3 mic with heterogeneous dimensions!! But so
# far is not failing, should it?
print(pwutils.magentaStr("\n==> Importing data - micrographs"))
p_imp_micros = new(emprot.ProtImportMicrographs,
filesPath=cls.dataset_xmipp.getFile('allMics'),
samplingRate=1.237,
voltage=300)
launch(p_imp_micros, wait=True)
cls.micros = p_imp_micros.outputMicrographs
# Micrographs SMALL - This is a mic with different dimensions
print(pwutils.magentaStr("\n==> Importing data - micrographs SMALL"))
p_imp_micros = new(emprot.ProtImportMicrographs,
filesPath=cls.dataset_xmipp.getFile('mic3'),
samplingRate=1.237,
voltage=300)
launch(p_imp_micros, wait=True)
cls.microsSmall = p_imp_micros.outputMicrographs
# Volumes
print(pwutils.magentaStr("\n==> Importing data - volumes"))
p_imp_volumes = new(emprot.ProtImportVolumes,
filesPath=cls.dataset_xmipp.getFile('volumes'),
samplingRate=9.896)
launch(p_imp_volumes, wait=True)
cls.vols = p_imp_volumes.outputVolumes
# Movies
print(pwutils.magentaStr("\n==> Importing data - movies"))
p_imp_movies = new(emprot.ProtImportMovies,
filesPath=cls.dataset_ribo.getFile('movies'),
samplingRate=2.37,
magnification=59000,
voltage=300,
sphericalAberration=2.0,
gainFile=cls.dataset_ribo.getFile('volume'),
darkFile=cls.dataset_ribo.getFile('volume'))
launch(p_imp_movies, wait=True)
cls.movies = p_imp_movies.outputMovies
# Particles
print(pwutils.magentaStr("\n==> Importing data - particles"))
p_imp_particles = new(emprot.ProtImportParticles,
filesPath=cls.dataset_mda.getFile('particles'),
samplingRate=3.5)
launch(p_imp_particles, wait=True)
cls.particles = p_imp_particles.outputParticles
# Particles with micId
print(
pwutils.magentaStr("\n==> Importing data - particles with micId"))
relionFile = 'import/case2/relion_it015_data.star'
pImpPartMicId = new(
emprot.ProtImportParticles,
objLabel='from relion (auto-refine 3d)',
importFrom=emprot.ProtImportParticles.IMPORT_FROM_RELION,
starFile=cls.datasetRelion.getFile(relionFile),
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
launch(pImpPartMicId, wait=True)
cls.partMicId = pImpPartMicId.outputParticles
cls.micsMicId = pImpPartMicId.outputMicrographs
def testSubset(self):
"""Test that the subset operation works as expected."""
print("\n", pwutils.greenStr(" Test Subset ".center(75, '-')))
def check(set0, n1=2, n2=2):
"""Simple checks on subsets, coming from split sets of set0."""
print(
pwutils.magentaStr("\n==> Check subset of %s" %
type(set0).__name__))
p_split1 = self.split(set0, n=n1, randomize=True)
p_split2 = self.split(set0, n=n2, randomize=True)
setFull = random.choice(list(self.outputs(p_split1)))
setSub = random.choice(list(self.outputs(p_split2)))
label = '%s - %s,%s ' % (set0.getClassName(), n1, n2)
# Launch intersection subset
p_subset = self.newProtocol(emprot.ProtSubSet)
p_subset.setObjLabel(label + 'intersection')
p_subset.inputFullSet.set(setFull)
p_subset.inputSubSet.set(setSub)
self.launchProtocol(p_subset)
# Launch difference subset
p_subset_diff = self.proj.copyProtocol(p_subset)
p_subset_diff.setOperation.set(p_subset_diff.SET_DIFFERENCE)
p_subset_diff.setObjLabel(label + 'difference')
self.launchProtocol(p_subset_diff)
setFullIds = setFull.getIdSet()
setSubIds = setSub.getIdSet()
n = len(setFull)
# Check intersection
outputs = [o for o in self.outputs(p_subset)]
n1 = 0
if outputs:
output = outputs[0]
# Check properties
self.assertTrue(
set0.equalAttributes(output,
ignore=['_mapperPath', '_size'],
verbose=True),
"Intersection subset attributes are wrong")
n1 = len(output)
for elem in output:
self.assertTrue(elem.getObjId() in setFullIds)
self.assertTrue(
elem.getObjId() in setSubIds,
'object id %s not in set: %s' %
(elem.getObjId(), setSubIds))
# Check difference
outputs = [o for o in self.outputs(p_subset_diff)]
n2 = 0
if outputs:
output_diff = outputs[0]
# Check properties
self.assertTrue(
set0.equalAttributes(output_diff,
ignore=['_mapperPath', '_size'],
verbose=True),
"In subset attributes are wrong")
n2 = len(output_diff)
for elem in output_diff:
self.assertTrue(elem.getObjId() in setFullIds)
self.assertTrue(elem.getObjId() not in setSubIds)
self.assertTrue(n >= n1)
self.assertTrue(n >= n2)
self.assertEqual(n, n1 + n2)
# We won't do these first two, there are too few elements.
# check(self.micros)
# check(self.vols)
check(self.movies)
check(self.particles)
check(self.particles, n1=3, n2=5)
def setUpClass(cls):
print "\n", greenStr(" Crop/Resize Set Up - Collect data ".center(75, '-'))
setupTestProject(cls)
TestXmippBase.setData('xmipp_tutorial')
cls.protImport = cls.runImportParticles(cls.particlesFn, 1.237, True)
cls.acquisition = cls.protImport.outputParticles.getAcquisition()
def _submit(hostConfig, submitDict):
""" Submit a protocol to a queue system. Return its job id.
"""
# Create forst the submission script to be launched
# formatting using the template
template = hostConfig.getSubmitTemplate() % submitDict
#FIXME: CREATE THE PATH FIRST
scripPath = submitDict['JOB_SCRIPT']
f = open(scripPath, 'w')
#Ensure the path exists
makeFilePath(scripPath)
# Add some line ends because in some clusters it fails
# to submit jobs if the submit script does not have end of line
f.write(template + '\n\n')
f.close()
# This should format the command using a template like:
# "qsub %(JOB_SCRIPT)s"
command = hostConfig.getSubmitCommand() % submitDict
gcmd = greenStr(command)
print "** Submiting to queue: '%s'" % gcmd
# zf = open('/home/jtq89441/Desktop/scipion.log','w+')
# zf.write('It works!%s'%submitDict)
# zf.close()
# ----------------------------------
DLS_SCIPION = '/dls_sw/apps/scipion/release-1.2.1-zo'
# command_for_recipe = 'module load %s &&'%DLS_SCIPION +'; '+ command
projpath = submitDict['JOB_COMMAND'].split()[4]
command_for_queue = '%s %s' % (command.split()[0], '/'.join(
[projpath, command.split()[1]]))
print 'command_for_queue: %s' % command_for_queue
zocolo_cmd = 'module load dials; dials.python /dls_sw/apps/scipion/scipion_1_2_1_dials/scipion/pyworkflow/protocol/generic_template.py %s' % command_for_queue
print zocolo_cmd
print '****Before Zocolo****'
msg_p = Popen(zocolo_cmd, shell=True)
print '****After Zocolo****'
# ------------------------------------
#Generating the recipe for ActiveMQ
# default_configuration = '/dls_sw/apps/zocalo/secrets/credentials-live.cfg'
# # override default stomp host
# try:
# StompTransport.load_configuration_file(default_configuration)
# except workflows.Error as e:
# print "Error: %s\n" % str(e)
#
# # StompTransport.add_command_line_options(parser)
# # (options, args) = parser.parse_args(sys.argv[1:])
# stomp = StompTransport()
#
# message = {'recipes': [],
# 'parameters': {},
# }
# # Build a custom recipe
# command_for_recipe = 'module load scipion/release-1.2.1-headless &&' + command
#
# recipe = {}
# recipe['1'] = {}
# recipe['1']['service'] = "motioncor2_runner"
# recipe['1']['queue'] = "motioncor2_runner"
# recipe['1']['parameters'] = {}
# recipe['1']['parameters']['arguments'] = command_for_recipe
# recipe['start'] = [[1, []]]
#
# message['custom_recipe'] = recipe
# print "******************************** THIS IS THE SUBMITTED RECIPE**********************************************"
#
# stomp.connect()
# test_valid_recipe = workflows.recipe.Recipe(recipe)
# test_valid_recipe.validate()
# print message
#
# stomp.send('processing_recipe',message)
# print("\nMotioncor2 job submitted")
## end of recipe generation
# Npn zocalo scipion send command
p = Popen(command, shell=True, stdout=PIPE)
out = p.communicate()[0]
# Try to parse the result of qsub, searching for a number (jobId)
s = re.search('(\d+)', out)
if s:
return int(s.group(0))
else:
print "** Couldn't parse %s ouput: %s" % (gcmd, redStr(out))
return UNKNOWN_JOBID
def setUpClass(cls):
print "\n", greenStr(" Crop/Resize Volumes Set Up - Collect data ".center(75, '-'))
setupTestProject(cls)
TestXmippBase.setData()
cls.protImport1 = cls.runImportVolumes(cls.volumes, 9.896)
cls.protImport2 = cls.runImportVolumes(cls.vol1, 9.896)
def testSubset(self):
"""Test that the subset operation works as expected."""
print "\n", greenStr(" Test Subset ".center(75, '-'))
def check(set0, n1=2, n2=2):
"Simple checks on subsets, coming from split sets of set0."
print magentaStr("\n==> Check subset of %s" % type(set0).__name__)
p_split1 = self.split(set0, n=n1, randomize=True)
p_split2 = self.split(set0, n=n2, randomize=True)
setFull = random.choice(list(self.outputs(p_split1)))
setSub = random.choice(list(self.outputs(p_split2)))
label = '%s - %s,%s ' % (set0.getClassName(), n1, n2)
# Launch intersection subset
p_subset = self.newProtocol(ProtSubSet)
p_subset.setObjLabel(label + 'intersection')
p_subset.inputFullSet.set(setFull)
p_subset.inputSubSet.set(setSub)
self.launchProtocol(p_subset)
# Launch difference subset
p_subset_diff = self.proj.copyProtocol(p_subset)
p_subset_diff.setOperation.set(p_subset_diff.SET_DIFFERENCE)
p_subset_diff.setObjLabel(label + 'difference')
self.launchProtocol(p_subset_diff)
setFullIds = setFull.getIdSet()
setSubIds = setSub.getIdSet()
n = len(setFull)
# Check intersection
outputs = [o for o in self.outputs(p_subset)]
n1 = 0
if outputs:
output = outputs[0]
n1 = len(output)
for elem in output:
self.assertTrue(elem.getObjId() in setFullIds)
self.assertTrue(elem.getObjId() in setSubIds,
'object id %s not in set: %s' % (elem.getObjId(), setSubIds))
# Check difference
outputs = [o for o in self.outputs(p_subset_diff)]
n2 = 0
if outputs:
output_diff = outputs[0]
n2 = len(output_diff)
for elem in output_diff:
self.assertTrue(elem.getObjId() in setFullIds)
self.assertTrue(elem.getObjId() not in setSubIds)
self.assertTrue(n >= n1)
self.assertTrue(n >= n2)
self.assertEqual(n, n1+n2)
# We won't do these first two, there are too few elements.
# check(self.micros)
# check(self.vols)
check(self.movies)
check(self.particles)
check(self.particles, n1=3, n2=5)
def addSuccess(self, test):
secs = self.toc()
sys.stderr.write(
"%s %s (%0.3f secs)\n" %
(pwutils.greenStr('[ RUN OK ]'), self.getTestName(test), secs))
def setUpClass(cls):
"""Prepare the data that we will use later on."""
print("\n", greenStr(" Set Up - Collect data ".center(75, '-')))
setupTestProject(cls) # defined in BaseTest, creates cls.proj
cls.xmippDataTest = DataSet.getDataSet('xmipp_tutorial')
def _importClasses(label, numClasses, randomSeed, numPart=None):
""" We import a set of classes with an alterate import particles
because there is not a import classes protocol
"""
pImpClasses = cls.proj.newProtocol(
ProtImportParticles,
filesPath=cls.xmippDataTest.getFile('particles'),
samplingRate=3.5)
pImpClasses.setObjLabel('Import %s' % label)
# we launch the protocol to obtain the particles
cls.proj.launchProtocol(pImpClasses, wait=True)
# fake importing classes handmade
partSet = pImpClasses.outputParticles
setOfClasses = pImpClasses._createSetOfClasses3D(partSet)
numOfPart = partSet.getSize() if numPart is None else numPart
partIds = list(partSet.getIdSet())
m = int(numOfPart / numClasses)
# random shuffle with a certain seed to get always the same classes
random.seed(randomSeed)
random.shuffle(partIds)
for clInx in list(range(numClasses)):
currIds = partIds[clInx * m:(clInx + 1) * m]
newClass = Class3D()
newClass.copyInfo(partSet)
newClass.setAcquisition(partSet.getAcquisition())
# newClass.setRepresentative(clRep.getRepresentative())
setOfClasses.append(newClass)
enabledClass = setOfClasses[newClass.getObjId()]
enabledClass.enableAppend()
for itemId in currIds:
item = partSet[itemId]
enabledClass.append(item)
setOfClasses.update(enabledClass)
# including the outputClasses as protocol output by hand
pImpClasses._defineOutputs(outputClasses=setOfClasses)
# relaunching the protocol to get the new output
cls.proj.launchProtocol(pImpClasses, wait=True)
return pImpClasses.outputClasses
cls.set1 = _importClasses('3 Classes I', 3, 65)
cls.set2 = _importClasses('3 Classes II', 3, 123)
cls.set3 = _importClasses('3 Classes III', 3, 256)
cls.set4 = _importClasses('3 Classes IV (not all part)', 3, 568, 70)
cls.set5 = _importClasses('5 Classes I', 5, 745)
cls.set6 = _importClasses('5 Classes II', 5, 1025)