def __init__(self,_DoShowAlignedImages,
_DoShowAverage,
_DoShowSpectra,
_DoShowSOMSpectra,
_ProtocolName,
):
import os,sys,shutil
scriptdir=os.path.split(os.path.dirname(os.popen('which xmipp_protocols','r').read()))[0]+'/protocols'
sys.path.append(scriptdir) # add default search path
import log
# Import the corresponding protocol, get WorkingDir and go there
pardir=os.path.abspath(os.getcwd())
shutil.copy(ProtocolName,'protocol.py')
import protocol
self._WorkingDir=protocol.WorkingDir
os.chdir(self._WorkingDir)
_LogDir=protocol.LogDir
_ProjectDir=protocol.ProjectDir
self._SelFileName=_ProjectDir+'/'+str(protocol.SelFileName)
self._SomName=protocol.SomName
self._SpectraName=protocol.SpectraName
self.mylog=log.init_log_system(_ProjectDir,
_LogDir,
sys.argv[0],
self._WorkingDir)
if (_DoShowAlignedImages):
self.visualize_AlignedImages(self._SelFileName)
if (_DoShowAverage):
self.visualize_Average(self._SelFileName)
if (_DoShowSpectra):
self.visualize_Spectra(self._SpectraName)
if (_DoShowSOMSpectra):
self.visualize_SOMSpectra(self._SomName,self._SpectraName)
# Return to parent dir and remove protocol.py(c)
os.chdir(pardir)
if (os.path.exists('protocol.py')):
os.remove('protocol.py')
if (os.path.exists('protocol.pyc')):
os.remove('protocol.pyc')
def __init__(self,
InSelFile,
InDocFile,
MissingDocFile,
WorkingDir,
DoDeleteWorkingDir,
ProjectDir,
LogDir,
OutputRootName,
NumberOfIterations,
DoKeepAngles,
RestartIter,
Symmetry,
Dimension,
MaximumResolution,
ExtraParamsMLtomo,
DoAlign,
AngularSampling,
AngularSearchRange,
DoPerturb,
NumberOfReferences,
SeedsSelfile,
InitialRegularization,
NumberRegularizationSteps,
MaskName,
NumberOfThreads,
DoParallel,
NumberOfMpi,
SystemFlavour):
import os,sys,shutil
scriptdir=os.path.split(os.path.dirname(os.popen('which xmipp_protocols','r').read()))[0]+'/protocols'
sys.path.append(scriptdir) # add default search path
import log,selfile
self.InSelFile=InSelFile
self.InDocFile=InDocFile
self.MissingDocFile=MissingDocFile
self.WorkingDir=WorkingDir
self.DoDeleteWorkingDir=DoDeleteWorkingDir
self.ProjectDir=ProjectDir
self.LogDir=LogDir
self.OutputRootName=OutputRootName
self.NumberOfIterations=NumberOfIterations
self.DoKeepAngles=DoKeepAngles
self.RestartIter=RestartIter
self.Symmetry=Symmetry
self.Dimension=Dimension
self.MaximumResolution=MaximumResolution
self.ExtraParamsMLtomo=ExtraParamsMLtomo
self.DoAlign=DoAlign
self.AngularSampling=AngularSampling
self.AngularSearchRange=AngularSearchRange
self.DoPerturb=DoPerturb
self.NumberOfReferences=NumberOfReferences
self.SeedsSelfile=SeedsSelfile
self.InitialRegularization=InitialRegularization
self.NumberRegularizationSteps=NumberRegularizationSteps
self.MaskName=MaskName
self.NumberOfThreads=NumberOfThreads
self.DoParallel=DoParallel
self.NumberOfMpi=NumberOfMpi
self.SystemFlavour=SystemFlavour
# Setup logging
self.log=log.init_log_system(self.ProjectDir,
self.LogDir,
sys.argv[0],
self.WorkingDir)
# This is not a restart
if (self.RestartIter < 1):
# Delete working directory if it exists, make a new one
if (DoDeleteWorkingDir):
if (self.WorkingDir==""):
raise RuntimeError,"No working directory given"
if os.path.exists(self.WorkingDir):
shutil.rmtree(self.WorkingDir)
if not os.path.exists(self.WorkingDir):
os.makedirs(self.WorkingDir)
# Backup script
log.make_backup_of_script_file(sys.argv[0],
os.path.abspath(self.WorkingDir))
# This protocol is executed from the ProjectDir
self.execute_MLtomo()
def projection_matching_protocol_basic(_SelFileName,
_ReferenceFileName,
_WorkingDir,
_ProjectDir,
_MaskRadius,
_InnerRadius,
_OuterRadius,
_AvailableMemory,
_ResolSam,
_NumberOfMpiProcesses,
_NumberofIterations=30,
_ContinueAtIteration=1,
_CleanUpFiles=True,
_LogDir="Logs",
_DoMask=True,
_DoSphericalMask=True,
_MaskFileName='',
_AngSamplingRateDeg='20x10 5x5 5x3',
_MaxChangeInAngles='20x1000 5x20 5x9',
_MaxChangeOffset='20x1000 10x10',
_Search5DShift='20x5 1',
_Search5DStep='20x2 1',
_DocFileName='',
_SymmetryGroup='c1',
_MinimumCrossCorrelation='-1',
_DiscardPercentage='0',
_DoReconstruction=True,
_PaddingFactor=1.0,
_FourierMaxFrequencyOfInterest='0.35',
_ConstantToAddToFiltration='0.35',
_NumberOfThreads=1,
_DoParallel=True,
_MpiJobSize='10',
_SystemFlavour=''):
# Import libraries and add Xmipp libs to default search path
import os, sys, shutil
scriptdir = os.path.split(
os.path.dirname(os.popen('which xmipp_protocols',
'r').read()))[0] + '/protocols'
sys.path.append(scriptdir)
import arg, log, logging, selfile
import launch_job
# global variables
_ReferenceVolumeName = 'reference_volume.vol'
_LibraryDir = "ReferenceLibrary"
_ProjectLibraryRootName = _LibraryDir + "/ref"
_ProjMatchDir = "ProjMatchClasses"
_ProjMatchName = 'proj_match'
_ProjMatchRootName = _ProjMatchDir + "/" + _ProjMatchName
_DocFileWithOriginalAngles = 'original_angles.doc'
_docfile_with_current_angles = 'current_angles.doc'
_ReconstructedVolumeBase = "reconstruction"
_FilteredReconstruction = "filtered_reconstruction"
_WorkingDir = os.getcwd() + '/' + os.path.basename(_WorkingDir)
_ReferenceFileName = os.path.abspath(_ReferenceFileName)
_user_suplied_ReferenceVolume = _ReferenceFileName
selfile_without_ext = (os.path.splitext(str(
os.path.basename(_SelFileName))))[0]
globalFourierMaxFrequencyOfInterest = float(_FourierMaxFrequencyOfInterest)
_MySystemFlavour = _SystemFlavour
_MyNumberOfMpiProcesses = _NumberOfMpiProcesses
_MyMpiJobSize = _MpiJobSize
_MyNumberOfThreads = _NumberOfThreads
# Set up logging
_mylog = log.init_log_system(_ProjectDir, _LogDir, sys.argv[0],
_WorkingDir)
# Uncomment next line to get Debug level logging
_mylog.setLevel(logging.DEBUG)
_mylog.debug("Debug level logging enabled")
#input files should exists
check_file_exists(_ReferenceFileName, _mylog)
_NumberofIterations += 1
create_working_directory(_mylog, _WorkingDir, _ProjMatchDir, _LibraryDir)
# Create a selfile with absolute pathname in the WorkingDir
mysel = selfile.selfile()
mysel.read(os.path.basename(_SelFileName))
newsel = mysel.make_abspath()
_SelFileName = os.path.abspath(_WorkingDir + '/' +
os.path.basename(_SelFileName))
newsel.write(_SelFileName)
# Set _OuterRadius
if (_OuterRadius < 0):
xdim, ydim = newsel.imgSize()
_OuterRadius = (xdim / 2) - 1
comment = " Outer radius set to: " + str(_OuterRadius)
print '* ' + comment
_mylog.info(comment)
else:
_OuterRadius = _OuterRadius
# Create a docfile with the current angles in the WorkingDir
if (_DocFileName == ''):
params=' -i ' + _SelFileName + \
' -o ' + _WorkingDir + '/' + \
_DocFileWithOriginalAngles
launch_job.launch_job("xmipp_header_extract", params, _mylog, False, 1,
1, '')
# Change to working dir
os.chdir(_WorkingDir)
_SelFileName = _WorkingDir + '/' + str(os.path.basename(_SelFileName))
##
##LOOP
##
#output of reconstruction cycle
#first value given by user
#these names are the input of the mask program
#in general is the output of the reconstruction plus filtration
_ReconstructedVolume = []
fill_name_vector("", _ReconstructedVolume, _NumberofIterations,
_ReconstructedVolumeBase)
_ReconstructedandfilteredVolume = []
fill_name_vector(_user_suplied_ReferenceVolume,
_ReconstructedandfilteredVolume, _NumberofIterations,
_FilteredReconstruction)
# Optimal angles from previous iteration or user-provided at the beginning
_DocFileInputAngles = []
fill_name_vector('../' + _DocFileWithOriginalAngles, _DocFileInputAngles,
_NumberofIterations + 1, _docfile_with_current_angles)
# Reconstructed and filtered volume of n-1 after masking called reference volume
_ReferenceVolume = []
fill_name_vector("", _ReferenceVolume, _NumberofIterations,
_ReferenceVolumeName)
for _iteration_number in range(_ContinueAtIteration, _NumberofIterations):
debug_string = "ITERATION: " + str(_iteration_number)
print "*", debug_string
_mylog.info(debug_string)
# Create working dir for this iteration and go there
Iteration_Working_Directory = _WorkingDir + '/Iter_' + str(
_iteration_number)
create_working_directory(_mylog, Iteration_Working_Directory,
_ProjMatchDir, _LibraryDir)
os.chdir(Iteration_Working_Directory)
#### changed here for this stack -- Dmitry
# Mask reference volume
if _iteration_number == 1:
execute_mask(
_DoMask,
_mylog,
_ProjectDir,
_ReconstructedandfilteredVolume[_iteration_number], #in
_MaskFileName,
_DoSphericalMask,
_MaskRadius,
_iteration_number,
_ReferenceVolume[_iteration_number]) #out
elif _DoReconstruction is True:
execute_mask(
_DoMask, _mylog, _ProjectDir,
os.path.join(
"../Iter_%d" % (_iteration_number - 1),
"Iter_%d_reconstruction" % (_iteration_number - 1)),
_MaskFileName, _DoSphericalMask, _MaskRadius,
_iteration_number,
os.path.join(
"../Iter_%d" % (_iteration_number),
"Iter_%d_reference_volume.vol" % (_iteration_number)))
else:
print "skipped masking, no reconstructed volume"
if _DoReconstruction is False and (_iteration_number > 1):
shutil.move(os.path.join("../Iter_%d" % (_iteration_number-1), "Iter_%d_reference_volume.vol" % (_iteration_number-1)), \
os.path.join("../Iter_%d" % (_iteration_number), "Iter_%d_reference_volume.vol" % (_iteration_number)))
# Parameters for projection matching
_AngSamplingRateDeg = arg.getComponentFromVector(
_AngSamplingRateDeg, _iteration_number - 1)
_MaxChangeOffset = arg.getComponentFromVector(_MaxChangeOffset,
_iteration_number - 1)
_MaxChangeInAngles = arg.getComponentFromVector(
_MaxChangeInAngles, _iteration_number - 1)
_Search5DShift = arg.getComponentFromVector(_Search5DShift,
_iteration_number - 1)
_Search5DStep = arg.getComponentFromVector(_Search5DStep,
_iteration_number - 1)
_MinimumCrossCorrelation = arg.getComponentFromVector(
_MinimumCrossCorrelation, _iteration_number - 1)
_DiscardPercentage = arg.getComponentFromVector(
_DiscardPercentage, _iteration_number - 1)
execute_projection_matching(
_mylog, _ProjectDir, _ReferenceVolume[_iteration_number],
_MaskFileName, _DocFileInputAngles[_iteration_number],
_DocFileInputAngles[_iteration_number + 1], _AngSamplingRateDeg,
_InnerRadius, _OuterRadius, _Search5DShift, _Search5DStep,
_MaxChangeOffset, _MaxChangeInAngles, _MinimumCrossCorrelation,
_DiscardPercentage, _DoParallel, _MyNumberOfMpiProcesses,
_MyNumberOfThreads, _MySystemFlavour, _MyMpiJobSize, _WorkingDir,
_SymmetryGroup, _AvailableMemory, _iteration_number,
_ProjectLibraryRootName, _ProjMatchRootName)
if _DoReconstruction is True:
execute_reconstruction(
_mylog, _SelFileName, _iteration_number, _DoParallel,
_MyNumberOfMpiProcesses, _MyNumberOfThreads, _MySystemFlavour,
_MyMpiJobSize, globalFourierMaxFrequencyOfInterest,
_SymmetryGroup, _ReconstructedVolume[_iteration_number],
_PaddingFactor)
else:
print "skipped reconstruction"
_ConstantToAddToFiltration = arg.getComponentFromVector(
_ConstantToAddToFiltration, _iteration_number - 1)
# Remove all class averages and reference projections
if (_CleanUpFiles):
execute_cleanup(_mylog, True, True, _ProjMatchDir, _LibraryDir)
def projection_matching_protocol_basic(
_SelFileName,
_ReferenceFileName,
_WorkingDir,
_ProjectDir,
_MaskRadius,
_InnerRadius,
_OuterRadius,
_AvailableMemory,
_ResolSam,
_NumberOfMpiProcesses,
_NumberofIterations=30,
_ContinueAtIteration=1,
_CleanUpFiles=True,
_LogDir="Logs",
_DoMask=True,
_DoSphericalMask=True,
_MaskFileName='',
_AngSamplingRateDeg='20x10 5x5 5x3',
_MaxChangeInAngles='20x1000 5x20 5x9',
_MaxChangeOffset='20x1000 10x10',
_Search5DShift='20x5 1',
_Search5DStep='20x2 1',
_DocFileName='',
_SymmetryGroup='c1',
_MinimumCrossCorrelation='-1',
_DiscardPercentage='0',
_DoReconstruction=True,
_PaddingFactor=1.0,
_FourierMaxFrequencyOfInterest='0.35',
_ConstantToAddToFiltration='0.35',
_NumberOfThreads=1,
_DoParallel=True,
_MpiJobSize='10',
_SystemFlavour=''
):
# Import libraries and add Xmipp libs to default search path
import os,sys,shutil
scriptdir=os.path.split(os.path.dirname(os.popen('which xmipp_protocols','r').read()))[0]+'/protocols'
sys.path.append(scriptdir)
import arg,log,logging,selfile
import launch_job
# global variables
_ReferenceVolumeName='reference_volume.vol'
_LibraryDir = "ReferenceLibrary"
_ProjectLibraryRootName= _LibraryDir + "/ref"
_ProjMatchDir = "ProjMatchClasses"
_ProjMatchName = 'proj_match'
_ProjMatchRootName= _ProjMatchDir + "/" + _ProjMatchName
_DocFileWithOriginalAngles='original_angles.doc'
_docfile_with_current_angles='current_angles.doc'
_ReconstructedVolumeBase="reconstruction"
_FilteredReconstruction="filtered_reconstruction"
_WorkingDir=os.getcwd()+'/'+os.path.basename(_WorkingDir)
_ReferenceFileName=os.path.abspath(_ReferenceFileName)
_user_suplied_ReferenceVolume=_ReferenceFileName
selfile_without_ext=(os.path.splitext(str(os.path.basename(_SelFileName))))[0]
globalFourierMaxFrequencyOfInterest=float(_FourierMaxFrequencyOfInterest)
_MySystemFlavour=_SystemFlavour
_MyNumberOfMpiProcesses=_NumberOfMpiProcesses
_MyMpiJobSize =_MpiJobSize
_MyNumberOfThreads =_NumberOfThreads
# Set up logging
_mylog=log.init_log_system(_ProjectDir, _LogDir, sys.argv[0], _WorkingDir)
# Uncomment next line to get Debug level logging
_mylog.setLevel(logging.DEBUG)
_mylog.debug("Debug level logging enabled")
#input files should exists
check_file_exists(_ReferenceFileName,_mylog)
_NumberofIterations +=1;
create_working_directory(_mylog, _WorkingDir, _ProjMatchDir, _LibraryDir)
# Create a selfile with absolute pathname in the WorkingDir
mysel=selfile.selfile()
mysel.read(os.path.basename(_SelFileName))
newsel=mysel.make_abspath()
_SelFileName=os.path.abspath(_WorkingDir + '/' + os.path.basename(_SelFileName))
newsel.write(_SelFileName)
# Set _OuterRadius
if (_OuterRadius < 0):
xdim,ydim=newsel.imgSize()
_OuterRadius = (xdim/2) - 1
comment = " Outer radius set to: " + str(_OuterRadius)
print '* ' + comment
_mylog.info(comment)
else:
_OuterRadius=_OuterRadius
# Create a docfile with the current angles in the WorkingDir
if (_DocFileName==''):
params=' -i ' + _SelFileName + \
' -o ' + _WorkingDir + '/' + \
_DocFileWithOriginalAngles
launch_job.launch_job("xmipp_header_extract", params, _mylog, False,1,1,'')
# Change to working dir
os.chdir(_WorkingDir)
_SelFileName=_WorkingDir+'/' + str(os.path.basename(_SelFileName))
##
##LOOP
##
#output of reconstruction cycle
#first value given by user
#these names are the input of the mask program
#in general is the output of the reconstruction plus filtration
_ReconstructedVolume=[]
fill_name_vector("",
_ReconstructedVolume,
_NumberofIterations,
_ReconstructedVolumeBase)
_ReconstructedandfilteredVolume=[]
fill_name_vector(_user_suplied_ReferenceVolume,
_ReconstructedandfilteredVolume,
_NumberofIterations,
_FilteredReconstruction)
# Optimal angles from previous iteration or user-provided at the beginning
_DocFileInputAngles=[]
fill_name_vector('../'+_DocFileWithOriginalAngles,
_DocFileInputAngles,
_NumberofIterations+1,
_docfile_with_current_angles)
# Reconstructed and filtered volume of n-1 after masking called reference volume
_ReferenceVolume=[]
fill_name_vector("",
_ReferenceVolume,
_NumberofIterations,
_ReferenceVolumeName)
for _iteration_number in range(_ContinueAtIteration, _NumberofIterations):
debug_string = "ITERATION: " + str(_iteration_number)
print "*", debug_string
_mylog.info(debug_string)
# Create working dir for this iteration and go there
Iteration_Working_Directory=_WorkingDir+'/Iter_' + str(_iteration_number)
create_working_directory(_mylog, Iteration_Working_Directory, _ProjMatchDir, _LibraryDir)
os.chdir(Iteration_Working_Directory)
#### changed here for this stack -- Dmitry
# Mask reference volume
if _iteration_number == 1:
execute_mask(_DoMask,
_mylog,
_ProjectDir,
_ReconstructedandfilteredVolume[_iteration_number],#in
_MaskFileName,
_DoSphericalMask,
_MaskRadius,
_iteration_number,
_ReferenceVolume[_iteration_number])#out
elif _DoReconstruction is True:
execute_mask(_DoMask,
_mylog,
_ProjectDir,
os.path.join("../Iter_%d" % (_iteration_number-1), "Iter_%d_reconstruction" % (_iteration_number-1)),
_MaskFileName,
_DoSphericalMask,
_MaskRadius,
_iteration_number,
os.path.join("../Iter_%d" % (_iteration_number), "Iter_%d_reference_volume.vol" % (_iteration_number)))
else:
print "skipped masking, no reconstructed volume"
if _DoReconstruction is False and (_iteration_number > 1):
shutil.move(os.path.join("../Iter_%d" % (_iteration_number-1), "Iter_%d_reference_volume.vol" % (_iteration_number-1)), \
os.path.join("../Iter_%d" % (_iteration_number), "Iter_%d_reference_volume.vol" % (_iteration_number)))
# Parameters for projection matching
_AngSamplingRateDeg=arg.getComponentFromVector(_AngSamplingRateDeg, _iteration_number-1)
_MaxChangeOffset=arg.getComponentFromVector(_MaxChangeOffset, _iteration_number-1)
_MaxChangeInAngles=arg.getComponentFromVector(_MaxChangeInAngles, _iteration_number-1)
_Search5DShift=arg.getComponentFromVector(_Search5DShift, _iteration_number-1)
_Search5DStep=arg.getComponentFromVector(_Search5DStep, _iteration_number-1)
_MinimumCrossCorrelation=arg.getComponentFromVector(_MinimumCrossCorrelation, _iteration_number-1)
_DiscardPercentage=arg.getComponentFromVector(_DiscardPercentage, _iteration_number-1)
execute_projection_matching(_mylog,
_ProjectDir,
_ReferenceVolume[_iteration_number],
_MaskFileName,
_DocFileInputAngles[_iteration_number],
_DocFileInputAngles[_iteration_number+1],
_AngSamplingRateDeg,
_InnerRadius,
_OuterRadius,
_Search5DShift,
_Search5DStep,
_MaxChangeOffset,
_MaxChangeInAngles,
_MinimumCrossCorrelation,
_DiscardPercentage,
_DoParallel,
_MyNumberOfMpiProcesses,
_MyNumberOfThreads,
_MySystemFlavour,
_MyMpiJobSize,
_WorkingDir,
_SymmetryGroup,
_AvailableMemory,
_iteration_number,
_ProjectLibraryRootName,
_ProjMatchRootName
)
if _DoReconstruction is True:
execute_reconstruction(_mylog,
_SelFileName,
_iteration_number,
_DoParallel,
_MyNumberOfMpiProcesses,
_MyNumberOfThreads,
_MySystemFlavour,
_MyMpiJobSize,
globalFourierMaxFrequencyOfInterest,
_SymmetryGroup,
_ReconstructedVolume[_iteration_number],
_PaddingFactor
)
else:
print "skipped reconstruction"
_ConstantToAddToFiltration=arg.getComponentFromVector(_ConstantToAddToFiltration, _iteration_number-1)
# Remove all class averages and reference projections
if (_CleanUpFiles):
execute_cleanup(_mylog, True, True, _ProjMatchDir, _LibraryDir)
def _reinit_log_system(self):
self._logger.info({'action': 'reload config, restart log system'})
self._config = utils.load_config(self._config_path, self._logger)
log.init_log_system(self._config['logging'])
def _init_log_system(self):
log.init_log_system(self._config['logging'])
def __init__(self,
InSelFile,
WorkingDir,
ProjectDir,
LogDir,
ReferenceImage,
NumberOfIterations,
DoFilter,
Highpass,
Lowpass,
NumberOfMpi,
SystemFlavour):
scriptdir=os.path.split(os.path.dirname(os.popen('which xmipp_protocols','r').read()))[0]+'/protocols'
sys.path.append(scriptdir) # add default search path
import log
self.WorkingDir=WorkingDir
self.ProjectDir=ProjectDir
self.InSelFile=InSelFile
self.ReferenceImage=ReferenceImage
self.NumberOfIterations=NumberOfIterations
self.DoFilter=DoFilter
self.Highpass=Highpass
self.Lowpass=Lowpass
self.NumberOfMpi=NumberOfMpi
self.SystemFlavour=SystemFlavour
# Setup logging
self.log=log.init_log_system(self.ProjectDir,
LogDir,
sys.argv[0],
self.WorkingDir)
# Create directory if does not exist
if not os.path.exists(self.WorkingDir):
os.makedirs(self.WorkingDir)
self.doStep1=True
self.doStep2=True
# Save parameters and compare to possible previous runs
self.saveAndCompareParameters([
"InSelFile",
"ReferenceImage",
"NumberOfIterations",
"DoFilter",
"Highpass",
"Lowpass"]);
# Backup script
log.make_backup_of_script_file(sys.argv[0],
os.path.abspath(self.WorkingDir))
# Update status
fh=open(self.WorkingDir + "/status.txt", "a")
fh.write("Step 0: Process started at " + time.asctime() + "\n")
fh.close()
# Run
self.preprocess()
self.execute_CLalign2D()
self.postprocess()
fh=open(self.WorkingDir + "/status.txt", "a")
fh.write("Step F: Process finished at " + time.asctime() + "\n")
fh.close()
