def __init__(self,_DoShowAlignedImages, _DoShowAverage, _DoShowSpectra, _DoShowSOMSpectra, _ProtocolName, ): import os,sys,shutil scriptdir=os.path.split(os.path.dirname(os.popen('which xmipp_protocols','r').read()))[0]+'/protocols' sys.path.append(scriptdir) # add default search path import log # Import the corresponding protocol, get WorkingDir and go there pardir=os.path.abspath(os.getcwd()) shutil.copy(ProtocolName,'protocol.py') import protocol self._WorkingDir=protocol.WorkingDir os.chdir(self._WorkingDir) _LogDir=protocol.LogDir _ProjectDir=protocol.ProjectDir self._SelFileName=_ProjectDir+'/'+str(protocol.SelFileName) self._SomName=protocol.SomName self._SpectraName=protocol.SpectraName self.mylog=log.init_log_system(_ProjectDir, _LogDir, sys.argv[0], self._WorkingDir) if (_DoShowAlignedImages): self.visualize_AlignedImages(self._SelFileName) if (_DoShowAverage): self.visualize_Average(self._SelFileName) if (_DoShowSpectra): self.visualize_Spectra(self._SpectraName) if (_DoShowSOMSpectra): self.visualize_SOMSpectra(self._SomName,self._SpectraName) # Return to parent dir and remove protocol.py(c) os.chdir(pardir) if (os.path.exists('protocol.py')): os.remove('protocol.py') if (os.path.exists('protocol.pyc')): os.remove('protocol.pyc')
def __init__(self, InSelFile, InDocFile, MissingDocFile, WorkingDir, DoDeleteWorkingDir, ProjectDir, LogDir, OutputRootName, NumberOfIterations, DoKeepAngles, RestartIter, Symmetry, Dimension, MaximumResolution, ExtraParamsMLtomo, DoAlign, AngularSampling, AngularSearchRange, DoPerturb, NumberOfReferences, SeedsSelfile, InitialRegularization, NumberRegularizationSteps, MaskName, NumberOfThreads, DoParallel, NumberOfMpi, SystemFlavour): import os,sys,shutil scriptdir=os.path.split(os.path.dirname(os.popen('which xmipp_protocols','r').read()))[0]+'/protocols' sys.path.append(scriptdir) # add default search path import log,selfile self.InSelFile=InSelFile self.InDocFile=InDocFile self.MissingDocFile=MissingDocFile self.WorkingDir=WorkingDir self.DoDeleteWorkingDir=DoDeleteWorkingDir self.ProjectDir=ProjectDir self.LogDir=LogDir self.OutputRootName=OutputRootName self.NumberOfIterations=NumberOfIterations self.DoKeepAngles=DoKeepAngles self.RestartIter=RestartIter self.Symmetry=Symmetry self.Dimension=Dimension self.MaximumResolution=MaximumResolution self.ExtraParamsMLtomo=ExtraParamsMLtomo self.DoAlign=DoAlign self.AngularSampling=AngularSampling self.AngularSearchRange=AngularSearchRange self.DoPerturb=DoPerturb self.NumberOfReferences=NumberOfReferences self.SeedsSelfile=SeedsSelfile self.InitialRegularization=InitialRegularization self.NumberRegularizationSteps=NumberRegularizationSteps self.MaskName=MaskName self.NumberOfThreads=NumberOfThreads self.DoParallel=DoParallel self.NumberOfMpi=NumberOfMpi self.SystemFlavour=SystemFlavour # Setup logging self.log=log.init_log_system(self.ProjectDir, self.LogDir, sys.argv[0], self.WorkingDir) # This is not a restart if (self.RestartIter < 1): # Delete working directory if it exists, make a new one if (DoDeleteWorkingDir): if (self.WorkingDir==""): raise RuntimeError,"No working directory given" if os.path.exists(self.WorkingDir): shutil.rmtree(self.WorkingDir) if not os.path.exists(self.WorkingDir): os.makedirs(self.WorkingDir) # Backup script log.make_backup_of_script_file(sys.argv[0], os.path.abspath(self.WorkingDir)) # This protocol is executed from the ProjectDir self.execute_MLtomo()
def projection_matching_protocol_basic(_SelFileName, _ReferenceFileName, _WorkingDir, _ProjectDir, _MaskRadius, _InnerRadius, _OuterRadius, _AvailableMemory, _ResolSam, _NumberOfMpiProcesses, _NumberofIterations=30, _ContinueAtIteration=1, _CleanUpFiles=True, _LogDir="Logs", _DoMask=True, _DoSphericalMask=True, _MaskFileName='', _AngSamplingRateDeg='20x10 5x5 5x3', _MaxChangeInAngles='20x1000 5x20 5x9', _MaxChangeOffset='20x1000 10x10', _Search5DShift='20x5 1', _Search5DStep='20x2 1', _DocFileName='', _SymmetryGroup='c1', _MinimumCrossCorrelation='-1', _DiscardPercentage='0', _DoReconstruction=True, _PaddingFactor=1.0, _FourierMaxFrequencyOfInterest='0.35', _ConstantToAddToFiltration='0.35', _NumberOfThreads=1, _DoParallel=True, _MpiJobSize='10', _SystemFlavour=''): # Import libraries and add Xmipp libs to default search path import os, sys, shutil scriptdir = os.path.split( os.path.dirname(os.popen('which xmipp_protocols', 'r').read()))[0] + '/protocols' sys.path.append(scriptdir) import arg, log, logging, selfile import launch_job # global variables _ReferenceVolumeName = 'reference_volume.vol' _LibraryDir = "ReferenceLibrary" _ProjectLibraryRootName = _LibraryDir + "/ref" _ProjMatchDir = "ProjMatchClasses" _ProjMatchName = 'proj_match' _ProjMatchRootName = _ProjMatchDir + "/" + _ProjMatchName _DocFileWithOriginalAngles = 'original_angles.doc' _docfile_with_current_angles = 'current_angles.doc' _ReconstructedVolumeBase = "reconstruction" _FilteredReconstruction = "filtered_reconstruction" _WorkingDir = os.getcwd() + '/' + os.path.basename(_WorkingDir) _ReferenceFileName = os.path.abspath(_ReferenceFileName) _user_suplied_ReferenceVolume = _ReferenceFileName selfile_without_ext = (os.path.splitext(str( os.path.basename(_SelFileName))))[0] globalFourierMaxFrequencyOfInterest = float(_FourierMaxFrequencyOfInterest) _MySystemFlavour = _SystemFlavour _MyNumberOfMpiProcesses = _NumberOfMpiProcesses _MyMpiJobSize = _MpiJobSize _MyNumberOfThreads = _NumberOfThreads # Set up logging _mylog = log.init_log_system(_ProjectDir, _LogDir, sys.argv[0], _WorkingDir) # Uncomment next line to get Debug level logging _mylog.setLevel(logging.DEBUG) _mylog.debug("Debug level logging enabled") #input files should exists check_file_exists(_ReferenceFileName, _mylog) _NumberofIterations += 1 create_working_directory(_mylog, _WorkingDir, _ProjMatchDir, _LibraryDir) # Create a selfile with absolute pathname in the WorkingDir mysel = selfile.selfile() mysel.read(os.path.basename(_SelFileName)) newsel = mysel.make_abspath() _SelFileName = os.path.abspath(_WorkingDir + '/' + os.path.basename(_SelFileName)) newsel.write(_SelFileName) # Set _OuterRadius if (_OuterRadius < 0): xdim, ydim = newsel.imgSize() _OuterRadius = (xdim / 2) - 1 comment = " Outer radius set to: " + str(_OuterRadius) print '* ' + comment _mylog.info(comment) else: _OuterRadius = _OuterRadius # Create a docfile with the current angles in the WorkingDir if (_DocFileName == ''): params=' -i ' + _SelFileName + \ ' -o ' + _WorkingDir + '/' + \ _DocFileWithOriginalAngles launch_job.launch_job("xmipp_header_extract", params, _mylog, False, 1, 1, '') # Change to working dir os.chdir(_WorkingDir) _SelFileName = _WorkingDir + '/' + str(os.path.basename(_SelFileName)) ## ##LOOP ## #output of reconstruction cycle #first value given by user #these names are the input of the mask program #in general is the output of the reconstruction plus filtration _ReconstructedVolume = [] fill_name_vector("", _ReconstructedVolume, _NumberofIterations, _ReconstructedVolumeBase) _ReconstructedandfilteredVolume = [] fill_name_vector(_user_suplied_ReferenceVolume, _ReconstructedandfilteredVolume, _NumberofIterations, _FilteredReconstruction) # Optimal angles from previous iteration or user-provided at the beginning _DocFileInputAngles = [] fill_name_vector('../' + _DocFileWithOriginalAngles, _DocFileInputAngles, _NumberofIterations + 1, _docfile_with_current_angles) # Reconstructed and filtered volume of n-1 after masking called reference volume _ReferenceVolume = [] fill_name_vector("", _ReferenceVolume, _NumberofIterations, _ReferenceVolumeName) for _iteration_number in range(_ContinueAtIteration, _NumberofIterations): debug_string = "ITERATION: " + str(_iteration_number) print "*", debug_string _mylog.info(debug_string) # Create working dir for this iteration and go there Iteration_Working_Directory = _WorkingDir + '/Iter_' + str( _iteration_number) create_working_directory(_mylog, Iteration_Working_Directory, _ProjMatchDir, _LibraryDir) os.chdir(Iteration_Working_Directory) #### changed here for this stack -- Dmitry # Mask reference volume if _iteration_number == 1: execute_mask( _DoMask, _mylog, _ProjectDir, _ReconstructedandfilteredVolume[_iteration_number], #in _MaskFileName, _DoSphericalMask, _MaskRadius, _iteration_number, _ReferenceVolume[_iteration_number]) #out elif _DoReconstruction is True: execute_mask( _DoMask, _mylog, _ProjectDir, os.path.join( "../Iter_%d" % (_iteration_number - 1), "Iter_%d_reconstruction" % (_iteration_number - 1)), _MaskFileName, _DoSphericalMask, _MaskRadius, _iteration_number, os.path.join( "../Iter_%d" % (_iteration_number), "Iter_%d_reference_volume.vol" % (_iteration_number))) else: print "skipped masking, no reconstructed volume" if _DoReconstruction is False and (_iteration_number > 1): shutil.move(os.path.join("../Iter_%d" % (_iteration_number-1), "Iter_%d_reference_volume.vol" % (_iteration_number-1)), \ os.path.join("../Iter_%d" % (_iteration_number), "Iter_%d_reference_volume.vol" % (_iteration_number))) # Parameters for projection matching _AngSamplingRateDeg = arg.getComponentFromVector( _AngSamplingRateDeg, _iteration_number - 1) _MaxChangeOffset = arg.getComponentFromVector(_MaxChangeOffset, _iteration_number - 1) _MaxChangeInAngles = arg.getComponentFromVector( _MaxChangeInAngles, _iteration_number - 1) _Search5DShift = arg.getComponentFromVector(_Search5DShift, _iteration_number - 1) _Search5DStep = arg.getComponentFromVector(_Search5DStep, _iteration_number - 1) _MinimumCrossCorrelation = arg.getComponentFromVector( _MinimumCrossCorrelation, _iteration_number - 1) _DiscardPercentage = arg.getComponentFromVector( _DiscardPercentage, _iteration_number - 1) execute_projection_matching( _mylog, _ProjectDir, _ReferenceVolume[_iteration_number], _MaskFileName, _DocFileInputAngles[_iteration_number], _DocFileInputAngles[_iteration_number + 1], _AngSamplingRateDeg, _InnerRadius, _OuterRadius, _Search5DShift, _Search5DStep, _MaxChangeOffset, _MaxChangeInAngles, _MinimumCrossCorrelation, _DiscardPercentage, _DoParallel, _MyNumberOfMpiProcesses, _MyNumberOfThreads, _MySystemFlavour, _MyMpiJobSize, _WorkingDir, _SymmetryGroup, _AvailableMemory, _iteration_number, _ProjectLibraryRootName, _ProjMatchRootName) if _DoReconstruction is True: execute_reconstruction( _mylog, _SelFileName, _iteration_number, _DoParallel, _MyNumberOfMpiProcesses, _MyNumberOfThreads, _MySystemFlavour, _MyMpiJobSize, globalFourierMaxFrequencyOfInterest, _SymmetryGroup, _ReconstructedVolume[_iteration_number], _PaddingFactor) else: print "skipped reconstruction" _ConstantToAddToFiltration = arg.getComponentFromVector( _ConstantToAddToFiltration, _iteration_number - 1) # Remove all class averages and reference projections if (_CleanUpFiles): execute_cleanup(_mylog, True, True, _ProjMatchDir, _LibraryDir)
def projection_matching_protocol_basic( _SelFileName, _ReferenceFileName, _WorkingDir, _ProjectDir, _MaskRadius, _InnerRadius, _OuterRadius, _AvailableMemory, _ResolSam, _NumberOfMpiProcesses, _NumberofIterations=30, _ContinueAtIteration=1, _CleanUpFiles=True, _LogDir="Logs", _DoMask=True, _DoSphericalMask=True, _MaskFileName='', _AngSamplingRateDeg='20x10 5x5 5x3', _MaxChangeInAngles='20x1000 5x20 5x9', _MaxChangeOffset='20x1000 10x10', _Search5DShift='20x5 1', _Search5DStep='20x2 1', _DocFileName='', _SymmetryGroup='c1', _MinimumCrossCorrelation='-1', _DiscardPercentage='0', _DoReconstruction=True, _PaddingFactor=1.0, _FourierMaxFrequencyOfInterest='0.35', _ConstantToAddToFiltration='0.35', _NumberOfThreads=1, _DoParallel=True, _MpiJobSize='10', _SystemFlavour='' ): # Import libraries and add Xmipp libs to default search path import os,sys,shutil scriptdir=os.path.split(os.path.dirname(os.popen('which xmipp_protocols','r').read()))[0]+'/protocols' sys.path.append(scriptdir) import arg,log,logging,selfile import launch_job # global variables _ReferenceVolumeName='reference_volume.vol' _LibraryDir = "ReferenceLibrary" _ProjectLibraryRootName= _LibraryDir + "/ref" _ProjMatchDir = "ProjMatchClasses" _ProjMatchName = 'proj_match' _ProjMatchRootName= _ProjMatchDir + "/" + _ProjMatchName _DocFileWithOriginalAngles='original_angles.doc' _docfile_with_current_angles='current_angles.doc' _ReconstructedVolumeBase="reconstruction" _FilteredReconstruction="filtered_reconstruction" _WorkingDir=os.getcwd()+'/'+os.path.basename(_WorkingDir) _ReferenceFileName=os.path.abspath(_ReferenceFileName) _user_suplied_ReferenceVolume=_ReferenceFileName selfile_without_ext=(os.path.splitext(str(os.path.basename(_SelFileName))))[0] globalFourierMaxFrequencyOfInterest=float(_FourierMaxFrequencyOfInterest) _MySystemFlavour=_SystemFlavour _MyNumberOfMpiProcesses=_NumberOfMpiProcesses _MyMpiJobSize =_MpiJobSize _MyNumberOfThreads =_NumberOfThreads # Set up logging _mylog=log.init_log_system(_ProjectDir, _LogDir, sys.argv[0], _WorkingDir) # Uncomment next line to get Debug level logging _mylog.setLevel(logging.DEBUG) _mylog.debug("Debug level logging enabled") #input files should exists check_file_exists(_ReferenceFileName,_mylog) _NumberofIterations +=1; create_working_directory(_mylog, _WorkingDir, _ProjMatchDir, _LibraryDir) # Create a selfile with absolute pathname in the WorkingDir mysel=selfile.selfile() mysel.read(os.path.basename(_SelFileName)) newsel=mysel.make_abspath() _SelFileName=os.path.abspath(_WorkingDir + '/' + os.path.basename(_SelFileName)) newsel.write(_SelFileName) # Set _OuterRadius if (_OuterRadius < 0): xdim,ydim=newsel.imgSize() _OuterRadius = (xdim/2) - 1 comment = " Outer radius set to: " + str(_OuterRadius) print '* ' + comment _mylog.info(comment) else: _OuterRadius=_OuterRadius # Create a docfile with the current angles in the WorkingDir if (_DocFileName==''): params=' -i ' + _SelFileName + \ ' -o ' + _WorkingDir + '/' + \ _DocFileWithOriginalAngles launch_job.launch_job("xmipp_header_extract", params, _mylog, False,1,1,'') # Change to working dir os.chdir(_WorkingDir) _SelFileName=_WorkingDir+'/' + str(os.path.basename(_SelFileName)) ## ##LOOP ## #output of reconstruction cycle #first value given by user #these names are the input of the mask program #in general is the output of the reconstruction plus filtration _ReconstructedVolume=[] fill_name_vector("", _ReconstructedVolume, _NumberofIterations, _ReconstructedVolumeBase) _ReconstructedandfilteredVolume=[] fill_name_vector(_user_suplied_ReferenceVolume, _ReconstructedandfilteredVolume, _NumberofIterations, _FilteredReconstruction) # Optimal angles from previous iteration or user-provided at the beginning _DocFileInputAngles=[] fill_name_vector('../'+_DocFileWithOriginalAngles, _DocFileInputAngles, _NumberofIterations+1, _docfile_with_current_angles) # Reconstructed and filtered volume of n-1 after masking called reference volume _ReferenceVolume=[] fill_name_vector("", _ReferenceVolume, _NumberofIterations, _ReferenceVolumeName) for _iteration_number in range(_ContinueAtIteration, _NumberofIterations): debug_string = "ITERATION: " + str(_iteration_number) print "*", debug_string _mylog.info(debug_string) # Create working dir for this iteration and go there Iteration_Working_Directory=_WorkingDir+'/Iter_' + str(_iteration_number) create_working_directory(_mylog, Iteration_Working_Directory, _ProjMatchDir, _LibraryDir) os.chdir(Iteration_Working_Directory) #### changed here for this stack -- Dmitry # Mask reference volume if _iteration_number == 1: execute_mask(_DoMask, _mylog, _ProjectDir, _ReconstructedandfilteredVolume[_iteration_number],#in _MaskFileName, _DoSphericalMask, _MaskRadius, _iteration_number, _ReferenceVolume[_iteration_number])#out elif _DoReconstruction is True: execute_mask(_DoMask, _mylog, _ProjectDir, os.path.join("../Iter_%d" % (_iteration_number-1), "Iter_%d_reconstruction" % (_iteration_number-1)), _MaskFileName, _DoSphericalMask, _MaskRadius, _iteration_number, os.path.join("../Iter_%d" % (_iteration_number), "Iter_%d_reference_volume.vol" % (_iteration_number))) else: print "skipped masking, no reconstructed volume" if _DoReconstruction is False and (_iteration_number > 1): shutil.move(os.path.join("../Iter_%d" % (_iteration_number-1), "Iter_%d_reference_volume.vol" % (_iteration_number-1)), \ os.path.join("../Iter_%d" % (_iteration_number), "Iter_%d_reference_volume.vol" % (_iteration_number))) # Parameters for projection matching _AngSamplingRateDeg=arg.getComponentFromVector(_AngSamplingRateDeg, _iteration_number-1) _MaxChangeOffset=arg.getComponentFromVector(_MaxChangeOffset, _iteration_number-1) _MaxChangeInAngles=arg.getComponentFromVector(_MaxChangeInAngles, _iteration_number-1) _Search5DShift=arg.getComponentFromVector(_Search5DShift, _iteration_number-1) _Search5DStep=arg.getComponentFromVector(_Search5DStep, _iteration_number-1) _MinimumCrossCorrelation=arg.getComponentFromVector(_MinimumCrossCorrelation, _iteration_number-1) _DiscardPercentage=arg.getComponentFromVector(_DiscardPercentage, _iteration_number-1) execute_projection_matching(_mylog, _ProjectDir, _ReferenceVolume[_iteration_number], _MaskFileName, _DocFileInputAngles[_iteration_number], _DocFileInputAngles[_iteration_number+1], _AngSamplingRateDeg, _InnerRadius, _OuterRadius, _Search5DShift, _Search5DStep, _MaxChangeOffset, _MaxChangeInAngles, _MinimumCrossCorrelation, _DiscardPercentage, _DoParallel, _MyNumberOfMpiProcesses, _MyNumberOfThreads, _MySystemFlavour, _MyMpiJobSize, _WorkingDir, _SymmetryGroup, _AvailableMemory, _iteration_number, _ProjectLibraryRootName, _ProjMatchRootName ) if _DoReconstruction is True: execute_reconstruction(_mylog, _SelFileName, _iteration_number, _DoParallel, _MyNumberOfMpiProcesses, _MyNumberOfThreads, _MySystemFlavour, _MyMpiJobSize, globalFourierMaxFrequencyOfInterest, _SymmetryGroup, _ReconstructedVolume[_iteration_number], _PaddingFactor ) else: print "skipped reconstruction" _ConstantToAddToFiltration=arg.getComponentFromVector(_ConstantToAddToFiltration, _iteration_number-1) # Remove all class averages and reference projections if (_CleanUpFiles): execute_cleanup(_mylog, True, True, _ProjMatchDir, _LibraryDir)
def _reinit_log_system(self): self._logger.info({'action': 'reload config, restart log system'}) self._config = utils.load_config(self._config_path, self._logger) log.init_log_system(self._config['logging'])
def _init_log_system(self): log.init_log_system(self._config['logging'])
def __init__(self, InSelFile, WorkingDir, ProjectDir, LogDir, ReferenceImage, NumberOfIterations, DoFilter, Highpass, Lowpass, NumberOfMpi, SystemFlavour): scriptdir=os.path.split(os.path.dirname(os.popen('which xmipp_protocols','r').read()))[0]+'/protocols' sys.path.append(scriptdir) # add default search path import log self.WorkingDir=WorkingDir self.ProjectDir=ProjectDir self.InSelFile=InSelFile self.ReferenceImage=ReferenceImage self.NumberOfIterations=NumberOfIterations self.DoFilter=DoFilter self.Highpass=Highpass self.Lowpass=Lowpass self.NumberOfMpi=NumberOfMpi self.SystemFlavour=SystemFlavour # Setup logging self.log=log.init_log_system(self.ProjectDir, LogDir, sys.argv[0], self.WorkingDir) # Create directory if does not exist if not os.path.exists(self.WorkingDir): os.makedirs(self.WorkingDir) self.doStep1=True self.doStep2=True # Save parameters and compare to possible previous runs self.saveAndCompareParameters([ "InSelFile", "ReferenceImage", "NumberOfIterations", "DoFilter", "Highpass", "Lowpass"]); # Backup script log.make_backup_of_script_file(sys.argv[0], os.path.abspath(self.WorkingDir)) # Update status fh=open(self.WorkingDir + "/status.txt", "a") fh.write("Step 0: Process started at " + time.asctime() + "\n") fh.close() # Run self.preprocess() self.execute_CLalign2D() self.postprocess() fh=open(self.WorkingDir + "/status.txt", "a") fh.write("Step F: Process finished at " + time.asctime() + "\n") fh.close()