def changeDir(log, path):
""" Change to Directory """
try:
os.chdir(path)
printLog("Changed to dir " + path, log)
except os.error, (errno, errstr):
printLog("Could not change to directory '%s': Error (%d): %s" % (path, errno, errstr), log, err=True, isError=True)
def initAngularReferenceFile(_log, BlockWithAllExpImages, CtfGroupDirectory, CtfGroupRootName, DocFileName, DocFileWithOriginalAngles, SelFileName):
'''Create Initial angular file. Either fill it with zeros or copy input'''
printLog("initAngularReferenceFile", _log)
import shutil
MD = MetaData()
if len(DocFileName) > 1:
#shutil.copy(DocFileName,DocFileWithOriginalAngles)
MD.read(DocFileName)
else:
MD.read(SelFileName)
MD.addLabel(MDL_ANGLE_ROT)
MD.addLabel(MDL_ANGLE_TILT)
MD.addLabel(MDL_ANGLE_PSI)
MD.write(BlockWithAllExpImages + '@' + DocFileWithOriginalAngles)
block_name = 'ctfGroup' + str(1).zfill(FILENAMENUMBERLENGTH) + \
'@' + join(CtfGroupDirectory, CtfGroupRootName) + '_images.sel'
blocklist = getBlocksInMetaDataFile(join(CtfGroupDirectory, CtfGroupRootName) + '_images.sel')
MDctf = MetaData()
mDaux = MetaData()
mdList = [MDL_IMAGE]
for block in blocklist:
#read blocks
MDctf.read(block + '@' + join(CtfGroupDirectory, CtfGroupRootName) + '_images.sel', mdList)
#add angles to blocks
mDaux.joinNatural(MD, MDctf)
#MDctf.intersection(MD, MDL_IMAGE)
block_name = block + '@' + DocFileWithOriginalAngles
MDctf.write(block_name, MD_APPEND)
def run(self):
try:
script = self.getParam("--script")
fromStep = self.getIntParam("--range", 0)
toStep = self.getIntParam("--range", 1)
project = XmippProject()
project.load()
# Create protocol instance from script and setup database
protocol = project.getProtocolFromModule(script)
protocol.runSetup(isMainLoop=False)
self.db = protocol.Db
# All nodes retrieve parallel steps to work on
self.steps = self.db.getStepsRange(fromStep, toStep)
comm = MPI.COMM_WORLD
self.rank = comm.Get_rank()
self.size = comm.Get_size()
if self.rank:
self.runWorker(comm)
else:
self.runMaster(comm)
# comm.Disconnect()
except Exception, e:
if hasattr(self, "db"):
printLog("Stopping MPI process because of error %s" % e, self.db.Log, out=True, err=True, isError=True)
self.db.updateRunState(SqliteDb.RUN_FAILED)
else:
print "Stopping MPI process because of error %s" % e
exit(1)
def deleteFile(log, filename, verbose=True):
if exists(filename):
os.remove(filename)
if verbose:
printLog('Deleted file %s' % filename, log)
else:
if verbose:
printLog('Do not need to delete %s; already gone' % filename, log)
def runSingleStep(self, _connection, _cursor, stepRow):
info = self._beginSingleStep(_connection, _cursor, stepRow)
try:
self._execSingleStep(stepRow, info)
except Exception as e:
err = " Step finished with error: %s: %s" % (info.stepStr, e)
printLog(err, self.Log, out=True, err=True, isError=True)
raise
self._endSingleStep(_connection, _cursor, stepRow, info)
def linkAcquisitionInfo(log, InputFile, dirDest):
''' Search for the acquisition info file and if
exists create a link in the specified folder'''
acquisitionInfo = findAcquisitionInfo(InputFile)
if acquisitionInfo is None:
printLog("Couldn't find acquisition_info.xmd from file path: '%(InputFile)s'" % locals(),
log, err=True, isError=True)
else:
createLink(log, acquisitionInfo, join(dirDest, ACQUISITION_INFO))
def createLink(log, source, dest):
try:
if exists(dest):
os.remove(dest)
destDir = split(dest)[0]
os.symlink(relpath(source, destDir), dest)
printLog("Linked '%s' to '%s'" % (source, dest))
except Exception, e:
printLog("Could not link '%s' to '%s'. Error: %s" % (source, dest, str(e)), log, err=True, isError=True)
def copyDir(log, source, dest):
try:
from shutil import copytree
if exists(dest):
deleteDir(log, dest)
copytree(source, dest, symlinks=True)
printLog("Copied '%s' to '%s'" % (source, dest))
except Exception, e:
printLog("Could not copy '%s' to '%s'. Error: %s" % (source, dest, str(e)), log, err=True, isError=True)
def createDir(log, path):
""" Create directory, does not add workingdir"""
from distutils.dir_util import mkpath
from distutils.errors import DistutilsFileError
try:
if not exists(path):
mkpath(path, 0777, True)
printLog("Created dir " + path, log)
except DistutilsFileError, e:
printLog("Couldn't create dir: '%(path)s': %(e)s" % locals(),
log, err=True, isError=True)
def _endSingleStep(self, _connection, _cursor, stepRow, info):
# Report step finish and update database
msg = greenStr(" Step finished: %s" % info.stepStr)
printLog(msg, self.Log, out=True, err=True)
sqlCommand = (
"""UPDATE %(TableSteps)s SET finish = CURRENT_TIMESTAMP
WHERE step_id=%(step_id)d
AND run_id=%(run_id)d"""
% info.dict
)
_cursor.execute(sqlCommand)
_connection.commit()
def _beginSingleStep(self, _connection, _cursor, stepRow):
info = self._getStepRowInfo(stepRow)
# Print
from pprint import pformat
msg = blueStr("-------- Step start: %s" % info.stepStr)
printLog(msg, self.Log, out=True, err=True)
printLog(headerStr((info.command.split())[-1]), self.Log, out=True)
printLog(pformat(info.args, indent=4, width=20), self.Log, out=False)
# pprint.PrettyPrinter(indent=4, width=20,stream=self.log).pprint(info.args)
# Set init time
sqlCommand = (
"""UPDATE %(TableSteps)s SET init = CURRENT_TIMESTAMP
WHERE step_id=%(step_id)d
AND run_id=%(run_id)d"""
% info.dict
)
_cursor.execute(sqlCommand)
_connection.commit()
return info
def copyFile(log, source, dest):
try:
copyfile(source, dest)
printLog("Copied '%s' to '%s'" % (source, dest), log)
except Exception, e:
printLog("Could not copy '%s' to '%s'. Error: %s" % (source, dest, str(e)), log, err=True, isError=True)
def deleteDir(log, path):
from distutils.dir_util import remove_tree
if exists(path):
remove_tree(path, True)
printLog("Deleted directory " + path, log)
def runSteps(self):
# Update run state to STARTED
self.updateRunState(SqliteDb.RUN_STARTED)
# Clean init and finish for unfinished and doAlways steps
sqlCommand = (
""" UPDATE %(TableSteps)s SET init=NULL, finish=NULL
WHERE run_id = %(run_id)d
AND (finish IS NULL OR execution_mode = %(execution_always)d)"""
% self.sqlDict
)
self.cur.execute(sqlCommand)
self.connection.commit()
# Select steps to run, include parallel ones
sqlCommand = (
""" SELECT step_id, iter, passDb, command, parameters, verifyFiles, execution_mode
FROM %(TableSteps)s
WHERE run_id = %(run_id)d
AND finish IS NULL
ORDER BY step_id """
% self.sqlDict
)
self.cur.execute(sqlCommand)
steps = self.cur.fetchall()
n = len(steps)
msg = "***************************** Protocol STARTED mode: %s" % self.runBehavior
printLog(msg, self.Log, out=True, err=True)
i = 0
final_status = (SqliteDb.RUN_FINISHED, "FINISHED")
while i < n:
# Execute the step
try:
# Detect if there are parallel steps and execute them in parallel
first = i
while i < n and steps[i]["execution_mode"] > SqliteDb.EXEC_MAINLOOP:
i += 1
if first < i: # There are parallel steps
mpiForParallelSteps = self.NumberOfMpi
fromStep = steps[first]["step_id"]
if i < n:
toStep = steps[i]["step_id"]
else:
toStep = 99999
self.runParallelSteps(fromStep, toStep, mpiForParallelSteps)
if i < n: # To be sure there are normal steps after parallels
self.runSingleStep(self.connection, self.cur, steps[i])
i += 1
except Exception as e:
msg = "Stopping batch execution"
if self.runAborted():
msg += " ABORTED by user request"
final_status = (SqliteDb.RUN_ABORTED, "ABORTED")
else:
msg += " since one of the steps could not be performed: %s" % str(e)
final_status = (SqliteDb.RUN_FAILED, "FAILED")
printLog(msg, self.Log, out=True, err=True, isError=True)
break
self.updateRunState(final_status[0])
msg = "***************************** Protocol %s" % final_status[1]
printLog(msg, self.Log, out=True, err=True)
def reconstruction(_log
, ARTReconstructionExtraCommand
, WBPReconstructionExtraCommand
, FourierReconstructionExtraCommand
, Iteration_number
, DoParallel
, maskedFileNamesIter
, MpiJobSize
, NumberOfMpi
, NumberOfThreads
, ReconstructionMethod
, FourierMaxFrequencyOfInterest
, ARTLambda
, SymmetryGroup
, ReconstructionXmd
, ReconstructedVolume
, ResolSam
, ResolutionXmdPrevIterMax
, PaddingFactor
, ConstantToAddToFiltration
):
#if inout metadata is empty create a Blanck image
if emptyMd(ReconstructionXmd):
from protlib_utils import printLog
img = Image()
img.read(maskedFileNamesIter, DATA)
#(x,y,z,n) = img.getDimensions()
printLog("Metadata %s is empty. Creating a Black file named %s" % (ReconstructionXmd, ReconstructedVolume))
#createEmptyFile(ReconstructedVolume,x,y,z,n)
img.initRandom()
img.write(ReconstructedVolume)
return
print '*********************************************************************'
print '* Reconstruct volume using '
if ReconstructionMethod == 'wbp':
program = 'xmipp_reconstruct_wbp'
parameters = ' -i ' + ReconstructionXmd + \
' --doc ' + ReconstructionXmd + \
' -o ' + ReconstructedVolume + \
' --sym ' + SymmetryGroup + \
' --weight --use_each_image '
parameters = parameters + WBPReconstructionExtraCommand
elif ReconstructionMethod == 'art':
program = 'xmipp_reconstruct_art'
parameters = ' -i ' + ReconstructionXmd + \
' -o ' + ReconstructedVolume + ' ' + \
' --sym ' + SymmetryGroup + \
' --thr ' + str(NumberOfThreads) + \
' --WLS '
if len(ARTLambda) > 1:
parameters = parameters + ' -l ' + ARTLambda + ' '
parameters = parameters + ARTReconstructionExtraCommand
NumberOfMpi = 1
NumberOfThreads = 1
DoParallel = False
elif ReconstructionMethod == 'fourier':
if FourierMaxFrequencyOfInterest == -1:
md = MetaData(ResolutionXmdPrevIterMax)
id = md.firstObject()
FourierMaxFrequencyOfInterest = md.getValue(MDL_RESOLUTION_FREQREAL, id)
FourierMaxFrequencyOfInterest = ResolSam / FourierMaxFrequencyOfInterest + float(ConstantToAddToFiltration)
if FourierMaxFrequencyOfInterest > 0.5:
FourierMaxFrequencyOfInterest = 0.5
elif FourierMaxFrequencyOfInterest < 0.:
FourierMaxFrequencyOfInterest = 0.001
program = 'xmipp_reconstruct_fourier'
parameters = ' -i ' + ReconstructionXmd + \
' -o ' + ReconstructedVolume + \
' --sym ' + SymmetryGroup + \
' --thr ' + str(NumberOfThreads) + \
' --weight ' + \
' --max_resolution ' + str(FourierMaxFrequencyOfInterest) + \
' --padding ' + str(PaddingFactor) + ' ' + str(PaddingFactor)
if (DoParallel):
parameters = parameters + ' --mpi_job_size ' + str(MpiJobSize)
runJob(_log
, program
, parameters
, NumberOfMpi
, NumberOfThreads
)
def executeCtfGroups (_log,
CTFDatName,
CtfGroupDirectory,
CtfGroupMaxDiff,
CtfGroupMaxResol,
CtfGroupRootName,
DataArePhaseFlipped,
DoAutoCtfGroup,
DoCtfCorrection,
PaddingFactor,
SamplingRate ,
SelFileName,
SplitDefocusDocFile,
WienerConstant,
) :
import glob, sys, shutil
from protlib_utils import runJob
if not os.path.exists(CtfGroupDirectory):
os.makedirs(CtfGroupDirectory)
printLog("executeCtfGroups01"+ CTFDatName, _log)
if(not DoCtfCorrection):
MD = MetaData(SelFileName)
block_name = 'ctfGroup' + str(1).zfill(FILENAMENUMBERLENGTH) + \
'@' + join(CtfGroupDirectory, CtfGroupRootName) + '_images.sel'
MD.write(block_name)
return 1
# print '*********************************************************************'
# print '* Make CTF groups'
# remove all entries not present in sel file by
# join between selfile and metadatafile
MDctfdata = MetaData();
MDctfdata.read(CTFDatName)
MDsel = MetaData();
MDsel.read(SelFileName)
MDctfdata.intersection(MDsel, MDL_IMAGE)
tmpCtfdat = uniqueFilename(CTFDatName)
MDctfdata.write(tmpCtfdat)
command = \
' --ctfdat ' + tmpCtfdat + \
' -o ' + CtfGroupDirectory + '/' + CtfGroupRootName + ':stk'\
' --wiener --wc ' + str(WienerConstant) + \
' --pad ' + str(PaddingFactor) + \
' --sampling_rate ' + str (SamplingRate)
if (DataArePhaseFlipped):
command += ' --phase_flipped '
if (DoAutoCtfGroup):
command += ' --error ' + str(CtfGroupMaxDiff) + \
' --resol ' + str(CtfGroupMaxResol)
else:
if (len(SplitDefocusDocFile) > 0):
command += ' --split ' + SplitDefocusDocFile
else:
message = "Error: for non-automated ctf grouping, please provide a docfile!"
print '* ', message
_log.info(message)
sys.exit()
if runJob(_log, "xmipp_ctf_group", command):
return 1
def renameFile(log, source, dest):
try:
os.rename(source, dest)
printLog("Renamed '%s' to '%s'" % (source, dest))
except Exception, e:
printLog("Could not rename '%s' to '%s'. Error: %s" % (source, dest, str(e)), log, err=True, isError=True)