Esempio n. 1
0
def changeDir(log, path):
    """ Change to Directory """
    try:
        os.chdir(path)
        printLog("Changed to dir " + path, log)
    except os.error, (errno, errstr):
        printLog("Could not change to directory '%s': Error (%d): %s" % (path, errno, errstr), log, err=True, isError=True)
def initAngularReferenceFile(_log, BlockWithAllExpImages, CtfGroupDirectory, CtfGroupRootName, DocFileName, DocFileWithOriginalAngles, SelFileName):
    '''Create Initial angular file. Either fill it with zeros or copy input'''
    printLog("initAngularReferenceFile", _log)
    import shutil
    
    MD = MetaData()

    if len(DocFileName) > 1:
        #shutil.copy(DocFileName,DocFileWithOriginalAngles)
        MD.read(DocFileName)
        
        
    else:
        MD.read(SelFileName)
        MD.addLabel(MDL_ANGLE_ROT)
        MD.addLabel(MDL_ANGLE_TILT)
        MD.addLabel(MDL_ANGLE_PSI)
    
    MD.write(BlockWithAllExpImages + '@' + DocFileWithOriginalAngles)
    block_name = 'ctfGroup' + str(1).zfill(FILENAMENUMBERLENGTH) + \
                    '@' + join(CtfGroupDirectory, CtfGroupRootName) + '_images.sel'
    blocklist = getBlocksInMetaDataFile(join(CtfGroupDirectory, CtfGroupRootName) + '_images.sel')
    
    MDctf = MetaData()
    mDaux = MetaData()
    mdList = [MDL_IMAGE]
    for block in blocklist:
        #read blocks
        MDctf.read(block + '@' + join(CtfGroupDirectory, CtfGroupRootName) + '_images.sel', mdList)
        #add angles to blocks
        mDaux.joinNatural(MD, MDctf)
        #MDctf.intersection(MD, MDL_IMAGE)
        block_name = block + '@' + DocFileWithOriginalAngles
        MDctf.write(block_name, MD_APPEND)
    def run(self):
        try:
            script = self.getParam("--script")
            fromStep = self.getIntParam("--range", 0)
            toStep = self.getIntParam("--range", 1)
            project = XmippProject()
            project.load()
            # Create protocol instance from script and setup database
            protocol = project.getProtocolFromModule(script)
            protocol.runSetup(isMainLoop=False)
            self.db = protocol.Db
            # All nodes retrieve parallel steps to work on
            self.steps = self.db.getStepsRange(fromStep, toStep)

            comm = MPI.COMM_WORLD
            self.rank = comm.Get_rank()
            self.size = comm.Get_size()

            if self.rank:
                self.runWorker(comm)
            else:
                self.runMaster(comm)

            # comm.Disconnect()

        except Exception, e:
            if hasattr(self, "db"):
                printLog("Stopping MPI process because of error %s" % e, self.db.Log, out=True, err=True, isError=True)
                self.db.updateRunState(SqliteDb.RUN_FAILED)
            else:
                print "Stopping MPI process because of error %s" % e
            exit(1)
Esempio n. 4
0
def deleteFile(log, filename, verbose=True):
    if exists(filename):
        os.remove(filename)
        if verbose:
            printLog('Deleted file %s' % filename, log)
    else:
        if verbose:
            printLog('Do not need to delete %s; already gone' % filename, log)
Esempio n. 5
0
 def runSingleStep(self, _connection, _cursor, stepRow):
     info = self._beginSingleStep(_connection, _cursor, stepRow)
     try:
         self._execSingleStep(stepRow, info)
     except Exception as e:
         err = "         Step finished with error: %s: %s" % (info.stepStr, e)
         printLog(err, self.Log, out=True, err=True, isError=True)
         raise
     self._endSingleStep(_connection, _cursor, stepRow, info)
Esempio n. 6
0
def linkAcquisitionInfo(log, InputFile, dirDest):
    ''' Search for the acquisition info file and if 
    exists create a link in the specified folder'''
    acquisitionInfo = findAcquisitionInfo(InputFile)
    if acquisitionInfo is None:
        printLog("Couldn't find acquisition_info.xmd from file path: '%(InputFile)s'" % locals(),
                 log, err=True, isError=True)
    else:        
        createLink(log, acquisitionInfo, join(dirDest, ACQUISITION_INFO))
Esempio n. 7
0
def createLink(log, source, dest):
    try:
        if exists(dest):
            os.remove(dest)
        destDir = split(dest)[0]
        os.symlink(relpath(source, destDir), dest)
        printLog("Linked '%s' to '%s'" % (source, dest))
    except Exception, e:
        printLog("Could not link '%s' to '%s'. Error: %s" % (source, dest, str(e)), log, err=True, isError=True)
Esempio n. 8
0
def copyDir(log, source, dest):
    try:
        from shutil import copytree
        if exists(dest):
            deleteDir(log, dest)
        copytree(source, dest, symlinks=True)
        printLog("Copied '%s' to '%s'" % (source, dest))
    except Exception, e:
        printLog("Could not copy '%s' to '%s'. Error: %s" % (source, dest, str(e)), log, err=True, isError=True)
Esempio n. 9
0
def createDir(log, path):
    """ Create directory, does not add workingdir"""
    from distutils.dir_util import mkpath
    from distutils.errors import DistutilsFileError
    try:
        if not exists(path):
            mkpath(path, 0777, True)
            printLog("Created dir " + path, log)
    except DistutilsFileError, e:
        printLog("Couldn't create dir: '%(path)s': %(e)s" % locals(),
                 log, err=True, isError=True)
Esempio n. 10
0
 def _endSingleStep(self, _connection, _cursor, stepRow, info):
     # Report step finish and update database
     msg = greenStr("         Step finished: %s" % info.stepStr)
     printLog(msg, self.Log, out=True, err=True)
     sqlCommand = (
         """UPDATE %(TableSteps)s SET finish = CURRENT_TIMESTAMP 
                     WHERE step_id=%(step_id)d
                       AND run_id=%(run_id)d"""
         % info.dict
     )
     _cursor.execute(sqlCommand)
     _connection.commit()
Esempio n. 11
0
    def _beginSingleStep(self, _connection, _cursor, stepRow):
        info = self._getStepRowInfo(stepRow)
        # Print
        from pprint import pformat

        msg = blueStr("-------- Step start:  %s" % info.stepStr)
        printLog(msg, self.Log, out=True, err=True)
        printLog(headerStr((info.command.split())[-1]), self.Log, out=True)
        printLog(pformat(info.args, indent=4, width=20), self.Log, out=False)
        # pprint.PrettyPrinter(indent=4, width=20,stream=self.log).pprint(info.args)
        # Set init time
        sqlCommand = (
            """UPDATE %(TableSteps)s SET init = CURRENT_TIMESTAMP 
                        WHERE step_id=%(step_id)d
                          AND run_id=%(run_id)d"""
            % info.dict
        )
        _cursor.execute(sqlCommand)
        _connection.commit()
        return info
Esempio n. 12
0
def copyFile(log, source, dest):
    try:
        copyfile(source, dest)
        printLog("Copied '%s' to '%s'" % (source, dest), log)
    except Exception, e:
        printLog("Could not copy '%s' to '%s'. Error: %s" % (source, dest, str(e)), log, err=True, isError=True)
Esempio n. 13
0
def deleteDir(log, path):
    from distutils.dir_util import remove_tree
    if exists(path):
        remove_tree(path, True)
        printLog("Deleted directory " + path, log)
Esempio n. 14
0
    def runSteps(self):
        # Update run state to STARTED
        self.updateRunState(SqliteDb.RUN_STARTED)
        # Clean init and finish for unfinished and doAlways steps
        sqlCommand = (
            """ UPDATE %(TableSteps)s  SET init=NULL, finish=NULL
                         WHERE run_id = %(run_id)d 
                         AND (finish IS NULL OR execution_mode = %(execution_always)d)"""
            % self.sqlDict
        )
        self.cur.execute(sqlCommand)
        self.connection.commit()
        # Select steps to run, include parallel ones
        sqlCommand = (
            """ SELECT step_id, iter, passDb, command, parameters, verifyFiles, execution_mode
                         FROM %(TableSteps)s 
                         WHERE run_id = %(run_id)d 
                         AND finish IS NULL
                         ORDER BY step_id """
            % self.sqlDict
        )
        self.cur.execute(sqlCommand)
        steps = self.cur.fetchall()
        n = len(steps)
        msg = "***************************** Protocol STARTED mode: %s" % self.runBehavior
        printLog(msg, self.Log, out=True, err=True)

        i = 0
        final_status = (SqliteDb.RUN_FINISHED, "FINISHED")

        while i < n:
            # Execute the step
            try:
                # Detect if there are parallel steps and execute them in parallel
                first = i
                while i < n and steps[i]["execution_mode"] > SqliteDb.EXEC_MAINLOOP:
                    i += 1
                if first < i:  # There are parallel steps
                    mpiForParallelSteps = self.NumberOfMpi
                    fromStep = steps[first]["step_id"]
                    if i < n:
                        toStep = steps[i]["step_id"]
                    else:
                        toStep = 99999
                    self.runParallelSteps(fromStep, toStep, mpiForParallelSteps)
                if i < n:  # To be sure there are normal steps after parallels
                    self.runSingleStep(self.connection, self.cur, steps[i])
                i += 1
            except Exception as e:
                msg = "Stopping batch execution"
                if self.runAborted():
                    msg += " ABORTED by user request"
                    final_status = (SqliteDb.RUN_ABORTED, "ABORTED")
                else:
                    msg += " since one of the steps could not be performed: %s" % str(e)
                    final_status = (SqliteDb.RUN_FAILED, "FAILED")
                printLog(msg, self.Log, out=True, err=True, isError=True)
                break

        self.updateRunState(final_status[0])
        msg = "***************************** Protocol %s" % final_status[1]
        printLog(msg, self.Log, out=True, err=True)
def reconstruction(_log
                   , ARTReconstructionExtraCommand
                   , WBPReconstructionExtraCommand
                   , FourierReconstructionExtraCommand
                   , Iteration_number
                   , DoParallel
                   , maskedFileNamesIter
                   , MpiJobSize
                   , NumberOfMpi
                   , NumberOfThreads
                   , ReconstructionMethod
                   , FourierMaxFrequencyOfInterest
                   , ARTLambda
                   , SymmetryGroup
                   , ReconstructionXmd
                   , ReconstructedVolume
                   , ResolSam
                   , ResolutionXmdPrevIterMax
                   , PaddingFactor
                   , ConstantToAddToFiltration
                   ):
    
    #if inout metadata is empty create a Blanck image
    if emptyMd(ReconstructionXmd):
        from protlib_utils import printLog
        img = Image()
        img.read(maskedFileNamesIter, DATA)
        #(x,y,z,n) = img.getDimensions()
        printLog("Metadata %s is empty. Creating a Black file named %s" % (ReconstructionXmd, ReconstructedVolume))
        #createEmptyFile(ReconstructedVolume,x,y,z,n)
        img.initRandom()
        img.write(ReconstructedVolume)
        return

    
    print '*********************************************************************'
    print '* Reconstruct volume using '
    if ReconstructionMethod == 'wbp':
        program = 'xmipp_reconstruct_wbp'
        parameters = ' -i ' + ReconstructionXmd + \
                    ' --doc ' + ReconstructionXmd + \
                    ' -o ' + ReconstructedVolume + \
                    ' --sym ' + SymmetryGroup + \
                    ' --weight --use_each_image '
        parameters = parameters + WBPReconstructionExtraCommand
                  
    elif ReconstructionMethod == 'art':
        program = 'xmipp_reconstruct_art'

        parameters = ' -i ' + ReconstructionXmd + \
                   ' -o ' + ReconstructedVolume + ' ' + \
                   ' --sym ' + SymmetryGroup + \
                   ' --thr ' + str(NumberOfThreads) + \
                   ' --WLS '
        if len(ARTLambda) > 1:
           parameters = parameters + ' -l ' + ARTLambda + ' '
        parameters = parameters + ARTReconstructionExtraCommand
        
        NumberOfMpi = 1
        NumberOfThreads = 1
        DoParallel = False
                
    elif ReconstructionMethod == 'fourier':
        
        if FourierMaxFrequencyOfInterest == -1:
                md = MetaData(ResolutionXmdPrevIterMax)
                id = md.firstObject()
                FourierMaxFrequencyOfInterest = md.getValue(MDL_RESOLUTION_FREQREAL, id)
                FourierMaxFrequencyOfInterest = ResolSam / FourierMaxFrequencyOfInterest + float(ConstantToAddToFiltration)
                if FourierMaxFrequencyOfInterest > 0.5:
                    FourierMaxFrequencyOfInterest = 0.5
                elif FourierMaxFrequencyOfInterest < 0.:
                    FourierMaxFrequencyOfInterest = 0.001

        program = 'xmipp_reconstruct_fourier'
        parameters = ' -i ' + ReconstructionXmd + \
                   ' -o ' + ReconstructedVolume + \
                   ' --sym ' + SymmetryGroup + \
                   ' --thr ' + str(NumberOfThreads) + \
                   ' --weight ' + \
                   ' --max_resolution ' + str(FourierMaxFrequencyOfInterest) + \
                   ' --padding ' + str(PaddingFactor) + ' ' + str(PaddingFactor)
 
    if (DoParallel):
        parameters = parameters + ' --mpi_job_size ' + str(MpiJobSize)
            
    runJob(_log
           , program
           , parameters
           , NumberOfMpi
           , NumberOfThreads
           )
def executeCtfGroups (_log,
                                 CTFDatName,
                                 CtfGroupDirectory,
                                 CtfGroupMaxDiff,
                                 CtfGroupMaxResol,
                                 CtfGroupRootName,
                                 DataArePhaseFlipped,
                                 DoAutoCtfGroup,
                                 DoCtfCorrection,
                                 PaddingFactor,
                                 SamplingRate ,
                                 SelFileName,
                                 SplitDefocusDocFile,
                                 WienerConstant,
                                 ) :
    import glob, sys, shutil
    from protlib_utils import runJob
    if not os.path.exists(CtfGroupDirectory):
        os.makedirs(CtfGroupDirectory)
    printLog("executeCtfGroups01"+ CTFDatName, _log)

    if(not DoCtfCorrection):
        MD = MetaData(SelFileName)
        block_name = 'ctfGroup' + str(1).zfill(FILENAMENUMBERLENGTH) + \
                    '@' + join(CtfGroupDirectory, CtfGroupRootName) + '_images.sel'
        MD.write(block_name)
        return 1

#    print '*********************************************************************'
#    print '* Make CTF groups'
#    remove all entries not present in sel file by
#    join between selfile and metadatafile
    MDctfdata = MetaData();
    MDctfdata.read(CTFDatName)

    MDsel = MetaData();
    MDsel.read(SelFileName)
    MDctfdata.intersection(MDsel, MDL_IMAGE)
    tmpCtfdat = uniqueFilename(CTFDatName)
    MDctfdata.write(tmpCtfdat)
    command = \
              ' --ctfdat ' + tmpCtfdat + \
              ' -o ' + CtfGroupDirectory + '/' + CtfGroupRootName + ':stk'\
              ' --wiener --wc ' + str(WienerConstant) + \
              ' --pad ' + str(PaddingFactor) + \
              ' --sampling_rate ' + str (SamplingRate)

    if (DataArePhaseFlipped):
        command += ' --phase_flipped '

    if (DoAutoCtfGroup):
        command += ' --error ' + str(CtfGroupMaxDiff) + \
                   ' --resol ' + str(CtfGroupMaxResol)
    else:
        if (len(SplitDefocusDocFile) > 0):
            command += ' --split ' + SplitDefocusDocFile
        else:
            message = "Error: for non-automated ctf grouping, please provide a docfile!"
            print '* ', message
            _log.info(message)
            sys.exit()
    
    if runJob(_log, "xmipp_ctf_group", command):
        return 1
Esempio n. 17
0
def renameFile(log, source, dest):
    try:
        os.rename(source, dest)
        printLog("Renamed '%s' to '%s'" % (source, dest))
    except Exception, e:
        printLog("Could not rename '%s' to '%s'. Error: %s" % (source, dest, str(e)), log, err=True, isError=True)