Example #1
0
    def __init__(self, job, time_created):

        self.job_json = get_job_JSON(job)
        self.time_created = time_created
        self.job_status = job.status
        self.job_application = getName(job.application)
        self.job_backend = getName(job.backend)
Example #2
0
def create_subjobs_graphics(jobid, subjob_attribute, fromDate, toDate):

    subjobs_in_time_range = get_subjobs_in_time_range(jobid, fromDate, toDate)

    if subjob_attribute == 'accumulate':
        # return some JSON here
        return get_accumulated_subjobs_JSON(subjobs_in_time_range)

    subjobs_attributes = {}

    for subjob in subjobs_in_time_range:

        if subjob_attribute == 'status':
            increment(subjobs_attributes, subjob.status)

        elif subjob_attribute == 'application':
            increment(
                subjobs_attributes, getName(subjob.application))

        elif subjob_attribute == 'backend':
            increment(subjobs_attributes, getName(subjob.backend))

        elif subjob_attribute == 'actualCE':
            increment(subjobs_attributes, subjob.backend.actualCE)

    if subjob_attribute == 'status':
        return get_pie_chart_json(subjobs_attributes, colors=True, jobs=False)
    else:
        return get_pie_chart_json(subjobs_attributes)
Example #3
0
def isCredentialRequired(credObj):
    """
    The logic to decide if a given invalid credential
    should trigger the deactivation of Ganga internal services.  
    """

    from Ganga.Runtime import Workspace_runtime
    from Ganga.Runtime import Repository_runtime

    if getName(credObj) == 'AfsToken':
        return Workspace_runtime.requiresAfsToken() or Repository_runtime.requiresAfsToken()

    if getName(credObj) == 'GridProxy':
        from Ganga.Core.GangaRepository import getRegistryProxy
        from Ganga.Runtime.GPIFunctions import typename
        from Ganga.GPIDev.Base.Proxy import stripProxy
        from Ganga.GPIDev.Lib.Job.Job import lazyLoadJobBackend, lazyLoadJobStatus
        for j in getRegistryProxy('jobs'):
            ji = stripProxy(j)
            this_status = lazyLoadJobStatus(ji)
            if this_status in ['submitted', 'running', 'completing']:
                this_backend = lazyLoadJobBackend(ji)
                if getName(this_backend) == 'LCG':
                    return True
        return False

    log.warning("Unknown credential object : %s" % credObj)
Example #4
0
def isCredentialRequired(credObj):
    """
    The logic to decide if a given invalid credential
    should trigger the deactivation of Ganga internal services.  
    """

    from Ganga.Runtime import Workspace_runtime
    from Ganga.Runtime import Repository_runtime

    if getName(credObj) == 'AfsToken':
        return Workspace_runtime.requiresAfsToken() or Repository_runtime.requiresAfsToken()

    if getName(credObj) == 'GridProxy':
        if Repository_runtime.requiresGridProxy() or Workspace_runtime.requiresGridProxy():
            return True
        from Ganga.GPI import jobs, typename
        from Ganga.GPIDev.Base.Proxy import stripProxy
        for j in jobs:
            ji = stripProxy(j)
            if ji.status in ['submitted', 'running', 'completing']:
                if ji.getNodeIndexCache() is not None and 'display:backend' in ji.getNodeIndexCache().keys():
                    if ji.getNodeIndexCache()['display:backend'] == 'LCG':
                        return True
                else:
                    if getName(ji.backend) == 'LCG':
                        return True
        return False

    log.warning("Unknown credential object : %s" % credObj)
Example #5
0
def isCredentialRequired(credObj):
    """
    The logic to decide if a given invalid credential
    should trigger the deactivation of Ganga internal services.  
    """

    from Ganga.Runtime import Workspace_runtime
    from Ganga.Runtime import Repository_runtime

    if getName(credObj) == "AfsToken":
        return Workspace_runtime.requiresAfsToken() or Repository_runtime.requiresAfsToken()

    if getName(credObj) == "GridProxy":
        if Repository_runtime.requiresGridProxy() or Workspace_runtime.requiresGridProxy():
            return True
        from Ganga.GPI import jobs, typename
        from Ganga.GPIDev.Base.Proxy import stripProxy

        for j in jobs:
            ji = stripProxy(j)
            if ji.status in ["submitted", "running", "completing"]:
                if getName(ji.backend) == "LCG":
                    return True
        return False

    log.warning("Unknown credential object : %s" % credObj)
Example #6
0
    def upload(self, files=[], opts=""):
        """
        Uploads multiple files to a remote grid storage.

        @param files is a list of local files to be uploaded to the grid.
               The elemement can be a file path or a File object.

        @return True if files are successfully uploaded; otherwise it returns False
        """
        status = False

        paths = []
        for f in files:
            if getName(f) == "File":
                paths.append("file://%s" % f.name)
            elif getName(f) == "str":
                paths.append("file://%s" % f)
            else:
                self.logger.warning("unknown file expression: %s" % repr(f))

        uploaded_files = self.impl_upload(files=paths, opts=opts)

        if len(uploaded_files) == len(files):
            status = self.impl_bookkeepUploadedFiles(uploaded_files, append=True, opts=opts)
        else:
            status = False

        if len(uploaded_files) == len(files):
            status = self.impl_bookkeepUploadedFiles(uploaded_files, append=True, opts=opts)
        else:
            status = False

        return status
Example #7
0
def getWNCodeForOutputPostprocessing(job, indent):

    # dict containing the list of outputfiles that need to be processed on the
    # WN for every file type
    outputFilesProcessedOnWN = {}
    patternsToZip = []

    if len(job.outputfiles) == 0:
        return ""
    else:
        for outputFile in job.outputfiles:

            outputfileClassName = getName(outputFile)
            backendClassName = getName(job.backend)

            if outputFile.compressed:
                patternsToZip.append(outputFile.namePattern)

            if outputfileClassName not in outputFilesProcessedOnWN.keys():
                outputFilesProcessedOnWN[outputfileClassName] = []

            if outputFilePostProcessingOnWN(job, outputfileClassName):
                outputFilesProcessedOnWN[outputfileClassName].append(outputFile)

    if not patternsToZip:
        if not any(outputFilesProcessedOnWN.values()):
            return ""

    logger.debug("Process: '%s' on WN" % str(outputFilePostProcessingOnWN))

    shortScript = """\n
import os, glob
for patternToZip in ###PATTERNSTOZIP###:
    for currentFile in glob.glob(os.path.join(os.getcwd(),patternToZip)):
        if os.path.isfile(currentFile):
            os.system("gzip %s" % currentFile)

postprocesslocations = open(os.path.join(os.getcwd(), '###POSTPROCESSLOCATIONSFILENAME###'), 'a+')  
"""

    from Ganga.GPIDev.Lib.File import FileUtils
    shortScript = FileUtils.indentScript(shortScript, '###INDENT###')

    insertScript = shortScript

    insertScript = insertScript.replace('###PATTERNSTOZIP###', str(patternsToZip))
    insertScript = insertScript.replace('###POSTPROCESSLOCATIONSFILENAME###', getConfig('Output')['PostProcessLocationsFileName'])

    for outputFileName in outputFilesProcessedOnWN.keys():

        if len(outputFilesProcessedOnWN[outputFileName]) > 0:

            insertScript += outputFilesProcessedOnWN[outputFileName][0].getWNInjectedScript(outputFilesProcessedOnWN[outputFileName], indent, patternsToZip, 'postprocesslocations')

    insertScript += """\n
###INDENT###postprocesslocations.close()
"""
    insertScript = insertScript.replace('###INDENT###', indent)

    return insertScript
Example #8
0
def notifyInvalidCredential(credObj):
    """
    The Core is notified when one of the monitored credentials is invalid
    @see ICredential.create()
    """

    # ignore this notification if the internal services are already stopped
    if not servicesEnabled:
        log.debug(
            "One of the monitored credential [%s] is invalid BUT the internal services are already disabled." % getName(credObj))
        return

    if isCredentialRequired(credObj):
        log.debug("One of the required credential for the internal services is invalid: [%s]."
                  "Disabling internal services ..." % getName(credObj))
        _tl = credObj.timeleft()
        if _tl == "-1":
            log.error('%s has been destroyed! Could not shutdown internal services.' % getName(credObj))
            return
        disableInternalServices()
        log.warning('%s is about to expire! '
                    'To protect against possible write errors all internal services has been disabled.'
                    'If you believe the problem has been solved type "reactivate()" to re-enable '
                    'interactions within this session.' % getName(credObj))
    else:
        log.debug("One of the monitored credential [%s] is invalid BUT it is not required by the internal services" % getName(credObj))
Example #9
0
def getWNCodeForDownloadingInputFiles(job, indent):
    """
    Generate the code to be run on the WN to download input files
    """

    from Ganga.GPIDev.Lib.Dataset.GangaDataset import GangaDataset

    if (
        job.inputfiles is None
        or len(job.inputfiles) == 0
        and (not job.inputdata or ((not isType(job.inputdata, GangaDataset)) or not job.inputdata.treat_as_inputfiles))
    ):
        return ""

    insertScript = """\n
"""

    # first, go over any LocalFiles in GangaDatasets to be transferred
    # The LocalFiles in inputfiles have already been dealt with
    if job.inputdata and isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles:
        for inputFile in job.inputdata.files:
            inputfileClassName = getName(inputFile)

            if inputfileClassName == "LocalFile":

                # special case for LocalFile
                if getName(job.backend) in ["Localhost", "Batch", "LSF", "Condor", "PBS"]:
                    # create symlink
                    shortScript += """
# create symbolic links for LocalFiles
for f in ###FILELIST###:
   os.symlink(f, os.path.basename(f)) 
"""
                    shortScript = FileUtils.indentScript(shortScript, "###INDENT####")

                    insertScript += shortScript

                    insertScript = insertScript.replace("###FILELIST###", "%s" % inputFile.getFilenameList())

    # if GangaDataset is used, check if they want the inputfiles transferred
    inputfiles_list = job.inputfiles
    if job.inputdata and isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles:
        inputfiles_list += job.inputdata.files

    for inputFile in inputfiles_list:

        inputfileClassName = getName(inputFile)

        if outputFilePostProcessingOnWN(job, inputfileClassName):
            inputFile.processWildcardMatches()
            if inputFile.subfiles:
                for subfile in inputFile.subfiles:
                    insertScript += subfile.getWNScriptDownloadCommand(indent)
            else:
                insertScript += inputFile.getWNScriptDownloadCommand(indent)

    insertScript = insertScript.replace("###INDENT###", indent)

    return insertScript
Example #10
0
def getWNCodeForOutputPostprocessing(job, indent):

    # dict containing the list of outputfiles that need to be processed on the
    # WN for every file type
    outputFilesProcessedOnWN = {}
    patternsToZip = []

    if len(job.outputfiles) == 0:
        return ""
    else:
        for outputFile in job.outputfiles:

            outputfileClassName = getName(outputFile)
            backendClassName = getName(job.backend)

            if outputFile.compressed:
                if outputfileClassName == 'LocalFile' and backendClassName not in ['Localhost', 'LSF', 'Interactive']:
                    patternsToZip.append(outputFile.namePattern)
                elif outputfileClassName != 'LocalFile' and outputFilePostProcessingOnWN(job, outputfileClassName):
                    patternsToZip.append(outputFile.namePattern)
                elif outputfileClassName != 'LocalFile' and outputFilePostProcessingOnClient(job, outputfileClassName) and backendClassName not in ['Localhost', 'LSF', 'Interactive']:
                    patternsToZip.append(outputFile.namePattern)

            if outputfileClassName not in outputFilesProcessedOnWN.keys():
                outputFilesProcessedOnWN[outputfileClassName] = []

            if outputFilePostProcessingOnWN(job, outputfileClassName):
                outputFilesProcessedOnWN[
                    outputfileClassName].append(outputFile)

    shortScript = """\n
import os, glob
for patternToZip in ###PATTERNSTOZIP###:
    for currentFile in glob.glob(os.path.join(os.getcwd(),patternToZip)):
        os.system("gzip %s" % currentFile)

postprocesslocations = file(os.path.join(os.getcwd(), '###POSTPROCESSLOCATIONSFILENAME###'), 'w')  
"""

    shortScript = FileUtils.indentScript(shortScript, '###INDENT###')

    insertScript = shortScript

    insertScript = insertScript.replace('###PATTERNSTOZIP###', str(patternsToZip))
    insertScript = insertScript.replace('###POSTPROCESSLOCATIONSFILENAME###', getConfig('Output')['PostProcessLocationsFileName'])

    for outputFileName in outputFilesProcessedOnWN.keys():

        if len(outputFilesProcessedOnWN[outputFileName]) > 0:

            insertScript += outputFilesProcessedOnWN[outputFileName][0].getWNInjectedScript(outputFilesProcessedOnWN[outputFileName], indent, patternsToZip, 'postprocesslocations')

    insertScript += """\n
###INDENT###postprocesslocations.close()
"""
    insertScript = insertScript.replace('###INDENT###', indent)

    return insertScript
Example #11
0
    def __str__(self, short=True, id=None):
        """Prints an overview over the currently running tasks"""
        if Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_show_help"]:
            self.help(short=True)
            Ganga.GPIDev.Lib.Registry.RegistrySlice.config.setUserValue("tasks_show_help", False)
            print("To show this help message again, type 'tasks.help()'.")
            print('')
            print(" The following is the output of " + markup("tasks.table()", fgcol("blue")))

        lenfstring = 0
        flist = []
        for thing in Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_columns"]:
            width = Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_columns_width"][thing]
            lenfstring += width
            flist.append("%" + str(width) + "s ")

        fstring = "|".join(flist)
        fstring += '\n'
        lenfstring += 27
        ds = fstring % ("#", "Type", "Name", "State", "Comment", "%4s: %4s/ %4s/ %4s/ %4s/ %4s/ %4s/ %4s" % ("Jobs", markup("done", overview_colours["completed"]), " " + markup("run", overview_colours["running"]), " " + markup("subd", overview_colours["submitted"]), " " + markup("attd", overview_colours["attempted"]), markup("fail", overview_colours["failed"]), markup("hold", overview_colours["hold"]), " " + markup("bad", overview_colours["bad"])), "Float")
        ds += "-" * lenfstring + "\n"


        from Ganga.GPIDev.Lib.Tasks import ITask

        if id is not None and type(id) is int:
            iterable_list = [self[id]]
        else:
            iterable_list = stripProxy(self).objects.values()

        for p in iterable_list:

            if isType(p, ITask):
                stat = "%4i: %4i/ %4i/  %4i/    --/ %4i/ %4i/ %4i" % (p.n_all(), p.n_status("completed"), p.n_status("running"), p.n_status("submitted"), p.n_status("failed"), p.n_status("hold"), p.n_status("bad"))
                ds += markup(fstring % (p.id, getName(p), p.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], p.status, p.comment, stat, p.float), status_colours[p.status])
            else:
                stat = "%4i: %4i/ %4i/    --/  %4i/ %4i/ %4i/ %4i" % (p.n_all(), p.n_status("completed"), p.n_status("running"), p.n_status("attempted"), p.n_status("failed"), p.n_status("hold"), p.n_status("bad"))
                ds +=  markup(fstring % (p.id, getName(p), p.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], p.status, p.comment, stat, p.float), status_colours[p.status])

            if short is True:
                continue

            for ti in range(0, len(p.transforms)):
                t = p.transforms[ti]

                if isType(p, ITask):
                    stat = "%4i: %4i/ %4i/ %4i/     --/ %4i/ %4i/ %4s" % (t.n_all(), t.n_status("completed"), t.n_status("running"), t.n_status("submitted"), t.n_status("failed"), t.n_status("hold"), t.n_status("bad"))
                    ds += "\n" + markup(fstring % ("%i.%i" % (p.id, ti), getName(t), t.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], t.status, "", stat, ""), status_colours[t.status])
                else:
                    stat = "%4i: %4i/ %4i/     --/ %4i/ %4i/ %4i/ %4s" % (t.n_all(), t.n_status("completed"), t.n_status("running"), t.n_status("attempted"), t.n_status("failed"), t.n_status("hold"), t.n_status("bad"))
                    ds += "zn" + markup(fstring % ("%i.%i" % (p.id, ti), getName(t), t.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], t.status, "", stat, ""), status_colours[t.status])

            ds += "-" * lenfstring + "\n"

        return ds + "\n"
Example #12
0
    def __atomic_set__(self, _obj, _val):

        obj = stripProxy(_obj)
        val = stripProxy(_val)

        from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList, GangaList

        checkSet = self._bind_method(obj, self._checkset_name)
        if checkSet is not None:
            checkSet(val)
        this_filter = self._bind_method(obj, self._filter_name)
        if this_filter:
            val = this_filter(val)

        # LOCKING
        obj._getWriteAccess()

        # self._check_getter()

        item = stripProxy(obj._schema[getName(self)])

        def cloneVal(v):
            if isType(v, list()):
                new_v = GangaList()
                for elem in v:
                    new_v.append(self.__cloneVal(elem, obj))
                return new_v
            else:
                return self.__cloneVal(v, obj)

        if item['sequence']:
            _preparable = True if item['preparable'] else False
            if len(val) == 0:
                val = GangaList()
            else:
                if isType(item, Schema.ComponentItem):
                    val = makeGangaList(val, cloneVal, parent=obj, preparable=_preparable)
                else:
                    val = makeGangaList(val, parent=obj, preparable=_preparable)
        else:
            if isType(item, Schema.ComponentItem):
                newListObj = []
                if isinstance(val, list):
                    for elem in val:
                        newListObj.append(cloneVal(elem))
                    val = newListObj
                else:
                    val = cloneVal(val)

        if hasattr(val, '_setParent'):
            val._setParent(obj)
        
        obj.setNodeAttribute(getName(self), val)

        obj._setDirty()
Example #13
0
 def index_write(self, id):
     """ write an index file for this object (must be locked).
         Should not raise any Errors """
     obj = self.objects[id]
     try:
         ifn = self.get_idxfn(id)
         new_idx_cache = self.registry.getIndexCache(obj)
         if new_idx_cache != obj._index_cache or not os.path.exists(ifn):
             obj._index_cache = new_idx_cache
             with open(ifn, "w") as this_file:
                 pickle_to_file((obj._category, getName(obj), obj._index_cache), this_file)
     except IOError as err:
         logger.error("Index saving to '%s' failed: %s %s" % (ifn, getName(err), str(err)))
Example #14
0
    def __defaultActiveBackendsFunc(self):
        log.debug("__defaultActiveBackendsFunc")
        active_backends = {}
        # FIXME: this is not thread safe: if the new jobs are added then
        # iteration exception is raised
        fixed_ids = self.registry.ids()
        #log.debug("Registry: %s" % str(self.registry))
        log.debug("Running over fixed_ids: %s" % str(fixed_ids))
        for i in fixed_ids:
            try:
                j = stripProxy(self.registry(i))
                #log.debug("Job #%s status: %s" % (str(i), str(j.status)))
                if j.status in ['submitted', 'running'] or (j.master and (j.status in ['submitting'])):
                    if self.enabled is True and self.alive is True:
                        ## This causes a Loading of the subjobs from disk!!!
                        #stripProxy(j)._getReadAccess()
                        if j.getNodeData():
                            #log.info("data: %s" % str(j.getNodeData()))
                            #log.info("node %s" % str(j.getNodeIndexCache()))
                            #log.info("__dict: %s" % str(j.__class__.__dict__['backend']))
                            #bn = j.__class__.__dict__['backend'].__name__
                            #log.info("bn: %s" % str(bn))
                            lazy_load_backend_str = 'display:backend'
                            if lazy_load_backend_str in j.getNodeData():
                                bn = j.getNodeData()[lazy_load_backend_str]
                            else:
                                bn = getName(j.backend)
                        else:
                            bn = getName(j.backend)
                        #log.debug("active_backends.setdefault: %s" % str(bn))
                        active_backends.setdefault(bn, [])
                        active_backends[bn].append(j)
            except RegistryKeyError as err:
                log.debug("RegistryKeyError: The job was most likely removed")
                log.debug("RegError %s" % str(err))
            except RegistryLockError as err:
                log.debug("RegistryLockError: The job was most likely removed")
                log.debug("Reg LockError%s" % str(err))

        summary = '{'
        for backend, these_jobs in active_backends.iteritems():
            summary += '"' + str(backend) + '" : ['
            for this_job in these_jobs:
                summary += stripProxy(this_job).getFQID('.') + ', '
            summary += '], '
        summary += '}'
        log.debug("Returning active_backends: %s" % summary)
        return active_backends
Example #15
0
def mangle_job_name(app):
    job = stripProxy(app).getJobObject()

    jobName = job.name
    jobIndex = job.getStringFQID()
    appName = str(getName(app))
    appVersion = None
    if hasattr(app, "version"):
        appVersion = str(app.version)

    result = ""
    addBracket = False
    if jobName:
        result += "%s__" % jobName
    if appName:
        # and not j.subjobs: #not necessary?
        if not job.master and job.splitter:
            result += "{Ganga_%s_(%s.%s)" % (appName, jobIndex, "%n")
        else:
            result += "{Ganga_%s_(%s)" % (appName, jobIndex)
        addBracket = True
    if appVersion:
        result += "_%s" % appVersion
    if not result:
        result = "{Ganga_Job}"
    elif addBracket:
        result += "}"
    return result
Example #16
0
def mangle_job_name(app):
    """ Create a safe job name to send to DIRAC (includes full fqid)
    Args:
        app (IApplication): This is the application belonging to the job of interest
    """
    job = app.getJobObject()

    jobName = job.name
    jobIndex = job.getStringFQID()
    appName = getName(app)
    appVersion = None
    if hasattr(app, 'version'):
        appVersion = str(app.version)

    result = ''
    addBracket = False
    if jobName:
        result += '%s__' % jobName
    if appName:
        # and not j.subjobs: #not necessary?
        if not job.master and job.splitter:
            result += '{Ganga_%s_(%s.%s)' % (appName, jobIndex, '%n')
        else:
            result += '{Ganga_%s_(%s)' % (appName, jobIndex)
        addBracket = True
    if appVersion:
        result += '_%s' % appVersion
    if not result:
        result = '{Ganga_Job}'
    elif addBracket:
        result += '}'
    return result
    def transition_update(self, new_status):
        # print "Transition Update of app ", self.id, " to ",new_status
        try:
            transform = self.getTransform()
            if self.tasks_id.startswith("00"):  # Master job
                if new_status == "new":  # something went wrong with submission
                    for sj in self._getParent().subjobs:
                        sj.application.transition_update(new_status)

                if transform:
                    stripProxy(transform).setMasterJobStatus(
                        self._getParent(), new_status)

            else:
                if transform:
                    stripProxy(transform).setAppStatus(self, new_status)

        except Exception as x:
            import traceback
            import sys
            logger.error(
                "Exception in call to transform[%s].setAppStatus(%i, %s)", self.tasks_id, self.id, new_status)
            logger.error( getName(x) + " : " + x)
            tb = sys.exc_info()[2]
            if tb:
                traceback.print_tb(tb)
            else:
                logger.error("No Traceback available")

            logger.error("%s", x)
def taskify(baseclass, name):
    smajor = baseclass._schema.version.major
    sminor = baseclass._schema.version.minor

    cat = baseclass._category

    if cat == "applications":
        schema_items = _app_schema
        taskclass = TaskApplication
    elif cat == "splitters":
        schema_items = _splitter_schema
        taskclass = TaskSplitter

    classdict = {
        "_schema": Schema(Version(smajor, sminor), dict(baseclass._schema.datadict.items() + schema_items)),
        "_category": cat,
        "_name": name,
        "__init__": __task__init__,
    }

    if '_exportmethods' in baseclass.__dict__:
        classdict['_exportmethods'] = baseclass.__dict__['_exportmethods']
    cls = classobj(name, (taskclass, baseclass), classdict)

    global handler_map
    # Use the same handlers as for the base class
    handler_map.append((getName(baseclass), name))

    return cls
Example #19
0
 def do_collective_operation(self, keep_going, method, *args, **kwds):
     """
     """
     if not isinstance(keep_going, bool):
         raise GangaException("The variable 'keep_going' must be a boolean. Probably you wanted to do %s(%s).%s()" % (self.name, keep_going, method))
     result = []
     for _id in self.objects.keys():
         obj = self.objects[_id]
         try:
             if isinstance(method, str):
                 doc = method
                 result.append(getattr(obj, method)(*args, **kwds))
             else:
                 try:
                     doc = method.__doc__
                 except AttributeError:
                     doc = str(method)
                 result.append(method(obj, *args, **kwds))
         except GangaException as x:
             if not keep_going:
                 raise
         except Exception as x:
             logger.exception('%s %s %s: %s %s', doc, self.name, _id, getName(x), x)
             if not keep_going:
                 raise
     return result
Example #20
0
    def _checkActiveBackends(self, activeBackendsFunc):

        log.debug("calling function _checkActiveBackends")
        activeBackends = activeBackendsFunc()

        summary = '{'
        for this_backend, these_jobs in activeBackends.iteritems():
            summary += '"' + this_backend + '" : ['
            for this_job in these_jobs:
                summary += str(stripProxy(this_job).getFQID('.')) + ', '
            summary += '], '
        summary += '}'
        log.debug("Active Backends: %s" % summary)

        for jList in activeBackends.values():

            #log.debug("backend: %s" % str(jList))
            backendObj = jList[0].backend
            b_name = getName(backendObj)
            if b_name in config:
                pRate = config[b_name]
            else:
                pRate = config['default_backend_poll_rate']

            # TODO: To include an if statement before adding entry to
            #       updateDict. Entry is added only if credential requirements
            #       of the particular backend is satisfied.
            #       This requires backends to hold relevant information on its
            #       credential requirements.
            #log.debug("addEntry: %s, %s, %s, %s" % (str(backendObj), str(self._checkBackend), str(jList), str(pRate)))
            self.updateDict_ts.addEntry(backendObj, self._checkBackend, jList, pRate)
            summary = str([stripProxy(x).getFQID('.') for x in jList])
            log.debug("jList: %s" % str(summary))
Example #21
0
 def index_load(self, id):
     """ load the index file for this object if necessary
         Loads if never loaded or timestamp changed. Creates object if necessary
         Returns True if this object has been changed, False if not
         Raise IOError on access or unpickling error 
         Raise OSError on stat error
         Raise PluginManagerError if the class name is not found"""
     #logger.debug("Loading index %s" % id)
     fn = self.get_idxfn(id)
     # index timestamp changed
     if self._cache_load_timestamp.get(id, 0) != os.stat(fn).st_ctime:
         try:
             with open(fn, 'r') as fobj:
                 cat, cls, cache = pickle_from_file(fobj)[0]
         except Exception as x:
             logger.debug("index_load Exception: %s" % str(x))
             raise IOError("Error on unpickling: %s %s" %(getName(x), x))
         if id in self.objects:
             obj = self.objects[id]
             if obj._data:
                 obj.__dict__["_registry_refresh"] = True
         else:
             obj = self._make_empty_object_(id, cat, cls)
         obj._index_cache = cache
         self._cache_load_timestamp[id] = os.stat(fn).st_ctime
         self._cached_cat[id] = cat
         self._cached_cls[id] = cls
         self._cached_obj[id] = cache
         return True
     elif id not in self.objects:
         self.objects[id] = self._make_empty_object_(
             id, self._cached_cat[id], self._cached_cls[id])
         self.objects[id]._index_cache = self._cached_obj[id]
         return True
     return False
Example #22
0
    def master_auto_resubmit(self, rjobs):
        '''Duplicate of the IBackend.master_resubmit but hooked into auto resubmission
        such that the monitoring server is used rather than the user server'''
        from Ganga.Core import IncompleteJobSubmissionError, GangaException
        from Ganga.Utility.logging import log_user_exception
        incomplete = 0

        def handleError(x):
            if incomplete:
                raise x
            else:
                return 0
        try:
            for sj in rjobs:
                fqid = sj.getFQID('.')
                logger.info("resubmitting job %s to %s backend", fqid, getName(sj.backend))
                try:
                    b = sj.backend
                    sj.updateStatus('submitting')
                    result = b._resubmit()
                    if result:
                        sj.updateStatus('submitted')
                        # sj._commit() # PENDING: TEMPORARY DISABLED
                        incomplete = 1
                    else:
                        return handleError(IncompleteJobSubmissionError(fqid, 'resubmission failed'))
                except Exception as x:
                    log_user_exception(logger, debug=isType(x, GangaException))
                    return handleError(IncompleteJobSubmissionError(fqid, str(x)))
        finally:
            master = self.getJobObject().master
            if master:
                master.updateMasterJobStatus()
        return 1
 def _on_attribute__set__(self, obj_type, attrib_name):
     r = copy.deepcopy(self)
     if getName(obj_type) == 'Job' and attrib_name == 'outputfiles':
         r.locations = []
         r.localDir = ''
         r.failureReason = ''
     return r
Example #24
0
    def __setitem__(self, category, _filter):

        if category not in self._dict:
            self._dict[category] = {}

        # the filter can be registered as a tuple: ('filtername',filterfunction)
        # or just as a function in which case the function name is used as an
        # alias
        if isType(_filter, tuple) and len(_filter) >= 2:
            filtername = _filter[0]
            filterfunc = _filter[1]
        else:
            try:
                filtername = getName(_filter)
                filterfunc = _filter
            except AttributeError as e:
                raise ValueError(
                    'FilterManager: Invalid component filter %s.' % _filter)

        if filtername in self._dict[category]:
            raise ValueError('FilterManager: %s component filter already exists for %s category ' % (filtername, category))

        if category not in config.options:
            config.addOption(category, "", "")
        config.overrideDefaultValue(category, filtername)
        self._dict[category][filtername] = filterfunc
Example #25
0
def prepareCommand(app):
    """
    Returns the command which is to be run on the worker node
    Args:
        app (GaudiExec): This expects only the GaudiExec app
    """

    all_opts_files = app.getOptsFiles()
    opts_names = []
    for opts_file in all_opts_files:
        if isinstance(opts_file, (LocalFile, DiracFile)):
            # Ideally this would NOT need the basename, however LocalFile is special in this regard.
            # TODO Fix this after fixing LocalFile
            opts_names.append(os.path.basename(opts_file.namePattern))
        else:
            raise ApplicationConfigurationError(None, "The filetype: %s is not yet supported for use as an opts file.\nPlease contact the Ganga devs is you wish this implemented." %
                                                getName(opts_file))

    sourceEnv = app.getEnvScript()

    if not app.useGaudiRun:
        full_cmd = sourceEnv + './run python %s' % app.getWrapperScriptName()
    else:
        full_cmd = sourceEnv + "./run gaudirun.py %s %s" % (' '.join(opts_names), GaudiExecDiracRTHandler.data_file)
        if app.extraOpts:
            full_cmd += ' ' + app.getExtraOptsFileName()
        if app.extraArgs:
            full_cmd += " " + " ".join(app.extraArgs)

    return full_cmd
 def _display_element(self, item):
     if hasattr(item, 'name') and item.name != None:
         return item.name
     elif type(item.command_input[0]) != str:
         return getName(item.command_input[0])
     else:
         return item.command_input[0]
Example #27
0
 def removeCallbackHook(self, func):
     func_name = getName(func)
     log.debug('Removing Callback hook function %s.' % func_name)
     if func_name in self.callbackHookDict:
         del self.callbackHookDict[func_name]
     else:
         log.error('Callback hook function does not exist.')
Example #28
0
 def setCallbackHook(self, func, argDict, enabled, timeout=0):
     func_name = getName(func)
     log.debug('Setting Callback hook function %s.' % func_name)
     log.debug('arg dict: %s' % str(argDict))
     if func_name in self.callbackHookDict:
         log.debug('Replacing existing callback hook function %s with %s' % (str(self.callbackHookDict[func_name]), func_name))
     self.callbackHookDict[func_name] = [func, CallbackHookEntry(argDict=argDict, enabled=enabled, timeout=timeout)]
Example #29
0
    def test_h_testXMLIndex(self):
        # Check index of job
        from Ganga.Core.GangaRepository.PickleStreamer import to_file, from_file

        from Ganga.GPI import jobs

        j = jobs(0)

        assert path.isfile(getIndexFile(j))

        with open(getIndexFile(j)) as handler:
            obj, errs = from_file(handler)

            assert isinstance(obj, tuple)

            from Ganga.GPIDev.Base.Proxy import stripProxy, getName
            raw_j = stripProxy(j)
            index_cache = raw_j._getRegistry().getIndexCache(raw_j)
            assert isinstance(index_cache, dict)

            index_cls = getName(raw_j)
            index_cat = raw_j._category
            this_index_cache = (index_cat, index_cls, index_cache)

            print("just-built index: %s" % str(this_index_cache))
            print("from disk: %s" % str(obj))

            assert this_index_cache == obj
Example #30
0
 def disableCallbackHook(self, func):
     func_name = getName(func)
     log.debug('Disabling Callback hook function %s.' % func_name)
     if func_name in self.callbackHookDict:
         self.callbackHookDict[func_name][1].enabled = False
     else:
         log.error('Callback hook function does not exist.')
Example #31
0
def outputFilePostProcessingTestForWhen(job, outputFileClassName, when):
    """
    Checks if the output files of a given job(we are interested in the backend)
    should be postprocessed 'when', depending on job.backend_output_postprocess dictionary
    """
    backendClassName = getName(job.backend)

    backend_output_postprocess = stripProxy(
        job).getBackendOutputPostprocessDict()
    if backendClassName in backend_output_postprocess:
        if outputFileClassName in backend_output_postprocess[backendClassName]:
            if backend_output_postprocess[backendClassName][
                    outputFileClassName] == when:
                return True

    return False
Example #32
0
    def printSummaryTree(self,
                         level=0,
                         verbosity_level=0,
                         whitespace_marker='',
                         out=sys.stdout,
                         selection='',
                         interactive=False):
        """
        This funtion displays a summary of the contents of this file.
        (Docs from Gaga.GPIDev.Base.Objects # TODO determine how mch of this may be duplicated from there)
        Args:
            level (int): the hierachy level we are currently at in the object tree.
            verbosity_level (int): How verbose the print should be. Currently this is always 0.
            whitespace_marker (str): If printing on multiple lines, this allows the default indentation to be replicated.
                                     The first line should never use this, as the substitution is 'name = %s' % printSummaryTree()
            out (stream): An output stream to print to. The last line of output should be printed without a newline.'
            selection: See VPrinter for an explaintion of this.
            interactive (bool): Is this printing code being called from the interactive IPython prompt?
        """
        parent = self._getParent()
        schema_entry = self.findSchemaParentSchemaEntry(parent)

        if parent is None:
            full_print(self, out)
            return

        if schema_entry:
            self_len = len(self)
            print_summary = schema_entry['summary_print']
            maxLen = schema_entry['summary_sequence_maxlen']

            if print_summary:
                fp = getattr(parent, print_summary)
                str_val = fp(self._list, verbosity_level)
                out.write(str_val)
                return

            if (maxLen != -1) and (self_len > maxLen):
                out.write(decorateListEntries(self_len,
                                              getName(type(self[0]))))
                return
            else:
                summary_print(self, out)
                return

        out.write(str(self._list))
        return
Example #33
0
    def __cloneVal(self, v, obj):

        item = obj._schema[getName(self)]

        if v is None:
            if item.hasProperty('category'):
                assertion = item['optional'] and (item['category'] != 'internal')
            else:
                assertion = item['optional']
            #assert(assertion)
            if assertion is False:
                logger.warning("Item: '%s'. of class type: '%s'. Has a Default value of 'None' but is NOT optional!!!" % (getName(self), type(obj)))
                logger.warning("Please contact the developers and make sure this is updated!")
            return None
        elif isinstance(v, str):
            return str(v)
        elif isinstance(v, int):
            return int(v)
        elif isinstance(v, dict):
            new_dict = {}
            for key, item in new_dict.iteritems():
                new_dict[key] = self.__cloneVal(v, obj)
            return new_dict
        else:
            if not isinstance(v, Node) and isinstance(v, (list, tuple)):
                try:
                    GangaList = _getGangaList()
                    new_v = GangaList()
                except ImportError:
                    new_v = []
                for elem in v:
                    new_v.append(self.__cloneVal(elem, obj))
                #return new_v
            elif not isinstance(v, Node):
                if inspect.isclass(v):
                    new_v = v()
                else:
                    new_v = v
                if not isinstance(new_v, Node):
                    logger.error("v: %s" % str(v))
                    raise GangaException("Error: found Object: %s of type: %s expected an object inheriting from Node!" % (str(v), str(type(v))))
                else:
                    new_v = self.__copyNodeObject(new_v, obj)
            else:
                new_v = self.__copyNodeObject(v, obj)

            return new_v
Example #34
0
    def createUnitCopyOutputDS(self, unit_id):
        """Create a the Copy Output dataset to use with this unit. Overload to handle more than the basics"""

        from Ganga.GPIDev.Lib.Tasks.TaskLocalCopy import TaskLocalCopy
        if isType(self.unit_copy_output, TaskLocalCopy):
            logger.warning(
                "Default implementation of createUnitCopyOutputDS can't handle datasets of type '%s'"
                % getName(self.unit_copy_output))
            return

        # create copies of the Copy Output DS and add Unit name to path
        self.units[unit_id].copy_output = self.unit_copy_output.clone()
        self.units[unit_id].copy_output.local_location = os.path.join(
            self.unit_copy_output.local_location,
            self.units[unit_id].name.replace(":", "_").replace(" ",
                                                               "").replace(
                                                                   ",", "_"))
Example #35
0
def getOutputSandboxPatternsForInteractive(job):

    patternsToSandbox = [getConfig('Output')['PostProcessLocationsFileName']]
    patternsToZip = []

    for outputFile in job.outputfiles:

        outputFileClassName = getName(outputFile)

        if outputFileClassName == 'LocalFile' or (outputFileClassName != 'LocalFile' and outputFilePostProcessingOnClient(job, outputFileClassName)):
            if outputFile.compressed:
                patternsToSandbox.append('%s.gz' % outputFile.namePattern)
                patternsToZip.append(outputFile.namePattern)
            else:
                patternsToSandbox.append(outputFile.namePattern)

    return (patternsToSandbox, patternsToZip)
Example #36
0
    def __init__(self, registry):
        GangaThread.__init__(self, name="JobRegistry_Monitor")
        log.debug("Constructing JobRegistry_Monitor")
        self.setDaemon(True)
        self.registry = registry
        self.__sleepCounter = 0.0
        self.__updateTimeStamp = time.time()
        self.progressCallback = lambda x: None
        self.callbackHookDict = {}
        self.clientCallbackDict = {}
        self.alive = True
        self.enabled = False
        # run the monitoring loop continuosly (steps=-1) or just a specified
        # number of steps(>0)
        self.steps = -1
        self.activeBackends = {}
        self.updateJobStatus = None
        self.errors = {}
        # Create the default backend update method and add to callback hook.
        self.makeUpdateJobStatusFunction()

        # Add credential checking to monitoring loop
        for _credObj in Credentials._allCredentials.itervalues():
            log.debug("Setting callback hook for %s" % getName(_credObj))
            self.setCallbackHook(self.makeCredCheckJobInsertor(_credObj), {},
                                 True,
                                 timeout=config['creds_poll_rate'])

        # Add low disk-space checking to monitoring loop
        log.debug("Setting callback hook for disk space checking")
        self.setCallbackHook(self.diskSpaceCheckJobInsertor, {},
                             True,
                             timeout=config['diskspace_poll_rate'])

        # synch objects
        # main loop mutex
        self.__mainLoopCond = threading.Condition()
        # cleanup synch
        self.__cleanUpEvent = threading.Event()
        # asynch mon loop running synch
        self.__monStepsTerminatedEvent = threading.Event()
        # event to signal the break of job lists iterators
        self.stopIter = threading.Event()
        self.stopIter.set()

        self._runningNow = False
Example #37
0
    def master_resubmit(self, rjobs, backend=None):
        """ Resubmit (previously submitted) job. Configuration phase is skipped.
        Default implementation works is an emulated-bulk operation.
        If you override this method for bulk optimization then make sure that you call updateMasterJobStatus() on the master job,
        so the master job will be monitored by the monitoring loop.
        """
        from Ganga.Core import IncompleteJobSubmissionError, GangaException
        from Ganga.Utility.logging import log_user_exception
        incomplete = 0

        def handleError(x):
            if incomplete:
                raise x
            else:
                return 0

        try:
            for sj in rjobs:
                fqid = sj.getFQID('.')
                logger.info("resubmitting job %s to %s backend", fqid,
                            getName(sj.backend))
                try:
                    b = sj.backend
                    sj.updateStatus('submitting')
                    if backend is None:
                        result = b.resubmit()
                    else:
                        result = b.resubmit(backend=backend)
                    if result:
                        sj.updateStatus('submitted')
                        # sj._commit() # PENDING: TEMPORARY DISABLED
                        incomplete = 1
                    else:
                        return handleError(
                            IncompleteJobSubmissionError(
                                fqid, 'resubmission failed'))
                except Exception as x:
                    log_user_exception(logger, debug=isType(x, GangaException))
                    return handleError(
                        IncompleteJobSubmissionError(fqid, str(x)))
        finally:
            master = self.getJobObject().master
            if master:
                master.updateMasterJobStatus()
        return 1
Example #38
0
    def getFilenameList(self):
        "return a list of filenames to be created as input.txt on the WN"
        filelist = []
        for f in self.files:
            if hasattr(f, 'accessURL'):
                filelist += f.accessURL()
            elif hasattr(f, 'getFilenameList'):
                filelist += f.getFilenameList()
            else:
                if isinstance(f, GangaObject):
                    logger.warning(
                        "accessURL or getFilenameList not implemented for File '%s'"
                        % getName(f))
                else:
                    logger.warning("Warning, not sure how to parse file: %s" %
                                   str(f))

        return filelist
Example #39
0
    def __set__(self, _obj, _val):
        ## self: attribute being changed or Ganga.GPIDev.Base.Objects.Descriptor in which case getName(self) gives the name of the attribute being changed
        ## _obj: parent class which 'owns' the attribute
        ## _val: value of the attribute which we're about to set

        obj_reg = None
        obj_prevState = None
        obj = _obj
        if isinstance(obj, GangaObject):
            obj_reg = obj._getRegistry()
            if obj_reg is not None and hasattr(obj_reg, 'isAutoFlushEnabled'):
                obj_prevState = obj_reg.isAutoFlushEnabled()
                if obj_prevState is True and hasattr(obj_reg, 'turnOffAutoFlushing'):
                    obj_reg.turnOffAutoFlushing()

        val_reg = None
        val_prevState = None
        val = _val
        if isinstance(val, GangaObject):
            val_reg = val._getRegistry()
            if val_reg is not None and hasattr(val_reg, 'isAutoFlushEnabled'):
                val_prevState = val_reg.isAutoFlushEnabled()
                if val_prevState is True and hasattr(val_reg, 'turnOffAutoFlushing'):
                    val_reg.turnOffAutoFlushing()

        if type(_val) is str:
            from Ganga.GPIDev.Base.Proxy import stripProxy, runtimeEvalString
            new_val = stripProxy(runtimeEvalString(_obj, getName(self), _val))
        else:
            new_val = _val

        self.__atomic_set__(_obj, new_val)

        if isinstance(new_val, Node):
            val._setDirty()

        if val_reg is not None:
            if val_prevState is True and hasattr(val_reg, 'turnOnAutoFlushing'):
                val_reg.turnOnAutoFlushing()

        if obj_reg is not None:
            if obj_prevState is True and hasattr(obj_reg, 'turnOnAutoFlushing'):
                obj_reg.turnOnAutoFlushing()
Example #40
0
    def findFiles(self, job):

        if not len(self.files):
            raise PostProcessException(
                'No files specified, %s will do nothing!' % getName(self))

        filepaths = []
        for f in self.files:
            filepath = os.path.join(job.outputdir, f)
            for expanded_file in glob.glob(filepath):
                filepaths.append(expanded_file)
            if not len(glob.glob(filepath)):
                if (self.filesMustExist):
                    logger.info(
                        'The files %s does not exist, %s will fail job(%s) (to ignore missing files set filesMustExist to False)', filepath, self._name, job.fqid)
                    self.result = False
                else:
                    logger.warning(
                        'Ignoring file %s as it does not exist.', filepath)
        return filepaths
Example #41
0
def getOutputSandboxPatterns(job):

    outputPatterns = []

    if len(job.outputfiles) > 0:

        outputPatterns.append(getConfig('Output')['PostProcessLocationsFileName'])

        for outputFile in job.outputfiles:

            outputFileClassName = getName(outputFile)

            if outputFilePostProcessingOnClient(job, outputFileClassName) or outputFileClassName == 'LocalFile':
                if outputFile.namePattern not in outputPatterns:
                    if outputFile.compressed:
                        outputPatterns.append('%s.gz' % outputFile.namePattern)
                    else:
                        outputPatterns.append(outputFile.namePattern)

    return outputPatterns
Example #42
0
def getOutputSandboxPatternsForInteractive(job):
    """
    This should be used from only from Interactive backend
    """

    patternsToSandbox = [getConfig('Output')['PostProcessLocationsFileName']]
    patternsToZip = []

    for outputFile in job.outputfiles:

        outputFileClassName = getName(outputFile)

        if outputFilePostProcessingOnClient(job, outputFileClassName):
            if outputFile.compressed:
                patternsToSandbox.append('%s.gz' % outputFile.namePattern)
                patternsToZip.append(outputFile.namePattern)
            else:
                patternsToSandbox.append(outputFile.namePattern)

    return (patternsToSandbox, patternsToZip)
Example #43
0
    def __get__(self, obj, cls):
        name = getName(self)

        # If obj is None then the getter was called on the class so return the Item
        if obj is None:
            return cls._schema[name]

        if self._getter_name:
            returnable = self._bind_method(obj, self._getter_name)()
            if isinstance(returnable, GangaObject):
                returnable._setParent(self)
            return returnable


        # First we want to try to get the information without prompting a load from disk

        # ._data takes priority ALWAYS over ._index_cache
        # This access should not cause the object to be loaded
        obj_data = obj.getNodeData()
        if name in obj_data:
            return obj_data[name]

        # Then try to get it from the index cache
        obj_index = obj.getNodeIndexCache()
        if name in obj_index:
            return obj_index[name]

        # Since we couldn't find the information in the cache, we will need to fully load the object

        # Guarantee that the object is now loaded from disk
        obj._getReadAccess()

        # First try to load the object from the attributes on disk
        if name in obj.getNodeData():
            return obj.getNodeAttribute(name)

        # Finally, get the default value from the schema
        if obj._schema.hasItem(name):
            return obj._schema.getDefaultValue(name)

        raise AttributeError('Could not find attribute {0} in {1}'.format(name, obj))
Example #44
0
    def master_auto_resubmit(self, rjobs):
        '''Duplicate of the IBackend.master_resubmit but hooked into auto resubmission
        such that the monitoring server is used rather than the user server
        Args:
            rjobs (list): This is a list of jobs which are to be auto-resubmitted'''

        incomplete = 0

        def handleError(x):
            if incomplete:
                raise x
            else:
                return 0

        try:
            for sj in rjobs:
                fqid = sj.getFQID('.')
                logger.info("resubmitting job %s to %s backend", fqid,
                            getName(sj.backend))
                try:
                    b = sj.backend
                    sj.updateStatus('submitting')
                    result = b._resubmit()
                    if result:
                        sj.updateStatus('submitted')
                        # sj._commit() # PENDING: TEMPORARY DISABLED
                        incomplete = 1
                    else:
                        return handleError(
                            IncompleteJobSubmissionError(
                                fqid, 'resubmission failed'))
                except Exception as x:
                    log_user_exception(logger,
                                       debug=isType(x, GangaDiracError))
                    return handleError(
                        IncompleteJobSubmissionError(fqid, str(x)))
        finally:
            master = self.getJobObject().master
            if master:
                master.updateMasterJobStatus()
        return 1
Example #45
0
        def credCheckJobInsertor():
            def cb_Success():
                self.enableCallbackHook(credCheckJobInsertor)

            def cb_Failure():
                self.enableCallbackHook(credCheckJobInsertor)
                self._handleError('%s checking failed!' % getName(credObj),
                                  getName(credObj), False)

            log.debug('Inserting %s checking function to Qin.' %
                      getName(credObj))
            _action = JobAction(function=self.makeCredChecker(credObj),
                                callback_Success=cb_Success,
                                callback_Failure=cb_Failure)
            self.disableCallbackHook(credCheckJobInsertor)
            try:
                Qin.put(_action)
            except Exception as err:
                log.debug("makeCred Err: %s" % str(err))
                cb_Failure("Put _action failure: %s" % str(_action), "unknown",
                           True)
Example #46
0
    def __defaultActiveBackendsFunc(self):
        log.debug("__defaultActiveBackendsFunc")
        active_backends = {}
        # FIXME: this is not thread safe: if the new jobs are added then
        # iteration exception is raised
        fixed_ids = self.registry.ids()
        #log.debug("Registry: %s" % str(self.registry))
        log.debug("Running over fixed_ids: %s" % str(fixed_ids))
        for i in fixed_ids:
            try:
                j = stripProxy(self.registry(i))

                job_status = lazyLoadJobStatus(j)
                backend_obj = lazyLoadJobBackend(j)
                backend_name = getName(backend_obj)

                if job_status in ['submitted', 'running'
                                  ] or (j.master and
                                        (job_status in ['submitting'])):
                    if self.enabled is True and self.alive is True:
                        active_backends.setdefault(backend_name, [])
                        active_backends[backend_name].append(j)
            except RegistryKeyError as err:
                log.debug("RegistryKeyError: The job was most likely removed")
                log.debug("RegError %s" % str(err))
            except RegistryLockError as err:
                log.debug("RegistryLockError: The job was most likely removed")
                log.debug("Reg LockError%s" % str(err))

        summary = '{'
        for backend, these_jobs in active_backends.iteritems():
            summary += '"' + str(backend) + '" : ['
            for this_job in these_jobs:
                #stripProxy(this_job)._getWriteAccess()
                summary += str(
                    stripProxy(this_job).id) + ', '  #getFQID('.')) + ', '
            summary += '], '
        summary += '}'
        log.debug("Returning active_backends: %s" % summary)
        return active_backends
Example #47
0
def getOutputSandboxPatterns(job):
    """
    Intended for grid backends where we have to set the outputsandbox patterns for the output file types that have to be processed on the client
    """
    outputPatterns = []

    if len(job.outputfiles) > 0:

        outputPatterns.append(getConfig('Output')['PostProcessLocationsFileName'])

        for outputFile in job.outputfiles:

            outputFileClassName = getName(outputFile)

            if outputFilePostProcessingOnClient(job, outputFileClassName):
                if outputFile.namePattern not in outputPatterns:
                    if outputFile.compressed:
                        outputPatterns.append('%s.gz' % outputFile.namePattern)
                    else:
                        outputPatterns.append(outputFile.namePattern)

    return outputPatterns
Example #48
0
 def overview(self):
     """ Get an ascii art overview over task status. Can be overridden """
     task = self._getParent()
     if not task is None:
         id = str(task.transforms.index(self))
     else:
         id = "?"
     o = markup("#%s: %s '%s'\n" % (id, getName(self), self.name),
                status_colours[self.status])
     i = 0
     partitions = sorted(self._partition_status.keys())
     for c in partitions:
         s = self._partition_status[c]
         if c in self.getPartitionApps():
             failures = self.getPartitionFailures(c)
             o += markup("%i:%i " % (c, failures), overview_colours[s])
         else:
             o += markup("%i " % c, overview_colours[s])
         i += 1
         if i % 20 == 0:
             o += "\n"
     logger.info(o)
Example #49
0
    def check(self):
        self.global_lock_acquire()
        try:
            f = open(self.cntfn)
            newcount = int(f.readline())
            f.close()
            assert newcount >= self.count
            sessions = os.listdir(self.sdir)
            prevnames = set()
            for session in sessions:
                if not session.endswith(self.name + ".locks"):
                    continue
                try:
                    sf = os.path.join(self.sdir, session)
                    fd = -1
                    if not self.afs:
                        fd = os.open(sf, os.O_RDONLY)
                        fcntl.lockf(fd, fcntl.LOCK_SH)  # ONLY NFS
                    sf_file = open(sf)
                    names = pickle.load(sf_file)
                    sf_file.close()
                    if not self.afs and fd > 0:
                        fcntl.lockf(fd, fcntl.LOCK_UN)  # ONLY NFS
                        os.close(fd)
                except Exception as x:
                    logger.warning(
                        "CHECKER: session file %s corrupted: %s %s" %
                        (session, getName(x), str(x)))
                    continue
                if not len(names & prevnames) == 0:
                    logger.error("Double-locked stuff: " +
                                 str(names & prevnames))
                    assert False
                # prevnames.union_update(names) Should be alias to update but
                # not in some versions of python
                prevnames.update(names)

        finally:
            self.global_lock_release()
Example #50
0
 def check(self):
     with open(self.cntfn) as f:
         newcount = int(f.readline())
     try:
         assert newcount >= self.count
     except AssertionError:
         raise RepositoryError(
             "Count in lock file: %s is now inconsistent!" % self.cntfn)
     sessions = os.listdir(self.sdir)
     prevnames = set()
     for session in sessions:
         if not session.endswith(self.name + ".locks"):
             continue
         fd = None
         try:
             sf = self._path_helper(session)
             if not self.afs:
                 fd = os.open(sf, os.O_RDWR)
                 fcntl.lockf(fd, fcntl.LOCK_SH)  # ONLY NFS
             with open(sf) as sf_file:
                 names = pickle.load(sf_file)
             if not self.afs and fd is not None:
                 fcntl.lockf(fd, fcntl.LOCK_UN)  # ONLY NFS
         except Exception as x:
             logger.warning("CHECKER: session file %s corrupted: %s %s" %
                            (session, getName(x), x))
             continue
         finally:
             if fd is not None:
                 os.close(fd)
         if not len(names & prevnames) == 0:
             logger.error("Double-locked stuff: " + names & prevnames)
             raise RepositoryError(
                 "Lock file has double-locked objects: %s" %
                 str(names & prevnames))
         # prevnames.union_update(names) Should be alias to update but
         # not in some versions of python
         prevnames.update(names)
Example #51
0
    def __init__(self, name, display_prefix):
        super(RegistrySlice, self).__init__()
        self.objects = oDict()
        self.name = name
        self._display_prefix = display_prefix
        self._display_columns = config[self._display_prefix + '_columns']
        self._display_columns_show_empty = config[self._display_prefix +
                                                  "_columns_show_empty"]
        self._display_columns_width = config[self._display_prefix +
                                             "_columns_width"]
        self._display_columns_functions = {}
        try:
            col_funcs = config[self._display_prefix + '_columns_functions']
            for this_col_func in col_funcs:
                self._display_columns_functions[this_col_func] = eval(
                    col_funcs[this_col_func])
        except Exception as x:
            logger.error(
                "Error on evaluating display column functions from config file: %s: %s"
                % (getName(x), x))

        from Ganga.Utility.ColourText import Effects
        self._colour_normal = Effects().normal
        self._proxyClass = None
Example #52
0
    def __copyNodeObject(self, v, obj):
        """This deals with the actual deepcopy of an object which has inherited from Node class"""

        item = obj._schema[getName(self)]
        GangaList = _getGangaList()
        if isinstance(v, GangaList):
            categories = v.getCategory()
            len_cat = len(categories)
            if (len_cat > 1) or ((len_cat == 1) and (categories[0] != item['category'])) and item['category'] != 'internal':
                # we pass on empty lists, as the catagory is yet to be defined
                from Ganga.GPIDev.Base.Proxy import GangaAttributeError
                raise GangaAttributeError('%s: attempt to assign a list containing incompatible objects %s to the property in category "%s"' % (getName(self), v, item['category']))
        else:
            if v._category not in [item['category'], 'internal'] and item['category'] != 'internal':
                from Ganga.GPIDev.Base.Proxy import GangaAttributeError
                raise GangaAttributeError('%s: attempt to assign an incompatible object %s to the property in category "%s found cat: %s"' % (getName(self), v, item['category'], v._category))


        v_copy = deepcopy(v)

        #logger.info("Cloned Object Parent: %s" % v_copy._getParent())
        #logger.info("Original: %s" % v_copy._getParent())

        return v_copy
Example #53
0
    def __deepcopy__(self, memo=None):
        true_parent = self._getParent()
        ## This triggers a read of the job from disk
        self._getReadAccess()
        classname = getName(self)
        category = self._category
        cls = self.__class__#allPlugins.find(category, classname)

        self_copy = cls()

        if self._schema is not None:
            for name, item in self._schema.allItems():
                if not item['copyable']:
                    setattr(self_copy, name, self._schema.getDefaultValue(name))
                else:
                    if hasattr(self, name):
                        setattr(self_copy, name, deepcopy(getattr(self, name)))
                    else:
                        setattr(self_copy, name, self._schema.getDefaultValue(name))

                this_attr = getattr(self_copy, name)
                if isinstance(this_attr, Node):
                    this_attr._setParent(self_copy)

                if item.isA(Schema.SharedItem):
                    self.__incrementShareRef(self_copy, name)

        global do_not_copy
        for k, v in self.__dict__.iteritems():
            if k not in do_not_copy:
                setattr(self_copy, k, deepcopy(v))

        if true_parent is not None:
            self._setParent(true_parent)
            self_copy._setParent(true_parent)
        return self_copy
Example #54
0
    def validatedSplit(self, job):
        """ Perform splitting using the split() method and validate the mutability
        invariants. If the invariants are broken (or exception occurs in the
        split() method) then SplittingError exception is raised. This method is
        called directly by the framework and should not be modified in the derived
        classes. """

        # try:
        subjobs = self.split(stripProxy(job))
        # except Exception,x:
        #raise SplittingError(x)
        #raise x
        # if not len(subjobs):
        #raise SplittingError('splitter did not create any subjobs')

        cnt = 0
        for s in subjobs:
            if not isType(s.backend, type(stripProxy(job.backend))):
                raise SplittingError(
                    'masterjob backend %s is not the same as the subjob (probable subjob id=%d) backend %s'
                    % (job.backend._name, cnt, getName(s.backend)))
            cnt += 1

        return subjobs
Example #55
0
    def __setattr__(self, attr, value):

        if attr == 'outputfiles':

            if value != []:
                if self.outputdata is not None:
                    logger.error(
                        'ITransform.outputdata is set, you can\'t set ITransform.outputfiles')
                    return
                elif self.outputsandbox != []:
                    logger.error(
                        'ITransform.outputsandbox is set, you can\'t set ITransform.outputfiles')
                    return

            # reduce duplicate values here, leave only duplicates for LCG,
            # where we can have replicas
            uniqueValuesDict = []
            uniqueValues = []

            for val in value:
                key = '%s%s' % (getName(val), val.namePattern)
                if key not in uniqueValuesDict:
                    uniqueValuesDict.append(key)
                    uniqueValues.append(val)
                elif getName(val) == 'LCGSEFile':
                    uniqueValues.append(val)

            super(ITransform, self).__setattr__(attr, uniqueValues)

        elif attr == 'inputfiles':

            if value != []:
                if self.inputsandbox != []:
                    logger.error(
                        'ITransform.inputsandbox is set, you can\'t set ITransform.inputfiles')
                    return

            super(ITransform, self).__setattr__(attr, value)

        elif attr == 'outputsandbox':

            if value != []:

                if getConfig('Output')['ForbidLegacyOutput']:
                    logger.error(
                        'Use of ITransform.outputsandbox is forbidden, please use ITransform.outputfiles')
                    return

                if self.outputfiles != []:
                    logger.error(
                        'ITransform.outputfiles is set, you can\'t set ITransform.outputsandbox')
                    return

            super(ITransform, self).__setattr__(attr, value)

        elif attr == 'inputsandbox':

            if value != []:

                if getConfig('Output')['ForbidLegacyInput']:
                    logger.error(
                        'Use of ITransform.inputsandbox is forbidden, please use ITransform.inputfiles')
                    return

                if self.inputfiles != []:
                    logger.error(
                        'ITransform.inputfiles is set, you can\'t set ITransform.inputsandbox')
                    return

            super(ITransform, self).__setattr__(attr, value)

        elif attr == 'outputdata':

            if value is not None:

                if getConfig('Output')['ForbidLegacyOutput']:
                    logger.error(
                        'Use of ITransform.outputdata is forbidden, please use ITransform.outputfiles')
                    return

                if self.outputfiles != []:
                    logger.error(
                        'ITransform.outputfiles is set, you can\'t set ITransform.outputdata')
                    return
            super(ITransform, self).__setattr__(attr, value)

        else:
            super(ITransform, self).__setattr__(attr, value)
Example #56
0
    def __str__(self, short=True, id=None):
        """Prints an overview over the currently running tasks"""
        if Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_show_help"]:
            self.help(short=True)
            Ganga.GPIDev.Lib.Registry.RegistrySlice.config.setUserValue(
                "tasks_show_help", False)
            print("To show this help message again, type 'tasks.help()'.")
            print('')
            print(" The following is the output of " +
                  markup("tasks.table()", fgcol("blue")))

        lenfstring = 0
        flist = []
        for thing in Ganga.GPIDev.Lib.Registry.RegistrySlice.config[
                "tasks_columns"]:
            width = Ganga.GPIDev.Lib.Registry.RegistrySlice.config[
                "tasks_columns_width"][thing]
            lenfstring += width
            flist.append("%" + str(width) + "s ")

        fstring = "|".join(flist)
        fstring += '\n'
        lenfstring += 27
        ds = fstring % ("#", "Type", "Name", "State", "Comment",
                        "%4s: %4s/ %4s/ %4s/ %4s/ %4s/ %4s/ %4s" %
                        ("Jobs", markup("done", overview_colours["completed"]),
                         " " + markup("run", overview_colours["running"]),
                         " " + markup("subd", overview_colours["submitted"]),
                         " " + markup("attd", overview_colours["attempted"]),
                         markup("fail", overview_colours["failed"]),
                         markup("hold", overview_colours["hold"]), " " +
                         markup("bad", overview_colours["bad"])), "Float")
        ds += "-" * lenfstring + "\n"

        from Ganga.GPIDev.Lib.Tasks import ITask

        if id is not None and type(id) is int:
            iterable_list = [self[id]]
        else:
            iterable_list = stripProxy(self).objects.values()

        for p in iterable_list:

            if isType(p, ITask):
                stat = "%4i: %4i/ %4i/  %4i/    --/ %4i/ %4i/ %4i" % (
                    p.n_all(), p.n_status("completed"), p.n_status("running"),
                    p.n_status("submitted"), p.n_status("failed"),
                    p.n_status("hold"), p.n_status("bad"))
                ds += markup(
                    fstring %
                    (p.id, getName(p),
                     p.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.
                            config['tasks_columns_width']['Name']], p.status,
                     p.comment, stat, p.float), status_colours[p.status])
            else:
                stat = "%4i: %4i/ %4i/    --/  %4i/ %4i/ %4i/ %4i" % (
                    p.n_all(), p.n_status("completed"), p.n_status("running"),
                    p.n_status("attempted"), p.n_status("failed"),
                    p.n_status("hold"), p.n_status("bad"))
                ds += markup(
                    fstring %
                    (p.id, getName(p),
                     p.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.
                            config['tasks_columns_width']['Name']], p.status,
                     p.comment, stat, p.float), status_colours[p.status])

            if short is True:
                continue

            for ti in range(0, len(p.transforms)):
                t = p.transforms[ti]

                if isType(p, ITask):
                    stat = "%4i: %4i/ %4i/ %4i/     --/ %4i/ %4i/ %4s" % (
                        t.n_all(), t.n_status("completed"),
                        t.n_status("running"), t.n_status("submitted"),
                        t.n_status("failed"), t.n_status("hold"),
                        t.n_status("bad"))
                    ds += "\n" + markup(
                        fstring %
                        ("%i.%i" % (p.id, ti), getName(t),
                         t.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.
                                config['tasks_columns_width']['Name']],
                         t.status, "", stat, ""), status_colours[t.status])
                else:
                    stat = "%4i: %4i/ %4i/     --/ %4i/ %4i/ %4i/ %4s" % (
                        t.n_all(), t.n_status("completed"),
                        t.n_status("running"), t.n_status("attempted"),
                        t.n_status("failed"), t.n_status("hold"),
                        t.n_status("bad"))
                    ds += "zn" + markup(
                        fstring %
                        ("%i.%i" % (p.id, ti), getName(t),
                         t.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.
                                config['tasks_columns_width']['Name']],
                         t.status, "", stat, ""), status_colours[t.status])

            ds += "-" * lenfstring + "\n"

        return ds + "\n"
Example #57
0
    def report_inner(job=None, isJob=False, isTask=False):

        userInfoDirName = "userreport"
        tempDirName = "reportsRepository"
        # job relevant info
        jobSummaryFileName = "jobsummary.txt"
        jobFullPrintFileName = "jobfullprint.txt"
        repositoryPath = "repository/$usr/LocalXML/6.0/jobs/$thousandsNumxxx"
        # task relevant info
        taskSummaryFileName = "tasksummary.txt"
        taskFullPrintFileName = "taskfullprint.txt"
        tasksRepositoryPath = "repository/$usr/LocalXML/6.0/tasks/$thousandsNumxxx"
        # user's info
        environFileName = "environ.txt"
        userConfigFileName = "userconfig.txt"
        defaultConfigFileName = "gangarc.txt"
        ipythonHistoryFileName = "ipythonhistory.txt"
        gangaLogFileName = "gangalog.txt"
        jobsListFileName = "jobslist.txt"
        tasksListFileName = "taskslist.txt"
        from Ganga.Utility import Config
        uploadFileServer = Config.getConfig('Feedback')['uploadServer']
        #uploadFileServer= "http://gangamon.cern.ch/django/errorreports/"
        #uploadFileServer= "http://ganga-ai-02.cern.ch/django/errorreports/"
        #uploadFileServer= "http://127.0.0.1:8000/errorreports"

        def printDictionary(dictionary, file=sys.stdout):
            for k, v in dictionary.iteritems():
                print('%s: %s' % (k, v), file=file)

                if k == 'PYTHONPATH':
                    global PYTHON_PATH
                    PYTHON_PATH = v

        def extractFileObjects(fileName, targetDirectoryName):
            try:
                fileToRead = open(fileName, 'r')
                try:
                    fileText = fileToRead.read()
                    import re
                    pattern = "File\(name=\'(.+?)\'"
                    matches = re.findall(pattern, fileText)

                    for fileName in matches:
                        fileName = os.path.expanduser(fileName)
                        targetFileName = os.path.join(
                            targetDirectoryName, os.path.basename(fileName))
                        shutil.copyfile(fileName, targetFileName)

                finally:
                    fileToRead.close()
            # except IOError, OSError:
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

        def writeErrorLog(errorMessage):
            try:
                fileToWrite = open(errorLogPath, 'a')
                try:
                    fileToWrite.write(errorMessage)
                    fileToWrite.write("\n")
                except Exception as err:
                    logger.debug("Err: %s" % str(err))
                    raise
                finally:
                    fileToWrite.close()
            except Exception as err2:
                logger.debug("Err: %s" % str(err2))
                pass

        def writeStringToFile(fileName, stringToWrite):

            try:
                # uncomment this to try the error logger
                #fileName = '~/' + fileName
                fileToWrite = open(fileName, 'w')
                try:
                    fileToWrite.write(stringToWrite)
                except Exception as err:
                    logger.debug("Err: %s" % str(err))
                    raise err
                finally:
                    fileToWrite.close()
            # except IOError:
            except Exception as err:
                logger.debug("Err2: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

        def renameDataFiles(directory):

            for fileName in os.listdir(directory):
                fullFileName = os.path.join(directory, fileName)
                if os.path.isfile(fullFileName):
                    if fileName == 'data':
                        os.rename(fullFileName, fullFileName + '.txt')
                else:
                    renameDataFiles(fullFileName)

        import shutil
        import tarfile
        import tempfile
        import os

        userHomeDir = os.getenv("HOME")
        tempDir = tempfile.mkdtemp()

        errorLogPath = os.path.join(tempDir, 'reportErrorLog.txt')

        fullPathTempDir = os.path.join(tempDir, tempDirName)
        fullLogDirName = ''
        # create temp dir and specific dir for the job/user

        try:
            if not os.path.exists(fullPathTempDir):
                os.mkdir(fullPathTempDir)

            import datetime
            now = datetime.datetime.now()
            userInfoDirName = userInfoDirName + \
                now.strftime("%Y-%m-%d-%H:%M:%S")
            fullLogDirName = os.path.join(fullPathTempDir, userInfoDirName)

            # if report directory exists -> delete it's content(we would like
            # last version of the report)
            if os.path.exists(fullLogDirName):
                shutil.rmtree(fullLogDirName)

            os.mkdir(fullLogDirName)
        # except OSError:
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # import os.environ in a file
        fullEnvironFileName = os.path.join(fullLogDirName, environFileName)

        try:
            inputFile = open(fullEnvironFileName, 'w')
            try:
                printDictionary(os.environ, file=inputFile)

                print('OS VERSION : ' + platform.platform(), file=inputFile)

            finally:
                inputFile.close()
        # except IOError
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # import user config in a file
        userConfigFullFileName = os.path.join(
            fullLogDirName, userConfigFileName)

        try:
            inputFile = open(userConfigFullFileName, 'w')
            try:

                print("#GANGA_VERSION = %s" %
                      config.System.GANGA_VERSION, file=inputFile)

                global GANGA_VERSION
                GANGA_VERSION = config.System.GANGA_VERSION

                # this gets the default values
                # Ganga.GPIDev.Lib.Config.Config.print_config_file()

                # this should get the changed values
                for c in config:
                    print(config[c], file=inputFile)

            finally:
                inputFile.close()
        # except IOError does not catch the exception ???
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # write gangarc - default configuration
        defaultConfigFullFileName = os.path.join(
            fullLogDirName, defaultConfigFileName)

        try:
            outputFile = open(os.path.join(userHomeDir, '.gangarc'), 'r')

            try:
                writeStringToFile(defaultConfigFullFileName, outputFile.read())
            finally:
                outputFile.close()

        # except IOError does not catch the exception ???
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # import ipython history in a file
        try:
            ipythonFile = open(
                os.path.join(os.environ['IPYTHONDIR'], 'history'), 'r')

            try:
                lastIPythonCommands = ipythonFile.readlines()[-20:]
                writeStringToFile(os.path.join(
                    fullLogDirName, ipythonHistoryFileName), '\n'.join(lastIPythonCommands))
                #writeStringToFile(os.path.join(fullLogDirName, ipythonHistoryFileName), ipythonFile.read())
            finally:
                ipythonFile.close()
        # except IOError does not catch the exception ???
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # import gangalog in a file
        userLogFileLocation = config["Logging"]._logfile
        userLogFileLocation = os.path.expanduser(userLogFileLocation)

        try:
            gangaLogFile = open(userLogFileLocation, 'r')
            try:
                writeStringToFile(
                    os.path.join(fullLogDirName, gangaLogFileName), gangaLogFile.read())
            finally:
                gangaLogFile.close()
        # except IOError:
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # import the result of jobs command in the report
        jobsListFullFileName = os.path.join(fullLogDirName, jobsListFileName)

        try:
            outputFile = open(jobsListFullFileName, 'w')
            try:

                from Ganga.GPI import jobs
                print(jobs, file=outputFile)

            finally:
                outputFile.close()

        # except IOError does not catch the exception ???
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # import the result of tasks command in the report
        tasksListFullFileName = os.path.join(fullLogDirName, tasksListFileName)

        try:
            outputFile = open(tasksListFullFileName, 'w')
            try:

                from Ganga.GPI import tasks
                print(tasks, file=outputFile)

            finally:
                outputFile.close()

        # except IOError does not catch the exception ???
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # save it here because we will change fullLogDirName, but we want this
        # to be the archive and to be deleted
        folderToArchive = fullLogDirName

        # import job relevant info
        if (job is not None and isJob):

            global JOB_REPORT, APPLICATION_NAME, BACKEND_NAME

            JOB_REPORT = True
            APPLICATION_NAME = getName(job.application)
            BACKEND_NAME = getName(job.backend)

            # create job folder
            jobFolder = 'job_%s' % str(job.fqid)
            fullLogDirName = os.path.join(fullLogDirName, jobFolder)
            os.mkdir(fullLogDirName)

            # import job summary in a file
            fullJobSummaryFileName = os.path.join(
                fullLogDirName, jobSummaryFileName)
            writeStringToFile(fullJobSummaryFileName, str(job))

            # import job full print in a file
            fullJobPrintFileName = os.path.join(
                fullLogDirName, jobFullPrintFileName)

            try:
                inputFile = open(fullJobPrintFileName, 'w')
                try:
                    full_print(job, inputFile)
                finally:
                    inputFile.close()
            # except IOError, OSError:
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

            # extract file objects
            try:
                fileObjectsPath = os.path.join(fullLogDirName, 'fileobjects')
                os.mkdir(fileObjectsPath)
                extractFileObjects(fullJobSummaryFileName, fileObjectsPath)
            # except OSError:
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

            # copy dir of the job ->input/output and subjobs
            try:
                parentDir, currentDir = os.path.split(job.inputdir[:-1])
                workspaceDir = os.path.join(fullLogDirName, 'workspace')
                shutil.copytree(parentDir, workspaceDir)
            # except IOError, OSError
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

            # copy shared area of the job
            try:

                if hasattr(job.application, 'is_prepared'):
                    if job.application.is_prepared is not None and job.application.is_prepared is not True:
                        import os
                        from Ganga.Utility.Config import getConfig
                        from Ganga.Utility.files import expandfilename
                        shared_path = os.path.join(expandfilename(getConfig(
                            'Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'])
                        shareddir = os.path.join(
                            shared_path, job.application.is_prepared.name)
                        if os.path.isdir(shareddir):

                            sharedAreaDir = os.path.join(
                                fullLogDirName, 'sharedarea')
                            shutil.copytree(shareddir, sharedAreaDir)
            # except IOError, OSError
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

            # copy repository job file
            try:
                indexFileName = str(job.id) + '.index'

                repositoryPath = repositoryPath.replace(
                    '$usr', os.getenv("USER"))

                # check if the job is subjob -> different way of forming the
                # path to the repository
                is_subjob = job.fqid.find('.') > -1

                if is_subjob:

                    jobid, subjobid = job.fqid.split(
                        '.')[0], job.fqid.split('.')[1]
                    repositoryPath = repositoryPath.replace(
                        '$thousandsNum', str(int(jobid) / 1000))
                    repositoryPath = os.path.join(repositoryPath, jobid)

                else:
                    repositoryPath = repositoryPath.replace(
                        '$thousandsNum', str(job.id / 1000))

                repositoryFullPath = os.path.join(
                    config.Configuration.gangadir, repositoryPath)
                indexFileSourcePath = os.path.join(
                    repositoryFullPath, indexFileName)
                repositoryFullPath = os.path.join(
                    repositoryFullPath, str(job.id))

                repositoryTargetPath = os.path.join(
                    fullLogDirName, 'repository', str(job.id))

                os.mkdir(os.path.join(fullLogDirName, 'repository'))

                shutil.copytree(repositoryFullPath, repositoryTargetPath)
                # data files are copied but can not be opened -> add .txt to
                # their file names
                renameDataFiles(repositoryTargetPath)

                if not is_subjob:
                    # copy .index file
                    indexFileTargetPath = os.path.join(
                        fullLogDirName, 'repository', indexFileName)
                    shutil.copyfile(indexFileSourcePath, indexFileTargetPath)

            # except OSError, IOError:
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

        # import task relevant info
        if (job is not None and isTask):
            # job is actually a task object
            task = job
            # create task folder
            taskFolder = 'task_%s' % str(task.id)
            fullLogDirName = os.path.join(fullLogDirName, taskFolder)
            os.mkdir(fullLogDirName)

            # import task summary in a file
            fullTaskSummaryFileName = os.path.join(
                fullLogDirName, taskSummaryFileName)
            writeStringToFile(fullTaskSummaryFileName, str(task))

            # import task full print in a file
            fullTaskPrintFileName = os.path.join(
                fullLogDirName, taskFullPrintFileName)

            try:
                inputFile = open(fullTaskPrintFileName, 'w')
                try:
                    full_print(task, inputFile)
                except Exception as err:
                    logger.debug("Err: %s" % str(err))
                    raise err
                finally:
                    inputFile.close()
            # except IOError, OSError:
            except Exception as err:
                logger.debug("Err2: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

            # copy shared area of the task
            try:
                if len(task.transforms) > 0:
                    if hasattr(task.transforms[0], 'application') and hasattr(task.transforms[0].application, 'is_prepared'):
                        if task.transforms[0].application.is_prepared is not None and task.transforms[0].application.is_prepared is not True:
                            import os
                            from Ganga.Utility.Config import getConfig
                            from Ganga.Utility.files import expandfilename
                            shared_path = os.path.join(expandfilename(getConfig(
                                'Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'])
                            shareddir = os.path.join(
                                shared_path, task.transforms[0].application.is_prepared.name)
                            if os.path.isdir(shareddir):

                                sharedAreaDir = os.path.join(
                                    fullLogDirName, 'sharedarea')
                                shutil.copytree(shareddir, sharedAreaDir)
            # except IOError, OSError
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

            # copy repository task file
            try:
                indexFileName = str(task.id) + '.index'

                tasksRepositoryPath = tasksRepositoryPath.replace(
                    '$usr', os.getenv("USER"))
                tasksRepositoryPath = tasksRepositoryPath.replace(
                    '$thousandsNum', str(task.id / 1000))

                repositoryFullPath = os.path.join(
                    config.Configuration.gangadir, tasksRepositoryPath)
                indexFileSourcePath = os.path.join(
                    repositoryFullPath, indexFileName)
                repositoryFullPath = os.path.join(
                    repositoryFullPath, str(task.id))

                repositoryTargetPath = os.path.join(
                    fullLogDirName, 'repository', str(task.id))

                os.mkdir(os.path.join(fullLogDirName, 'repository'))

                shutil.copytree(repositoryFullPath, repositoryTargetPath)
                # data files are copied but can not be opened -> add .txt to
                # their file names
                renameDataFiles(repositoryTargetPath)

                # copy .index file
                indexFileTargetPath = os.path.join(
                    fullLogDirName, 'repository', indexFileName)
                shutil.copyfile(indexFileSourcePath, indexFileTargetPath)

            # except OSError, IOError:
            except Exception as err:
                logger.debug("Err %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

        resultArchive = '%s.tar.gz' % folderToArchive

        try:
            resultFile = tarfile.TarFile.open(resultArchive, 'w:gz')
            try:
                resultFile.add(
                    folderToArchive, arcname=os.path.basename(folderToArchive))
                # put the error log in the archive
                if(os.path.exists(errorLogPath)):
                    resultFile.add(
                        errorLogPath, arcname=os.path.basename(errorLogPath))
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                raise
            finally:
                resultFile.close()
        except Exception as err:
            logger.debug("Err2: %s" % str(err))
            raise  # pass

        # remove temp dir
        if(os.path.exists(folderToArchive)):
            shutil.rmtree(folderToArchive)

        # print the error if there is something
        if os.path.exists(errorLogPath):
            logger.error('')
            logger.error('An error occured while collecting report information : ' + open(errorLogPath, 'r').read())
            logger.error('')

        # delete the errorfile from user's pc
        if(os.path.exists(errorLogPath)):
            os.remove(errorLogPath)

        # return the path to the archive and the path to the upload server
        return (resultArchive, uploadFileServer, tempDir)
Example #58
0
 def info(self):
     logger.info(markup("%s '%s'" % (getName(self), self.name), status_colours[self.status]))
     logger.info("* backend: %s" % getName(self.backend))
     logger.info("Application:")
     self.application.printTree()
Example #59
0
    def update_index(self, this_id=None, verbose=False, firstRun=False):
        """ Update the list of available objects
        Raise RepositoryError"""
        # First locate and load the index files
        logger.debug("updating index...")
        objs = self.get_index_listing()
        changed_ids = []
        deleted_ids = set(self.objects.keys())
        summary = []
        if firstRun:
            self._read_master_cache()
        logger.debug("Iterating over Items")

        locked_ids = self.sessionlock.locked

        for this_id in objs.keys():
            deleted_ids.discard(this_id)
            # Make sure we do not overwrite older jobs if someone deleted the
            # count file
            if this_id > self.sessionlock.count:
                self.sessionlock.count = this_id + 1
            # Locked IDs can be ignored
            if this_id in locked_ids:
                continue
            # Now we treat unlocked IDs
            try:
                # if this succeeds, all is well and we are done
                if self.index_load(this_id):
                    changed_ids.append(this_id)
                continue
            except IOError as err:
                logger.debug("IOError: Failed to load index %i: %s" %
                             (this_id, str(err)))
            except OSError as err:
                logger.debug("OSError: Failed to load index %i: %s" %
                             (this_id, str(err)))
            except PluginManagerError as err:
                # Probably should be DEBUG
                logger.debug(
                    "PluginManagerError: Failed to load index %i: %s" %
                    (this_id, str(err)))
                # This is a FATAL error - do not try to load the main file, it
                # will fail as well
                summary.append((this_id, err))
                continue

            # this is bad - no or corrupted index but object not loaded yet!
            # Try to load it!
            if not this_id in self.objects:
                try:
                    self.load([this_id])
                    changed_ids.append(this_id)
                    # Write out a new index if the file can be locked
                    if len(self.lock([this_id])) != 0:
                        self.index_write(this_id)
                        self.unlock([this_id])
                except KeyError as err:
                    logger.debug("update Error: %s" % str(err))
                    # deleted job
                    if this_id in self.objects:
                        self._internal_del__(this_id)
                        changed_ids.append(this_id)
                except Exception as x:
                    ## WE DO NOT CARE what type of error occured here and it can be
                    ## due to corruption so could be one of MANY exception types
                    ## If the job is not accessible this should NOT cause the loading of ganga to fail!
                    ## we can't reasonably write all possible exceptions here!
                    logger.debug("update_index: Failed to load id %i: %s" %
                                 (this_id, str(x)))
                    summary.append((this_id, str(x)))

        logger.debug("Iterated over Items")

        # Check deleted files:
        for this_id in deleted_ids:
            self._internal_del__(this_id)
            changed_ids.append(this_id)
        if len(deleted_ids) > 0:
            logger.warning(
                "Registry '%s': Job %s externally deleted." %
                (self.registry.name, ",".join(map(str, list(deleted_ids)))))

        if len(summary) > 0:
            cnt = {}
            examples = {}
            for this_id, x in summary:
                if this_id in self.known_bad_ids:
                    continue
                cnt[getName(x)] = cnt.get(getName(x), []) + [str(this_id)]
                examples[getName(x)] = str(x)
                self.known_bad_ids.append(this_id)
                # add object to incomplete_objects
                if not this_id in self.incomplete_objects:
                    self.incomplete_objects.append(this_id)

            for exc, ids in cnt.items():
                logger.error(
                    "Registry '%s': Failed to load %i jobs (IDs: %s) due to '%s' (first error: %s)"
                    % (self.registry.name, len(ids), ",".join(ids), exc,
                       examples[exc]))

            if self.printed_explanation is False:
                logger.error(
                    "If you want to delete the incomplete objects, you can type:\n"
                )
                logger.error(
                    "'for i in %s.incomplete_ids(): %s(i).remove()'\n (then press 'Enter' twice)"
                    % (self.registry.name, self.registry.name))
                logger.error(
                    "WARNING!!! This will result in corrupt jobs being completely deleted!!!"
                )
                self.printed_explanation = True
        logger.debug("updated index done")

        if len(changed_ids) != 0:
            isShutdown = not firstRun
            self._write_master_cache(shutdown=isShutdown)

        return changed_ids
Example #60
0
    def getWNInjectedScript(self, outputFiles, indent, patternsToZip,
                            postProcessLocationsFP):
        """
        Returns script that have to be injected in the jobscript for postprocessing on the WN
        """

        script_path = os.path.dirname(
            os.path.abspath(inspect.getfile(inspect.currentframe())))
        script_location = os.path.join(script_path, 'uploadScript.py.template')

        upload_script = FileUtils.loadScript(script_location, '')

        WNscript_location = os.path.join(script_path,
                                         'WNInjectTemplate.py.template')
        script = FileUtils.loadScript(WNscript_location, '')

        if not self.remoteDir:
            try:
                job = self.getJobObject()
                lfn_folder = os.path.join("GangaJob_%s" % job.getFQID('.'),
                                          "OutputFiles")
            except AssertionError:
                t = datetime.datetime.now()
                this_date = t.strftime("%H.%M_%A_%d_%B_%Y")
                lfn_folder = os.path.join('GangaFiles_%s' % this_date)
            lfn_base = os.path.join(
                DiracFile.diracLFNBase(self.credential_requirements),
                lfn_folder)
        else:
            lfn_base = oa.path.join(
                DiracFile.diracLFNBase(self.credential_requirements),
                self.remoteDir)

        for this_file in outputFiles:
            isCompressed = this_file.namePattern in patternsToZip

            if not regex.search(this_file.namePattern) is None:
                script += self._WN_wildcard_script(this_file.namePattern,
                                                   lfn_base, str(isCompressed))
            else:
                script += '###INDENT###print("Uploading: %s as: %s")\n' % (
                    this_file.namePattern,
                    str(os.path.join(lfn_base, this_file.namePattern)))
                script += '###INDENT###processes.append(uploadFile("%s", "%s", %s))\n' % (
                    this_file.namePattern, lfn_base, str(isCompressed))

        if stripProxy(self)._parent is not None and stripProxy(
                self).getJobObject() and getName(
                    stripProxy(self).getJobObject().backend) != 'Dirac':
            script_env = self._getDiracEnvStr()
        else:
            script_env = str(None)

        script = '\n'.join(
            [str('###INDENT###' + str(line)) for line in script.split('\n')])

        replace_dict = {
            '###UPLOAD_SCRIPT###': upload_script,
            '###STORAGE_ELEMENTS###': str(configDirac['allDiracSE']),
            '###INDENT###': indent,
            '###LOCATIONSFILE###': postProcessLocationsFP,
            '###DIRAC_ENV###': script_env
        }

        for k, v in replace_dict.iteritems():
            script = script.replace(str(k), str(v))

        return script