def __init__(self, job, time_created): self.job_json = get_job_JSON(job) self.time_created = time_created self.job_status = job.status self.job_application = getName(job.application) self.job_backend = getName(job.backend)
def create_subjobs_graphics(jobid, subjob_attribute, fromDate, toDate): subjobs_in_time_range = get_subjobs_in_time_range(jobid, fromDate, toDate) if subjob_attribute == 'accumulate': # return some JSON here return get_accumulated_subjobs_JSON(subjobs_in_time_range) subjobs_attributes = {} for subjob in subjobs_in_time_range: if subjob_attribute == 'status': increment(subjobs_attributes, subjob.status) elif subjob_attribute == 'application': increment( subjobs_attributes, getName(subjob.application)) elif subjob_attribute == 'backend': increment(subjobs_attributes, getName(subjob.backend)) elif subjob_attribute == 'actualCE': increment(subjobs_attributes, subjob.backend.actualCE) if subjob_attribute == 'status': return get_pie_chart_json(subjobs_attributes, colors=True, jobs=False) else: return get_pie_chart_json(subjobs_attributes)
def isCredentialRequired(credObj): """ The logic to decide if a given invalid credential should trigger the deactivation of Ganga internal services. """ from Ganga.Runtime import Workspace_runtime from Ganga.Runtime import Repository_runtime if getName(credObj) == 'AfsToken': return Workspace_runtime.requiresAfsToken() or Repository_runtime.requiresAfsToken() if getName(credObj) == 'GridProxy': from Ganga.Core.GangaRepository import getRegistryProxy from Ganga.Runtime.GPIFunctions import typename from Ganga.GPIDev.Base.Proxy import stripProxy from Ganga.GPIDev.Lib.Job.Job import lazyLoadJobBackend, lazyLoadJobStatus for j in getRegistryProxy('jobs'): ji = stripProxy(j) this_status = lazyLoadJobStatus(ji) if this_status in ['submitted', 'running', 'completing']: this_backend = lazyLoadJobBackend(ji) if getName(this_backend) == 'LCG': return True return False log.warning("Unknown credential object : %s" % credObj)
def isCredentialRequired(credObj): """ The logic to decide if a given invalid credential should trigger the deactivation of Ganga internal services. """ from Ganga.Runtime import Workspace_runtime from Ganga.Runtime import Repository_runtime if getName(credObj) == 'AfsToken': return Workspace_runtime.requiresAfsToken() or Repository_runtime.requiresAfsToken() if getName(credObj) == 'GridProxy': if Repository_runtime.requiresGridProxy() or Workspace_runtime.requiresGridProxy(): return True from Ganga.GPI import jobs, typename from Ganga.GPIDev.Base.Proxy import stripProxy for j in jobs: ji = stripProxy(j) if ji.status in ['submitted', 'running', 'completing']: if ji.getNodeIndexCache() is not None and 'display:backend' in ji.getNodeIndexCache().keys(): if ji.getNodeIndexCache()['display:backend'] == 'LCG': return True else: if getName(ji.backend) == 'LCG': return True return False log.warning("Unknown credential object : %s" % credObj)
def isCredentialRequired(credObj): """ The logic to decide if a given invalid credential should trigger the deactivation of Ganga internal services. """ from Ganga.Runtime import Workspace_runtime from Ganga.Runtime import Repository_runtime if getName(credObj) == "AfsToken": return Workspace_runtime.requiresAfsToken() or Repository_runtime.requiresAfsToken() if getName(credObj) == "GridProxy": if Repository_runtime.requiresGridProxy() or Workspace_runtime.requiresGridProxy(): return True from Ganga.GPI import jobs, typename from Ganga.GPIDev.Base.Proxy import stripProxy for j in jobs: ji = stripProxy(j) if ji.status in ["submitted", "running", "completing"]: if getName(ji.backend) == "LCG": return True return False log.warning("Unknown credential object : %s" % credObj)
def upload(self, files=[], opts=""): """ Uploads multiple files to a remote grid storage. @param files is a list of local files to be uploaded to the grid. The elemement can be a file path or a File object. @return True if files are successfully uploaded; otherwise it returns False """ status = False paths = [] for f in files: if getName(f) == "File": paths.append("file://%s" % f.name) elif getName(f) == "str": paths.append("file://%s" % f) else: self.logger.warning("unknown file expression: %s" % repr(f)) uploaded_files = self.impl_upload(files=paths, opts=opts) if len(uploaded_files) == len(files): status = self.impl_bookkeepUploadedFiles(uploaded_files, append=True, opts=opts) else: status = False if len(uploaded_files) == len(files): status = self.impl_bookkeepUploadedFiles(uploaded_files, append=True, opts=opts) else: status = False return status
def getWNCodeForOutputPostprocessing(job, indent): # dict containing the list of outputfiles that need to be processed on the # WN for every file type outputFilesProcessedOnWN = {} patternsToZip = [] if len(job.outputfiles) == 0: return "" else: for outputFile in job.outputfiles: outputfileClassName = getName(outputFile) backendClassName = getName(job.backend) if outputFile.compressed: patternsToZip.append(outputFile.namePattern) if outputfileClassName not in outputFilesProcessedOnWN.keys(): outputFilesProcessedOnWN[outputfileClassName] = [] if outputFilePostProcessingOnWN(job, outputfileClassName): outputFilesProcessedOnWN[outputfileClassName].append(outputFile) if not patternsToZip: if not any(outputFilesProcessedOnWN.values()): return "" logger.debug("Process: '%s' on WN" % str(outputFilePostProcessingOnWN)) shortScript = """\n import os, glob for patternToZip in ###PATTERNSTOZIP###: for currentFile in glob.glob(os.path.join(os.getcwd(),patternToZip)): if os.path.isfile(currentFile): os.system("gzip %s" % currentFile) postprocesslocations = open(os.path.join(os.getcwd(), '###POSTPROCESSLOCATIONSFILENAME###'), 'a+') """ from Ganga.GPIDev.Lib.File import FileUtils shortScript = FileUtils.indentScript(shortScript, '###INDENT###') insertScript = shortScript insertScript = insertScript.replace('###PATTERNSTOZIP###', str(patternsToZip)) insertScript = insertScript.replace('###POSTPROCESSLOCATIONSFILENAME###', getConfig('Output')['PostProcessLocationsFileName']) for outputFileName in outputFilesProcessedOnWN.keys(): if len(outputFilesProcessedOnWN[outputFileName]) > 0: insertScript += outputFilesProcessedOnWN[outputFileName][0].getWNInjectedScript(outputFilesProcessedOnWN[outputFileName], indent, patternsToZip, 'postprocesslocations') insertScript += """\n ###INDENT###postprocesslocations.close() """ insertScript = insertScript.replace('###INDENT###', indent) return insertScript
def notifyInvalidCredential(credObj): """ The Core is notified when one of the monitored credentials is invalid @see ICredential.create() """ # ignore this notification if the internal services are already stopped if not servicesEnabled: log.debug( "One of the monitored credential [%s] is invalid BUT the internal services are already disabled." % getName(credObj)) return if isCredentialRequired(credObj): log.debug("One of the required credential for the internal services is invalid: [%s]." "Disabling internal services ..." % getName(credObj)) _tl = credObj.timeleft() if _tl == "-1": log.error('%s has been destroyed! Could not shutdown internal services.' % getName(credObj)) return disableInternalServices() log.warning('%s is about to expire! ' 'To protect against possible write errors all internal services has been disabled.' 'If you believe the problem has been solved type "reactivate()" to re-enable ' 'interactions within this session.' % getName(credObj)) else: log.debug("One of the monitored credential [%s] is invalid BUT it is not required by the internal services" % getName(credObj))
def getWNCodeForDownloadingInputFiles(job, indent): """ Generate the code to be run on the WN to download input files """ from Ganga.GPIDev.Lib.Dataset.GangaDataset import GangaDataset if ( job.inputfiles is None or len(job.inputfiles) == 0 and (not job.inputdata or ((not isType(job.inputdata, GangaDataset)) or not job.inputdata.treat_as_inputfiles)) ): return "" insertScript = """\n """ # first, go over any LocalFiles in GangaDatasets to be transferred # The LocalFiles in inputfiles have already been dealt with if job.inputdata and isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles: for inputFile in job.inputdata.files: inputfileClassName = getName(inputFile) if inputfileClassName == "LocalFile": # special case for LocalFile if getName(job.backend) in ["Localhost", "Batch", "LSF", "Condor", "PBS"]: # create symlink shortScript += """ # create symbolic links for LocalFiles for f in ###FILELIST###: os.symlink(f, os.path.basename(f)) """ shortScript = FileUtils.indentScript(shortScript, "###INDENT####") insertScript += shortScript insertScript = insertScript.replace("###FILELIST###", "%s" % inputFile.getFilenameList()) # if GangaDataset is used, check if they want the inputfiles transferred inputfiles_list = job.inputfiles if job.inputdata and isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles: inputfiles_list += job.inputdata.files for inputFile in inputfiles_list: inputfileClassName = getName(inputFile) if outputFilePostProcessingOnWN(job, inputfileClassName): inputFile.processWildcardMatches() if inputFile.subfiles: for subfile in inputFile.subfiles: insertScript += subfile.getWNScriptDownloadCommand(indent) else: insertScript += inputFile.getWNScriptDownloadCommand(indent) insertScript = insertScript.replace("###INDENT###", indent) return insertScript
def getWNCodeForOutputPostprocessing(job, indent): # dict containing the list of outputfiles that need to be processed on the # WN for every file type outputFilesProcessedOnWN = {} patternsToZip = [] if len(job.outputfiles) == 0: return "" else: for outputFile in job.outputfiles: outputfileClassName = getName(outputFile) backendClassName = getName(job.backend) if outputFile.compressed: if outputfileClassName == 'LocalFile' and backendClassName not in ['Localhost', 'LSF', 'Interactive']: patternsToZip.append(outputFile.namePattern) elif outputfileClassName != 'LocalFile' and outputFilePostProcessingOnWN(job, outputfileClassName): patternsToZip.append(outputFile.namePattern) elif outputfileClassName != 'LocalFile' and outputFilePostProcessingOnClient(job, outputfileClassName) and backendClassName not in ['Localhost', 'LSF', 'Interactive']: patternsToZip.append(outputFile.namePattern) if outputfileClassName not in outputFilesProcessedOnWN.keys(): outputFilesProcessedOnWN[outputfileClassName] = [] if outputFilePostProcessingOnWN(job, outputfileClassName): outputFilesProcessedOnWN[ outputfileClassName].append(outputFile) shortScript = """\n import os, glob for patternToZip in ###PATTERNSTOZIP###: for currentFile in glob.glob(os.path.join(os.getcwd(),patternToZip)): os.system("gzip %s" % currentFile) postprocesslocations = file(os.path.join(os.getcwd(), '###POSTPROCESSLOCATIONSFILENAME###'), 'w') """ shortScript = FileUtils.indentScript(shortScript, '###INDENT###') insertScript = shortScript insertScript = insertScript.replace('###PATTERNSTOZIP###', str(patternsToZip)) insertScript = insertScript.replace('###POSTPROCESSLOCATIONSFILENAME###', getConfig('Output')['PostProcessLocationsFileName']) for outputFileName in outputFilesProcessedOnWN.keys(): if len(outputFilesProcessedOnWN[outputFileName]) > 0: insertScript += outputFilesProcessedOnWN[outputFileName][0].getWNInjectedScript(outputFilesProcessedOnWN[outputFileName], indent, patternsToZip, 'postprocesslocations') insertScript += """\n ###INDENT###postprocesslocations.close() """ insertScript = insertScript.replace('###INDENT###', indent) return insertScript
def __str__(self, short=True, id=None): """Prints an overview over the currently running tasks""" if Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_show_help"]: self.help(short=True) Ganga.GPIDev.Lib.Registry.RegistrySlice.config.setUserValue("tasks_show_help", False) print("To show this help message again, type 'tasks.help()'.") print('') print(" The following is the output of " + markup("tasks.table()", fgcol("blue"))) lenfstring = 0 flist = [] for thing in Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_columns"]: width = Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_columns_width"][thing] lenfstring += width flist.append("%" + str(width) + "s ") fstring = "|".join(flist) fstring += '\n' lenfstring += 27 ds = fstring % ("#", "Type", "Name", "State", "Comment", "%4s: %4s/ %4s/ %4s/ %4s/ %4s/ %4s/ %4s" % ("Jobs", markup("done", overview_colours["completed"]), " " + markup("run", overview_colours["running"]), " " + markup("subd", overview_colours["submitted"]), " " + markup("attd", overview_colours["attempted"]), markup("fail", overview_colours["failed"]), markup("hold", overview_colours["hold"]), " " + markup("bad", overview_colours["bad"])), "Float") ds += "-" * lenfstring + "\n" from Ganga.GPIDev.Lib.Tasks import ITask if id is not None and type(id) is int: iterable_list = [self[id]] else: iterable_list = stripProxy(self).objects.values() for p in iterable_list: if isType(p, ITask): stat = "%4i: %4i/ %4i/ %4i/ --/ %4i/ %4i/ %4i" % (p.n_all(), p.n_status("completed"), p.n_status("running"), p.n_status("submitted"), p.n_status("failed"), p.n_status("hold"), p.n_status("bad")) ds += markup(fstring % (p.id, getName(p), p.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], p.status, p.comment, stat, p.float), status_colours[p.status]) else: stat = "%4i: %4i/ %4i/ --/ %4i/ %4i/ %4i/ %4i" % (p.n_all(), p.n_status("completed"), p.n_status("running"), p.n_status("attempted"), p.n_status("failed"), p.n_status("hold"), p.n_status("bad")) ds += markup(fstring % (p.id, getName(p), p.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], p.status, p.comment, stat, p.float), status_colours[p.status]) if short is True: continue for ti in range(0, len(p.transforms)): t = p.transforms[ti] if isType(p, ITask): stat = "%4i: %4i/ %4i/ %4i/ --/ %4i/ %4i/ %4s" % (t.n_all(), t.n_status("completed"), t.n_status("running"), t.n_status("submitted"), t.n_status("failed"), t.n_status("hold"), t.n_status("bad")) ds += "\n" + markup(fstring % ("%i.%i" % (p.id, ti), getName(t), t.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], t.status, "", stat, ""), status_colours[t.status]) else: stat = "%4i: %4i/ %4i/ --/ %4i/ %4i/ %4i/ %4s" % (t.n_all(), t.n_status("completed"), t.n_status("running"), t.n_status("attempted"), t.n_status("failed"), t.n_status("hold"), t.n_status("bad")) ds += "zn" + markup(fstring % ("%i.%i" % (p.id, ti), getName(t), t.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], t.status, "", stat, ""), status_colours[t.status]) ds += "-" * lenfstring + "\n" return ds + "\n"
def __atomic_set__(self, _obj, _val): obj = stripProxy(_obj) val = stripProxy(_val) from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList, GangaList checkSet = self._bind_method(obj, self._checkset_name) if checkSet is not None: checkSet(val) this_filter = self._bind_method(obj, self._filter_name) if this_filter: val = this_filter(val) # LOCKING obj._getWriteAccess() # self._check_getter() item = stripProxy(obj._schema[getName(self)]) def cloneVal(v): if isType(v, list()): new_v = GangaList() for elem in v: new_v.append(self.__cloneVal(elem, obj)) return new_v else: return self.__cloneVal(v, obj) if item['sequence']: _preparable = True if item['preparable'] else False if len(val) == 0: val = GangaList() else: if isType(item, Schema.ComponentItem): val = makeGangaList(val, cloneVal, parent=obj, preparable=_preparable) else: val = makeGangaList(val, parent=obj, preparable=_preparable) else: if isType(item, Schema.ComponentItem): newListObj = [] if isinstance(val, list): for elem in val: newListObj.append(cloneVal(elem)) val = newListObj else: val = cloneVal(val) if hasattr(val, '_setParent'): val._setParent(obj) obj.setNodeAttribute(getName(self), val) obj._setDirty()
def index_write(self, id): """ write an index file for this object (must be locked). Should not raise any Errors """ obj = self.objects[id] try: ifn = self.get_idxfn(id) new_idx_cache = self.registry.getIndexCache(obj) if new_idx_cache != obj._index_cache or not os.path.exists(ifn): obj._index_cache = new_idx_cache with open(ifn, "w") as this_file: pickle_to_file((obj._category, getName(obj), obj._index_cache), this_file) except IOError as err: logger.error("Index saving to '%s' failed: %s %s" % (ifn, getName(err), str(err)))
def __defaultActiveBackendsFunc(self): log.debug("__defaultActiveBackendsFunc") active_backends = {} # FIXME: this is not thread safe: if the new jobs are added then # iteration exception is raised fixed_ids = self.registry.ids() #log.debug("Registry: %s" % str(self.registry)) log.debug("Running over fixed_ids: %s" % str(fixed_ids)) for i in fixed_ids: try: j = stripProxy(self.registry(i)) #log.debug("Job #%s status: %s" % (str(i), str(j.status))) if j.status in ['submitted', 'running'] or (j.master and (j.status in ['submitting'])): if self.enabled is True and self.alive is True: ## This causes a Loading of the subjobs from disk!!! #stripProxy(j)._getReadAccess() if j.getNodeData(): #log.info("data: %s" % str(j.getNodeData())) #log.info("node %s" % str(j.getNodeIndexCache())) #log.info("__dict: %s" % str(j.__class__.__dict__['backend'])) #bn = j.__class__.__dict__['backend'].__name__ #log.info("bn: %s" % str(bn)) lazy_load_backend_str = 'display:backend' if lazy_load_backend_str in j.getNodeData(): bn = j.getNodeData()[lazy_load_backend_str] else: bn = getName(j.backend) else: bn = getName(j.backend) #log.debug("active_backends.setdefault: %s" % str(bn)) active_backends.setdefault(bn, []) active_backends[bn].append(j) except RegistryKeyError as err: log.debug("RegistryKeyError: The job was most likely removed") log.debug("RegError %s" % str(err)) except RegistryLockError as err: log.debug("RegistryLockError: The job was most likely removed") log.debug("Reg LockError%s" % str(err)) summary = '{' for backend, these_jobs in active_backends.iteritems(): summary += '"' + str(backend) + '" : [' for this_job in these_jobs: summary += stripProxy(this_job).getFQID('.') + ', ' summary += '], ' summary += '}' log.debug("Returning active_backends: %s" % summary) return active_backends
def mangle_job_name(app): job = stripProxy(app).getJobObject() jobName = job.name jobIndex = job.getStringFQID() appName = str(getName(app)) appVersion = None if hasattr(app, "version"): appVersion = str(app.version) result = "" addBracket = False if jobName: result += "%s__" % jobName if appName: # and not j.subjobs: #not necessary? if not job.master and job.splitter: result += "{Ganga_%s_(%s.%s)" % (appName, jobIndex, "%n") else: result += "{Ganga_%s_(%s)" % (appName, jobIndex) addBracket = True if appVersion: result += "_%s" % appVersion if not result: result = "{Ganga_Job}" elif addBracket: result += "}" return result
def mangle_job_name(app): """ Create a safe job name to send to DIRAC (includes full fqid) Args: app (IApplication): This is the application belonging to the job of interest """ job = app.getJobObject() jobName = job.name jobIndex = job.getStringFQID() appName = getName(app) appVersion = None if hasattr(app, 'version'): appVersion = str(app.version) result = '' addBracket = False if jobName: result += '%s__' % jobName if appName: # and not j.subjobs: #not necessary? if not job.master and job.splitter: result += '{Ganga_%s_(%s.%s)' % (appName, jobIndex, '%n') else: result += '{Ganga_%s_(%s)' % (appName, jobIndex) addBracket = True if appVersion: result += '_%s' % appVersion if not result: result = '{Ganga_Job}' elif addBracket: result += '}' return result
def transition_update(self, new_status): # print "Transition Update of app ", self.id, " to ",new_status try: transform = self.getTransform() if self.tasks_id.startswith("00"): # Master job if new_status == "new": # something went wrong with submission for sj in self._getParent().subjobs: sj.application.transition_update(new_status) if transform: stripProxy(transform).setMasterJobStatus( self._getParent(), new_status) else: if transform: stripProxy(transform).setAppStatus(self, new_status) except Exception as x: import traceback import sys logger.error( "Exception in call to transform[%s].setAppStatus(%i, %s)", self.tasks_id, self.id, new_status) logger.error( getName(x) + " : " + x) tb = sys.exc_info()[2] if tb: traceback.print_tb(tb) else: logger.error("No Traceback available") logger.error("%s", x)
def taskify(baseclass, name): smajor = baseclass._schema.version.major sminor = baseclass._schema.version.minor cat = baseclass._category if cat == "applications": schema_items = _app_schema taskclass = TaskApplication elif cat == "splitters": schema_items = _splitter_schema taskclass = TaskSplitter classdict = { "_schema": Schema(Version(smajor, sminor), dict(baseclass._schema.datadict.items() + schema_items)), "_category": cat, "_name": name, "__init__": __task__init__, } if '_exportmethods' in baseclass.__dict__: classdict['_exportmethods'] = baseclass.__dict__['_exportmethods'] cls = classobj(name, (taskclass, baseclass), classdict) global handler_map # Use the same handlers as for the base class handler_map.append((getName(baseclass), name)) return cls
def do_collective_operation(self, keep_going, method, *args, **kwds): """ """ if not isinstance(keep_going, bool): raise GangaException("The variable 'keep_going' must be a boolean. Probably you wanted to do %s(%s).%s()" % (self.name, keep_going, method)) result = [] for _id in self.objects.keys(): obj = self.objects[_id] try: if isinstance(method, str): doc = method result.append(getattr(obj, method)(*args, **kwds)) else: try: doc = method.__doc__ except AttributeError: doc = str(method) result.append(method(obj, *args, **kwds)) except GangaException as x: if not keep_going: raise except Exception as x: logger.exception('%s %s %s: %s %s', doc, self.name, _id, getName(x), x) if not keep_going: raise return result
def _checkActiveBackends(self, activeBackendsFunc): log.debug("calling function _checkActiveBackends") activeBackends = activeBackendsFunc() summary = '{' for this_backend, these_jobs in activeBackends.iteritems(): summary += '"' + this_backend + '" : [' for this_job in these_jobs: summary += str(stripProxy(this_job).getFQID('.')) + ', ' summary += '], ' summary += '}' log.debug("Active Backends: %s" % summary) for jList in activeBackends.values(): #log.debug("backend: %s" % str(jList)) backendObj = jList[0].backend b_name = getName(backendObj) if b_name in config: pRate = config[b_name] else: pRate = config['default_backend_poll_rate'] # TODO: To include an if statement before adding entry to # updateDict. Entry is added only if credential requirements # of the particular backend is satisfied. # This requires backends to hold relevant information on its # credential requirements. #log.debug("addEntry: %s, %s, %s, %s" % (str(backendObj), str(self._checkBackend), str(jList), str(pRate))) self.updateDict_ts.addEntry(backendObj, self._checkBackend, jList, pRate) summary = str([stripProxy(x).getFQID('.') for x in jList]) log.debug("jList: %s" % str(summary))
def index_load(self, id): """ load the index file for this object if necessary Loads if never loaded or timestamp changed. Creates object if necessary Returns True if this object has been changed, False if not Raise IOError on access or unpickling error Raise OSError on stat error Raise PluginManagerError if the class name is not found""" #logger.debug("Loading index %s" % id) fn = self.get_idxfn(id) # index timestamp changed if self._cache_load_timestamp.get(id, 0) != os.stat(fn).st_ctime: try: with open(fn, 'r') as fobj: cat, cls, cache = pickle_from_file(fobj)[0] except Exception as x: logger.debug("index_load Exception: %s" % str(x)) raise IOError("Error on unpickling: %s %s" %(getName(x), x)) if id in self.objects: obj = self.objects[id] if obj._data: obj.__dict__["_registry_refresh"] = True else: obj = self._make_empty_object_(id, cat, cls) obj._index_cache = cache self._cache_load_timestamp[id] = os.stat(fn).st_ctime self._cached_cat[id] = cat self._cached_cls[id] = cls self._cached_obj[id] = cache return True elif id not in self.objects: self.objects[id] = self._make_empty_object_( id, self._cached_cat[id], self._cached_cls[id]) self.objects[id]._index_cache = self._cached_obj[id] return True return False
def master_auto_resubmit(self, rjobs): '''Duplicate of the IBackend.master_resubmit but hooked into auto resubmission such that the monitoring server is used rather than the user server''' from Ganga.Core import IncompleteJobSubmissionError, GangaException from Ganga.Utility.logging import log_user_exception incomplete = 0 def handleError(x): if incomplete: raise x else: return 0 try: for sj in rjobs: fqid = sj.getFQID('.') logger.info("resubmitting job %s to %s backend", fqid, getName(sj.backend)) try: b = sj.backend sj.updateStatus('submitting') result = b._resubmit() if result: sj.updateStatus('submitted') # sj._commit() # PENDING: TEMPORARY DISABLED incomplete = 1 else: return handleError(IncompleteJobSubmissionError(fqid, 'resubmission failed')) except Exception as x: log_user_exception(logger, debug=isType(x, GangaException)) return handleError(IncompleteJobSubmissionError(fqid, str(x))) finally: master = self.getJobObject().master if master: master.updateMasterJobStatus() return 1
def _on_attribute__set__(self, obj_type, attrib_name): r = copy.deepcopy(self) if getName(obj_type) == 'Job' and attrib_name == 'outputfiles': r.locations = [] r.localDir = '' r.failureReason = '' return r
def __setitem__(self, category, _filter): if category not in self._dict: self._dict[category] = {} # the filter can be registered as a tuple: ('filtername',filterfunction) # or just as a function in which case the function name is used as an # alias if isType(_filter, tuple) and len(_filter) >= 2: filtername = _filter[0] filterfunc = _filter[1] else: try: filtername = getName(_filter) filterfunc = _filter except AttributeError as e: raise ValueError( 'FilterManager: Invalid component filter %s.' % _filter) if filtername in self._dict[category]: raise ValueError('FilterManager: %s component filter already exists for %s category ' % (filtername, category)) if category not in config.options: config.addOption(category, "", "") config.overrideDefaultValue(category, filtername) self._dict[category][filtername] = filterfunc
def prepareCommand(app): """ Returns the command which is to be run on the worker node Args: app (GaudiExec): This expects only the GaudiExec app """ all_opts_files = app.getOptsFiles() opts_names = [] for opts_file in all_opts_files: if isinstance(opts_file, (LocalFile, DiracFile)): # Ideally this would NOT need the basename, however LocalFile is special in this regard. # TODO Fix this after fixing LocalFile opts_names.append(os.path.basename(opts_file.namePattern)) else: raise ApplicationConfigurationError(None, "The filetype: %s is not yet supported for use as an opts file.\nPlease contact the Ganga devs is you wish this implemented." % getName(opts_file)) sourceEnv = app.getEnvScript() if not app.useGaudiRun: full_cmd = sourceEnv + './run python %s' % app.getWrapperScriptName() else: full_cmd = sourceEnv + "./run gaudirun.py %s %s" % (' '.join(opts_names), GaudiExecDiracRTHandler.data_file) if app.extraOpts: full_cmd += ' ' + app.getExtraOptsFileName() if app.extraArgs: full_cmd += " " + " ".join(app.extraArgs) return full_cmd
def _display_element(self, item): if hasattr(item, 'name') and item.name != None: return item.name elif type(item.command_input[0]) != str: return getName(item.command_input[0]) else: return item.command_input[0]
def removeCallbackHook(self, func): func_name = getName(func) log.debug('Removing Callback hook function %s.' % func_name) if func_name in self.callbackHookDict: del self.callbackHookDict[func_name] else: log.error('Callback hook function does not exist.')
def setCallbackHook(self, func, argDict, enabled, timeout=0): func_name = getName(func) log.debug('Setting Callback hook function %s.' % func_name) log.debug('arg dict: %s' % str(argDict)) if func_name in self.callbackHookDict: log.debug('Replacing existing callback hook function %s with %s' % (str(self.callbackHookDict[func_name]), func_name)) self.callbackHookDict[func_name] = [func, CallbackHookEntry(argDict=argDict, enabled=enabled, timeout=timeout)]
def test_h_testXMLIndex(self): # Check index of job from Ganga.Core.GangaRepository.PickleStreamer import to_file, from_file from Ganga.GPI import jobs j = jobs(0) assert path.isfile(getIndexFile(j)) with open(getIndexFile(j)) as handler: obj, errs = from_file(handler) assert isinstance(obj, tuple) from Ganga.GPIDev.Base.Proxy import stripProxy, getName raw_j = stripProxy(j) index_cache = raw_j._getRegistry().getIndexCache(raw_j) assert isinstance(index_cache, dict) index_cls = getName(raw_j) index_cat = raw_j._category this_index_cache = (index_cat, index_cls, index_cache) print("just-built index: %s" % str(this_index_cache)) print("from disk: %s" % str(obj)) assert this_index_cache == obj
def disableCallbackHook(self, func): func_name = getName(func) log.debug('Disabling Callback hook function %s.' % func_name) if func_name in self.callbackHookDict: self.callbackHookDict[func_name][1].enabled = False else: log.error('Callback hook function does not exist.')
def outputFilePostProcessingTestForWhen(job, outputFileClassName, when): """ Checks if the output files of a given job(we are interested in the backend) should be postprocessed 'when', depending on job.backend_output_postprocess dictionary """ backendClassName = getName(job.backend) backend_output_postprocess = stripProxy( job).getBackendOutputPostprocessDict() if backendClassName in backend_output_postprocess: if outputFileClassName in backend_output_postprocess[backendClassName]: if backend_output_postprocess[backendClassName][ outputFileClassName] == when: return True return False
def printSummaryTree(self, level=0, verbosity_level=0, whitespace_marker='', out=sys.stdout, selection='', interactive=False): """ This funtion displays a summary of the contents of this file. (Docs from Gaga.GPIDev.Base.Objects # TODO determine how mch of this may be duplicated from there) Args: level (int): the hierachy level we are currently at in the object tree. verbosity_level (int): How verbose the print should be. Currently this is always 0. whitespace_marker (str): If printing on multiple lines, this allows the default indentation to be replicated. The first line should never use this, as the substitution is 'name = %s' % printSummaryTree() out (stream): An output stream to print to. The last line of output should be printed without a newline.' selection: See VPrinter for an explaintion of this. interactive (bool): Is this printing code being called from the interactive IPython prompt? """ parent = self._getParent() schema_entry = self.findSchemaParentSchemaEntry(parent) if parent is None: full_print(self, out) return if schema_entry: self_len = len(self) print_summary = schema_entry['summary_print'] maxLen = schema_entry['summary_sequence_maxlen'] if print_summary: fp = getattr(parent, print_summary) str_val = fp(self._list, verbosity_level) out.write(str_val) return if (maxLen != -1) and (self_len > maxLen): out.write(decorateListEntries(self_len, getName(type(self[0])))) return else: summary_print(self, out) return out.write(str(self._list)) return
def __cloneVal(self, v, obj): item = obj._schema[getName(self)] if v is None: if item.hasProperty('category'): assertion = item['optional'] and (item['category'] != 'internal') else: assertion = item['optional'] #assert(assertion) if assertion is False: logger.warning("Item: '%s'. of class type: '%s'. Has a Default value of 'None' but is NOT optional!!!" % (getName(self), type(obj))) logger.warning("Please contact the developers and make sure this is updated!") return None elif isinstance(v, str): return str(v) elif isinstance(v, int): return int(v) elif isinstance(v, dict): new_dict = {} for key, item in new_dict.iteritems(): new_dict[key] = self.__cloneVal(v, obj) return new_dict else: if not isinstance(v, Node) and isinstance(v, (list, tuple)): try: GangaList = _getGangaList() new_v = GangaList() except ImportError: new_v = [] for elem in v: new_v.append(self.__cloneVal(elem, obj)) #return new_v elif not isinstance(v, Node): if inspect.isclass(v): new_v = v() else: new_v = v if not isinstance(new_v, Node): logger.error("v: %s" % str(v)) raise GangaException("Error: found Object: %s of type: %s expected an object inheriting from Node!" % (str(v), str(type(v)))) else: new_v = self.__copyNodeObject(new_v, obj) else: new_v = self.__copyNodeObject(v, obj) return new_v
def createUnitCopyOutputDS(self, unit_id): """Create a the Copy Output dataset to use with this unit. Overload to handle more than the basics""" from Ganga.GPIDev.Lib.Tasks.TaskLocalCopy import TaskLocalCopy if isType(self.unit_copy_output, TaskLocalCopy): logger.warning( "Default implementation of createUnitCopyOutputDS can't handle datasets of type '%s'" % getName(self.unit_copy_output)) return # create copies of the Copy Output DS and add Unit name to path self.units[unit_id].copy_output = self.unit_copy_output.clone() self.units[unit_id].copy_output.local_location = os.path.join( self.unit_copy_output.local_location, self.units[unit_id].name.replace(":", "_").replace(" ", "").replace( ",", "_"))
def getOutputSandboxPatternsForInteractive(job): patternsToSandbox = [getConfig('Output')['PostProcessLocationsFileName']] patternsToZip = [] for outputFile in job.outputfiles: outputFileClassName = getName(outputFile) if outputFileClassName == 'LocalFile' or (outputFileClassName != 'LocalFile' and outputFilePostProcessingOnClient(job, outputFileClassName)): if outputFile.compressed: patternsToSandbox.append('%s.gz' % outputFile.namePattern) patternsToZip.append(outputFile.namePattern) else: patternsToSandbox.append(outputFile.namePattern) return (patternsToSandbox, patternsToZip)
def __init__(self, registry): GangaThread.__init__(self, name="JobRegistry_Monitor") log.debug("Constructing JobRegistry_Monitor") self.setDaemon(True) self.registry = registry self.__sleepCounter = 0.0 self.__updateTimeStamp = time.time() self.progressCallback = lambda x: None self.callbackHookDict = {} self.clientCallbackDict = {} self.alive = True self.enabled = False # run the monitoring loop continuosly (steps=-1) or just a specified # number of steps(>0) self.steps = -1 self.activeBackends = {} self.updateJobStatus = None self.errors = {} # Create the default backend update method and add to callback hook. self.makeUpdateJobStatusFunction() # Add credential checking to monitoring loop for _credObj in Credentials._allCredentials.itervalues(): log.debug("Setting callback hook for %s" % getName(_credObj)) self.setCallbackHook(self.makeCredCheckJobInsertor(_credObj), {}, True, timeout=config['creds_poll_rate']) # Add low disk-space checking to monitoring loop log.debug("Setting callback hook for disk space checking") self.setCallbackHook(self.diskSpaceCheckJobInsertor, {}, True, timeout=config['diskspace_poll_rate']) # synch objects # main loop mutex self.__mainLoopCond = threading.Condition() # cleanup synch self.__cleanUpEvent = threading.Event() # asynch mon loop running synch self.__monStepsTerminatedEvent = threading.Event() # event to signal the break of job lists iterators self.stopIter = threading.Event() self.stopIter.set() self._runningNow = False
def master_resubmit(self, rjobs, backend=None): """ Resubmit (previously submitted) job. Configuration phase is skipped. Default implementation works is an emulated-bulk operation. If you override this method for bulk optimization then make sure that you call updateMasterJobStatus() on the master job, so the master job will be monitored by the monitoring loop. """ from Ganga.Core import IncompleteJobSubmissionError, GangaException from Ganga.Utility.logging import log_user_exception incomplete = 0 def handleError(x): if incomplete: raise x else: return 0 try: for sj in rjobs: fqid = sj.getFQID('.') logger.info("resubmitting job %s to %s backend", fqid, getName(sj.backend)) try: b = sj.backend sj.updateStatus('submitting') if backend is None: result = b.resubmit() else: result = b.resubmit(backend=backend) if result: sj.updateStatus('submitted') # sj._commit() # PENDING: TEMPORARY DISABLED incomplete = 1 else: return handleError( IncompleteJobSubmissionError( fqid, 'resubmission failed')) except Exception as x: log_user_exception(logger, debug=isType(x, GangaException)) return handleError( IncompleteJobSubmissionError(fqid, str(x))) finally: master = self.getJobObject().master if master: master.updateMasterJobStatus() return 1
def getFilenameList(self): "return a list of filenames to be created as input.txt on the WN" filelist = [] for f in self.files: if hasattr(f, 'accessURL'): filelist += f.accessURL() elif hasattr(f, 'getFilenameList'): filelist += f.getFilenameList() else: if isinstance(f, GangaObject): logger.warning( "accessURL or getFilenameList not implemented for File '%s'" % getName(f)) else: logger.warning("Warning, not sure how to parse file: %s" % str(f)) return filelist
def __set__(self, _obj, _val): ## self: attribute being changed or Ganga.GPIDev.Base.Objects.Descriptor in which case getName(self) gives the name of the attribute being changed ## _obj: parent class which 'owns' the attribute ## _val: value of the attribute which we're about to set obj_reg = None obj_prevState = None obj = _obj if isinstance(obj, GangaObject): obj_reg = obj._getRegistry() if obj_reg is not None and hasattr(obj_reg, 'isAutoFlushEnabled'): obj_prevState = obj_reg.isAutoFlushEnabled() if obj_prevState is True and hasattr(obj_reg, 'turnOffAutoFlushing'): obj_reg.turnOffAutoFlushing() val_reg = None val_prevState = None val = _val if isinstance(val, GangaObject): val_reg = val._getRegistry() if val_reg is not None and hasattr(val_reg, 'isAutoFlushEnabled'): val_prevState = val_reg.isAutoFlushEnabled() if val_prevState is True and hasattr(val_reg, 'turnOffAutoFlushing'): val_reg.turnOffAutoFlushing() if type(_val) is str: from Ganga.GPIDev.Base.Proxy import stripProxy, runtimeEvalString new_val = stripProxy(runtimeEvalString(_obj, getName(self), _val)) else: new_val = _val self.__atomic_set__(_obj, new_val) if isinstance(new_val, Node): val._setDirty() if val_reg is not None: if val_prevState is True and hasattr(val_reg, 'turnOnAutoFlushing'): val_reg.turnOnAutoFlushing() if obj_reg is not None: if obj_prevState is True and hasattr(obj_reg, 'turnOnAutoFlushing'): obj_reg.turnOnAutoFlushing()
def findFiles(self, job): if not len(self.files): raise PostProcessException( 'No files specified, %s will do nothing!' % getName(self)) filepaths = [] for f in self.files: filepath = os.path.join(job.outputdir, f) for expanded_file in glob.glob(filepath): filepaths.append(expanded_file) if not len(glob.glob(filepath)): if (self.filesMustExist): logger.info( 'The files %s does not exist, %s will fail job(%s) (to ignore missing files set filesMustExist to False)', filepath, self._name, job.fqid) self.result = False else: logger.warning( 'Ignoring file %s as it does not exist.', filepath) return filepaths
def getOutputSandboxPatterns(job): outputPatterns = [] if len(job.outputfiles) > 0: outputPatterns.append(getConfig('Output')['PostProcessLocationsFileName']) for outputFile in job.outputfiles: outputFileClassName = getName(outputFile) if outputFilePostProcessingOnClient(job, outputFileClassName) or outputFileClassName == 'LocalFile': if outputFile.namePattern not in outputPatterns: if outputFile.compressed: outputPatterns.append('%s.gz' % outputFile.namePattern) else: outputPatterns.append(outputFile.namePattern) return outputPatterns
def getOutputSandboxPatternsForInteractive(job): """ This should be used from only from Interactive backend """ patternsToSandbox = [getConfig('Output')['PostProcessLocationsFileName']] patternsToZip = [] for outputFile in job.outputfiles: outputFileClassName = getName(outputFile) if outputFilePostProcessingOnClient(job, outputFileClassName): if outputFile.compressed: patternsToSandbox.append('%s.gz' % outputFile.namePattern) patternsToZip.append(outputFile.namePattern) else: patternsToSandbox.append(outputFile.namePattern) return (patternsToSandbox, patternsToZip)
def __get__(self, obj, cls): name = getName(self) # If obj is None then the getter was called on the class so return the Item if obj is None: return cls._schema[name] if self._getter_name: returnable = self._bind_method(obj, self._getter_name)() if isinstance(returnable, GangaObject): returnable._setParent(self) return returnable # First we want to try to get the information without prompting a load from disk # ._data takes priority ALWAYS over ._index_cache # This access should not cause the object to be loaded obj_data = obj.getNodeData() if name in obj_data: return obj_data[name] # Then try to get it from the index cache obj_index = obj.getNodeIndexCache() if name in obj_index: return obj_index[name] # Since we couldn't find the information in the cache, we will need to fully load the object # Guarantee that the object is now loaded from disk obj._getReadAccess() # First try to load the object from the attributes on disk if name in obj.getNodeData(): return obj.getNodeAttribute(name) # Finally, get the default value from the schema if obj._schema.hasItem(name): return obj._schema.getDefaultValue(name) raise AttributeError('Could not find attribute {0} in {1}'.format(name, obj))
def master_auto_resubmit(self, rjobs): '''Duplicate of the IBackend.master_resubmit but hooked into auto resubmission such that the monitoring server is used rather than the user server Args: rjobs (list): This is a list of jobs which are to be auto-resubmitted''' incomplete = 0 def handleError(x): if incomplete: raise x else: return 0 try: for sj in rjobs: fqid = sj.getFQID('.') logger.info("resubmitting job %s to %s backend", fqid, getName(sj.backend)) try: b = sj.backend sj.updateStatus('submitting') result = b._resubmit() if result: sj.updateStatus('submitted') # sj._commit() # PENDING: TEMPORARY DISABLED incomplete = 1 else: return handleError( IncompleteJobSubmissionError( fqid, 'resubmission failed')) except Exception as x: log_user_exception(logger, debug=isType(x, GangaDiracError)) return handleError( IncompleteJobSubmissionError(fqid, str(x))) finally: master = self.getJobObject().master if master: master.updateMasterJobStatus() return 1
def credCheckJobInsertor(): def cb_Success(): self.enableCallbackHook(credCheckJobInsertor) def cb_Failure(): self.enableCallbackHook(credCheckJobInsertor) self._handleError('%s checking failed!' % getName(credObj), getName(credObj), False) log.debug('Inserting %s checking function to Qin.' % getName(credObj)) _action = JobAction(function=self.makeCredChecker(credObj), callback_Success=cb_Success, callback_Failure=cb_Failure) self.disableCallbackHook(credCheckJobInsertor) try: Qin.put(_action) except Exception as err: log.debug("makeCred Err: %s" % str(err)) cb_Failure("Put _action failure: %s" % str(_action), "unknown", True)
def __defaultActiveBackendsFunc(self): log.debug("__defaultActiveBackendsFunc") active_backends = {} # FIXME: this is not thread safe: if the new jobs are added then # iteration exception is raised fixed_ids = self.registry.ids() #log.debug("Registry: %s" % str(self.registry)) log.debug("Running over fixed_ids: %s" % str(fixed_ids)) for i in fixed_ids: try: j = stripProxy(self.registry(i)) job_status = lazyLoadJobStatus(j) backend_obj = lazyLoadJobBackend(j) backend_name = getName(backend_obj) if job_status in ['submitted', 'running' ] or (j.master and (job_status in ['submitting'])): if self.enabled is True and self.alive is True: active_backends.setdefault(backend_name, []) active_backends[backend_name].append(j) except RegistryKeyError as err: log.debug("RegistryKeyError: The job was most likely removed") log.debug("RegError %s" % str(err)) except RegistryLockError as err: log.debug("RegistryLockError: The job was most likely removed") log.debug("Reg LockError%s" % str(err)) summary = '{' for backend, these_jobs in active_backends.iteritems(): summary += '"' + str(backend) + '" : [' for this_job in these_jobs: #stripProxy(this_job)._getWriteAccess() summary += str( stripProxy(this_job).id) + ', ' #getFQID('.')) + ', ' summary += '], ' summary += '}' log.debug("Returning active_backends: %s" % summary) return active_backends
def getOutputSandboxPatterns(job): """ Intended for grid backends where we have to set the outputsandbox patterns for the output file types that have to be processed on the client """ outputPatterns = [] if len(job.outputfiles) > 0: outputPatterns.append(getConfig('Output')['PostProcessLocationsFileName']) for outputFile in job.outputfiles: outputFileClassName = getName(outputFile) if outputFilePostProcessingOnClient(job, outputFileClassName): if outputFile.namePattern not in outputPatterns: if outputFile.compressed: outputPatterns.append('%s.gz' % outputFile.namePattern) else: outputPatterns.append(outputFile.namePattern) return outputPatterns
def overview(self): """ Get an ascii art overview over task status. Can be overridden """ task = self._getParent() if not task is None: id = str(task.transforms.index(self)) else: id = "?" o = markup("#%s: %s '%s'\n" % (id, getName(self), self.name), status_colours[self.status]) i = 0 partitions = sorted(self._partition_status.keys()) for c in partitions: s = self._partition_status[c] if c in self.getPartitionApps(): failures = self.getPartitionFailures(c) o += markup("%i:%i " % (c, failures), overview_colours[s]) else: o += markup("%i " % c, overview_colours[s]) i += 1 if i % 20 == 0: o += "\n" logger.info(o)
def check(self): self.global_lock_acquire() try: f = open(self.cntfn) newcount = int(f.readline()) f.close() assert newcount >= self.count sessions = os.listdir(self.sdir) prevnames = set() for session in sessions: if not session.endswith(self.name + ".locks"): continue try: sf = os.path.join(self.sdir, session) fd = -1 if not self.afs: fd = os.open(sf, os.O_RDONLY) fcntl.lockf(fd, fcntl.LOCK_SH) # ONLY NFS sf_file = open(sf) names = pickle.load(sf_file) sf_file.close() if not self.afs and fd > 0: fcntl.lockf(fd, fcntl.LOCK_UN) # ONLY NFS os.close(fd) except Exception as x: logger.warning( "CHECKER: session file %s corrupted: %s %s" % (session, getName(x), str(x))) continue if not len(names & prevnames) == 0: logger.error("Double-locked stuff: " + str(names & prevnames)) assert False # prevnames.union_update(names) Should be alias to update but # not in some versions of python prevnames.update(names) finally: self.global_lock_release()
def check(self): with open(self.cntfn) as f: newcount = int(f.readline()) try: assert newcount >= self.count except AssertionError: raise RepositoryError( "Count in lock file: %s is now inconsistent!" % self.cntfn) sessions = os.listdir(self.sdir) prevnames = set() for session in sessions: if not session.endswith(self.name + ".locks"): continue fd = None try: sf = self._path_helper(session) if not self.afs: fd = os.open(sf, os.O_RDWR) fcntl.lockf(fd, fcntl.LOCK_SH) # ONLY NFS with open(sf) as sf_file: names = pickle.load(sf_file) if not self.afs and fd is not None: fcntl.lockf(fd, fcntl.LOCK_UN) # ONLY NFS except Exception as x: logger.warning("CHECKER: session file %s corrupted: %s %s" % (session, getName(x), x)) continue finally: if fd is not None: os.close(fd) if not len(names & prevnames) == 0: logger.error("Double-locked stuff: " + names & prevnames) raise RepositoryError( "Lock file has double-locked objects: %s" % str(names & prevnames)) # prevnames.union_update(names) Should be alias to update but # not in some versions of python prevnames.update(names)
def __init__(self, name, display_prefix): super(RegistrySlice, self).__init__() self.objects = oDict() self.name = name self._display_prefix = display_prefix self._display_columns = config[self._display_prefix + '_columns'] self._display_columns_show_empty = config[self._display_prefix + "_columns_show_empty"] self._display_columns_width = config[self._display_prefix + "_columns_width"] self._display_columns_functions = {} try: col_funcs = config[self._display_prefix + '_columns_functions'] for this_col_func in col_funcs: self._display_columns_functions[this_col_func] = eval( col_funcs[this_col_func]) except Exception as x: logger.error( "Error on evaluating display column functions from config file: %s: %s" % (getName(x), x)) from Ganga.Utility.ColourText import Effects self._colour_normal = Effects().normal self._proxyClass = None
def __copyNodeObject(self, v, obj): """This deals with the actual deepcopy of an object which has inherited from Node class""" item = obj._schema[getName(self)] GangaList = _getGangaList() if isinstance(v, GangaList): categories = v.getCategory() len_cat = len(categories) if (len_cat > 1) or ((len_cat == 1) and (categories[0] != item['category'])) and item['category'] != 'internal': # we pass on empty lists, as the catagory is yet to be defined from Ganga.GPIDev.Base.Proxy import GangaAttributeError raise GangaAttributeError('%s: attempt to assign a list containing incompatible objects %s to the property in category "%s"' % (getName(self), v, item['category'])) else: if v._category not in [item['category'], 'internal'] and item['category'] != 'internal': from Ganga.GPIDev.Base.Proxy import GangaAttributeError raise GangaAttributeError('%s: attempt to assign an incompatible object %s to the property in category "%s found cat: %s"' % (getName(self), v, item['category'], v._category)) v_copy = deepcopy(v) #logger.info("Cloned Object Parent: %s" % v_copy._getParent()) #logger.info("Original: %s" % v_copy._getParent()) return v_copy
def __deepcopy__(self, memo=None): true_parent = self._getParent() ## This triggers a read of the job from disk self._getReadAccess() classname = getName(self) category = self._category cls = self.__class__#allPlugins.find(category, classname) self_copy = cls() if self._schema is not None: for name, item in self._schema.allItems(): if not item['copyable']: setattr(self_copy, name, self._schema.getDefaultValue(name)) else: if hasattr(self, name): setattr(self_copy, name, deepcopy(getattr(self, name))) else: setattr(self_copy, name, self._schema.getDefaultValue(name)) this_attr = getattr(self_copy, name) if isinstance(this_attr, Node): this_attr._setParent(self_copy) if item.isA(Schema.SharedItem): self.__incrementShareRef(self_copy, name) global do_not_copy for k, v in self.__dict__.iteritems(): if k not in do_not_copy: setattr(self_copy, k, deepcopy(v)) if true_parent is not None: self._setParent(true_parent) self_copy._setParent(true_parent) return self_copy
def validatedSplit(self, job): """ Perform splitting using the split() method and validate the mutability invariants. If the invariants are broken (or exception occurs in the split() method) then SplittingError exception is raised. This method is called directly by the framework and should not be modified in the derived classes. """ # try: subjobs = self.split(stripProxy(job)) # except Exception,x: #raise SplittingError(x) #raise x # if not len(subjobs): #raise SplittingError('splitter did not create any subjobs') cnt = 0 for s in subjobs: if not isType(s.backend, type(stripProxy(job.backend))): raise SplittingError( 'masterjob backend %s is not the same as the subjob (probable subjob id=%d) backend %s' % (job.backend._name, cnt, getName(s.backend))) cnt += 1 return subjobs
def __setattr__(self, attr, value): if attr == 'outputfiles': if value != []: if self.outputdata is not None: logger.error( 'ITransform.outputdata is set, you can\'t set ITransform.outputfiles') return elif self.outputsandbox != []: logger.error( 'ITransform.outputsandbox is set, you can\'t set ITransform.outputfiles') return # reduce duplicate values here, leave only duplicates for LCG, # where we can have replicas uniqueValuesDict = [] uniqueValues = [] for val in value: key = '%s%s' % (getName(val), val.namePattern) if key not in uniqueValuesDict: uniqueValuesDict.append(key) uniqueValues.append(val) elif getName(val) == 'LCGSEFile': uniqueValues.append(val) super(ITransform, self).__setattr__(attr, uniqueValues) elif attr == 'inputfiles': if value != []: if self.inputsandbox != []: logger.error( 'ITransform.inputsandbox is set, you can\'t set ITransform.inputfiles') return super(ITransform, self).__setattr__(attr, value) elif attr == 'outputsandbox': if value != []: if getConfig('Output')['ForbidLegacyOutput']: logger.error( 'Use of ITransform.outputsandbox is forbidden, please use ITransform.outputfiles') return if self.outputfiles != []: logger.error( 'ITransform.outputfiles is set, you can\'t set ITransform.outputsandbox') return super(ITransform, self).__setattr__(attr, value) elif attr == 'inputsandbox': if value != []: if getConfig('Output')['ForbidLegacyInput']: logger.error( 'Use of ITransform.inputsandbox is forbidden, please use ITransform.inputfiles') return if self.inputfiles != []: logger.error( 'ITransform.inputfiles is set, you can\'t set ITransform.inputsandbox') return super(ITransform, self).__setattr__(attr, value) elif attr == 'outputdata': if value is not None: if getConfig('Output')['ForbidLegacyOutput']: logger.error( 'Use of ITransform.outputdata is forbidden, please use ITransform.outputfiles') return if self.outputfiles != []: logger.error( 'ITransform.outputfiles is set, you can\'t set ITransform.outputdata') return super(ITransform, self).__setattr__(attr, value) else: super(ITransform, self).__setattr__(attr, value)
def __str__(self, short=True, id=None): """Prints an overview over the currently running tasks""" if Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_show_help"]: self.help(short=True) Ganga.GPIDev.Lib.Registry.RegistrySlice.config.setUserValue( "tasks_show_help", False) print("To show this help message again, type 'tasks.help()'.") print('') print(" The following is the output of " + markup("tasks.table()", fgcol("blue"))) lenfstring = 0 flist = [] for thing in Ganga.GPIDev.Lib.Registry.RegistrySlice.config[ "tasks_columns"]: width = Ganga.GPIDev.Lib.Registry.RegistrySlice.config[ "tasks_columns_width"][thing] lenfstring += width flist.append("%" + str(width) + "s ") fstring = "|".join(flist) fstring += '\n' lenfstring += 27 ds = fstring % ("#", "Type", "Name", "State", "Comment", "%4s: %4s/ %4s/ %4s/ %4s/ %4s/ %4s/ %4s" % ("Jobs", markup("done", overview_colours["completed"]), " " + markup("run", overview_colours["running"]), " " + markup("subd", overview_colours["submitted"]), " " + markup("attd", overview_colours["attempted"]), markup("fail", overview_colours["failed"]), markup("hold", overview_colours["hold"]), " " + markup("bad", overview_colours["bad"])), "Float") ds += "-" * lenfstring + "\n" from Ganga.GPIDev.Lib.Tasks import ITask if id is not None and type(id) is int: iterable_list = [self[id]] else: iterable_list = stripProxy(self).objects.values() for p in iterable_list: if isType(p, ITask): stat = "%4i: %4i/ %4i/ %4i/ --/ %4i/ %4i/ %4i" % ( p.n_all(), p.n_status("completed"), p.n_status("running"), p.n_status("submitted"), p.n_status("failed"), p.n_status("hold"), p.n_status("bad")) ds += markup( fstring % (p.id, getName(p), p.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice. config['tasks_columns_width']['Name']], p.status, p.comment, stat, p.float), status_colours[p.status]) else: stat = "%4i: %4i/ %4i/ --/ %4i/ %4i/ %4i/ %4i" % ( p.n_all(), p.n_status("completed"), p.n_status("running"), p.n_status("attempted"), p.n_status("failed"), p.n_status("hold"), p.n_status("bad")) ds += markup( fstring % (p.id, getName(p), p.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice. config['tasks_columns_width']['Name']], p.status, p.comment, stat, p.float), status_colours[p.status]) if short is True: continue for ti in range(0, len(p.transforms)): t = p.transforms[ti] if isType(p, ITask): stat = "%4i: %4i/ %4i/ %4i/ --/ %4i/ %4i/ %4s" % ( t.n_all(), t.n_status("completed"), t.n_status("running"), t.n_status("submitted"), t.n_status("failed"), t.n_status("hold"), t.n_status("bad")) ds += "\n" + markup( fstring % ("%i.%i" % (p.id, ti), getName(t), t.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice. config['tasks_columns_width']['Name']], t.status, "", stat, ""), status_colours[t.status]) else: stat = "%4i: %4i/ %4i/ --/ %4i/ %4i/ %4i/ %4s" % ( t.n_all(), t.n_status("completed"), t.n_status("running"), t.n_status("attempted"), t.n_status("failed"), t.n_status("hold"), t.n_status("bad")) ds += "zn" + markup( fstring % ("%i.%i" % (p.id, ti), getName(t), t.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice. config['tasks_columns_width']['Name']], t.status, "", stat, ""), status_colours[t.status]) ds += "-" * lenfstring + "\n" return ds + "\n"
def report_inner(job=None, isJob=False, isTask=False): userInfoDirName = "userreport" tempDirName = "reportsRepository" # job relevant info jobSummaryFileName = "jobsummary.txt" jobFullPrintFileName = "jobfullprint.txt" repositoryPath = "repository/$usr/LocalXML/6.0/jobs/$thousandsNumxxx" # task relevant info taskSummaryFileName = "tasksummary.txt" taskFullPrintFileName = "taskfullprint.txt" tasksRepositoryPath = "repository/$usr/LocalXML/6.0/tasks/$thousandsNumxxx" # user's info environFileName = "environ.txt" userConfigFileName = "userconfig.txt" defaultConfigFileName = "gangarc.txt" ipythonHistoryFileName = "ipythonhistory.txt" gangaLogFileName = "gangalog.txt" jobsListFileName = "jobslist.txt" tasksListFileName = "taskslist.txt" from Ganga.Utility import Config uploadFileServer = Config.getConfig('Feedback')['uploadServer'] #uploadFileServer= "http://gangamon.cern.ch/django/errorreports/" #uploadFileServer= "http://ganga-ai-02.cern.ch/django/errorreports/" #uploadFileServer= "http://127.0.0.1:8000/errorreports" def printDictionary(dictionary, file=sys.stdout): for k, v in dictionary.iteritems(): print('%s: %s' % (k, v), file=file) if k == 'PYTHONPATH': global PYTHON_PATH PYTHON_PATH = v def extractFileObjects(fileName, targetDirectoryName): try: fileToRead = open(fileName, 'r') try: fileText = fileToRead.read() import re pattern = "File\(name=\'(.+?)\'" matches = re.findall(pattern, fileText) for fileName in matches: fileName = os.path.expanduser(fileName) targetFileName = os.path.join( targetDirectoryName, os.path.basename(fileName)) shutil.copyfile(fileName, targetFileName) finally: fileToRead.close() # except IOError, OSError: except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) def writeErrorLog(errorMessage): try: fileToWrite = open(errorLogPath, 'a') try: fileToWrite.write(errorMessage) fileToWrite.write("\n") except Exception as err: logger.debug("Err: %s" % str(err)) raise finally: fileToWrite.close() except Exception as err2: logger.debug("Err: %s" % str(err2)) pass def writeStringToFile(fileName, stringToWrite): try: # uncomment this to try the error logger #fileName = '~/' + fileName fileToWrite = open(fileName, 'w') try: fileToWrite.write(stringToWrite) except Exception as err: logger.debug("Err: %s" % str(err)) raise err finally: fileToWrite.close() # except IOError: except Exception as err: logger.debug("Err2: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) def renameDataFiles(directory): for fileName in os.listdir(directory): fullFileName = os.path.join(directory, fileName) if os.path.isfile(fullFileName): if fileName == 'data': os.rename(fullFileName, fullFileName + '.txt') else: renameDataFiles(fullFileName) import shutil import tarfile import tempfile import os userHomeDir = os.getenv("HOME") tempDir = tempfile.mkdtemp() errorLogPath = os.path.join(tempDir, 'reportErrorLog.txt') fullPathTempDir = os.path.join(tempDir, tempDirName) fullLogDirName = '' # create temp dir and specific dir for the job/user try: if not os.path.exists(fullPathTempDir): os.mkdir(fullPathTempDir) import datetime now = datetime.datetime.now() userInfoDirName = userInfoDirName + \ now.strftime("%Y-%m-%d-%H:%M:%S") fullLogDirName = os.path.join(fullPathTempDir, userInfoDirName) # if report directory exists -> delete it's content(we would like # last version of the report) if os.path.exists(fullLogDirName): shutil.rmtree(fullLogDirName) os.mkdir(fullLogDirName) # except OSError: except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # import os.environ in a file fullEnvironFileName = os.path.join(fullLogDirName, environFileName) try: inputFile = open(fullEnvironFileName, 'w') try: printDictionary(os.environ, file=inputFile) print('OS VERSION : ' + platform.platform(), file=inputFile) finally: inputFile.close() # except IOError except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # import user config in a file userConfigFullFileName = os.path.join( fullLogDirName, userConfigFileName) try: inputFile = open(userConfigFullFileName, 'w') try: print("#GANGA_VERSION = %s" % config.System.GANGA_VERSION, file=inputFile) global GANGA_VERSION GANGA_VERSION = config.System.GANGA_VERSION # this gets the default values # Ganga.GPIDev.Lib.Config.Config.print_config_file() # this should get the changed values for c in config: print(config[c], file=inputFile) finally: inputFile.close() # except IOError does not catch the exception ??? except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # write gangarc - default configuration defaultConfigFullFileName = os.path.join( fullLogDirName, defaultConfigFileName) try: outputFile = open(os.path.join(userHomeDir, '.gangarc'), 'r') try: writeStringToFile(defaultConfigFullFileName, outputFile.read()) finally: outputFile.close() # except IOError does not catch the exception ??? except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # import ipython history in a file try: ipythonFile = open( os.path.join(os.environ['IPYTHONDIR'], 'history'), 'r') try: lastIPythonCommands = ipythonFile.readlines()[-20:] writeStringToFile(os.path.join( fullLogDirName, ipythonHistoryFileName), '\n'.join(lastIPythonCommands)) #writeStringToFile(os.path.join(fullLogDirName, ipythonHistoryFileName), ipythonFile.read()) finally: ipythonFile.close() # except IOError does not catch the exception ??? except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # import gangalog in a file userLogFileLocation = config["Logging"]._logfile userLogFileLocation = os.path.expanduser(userLogFileLocation) try: gangaLogFile = open(userLogFileLocation, 'r') try: writeStringToFile( os.path.join(fullLogDirName, gangaLogFileName), gangaLogFile.read()) finally: gangaLogFile.close() # except IOError: except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # import the result of jobs command in the report jobsListFullFileName = os.path.join(fullLogDirName, jobsListFileName) try: outputFile = open(jobsListFullFileName, 'w') try: from Ganga.GPI import jobs print(jobs, file=outputFile) finally: outputFile.close() # except IOError does not catch the exception ??? except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # import the result of tasks command in the report tasksListFullFileName = os.path.join(fullLogDirName, tasksListFileName) try: outputFile = open(tasksListFullFileName, 'w') try: from Ganga.GPI import tasks print(tasks, file=outputFile) finally: outputFile.close() # except IOError does not catch the exception ??? except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # save it here because we will change fullLogDirName, but we want this # to be the archive and to be deleted folderToArchive = fullLogDirName # import job relevant info if (job is not None and isJob): global JOB_REPORT, APPLICATION_NAME, BACKEND_NAME JOB_REPORT = True APPLICATION_NAME = getName(job.application) BACKEND_NAME = getName(job.backend) # create job folder jobFolder = 'job_%s' % str(job.fqid) fullLogDirName = os.path.join(fullLogDirName, jobFolder) os.mkdir(fullLogDirName) # import job summary in a file fullJobSummaryFileName = os.path.join( fullLogDirName, jobSummaryFileName) writeStringToFile(fullJobSummaryFileName, str(job)) # import job full print in a file fullJobPrintFileName = os.path.join( fullLogDirName, jobFullPrintFileName) try: inputFile = open(fullJobPrintFileName, 'w') try: full_print(job, inputFile) finally: inputFile.close() # except IOError, OSError: except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # extract file objects try: fileObjectsPath = os.path.join(fullLogDirName, 'fileobjects') os.mkdir(fileObjectsPath) extractFileObjects(fullJobSummaryFileName, fileObjectsPath) # except OSError: except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # copy dir of the job ->input/output and subjobs try: parentDir, currentDir = os.path.split(job.inputdir[:-1]) workspaceDir = os.path.join(fullLogDirName, 'workspace') shutil.copytree(parentDir, workspaceDir) # except IOError, OSError except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # copy shared area of the job try: if hasattr(job.application, 'is_prepared'): if job.application.is_prepared is not None and job.application.is_prepared is not True: import os from Ganga.Utility.Config import getConfig from Ganga.Utility.files import expandfilename shared_path = os.path.join(expandfilename(getConfig( 'Configuration')['gangadir']), 'shared', getConfig('Configuration')['user']) shareddir = os.path.join( shared_path, job.application.is_prepared.name) if os.path.isdir(shareddir): sharedAreaDir = os.path.join( fullLogDirName, 'sharedarea') shutil.copytree(shareddir, sharedAreaDir) # except IOError, OSError except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # copy repository job file try: indexFileName = str(job.id) + '.index' repositoryPath = repositoryPath.replace( '$usr', os.getenv("USER")) # check if the job is subjob -> different way of forming the # path to the repository is_subjob = job.fqid.find('.') > -1 if is_subjob: jobid, subjobid = job.fqid.split( '.')[0], job.fqid.split('.')[1] repositoryPath = repositoryPath.replace( '$thousandsNum', str(int(jobid) / 1000)) repositoryPath = os.path.join(repositoryPath, jobid) else: repositoryPath = repositoryPath.replace( '$thousandsNum', str(job.id / 1000)) repositoryFullPath = os.path.join( config.Configuration.gangadir, repositoryPath) indexFileSourcePath = os.path.join( repositoryFullPath, indexFileName) repositoryFullPath = os.path.join( repositoryFullPath, str(job.id)) repositoryTargetPath = os.path.join( fullLogDirName, 'repository', str(job.id)) os.mkdir(os.path.join(fullLogDirName, 'repository')) shutil.copytree(repositoryFullPath, repositoryTargetPath) # data files are copied but can not be opened -> add .txt to # their file names renameDataFiles(repositoryTargetPath) if not is_subjob: # copy .index file indexFileTargetPath = os.path.join( fullLogDirName, 'repository', indexFileName) shutil.copyfile(indexFileSourcePath, indexFileTargetPath) # except OSError, IOError: except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # import task relevant info if (job is not None and isTask): # job is actually a task object task = job # create task folder taskFolder = 'task_%s' % str(task.id) fullLogDirName = os.path.join(fullLogDirName, taskFolder) os.mkdir(fullLogDirName) # import task summary in a file fullTaskSummaryFileName = os.path.join( fullLogDirName, taskSummaryFileName) writeStringToFile(fullTaskSummaryFileName, str(task)) # import task full print in a file fullTaskPrintFileName = os.path.join( fullLogDirName, taskFullPrintFileName) try: inputFile = open(fullTaskPrintFileName, 'w') try: full_print(task, inputFile) except Exception as err: logger.debug("Err: %s" % str(err)) raise err finally: inputFile.close() # except IOError, OSError: except Exception as err: logger.debug("Err2: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # copy shared area of the task try: if len(task.transforms) > 0: if hasattr(task.transforms[0], 'application') and hasattr(task.transforms[0].application, 'is_prepared'): if task.transforms[0].application.is_prepared is not None and task.transforms[0].application.is_prepared is not True: import os from Ganga.Utility.Config import getConfig from Ganga.Utility.files import expandfilename shared_path = os.path.join(expandfilename(getConfig( 'Configuration')['gangadir']), 'shared', getConfig('Configuration')['user']) shareddir = os.path.join( shared_path, task.transforms[0].application.is_prepared.name) if os.path.isdir(shareddir): sharedAreaDir = os.path.join( fullLogDirName, 'sharedarea') shutil.copytree(shareddir, sharedAreaDir) # except IOError, OSError except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # copy repository task file try: indexFileName = str(task.id) + '.index' tasksRepositoryPath = tasksRepositoryPath.replace( '$usr', os.getenv("USER")) tasksRepositoryPath = tasksRepositoryPath.replace( '$thousandsNum', str(task.id / 1000)) repositoryFullPath = os.path.join( config.Configuration.gangadir, tasksRepositoryPath) indexFileSourcePath = os.path.join( repositoryFullPath, indexFileName) repositoryFullPath = os.path.join( repositoryFullPath, str(task.id)) repositoryTargetPath = os.path.join( fullLogDirName, 'repository', str(task.id)) os.mkdir(os.path.join(fullLogDirName, 'repository')) shutil.copytree(repositoryFullPath, repositoryTargetPath) # data files are copied but can not be opened -> add .txt to # their file names renameDataFiles(repositoryTargetPath) # copy .index file indexFileTargetPath = os.path.join( fullLogDirName, 'repository', indexFileName) shutil.copyfile(indexFileSourcePath, indexFileTargetPath) # except OSError, IOError: except Exception as err: logger.debug("Err %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) resultArchive = '%s.tar.gz' % folderToArchive try: resultFile = tarfile.TarFile.open(resultArchive, 'w:gz') try: resultFile.add( folderToArchive, arcname=os.path.basename(folderToArchive)) # put the error log in the archive if(os.path.exists(errorLogPath)): resultFile.add( errorLogPath, arcname=os.path.basename(errorLogPath)) except Exception as err: logger.debug("Err: %s" % str(err)) raise finally: resultFile.close() except Exception as err: logger.debug("Err2: %s" % str(err)) raise # pass # remove temp dir if(os.path.exists(folderToArchive)): shutil.rmtree(folderToArchive) # print the error if there is something if os.path.exists(errorLogPath): logger.error('') logger.error('An error occured while collecting report information : ' + open(errorLogPath, 'r').read()) logger.error('') # delete the errorfile from user's pc if(os.path.exists(errorLogPath)): os.remove(errorLogPath) # return the path to the archive and the path to the upload server return (resultArchive, uploadFileServer, tempDir)
def info(self): logger.info(markup("%s '%s'" % (getName(self), self.name), status_colours[self.status])) logger.info("* backend: %s" % getName(self.backend)) logger.info("Application:") self.application.printTree()
def update_index(self, this_id=None, verbose=False, firstRun=False): """ Update the list of available objects Raise RepositoryError""" # First locate and load the index files logger.debug("updating index...") objs = self.get_index_listing() changed_ids = [] deleted_ids = set(self.objects.keys()) summary = [] if firstRun: self._read_master_cache() logger.debug("Iterating over Items") locked_ids = self.sessionlock.locked for this_id in objs.keys(): deleted_ids.discard(this_id) # Make sure we do not overwrite older jobs if someone deleted the # count file if this_id > self.sessionlock.count: self.sessionlock.count = this_id + 1 # Locked IDs can be ignored if this_id in locked_ids: continue # Now we treat unlocked IDs try: # if this succeeds, all is well and we are done if self.index_load(this_id): changed_ids.append(this_id) continue except IOError as err: logger.debug("IOError: Failed to load index %i: %s" % (this_id, str(err))) except OSError as err: logger.debug("OSError: Failed to load index %i: %s" % (this_id, str(err))) except PluginManagerError as err: # Probably should be DEBUG logger.debug( "PluginManagerError: Failed to load index %i: %s" % (this_id, str(err))) # This is a FATAL error - do not try to load the main file, it # will fail as well summary.append((this_id, err)) continue # this is bad - no or corrupted index but object not loaded yet! # Try to load it! if not this_id in self.objects: try: self.load([this_id]) changed_ids.append(this_id) # Write out a new index if the file can be locked if len(self.lock([this_id])) != 0: self.index_write(this_id) self.unlock([this_id]) except KeyError as err: logger.debug("update Error: %s" % str(err)) # deleted job if this_id in self.objects: self._internal_del__(this_id) changed_ids.append(this_id) except Exception as x: ## WE DO NOT CARE what type of error occured here and it can be ## due to corruption so could be one of MANY exception types ## If the job is not accessible this should NOT cause the loading of ganga to fail! ## we can't reasonably write all possible exceptions here! logger.debug("update_index: Failed to load id %i: %s" % (this_id, str(x))) summary.append((this_id, str(x))) logger.debug("Iterated over Items") # Check deleted files: for this_id in deleted_ids: self._internal_del__(this_id) changed_ids.append(this_id) if len(deleted_ids) > 0: logger.warning( "Registry '%s': Job %s externally deleted." % (self.registry.name, ",".join(map(str, list(deleted_ids))))) if len(summary) > 0: cnt = {} examples = {} for this_id, x in summary: if this_id in self.known_bad_ids: continue cnt[getName(x)] = cnt.get(getName(x), []) + [str(this_id)] examples[getName(x)] = str(x) self.known_bad_ids.append(this_id) # add object to incomplete_objects if not this_id in self.incomplete_objects: self.incomplete_objects.append(this_id) for exc, ids in cnt.items(): logger.error( "Registry '%s': Failed to load %i jobs (IDs: %s) due to '%s' (first error: %s)" % (self.registry.name, len(ids), ",".join(ids), exc, examples[exc])) if self.printed_explanation is False: logger.error( "If you want to delete the incomplete objects, you can type:\n" ) logger.error( "'for i in %s.incomplete_ids(): %s(i).remove()'\n (then press 'Enter' twice)" % (self.registry.name, self.registry.name)) logger.error( "WARNING!!! This will result in corrupt jobs being completely deleted!!!" ) self.printed_explanation = True logger.debug("updated index done") if len(changed_ids) != 0: isShutdown = not firstRun self._write_master_cache(shutdown=isShutdown) return changed_ids
def getWNInjectedScript(self, outputFiles, indent, patternsToZip, postProcessLocationsFP): """ Returns script that have to be injected in the jobscript for postprocessing on the WN """ script_path = os.path.dirname( os.path.abspath(inspect.getfile(inspect.currentframe()))) script_location = os.path.join(script_path, 'uploadScript.py.template') upload_script = FileUtils.loadScript(script_location, '') WNscript_location = os.path.join(script_path, 'WNInjectTemplate.py.template') script = FileUtils.loadScript(WNscript_location, '') if not self.remoteDir: try: job = self.getJobObject() lfn_folder = os.path.join("GangaJob_%s" % job.getFQID('.'), "OutputFiles") except AssertionError: t = datetime.datetime.now() this_date = t.strftime("%H.%M_%A_%d_%B_%Y") lfn_folder = os.path.join('GangaFiles_%s' % this_date) lfn_base = os.path.join( DiracFile.diracLFNBase(self.credential_requirements), lfn_folder) else: lfn_base = oa.path.join( DiracFile.diracLFNBase(self.credential_requirements), self.remoteDir) for this_file in outputFiles: isCompressed = this_file.namePattern in patternsToZip if not regex.search(this_file.namePattern) is None: script += self._WN_wildcard_script(this_file.namePattern, lfn_base, str(isCompressed)) else: script += '###INDENT###print("Uploading: %s as: %s")\n' % ( this_file.namePattern, str(os.path.join(lfn_base, this_file.namePattern))) script += '###INDENT###processes.append(uploadFile("%s", "%s", %s))\n' % ( this_file.namePattern, lfn_base, str(isCompressed)) if stripProxy(self)._parent is not None and stripProxy( self).getJobObject() and getName( stripProxy(self).getJobObject().backend) != 'Dirac': script_env = self._getDiracEnvStr() else: script_env = str(None) script = '\n'.join( [str('###INDENT###' + str(line)) for line in script.split('\n')]) replace_dict = { '###UPLOAD_SCRIPT###': upload_script, '###STORAGE_ELEMENTS###': str(configDirac['allDiracSE']), '###INDENT###': indent, '###LOCATIONSFILE###': postProcessLocationsFP, '###DIRAC_ENV###': script_env } for k, v in replace_dict.iteritems(): script = script.replace(str(k), str(v)) return script