def startUpRegistries(my_interface=None): # Startup the registries and export them to the GPI, also add jobtree and shareref from Ganga.Runtime.GPIexport import exportToInterface if not my_interface: import Ganga.GPI my_interface = Ganga.GPI # import default runtime modules global _runtime_interface _runtime_interface = my_interface # bootstrap user-defined runtime modules and enable transient named # template registries # bootstrap runtime modules from Ganga.GPIDev.Lib.JobTree import TreeError for n, k, d in bootstrap(): # make all repository proxies visible in GPI exportToInterface(my_interface, n, k, 'Objects', d) # JobTree from Ganga.Core.GangaRepository import getRegistry jobtree = getRegistry("jobs").getJobTree() exportToInterface(my_interface, 'jobtree', jobtree, 'Objects', 'Logical tree view of the jobs') exportToInterface(my_interface, 'TreeError', TreeError, 'Exceptions') # ShareRef shareref = getRegistry("prep").getShareRef() exportToInterface(my_interface, 'shareref', shareref, 'Objects', 'Mechanism for tracking use of shared directory resources')
def startUpRegistries(my_interface=None): # Startup the registries and export them to the GPI, also add jobtree and shareref from Ganga.Runtime.GPIexport import exportToInterface if not my_interface: import Ganga.GPI my_interface = Ganga.GPI # import default runtime modules global _runtime_interface _runtime_interface = my_interface # bootstrap user-defined runtime modules and enable transient named # template registries # bootstrap runtime modules from Ganga.GPIDev.Lib.JobTree import TreeError for n, k, d in bootstrap(): # make all repository proxies visible in GPI exportToInterface(my_interface, n, k, 'Objects', d) # JobTree from Ganga.Core.GangaRepository import getRegistry jobtree = getRegistry("jobs").getJobTree() exportToInterface(my_interface, 'jobtree', jobtree, 'Objects', 'Logical tree view of the jobs') exportToInterface(my_interface, 'TreeError', TreeError, 'Exceptions') # ShareRef shareref = getRegistry("prep").getShareRef() exportToInterface( my_interface, 'shareref', shareref, 'Objects', 'Mechanism for tracking use of shared directory resources')
def _copy(self, unprepare=None): global proxyRef logger.debug("unprepare is %s", str(unprepare)) if unprepare is None: if prepconfig["unprepare_on_copy"] is True: if hasattr(self, "is_prepared") or hasattr(self, "application"): unprepare = True def _getSharedPath(): Config_conf = getConfig("Configuration") return os.path.join(expandfilename(Config_conf["gangadir"]), "shared", Config_conf["user"]) if hasattr(self, "application"): if hasattr(self.application, "is_prepared"): from Ganga.Utility.files import expandfilename if self.application.is_prepared not in [None, True]: if hasattr(self.application.is_prepared, "name"): shared_path = _getSharedPath() if os.path.isdir(os.path.join(shared_path, self.application.is_prepared.name)): from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) logger.debug("increasing counter from proxy.py") shareref.increase(self.application.is_prepared.name) logger.debug("Found ShareDir directory: %s" % self.application.is_prepared.name) elif self.application.is_prepared not in [None, True]: shared_path = _getSharedPath() if not os.path.isdir(os.path.join(shared_path, self.application.is_prepared.name)): logger.error("ShareDir directory not found: %s" % self.application.is_prepared.name) logger.error("Unpreparing Job #%s" % self.id) from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) shareref.increase(self.application.is_prepared.name) self.unprepare() if unprepare is True: if hasattr(self, "is_prepared"): from Ganga.Utility.files import expandfilename if self.is_prepared not in [None, True]: if hasattr(self.is_prepared, "name"): shared_path = _getSharedPath() if not os.path.isdir(os.path.join(shared_path, self.is_prepared.name)): logger.error("ShareDir directory not found: %s" % self.is_prepared.name) logger.error("Unpreparing %s application" % getattr(self, proxyRef)._name) self.unprepare() c = getattr(self, proxyRef).clone() if hasattr(c, "is_prepared") and c._getRegistry() is None: from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) shareref.increase(self.is_prepared.name) c._auto__init__(unprepare=True) else: c = getattr(self, proxyRef).clone() c._auto__init__() return GPIProxyObjectFactory(c)
def add(self, input): from Ganga.Core.GangaRepository import getRegistry if not isType(input, list): input = [input] for item in input: if isType(item, str): if os.path.isfile(expandfilename(item)): logger.info('Copying file %s to shared directory %s' % (item, self.name)) shutil.copy2(expandfilename(item), os.path.join(getSharedPath(), self.name)) shareref = getRegistry("prep").getShareRef() shareref.increase(self.name) shareref.decrease(self.name) else: logger.error('File %s not found' % expandfilename(item)) elif isType(item, File) and item.name is not '' and os.path.isfile( expandfilename(item.name)): logger.info('Copying file object %s to shared directory %s' % (item.name, self.name)) shutil.copy2(expandfilename(item.name), os.path.join(getSharedPath(), self.name)) shareref = getRegistry("prep").getShareRef() shareref.increase(self.name) shareref.decrease(self.name) else: logger.error('File %s not found' % expandfilename(item.name))
def _copy(self, unprepare=None): logger.debug('unprepare is %s', str(unprepare)) if unprepare is None: if prepconfig['unprepare_on_copy'] is True: if hasattr(self, 'is_prepared') or hasattr(self, 'application'): unprepare = True def _getSharedPath(): Config_conf = getConfig('Configuration') return os.path.join(expandfilename(Config_conf['gangadir']), 'shared', Config_conf['user']) if hasattr(self, 'application'): if hasattr(self.application, 'is_prepared'): from Ganga.Utility.files import expandfilename if self.application.is_prepared not in [None, True]: if hasattr(self.application.is_prepared, 'name'): shared_path = _getSharedPath() if os.path.isdir(os.path.join(shared_path, self.application.is_prepared.name)): from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) logger.debug('increasing counter from proxy.py') shareref.increase(self.application.is_prepared.name) logger.debug('Found ShareDir directory: %s' % self.application.is_prepared.name) elif self.application.is_prepared not in [None, True]: shared_path = _getSharedPath() if not os.path.isdir(os.path.join(shared_path, self.application.is_prepared.name)): logger.error('ShareDir directory not found: %s' % self.application.is_prepared.name) logger.error('Unpreparing Job #%s' % self.id) from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) shareref.increase(self.application.is_prepared.name) self.unprepare() if unprepare is True: if hasattr(self, 'is_prepared'): from Ganga.Utility.files import expandfilename if self.is_prepared not in [None, True]: if hasattr(self.is_prepared, 'name'): shared_path = _getSharedPath() if not os.path.isdir(os.path.join(shared_path, self.is_prepared.name)): logger.error('ShareDir directory not found: %s' % self.is_prepared.name) logger.error('Unpreparing %s application' % getName(stripProxy(self))) self.unprepare() c = stripProxy(self).clone() if hasattr(c, 'is_prepared') and c._getRegistry() is None: from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) shareref.increase(self.is_prepared.name) stripProxy(c)._auto__init__(unprepare=True) else: c = stripProxy(self).clone() stripProxy(c)._auto__init__() return addProxy(c)
def test_all_exceptions(gpi): """Create all exceptions and make sure they behave correctly""" import Ganga.Core.exceptions test_str = "My Test Error" def exception_test(err_name): """Run tests on the given exception""" err_type = getattr(Ganga.Core.exceptions, err_name) err_obj = err_type(test_str) assert test_str in str(err_obj) err_list = [ "GangaException", "GangaFileError", "PluginError", "ApplicationConfigurationError", "ApplicationPrepareError", "IncompleteJobSubmissionError", "IncompleteKillError", "JobManagerError", "GangaAttributeError", "GangaValueError", "GangaIOError", "SplitterError", "ProtectedAttributeError", "ReadOnlyObjectError", "TypeMismatchError", "SchemaError", "SchemaVersionError", "CredentialsError", "CredentialRenewalError", "InvalidCredentialError", "ExpiredCredentialError" ] for e in err_list: exception_test(e) # check the BackendError from Ganga.Core.exceptions import BackendError err = BackendError("TestBackend", test_str) assert "TestBackend" in str(err) assert test_str in str(err) # check the InaccessibleObjectError from Ganga.Core.exceptions import InaccessibleObjectError, JobManagerError from Ganga.Core.GangaRepository import getRegistry err = InaccessibleObjectError( getRegistry('jobs').repository, 0, JobManagerError("My JobManagerError")) assert "jobs" in str(err) assert "#0" in str(err) assert "My JobManagerError" in str(err) # check the RepositoryError from Ganga.Core.exceptions import RepositoryError from Ganga.Core.GangaRepository import getRegistry RepositoryError(getRegistry('jobs').repository, test_str) # Construct another to check the except clause in the exception is called RepositoryError(getRegistry('jobs').repository, test_str)
def shutdown(): # Shutdown method for all repgistries in order from Ganga.Utility.logging import getLogger logger = getLogger() logger.info('Registry Shutdown') #import traceback #traceback.print_stack() # Flush all repos before we shut them down flush_all() # shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown() # method to perform actions on the box and/or job registries. logger.debug(started_registries) all_registries = getRegistries() try: if 'prep' in started_registries: registry = getRegistry('prep') registry.shutdown() # in case this is called repeatedly, only call shutdown once started_registries.remove(registry.name) except Exception as err: logger.error("Err: %s" % err) logger.error("Failed to Shutdown prep Repository!!! please check for stale lock files") logger.error("Trying to shutdown cleanly regardless") for registry in getRegistries(): thisName = registry.name try: if not thisName in started_registries: continue # in case this is called repeatedly, only call shutdown once started_registries.remove(thisName) registry.shutdown() # flush and release locks except Exception as x: logger.error("Failed to Shutdown Repository: %s !!! please check for stale lock files" % thisName) logger.error("%s" % x) logger.error("Trying to Shutdown cleanly regardless") for registry in all_registries: my_reg = [registry] if hasattr(registry, 'metadata'): if registry.metadata: my_reg.append(registry.metadata) assigned_attrs = ['location', 'type'] for this_reg in my_reg: for attr in assigned_attrs: if hasattr(registry, attr): delattr(registry, attr) from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers removeGlobalSessionFileHandlers() removeGlobalSessionFiles() removeRegistries()
def test_F_decrement(self): """Test that the ref counter is decremented as appropriate""" from Ganga.GPI import jobs j=jobs[-1] shareRef = getRegistry('prep').getShareRef() refDict = shareRef.name assert refDict[j.application.is_prepared.name] is 2 this_ref = j.application.is_prepared.name this_path = j.application.is_prepared.path() j.unprepare() assert refDict[this_ref] is 1 j2=jobs[-2].copy() assert refDict[this_ref] is 2 j2.remove() assert refDict[this_ref] is 1 jobs[-2].remove() assert refDict[this_ref] is 0 assert not path.isdir(this_path)
def test_F_decrement(self): """Test that the ref counter is decremented as appropriate""" from Ganga.GPI import jobs j = jobs[-1] shareRef = getRegistry('prep').getShareRef() refDict = shareRef.name assert refDict[j.application.is_prepared.name] is 2 this_ref = j.application.is_prepared.name this_path = j.application.is_prepared.path() j.unprepare() assert refDict[this_ref] is 1 j2 = jobs[-2].copy() assert refDict[this_ref] is 2 j2.remove() assert refDict[this_ref] is 1 jobs[-2].remove() assert refDict[this_ref] is 0 assert not path.isdir(this_path)
def __prep_set__(obj, val): # if we set is_prepared to None in the GPI, that should effectively # unprepare the application if val is None: if stripProxy(obj).is_prepared is not None: logger.info('Unpreparing application.') stripProxy(obj).unprepare() # Replace is_prepared on an application for another ShareDir object if hasattr(stripProxy(obj), '_getRegistry'): from Ganga.GPIDev.Lib.File import ShareDir if stripProxy(obj)._getRegistry() is not None and isType( val, ShareDir): logger.debug( 'Overwriting is_prepared attribute with a ShareDir object') # it's safe to unprepare 'not-prepared' applications. stripProxy(obj).unprepare() from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory( getRegistry("prep").getShareRef()) shareref.increase(val.name) if type(val) is str: logger.error("Setting string type to 'is_prepared'") import traceback traceback.print_stack()
def shutdown(): logger.debug("registry shutdown") # shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown() # method to perform actions on the box and/or job registries. logger.debug(started_registries) try: if "prep" in started_registries: registry = getRegistry("prep") registry.shutdown() # in case this is called repeatedly, only call shutdown once started_registries.remove(registry.name) except: logger.error("Failed to Shutdown prep Repository!!! please check for stale lock files") logger.error("Trying to shutdown cleanly regardless") pass for registry in getRegistries(): thisName = registry.name try: if not thisName in started_registries: continue # in case this is called repeatedly, only call shutdown once started_registries.remove(thisName) registry.shutdown() # flush and release locks except Exception as x: logger.error("Failed to Shutdown Repository: %s !!! please check for stale lock files" % thisName) logger.error("%s" % str(x)) logger.error("Trying to Shutdown cleanly regardless") pass from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers removeGlobalSessionFileHandlers() removeGlobalSessionFiles()
def __prep_set__(obj, val): # if we set is_prepared to None in the GPI, that should effectively # unprepare the application if val is None: if stripProxy(obj).is_prepared is not None: logger.info("Unpreparing application.") stripProxy(obj).unprepare() # Replace is_prepared on an application for another ShareDir object if hasattr(stripProxy(obj), "_getRegistry"): from Ganga.GPIDev.Lib.File import ShareDir if stripProxy(obj)._getRegistry() is not None and isType(val, ShareDir): logger.debug("Overwriting is_prepared attribute with a ShareDir object") # it's safe to unprepare 'not-prepared' applications. stripProxy(obj).unprepare() from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) shareref.increase(val.name) if type(val) is str: logger.error("Setting string type to 'is_prepared'") import traceback traceback.print_stack()
def __app_set__(obj, val): if not hasattr(obj, 'application') or obj.application is None: return if hasattr(obj.application, '_is_prepared'): #a=Job(); a.prepare(); a.application=Executable() if obj.application.is_prepared not in [None, True] and\ hasattr(val, 'is_prepared') and val.is_prepared is None: logger.debug('Overwriting a prepared application with one that is unprepared') obj.application.unprepare() #a=Job(); b=Executable(); b.prepare(); a.application=b elif obj.application.is_prepared is not True: if hasattr(val, 'is_prepared'): if val.is_prepared not in [None, True]: from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) logger.debug('Overwriting application with a prepared one') if stripProxy(obj.application) != val: stripProxy(obj.application).unprepare() shareref.increase(val.is_prepared.name) # check that the shared directory actually exists before # assigning the (prepared) application to a job if hasattr(val, 'is_prepared'): if val.is_prepared not in [None, True]: if hasattr(val.is_prepared, 'name'): from Ganga.Utility.files import expandfilename Config_conf = getConfig('Configuration') shared_path = os.path.join(expandfilename(Config_conf['gangadir']), 'shared', Config_conf['user']) if not os.path.isdir(os.path.join(shared_path, val.is_prepared.name)): logger.error('ShareDir directory not found: %s' % val.is_prepared.name)
def run(self): server, port = getHttpServer() server.socket.settimeout(1) reg = getRegistry("jobs") """ try: server = HTTPServer(('pclcg35.cern.ch', 1234), GetHandler) server.socket.settimeout(1) except Exception: return print "Another Ganga session is already started with --webgui option" process_details = getProcessDetails() print reg.repository.get_other_sessions() print "Process id : %s, hostname : %s" % (process_details[0], process_details[1]) self.stop() self.unregister() return """ print 'Starting web gui monitoring server, please wait ...' # initialization reg = getRegistry("jobs") #calling here first time will take all jobs reg.pollChangedJobs("WebGUI") #fill jobs dictionary at the begining fill_jobs_dictionary() print 'Web gui monitoring server started successfully' print print 'You can monitor your jobs at the following location: ' + getMonitoringLink(port) #server.serve_forever() try: while not self.should_stop(): server.handle_request() finally: pass
def test_0_testRegistryAsserts(self): from Ganga.GPI import jobs, box, tasks, prep from Ganga.GPIDev.Base.Proxy import stripProxy from Ganga.Core.GangaRepository import getRegistryProxy, getRegistrySlice, getRegistry assert getRegistryProxy('jobs') is jobs assert getRegistryProxy('box') is box assert getRegistryProxy('tasks') is tasks assert getRegistryProxy('prep') is prep assert getRegistrySlice('jobs') is stripProxy(jobs) assert getRegistrySlice('box') is stripProxy(box) assert getRegistrySlice('tasks') is stripProxy(tasks) assert getRegistrySlice('prep') is stripProxy(prep) assert getRegistry('jobs') is stripProxy(jobs).objects assert getRegistry('tasks') is stripProxy(tasks).objects assert getRegistry('box') is stripProxy(box).objects
def listShareDirContents(self, shared_directory_name): """ Function which is used to list the contents of the prepared sandbox folder managed by this app Args: shared_directory_name (str): full name of directory managed by this app """ shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) shareref.ls(shared_directory_name)
def run(self): server, port = getHttpServer() server.socket.settimeout(1) reg = getRegistry("jobs") """ try: server = HTTPServer(('pclcg35.cern.ch', 1234), GetHandler) server.socket.settimeout(1) except Exception: return print("Another Ganga session is already started with --webgui option" ) process_details = getProcessDetails() print(reg.repository.get_other_sessions()) print("Process id : %s, hostname : %s" % (process_details[0], process_details[1])) self.stop() self.unregister() return """ logger.info('Starting web gui monitoring server, please wait ...') # initialization reg = getRegistry("jobs") # calling here first time will take all jobs reg.pollChangedJobs("WebGUI") # fill jobs dictionary at the begining fill_jobs_dictionary() logger.info('Web gui monitoring server started successfully') logger.info('You can monitor your jobs at the following location: ' + getMonitoringLink(port)) # server.serve_forever() try: while not self.should_stop(): server.handle_request() finally: pass
def _auto__init__(self, registry=None): if registry is None: from Ganga.Core.GangaRepository import getRegistry registry = getRegistry(self.default_registry) # register the job (it will also commit it) # job gets its id now registry._add(self) self.creation_date = time.strftime('%Y%m%d%H%M%S') self.startup() self.status = 'new'
def _copy(self, unprepare=None): logger.debug('unprepare is %s', str(unprepare)) if unprepare is None: if prepconfig['unprepare_on_copy'] is True: if hasattr(self,'is_prepared') or hasattr(self,'application'): unprepare = True if hasattr(self,'application'): if hasattr(self.application,'is_prepared'): if self.application.is_prepared is not None and self.application.is_prepared \ is not True and os.path.isdir(os.path.join(shared_path,self.application.is_prepared.name)): from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) logger.debug('increasing counter from proxy.py') shareref.increase(self.application.is_prepared.name) logger.debug('Found ShareDir directory: %s' % self.application.is_prepared.name) elif self.application.is_prepared is not None and self.application.is_prepared \ is not True and not os.path.isdir(os.path.join(shared_path,self.application.is_prepared.name)): logger.error('ShareDir directory not found: %s' % self.application.is_prepared.name) logger.error('Unpreparing Job #%s' % self.id) from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) shareref.increase(self.application.is_prepared.name) self.unprepare() if hasattr(self,'is_prepared') and self.is_prepared is not None and self.is_prepared \ is not True and not os.path.isdir(os.path.join(shared_path,self.is_prepared.name)): logger.error('ShareDir directory not found: %s' % self.is_prepared.name) logger.error('Unpreparing %s application' % self._impl._name) self.unprepare() if unprepare is True: c = self._impl.clone() if hasattr(c,'is_prepared') and c._getRegistry() is None: from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) shareref.increase(self.is_prepared.name) c._auto__init__(unprepare=True) else: c = self._impl.clone() c._auto__init__() return GPIProxyObjectFactory(c)
def test_all_exceptions(gpi): """Create all exceptions and make sure they behave correctly""" import Ganga.Core.exceptions test_str = "My Test Error" def exception_test(err_name): """Run tests on the given exception""" err_type = getattr(Ganga.Core.exceptions, err_name) err_obj = err_type(test_str) assert test_str in str(err_obj) err_list = ["GangaException", "GangaFileError", "PluginError", "ApplicationConfigurationError", "ApplicationPrepareError", "IncompleteJobSubmissionError", "IncompleteKillError", "JobManagerError", "GangaAttributeError", "GangaValueError", "GangaIOError", "SplitterError", "ProtectedAttributeError", "ReadOnlyObjectError", "TypeMismatchError", "SchemaError", "SchemaVersionError", "CredentialsError", "CredentialRenewalError", "InvalidCredentialError", "ExpiredCredentialError"] for e in err_list: exception_test(e) # check the BackendError from Ganga.Core.exceptions import BackendError err = BackendError("TestBackend", test_str) assert "TestBackend" in str(err) assert test_str in str(err) # check the InaccessibleObjectError from Ganga.Core.exceptions import InaccessibleObjectError, JobManagerError from Ganga.Core.GangaRepository import getRegistry err = InaccessibleObjectError(getRegistry('jobs').repository, 0, JobManagerError("My JobManagerError")) assert "jobs" in str(err) assert "#0" in str(err) assert "My JobManagerError" in str(err) # check the RepositoryError from Ganga.Core.exceptions import RepositoryError from Ganga.Core.GangaRepository import getRegistry RepositoryError(getRegistry('jobs').repository, test_str) # Construct another to check the except clause in the exception is called RepositoryError(getRegistry('jobs').repository, test_str)
def __incrementShareRef(obj, attr_name): shared_dir = getattr(obj, attr_name) if hasattr(shared_dir, 'name'): from Ganga.Core.GangaRepository import getRegistry from Ganga.GPIDev.Base.Proxy import GPIProxyObjectFactory shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) logger.debug("Increasing shareref") shareref.increase(shared_dir.name)
def decrementShareCounter(self, shared_directory_name, remove=0): """ Function which is used to decrement the number of (sub)jobs which share the prepared sandbox managed by this app Args: shared_directory_name (str): full name of directory managed by this app """ remove = remove logger.debug('Decrementing shared directory reference counter') shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) shareref.decrease(shared_directory_name, remove)
def incrementShareCounter(self, shared_directory_name): """ Function which is used to increment the number of (sub)jobs which share the prepared sandbox managed by this app Args: shared_directory_name (str): full name of directory managed by this app """ logger.debug('Incrementing shared directory reference counter') shareref = getRegistry("prep").getShareRef() logger.debug('within incrementShareCounter, calling increase') shareref.increase(shared_directory_name)
def add(self, input): if not isType(input, list): input = [input] for item in input: if isType(item, str): if os.path.isfile(expandfilename(item)): logger.info('Copying file %s to shared directory %s'%(item, self.name)) shutil.copy2(expandfilename(item), os.path.join(shared_path,self.name)) shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) shareref.increase(self.name) shareref.decrease(self.name) else: logger.error('File %s not found' % expandfilename(item)) elif isType(item,File) and item.name is not '' and os.path.isfile(expandfilename(item.name)): logger.info('Copying file object %s to shared directory %s'%(item.name,self.name)) shutil.copy2(expandfilename(item.name), os.path.join(shared_path,self.name)) shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) shareref.increase(self.name) shareref.decrease(self.name) else: logger.error('File %s not found' % expandfilename(item.name))
def add(self, input): from Ganga.Core.GangaRepository import getRegistry if not isType(input, list): input = [input] for item in input: if isType(item, str): if os.path.isfile(expandfilename(item)): logger.info('Copying file %s to shared directory %s' % (item, self.name)) shutil.copy2(expandfilename(item), os.path.join(getSharedPath(), self.name)) shareref = getRegistry("prep").getShareRef() shareref.increase(self.name) shareref.decrease(self.name) else: logger.error('File %s not found' % expandfilename(item)) elif isType(item, File) and item.name is not '' and os.path.isfile(expandfilename(item.name)): logger.info('Copying file object %s to shared directory %s' % (item.name, self.name)) shutil.copy2(expandfilename(item.name), os.path.join(getSharedPath(), self.name)) shareref = getRegistry("prep").getShareRef() shareref.increase(self.name) shareref.decrease(self.name) else: logger.error('File %s not found' % expandfilename(item.name))
def shutdown(): logger.debug('registry shutdown') #shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown() #method to perform actions on the box and/or job registries. logger.debug(started_registries) if 'prep' in started_registries: registry = getRegistry('prep') registry.shutdown() started_registries.remove(registry.name) # in case this is called repeatedly, only call shutdown once for registry in getRegistries(): if not registry.name in started_registries: continue started_registries.remove(registry.name) # in case this is called repeatedly, only call shutdown once registry.shutdown() # flush and release locks
def test_E_increment(self): """Test that the ref counter is incremented as appropriate""" from Ganga.GPI import jobs j=jobs[-1] shareRef = getRegistry('prep').getShareRef() refDict = shareRef.name assert j.application.is_prepared.name in refDict j2 = j.copy() assert refDict[j.application.is_prepared.name] is 2
def test_E_increment(self): """Test that the ref counter is incremented as appropriate""" from Ganga.GPI import jobs j = jobs[-1] shareRef = getRegistry('prep').getShareRef() refDict = shareRef.name assert j.application.is_prepared.name in refDict j2 = j.copy() assert refDict[j.application.is_prepared.name] is 2
def __deepcopy__(self, memo = None): self._getReadAccess() c = super(GangaObject,self).__deepcopy__(memo) for name,item in self._schema.allItems(): if not item['copyable']: setattr(c,name,self._schema.getDefaultValue(name)) if item.isA(Schema.SharedItem): shared_dir=getattr(c,name) try: from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) logger.debug("Increasing shareref") shareref.increase(shared_dir.name) except AttributeError: pass return c
def update_jobs_dictionary(): reg = getRegistry("jobs") #get the changed jobs changed_ids = reg.pollChangedJobs("WebGUI") for job_id in changed_ids: try: job = jobs(job_id) try: jobs_dictionary[job_id] = JobRelatedInfo(job, job.time.timestamps['new']) except RegistryKeyError: jobs_dictionary[job_id] = JobRelatedInfo(job, None) except RegistryKeyError: del jobs_dictionary[job_id]
def update_jobs_dictionary(): from Ganga.Core.GangaRepository import getRegistryProxy reg = getRegistry("jobs") # get the changed jobs changed_ids = reg.pollChangedJobs("WebGUI") for job_id in changed_ids: try: job = getRegistryProxy('jobs')(job_id) try: jobs_dictionary[job_id] = JobRelatedInfo( job, job.time.timestamps['new']) except RegistryKeyError: jobs_dictionary[job_id] = JobRelatedInfo(job, None) except RegistryKeyError: del jobs_dictionary[job_id]
def __app_set__(obj, val): if hasattr(obj.application, '_is_prepared'): #a=Job(); a.prepare(); a.application=Executable() if obj.application.is_prepared not in [None, True] and\ hasattr(val, 'is_prepared') and val.is_prepared is None: logger.debug( 'Overwriting a prepared application with one that is unprepared' ) obj.application.unprepare() #a=Job(); b=Executable(); b.prepare(); a.application=b elif obj.application.is_prepared is not True: if hasattr(val, 'is_prepared'): if val.is_prepared not in [None, True]: from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory( getRegistry("prep").getShareRef()) logger.debug('Overwriting application with a prepared one') if stripProxy(obj.application) != val: stripProxy(obj.application).unprepare() shareref.increase(val.is_prepared.name) # check that the shared directory actually exists before # assigning the (prepared) application to a job if hasattr(val, 'is_prepared'): if val.is_prepared not in [None, True]: if hasattr(val.is_prepared, 'name'): from Ganga.Utility.files import expandfilename Config_conf = getConfig('Configuration') shared_path = os.path.join( expandfilename(Config_conf['gangadir']), 'shared', Config_conf['user']) if not os.path.isdir( os.path.join(shared_path, val.is_prepared.name)): logger.error('ShareDir directory not found: %s' % val.is_prepared.name)
def __deepcopy__(self, memo=None): self = stripProxy(self) self._getReadAccess() c = super(GangaObject, self).__deepcopy__(memo) if self._schema is not None: for name, item in self._schema.allItems(): if not item['copyable']: setattr(c, name, self._schema.getDefaultValue(name)) if item.isA(Schema.SharedItem): shared_dir = getattr(c, name) if hasattr(shared_dir, 'name'): from Ganga.Core.GangaRepository import getRegistry from Ganga.GPIDev.Base.Proxy import GPIProxyObjectFactory shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) logger.debug("Increasing shareref") shareref.increase(shared_dir.name) return c
def shutdown(): from Ganga.Utility.logging import getLogger logger = getLogger() logger.info('Registry Shutdown') #import traceback #traceback.print_stack() # shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown() # method to perform actions on the box and/or job registries. logger.debug(started_registries) try: if 'prep' in started_registries: registry = getRegistry('prep') registry.shutdown() # in case this is called repeatedly, only call shutdown once started_registries.remove(registry.name) except Exception as err: logger.debug("Err: %s" % str(err)) logger.error("Failed to Shutdown prep Repository!!! please check for stale lock files") logger.error("Trying to shutdown cleanly regardless") for registry in getRegistries(): thisName = registry.name try: if not thisName in started_registries: continue # in case this is called repeatedly, only call shutdown once started_registries.remove(thisName) registry.shutdown() # flush and release locks except Exception as x: logger.error("Failed to Shutdown Repository: %s !!! please check for stale lock files" % thisName) logger.error("%s" % str(x)) logger.error("Trying to Shutdown cleanly regardless") from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers removeGlobalSessionFileHandlers() removeGlobalSessionFiles()
def listShareDirContents(self, shared_directory_name): shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) shareref.ls(shared_directory_name)
def listShareDirs(self): shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) shareref
def _thread_main(self): """ This is an internal function; the main loop of the background thread """ from Ganga.Core.GangaRepository import getRegistry while getRegistry("jobs").hasStarted() is not True: time.sleep(0.1) if self._main_thread is None or self._main_thread.should_stop(): return while True: from Ganga.Core import monitoring_component if (not monitoring_component is None and monitoring_component.enabled) or config['ForceTaskMonitoring']: break time.sleep(0.1) if self._main_thread is None or self._main_thread.should_stop(): return # setup the tasks - THIS IS INCOMPATIBLE WITH CONCURRENCY # and must go away soon for tid in self.ids(): try: self[tid].startup() except Exception as err: logger.error("Unknown/Unexpected Error in starting up tasks main loop") logger.error("Exiting: err=%s" % str(err)) return logger.debug("Entering main loop") # Main loop while self._main_thread is not None and not self._main_thread.should_stop(): # If monitoring is enabled (or forced for Tasks) loop over each one and update if (config['ForceTaskMonitoring'] or monitoring_component.enabled) and not config['disableTaskMon']: for tid in self.ids(): logger.debug("Running over tid: %s" % str(tid)) try: p = self[tid] p.update() except Exception as x: logger.error( "Exception occurred in task monitoring loop: %s %s\nThe offending task was paused." % (x.__class__, x)) type_, value_, traceback_ = sys.exc_info() logger.error("Full traceback:\n %s" % ' '.join( traceback.format_exception(type_, value_, traceback_))) p.pause() if self._main_thread.should_stop(): break if self._main_thread.should_stop(): break logger.debug("TaskRegistry Sleeping for: %s seconds" % str(config['TaskLoopFrequency'])) # Sleep interruptible for 10 seconds for i in range(0, int(config['TaskLoopFrequency'] * 100)): if self._main_thread.should_stop(): break time.sleep(0.01)
def decrementShareCounter(self, shared_directory_name, remove=0): remove = remove logger.debug('Decrementing shared directory reference counter') shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) shareref.decrease(shared_directory_name, remove)
def listShareDirContents(self,shared_directory_name): shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) shareref.ls(shared_directory_name)
def shutdown(): # Shutdown method for all repgistries in order from Ganga.Utility.logging import getLogger logger = getLogger() logger.info('Registry Shutdown') #import traceback #traceback.print_stack() # Flush all repos before we shut them down flush_all() # shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown() # method to perform actions on the box and/or job registries. logger.debug(started_registries) all_registries = getRegistries() try: if 'prep' in started_registries: registry = getRegistry('prep') registry.shutdown() # in case this is called repeatedly, only call shutdown once started_registries.remove(registry.name) except Exception as err: logger.error("Err: %s" % err) logger.error( "Failed to Shutdown prep Repository!!! please check for stale lock files" ) logger.error("Trying to shutdown cleanly regardless") for registry in getRegistries(): thisName = registry.name try: if not thisName in started_registries: continue # in case this is called repeatedly, only call shutdown once started_registries.remove(thisName) registry.shutdown() # flush and release locks except Exception as x: logger.error( "Failed to Shutdown Repository: %s !!! please check for stale lock files" % thisName) logger.error("%s" % x) logger.error("Trying to Shutdown cleanly regardless") for registry in all_registries: my_reg = [registry] if hasattr(registry, 'metadata'): if registry.metadata: my_reg.append(registry.metadata) assigned_attrs = ['location', 'type'] for this_reg in my_reg: for attr in assigned_attrs: if hasattr(registry, attr): delattr(registry, attr) from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers removeGlobalSessionFileHandlers() removeGlobalSessionFiles() removeRegistries()
def decrementShareCounter(self, shared_directory_name, remove=0): remove=remove logger.debug('Decrementing shared directory reference counter') shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) shareref.decrease(shared_directory_name, remove)
convert_merger_to_postprocessor, 'Functions') exportToPublicInterface('config', Ganga.GPIDev.Lib.Config.config, 'Objects', 'access to Ganga configuration') exportToPublicInterface('ConfigError', Ganga.GPIDev.Lib.Config.ConfigError, 'Exceptions') exportToPublicInterface('report', report, 'Functions') # ------------------------------------------------------------------------------------ # bootstrap the repositories and connect to them for n, k, d in Repository_runtime.bootstrap(): # make all repository proxies visible in GPI exportToPublicInterface(n, k, 'Objects', d) # ------------------------------------------------------------------------------------ # JobTree jobtree = GPIProxyObjectFactory(getRegistry("jobs").getJobTree()) exportToPublicInterface('jobtree', jobtree, 'Objects', 'Logical tree view of the jobs') exportToPublicInterface('TreeError', TreeError, 'Exceptions') # ------------------------------------------------------------------------------------ # ShareRef shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) exportToPublicInterface( 'shareref', shareref, 'Objects', 'Mechanism for tracking use of shared directory resources') # ------------------------------------------------------------------------------------ # bootstrap the workspace Workspace_runtime.bootstrap()
def incrementShareCounter(self, shared_directory_name): logger.debug('Incrementing shared directory reference counter') shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) logger.debug('within incrementShareCounter, calling increase') shareref.increase(shared_directory_name)
def _thread_main(self): """ This is an internal function; the main loop of the background thread """ from Ganga.Core.GangaRepository import getRegistry while getRegistry("jobs").hasStarted() is not True: time.sleep(0.1) if self._main_thread is None or self._main_thread.should_stop(): return while True: from Ganga.Core import monitoring_component if (not monitoring_component is None and monitoring_component.enabled ) or config['ForceTaskMonitoring']: break time.sleep(0.1) if self._main_thread is None or self._main_thread.should_stop(): return # setup the tasks - THIS IS INCOMPATIBLE WITH CONCURRENCY # and must go away soon for tid in self.ids(): try: self[tid].startup() except Exception as err: logger.error( "Unknown/Unexpected Error in starting up tasks main loop") logger.error("Exiting: err=%s" % str(err)) return logger.debug("Entering main loop") # Main loop while self._main_thread is not None and not self._main_thread.should_stop( ): # If monitoring is enabled (or forced for Tasks) loop over each one and update if (config['ForceTaskMonitoring'] or monitoring_component.enabled ) and not config['disableTaskMon']: for tid in self.ids(): logger.debug("Running over tid: %s" % str(tid)) try: p = self[tid] p.update() except Exception as x: logger.error( "Exception occurred in task monitoring loop: %s %s\nThe offending task was paused." % (x.__class__, x)) type_, value_, traceback_ = sys.exc_info() logger.error("Full traceback:\n %s" % ' '.join( traceback.format_exception(type_, value_, traceback_))) p.pause() if self._main_thread.should_stop(): break if self._main_thread.should_stop(): break logger.debug("TaskRegistry Sleeping for: %s seconds" % str(config['TaskLoopFrequency'])) # Sleep interruptible for 10 seconds for i in range(0, int(config['TaskLoopFrequency'] * 100)): if self._main_thread.should_stop(): break time.sleep(0.01)
def _copy(self, unprepare=None): logger.debug('unprepare is %s', str(unprepare)) if unprepare is None: if prepconfig['unprepare_on_copy'] is True: if hasattr(self, 'is_prepared') or hasattr( self, 'application'): unprepare = True def _getSharedPath(): Config_conf = getConfig('Configuration') return os.path.join(expandfilename(Config_conf['gangadir']), 'shared', Config_conf['user']) if hasattr(self, 'application'): if hasattr(self.application, 'is_prepared'): from Ganga.Utility.files import expandfilename if self.application.is_prepared not in [None, True]: if hasattr(self.application.is_prepared, 'name'): shared_path = _getSharedPath() if os.path.isdir( os.path.join( shared_path, self.application.is_prepared.name)): from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory( getRegistry("prep").getShareRef()) logger.debug('increasing counter from proxy.py') shareref.increase( self.application.is_prepared.name) logger.debug('Found ShareDir directory: %s' % self.application.is_prepared.name) elif self.application.is_prepared not in [None, True]: shared_path = _getSharedPath() if not os.path.isdir( os.path.join(shared_path, self.application.is_prepared.name)): logger.error('ShareDir directory not found: %s' % self.application.is_prepared.name) logger.error('Unpreparing Job #%s' % self.id) from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory( getRegistry("prep").getShareRef()) shareref.increase(self.application.is_prepared.name) self.unprepare() if unprepare is True: if hasattr(self, 'is_prepared'): from Ganga.Utility.files import expandfilename if self.is_prepared not in [None, True]: if hasattr(self.is_prepared, 'name'): shared_path = _getSharedPath() if not os.path.isdir( os.path.join(shared_path, self.is_prepared.name)): logger.error('ShareDir directory not found: %s' % self.is_prepared.name) logger.error('Unpreparing %s application' % getName(stripProxy(self))) self.unprepare() c = stripProxy(self).clone() if hasattr(c, 'is_prepared') and c._getRegistry() is None: from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory( getRegistry("prep").getShareRef()) shareref.increase(self.is_prepared.name) stripProxy(c)._auto__init__(unprepare=True) else: c = stripProxy(self).clone() stripProxy(c)._auto__init__() return GPIProxyObjectFactory(c)
def _thread_main(self): """ This is an internal function; the main loop of the background thread """ # Add runtime handlers for all the taskified applications, since now # all the backends are loaded from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from .TaskApplication import handler_map for basename, name in handler_map: for backend in allHandlers.getAllBackends(basename): allHandlers.add(name, backend, allHandlers.get(basename, backend)) from Ganga.Core.GangaRepository import getRegistry while getRegistry("jobs").hasStarted() is not True: time.sleep(0.1) if self._main_thread is None or self._main_thread.should_stop(): return while True: from Ganga.Core import monitoring_component if (not monitoring_component is None and monitoring_component.enabled ) or config['ForceTaskMonitoring']: break time.sleep(0.1) if self._main_thread is None or self._main_thread.should_stop(): return # setup the tasks - THIS IS INCOMPATIBLE WITH CONCURRENCY # and must go away soon for tid in self.ids(): try: self[tid]._getWriteAccess() self[tid].startup() except RegistryError: continue except Exception as err: logger.error( "Unknown/Unexpected Error in starting up tasks main loop") logger.error("Exiting: err=%s" % str(err)) return logger.debug("Entering main loop") # Main loop while self._main_thread is not None and not self._main_thread.should_stop( ): # For each task try to run it if config['ForceTaskMonitoring'] or monitoring_component.enabled: for tid in self.ids(): logger.debug("Running over tid: %s" % str(tid)) try: from Ganga.GPIDev.Lib.Tasks import ITask if isType(self[tid], ITask): # for new ITasks, always need write access self[tid]._getWriteAccess() p = self[tid] else: if self[tid].status in [ "running", "running/pause" ]: self[tid]._getWriteAccess() p = self[tid] elif self[tid].status is 'completed' and ( self[tid].n_status('ready') or self[tid].n_status('running')): self[tid].updateStatus() continue else: continue except RegistryError: # could not acquire lock continue if self._main_thread.should_stop(): break try: from Ganga.GPIDev.Lib.Tasks import ITask if isType(self[tid], ITask): # for new ITasks, always call update() p.update() else: # TODO: Make this user-configurable and add better # error message if (p.n_status("failed") * 100.0 / (20 + p.n_status("completed")) > 20): p.pause() logger.error( "Task %s paused - %i jobs have failed while only %i jobs have completed successfully." % (p.name, p.n_status("failed"), p.n_status("completed"))) logger.error( "Please investigate the cause of the failing jobs and then remove the previously failed jobs using job.remove()" ) logger.error( "You can then continue to run this task with tasks(%i).run()" % p.id) continue numjobs = p.submitJobs() if numjobs > 0: self._flush([p]) # finalise any required transforms p.finaliseTransforms() p.updateStatus() except Exception as x: logger.error( "Exception occurred in task monitoring loop: %s %s\nThe offending task was paused." % (x.__class__, x)) type_, value_, traceback_ = sys.exc_info() logger.error("Full traceback:\n %s" % ' '.join( traceback.format_exception(type_, value_, traceback_))) p.pause() if self._main_thread.should_stop(): break if self._main_thread.should_stop(): break logger.debug("TaskRegistry Sleeping for: %s seconds" % str(config['TaskLoopFrequency'])) # Sleep interruptible for 10 seconds for i in range(0, int(config['TaskLoopFrequency'] * 100)): if self._main_thread.should_stop(): break time.sleep(0.01)