Esempio n. 1
0
def startUpRegistries(my_interface=None):
    # Startup the registries and export them to the GPI, also add jobtree and shareref
    from Ganga.Runtime.GPIexport import exportToInterface
    if not my_interface:
        import Ganga.GPI
        my_interface = Ganga.GPI
    # import default runtime modules

    global _runtime_interface
    _runtime_interface = my_interface

    # bootstrap user-defined runtime modules and enable transient named
    # template registries

    # bootstrap runtime modules
    from Ganga.GPIDev.Lib.JobTree import TreeError

    for n, k, d in bootstrap():
        # make all repository proxies visible in GPI
        exportToInterface(my_interface, n, k, 'Objects', d)

    # JobTree
    from Ganga.Core.GangaRepository import getRegistry
    jobtree = getRegistry("jobs").getJobTree()
    exportToInterface(my_interface, 'jobtree', jobtree, 'Objects', 'Logical tree view of the jobs')
    exportToInterface(my_interface, 'TreeError', TreeError, 'Exceptions')

    # ShareRef
    shareref = getRegistry("prep").getShareRef()
    exportToInterface(my_interface, 'shareref', shareref, 'Objects', 'Mechanism for tracking use of shared directory resources')
Esempio n. 2
0
def startUpRegistries(my_interface=None):
    # Startup the registries and export them to the GPI, also add jobtree and shareref
    from Ganga.Runtime.GPIexport import exportToInterface
    if not my_interface:
        import Ganga.GPI
        my_interface = Ganga.GPI
    # import default runtime modules

    global _runtime_interface
    _runtime_interface = my_interface

    # bootstrap user-defined runtime modules and enable transient named
    # template registries

    # bootstrap runtime modules
    from Ganga.GPIDev.Lib.JobTree import TreeError

    for n, k, d in bootstrap():
        # make all repository proxies visible in GPI
        exportToInterface(my_interface, n, k, 'Objects', d)

    # JobTree
    from Ganga.Core.GangaRepository import getRegistry
    jobtree = getRegistry("jobs").getJobTree()
    exportToInterface(my_interface, 'jobtree', jobtree, 'Objects',
                      'Logical tree view of the jobs')
    exportToInterface(my_interface, 'TreeError', TreeError, 'Exceptions')

    # ShareRef
    shareref = getRegistry("prep").getShareRef()
    exportToInterface(
        my_interface, 'shareref', shareref, 'Objects',
        'Mechanism for tracking use of shared directory resources')
Esempio n. 3
0
    def _copy(self, unprepare=None):
        global proxyRef
        logger.debug("unprepare is %s", str(unprepare))
        if unprepare is None:
            if prepconfig["unprepare_on_copy"] is True:
                if hasattr(self, "is_prepared") or hasattr(self, "application"):
                    unprepare = True

        def _getSharedPath():
            Config_conf = getConfig("Configuration")
            return os.path.join(expandfilename(Config_conf["gangadir"]), "shared", Config_conf["user"])

        if hasattr(self, "application"):
            if hasattr(self.application, "is_prepared"):
                from Ganga.Utility.files import expandfilename

                if self.application.is_prepared not in [None, True]:
                    if hasattr(self.application.is_prepared, "name"):
                        shared_path = _getSharedPath()
                        if os.path.isdir(os.path.join(shared_path, self.application.is_prepared.name)):
                            from Ganga.Core.GangaRepository import getRegistry

                            shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
                            logger.debug("increasing counter from proxy.py")
                            shareref.increase(self.application.is_prepared.name)
                            logger.debug("Found ShareDir directory: %s" % self.application.is_prepared.name)
                elif self.application.is_prepared not in [None, True]:
                    shared_path = _getSharedPath()
                    if not os.path.isdir(os.path.join(shared_path, self.application.is_prepared.name)):
                        logger.error("ShareDir directory not found: %s" % self.application.is_prepared.name)
                        logger.error("Unpreparing Job #%s" % self.id)
                        from Ganga.Core.GangaRepository import getRegistry

                        shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
                        shareref.increase(self.application.is_prepared.name)
                        self.unprepare()

        if unprepare is True:
            if hasattr(self, "is_prepared"):
                from Ganga.Utility.files import expandfilename

                if self.is_prepared not in [None, True]:
                    if hasattr(self.is_prepared, "name"):
                        shared_path = _getSharedPath()
                        if not os.path.isdir(os.path.join(shared_path, self.is_prepared.name)):
                            logger.error("ShareDir directory not found: %s" % self.is_prepared.name)
                            logger.error("Unpreparing %s application" % getattr(self, proxyRef)._name)
                            self.unprepare()

            c = getattr(self, proxyRef).clone()
            if hasattr(c, "is_prepared") and c._getRegistry() is None:
                from Ganga.Core.GangaRepository import getRegistry

                shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
                shareref.increase(self.is_prepared.name)
            c._auto__init__(unprepare=True)
        else:
            c = getattr(self, proxyRef).clone()
            c._auto__init__()
        return GPIProxyObjectFactory(c)
Esempio n. 4
0
 def add(self, input):
     from Ganga.Core.GangaRepository import getRegistry
     if not isType(input, list):
         input = [input]
     for item in input:
         if isType(item, str):
             if os.path.isfile(expandfilename(item)):
                 logger.info('Copying file %s to shared directory %s' %
                             (item, self.name))
                 shutil.copy2(expandfilename(item),
                              os.path.join(getSharedPath(), self.name))
                 shareref = getRegistry("prep").getShareRef()
                 shareref.increase(self.name)
                 shareref.decrease(self.name)
             else:
                 logger.error('File %s not found' % expandfilename(item))
         elif isType(item, File) and item.name is not '' and os.path.isfile(
                 expandfilename(item.name)):
             logger.info('Copying file object %s to shared directory %s' %
                         (item.name, self.name))
             shutil.copy2(expandfilename(item.name),
                          os.path.join(getSharedPath(), self.name))
             shareref = getRegistry("prep").getShareRef()
             shareref.increase(self.name)
             shareref.decrease(self.name)
         else:
             logger.error('File %s not found' % expandfilename(item.name))
Esempio n. 5
0
    def _copy(self, unprepare=None):
        logger.debug('unprepare is %s', str(unprepare))
        if unprepare is None:
            if prepconfig['unprepare_on_copy'] is True:
                if hasattr(self, 'is_prepared') or hasattr(self, 'application'):
                    unprepare = True

        def _getSharedPath():
            Config_conf = getConfig('Configuration')
            return os.path.join(expandfilename(Config_conf['gangadir']), 'shared', Config_conf['user'])

        if hasattr(self, 'application'):
            if hasattr(self.application, 'is_prepared'):
                from Ganga.Utility.files import expandfilename
                if self.application.is_prepared not in [None, True]:
                    if hasattr(self.application.is_prepared, 'name'):
                        shared_path = _getSharedPath()
                        if os.path.isdir(os.path.join(shared_path, self.application.is_prepared.name)):
                            from Ganga.Core.GangaRepository import getRegistry
                            shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
                            logger.debug('increasing counter from proxy.py')
                            shareref.increase(self.application.is_prepared.name)
                            logger.debug('Found ShareDir directory: %s' % self.application.is_prepared.name)
                elif self.application.is_prepared not in [None, True]:
                    shared_path = _getSharedPath()
                    if not os.path.isdir(os.path.join(shared_path, self.application.is_prepared.name)):
                        logger.error('ShareDir directory not found: %s' % self.application.is_prepared.name)
                        logger.error('Unpreparing Job #%s' % self.id)
                        from Ganga.Core.GangaRepository import getRegistry
                        shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
                        shareref.increase(self.application.is_prepared.name)
                        self.unprepare()

        if unprepare is True:
            if hasattr(self, 'is_prepared'):
                from Ganga.Utility.files import expandfilename
                if self.is_prepared not in [None, True]:
                    if hasattr(self.is_prepared, 'name'):
                        shared_path = _getSharedPath()
                        if not os.path.isdir(os.path.join(shared_path, self.is_prepared.name)):
                            logger.error('ShareDir directory not found: %s' % self.is_prepared.name)
                            logger.error('Unpreparing %s application' % getName(stripProxy(self)))
                            self.unprepare()

            c = stripProxy(self).clone()
            if hasattr(c, 'is_prepared') and c._getRegistry() is None:
                from Ganga.Core.GangaRepository import getRegistry
                shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
                shareref.increase(self.is_prepared.name)
            stripProxy(c)._auto__init__(unprepare=True)
        else:
            c = stripProxy(self).clone()
            stripProxy(c)._auto__init__()

        return addProxy(c)
Esempio n. 6
0
def test_all_exceptions(gpi):
    """Create all exceptions and make sure they behave correctly"""

    import Ganga.Core.exceptions
    test_str = "My Test Error"

    def exception_test(err_name):
        """Run tests on the given exception"""
        err_type = getattr(Ganga.Core.exceptions, err_name)
        err_obj = err_type(test_str)
        assert test_str in str(err_obj)

    err_list = [
        "GangaException", "GangaFileError", "PluginError",
        "ApplicationConfigurationError", "ApplicationPrepareError",
        "IncompleteJobSubmissionError", "IncompleteKillError",
        "JobManagerError", "GangaAttributeError", "GangaValueError",
        "GangaIOError", "SplitterError", "ProtectedAttributeError",
        "ReadOnlyObjectError", "TypeMismatchError", "SchemaError",
        "SchemaVersionError", "CredentialsError", "CredentialRenewalError",
        "InvalidCredentialError", "ExpiredCredentialError"
    ]

    for e in err_list:
        exception_test(e)

    # check the BackendError
    from Ganga.Core.exceptions import BackendError
    err = BackendError("TestBackend", test_str)
    assert "TestBackend" in str(err)
    assert test_str in str(err)

    # check the InaccessibleObjectError
    from Ganga.Core.exceptions import InaccessibleObjectError, JobManagerError
    from Ganga.Core.GangaRepository import getRegistry
    err = InaccessibleObjectError(
        getRegistry('jobs').repository, 0,
        JobManagerError("My JobManagerError"))
    assert "jobs" in str(err)
    assert "#0" in str(err)
    assert "My JobManagerError" in str(err)

    # check the RepositoryError
    from Ganga.Core.exceptions import RepositoryError
    from Ganga.Core.GangaRepository import getRegistry
    RepositoryError(getRegistry('jobs').repository, test_str)

    # Construct another to check the except clause in the exception is called
    RepositoryError(getRegistry('jobs').repository, test_str)
Esempio n. 7
0
def shutdown():
    # Shutdown method for all repgistries in order
    from Ganga.Utility.logging import getLogger
    logger = getLogger()
    logger.info('Registry Shutdown')
    #import traceback
    #traceback.print_stack()

    # Flush all repos before we shut them down
    flush_all()

    # shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown()
    # method to perform actions on the box and/or job registries.
    logger.debug(started_registries)

    all_registries = getRegistries()

    try:
        if 'prep' in started_registries:
            registry = getRegistry('prep')
            registry.shutdown()
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(registry.name)
    except Exception as err:
        logger.error("Err: %s" % err)
        logger.error("Failed to Shutdown prep Repository!!! please check for stale lock files")
        logger.error("Trying to shutdown cleanly regardless")

    for registry in getRegistries():
        thisName = registry.name
        try:
            if not thisName in started_registries:
                continue
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(thisName)
            registry.shutdown()  # flush and release locks
        except Exception as x:
            logger.error("Failed to Shutdown Repository: %s !!! please check for stale lock files" % thisName)
            logger.error("%s" % x)
            logger.error("Trying to Shutdown cleanly regardless")


    for registry in all_registries:

        my_reg = [registry]
        if hasattr(registry, 'metadata'):
            if registry.metadata:
                my_reg.append(registry.metadata)

        assigned_attrs = ['location', 'type']
        for this_reg in my_reg:
            for attr in assigned_attrs:
                if hasattr(registry, attr):
                    delattr(registry, attr)

    from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
    removeGlobalSessionFileHandlers()
    removeGlobalSessionFiles()

    removeRegistries()
Esempio n. 8
0
    def test_F_decrement(self):
        """Test that the ref counter is decremented as appropriate"""

        from Ganga.GPI import jobs

        j=jobs[-1]

        shareRef = getRegistry('prep').getShareRef()

        refDict = shareRef.name

        assert refDict[j.application.is_prepared.name] is 2

        this_ref = j.application.is_prepared.name
        this_path = j.application.is_prepared.path()

        j.unprepare()

        assert refDict[this_ref] is 1

        j2=jobs[-2].copy()

        assert refDict[this_ref] is 2

        j2.remove()

        assert refDict[this_ref] is 1

        jobs[-2].remove()

        assert refDict[this_ref] is 0

        assert not path.isdir(this_path)
Esempio n. 9
0
    def test_F_decrement(self):
        """Test that the ref counter is decremented as appropriate"""

        from Ganga.GPI import jobs

        j = jobs[-1]

        shareRef = getRegistry('prep').getShareRef()

        refDict = shareRef.name

        assert refDict[j.application.is_prepared.name] is 2

        this_ref = j.application.is_prepared.name
        this_path = j.application.is_prepared.path()

        j.unprepare()

        assert refDict[this_ref] is 1

        j2 = jobs[-2].copy()

        assert refDict[this_ref] is 2

        j2.remove()

        assert refDict[this_ref] is 1

        jobs[-2].remove()

        assert refDict[this_ref] is 0

        assert not path.isdir(this_path)
Esempio n. 10
0
    def __prep_set__(obj, val):

        # if we set is_prepared to None in the GPI, that should effectively
        # unprepare the application
        if val is None:
            if stripProxy(obj).is_prepared is not None:
                logger.info('Unpreparing application.')
                stripProxy(obj).unprepare()

        # Replace is_prepared on an application for another ShareDir object
        if hasattr(stripProxy(obj), '_getRegistry'):
            from Ganga.GPIDev.Lib.File import ShareDir
            if stripProxy(obj)._getRegistry() is not None and isType(
                    val, ShareDir):
                logger.debug(
                    'Overwriting is_prepared attribute with a ShareDir object')
                # it's safe to unprepare 'not-prepared' applications.
                stripProxy(obj).unprepare()
                from Ganga.Core.GangaRepository import getRegistry
                shareref = GPIProxyObjectFactory(
                    getRegistry("prep").getShareRef())
                shareref.increase(val.name)

        if type(val) is str:
            logger.error("Setting string type to 'is_prepared'")
            import traceback
            traceback.print_stack()
Esempio n. 11
0
def shutdown():
    logger.debug("registry shutdown")
    # shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown()
    # method to perform actions on the box and/or job registries.
    logger.debug(started_registries)
    try:
        if "prep" in started_registries:
            registry = getRegistry("prep")
            registry.shutdown()
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(registry.name)
    except:
        logger.error("Failed to Shutdown prep Repository!!! please check for stale lock files")
        logger.error("Trying to shutdown cleanly regardless")
        pass

    for registry in getRegistries():
        thisName = registry.name
        try:
            if not thisName in started_registries:
                continue
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(thisName)
            registry.shutdown()  # flush and release locks
        except Exception as x:
            logger.error("Failed to Shutdown Repository: %s !!! please check for stale lock files" % thisName)
            logger.error("%s" % str(x))
            logger.error("Trying to Shutdown cleanly regardless")
            pass

    from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers

    removeGlobalSessionFileHandlers()
    removeGlobalSessionFiles()
Esempio n. 12
0
    def __prep_set__(obj, val):

        # if we set is_prepared to None in the GPI, that should effectively
        # unprepare the application
        if val is None:
            if stripProxy(obj).is_prepared is not None:
                logger.info("Unpreparing application.")
                stripProxy(obj).unprepare()

        # Replace is_prepared on an application for another ShareDir object
        if hasattr(stripProxy(obj), "_getRegistry"):
            from Ganga.GPIDev.Lib.File import ShareDir

            if stripProxy(obj)._getRegistry() is not None and isType(val, ShareDir):
                logger.debug("Overwriting is_prepared attribute with a ShareDir object")
                # it's safe to unprepare 'not-prepared' applications.
                stripProxy(obj).unprepare()
                from Ganga.Core.GangaRepository import getRegistry

                shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
                shareref.increase(val.name)

        if type(val) is str:
            logger.error("Setting string type to 'is_prepared'")
            import traceback

            traceback.print_stack()
Esempio n. 13
0
    def __app_set__(obj, val):

        if not hasattr(obj, 'application') or obj.application is None:
            return

        if hasattr(obj.application, '_is_prepared'):

            #a=Job(); a.prepare(); a.application=Executable()
            if obj.application.is_prepared not in [None, True] and\
                 hasattr(val, 'is_prepared') and val.is_prepared is None:
                 logger.debug('Overwriting a prepared application with one that is unprepared')
                 obj.application.unprepare()

            #a=Job(); b=Executable(); b.prepare(); a.application=b
        elif obj.application.is_prepared is not True:
            if hasattr(val, 'is_prepared'):
                if val.is_prepared not in [None, True]:
                    from Ganga.Core.GangaRepository import getRegistry
                    shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
                    logger.debug('Overwriting application with a prepared one')
                    if stripProxy(obj.application) != val:
                        stripProxy(obj.application).unprepare()
                        shareref.increase(val.is_prepared.name)

            # check that the shared directory actually exists before
            # assigning the (prepared) application to a job
            if hasattr(val, 'is_prepared'):
                if val.is_prepared not in [None, True]:
                    if hasattr(val.is_prepared, 'name'):
                        from Ganga.Utility.files import expandfilename
                        Config_conf = getConfig('Configuration')
                        shared_path = os.path.join(expandfilename(Config_conf['gangadir']), 'shared', Config_conf['user'])
                        if not os.path.isdir(os.path.join(shared_path, val.is_prepared.name)):
                            logger.error('ShareDir directory not found: %s' % val.is_prepared.name)
Esempio n. 14
0
    def run(self):
        

        server, port = getHttpServer()
        server.socket.settimeout(1)
        reg = getRegistry("jobs") 

        """
        try:
            server = HTTPServer(('pclcg35.cern.ch', 1234), GetHandler)
            server.socket.settimeout(1)
        except Exception:
            return
        
            print "Another Ganga session is already started with --webgui option"       
            process_details = getProcessDetails()

            print reg.repository.get_other_sessions()
            print "Process id : %s, hostname : %s" % (process_details[0], process_details[1])
            
            self.stop()
            self.unregister()   
            return      
        """
        
        print 'Starting web gui monitoring server, please wait ...'

        #   initialization

        reg = getRegistry("jobs") 
        #calling here first time will take all jobs
        reg.pollChangedJobs("WebGUI")
        #fill jobs dictionary at the begining
        fill_jobs_dictionary()  

        print 'Web gui monitoring server started successfully'
        print
        print 'You can monitor your jobs at the following location: ' + getMonitoringLink(port)

        #server.serve_forever()
        
        try:
                while not self.should_stop():
                        server.handle_request()
        finally:
                pass
Esempio n. 15
0
    def test_0_testRegistryAsserts(self):
        from Ganga.GPI import jobs, box, tasks, prep
        from Ganga.GPIDev.Base.Proxy import stripProxy
        from Ganga.Core.GangaRepository import getRegistryProxy, getRegistrySlice, getRegistry
        assert getRegistryProxy('jobs') is jobs
        assert getRegistryProxy('box') is box
        assert getRegistryProxy('tasks') is tasks
        assert getRegistryProxy('prep') is prep

        assert getRegistrySlice('jobs') is stripProxy(jobs)
        assert getRegistrySlice('box') is stripProxy(box)
        assert getRegistrySlice('tasks') is stripProxy(tasks)
        assert getRegistrySlice('prep') is stripProxy(prep)

        assert getRegistry('jobs') is stripProxy(jobs).objects
        assert getRegistry('tasks') is stripProxy(tasks).objects
        assert getRegistry('box') is stripProxy(box).objects
Esempio n. 16
0
 def listShareDirContents(self, shared_directory_name):
     """
     Function which is used to list the contents of the prepared sandbox folder managed by this app
     Args:
         shared_directory_name (str): full name of directory managed by this app
     """
     shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
     shareref.ls(shared_directory_name)
Esempio n. 17
0
    def test_0_testRegistryAsserts(self):
        from Ganga.GPI import jobs, box, tasks, prep
        from Ganga.GPIDev.Base.Proxy import stripProxy
        from Ganga.Core.GangaRepository import getRegistryProxy, getRegistrySlice, getRegistry
        assert getRegistryProxy('jobs') is jobs
        assert getRegistryProxy('box') is box
        assert getRegistryProxy('tasks') is tasks
        assert getRegistryProxy('prep') is prep

        assert getRegistrySlice('jobs') is stripProxy(jobs)
        assert getRegistrySlice('box') is stripProxy(box)
        assert getRegistrySlice('tasks') is stripProxy(tasks)
        assert getRegistrySlice('prep') is stripProxy(prep)

        assert getRegistry('jobs') is stripProxy(jobs).objects
        assert getRegistry('tasks') is stripProxy(tasks).objects
        assert getRegistry('box') is stripProxy(box).objects
Esempio n. 18
0
 def listShareDirContents(self, shared_directory_name):
     """
     Function which is used to list the contents of the prepared sandbox folder managed by this app
     Args:
         shared_directory_name (str): full name of directory managed by this app
     """
     shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
     shareref.ls(shared_directory_name)
Esempio n. 19
0
    def run(self):

        server, port = getHttpServer()
        server.socket.settimeout(1)
        reg = getRegistry("jobs")

        """
        try:
            server = HTTPServer(('pclcg35.cern.ch', 1234), GetHandler)
            server.socket.settimeout(1)
        except Exception:
            return
        
            print("Another Ganga session is already started with --webgui option" )      
            process_details = getProcessDetails()

            print(reg.repository.get_other_sessions())
            print("Process id : %s, hostname : %s" % (process_details[0], process_details[1]))
            
            self.stop()
            self.unregister()   
            return      
        """

        logger.info('Starting web gui monitoring server, please wait ...')

        #   initialization

        reg = getRegistry("jobs")
        # calling here first time will take all jobs
        reg.pollChangedJobs("WebGUI")
        # fill jobs dictionary at the begining
        fill_jobs_dictionary()

        logger.info('Web gui monitoring server started successfully')
        logger.info('You can monitor your jobs at the following location: ' + getMonitoringLink(port))

        # server.serve_forever()

        try:
            while not self.should_stop():
                server.handle_request()
        finally:
            pass
Esempio n. 20
0
 def _auto__init__(self, registry=None):
     if registry is None:
         from Ganga.Core.GangaRepository import getRegistry
         registry = getRegistry(self.default_registry)
     # register the job (it will also commit it)
     # job gets its id now
     registry._add(self)
     self.creation_date = time.strftime('%Y%m%d%H%M%S')
     self.startup()
     self.status = 'new'
Esempio n. 21
0
 def _auto__init__(self, registry=None):
     if registry is None:
         from Ganga.Core.GangaRepository import getRegistry
         registry = getRegistry(self.default_registry)
     # register the job (it will also commit it)
     # job gets its id now
     registry._add(self)
     self.creation_date = time.strftime('%Y%m%d%H%M%S')
     self.startup()
     self.status = 'new'
Esempio n. 22
0
    def _copy(self, unprepare=None):
        logger.debug('unprepare is %s', str(unprepare))
        if unprepare is None:
            if prepconfig['unprepare_on_copy'] is True:
                if hasattr(self,'is_prepared') or hasattr(self,'application'):
                    unprepare = True

        if hasattr(self,'application'):
            if hasattr(self.application,'is_prepared'):
                if self.application.is_prepared is not None and self.application.is_prepared \
                           is not True and os.path.isdir(os.path.join(shared_path,self.application.is_prepared.name)):
                    from Ganga.Core.GangaRepository import getRegistry
                    shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) 
                    logger.debug('increasing counter from proxy.py')
                    shareref.increase(self.application.is_prepared.name)
                    logger.debug('Found ShareDir directory: %s' % self.application.is_prepared.name)
                elif self.application.is_prepared is not None and self.application.is_prepared \
                           is not True and not os.path.isdir(os.path.join(shared_path,self.application.is_prepared.name)):
                    logger.error('ShareDir directory not found: %s' % self.application.is_prepared.name)
                    logger.error('Unpreparing Job #%s' % self.id)
                    from Ganga.Core.GangaRepository import getRegistry
                    shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) 
                    shareref.increase(self.application.is_prepared.name)
                    self.unprepare()
    
        if hasattr(self,'is_prepared') and self.is_prepared is not None and self.is_prepared \
                           is not True and not os.path.isdir(os.path.join(shared_path,self.is_prepared.name)):
            logger.error('ShareDir directory not found: %s' % self.is_prepared.name)
            logger.error('Unpreparing %s application' % self._impl._name)
            self.unprepare()

            
        if unprepare is True:
            c = self._impl.clone()
            if hasattr(c,'is_prepared') and c._getRegistry() is None:
                from Ganga.Core.GangaRepository import getRegistry
                shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) 
                shareref.increase(self.is_prepared.name)
            c._auto__init__(unprepare=True)
        else:
            c = self._impl.clone()
            c._auto__init__()
        return GPIProxyObjectFactory(c)
Esempio n. 23
0
def test_all_exceptions(gpi):
    """Create all exceptions and make sure they behave correctly"""

    import Ganga.Core.exceptions
    test_str = "My Test Error"

    def exception_test(err_name):
        """Run tests on the given exception"""
        err_type = getattr(Ganga.Core.exceptions, err_name)
        err_obj = err_type(test_str)
        assert test_str in str(err_obj)

    err_list = ["GangaException", "GangaFileError", "PluginError", "ApplicationConfigurationError",
                "ApplicationPrepareError", "IncompleteJobSubmissionError", "IncompleteKillError", "JobManagerError",
                "GangaAttributeError", "GangaValueError", "GangaIOError", "SplitterError", "ProtectedAttributeError",
                "ReadOnlyObjectError", "TypeMismatchError", "SchemaError", "SchemaVersionError", "CredentialsError",
                "CredentialRenewalError", "InvalidCredentialError", "ExpiredCredentialError"]

    for e in err_list:
        exception_test(e)

    # check the BackendError
    from Ganga.Core.exceptions import BackendError
    err = BackendError("TestBackend", test_str)
    assert "TestBackend" in str(err)
    assert test_str in str(err)

    # check the InaccessibleObjectError
    from Ganga.Core.exceptions import InaccessibleObjectError, JobManagerError
    from Ganga.Core.GangaRepository import getRegistry
    err = InaccessibleObjectError(getRegistry('jobs').repository, 0, JobManagerError("My JobManagerError"))
    assert "jobs" in str(err)
    assert "#0" in str(err)
    assert "My JobManagerError" in str(err)

    # check the RepositoryError
    from Ganga.Core.exceptions import RepositoryError
    from Ganga.Core.GangaRepository import getRegistry
    RepositoryError(getRegistry('jobs').repository, test_str)

    # Construct another to check the except clause in the exception is called
    RepositoryError(getRegistry('jobs').repository, test_str)
Esempio n. 24
0
    def __incrementShareRef(obj, attr_name):
        shared_dir = getattr(obj, attr_name)

        if hasattr(shared_dir, 'name'):

            from Ganga.Core.GangaRepository import getRegistry
            from Ganga.GPIDev.Base.Proxy import GPIProxyObjectFactory
            shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())

            logger.debug("Increasing shareref")
            shareref.increase(shared_dir.name)
Esempio n. 25
0
 def decrementShareCounter(self, shared_directory_name, remove=0):
     """
     Function which is used to decrement the number of (sub)jobs which share the prepared sandbox
     managed by this app
     Args:
         shared_directory_name (str): full name of directory managed by this app
     """
     remove = remove
     logger.debug('Decrementing shared directory reference counter')
     shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
     shareref.decrease(shared_directory_name, remove)
Esempio n. 26
0
 def incrementShareCounter(self, shared_directory_name):
     """
     Function which is used to increment the number of (sub)jobs which share the prepared sandbox
     managed by this app
     Args:
         shared_directory_name (str): full name of directory managed by this app
     """
     logger.debug('Incrementing shared directory reference counter')
     shareref = getRegistry("prep").getShareRef()
     logger.debug('within incrementShareCounter, calling increase')
     shareref.increase(shared_directory_name)
Esempio n. 27
0
 def decrementShareCounter(self, shared_directory_name, remove=0):
     """
     Function which is used to decrement the number of (sub)jobs which share the prepared sandbox
     managed by this app
     Args:
         shared_directory_name (str): full name of directory managed by this app
     """
     remove = remove
     logger.debug('Decrementing shared directory reference counter')
     shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
     shareref.decrease(shared_directory_name, remove)
Esempio n. 28
0
 def incrementShareCounter(self, shared_directory_name):
     """
     Function which is used to increment the number of (sub)jobs which share the prepared sandbox
     managed by this app
     Args:
         shared_directory_name (str): full name of directory managed by this app
     """
     logger.debug('Incrementing shared directory reference counter')
     shareref = getRegistry("prep").getShareRef()
     logger.debug('within incrementShareCounter, calling increase')
     shareref.increase(shared_directory_name)
Esempio n. 29
0
 def add(self, input):
     if not isType(input, list):
         input = [input] 
     for item in input:
         if isType(item, str):
             if os.path.isfile(expandfilename(item)):
                 logger.info('Copying file %s to shared directory %s'%(item, self.name))
                 shutil.copy2(expandfilename(item), os.path.join(shared_path,self.name))
                 shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
                 shareref.increase(self.name)
                 shareref.decrease(self.name)
             else:
                 logger.error('File %s not found' % expandfilename(item))
         elif isType(item,File) and item.name is not '' and os.path.isfile(expandfilename(item.name)):
             logger.info('Copying file object %s to shared directory %s'%(item.name,self.name))
             shutil.copy2(expandfilename(item.name), os.path.join(shared_path,self.name))
             shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
             shareref.increase(self.name)
             shareref.decrease(self.name)
         else:
             logger.error('File %s not found' % expandfilename(item.name))
Esempio n. 30
0
 def add(self, input):
     from Ganga.Core.GangaRepository import getRegistry
     if not isType(input, list):
         input = [input]
     for item in input:
         if isType(item, str):
             if os.path.isfile(expandfilename(item)):
                 logger.info('Copying file %s to shared directory %s' % (item, self.name))
                 shutil.copy2(expandfilename(item), os.path.join(getSharedPath(), self.name))
                 shareref = getRegistry("prep").getShareRef()
                 shareref.increase(self.name)
                 shareref.decrease(self.name)
             else:
                 logger.error('File %s not found' % expandfilename(item))
         elif isType(item, File) and item.name is not '' and os.path.isfile(expandfilename(item.name)):
             logger.info('Copying file object %s to shared directory %s' % (item.name, self.name))
             shutil.copy2(expandfilename(item.name), os.path.join(getSharedPath(), self.name))
             shareref = getRegistry("prep").getShareRef()
             shareref.increase(self.name)
             shareref.decrease(self.name)
         else:
             logger.error('File %s not found' % expandfilename(item.name))
Esempio n. 31
0
def shutdown():
    logger.debug('registry shutdown')
    #shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown()
    #method to perform actions on the box and/or job registries.
    logger.debug(started_registries)
    if 'prep' in started_registries: 
        registry = getRegistry('prep')
        registry.shutdown()
        started_registries.remove(registry.name) # in case this is called repeatedly, only call shutdown once
        
    for registry in getRegistries():
        if not registry.name in started_registries: continue
        started_registries.remove(registry.name) # in case this is called repeatedly, only call shutdown once
        registry.shutdown() # flush and release locks
Esempio n. 32
0
    def test_E_increment(self):
        """Test that the ref counter is incremented as appropriate"""

        from Ganga.GPI import jobs

        j=jobs[-1]

        shareRef = getRegistry('prep').getShareRef()

        refDict = shareRef.name

        assert j.application.is_prepared.name in refDict

        j2 = j.copy()

        assert refDict[j.application.is_prepared.name] is 2
Esempio n. 33
0
    def test_E_increment(self):
        """Test that the ref counter is incremented as appropriate"""

        from Ganga.GPI import jobs

        j = jobs[-1]

        shareRef = getRegistry('prep').getShareRef()

        refDict = shareRef.name

        assert j.application.is_prepared.name in refDict

        j2 = j.copy()

        assert refDict[j.application.is_prepared.name] is 2
Esempio n. 34
0
 def __deepcopy__(self, memo = None):
     self._getReadAccess()
     c = super(GangaObject,self).__deepcopy__(memo)
     for name,item in self._schema.allItems():
         if not item['copyable']:
             setattr(c,name,self._schema.getDefaultValue(name))
         if item.isA(Schema.SharedItem):
             shared_dir=getattr(c,name)
             try:
                 from Ganga.Core.GangaRepository import getRegistry
                 shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
                 logger.debug("Increasing shareref")
                 shareref.increase(shared_dir.name)
             except AttributeError:
                 pass
     return c
Esempio n. 35
0
def update_jobs_dictionary():

    reg = getRegistry("jobs") 
    #get the changed jobs
    changed_ids = reg.pollChangedJobs("WebGUI")

    for job_id in changed_ids:
        try:

            job = jobs(job_id)

            try:        
                jobs_dictionary[job_id] = JobRelatedInfo(job, job.time.timestamps['new']) 
            except RegistryKeyError:
                jobs_dictionary[job_id] = JobRelatedInfo(job, None)       

        except RegistryKeyError:

            del jobs_dictionary[job_id]
Esempio n. 36
0
def update_jobs_dictionary():
    from Ganga.Core.GangaRepository import getRegistryProxy

    reg = getRegistry("jobs")
    # get the changed jobs
    changed_ids = reg.pollChangedJobs("WebGUI")

    for job_id in changed_ids:
        try:

            job = getRegistryProxy('jobs')(job_id)

            try:
                jobs_dictionary[job_id] = JobRelatedInfo(
                    job, job.time.timestamps['new'])
            except RegistryKeyError:
                jobs_dictionary[job_id] = JobRelatedInfo(job, None)

        except RegistryKeyError:

            del jobs_dictionary[job_id]
Esempio n. 37
0
    def __app_set__(obj, val):

        if hasattr(obj.application, '_is_prepared'):

            #a=Job(); a.prepare(); a.application=Executable()
            if obj.application.is_prepared not in [None, True] and\
                 hasattr(val, 'is_prepared') and val.is_prepared is None:
                logger.debug(
                    'Overwriting a prepared application with one that is unprepared'
                )
                obj.application.unprepare()

            #a=Job(); b=Executable(); b.prepare(); a.application=b
        elif obj.application.is_prepared is not True:
            if hasattr(val, 'is_prepared'):
                if val.is_prepared not in [None, True]:
                    from Ganga.Core.GangaRepository import getRegistry
                    shareref = GPIProxyObjectFactory(
                        getRegistry("prep").getShareRef())
                    logger.debug('Overwriting application with a prepared one')
                    if stripProxy(obj.application) != val:
                        stripProxy(obj.application).unprepare()
                        shareref.increase(val.is_prepared.name)

            # check that the shared directory actually exists before
            # assigning the (prepared) application to a job
            if hasattr(val, 'is_prepared'):
                if val.is_prepared not in [None, True]:
                    if hasattr(val.is_prepared, 'name'):
                        from Ganga.Utility.files import expandfilename
                        Config_conf = getConfig('Configuration')
                        shared_path = os.path.join(
                            expandfilename(Config_conf['gangadir']), 'shared',
                            Config_conf['user'])
                        if not os.path.isdir(
                                os.path.join(shared_path,
                                             val.is_prepared.name)):
                            logger.error('ShareDir directory not found: %s' %
                                         val.is_prepared.name)
Esempio n. 38
0
    def __deepcopy__(self, memo=None):
        self = stripProxy(self)
        self._getReadAccess()
        c = super(GangaObject, self).__deepcopy__(memo)
        if self._schema is not None:
            for name, item in self._schema.allItems():
                if not item['copyable']:
                    setattr(c, name, self._schema.getDefaultValue(name))

                if item.isA(Schema.SharedItem):

                    shared_dir = getattr(c, name)

                    if hasattr(shared_dir, 'name'):

                        from Ganga.Core.GangaRepository import getRegistry
                        from Ganga.GPIDev.Base.Proxy import GPIProxyObjectFactory
                        shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())

                        logger.debug("Increasing shareref")
                        shareref.increase(shared_dir.name)
        return c
Esempio n. 39
0
def shutdown():
    from Ganga.Utility.logging import getLogger
    logger = getLogger()
    logger.info('Registry Shutdown')
    #import traceback
    #traceback.print_stack()
    # shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown()
    # method to perform actions on the box and/or job registries.
    logger.debug(started_registries)
    try:
        if 'prep' in started_registries:
            registry = getRegistry('prep')
            registry.shutdown()
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(registry.name)
    except Exception as err:
        logger.debug("Err: %s" % str(err))
        logger.error("Failed to Shutdown prep Repository!!! please check for stale lock files")
        logger.error("Trying to shutdown cleanly regardless")

    for registry in getRegistries():
        thisName = registry.name
        try:
            if not thisName in started_registries:
                continue
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(thisName)
            registry.shutdown()  # flush and release locks
        except Exception as x:
            logger.error("Failed to Shutdown Repository: %s !!! please check for stale lock files" % thisName)
            logger.error("%s" % str(x))
            logger.error("Trying to Shutdown cleanly regardless")

    from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
    removeGlobalSessionFileHandlers()
    removeGlobalSessionFiles()
Esempio n. 40
0
def shutdown():
    from Ganga.Utility.logging import getLogger
    logger = getLogger()
    logger.info('Registry Shutdown')
    #import traceback
    #traceback.print_stack()
    # shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown()
    # method to perform actions on the box and/or job registries.
    logger.debug(started_registries)
    try:
        if 'prep' in started_registries:
            registry = getRegistry('prep')
            registry.shutdown()
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(registry.name)
    except Exception as err:
        logger.debug("Err: %s" % str(err))
        logger.error("Failed to Shutdown prep Repository!!! please check for stale lock files")
        logger.error("Trying to shutdown cleanly regardless")

    for registry in getRegistries():
        thisName = registry.name
        try:
            if not thisName in started_registries:
                continue
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(thisName)
            registry.shutdown()  # flush and release locks
        except Exception as x:
            logger.error("Failed to Shutdown Repository: %s !!! please check for stale lock files" % thisName)
            logger.error("%s" % str(x))
            logger.error("Trying to Shutdown cleanly regardless")

    from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
    removeGlobalSessionFileHandlers()
    removeGlobalSessionFiles()
Esempio n. 41
0
 def listShareDirContents(self, shared_directory_name):
     shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
     shareref.ls(shared_directory_name)
Esempio n. 42
0
 def listShareDirs(self):
     shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
     shareref
Esempio n. 43
0
    def _thread_main(self):
        """ This is an internal function; the main loop of the background thread """
        from Ganga.Core.GangaRepository import getRegistry
        while getRegistry("jobs").hasStarted() is not True:
            time.sleep(0.1)
            if self._main_thread is None or self._main_thread.should_stop():
                return

        while True:
            from Ganga.Core import monitoring_component
            if (not monitoring_component is None and monitoring_component.enabled) or config['ForceTaskMonitoring']:
                break
            time.sleep(0.1)
            if self._main_thread is None or self._main_thread.should_stop():
                return

        # setup the tasks - THIS IS INCOMPATIBLE WITH CONCURRENCY
        # and must go away soon
        for tid in self.ids():
            try:
                self[tid].startup()
            except Exception as err:
                logger.error("Unknown/Unexpected Error in starting up tasks main loop")
                logger.error("Exiting: err=%s" % str(err))
                return

        logger.debug("Entering main loop")

        # Main loop
        while self._main_thread is not None and not self._main_thread.should_stop():

            # If monitoring is enabled (or forced for Tasks) loop over each one and update
            if (config['ForceTaskMonitoring'] or monitoring_component.enabled) and not config['disableTaskMon']:
                for tid in self.ids():

                    logger.debug("Running over tid: %s" % str(tid))

                    try:
                        p = self[tid]
                        p.update()

                    except Exception as x:
                        logger.error(
                            "Exception occurred in task monitoring loop: %s %s\nThe offending task was paused." % (x.__class__, x))
                        type_, value_, traceback_ = sys.exc_info()
                        logger.error("Full traceback:\n %s" % ' '.join(
                            traceback.format_exception(type_, value_, traceback_)))
                        p.pause()

                    if self._main_thread.should_stop():
                        break

                if self._main_thread.should_stop():
                    break

            logger.debug("TaskRegistry Sleeping for: %s seconds" % str(config['TaskLoopFrequency']))

            # Sleep interruptible for 10 seconds
            for i in range(0, int(config['TaskLoopFrequency'] * 100)):
                if self._main_thread.should_stop():
                    break
                time.sleep(0.01)
Esempio n. 44
0
 def decrementShareCounter(self, shared_directory_name, remove=0):
     remove = remove
     logger.debug('Decrementing shared directory reference counter')
     shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
     shareref.decrease(shared_directory_name, remove)
Esempio n. 45
0
 def listShareDirs(self):
     shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
     shareref
Esempio n. 46
0
 def listShareDirContents(self,shared_directory_name):
     shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
     shareref.ls(shared_directory_name)
Esempio n. 47
0
def shutdown():
    # Shutdown method for all repgistries in order
    from Ganga.Utility.logging import getLogger
    logger = getLogger()
    logger.info('Registry Shutdown')
    #import traceback
    #traceback.print_stack()

    # Flush all repos before we shut them down
    flush_all()

    # shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown()
    # method to perform actions on the box and/or job registries.
    logger.debug(started_registries)

    all_registries = getRegistries()

    try:
        if 'prep' in started_registries:
            registry = getRegistry('prep')
            registry.shutdown()
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(registry.name)
    except Exception as err:
        logger.error("Err: %s" % err)
        logger.error(
            "Failed to Shutdown prep Repository!!! please check for stale lock files"
        )
        logger.error("Trying to shutdown cleanly regardless")

    for registry in getRegistries():
        thisName = registry.name
        try:
            if not thisName in started_registries:
                continue
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(thisName)
            registry.shutdown()  # flush and release locks
        except Exception as x:
            logger.error(
                "Failed to Shutdown Repository: %s !!! please check for stale lock files"
                % thisName)
            logger.error("%s" % x)
            logger.error("Trying to Shutdown cleanly regardless")

    for registry in all_registries:

        my_reg = [registry]
        if hasattr(registry, 'metadata'):
            if registry.metadata:
                my_reg.append(registry.metadata)

        assigned_attrs = ['location', 'type']
        for this_reg in my_reg:
            for attr in assigned_attrs:
                if hasattr(registry, attr):
                    delattr(registry, attr)

    from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
    removeGlobalSessionFileHandlers()
    removeGlobalSessionFiles()

    removeRegistries()
Esempio n. 48
0
 def decrementShareCounter(self, shared_directory_name, remove=0):
     remove=remove
     logger.debug('Decrementing shared directory reference counter')
     shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
     shareref.decrease(shared_directory_name, remove)
Esempio n. 49
0
                        convert_merger_to_postprocessor, 'Functions')
exportToPublicInterface('config', Ganga.GPIDev.Lib.Config.config, 'Objects',
                        'access to Ganga configuration')
exportToPublicInterface('ConfigError', Ganga.GPIDev.Lib.Config.ConfigError,
                        'Exceptions')
exportToPublicInterface('report', report, 'Functions')

# ------------------------------------------------------------------------------------
# bootstrap the repositories and connect to them
for n, k, d in Repository_runtime.bootstrap():
    # make all repository proxies visible in GPI
    exportToPublicInterface(n, k, 'Objects', d)

# ------------------------------------------------------------------------------------
# JobTree
jobtree = GPIProxyObjectFactory(getRegistry("jobs").getJobTree())
exportToPublicInterface('jobtree', jobtree, 'Objects',
                        'Logical tree view of the jobs')
exportToPublicInterface('TreeError', TreeError, 'Exceptions')

# ------------------------------------------------------------------------------------
# ShareRef
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
exportToPublicInterface(
    'shareref', shareref, 'Objects',
    'Mechanism for tracking use of shared directory resources')

# ------------------------------------------------------------------------------------
# bootstrap the workspace
Workspace_runtime.bootstrap()
Esempio n. 50
0
 def incrementShareCounter(self, shared_directory_name):
     logger.debug('Incrementing shared directory reference counter')
     shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
     logger.debug('within incrementShareCounter, calling increase')
     shareref.increase(shared_directory_name)
Esempio n. 51
0
    def _thread_main(self):
        """ This is an internal function; the main loop of the background thread """
        from Ganga.Core.GangaRepository import getRegistry
        while getRegistry("jobs").hasStarted() is not True:
            time.sleep(0.1)
            if self._main_thread is None or self._main_thread.should_stop():
                return

        while True:
            from Ganga.Core import monitoring_component
            if (not monitoring_component is None
                    and monitoring_component.enabled
                ) or config['ForceTaskMonitoring']:
                break
            time.sleep(0.1)
            if self._main_thread is None or self._main_thread.should_stop():
                return

        # setup the tasks - THIS IS INCOMPATIBLE WITH CONCURRENCY
        # and must go away soon
        for tid in self.ids():
            try:
                self[tid].startup()
            except Exception as err:
                logger.error(
                    "Unknown/Unexpected Error in starting up tasks main loop")
                logger.error("Exiting: err=%s" % str(err))
                return

        logger.debug("Entering main loop")

        # Main loop
        while self._main_thread is not None and not self._main_thread.should_stop(
        ):

            # If monitoring is enabled (or forced for Tasks) loop over each one and update
            if (config['ForceTaskMonitoring'] or monitoring_component.enabled
                ) and not config['disableTaskMon']:
                for tid in self.ids():

                    logger.debug("Running over tid: %s" % str(tid))

                    try:
                        p = self[tid]
                        p.update()

                    except Exception as x:
                        logger.error(
                            "Exception occurred in task monitoring loop: %s %s\nThe offending task was paused."
                            % (x.__class__, x))
                        type_, value_, traceback_ = sys.exc_info()
                        logger.error("Full traceback:\n %s" % ' '.join(
                            traceback.format_exception(type_, value_,
                                                       traceback_)))
                        p.pause()

                    if self._main_thread.should_stop():
                        break

                if self._main_thread.should_stop():
                    break

            logger.debug("TaskRegistry Sleeping for: %s seconds" %
                         str(config['TaskLoopFrequency']))

            # Sleep interruptible for 10 seconds
            for i in range(0, int(config['TaskLoopFrequency'] * 100)):
                if self._main_thread.should_stop():
                    break
                time.sleep(0.01)
Esempio n. 52
0
    def _copy(self, unprepare=None):
        logger.debug('unprepare is %s', str(unprepare))
        if unprepare is None:
            if prepconfig['unprepare_on_copy'] is True:
                if hasattr(self, 'is_prepared') or hasattr(
                        self, 'application'):
                    unprepare = True

        def _getSharedPath():
            Config_conf = getConfig('Configuration')
            return os.path.join(expandfilename(Config_conf['gangadir']),
                                'shared', Config_conf['user'])

        if hasattr(self, 'application'):
            if hasattr(self.application, 'is_prepared'):
                from Ganga.Utility.files import expandfilename
                if self.application.is_prepared not in [None, True]:
                    if hasattr(self.application.is_prepared, 'name'):
                        shared_path = _getSharedPath()
                        if os.path.isdir(
                                os.path.join(
                                    shared_path,
                                    self.application.is_prepared.name)):
                            from Ganga.Core.GangaRepository import getRegistry
                            shareref = GPIProxyObjectFactory(
                                getRegistry("prep").getShareRef())
                            logger.debug('increasing counter from proxy.py')
                            shareref.increase(
                                self.application.is_prepared.name)
                            logger.debug('Found ShareDir directory: %s' %
                                         self.application.is_prepared.name)
                elif self.application.is_prepared not in [None, True]:
                    shared_path = _getSharedPath()
                    if not os.path.isdir(
                            os.path.join(shared_path,
                                         self.application.is_prepared.name)):
                        logger.error('ShareDir directory not found: %s' %
                                     self.application.is_prepared.name)
                        logger.error('Unpreparing Job #%s' % self.id)
                        from Ganga.Core.GangaRepository import getRegistry
                        shareref = GPIProxyObjectFactory(
                            getRegistry("prep").getShareRef())
                        shareref.increase(self.application.is_prepared.name)
                        self.unprepare()

        if unprepare is True:
            if hasattr(self, 'is_prepared'):
                from Ganga.Utility.files import expandfilename
                if self.is_prepared not in [None, True]:
                    if hasattr(self.is_prepared, 'name'):
                        shared_path = _getSharedPath()
                        if not os.path.isdir(
                                os.path.join(shared_path,
                                             self.is_prepared.name)):
                            logger.error('ShareDir directory not found: %s' %
                                         self.is_prepared.name)
                            logger.error('Unpreparing %s application' %
                                         getName(stripProxy(self)))
                            self.unprepare()

            c = stripProxy(self).clone()
            if hasattr(c, 'is_prepared') and c._getRegistry() is None:
                from Ganga.Core.GangaRepository import getRegistry
                shareref = GPIProxyObjectFactory(
                    getRegistry("prep").getShareRef())
                shareref.increase(self.is_prepared.name)
            stripProxy(c)._auto__init__(unprepare=True)
        else:
            c = stripProxy(self).clone()
            stripProxy(c)._auto__init__()
        return GPIProxyObjectFactory(c)
Esempio n. 53
0
    def _thread_main(self):
        """ This is an internal function; the main loop of the background thread """
        # Add runtime handlers for all the taskified applications, since now
        # all the backends are loaded
        from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
        from .TaskApplication import handler_map
        for basename, name in handler_map:
            for backend in allHandlers.getAllBackends(basename):
                allHandlers.add(name, backend,
                                allHandlers.get(basename, backend))

        from Ganga.Core.GangaRepository import getRegistry
        while getRegistry("jobs").hasStarted() is not True:
            time.sleep(0.1)
            if self._main_thread is None or self._main_thread.should_stop():
                return

        while True:
            from Ganga.Core import monitoring_component
            if (not monitoring_component is None
                    and monitoring_component.enabled
                ) or config['ForceTaskMonitoring']:
                break
            time.sleep(0.1)
            if self._main_thread is None or self._main_thread.should_stop():
                return

        # setup the tasks - THIS IS INCOMPATIBLE WITH CONCURRENCY
        # and must go away soon
        for tid in self.ids():
            try:
                self[tid]._getWriteAccess()
                self[tid].startup()
            except RegistryError:
                continue
            except Exception as err:
                logger.error(
                    "Unknown/Unexpected Error in starting up tasks main loop")
                logger.error("Exiting: err=%s" % str(err))
                return

        logger.debug("Entering main loop")

        # Main loop
        while self._main_thread is not None and not self._main_thread.should_stop(
        ):

            # For each task try to run it
            if config['ForceTaskMonitoring'] or monitoring_component.enabled:
                for tid in self.ids():

                    logger.debug("Running over tid: %s" % str(tid))

                    try:
                        from Ganga.GPIDev.Lib.Tasks import ITask
                        if isType(self[tid], ITask):
                            # for new ITasks, always need write access
                            self[tid]._getWriteAccess()
                            p = self[tid]
                        else:
                            if self[tid].status in [
                                    "running", "running/pause"
                            ]:
                                self[tid]._getWriteAccess()
                                p = self[tid]
                            elif self[tid].status is 'completed' and (
                                    self[tid].n_status('ready')
                                    or self[tid].n_status('running')):
                                self[tid].updateStatus()
                                continue
                            else:
                                continue
                    except RegistryError:
                        # could not acquire lock
                        continue

                    if self._main_thread.should_stop():
                        break

                    try:
                        from Ganga.GPIDev.Lib.Tasks import ITask
                        if isType(self[tid], ITask):
                            # for new ITasks, always call update()
                            p.update()
                        else:
                            # TODO: Make this user-configurable and add better
                            # error message
                            if (p.n_status("failed") * 100.0 /
                                (20 + p.n_status("completed")) > 20):
                                p.pause()
                                logger.error(
                                    "Task %s paused - %i jobs have failed while only %i jobs have completed successfully."
                                    % (p.name, p.n_status("failed"),
                                       p.n_status("completed")))
                                logger.error(
                                    "Please investigate the cause of the failing jobs and then remove the previously failed jobs using job.remove()"
                                )
                                logger.error(
                                    "You can then continue to run this task with tasks(%i).run()"
                                    % p.id)
                                continue
                            numjobs = p.submitJobs()
                            if numjobs > 0:
                                self._flush([p])
                            # finalise any required transforms
                            p.finaliseTransforms()
                            p.updateStatus()

                    except Exception as x:
                        logger.error(
                            "Exception occurred in task monitoring loop: %s %s\nThe offending task was paused."
                            % (x.__class__, x))
                        type_, value_, traceback_ = sys.exc_info()
                        logger.error("Full traceback:\n %s" % ' '.join(
                            traceback.format_exception(type_, value_,
                                                       traceback_)))
                        p.pause()

                    if self._main_thread.should_stop():
                        break
                if self._main_thread.should_stop():
                    break

            logger.debug("TaskRegistry Sleeping for: %s seconds" %
                         str(config['TaskLoopFrequency']))

            # Sleep interruptible for 10 seconds
            for i in range(0, int(config['TaskLoopFrequency'] * 100)):
                if self._main_thread.should_stop():
                    break
                time.sleep(0.01)