Ejemplo n.º 1
0
def bootstrap(reg_slice, interactive_session, my_interface=None):
    """
    Create local subsystems. In the future this procedure should be enhanced to connect to remote subsystems.
    FIXME: this procedure should be moved to the Runtime package.

    This function will change the default value of autostart of the monitoring, depending if the session is interactive or batch.
    The autostart value may be overriden in the config file, so warn if it differs from the default.
    """
    from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import JobRegistry_Monitor, config
    from Ganga.Utility.logging import getLogger

    logger = getLogger()

    from Ganga.Core.GangaThread import GangaThreadPool

    # start the internal services coordinator
    from Ganga.Core.InternalServices import Coordinator
    Coordinator.bootstrap()

    # backend-specific setup (e.g. Remote: setup any remote ssh pipes)
    # for j in reg_slice:
    #    if hasattr(j,'status') and j.status in ['submitted','running']:
    #        if hasattr(j,'backend'): # protect: EmptyGangaObject does not have backend either
    #            if hasattr(j.backend,'setup'): # protect: EmptyGangaObject does not have setup() method
    #                j.backend.setup()

    start_jobregistry_monitor(reg_slice)

    # Set the shutdown policy depending on whether we're interactive or not
    if config['forced_shutdown_policy'] in ['interactive', 'batch']:
        GangaThreadPool.shutdown_policy = config['forced_shutdown_policy']
    else:
        if interactive_session:
            GangaThreadPool.shutdown_policy = 'interactive'
        else:
            GangaThreadPool.shutdown_policy = 'batch'

    # export to GPI moved to Runtime bootstrap

    autostart_default = interactive_session
    config.overrideDefaultValue('autostart', bool(autostart_default))

    if config['autostart'] is not autostart_default:
        msg = 'monitoring loop %s (the default setting for %s session is %s)'
        val = {
            True: ('enabled', 'batch', 'disabled'),
            False: ('disabled', 'interactive', 'enabled')
        }
        logger.warning(msg % val[config['autostart']])

    if config['autostart']:
        monitoring_component.enableMonitoring()

    if not my_interface:
        import Ganga.GPI
        my_interface = Ganga.GPI
    from Ganga.Runtime.GPIexport import exportToInterface
    exportToInterface(my_interface, 'runMonitoring',
                      monitoring_component.runMonitoring, 'Functions')
Ejemplo n.º 2
0
def bootstrap(reg_slice, interactive_session, my_interface=None):
    """
    Create local subsystems. In the future this procedure should be enhanced to connect to remote subsystems.
    FIXME: this procedure should be moved to the Runtime package.

    This function will change the default value of autostart of the monitoring, depending if the session is interactive or batch.
    The autostart value may be overriden in the config file, so warn if it differs from the default.
    """
    from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import JobRegistry_Monitor, config
    from Ganga.Utility.logging import getLogger

    logger = getLogger()

    from Ganga.Core.GangaThread import GangaThreadPool

    # start the internal services coordinator
    from Ganga.Core.InternalServices import Coordinator

    Coordinator.bootstrap()

    # backend-specific setup (e.g. Remote: setup any remote ssh pipes)
    # for j in reg_slice:
    #    if hasattr(j,'status') and j.status in ['submitted','running']:
    #        if hasattr(j,'backend'): # protect: EmptyGangaObject does not have backend either
    #            if hasattr(j.backend,'setup'): # protect: EmptyGangaObject does not have setup() method
    #                j.backend.setup()

    start_jobregistry_monitor(reg_slice)

    # Set the shutdown policy depending on whether we're interactive or not
    if config["forced_shutdown_policy"] in ["interactive", "batch"]:
        GangaThreadPool.shutdown_policy = config["forced_shutdown_policy"]
    else:
        if interactive_session:
            GangaThreadPool.shutdown_policy = "interactive"
        else:
            GangaThreadPool.shutdown_policy = "batch"

    # export to GPI moved to Runtime bootstrap

    autostart_default = interactive_session
    config.overrideDefaultValue("autostart", bool(autostart_default))

    if config["autostart"] is not autostart_default:
        msg = "monitoring loop %s (the default setting for %s session is %s)"
        val = {True: ("enabled", "batch", "disabled"), False: ("disabled", "interactive", "enabled")}
        logger.warning(msg % val[config["autostart"]])

    if config["autostart"]:
        monitoring_component.enableMonitoring()

    if not my_interface:
        import Ganga.GPI

        my_interface = Ganga.GPI
    from Ganga.Runtime.GPIexport import exportToInterface

    exportToInterface(my_interface, "runMonitoring", monitoring_component.runMonitoring, "Functions")
Ejemplo n.º 3
0
def startUpRegistries(my_interface=None):
    # Startup the registries and export them to the GPI, also add jobtree and shareref
    from Ganga.Runtime.GPIexport import exportToInterface
    if not my_interface:
        import Ganga.GPI
        my_interface = Ganga.GPI
    # import default runtime modules

    global _runtime_interface
    _runtime_interface = my_interface

    # bootstrap user-defined runtime modules and enable transient named
    # template registries

    # bootstrap runtime modules
    from Ganga.GPIDev.Lib.JobTree import TreeError

    for n, k, d in bootstrap():
        # make all repository proxies visible in GPI
        exportToInterface(my_interface, n, k, 'Objects', d)

    # JobTree
    from Ganga.Core.GangaRepository import getRegistry
    jobtree = getRegistry("jobs").getJobTree()
    exportToInterface(my_interface, 'jobtree', jobtree, 'Objects', 'Logical tree view of the jobs')
    exportToInterface(my_interface, 'TreeError', TreeError, 'Exceptions')

    # ShareRef
    shareref = getRegistry("prep").getShareRef()
    exportToInterface(my_interface, 'shareref', shareref, 'Objects', 'Mechanism for tracking use of shared directory resources')
Ejemplo n.º 4
0
def bootstrap(reg_slice, interactive_session, my_interface=None):
    """Create local subsystems.

    This function will change the default value of autostart of the monitoring, depending if the session is interactive
    or batch. The autostart value may be overridden in the config file, so warn if it differs from the default.

    Args:
        reg_slice (RegistrySlice): A registry slice encompassing the Registry to monitor,
            e.g. from getRegistrySlice('jobs') -> JobRegistry.getSlice()
        interactive_session (bool): Flag indicating an interactive session or not
        my_interface (Optional[module]): Public interface to add the runMonitoring function to, None (default) will set
            it to Ganga.GPI
    """
    # Must do some Ganga imports here to avoid circular importing
    from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import JobRegistry_Monitor
    from Ganga.Utility.Config import getConfig
    from Ganga.Runtime.GPIexport import exportToInterface
    from Ganga.Utility.logging import getLogger
    global monitoring_component

    # start the monitoring loop
    monitoring_component = JobRegistry_Monitor(reg_slice)
    monitoring_component.start()

    # override the default monitoring autostart value with the setting from interactive session
    config = getConfig("PollThread")
    config.overrideDefaultValue('autostart', interactive_session)

    # has the user changed monitoring autostart from the default? if so, warn them
    if config['autostart'] != interactive_session:
        if config['autostart']:
            getLogger().warning(
                'Monitoring loop enabled (the default setting for a batch session is disabled)'
            )
        else:
            getLogger().warning(
                'Monitoring loop disabled (the default setting for an interactive session is enabled)'
            )

    # Enable job monitoring if requested
    if config['autostart']:
        monitoring_component.enableMonitoring()

    # export the runMonitoring function to the public interface
    if not my_interface:
        import Ganga.GPI
        my_interface = Ganga.GPI

    exportToInterface(my_interface, 'runMonitoring',
                      monitoring_component.runMonitoring, 'Functions')
Ejemplo n.º 5
0
def startUpRegistries(my_interface=None):
    # Startup the registries and export them to the GPI, also add jobtree and shareref
    from Ganga.Runtime.GPIexport import exportToInterface
    if not my_interface:
        import Ganga.GPI
        my_interface = Ganga.GPI
    # import default runtime modules

    global _runtime_interface
    _runtime_interface = my_interface

    # bootstrap user-defined runtime modules and enable transient named
    # template registries

    # bootstrap runtime modules
    from Ganga.GPIDev.Lib.JobTree import TreeError

    for n, k, d in bootstrap():
        # make all repository proxies visible in GPI
        exportToInterface(my_interface, n, k, 'Objects', d)

    # JobTree
    from Ganga.Core.GangaRepository import getRegistry
    jobtree = getRegistry("jobs").getJobTree()
    exportToInterface(my_interface, 'jobtree', jobtree, 'Objects',
                      'Logical tree view of the jobs')
    exportToInterface(my_interface, 'TreeError', TreeError, 'Exceptions')

    # ShareRef
    shareref = getRegistry("prep").getShareRef()
    exportToInterface(
        my_interface, 'shareref', shareref, 'Objects',
        'Mechanism for tracking use of shared directory resources')
Ejemplo n.º 6
0
def bootstrap(reg_slice, interactive_session, my_interface=None):
    """Create local subsystems.

    This function will change the default value of autostart of the monitoring, depending if the session is interactive
    or batch. The autostart value may be overridden in the config file, so warn if it differs from the default.

    Args:
        reg_slice (RegistrySlice): A registry slice encompassing the Registry to monitor,
            e.g. from getRegistrySlice('jobs') -> JobRegistry.getSlice()
        interactive_session (bool): Flag indicating an interactive session or not
        my_interface (Optional[module]): Public interface to add the runMonitoring function to, None (default) will set
            it to Ganga.GPI
    """
    # Must do some Ganga imports here to avoid circular importing
    from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import JobRegistry_Monitor
    from Ganga.Utility.Config import getConfig
    from Ganga.Runtime.GPIexport import exportToInterface
    from Ganga.Utility.logging import getLogger
    global monitoring_component

    # start the monitoring loop
    monitoring_component = JobRegistry_Monitor(reg_slice)
    monitoring_component.start()

    # override the default monitoring autostart value with the setting from interactive session
    config = getConfig("PollThread")
    config.overrideDefaultValue('autostart', interactive_session)

    # has the user changed monitoring autostart from the default? if so, warn them
    if config['autostart'] != interactive_session:
        if config['autostart']:
            getLogger().warning('Monitoring loop enabled (the default setting for a batch session is disabled)')
        else:
            getLogger().warning('Monitoring loop disabled (the default setting for an interactive session is enabled)')

    # Enable job monitoring if requested
    if config['autostart']:
        monitoring_component.enableMonitoring()

    # export the runMonitoring function to the public interface
    if not my_interface:
        import Ganga.GPI
        my_interface = Ganga.GPI

    exportToInterface(my_interface, 'runMonitoring', monitoring_component.runMonitoring, 'Functions')
Ejemplo n.º 7
0
def autoPopulateGPI(my_interface=None):
    """
    Fully expose all plugins registered with the interface in a single line.
    By default only populate GPI, but also populate any other interface requested
    """
    if not my_interface:
        import Ganga.GPI
        my_interface = Ganga.GPI
    from Ganga.Runtime.GPIexport import exportToInterface
    from Ganga.Utility.Plugin import allPlugins
    # make all plugins visible in GPI
    for k in allPlugins.allCategories():
        for n in allPlugins.allClasses(k):
            cls = allPlugins.find(k, n)
            if not cls._declared_property('hidden'):
                if n != cls.__name__:
                    exportToInterface(my_interface, cls.__name__, cls, 'Classes')
                exportToInterface(my_interface, n, cls, 'Classes')
Ejemplo n.º 8
0
def autoPopulateGPI(my_interface=None):
    """
    Fully expose all plugins registered with the interface in a single line.
    By default only populate GPI, but also populate any other interface requested
    """
    if not my_interface:
        import Ganga.GPI
        my_interface = Ganga.GPI
    from Ganga.Runtime.GPIexport import exportToInterface
    from Ganga.Utility.Plugin import allPlugins
    # make all plugins visible in GPI
    for k in allPlugins.allCategories():
        for n in allPlugins.allClasses(k):
            cls = allPlugins.find(k, n)
            if not cls._declared_property('hidden'):
                if n != cls.__name__:
                    exportToInterface(my_interface, cls.__name__, cls, 'Classes')
                exportToInterface(my_interface, n, cls, 'Classes')
Ejemplo n.º 9
0
def startUpQueues(my_interface=None):
    from Ganga.Utility.logging import getLogger
    logger = getLogger()
    global _global_queues
    global _queues_interface
    if not my_interface:
        import Ganga.GPI
        my_interface = Ganga.GPI
    _queues_additional = my_interface
    if _global_queues is None:
        logger.debug("Starting Queues")
        # start queues
        from Ganga.Runtime.GPIexport import exportToInterface
        from Ganga.Core.GangaThread.WorkerThreads.ThreadPoolQueueMonitor import ThreadPoolQueueMonitor
        _global_queues = ThreadPoolQueueMonitor()
        exportToInterface(my_interface, 'queues', _global_queues, 'Objects')

    else:
        logger.error("Cannot Start queues if they've already started")
Ejemplo n.º 10
0
            logger.warning("do not understand option %s in [Plugins]", opt)
            logger.debug('Reason: want %s' % str(err))
        else:
            if tag == 'default':
                try:
                    allPlugins.setDefault(category, default_plugins_cfg[opt])
                except Ganga.Utility.Plugin.PluginManagerError as x:
                    logger.warning('cannot set the default plugin "%s": %s' % (opt, x))
            else:
                logger.warning("do not understand option %s in [Plugins]", opt)


    # set alias for default Batch plugin (it will not appear in the
    # configuration)

    batch_default_name = getConfig('Configuration').getEffectiveOption('Batch')
    try:
        batch_default = allPlugins.find('backends', batch_default_name)
    except Exception as x:
        raise Ganga.Utility.Config.ConfigError('Check configuration. Unable to set default Batch backend alias (%s)' % str(x))
    else:
        allPlugins.add(batch_default, 'backends', 'Batch')
        from Ganga.Runtime.GPIexport import exportToInterface
        if not my_interface:
            import Ganga.GPI
            my_interface = Ganga.GPI
        exportToInterface(my_interface, 'Batch', batch_default, 'Classes')



Ejemplo n.º 11
0
            logger.warning("do not understand option %s in [Plugins]", opt)
            logger.debug('Reason: want %s' % str(err))
        else:
            if tag == 'default':
                try:
                    allPlugins.setDefault(category, default_plugins_cfg[opt])
                except Ganga.Utility.Plugin.PluginManagerError as x:
                    logger.warning('cannot set the default plugin "%s": %s' % (opt, x))
            else:
                logger.warning("do not understand option %s in [Plugins]", opt)


    # set alias for default Batch plugin (it will not appear in the
    # configuration)

    batch_default_name = getConfig('Configuration').getEffectiveOption('Batch')
    try:
        batch_default = allPlugins.find('backends', batch_default_name)
    except Exception as x:
        raise Ganga.Utility.Config.ConfigError('Check configuration. Unable to set default Batch backend alias (%s)' % str(x))
    else:
        allPlugins.add(batch_default, 'backends', 'Batch')
        from Ganga.Runtime.GPIexport import exportToInterface
        if not my_interface:
            import Ganga.GPI
            my_interface = Ganga.GPI
        exportToInterface(my_interface, 'Batch', batch_default, 'Classes')