Ejemplo n.º 1
0
    def __init__(self, suite, suite_dir):
        # Suite only needed for back-compat with old clients (see below):
        self.suite = suite

        # Figure out the ports we are allowed to use.
        base_port = GLOBAL_CFG.get(["communication", "base port"])
        max_ports = GLOBAL_CFG.get(["communication", "maximum number of ports"])
        self.ok_ports = range(int(base_port), int(base_port) + int(max_ports))
        random.shuffle(self.ok_ports)
        comms_options = GLOBAL_CFG.get(["communication", "options"])
        # HTTP Digest Auth uses MD5 - pretty secure in this use case.
        # Extending it with extra algorithms is allowed, but won't be
        # supported by most browsers. requests and urllib2 are OK though.
        self.hash_algorithm = "MD5"
        if "SHA1" in comms_options:
            # Note 'SHA' rather than 'SHA1'.
            self.hash_algorithm = "SHA"

        self.reg_db = RegistrationDB()
        try:
            self.cert = self.reg_db.load_item(suite, USER, None, "certificate", create_ok=True)
            self.pkey = self.reg_db.load_item(suite, USER, None, "private_key", create_ok=True)
        except PassphraseError:
            # No OpenSSL installed.
            self.cert = None
            self.pkey = None
        self.suite = suite
        passphrase = self.reg_db.load_passphrase(suite, USER, None)
        userpassdict = {"cylc": passphrase, "anon": NO_PASSPHRASE}
        get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(userpassdict, algorithm=self.hash_algorithm)
        self.get_ha1 = get_ha1
        del passphrase
        del userpassdict
        self.client_reporter = CommsClientReporter.get_inst()
        self.start()
Ejemplo n.º 2
0
def get_suite_host():
    from cylc.cfgspec.globalcfg import GLOBAL_CFG, GlobalConfigError
    global suite_host
    if suite_host is None:
        hardwired = GLOBAL_CFG.get(['suite host self-identification', 'host'])
        method = GLOBAL_CFG.get(['suite host self-identification', 'method'])
        # the following is for suite host self-identfication in task job
        # scripts:
        if method == 'name':
            suite_host = hostname
        elif method == 'address':
            suite_host = get_host_ip_address()
        elif method == 'hardwired':
            if not hardwired:
                raise GlobalConfigError(
                    'ERROR, no hardwired hostname is configured (%s)' %
                    ['suite host self-identification', 'host']
                )
            suite_host = hardwired
        else:
            raise GlobalConfigError(
                'ERROR, unknown host method (%s): %s' % (
                    ['suite host self-identification', 'method'], method)
            )
    return suite_host
Ejemplo n.º 3
0
    def __init__( self, suite ):

        sodir = GLOBAL_CFG.get_derived_host_item( suite, 'suite log directory' )
        self.opath = os.path.join( sodir, 'out' ) 
        self.epath = os.path.join( sodir, 'err' ) 

        # use same archive length as logging (TODO: document this)
        self.roll_at_startup = GLOBAL_CFG.get( ['suite logging','roll over at start-up'] )
        self.arclen = GLOBAL_CFG.get( ['suite logging','rolling archive length'] )
Ejemplo n.º 4
0
    def __init__( self, suite ):

        self.ldir = GLOBAL_CFG.get_derived_host_item( suite, 'suite log directory' )
        self.path = os.path.join( self.ldir, 'log' ) 

        self.err_path = os.path.join( self.ldir, 'err' )
        self.roll_at_startup = GLOBAL_CFG.get( ['suite logging','roll over at start-up'] )
        self.n_keep = GLOBAL_CFG.get( ['suite logging','rolling archive length'] )
        self.max_bytes = GLOBAL_CFG.get( ['suite logging','maximum size in bytes'] )
Ejemplo n.º 5
0
    def __init__(self, suite):

        sodir = GLOBAL_CFG.get_derived_host_item(suite, 'suite log directory')
        self.opath = os.path.join(sodir, 'out')
        self.epath = os.path.join(sodir, 'err')

        # use same archive length as logging (TODO: document this)
        self.roll_at_startup = GLOBAL_CFG.get(
            ['suite logging', 'roll over at start-up'])
        self.arclen = GLOBAL_CFG.get(
            ['suite logging', 'rolling archive length'])
Ejemplo n.º 6
0
def get_port(suite,
             owner=user,
             host=get_hostname(),
             pphrase=None,
             pyro_timeout=None):
    # Scan ports until a particular suite is found.

    pyro_base_port = GLOBAL_CFG.get(['pyro', 'base port'])
    pyro_port_range = GLOBAL_CFG.get(['pyro', 'maximum number of ports'])

    for port in range(pyro_base_port, pyro_base_port + pyro_port_range):
        uri = cylcid_uri(host, port)
        try:
            proxy = Pyro.core.getProxyForURI(uri)
        except Pyro.errors.URIError, x:
            # No such host?
            raise SuiteNotFoundError, x

        if pyro_timeout:  # convert from string
            pyro_timeout = float(pyro_timeout)

        proxy._setTimeout(pyro_timeout)
        proxy._setIdentification(pphrase)

        before = datetime.datetime.now()
        try:
            name, xowner = proxy.id()
        except Pyro.errors.TimeoutError:
            warn_timeout(host, port, pyro_timeout)
            pass
        except Pyro.errors.ConnectionDeniedError:
            #print >> sys.stderr, "Wrong suite or wrong passphrase at " + portid( host, port )
            pass
        except Pyro.errors.ProtocolError:
            #print >> sys.stderr, "No Suite Found at " + portid( host, port )
            pass
        except Pyro.errors.NamingError:
            #print >> sys.stderr, "Non-cylc pyro server found at " + portid( host, port )
            pass
        else:
            if flags.verbose:
                after = datetime.datetime.now()
                print "Pyro connection on port " + str(port) + " took: " + str(
                    after - before)
            if name == suite and xowner == owner:
                if flags.verbose:
                    print suite, owner, host, port
                # RESULT
                return port
            else:
                # ID'd some other suite.
                #print 'OTHER SUITE:', name, xowner, host, port
                pass
Ejemplo n.º 7
0
    def write_environment_1( self, BUFFER=None ):
        if not BUFFER:
            BUFFER = self.FILE

        BUFFER.write( "\n\n# CYLC SUITE ENVIRONMENT:" )

        # write the static suite variables
        for var, val in sorted(self.__class__.suite_env.items()):
            BUFFER.write( "\nexport " + var + "=" + str(val) )

        if str(self.__class__.suite_env.get('CYLC_UTC')) == 'True':
            BUFFER.write( "\nexport TZ=UTC" )

        BUFFER.write("\n")
        # override and write task-host-specific suite variables
        suite_work_dir = GLOBAL_CFG.get_derived_host_item( self.suite, 'suite work directory', self.host, self.owner )
        st_env = deepcopy( self.__class__.suite_task_env ) 
        st_env[ 'CYLC_SUITE_RUN_DIR'    ] = GLOBAL_CFG.get_derived_host_item( self.suite, 'suite run directory', self.host, self.owner )
        st_env[ 'CYLC_SUITE_WORK_DIR'   ] = suite_work_dir
        st_env[ 'CYLC_SUITE_SHARE_DIR'  ] = GLOBAL_CFG.get_derived_host_item( self.suite, 'suite share directory', self.host, self.owner )
        st_env[ 'CYLC_SUITE_SHARE_PATH' ] = '$CYLC_SUITE_SHARE_DIR' # DEPRECATED
        rsp = self.jobconfig['remote suite path']
        if rsp:
            st_env[ 'CYLC_SUITE_DEF_PATH' ] = rsp
        else:
            # replace home dir with '$HOME' for evaluation on the task host
            st_env[ 'CYLC_SUITE_DEF_PATH' ] = re.sub( os.environ['HOME'], '$HOME', st_env['CYLC_SUITE_DEF_PATH'] )
        for var, val in sorted(st_env.items()):
            BUFFER.write( "\nexport " + var + "=" + str(val) )

        task_work_dir  = os.path.join( suite_work_dir, self.jobconfig['work sub-directory'] )

        use_login_shell = GLOBAL_CFG.get_host_item( 'use login shell', self.host, self.owner )
        comms = GLOBAL_CFG.get_host_item( 'task communication method', self.host, self.owner )

        BUFFER.write( "\n\n# CYLC TASK ENVIRONMENT:" )
        BUFFER.write( "\nexport CYLC_TASK_COMMS_METHOD=" + comms )
        BUFFER.write( "\nexport CYLC_TASK_CYCLE_POINT=" + self.point_string )
        BUFFER.write( "\nexport CYLC_TASK_CYCLE_TIME=" + self.point_string )
        BUFFER.write( "\nexport CYLC_TASK_ID=" + self.task_id )
        BUFFER.write( "\nexport CYLC_TASK_IS_COLDSTART=" + str( self.jobconfig['is cold-start']) )
        BUFFER.write( "\nexport CYLC_TASK_LOG_ROOT=" + self.log_root )
        BUFFER.write( "\nexport CYLC_TASK_MSG_MAX_TRIES=" + str( GLOBAL_CFG.get( ['task messaging','maximum number of tries'])) )
        BUFFER.write( "\nexport CYLC_TASK_MSG_RETRY_INTVL=" + str( GLOBAL_CFG.get( ['task messaging','retry interval in seconds'])) )
        BUFFER.write( "\nexport CYLC_TASK_MSG_TIMEOUT=" + str( GLOBAL_CFG.get( ['task messaging','connection timeout in seconds'])) )
        BUFFER.write( "\nexport CYLC_TASK_NAME=" + self.task_name )
        BUFFER.write( '\nexport CYLC_TASK_NAMESPACE_HIERARCHY="' + ' '.join( self.jobconfig['namespace hierarchy']) + '"')
        BUFFER.write( "\nexport CYLC_TASK_SSH_LOGIN_SHELL=" + str(use_login_shell) )
        BUFFER.write( "\nexport CYLC_TASK_SUBMIT_NUMBER=" + str(self.jobconfig['absolute submit number']) )
        BUFFER.write( "\nexport CYLC_TASK_TRY_NUMBER=" + str(self.jobconfig['try number']) )
        BUFFER.write( "\nexport CYLC_TASK_WORK_DIR=" + task_work_dir )
        BUFFER.write( "\nexport CYLC_TASK_WORK_PATH=$CYLC_TASK_WORK_DIR") # DEPRECATED
Ejemplo n.º 8
0
    def __init__(self, suite):

        self.ldir = GLOBAL_CFG.get_derived_host_item(
            suite, 'suite log directory')
        self.path = os.path.join(self.ldir, 'log')

        self.err_path = os.path.join(self.ldir, 'err')
        self.roll_at_startup = GLOBAL_CFG.get(
            ['suite logging', 'roll over at start-up'])
        self.n_keep = GLOBAL_CFG.get(
            ['suite logging', 'rolling archive length'])
        self.max_bytes = GLOBAL_CFG.get(
            ['suite logging', 'maximum size in bytes'])
Ejemplo n.º 9
0
    def __init__(self, suite):
        # Suite only needed for back-compat with old clients (see below):
        self.suite = suite
        self.engine = None
        self.port = None

        # Figure out the ports we are allowed to use.
        base_port = GLOBAL_CFG.get(['communication', 'base port'])
        max_ports = GLOBAL_CFG.get(
            ['communication', 'maximum number of ports'])
        self.ok_ports = range(int(base_port), int(base_port) + int(max_ports))
        random.shuffle(self.ok_ports)

        comms_options = GLOBAL_CFG.get(['communication', 'options'])

        # HTTP Digest Auth uses MD5 - pretty secure in this use case.
        # Extending it with extra algorithms is allowed, but won't be
        # supported by most browsers. requests and urllib2 are OK though.
        self.hash_algorithm = "MD5"
        if "SHA1" in comms_options:
            # Note 'SHA' rather than 'SHA1'.
            self.hash_algorithm = "SHA"

        self.srv_files_mgr = SuiteSrvFilesManager()
        self.comms_method = GLOBAL_CFG.get(['communication', 'method'])
        self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(
            {
                'cylc':
                self.srv_files_mgr.get_auth_item(
                    self.srv_files_mgr.FILE_BASE_PASSPHRASE,
                    suite,
                    content=True),
                'anon':
                NO_PASSPHRASE
            },
            algorithm=self.hash_algorithm)
        if self.comms_method == 'http':
            self.cert = None
            self.pkey = None
        else:  # if self.comms_method in [None, 'https']:
            try:
                self.cert = self.srv_files_mgr.get_auth_item(
                    self.srv_files_mgr.FILE_BASE_SSL_CERT, suite)
                self.pkey = self.srv_files_mgr.get_auth_item(
                    self.srv_files_mgr.FILE_BASE_SSL_PEM, suite)
            except SuiteServiceFileError:
                ERR.error("no HTTPS/OpenSSL support. Aborting...")
                raise CylcError("No HTTPS support. "
                                "Configure user's global.rc to use HTTP.")
        self.start()
Ejemplo n.º 10
0
def get_port( suite, owner=user, host=get_hostname(), pphrase=None, pyro_timeout=None ):
    # Scan ports until a particular suite is found.

    pyro_base_port = GLOBAL_CFG.get( ['pyro','base port'] )
    pyro_port_range = GLOBAL_CFG.get( ['pyro','maximum number of ports'] )

    for port in range( pyro_base_port, pyro_base_port + pyro_port_range ):
        uri = cylcid_uri( host, port )
        try:
            proxy = Pyro.core.getProxyForURI(uri)
        except Pyro.errors.URIError, x:
            # No such host?
            raise SuiteNotFoundError, x

        if pyro_timeout: # convert from string
            pyro_timeout = float( pyro_timeout )

        proxy._setTimeout(pyro_timeout)
        proxy._setIdentification( pphrase )

        before = datetime.datetime.now()
        try:
            name, xowner = proxy.id()
        except Pyro.errors.TimeoutError:
            warn_timeout( host, port, pyro_timeout )
            pass
        except Pyro.errors.ConnectionDeniedError:
            #print >> sys.stderr, "Wrong suite or wrong passphrase at " + portid( host, port )
            pass
        except Pyro.errors.ProtocolError:
            #print >> sys.stderr, "No Suite Found at " + portid( host, port )
            pass
        except Pyro.errors.NamingError:
            #print >> sys.stderr, "Non-cylc pyro server found at " + portid( host, port )
            pass
        else:
            if flags.verbose:
                after = datetime.datetime.now()
                print "Pyro connection on port " +str(port) + " took: " + str( after - before )
            if name == suite and xowner == owner:
                if flags.verbose:
                    print suite, owner, host, port
                # RESULT
                return port
            else:
                # ID'd some other suite.
                #print 'OTHER SUITE:', name, xowner, host, port
                pass
Ejemplo n.º 11
0
 def __init__(self, suite, host, owner):
     self.suite = suite
     self.host = host
     self.owner = owner
     self.locn = None
     self.local_path = os.path.join(
         GLOBAL_CFG.get(['pyro', 'ports directory']), suite)
Ejemplo n.º 12
0
 def __init__(self, hosts=None, poll_interval=None, is_compact=False):
     # We can't use gobject.threads_init() for panel applets.
     warnings.filterwarnings('ignore', 'use the new', Warning)
     setup_icons()
     if not hosts:
         try:
             hosts = GLOBAL_CFG.get(["suite host scanning", "hosts"])
         except KeyError:
             hosts = ["localhost"]
     self.is_compact = is_compact
     self.hosts = hosts
     dot_hbox = gtk.HBox()
     dot_hbox.show()
     dot_eb = gtk.EventBox()
     dot_eb.show()
     dot_eb.add(dot_hbox)
     image = gtk.image_new_from_stock("gcylc", gtk.ICON_SIZE_MENU)
     image.show()
     image_eb = gtk.EventBox()
     image_eb.show()
     image_eb.connect("button-press-event", self._on_button_press_event)
     image_eb.add(image)
     self.top_hbox = gtk.HBox()
     self.top_hbox.pack_start(image_eb, expand=False, fill=False)
     self.top_hbox.pack_start(dot_eb, expand=False, fill=False, padding=2)
     self.top_hbox.show()
     self.updater = ScanPanelAppletUpdater(hosts, dot_hbox, image,
                                           self.is_compact,
                                           poll_interval=poll_interval)
     self.top_hbox.connect("destroy", self.stop)
     if gsfg.get(["activate on startup"]):
         self.updater.start()
Ejemplo n.º 13
0
 def unregister(self, exp):
     """Un-register a suite."""
     unregistered_set = set()
     skipped_set = set()
     ports_d = GLOBAL_CFG.get(['pyro', 'ports directory'])
     for name in sorted(self.list_all_suites()):
         if not re.match(exp + r'\Z', name):
             continue
         try:
             data = self.get_suite_data(name)
         except RegistrationError:
             continue
         if os.path.exists(os.path.join(ports_d, name)):
             skipped_set.add((name, data['path']))
             print >> sys.stderr, (
                 'SKIP UNREGISTER %s: port file exists' % (name))
             continue
         for base_name in ['passphrase', 'suite.rc.processed']:
             try:
                 os.unlink(os.path.join(data['path'], base_name))
             except OSError:
                 pass
         unregistered_set.add((name, data['path']))
         print 'UNREGISTER %s:%s' % (name, data['path'])
         os.unlink(os.path.join(self.dbpath, name))
     return unregistered_set, skipped_set
Ejemplo n.º 14
0
 def __init__(self, hosts=None, owner=None, poll_interval=None,
              is_compact=False):
     # We can't use gobject.threads_init() for panel applets.
     warnings.filterwarnings('ignore', 'use the new', Warning)
     setup_icons()
     if not hosts:
         try:
             hosts = GLOBAL_CFG.get(["suite host scanning", "hosts"])
         except KeyError:
             hosts = ["localhost"]
     self.is_compact = is_compact
     self.hosts = hosts
     if owner is None:
         owner = user
     self.owner = owner
     dot_hbox = gtk.HBox()
     dot_hbox.show()
     dot_eb = gtk.EventBox()
     dot_eb.show()
     dot_eb.add(dot_hbox)
     image = gtk.image_new_from_stock("gcylc", gtk.ICON_SIZE_MENU)
     image.show()
     image_eb = gtk.EventBox()
     image_eb.show()
     image_eb.connect("button-press-event", self._on_button_press_event)
     image_eb.add(image)
     self.top_hbox = gtk.HBox()
     self.top_hbox.pack_start(image_eb, expand=False, fill=False)
     self.top_hbox.pack_start(dot_eb, expand=False, fill=False, padding=2)
     self.top_hbox.show()
     self.updater = ScanPanelAppletUpdater(hosts, dot_hbox, image,
                                           self.is_compact,
                                           owner=owner,
                                           poll_interval=poll_interval)
     self.top_hbox.connect("destroy", self.stop)
Ejemplo n.º 15
0
def prompt(question, force=False, gui=False, no_force=False, no_abort=False):
    """Interactive Yes/No prompt for cylc CLI scripts.

    For convenience, on No we just exit rather than return.
    If force is True don't prompt, just return immediately.

    """
    if (force or GLOBAL_CFG.get(['disable interactive command prompts'])) and (
            not no_force):
        return True
    if gui:
        import gtk
        dialog = gtk.MessageDialog(
            None, gtk.DIALOG_DESTROY_WITH_PARENT,
            gtk.MESSAGE_QUESTION, gtk.BUTTONS_YES_NO,
            question
        )
        gui_response = dialog.run()
        response_no = (gui_response != gtk.RESPONSE_YES)
    else:
        cli_response = raw_input('%s (y/n)? ' % question)
        response_no = (cli_response not in ['y', 'Y'])
    if response_no:
        if no_abort:
            return False
        else:
            sys.exit(0)
    else:
        return True
Ejemplo n.º 16
0
def prompt(question, force=False, gui=False, no_force=False, no_abort=False):
    """Interactive Yes/No prompt for cylc CLI scripts.

    For convenience, on No we just exit rather than return.
    If force is True don't prompt, just return immediately.

    """
    if (force or GLOBAL_CFG.get(['disable interactive command prompts'])) and (
            not no_force):
        return True
    if gui:
        import gtk
        dialog = gtk.MessageDialog(
            None, gtk.DIALOG_DESTROY_WITH_PARENT,
            gtk.MESSAGE_QUESTION, gtk.BUTTONS_YES_NO,
            question
        )
        gui_response = dialog.run()
        response_no = (gui_response != gtk.RESPONSE_YES)
    else:
        cli_response = raw_input('%s (y/n)? ' % question)
        response_no = (cli_response not in ['y', 'Y'])
    if response_no:
        if no_abort:
            return False
        else:
            sys.exit(0)
    else:
        return True
Ejemplo n.º 17
0
def handle_proxies():
    """Unset proxies if the configuration matches this."""
    from cylc.cfgspec.globalcfg import GLOBAL_CFG
    if not GLOBAL_CFG.get(['communication', 'proxies on']):
        import os
        os.environ.pop("http_proxy", None)
        os.environ.pop("https_proxy", None)
Ejemplo n.º 18
0
def handle_proxies():
    """Unset proxies if the configuration matches this."""
    from cylc.cfgspec.globalcfg import GLOBAL_CFG
    if not GLOBAL_CFG.get(['communication', 'proxies on']):
        import os
        os.environ.pop("http_proxy", None)
        os.environ.pop("https_proxy", None)
Ejemplo n.º 19
0
 def _load_port_file(self):
     """Load port, host, etc from port file."""
     # GLOBAL_CFG is expensive to import, so only load on demand
     from cylc.cfgspec.globalcfg import GLOBAL_CFG
     port_file_path = os.path.join(
         GLOBAL_CFG.get(['communication', 'ports directory']), self.suite)
     out = ""
     if is_remote_host(self.host) or is_remote_user(self.owner):
         # Only load these modules on demand, as they may be expensive
         import shlex
         from subprocess import Popen, PIPE
         ssh_tmpl = str(GLOBAL_CFG.get_host_item(
             'remote shell template', self.host, self.owner))
         ssh_tmpl = ssh_tmpl.replace(' %s', '')
         user_at_host = ''
         if self.owner:
             user_at_host = self.owner + '@'
         if self.host:
             user_at_host += self.host
         else:
             user_at_host += 'localhost'
         r_port_file_path = port_file_path.replace(
             os.environ['HOME'], '$HOME')
         command = shlex.split(ssh_tmpl) + [
             user_at_host, 'cat', r_port_file_path]
         proc = Popen(command, stdout=PIPE, stderr=PIPE)
         out, err = proc.communicate()
         ret_code = proc.wait()
         if ret_code:
             if cylc.flags.debug:
                 print >> sys.stderr, {
                     "code": ret_code,
                     "command": command,
                     "stdout": out,
                     "stderr": err}
             if self.port is None:
                 raise PortFileError(
                     "Port file '%s:%s' not found - suite not running?." %
                     (user_at_host, r_port_file_path))
     else:
         try:
             out = open(port_file_path).read()
         except IOError:
             if self.port is None:
                 raise PortFileError(
                     "Port file '%s' not found - suite not running?." %
                     (port_file_path))
     lines = out.splitlines()
     if self.port is None:
         try:
             self.port = int(lines[0])
         except (IndexError, ValueError):
             raise PortFileError(
                 "ERROR, bad content in port file: %s" % port_file_path)
     if self.host is None:
         if len(lines) >= 2:
             self.host = lines[1].strip()
         else:
             self.host = get_hostname()
Ejemplo n.º 20
0
def prompt(reason, force=False):
    if force or GLOBAL_CFG.get(["disable interactive command prompts"]):
        return
    response = raw_input(reason + " (y/n)? ")
    if response == "y":
        return
    else:
        sys.exit(0)
Ejemplo n.º 21
0
def prompt(reason, force=False):
    if force or GLOBAL_CFG.get(['disable interactive command prompts']):
        return
    response = raw_input(reason + ' (y/n)? ')
    if response == 'y':
        return
    else:
        sys.exit(0)
Ejemplo n.º 22
0
def prompt( reason, force=False ):
    if force or GLOBAL_CFG.get( ['disable interactive command prompts'] ):
        return
    response = raw_input( reason + ' (y/n)? ' )
    if response == 'y':
        return
    else:
        sys.exit(0)
Ejemplo n.º 23
0
    def __init__(self, suite, host, owner):
        self.suite = suite
        self.host = host
        self.owner = owner
        self.locn = None

        self.local_path = os.path.join(
            GLOBAL_CFG.get(['pyro', 'ports directory']), suite)
Ejemplo n.º 24
0
    def __init__(self, suite):

        Pyro.config.PYRO_MULTITHREADED = 1
        # Use dns names instead of fixed ip addresses from /etc/hosts
        # (see the Userguide "Networking Issues" section).
        Pyro.config.PYRO_DNS_URI = True

        # Base Pyro socket number.
        Pyro.config.PYRO_PORT = GLOBAL_CFG.get(['pyro', 'base port'])
        # Max number of sockets starting at base.
        Pyro.config.PYRO_PORT_RANGE = GLOBAL_CFG.get(
            ['pyro', 'maximum number of ports'])

        Pyro.core.initServer()
        self.daemon = None
        # Suite only needed for back-compat with old clients (see below):
        self.suite = suite
Ejemplo n.º 25
0
 def __init__(self, pool_size=None):
     self.pool_size = (pool_size or GLOBAL_CFG.get(["process pool size"])
                       or multiprocessing.cpu_count())
     # (The Pool class defaults to cpu_count anyway, but does not
     # expose the result via its public interface).
     LOG.debug("Initializing process pool, size %d" % self.pool_size)
     self.pool = multiprocessing.Pool(processes=self.pool_size)
     self.results = {}
Ejemplo n.º 26
0
def get_host_ip_address():
    from cylc.cfgspec.globalcfg import GLOBAL_CFG
    global host_ip_address
    if host_ip_address is None:
        target = GLOBAL_CFG.get(['suite host self-identification', 'target'])
        # external IP address of the suite host:
        host_ip_address = get_local_ip_address(target)
    return host_ip_address
Ejemplo n.º 27
0
def get_host_ip_address():
    from cylc.cfgspec.globalcfg import GLOBAL_CFG
    global host_ip_address
    if host_ip_address is None:
        target = GLOBAL_CFG.get( ['suite host self-identification','target'] )
        # external IP address of the suite host:
        host_ip_address = get_local_ip_address( target )
    return host_ip_address
Ejemplo n.º 28
0
    def __init__(self, suite, test_params=None):
        if SuiteLog.__INSTANCE:
            raise Exception("Attempting to initiate a second singleton"
                            "instance.")

        self._group = None
        if not test_params:
            self.is_test = False
            self.max_bytes = GLOBAL_CFG.get(
                ['suite logging', 'maximum size in bytes'])
            self.roll_at_startup = GLOBAL_CFG.get(
                ['suite logging', 'roll over at start-up'])
            self.archive_length = GLOBAL_CFG.get(
                ['suite logging', 'rolling archive length'])
        else:
            self.is_test = True
            self.max_bytes = test_params['max_bytes']
            self.roll_at_startup = test_params['roll_at_startup']
            self.archive_length = 4

        # Log paths.
        if test_params:
            self.ldir = test_params['ldir']
        else:
            self.ldir = GLOBAL_CFG.get_derived_host_item(
                suite, 'suite log directory')
        self.log_paths = {}
        self.log_paths[self.LOG] = os.path.join(self.ldir, self.LOG)
        self.log_paths[self.OUT] = os.path.join(self.ldir, self.OUT)
        self.log_paths[self.ERR] = os.path.join(self.ldir, self.ERR)

        # The loggers.
        self.loggers = {}
        self.loggers[self.LOG] = None
        self.loggers[self.OUT] = None
        self.loggers[self.ERR] = None

        # Filename stamp functions.
        if self.is_test:
            self.stamp = lambda: get_current_time_string(True, True, True
                                                         ).replace('.', '-')
        else:
            self.stamp = lambda: get_current_time_string(False, True, True)

        SuiteLog.__INSTANCE = self
Ejemplo n.º 29
0
    def __init__(self, suite, test_params=None):
        if SuiteLog.__INSTANCE:
            raise Exception("Attempting to initiate a second singleton"
                            "instance.")

        self._group = None
        if not test_params:
            self.is_test = False
            self.max_bytes = GLOBAL_CFG.get(
                ['suite logging', 'maximum size in bytes'])
            self.roll_at_startup = GLOBAL_CFG.get(
                ['suite logging', 'roll over at start-up'])
            self.archive_length = GLOBAL_CFG.get(
                ['suite logging', 'rolling archive length'])
        else:
            self.is_test = True
            self.max_bytes = test_params['max_bytes']
            self.roll_at_startup = test_params['roll_at_startup']
            self.archive_length = 4

        # Log paths.
        if test_params:
            self.ldir = test_params['ldir']
        else:
            self.ldir = GLOBAL_CFG.get_derived_host_item(
                suite, 'suite log directory')
        self.log_paths = {}
        self.log_paths[self.LOG] = os.path.join(self.ldir, self.LOG)
        self.log_paths[self.OUT] = os.path.join(self.ldir, self.OUT)
        self.log_paths[self.ERR] = os.path.join(self.ldir, self.ERR)

        # The loggers.
        self.loggers = {}
        self.loggers[self.LOG] = None
        self.loggers[self.OUT] = None
        self.loggers[self.ERR] = None

        # Filename stamp functions.
        if self.is_test:
            self.stamp = lambda: get_current_time_string(True, True, True
                                                         ).replace('.', '-')
        else:
            self.stamp = lambda: get_current_time_string(False, True, True)

        SuiteLog.__INSTANCE = self
Ejemplo n.º 30
0
def get_suite_host():
    from cylc.cfgspec.globalcfg import GLOBAL_CFG
    global suite_host
    if suite_host is None:
        hardwired = GLOBAL_CFG.get( ['suite host self-identification','host'] )
        method = GLOBAL_CFG.get( ['suite host self-identification','method'] )
        # the following is for suite host self-identfication in task job scripts:
        if method == 'name':
            suite_host = hostname
        elif method == 'address':
            suite_host = host_ip_address
        elif method == 'hardwired':
            if not hardwired:
                sys.exit( 'ERROR, no hardwired hostname is configured' )
            suite_host = hardwired
        else:
            sys.exit( 'ERROR, unknown host method: ' + method )
    return suite_host
Ejemplo n.º 31
0
def get_suite_host():
    from cylc.cfgspec.globalcfg import GLOBAL_CFG
    global suite_host
    if suite_host is None:
        hardwired = GLOBAL_CFG.get( ['suite host self-identification','host'] )
        method = GLOBAL_CFG.get( ['suite host self-identification','method'] )
        # the following is for suite host self-identfication in task job scripts:
        if method == 'name':
            suite_host = hostname
        elif method == 'address':
            suite_host = get_host_ip_address()
        elif method == 'hardwired':
            if not hardwired:
                sys.exit( 'ERROR, no hardwired hostname is configured' )
            suite_host = hardwired
        else:
            sys.exit( 'ERROR, unknown host method: ' + method )
    return suite_host
Ejemplo n.º 32
0
 def __init__(self, suite, port):
     self.suite = suite
     # The ports directory is assumed to exist.
     pdir = GLOBAL_CFG.get(['pyro', 'ports directory'])
     self.local_path = os.path.join(pdir, suite)
     try:
         self.port = str(int(port))
     except ValueError, x:
         print >> sys.stderr, x
         raise PortFileError("ERROR, illegal port number: %s" % port)
    def remote_host_select(self, host_str):
        """Evaluate a task host string.

        Arguments:
            host_str (str):
                An explicit host name, a command in back-tick or $(command)
                format, or an environment variable holding a hostname.

        Return (str):
            None if evaluate of host_str is still taking place.
            'localhost' if host_str is not defined or if the evaluated host
            name is equivalent to 'localhost'.
            Otherwise, return the evaluated host name on success.

        Raise TaskRemoteMgmtError on error.

        """
        if not host_str:
            return 'localhost'

        # Host selection command: $(command) or `command`
        match = REC_COMMAND.match(host_str)
        if match:
            cmd_str = match.groups()[1]
            if cmd_str in self.remote_host_str_map:
                # Command recently launched
                value = self.remote_host_str_map[cmd_str]
                if isinstance(value, TaskRemoteMgmtError):
                    raise value  # command failed
                elif value is None:
                    return  # command not yet ready
                else:
                    host_str = value  # command succeeded
            else:
                # Command not launched (or already reset)
                timeout = GLOBAL_CFG.get(['task host select command timeout'])
                if timeout:
                    cmd = ['timeout', str(int(timeout)), 'bash', '-c', cmd_str]
                else:
                    cmd = ['bash', '-c', cmd_str]
                self.proc_pool.put_command(
                    SuiteProcContext('remote-host-select',
                                     cmd,
                                     env=dict(os.environ)),
                    self._remote_host_select_callback, [cmd_str])
                self.remote_host_str_map[cmd_str] = None
                return self.remote_host_str_map[cmd_str]

        # Environment variable substitution
        host_str = os.path.expandvars(host_str)
        # Remote?
        if is_remote_host(host_str):
            return host_str
        else:
            return 'localhost'
Ejemplo n.º 34
0
    def __init__(self, suite, suite_dir):
        # Suite only needed for back-compat with old clients (see below):
        self.suite = suite

        Pyro.config.PYRO_MULTITHREADED = 1
        # Use dns names instead of fixed ip addresses from /etc/hosts
        # (see the Userguide "Networking Issues" section).
        Pyro.config.PYRO_DNS_URI = True

        # Base Pyro socket number.
        Pyro.config.PYRO_PORT = GLOBAL_CFG.get(['pyro', 'base port'])
        # Max number of sockets starting at base.
        Pyro.config.PYRO_PORT_RANGE = GLOBAL_CFG.get(
            ['pyro', 'maximum number of ports'])

        Pyro.core.initServer()
        self.daemon = Pyro.core.Daemon()
        cval = ConnValidator()
        self.daemon.setNewConnectionValidator(cval)
        cval.set_pphrase(RegistrationDB.load_passphrase_from_dir(suite_dir))
Ejemplo n.º 35
0
    def __init__(self, suite, suite_dir):
        # Suite only needed for back-compat with old clients (see below):
        self.suite = suite

        Pyro.config.PYRO_MULTITHREADED = 1
        # Use dns names instead of fixed ip addresses from /etc/hosts
        # (see the Userguide "Networking Issues" section).
        Pyro.config.PYRO_DNS_URI = True

        # Base Pyro socket number.
        Pyro.config.PYRO_PORT = GLOBAL_CFG.get(['pyro', 'base port'])
        # Max number of sockets starting at base.
        Pyro.config.PYRO_PORT_RANGE = GLOBAL_CFG.get(
            ['pyro', 'maximum number of ports'])

        Pyro.core.initServer()
        self.daemon = Pyro.core.Daemon()
        cval = ConnValidator()
        self.daemon.setNewConnectionValidator(cval)
        cval.set_pphrase(RegistrationDB.load_passphrase_from_dir(suite_dir))
Ejemplo n.º 36
0
 def __init__(self, pool_size=None):
     self.pool_size = (
         pool_size or
         GLOBAL_CFG.get(["process pool size"]) or
         multiprocessing.cpu_count())
     # (The Pool class defaults to cpu_count anyway, but does not
     # expose the result via its public interface).
     LOG.debug(
         "Initializing process pool, size %d" % self.pool_size)
     self.pool = multiprocessing.Pool(processes=self.pool_size)
     self.results = {}
Ejemplo n.º 37
0
 def get_events_conf(config, key, default=None):
     """Return a named [cylc][[events]] configuration."""
     for getter in [
             config.cfg['cylc']['events'],
             GLOBAL_CFG.get(['cylc', 'events'])]:
         try:
             value = getter[key]
         except KeyError:
             pass
         else:
             if value is not None:
                 return value
     return default
Ejemplo n.º 38
0
    def __init__(self, suite):
        # Suite only needed for back-compat with old clients (see below):
        self.suite = suite

        # Figure out the ports we are allowed to use.
        base_port = GLOBAL_CFG.get(['communication', 'base port'])
        max_ports = GLOBAL_CFG.get(
            ['communication', 'maximum number of ports'])
        self.ok_ports = range(int(base_port), int(base_port) + int(max_ports))
        random.shuffle(self.ok_ports)

        comms_options = GLOBAL_CFG.get(['communication', 'options'])
        # HTTP Digest Auth uses MD5 - pretty secure in this use case.
        # Extending it with extra algorithms is allowed, but won't be
        # supported by most browsers. requests and urllib2 are OK though.
        self.hash_algorithm = "MD5"
        if "SHA1" in comms_options:
            # Note 'SHA' rather than 'SHA1'.
            self.hash_algorithm = "SHA"

        self.srv_files_mgr = SuiteSrvFilesManager()
        self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(
            {
                'cylc': self.srv_files_mgr.get_auth_item(
                    self.srv_files_mgr.FILE_BASE_PASSPHRASE,
                    suite, content=True),
                'anon': NO_PASSPHRASE
            },
            algorithm=self.hash_algorithm)
        try:
            self.cert = self.srv_files_mgr.get_auth_item(
                self.srv_files_mgr.FILE_BASE_SSL_CERT, suite)
            self.pkey = self.srv_files_mgr.get_auth_item(
                self.srv_files_mgr.FILE_BASE_SSL_PEM, suite)
        except SuiteServiceFileError:
            self.cert = None
            self.pkey = None
        self.client_reporter = CommsClientReporter.get_inst()
        self.start()
Ejemplo n.º 39
0
 def __init__(self, suite, run_mode='live', ict=None, stop_point=None):
     self.run_mode = run_mode
     self.set_cts(ict, stop_point)
     self.dir_name = GLOBAL_CFG.get_derived_host_item(
         suite, 'suite state directory')
     self.file_name = os.path.join(self.dir_name, self.BASE_NAME)
     self.arch_len = GLOBAL_CFG.get(['state dump rolling archive length'])
     if not self.arch_len or int(self.arch_len) <= 1:
         self.arch_len = 1
     self.arch_files = []
     self.pool = None
     self.wireless = None
     self.log = logging.getLogger('main')
Ejemplo n.º 40
0
 def get_events_conf(config, key, default=None):
     """Return a named [cylc][[events]] configuration."""
     for getter in [
             config.cfg['cylc']['events'],
             GLOBAL_CFG.get(['cylc', 'events'])]:
         try:
             value = getter[key]
         except KeyError:
             pass
         else:
             if value is not None:
                 return value
     return default
Ejemplo n.º 41
0
 def __init__(self, suite, run_mode='live', ict=None, stop_point=None):
     self.run_mode = run_mode
     self.cts_str = None
     self.set_cts(ict, stop_point)
     self.dir_name = GLOBAL_CFG.get_derived_host_item(
         suite, 'suite state directory')
     self.file_name = os.path.join(self.dir_name, self.BASE_NAME)
     self.arch_len = GLOBAL_CFG.get(['state dump rolling archive length'])
     if not self.arch_len or int(self.arch_len) <= 1:
         self.arch_len = 1
     self.arch_files = []
     self.pool = None
     self.log = logging.getLogger('main')
Ejemplo n.º 42
0
def scan_all(hosts=None, reg_db_path=None, timeout=None):
    """Scan all hosts."""
    if not hosts:
        hosts = GLOBAL_CFG.get(["suite host scanning", "hosts"])
    # Ensure that it does "localhost" only once
    hosts = set(hosts)
    for host in list(hosts):
        if not is_remote_host(host):
            hosts.remove(host)
            hosts.add("localhost")
    proc_pool_size = GLOBAL_CFG.get(["process pool size"])
    if proc_pool_size is None:
        proc_pool_size = cpu_count()
    if proc_pool_size > len(hosts):
        proc_pool_size = len(hosts)
    proc_pool = Pool(proc_pool_size)
    async_results = {}
    for host in hosts:
        async_results[host] = proc_pool.apply_async(
            scan, [host, reg_db_path, timeout])
    proc_pool.close()
    scan_results = []
    scan_results_hosts = []
    while async_results:
        sleep(0.05)
        for host, async_result in async_results.items():
            if async_result.ready():
                async_results.pop(host)
                try:
                    res = async_result.get()
                except Exception:
                    if cylc.flags.debug:
                        traceback.print_exc()
                else:
                    scan_results.extend(res)
                    scan_results_hosts.extend([host] * len(res))
    proc_pool.join()
    return zip(scan_results_hosts, scan_results)
Ejemplo n.º 43
0
def scan_all(hosts=None, reg_db_path=None, pyro_timeout=None):
    """Scan all hosts."""
    if not hosts:
        hosts = GLOBAL_CFG.get(["suite host scanning", "hosts"])
    # Ensure that it does "localhost" only once
    hosts = set(hosts)
    for host in list(hosts):
        if not is_remote_host(host):
            hosts.remove(host)
            hosts.add("localhost")
    proc_pool_size = GLOBAL_CFG.get(["process pool size"])
    if proc_pool_size is None:
        proc_pool_size = cpu_count()
    if proc_pool_size > len(hosts):
        proc_pool_size = len(hosts)
    proc_pool = Pool(proc_pool_size)
    async_results = {}
    for host in hosts:
        async_results[host] = proc_pool.apply_async(
            scan, [host, reg_db_path, pyro_timeout])
    proc_pool.close()
    scan_results = []
    scan_results_hosts = []
    while async_results:
        sleep(0.05)
        for host, async_result in async_results.items():
            if async_result.ready():
                async_results.pop(host)
                try:
                    res = async_result.get()
                except Exception:
                    if cylc.flags.debug:
                        traceback.print_exc()
                else:
                    scan_results.extend(res)
                    scan_results_hosts.extend([host] * len(res))
    proc_pool.join()
    return zip(scan_results_hosts, scan_results)
Ejemplo n.º 44
0
 def _get_events_conf(self, itask, key, default=None):
     """Return an events setting from suite then global configuration."""
     for getter in [
             self.broadcast_mgr.get_broadcast(itask.identity).get("events"),
             itask.tdef.rtconfig["events"],
             GLOBAL_CFG.get()["task events"]]:
         try:
             value = getter.get(key)
         except (AttributeError, ItemNotFoundError, KeyError):
             pass
         else:
             if value is not None:
                 return value
     return default
Ejemplo n.º 45
0
    def __init__(self, suite, port):
        self.suite = suite

        # the ports directory is assumed to exist

        pdir = GLOBAL_CFG.get(['pyro', 'ports directory'])

        self.local_path = os.path.join(pdir, suite)

        try:
            self.port = str(int(port))
        except ValueError, x:
            print >> sys.stderr, x
            raise PortFileError("ERROR, illegal port number: " + str(port))
Ejemplo n.º 46
0
def scan(host=get_hostname(), db=None, pyro_timeout=None):
    #print 'SCANNING PORTS'
    # scan all cylc Pyro ports for cylc suites

    pyro_base_port = GLOBAL_CFG.get( ['pyro','base port'] )
    pyro_port_range = GLOBAL_CFG.get( ['pyro','maximum number of ports'] )

    # In non-verbose mode print nothing (scan is used by cylc db viewer).

    # load my suite passphrases
    reg = localdb(db)
    reg_suites = reg.get_list()
    my_passphrases = {}
    for item in reg_suites:
        rg = item[0]
        di = item[1]
        try:
            pp = passphrase( rg, user, host ).get( suitedir=di )
        except Exception, x:
            #print >> sys.stderr, x
            # no passphrase defined for this suite
            pass
        else:
            my_passphrases[ rg ] = pp
Ejemplo n.º 47
0
def get_suite_host():
    from cylc.cfgspec.globalcfg import GLOBAL_CFG, GlobalConfigError

    global suite_host
    if suite_host is None:
        hardwired = GLOBAL_CFG.get(["suite host self-identification", "host"])
        method = GLOBAL_CFG.get(["suite host self-identification", "method"])
        # the following is for suite host self-identfication in task job
        # scripts:
        if method == "name":
            suite_host = hostname
        elif method == "address":
            suite_host = get_host_ip_address()
        elif method == "hardwired":
            if not hardwired:
                raise GlobalConfigError(
                    "ERROR, no hardwired hostname is configured (%s)" % ["suite host self-identification", "host"]
                )
            suite_host = hardwired
        else:
            raise GlobalConfigError(
                "ERROR, unknown host method (%s): %s" % (["suite host self-identification", "method"], method)
            )
    return suite_host
Ejemplo n.º 48
0
def scan(host=get_hostname(), db=None, pyro_timeout=None, silent=False):
    #print 'SCANNING PORTS'
    # scan all cylc Pyro ports for cylc suites

    pyro_base_port = GLOBAL_CFG.get(['pyro', 'base port'])
    pyro_port_range = GLOBAL_CFG.get(['pyro', 'maximum number of ports'])

    # In non-verbose mode print nothing (scan is used by cylc db viewer).

    # load my suite passphrases
    reg = localdb(db)
    reg_suites = reg.get_list()
    my_passphrases = {}
    for item in reg_suites:
        rg = item[0]
        di = item[1]
        try:
            pp = passphrase(rg, user, host).get(suitedir=di)
        except Exception, x:
            #print >> sys.stderr, x
            # no passphrase defined for this suite
            pass
        else:
            my_passphrases[rg] = pp
Ejemplo n.º 49
0
 def _call_server_get_url(self, function, **kwargs):
     """Build request URL."""
     comms_protocol = self.comms_protocol
     if comms_protocol is None:
         # Use standard setting from global configuration
         from cylc.cfgspec.globalcfg import GLOBAL_CFG
         comms_protocol = GLOBAL_CFG.get(['communication', 'method'])
     url = '%s://%s:%s/%s' % (
         comms_protocol, self.host, self.port, function)
     # If there are any parameters left in the dict after popping,
     # append them to the url.
     if kwargs:
         import urllib
         params = urllib.urlencode(kwargs, doseq=True)
         url += "?" + params
     return url
Ejemplo n.º 50
0
def get_task_host(cfg_item):
    """Evaluate a task host string.

    E.g.:

    [runtime]
        [[NAME]]
            [[[remote]]]
                host = cfg_item

    cfg_item -- An explicit host name, a command in back-tick or $(command)
                format, or an environment variable holding a hostname.

    Return "localhost" if cfg_item is not defined or if the evaluated host name
    is equivalent to "localhost". Otherwise, return the evaluated host name on
    success.

    """

    host = cfg_item

    if not host:
        return "localhost"

    # 1) host selection command: $(command) or `command`
    match = REC_COMMAND.match(host)
    if match:
        # extract the command and execute it
        hs_command = match.groups()[1]
        timeout = GLOBAL_CFG.get(["task host select command timeout"])
        is_ok, outlines = run_get_stdout(hs_command, timeout)
        if is_ok:
            # host selection command succeeded
            host = outlines[0]
        else:
            # host selection command failed
            raise HostSelectError(host, "\n".join(outlines))

    # 2) environment variable: ${VAR} or $VAR
    # (any quotes are stripped by file parsing)
    match = REC_ENVIRON.match(host)
    if match:
        name = match.groups()[0]
        try:
            host = os.environ[name]
        except KeyError, exc:
            raise HostSelectError(host, "Variable not defined: " + str(exc))
Ejemplo n.º 51
0
def get_task_host(cfg_item):
    """Evaluate a task host string.

    E.g.:

    [runtime]
        [[NAME]]
            [[[remote]]]
                host = cfg_item

    cfg_item -- An explicit host name, a command in back-tick or $(command)
                format, or an environment variable holding a hostname.

    Return "localhost" if cfg_item is not defined or if the evaluated host name
    is equivalent to "localhost". Otherwise, return the evaluated host name on
    success.

    """

    host = cfg_item

    if not host:
        return "localhost"

    # 1) host selection command: $(command) or `command`
    match = REC_COMMAND.match(host)
    if match:
        # extract the command and execute it
        hs_command = match.groups()[1]
        timeout = GLOBAL_CFG.get(["task host select command timeout"])
        is_ok, outlines = run_get_stdout(hs_command, timeout)
        if is_ok:
            # host selection command succeeded
            host = outlines[0]
        else:
            # host selection command failed
            raise HostSelectError(host, "\n".join(outlines))

    # 2) environment variable: ${VAR} or $VAR
    # (any quotes are stripped by file parsing)
    match = REC_ENVIRON.match(host)
    if match:
        name = match.groups()[0]
        try:
            host = os.environ[name]
        except KeyError, exc:
            raise HostSelectError(host, "Variable not defined: " + str(exc))
Ejemplo n.º 52
0
 def _compile_request(self, func_dict, host, comms_protocol=None):
     """Build request URL."""
     payload = func_dict.pop("payload", None)
     method = func_dict.pop("method", self.METHOD)
     function = func_dict.pop("function", None)
     if comms_protocol is None:
         # Use standard setting from global configuration
         from cylc.cfgspec.globalcfg import GLOBAL_CFG
         comms_protocol = GLOBAL_CFG.get(['communication', 'method'])
     url = '%s://%s:%s/%s' % (comms_protocol, host, self.port, function)
     # If there are any parameters left in the dict after popping,
     # append them to the url.
     if func_dict:
         import urllib
         params = urllib.urlencode(func_dict, doseq=True)
         url += "?" + params
     return (method, url, payload, comms_protocol)
Ejemplo n.º 53
0
    def _write_environment_1(self, handle, job_conf):
        """Suite and task environment."""
        handle.write("\n\n# CYLC SUITE ENVIRONMENT:")

        # write the static suite variables
        for var, val in sorted(self.suite_env.items()):
            handle.write("\nexport " + var + "=" + str(val))

        if str(self.suite_env.get('CYLC_UTC')) == 'True':
            handle.write("\nexport TZ=UTC")

        handle.write("\n")
        # override and write task-host-specific suite variables
        suite_work_dir = GLOBAL_CFG.get_derived_host_item(
            job_conf['suite name'], 'suite work directory',
            job_conf['host'], job_conf['owner'])
        st_env = {}
        st_env['CYLC_SUITE_RUN_DIR'] = GLOBAL_CFG.get_derived_host_item(
            job_conf['suite name'], 'suite run directory',
            job_conf['host'], job_conf['owner'])
        st_env['CYLC_SUITE_WORK_DIR'] = suite_work_dir
        st_env['CYLC_SUITE_SHARE_DIR'] = GLOBAL_CFG.get_derived_host_item(
            job_conf['suite name'], 'suite share directory',
            job_conf['host'], job_conf['owner'])
        # DEPRECATED
        st_env['CYLC_SUITE_SHARE_PATH'] = '$CYLC_SUITE_SHARE_DIR'
        rsp = job_conf['remote suite path']
        if rsp:
            st_env['CYLC_SUITE_DEF_PATH'] = rsp
        else:
            # replace home dir with '$HOME' for evaluation on the task host
            st_env['CYLC_SUITE_DEF_PATH'] = re.sub(
                os.environ['HOME'], '$HOME',
                self.suite_env['CYLC_SUITE_DEF_PATH_ON_SUITE_HOST'])
        for var, val in sorted(st_env.items()):
            handle.write("\nexport " + var + "=" + str(val))

        task_work_dir = os.path.join(
            suite_work_dir, job_conf['work sub-directory'])

        use_login_shell = GLOBAL_CFG.get_host_item(
            'use login shell', job_conf['host'], job_conf['owner'])
        comms = GLOBAL_CFG.get_host_item(
            'task communication method', job_conf['host'], job_conf['owner'])

        task_name, point_string = TaskID.split(job_conf['task id'])
        handle.write("\n\n# CYLC TASK ENVIRONMENT:")
        handle.write("\nexport CYLC_TASK_COMMS_METHOD=" + comms)
        handle.write("\nexport CYLC_TASK_CYCLE_POINT=" + point_string)
        handle.write("\nexport CYLC_TASK_CYCLE_TIME=" + point_string)
        handle.write("\nexport CYLC_TASK_ID=" + job_conf['task id'])
        handle.write(
            "\nexport CYLC_TASK_IS_COLDSTART=" +
            str(job_conf['is cold-start']))
        handle.write(
            "\nexport CYLC_TASK_LOG_ROOT=" + job_conf['job file path'])
        handle.write(
            "\nexport CYLC_TASK_MSG_MAX_TRIES=" +
            str(GLOBAL_CFG.get(['task messaging', 'maximum number of tries'])))
        handle.write(
            "\nexport CYLC_TASK_MSG_RETRY_INTVL=%f" %
            GLOBAL_CFG.get(['task messaging', 'retry interval']))
        handle.write(
            "\nexport CYLC_TASK_MSG_TIMEOUT=%f" %
            GLOBAL_CFG.get(['task messaging', 'connection timeout']))
        handle.write("\nexport CYLC_TASK_NAME=" + task_name)
        handle.write(
            '\nexport CYLC_TASK_NAMESPACE_HIERARCHY="' +
            ' '.join(job_conf['namespace hierarchy']) + '"')
        handle.write(
            "\nexport CYLC_TASK_SSH_LOGIN_SHELL=" + str(use_login_shell))
        handle.write(
            "\nexport CYLC_TASK_SUBMIT_NUMBER=" + str(job_conf['submit num']))
        handle.write(
            "\nexport CYLC_TASK_TRY_NUMBER=" +
            str(job_conf['try number']))
        handle.write("\nexport CYLC_TASK_WORK_DIR=" + task_work_dir)
        # DEPRECATED
        handle.write("\nexport CYLC_TASK_WORK_PATH=$CYLC_TASK_WORK_DIR")
        handle.write("\nexport %s=$$" % (TaskMessage.CYLC_JOB_PID))
Ejemplo n.º 54
0
    def _write_environment_1(self, handle, job_conf):
        """Suite and task environment."""
        handle.write("\n\n# CYLC SUITE ENVIRONMENT:")

        # write the static suite variables
        for var, val in sorted(self.suite_env.items()):
            handle.write("\nexport " + var + "=" + str(val))

        if str(self.suite_env.get("CYLC_UTC")) == "True":
            handle.write("\nexport TZ=UTC")

        handle.write("\n")
        # override and write task-host-specific suite variables
        suite_work_dir = GLOBAL_CFG.get_derived_host_item(
            job_conf["suite name"], "suite work directory", job_conf["host"], job_conf["owner"]
        )
        st_env = {}
        st_env["CYLC_SUITE_RUN_DIR"] = GLOBAL_CFG.get_derived_host_item(
            job_conf["suite name"], "suite run directory", job_conf["host"], job_conf["owner"]
        )
        st_env["CYLC_SUITE_WORK_DIR"] = suite_work_dir
        st_env["CYLC_SUITE_SHARE_DIR"] = GLOBAL_CFG.get_derived_host_item(
            job_conf["suite name"], "suite share directory", job_conf["host"], job_conf["owner"]
        )
        # DEPRECATED
        st_env["CYLC_SUITE_SHARE_PATH"] = "$CYLC_SUITE_SHARE_DIR"
        rsp = job_conf["remote suite path"]
        if rsp:
            st_env["CYLC_SUITE_DEF_PATH"] = rsp
        else:
            # replace home dir with '$HOME' for evaluation on the task host
            st_env["CYLC_SUITE_DEF_PATH"] = re.sub(
                os.environ["HOME"], "$HOME", self.suite_env["CYLC_SUITE_DEF_PATH_ON_SUITE_HOST"]
            )
        for var, val in sorted(st_env.items()):
            handle.write("\nexport " + var + "=" + str(val))

        task_work_dir = os.path.join(suite_work_dir, job_conf["work sub-directory"])

        use_login_shell = GLOBAL_CFG.get_host_item("use login shell", job_conf["host"], job_conf["owner"])
        comms = GLOBAL_CFG.get_host_item("task communication method", job_conf["host"], job_conf["owner"])

        task_name, point_string = TaskID.split(job_conf["task id"])
        handle.write("\n\n# CYLC TASK ENVIRONMENT:")
        handle.write("\nexport CYLC_TASK_COMMS_METHOD=" + comms)
        handle.write("\nexport CYLC_TASK_CYCLE_POINT=" + point_string)
        handle.write("\nexport CYLC_TASK_CYCLE_TIME=" + point_string)
        handle.write("\nexport CYLC_TASK_ID=" + job_conf["task id"])
        handle.write("\nexport CYLC_TASK_IS_COLDSTART=" + str(job_conf["is cold-start"]))
        handle.write("\nexport CYLC_TASK_LOG_ROOT=" + job_conf["job file path"])
        handle.write(
            "\nexport CYLC_TASK_MSG_MAX_TRIES=" + str(GLOBAL_CFG.get(["task messaging", "maximum number of tries"]))
        )
        handle.write("\nexport CYLC_TASK_MSG_RETRY_INTVL=" + str(GLOBAL_CFG.get(["task messaging", "retry interval"])))
        handle.write("\nexport CYLC_TASK_MSG_TIMEOUT=" + str(GLOBAL_CFG.get(["task messaging", "connection timeout"])))
        handle.write("\nexport CYLC_TASK_NAME=" + task_name)
        handle.write('\nexport CYLC_TASK_NAMESPACE_HIERARCHY="' + " ".join(job_conf["namespace hierarchy"]) + '"')
        handle.write("\nexport CYLC_TASK_SSH_LOGIN_SHELL=" + str(use_login_shell))
        handle.write("\nexport CYLC_TASK_SUBMIT_NUMBER=" + str(job_conf["absolute submit number"]))
        handle.write("\nexport CYLC_TASK_TRY_NUMBER=" + str(job_conf["try number"]))
        handle.write("\nexport CYLC_TASK_WORK_DIR=" + task_work_dir)
        # DEPRECATED
        handle.write("\nexport CYLC_TASK_WORK_PATH=$CYLC_TASK_WORK_DIR")
        handle.write("\nexport CYLC_JOB_PID=$$")
Ejemplo n.º 55
0
coercers["interval"] = coerce_interval
coercers["interval_minutes"] = lambda *a: coerce_interval(*a, back_comp_unit_factor=60)
coercers["interval_seconds"] = coerce_interval
coercers["interval_list"] = coerce_interval_list
coercers["interval_minutes_list"] = lambda *a: coerce_interval_list(*a, back_comp_unit_factor=60)
coercers["interval_seconds_list"] = coerce_interval_list
coercers["parameter_list"] = _coerce_parameter_list


SPEC = {
    "title": vdr(vtype="string", default=""),
    "description": vdr(vtype="string", default=""),
    "group": vdr(vtype="string", default="(ungrouped)"),
    "URL": vdr(vtype="string", default=""),
    "cylc": {
        "UTC mode": vdr(vtype="boolean", default=GLOBAL_CFG.get(["cylc", "UTC mode"])),
        "cycle point format": vdr(vtype="cycletime_format", default=None),
        "cycle point num expanded year digits": vdr(vtype="integer", default=0),
        "cycle point time zone": vdr(vtype="cycletime_time_zone", default=None),
        "required run mode": vdr(vtype="string", options=["live", "dummy", "simulation", ""]),
        "force run mode": vdr(vtype="string", options=["live", "dummy", "simulation", ""]),
        "abort if any task fails": vdr(vtype="boolean", default=False),
        "health check interval": vdr(vtype="interval", default=None),
        "log resolved dependencies": vdr(vtype="boolean", default=False),
        "disable automatic shutdown": vdr(vtype="boolean", default=False),
        "environment": {"__MANY__": vdr(vtype="string")},
        "parameters": {"__MANY__": vdr(vtype="parameter_list")},
        "parameter templates": {"__MANY__": vdr(vtype="string")},
        "events": {
            "handlers": vdr(vtype="string_list"),
            "handler events": vdr(vtype="string_list"),
Ejemplo n.º 56
0
Archivo: gscan.py Proyecto: kaday/cylc
    def __init__(self, hosts=None, owner=None, poll_interval=None):
        gobject.threads_init()
        set_exception_hook_dialog("cylc gscan")
        setup_icons()
        if not hosts:
            hosts = GLOBAL_CFG.get(["suite host scanning", "hosts"])
        self.hosts = hosts
        if owner is None:
            owner = user
        self.owner = owner
        self.window = gtk.Window()
        self.window.set_title("cylc gscan")
        self.window.set_icon(get_icon())
        self.vbox = gtk.VBox()
        self.vbox.show()

        self.theme_name = gcfg.get(['use theme'])
        self.theme = gcfg.get(['themes', self.theme_name])

        self.dots = DotMaker(self.theme)
        suite_treemodel = gtk.TreeStore(str, str, bool, str, int, str, str)
        self._prev_tooltip_location_id = None
        self.suite_treeview = gtk.TreeView(suite_treemodel)

        # Construct the host column.
        host_name_column = gtk.TreeViewColumn("Host")
        cell_text_host = gtk.CellRendererText()
        host_name_column.pack_start(cell_text_host, expand=False)
        host_name_column.set_cell_data_func(cell_text_host,
                                            self._set_cell_text_host)
        host_name_column.set_sort_column_id(0)
        host_name_column.set_visible(False)
        host_name_column.set_resizable(True)

        # Construct the suite name column.
        suite_name_column = gtk.TreeViewColumn("Suite")
        cell_text_name = gtk.CellRendererText()
        suite_name_column.pack_start(cell_text_name, expand=False)
        suite_name_column.set_cell_data_func(cell_text_name,
                                             self._set_cell_text_name)
        suite_name_column.set_sort_column_id(1)
        suite_name_column.set_resizable(True)

        # Construct the suite title column.
        suite_title_column = gtk.TreeViewColumn("Title")
        cell_text_title = gtk.CellRendererText()
        suite_title_column.pack_start(cell_text_title, expand=False)
        suite_title_column.set_cell_data_func(cell_text_title,
                                              self._set_cell_text_title)
        suite_title_column.set_sort_column_id(3)
        suite_title_column.set_visible(False)
        suite_title_column.set_resizable(True)

        # Construct the update time column.
        time_column = gtk.TreeViewColumn("Updated")
        cell_text_time = gtk.CellRendererText()
        time_column.pack_start(cell_text_time, expand=False)
        time_column.set_cell_data_func(cell_text_time,
                                       self._set_cell_text_time)
        time_column.set_sort_column_id(4)
        time_column.set_visible(False)
        time_column.set_resizable(True)

        self.suite_treeview.append_column(host_name_column)
        self.suite_treeview.append_column(suite_name_column)
        self.suite_treeview.append_column(suite_title_column)
        self.suite_treeview.append_column(time_column)

        # Construct the status column.
        status_column = gtk.TreeViewColumn("Status")
        status_column.set_sort_column_id(5)
        status_column.set_resizable(True)
        status_column_info = 6
        cycle_column_info = 5
        cell_text_cycle = gtk.CellRendererText()
        status_column.pack_start(cell_text_cycle, expand=False)
        status_column.set_cell_data_func(cell_text_cycle,
                                         self._set_cell_text_cycle,
                                         cycle_column_info)
        self.suite_treeview.append_column(status_column)
        distinct_states = len(task_state.legal)
        for i in range(distinct_states):
            cell_pixbuf_state = gtk.CellRendererPixbuf()
            status_column.pack_start(cell_pixbuf_state, expand=False)
            status_column.set_cell_data_func(cell_pixbuf_state,
                                             self._set_cell_pixbuf_state,
                                             (status_column_info, i))

        self.suite_treeview.show()
        if hasattr(self.suite_treeview, "set_has_tooltip"):
            self.suite_treeview.set_has_tooltip(True)
            try:
                self.suite_treeview.connect('query-tooltip',
                                            self._on_query_tooltip)
            except TypeError:
                # Lower PyGTK version.
                pass
        self.suite_treeview.connect("button-press-event",
                                    self._on_button_press_event)
        scrolled_window = gtk.ScrolledWindow()
        scrolled_window.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC)
        scrolled_window.add(self.suite_treeview)
        scrolled_window.show()
        self.vbox.pack_start(scrolled_window, expand=True, fill=True)
        self.updater = ScanAppUpdater(self.hosts,
                                      suite_treemodel,
                                      self.suite_treeview,
                                      owner=self.owner,
                                      poll_interval=poll_interval)
        self.updater.start()
        self.window.add(self.vbox)
        self.window.connect("destroy", self._on_destroy_event)
        self.window.set_default_size(300, 150)
        self.suite_treeview.grab_focus()
        self.window.show()
Ejemplo n.º 57
0
def scan_many(items=None, timeout=None, updater=None):
    """Call "identify" method of suites on many host:port.

    Args:
        items (list): list of 'host' string or ('host', port) tuple to scan.
        timeout (float): connection timeout, default is CONNECT_TIMEOUT.
        updater (object): quit scan cleanly if updater.quit is set.

    Return:
        list: [(host, port, identify_result), ...]
    """
    try:
        timeout = float(timeout)
    except (TypeError, ValueError):
        timeout = CONNECT_TIMEOUT
    my_uuid = uuid4()
    # Determine hosts to scan
    if not items:
        items = GLOBAL_CFG.get(["suite host scanning", "hosts"])
    # Ensure that it does "localhost" only once
    items = set(items)
    for item in list(items):
        if not isinstance(item, tuple) and not is_remote_host(item):
            items.remove(item)
            items.add("localhost")
    # To do and wait (submitted, waiting for results) sets
    todo_set = set()
    wait_set = set()
    # Determine ports to scan
    base_port = None
    max_ports = None
    for item in items:
        if isinstance(item, tuple):
            # Assume item is ("host", port)
            todo_set.add(item)
        else:
            # Full port range for a host
            if base_port is None or max_ports is None:
                base_port = GLOBAL_CFG.get(['communication', 'base port'])
                max_ports = GLOBAL_CFG.get(
                    ['communication', 'maximum number of ports'])
            for port in range(base_port, base_port + max_ports):
                todo_set.add((item, port))
    proc_items = []
    results = []
    # Number of child processes
    max_procs = GLOBAL_CFG.get(["process pool size"])
    if max_procs is None:
        max_procs = cpu_count()
    try:
        while todo_set or proc_items:
            no_action = True
            # Get results back from child processes where possible
            busy_proc_items = []
            while proc_items:
                if updater and updater.quit:
                    raise KeyboardInterrupt()
                proc, my_conn, terminate_time = proc_items.pop()
                if my_conn.poll():
                    host, port, result = my_conn.recv()
                    if result is None:
                        # Can't connect, ignore
                        wait_set.remove((host, port))
                    elif result == MSG_TIMEOUT:
                        # Connection timeout, leave in "wait_set"
                        pass
                    else:
                        # Connection success
                        results.append((host, port, result))
                        wait_set.remove((host, port))
                    if todo_set:
                        # Immediately give the child process something to do
                        host, port = todo_set.pop()
                        wait_set.add((host, port))
                        my_conn.send((host, port))
                        busy_proc_items.append(
                            (proc, my_conn, time() + INACTIVITY_TIMEOUT))
                    else:
                        # Or quit if there is nothing left to do
                        my_conn.send(MSG_QUIT)
                        my_conn.close()
                        proc.join()
                    no_action = False
                elif time() > terminate_time:
                    # Terminate child process if it is taking too long
                    proc.terminate()
                    proc.join()
                    no_action = False
                else:
                    busy_proc_items.append((proc, my_conn, terminate_time))
            proc_items += busy_proc_items
            # Create some child processes where necessary
            while len(proc_items) < max_procs and todo_set:
                if updater and updater.quit:
                    raise KeyboardInterrupt()
                my_conn, conn = Pipe()
                try:
                    proc = Process(target=_scan_worker,
                                   args=(conn, timeout, my_uuid))
                except OSError:
                    # Die if unable to start any worker process.
                    # OK to wait and see if any worker process already running.
                    if not proc_items:
                        raise
                    if cylc.flags.debug:
                        traceback.print_exc()
                else:
                    proc.start()
                    host, port = todo_set.pop()
                    wait_set.add((host, port))
                    my_conn.send((host, port))
                    proc_items.append(
                        (proc, my_conn, time() + INACTIVITY_TIMEOUT))
                    no_action = False
            if no_action:
                sleep(SLEEP_INTERVAL)
    except KeyboardInterrupt:
        return []
    # Report host:port with no results
    if wait_set:
        sys.stderr.write(
            'WARNING, scan timed out, no result for the following:\n')
        for key in sorted(wait_set):
            sys.stderr.write('  %s:%s\n' % key)
    return results
Ejemplo n.º 58
0
def scan_all(hosts=None, timeout=None):
    """Scan all hosts."""
    try:
        timeout = float(timeout)
    except:
        timeout = CONNECT_TIMEOUT
    my_uuid = uuid4()
    # Determine hosts to scan
    if not hosts:
        hosts = GLOBAL_CFG.get(["suite host scanning", "hosts"])
    # Ensure that it does "localhost" only once
    hosts = set(hosts)
    for host in list(hosts):
        if not is_remote_host(host):
            hosts.remove(host)
            hosts.add("localhost")
    # Determine ports to scan
    base_port = GLOBAL_CFG.get(['communication', 'base port'])
    max_ports = GLOBAL_CFG.get(['communication', 'maximum number of ports'])
    # Number of child processes
    max_procs = GLOBAL_CFG.get(["process pool size"])
    if max_procs is None:
        max_procs = cpu_count()
    # To do and wait (submitted, waiting for results) sets
    todo_set = set()
    wait_set = set()
    for host in hosts:
        for port in range(base_port, base_port + max_ports):
            todo_set.add((host, port))
    proc_items = []
    results = []
    while todo_set or proc_items:
        no_action = True
        # Get results back from child processes where possible
        busy_proc_items = []
        while proc_items:
            proc, my_conn, terminate_time = proc_items.pop()
            if my_conn.poll():
                host, port, result = my_conn.recv()
                if result is None:
                    # Can't connect, ignore
                    wait_set.remove((host, port))
                elif result == MSG_TIMEOUT:
                    # Connection timeout, leave in "wait_set"
                    pass
                else:
                    # Connection success
                    results.append((host, port, result))
                    wait_set.remove((host, port))
                if todo_set:
                    # Immediately give the child process something to do
                    host, port = todo_set.pop()
                    wait_set.add((host, port))
                    my_conn.send((host, port))
                    busy_proc_items.append(
                        (proc, my_conn, time() + INACTIVITY_TIMEOUT))
                else:
                    # Or quit if there is nothing left to do
                    my_conn.send(MSG_QUIT)
                    my_conn.close()
                    proc.join()
                no_action = False
            elif time() > terminate_time:
                # Terminate child process if it is taking too long
                proc.terminate()
                proc.join()
                no_action = False
            else:
                busy_proc_items.append((proc, my_conn, terminate_time))
        proc_items += busy_proc_items
        # Create some child processes where necessary
        while len(proc_items) < max_procs and todo_set:
            my_conn, conn = Pipe()
            try:
                proc = Process(target=_scan1_impl,
                               args=(conn, timeout, my_uuid))
            except OSError:
                # Die if unable to start any worker process.
                # OK to wait and see if any worker process already running.
                if not proc_items:
                    raise
                if cylc.flags.debug:
                    traceback.print_exc()
            else:
                proc.start()
                host, port = todo_set.pop()
                wait_set.add((host, port))
                my_conn.send((host, port))
                proc_items.append((proc, my_conn, time() + INACTIVITY_TIMEOUT))
                no_action = False
        if no_action:
            sleep(SLEEP_INTERVAL)
    # Report host:port with no results
    if wait_set:
        print >> sys.stderr, (
            'WARNING, scan timed out, no result for the following:')
        for key in sorted(wait_set):
            print >> sys.stderr, '  %s:%s' % key
    return results