Ejemplo n.º 1
0
    def register(self, name, dir):
        name = RegPath(name).get()
        for suite in self.list_all_suites():
            if name == suite:
                raise RegistrationError("ERROR: " + name +
                                        " is already registered.")
            elif suite.startswith(name + RegPath.delimiter):
                raise RegistrationError("ERROR: " + name +
                                        " is a registered group.")
            elif name.startswith(suite + RegPath.delimiter):
                # suite starts with, to some level, an existing suite name
                raise RegistrationError("ERROR: " + suite +
                                        " is a registered suite.")
        dir = dir.rstrip('/')  # strip trailing '/'
        dir = re.sub('^\./', '', dir)  # strip leading './'
        if not dir.startswith('/'):
            # On AIX on GPFS os.path.abspath(dir) returns the path with
            # full 'fileset' prefix. Manual use of $PWD to absolutize a
            # relative path gives a cleaner result.
            dir = os.path.join(os.environ['PWD'], dir)
        title = self.get_suite_title(name, path=dir)
        title = title.split('\n')[0]  # use the first of multiple lines
        print 'REGISTER', name + ':', dir
        with open(os.path.join(self.dbpath, name), 'w') as file:
            file.write('path=' + dir + '\n')
            file.write('title=' + title + '\n')

        # create a new passphrase for the suite if necessary
        passphrase(name, user, get_hostname()).generate(dir)
Ejemplo n.º 2
0
    def cache_passphrase(self, suite, owner, host, passphrase):
        """Cache and dump passphrase for a remote suite in standard location.

        Save passphrase to ~/.cylc/passphrases/owner@host/suite if possible.
        This is normally called on a successful authentication, and will cache
        the remote passphrase in memory as well.
        """
        if owner is None:
            owner = USER
        if host is None:
            host = get_hostname()
        path = os.path.expanduser(os.path.join(
            '~', '.cylc', self.PASSPHRASES_DIR_BASE, owner + "@" + host, suite
        ))
        self.cached_passphrases[(suite, owner, host)] = passphrase
        # Dump to a file only for remote suites loaded via SSH.
        if self.can_disk_cache_passphrases.get((suite, owner, host)):
            # Although not desirable, failing to dump the passphrase to a file
            # is not disastrous.
            try:
                self._dump_passphrase_to_dir(path, passphrase)
            except (IOError, OSError):
                if cylc.flags.debug:
                    import traceback
                    traceback.print_exc()
Ejemplo n.º 3
0
    def register( self, name, dir ):
        name = RegPath(name).get()
        for suite in self.list_all_suites():
            if name == suite:
                raise RegistrationError, "ERROR: " + name + " is already registered."
            elif suite.startswith( name + RegPath.delimiter ):
                raise RegistrationError, "ERROR: " + name + " is a registered group."
            elif name.startswith( suite + RegPath.delimiter ):
                # suite starts with, to some level, an existing suite name
                raise RegistrationError, "ERROR: " + suite + " is a registered suite."
        dir = dir.rstrip( '/' )  # strip trailing '/'
        dir = re.sub( '^\./', '', dir ) # strip leading './'
        if not dir.startswith( '/' ):
            # On AIX on GPFS os.path.abspath(dir) returns the path with
            # full 'fileset' prefix. Manual use of $PWD to absolutize a
            # relative path gives a cleaner result.
            dir = os.path.join( os.environ['PWD'], dir )
        title = self.get_suite_title(name, path=dir)
        title = title.split('\n')[0] # use the first of multiple lines
        print 'REGISTER', name + ':', dir
        with open( os.path.join( self.dbpath, name ), 'w' ) as file:
            file.write( 'path=' + dir + '\n' )
            file.write( 'title=' + title + '\n' )

        # create a new passphrase for the suite if necessary
        passphrase(name,user,get_hostname()).generate(dir)
Ejemplo n.º 4
0
 def _load_port_file(self):
     """Load port, host, etc from port file."""
     # GLOBAL_CFG is expensive to import, so only load on demand
     from cylc.cfgspec.globalcfg import GLOBAL_CFG
     port_file_path = os.path.join(
         GLOBAL_CFG.get(['communication', 'ports directory']), self.suite)
     out = ""
     if is_remote_host(self.host) or is_remote_user(self.owner):
         # Only load these modules on demand, as they may be expensive
         import shlex
         from subprocess import Popen, PIPE
         ssh_tmpl = str(GLOBAL_CFG.get_host_item(
             'remote shell template', self.host, self.owner))
         ssh_tmpl = ssh_tmpl.replace(' %s', '')
         user_at_host = ''
         if self.owner:
             user_at_host = self.owner + '@'
         if self.host:
             user_at_host += self.host
         else:
             user_at_host += 'localhost'
         r_port_file_path = port_file_path.replace(
             os.environ['HOME'], '$HOME')
         command = shlex.split(ssh_tmpl) + [
             user_at_host, 'cat', r_port_file_path]
         proc = Popen(command, stdout=PIPE, stderr=PIPE)
         out, err = proc.communicate()
         ret_code = proc.wait()
         if ret_code:
             if cylc.flags.debug:
                 print >> sys.stderr, {
                     "code": ret_code,
                     "command": command,
                     "stdout": out,
                     "stderr": err}
             if self.port is None:
                 raise PortFileError(
                     "Port file '%s:%s' not found - suite not running?." %
                     (user_at_host, r_port_file_path))
     else:
         try:
             out = open(port_file_path).read()
         except IOError:
             if self.port is None:
                 raise PortFileError(
                     "Port file '%s' not found - suite not running?." %
                     (port_file_path))
     lines = out.splitlines()
     if self.port is None:
         try:
             self.port = int(lines[0])
         except (IndexError, ValueError):
             raise PortFileError(
                 "ERROR, bad content in port file: %s" % port_file_path)
     if self.host is None:
         if len(lines) >= 2:
             self.host = lines[1].strip()
         else:
             self.host = get_hostname()
Ejemplo n.º 5
0
 def __init__(self, suite, owner=None, host=None):
     self.suite = suite
     self.owner = owner
     self.host = host
     if self.owner is None:
         self.owner = user
     if self.host is None:
         self.host = get_hostname()
     self.location = None
Ejemplo n.º 6
0
 def __init__(self,
              suite,
              task_id,
              owner=user,
              host=get_hostname(),
              pyro_timeout=None,
              port=None):
     self.target_server_object = task_id
     super(TaskMessageClient, self).__init__(suite, owner, host,
                                             pyro_timeout, port)
Ejemplo n.º 7
0
 def _get_headers(self):
     """Return HTTP headers identifying the client."""
     user_agent_string = (
         "cylc/%s prog_name/%s uuid/%s" % (
             CYLC_VERSION, self.prog_name, self.my_uuid
         )
     )
     auth_info = "%s@%s" % (USER, get_hostname())
     return {"User-Agent": user_agent_string,
             "From": auth_info}
Ejemplo n.º 8
0
 def _get_headers(self):
     """Return HTTP headers identifying the client."""
     user_agent_string = (
         "cylc/%s prog_name/%s uuid/%s" % (
             CYLC_VERSION, self.prog_name, self.my_uuid
         )
     )
     auth_info = "%s@%s" % (USER, get_hostname())
     return {"User-Agent": user_agent_string,
             "From": auth_info}
Ejemplo n.º 9
0
def _ws_init(service_inst, *args, **kwargs):
    """Start quick web service."""
    # cherrypy.config["tools.encode.on"] = True
    # cherrypy.config["tools.encode.encoding"] = "utf-8"
    cherrypy.config["server.socket_host"] = "0.0.0.0"
    cherrypy.config["engine.autoreload.on"] = False
    if can_ssl():
        cherrypy.config["server.ssl_module"] = "pyopenSSL"
        cherrypy.config["server.ssl_certificate"] = service_inst.cert
        cherrypy.config["server.ssl_private_key"] = service_inst.pkey
    else:
        sys.stderr.write("WARNING: no HTTPS support: cannot import OpenSSL\n")
    cherrypy.config["log.screen"] = None
    key = binascii.hexlify(os.urandom(16))
    cherrypy.config.update(
        {
            "tools.auth_digest.on": True,
            "tools.auth_digest.realm": service_inst.suite,
            "tools.auth_digest.get_ha1": service_inst.get_ha1,
            "tools.auth_digest.key": key,
            "tools.auth_digest.algorithm": service_inst.hash_algorithm,
        }
    )
    cherrypy.tools.connect_log = cherrypy.Tool("on_end_resource", service_inst.report_connection_if_denied)
    cherrypy.config["tools.connect_log.on"] = True
    host = get_hostname()
    service_inst.engine = cherrypy.engine

    for port in service_inst.ok_ports:
        my_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            my_socket.bind((host, port))
        except socket.error:
            # Host busy.
            my_socket.close()
            continue
        my_socket.close()
        cherrypy.config["server.socket_port"] = port
        try:
            cherrypy.engine.start()
            cherrypy.engine.wait(cherrypy.engine.states.STARTED)
            if cherrypy.engine.state != cherrypy.engine.states.STARTED:
                continue
        except (socket.error, IOError):
            pass
        except:
            import traceback

            traceback.print_exc()
        else:
            service_inst.port = port
            return
        # We need to reinitialise the httpserver for each port attempt.
        cherrypy.server.httpserver = None
    raise Exception("No available ports")
Ejemplo n.º 10
0
 def call_server_func(self, category, fname, **fargs):
     """Call server_object.fname(*fargs, **fargs)."""
     if self.host is None and self.port is not None:
         self.host = get_hostname()
     try:
         self._load_contact_info()
     except (IOError, ValueError, SuiteServiceFileError):
         raise ConnectionInfoError(self.suite)
     handle_proxies()
     payload = fargs.pop("payload", None)
     method = fargs.pop("method", self.METHOD)
     host = self.host
     if host == "localhost":
         host = get_hostname().split(".")[0]
     url = 'https://%s:%s/%s/%s' % (host, self.port, category, fname)
     if fargs:
         import urllib
         params = urllib.urlencode(fargs, doseq=True)
         url += "?" + params
     return self._get_data_from_url(url, payload, method=method)
Ejemplo n.º 11
0
 def get(self, pfile=None, suitedir=None):
     ppfile = self.get_passphrase_file(pfile, suitedir)
     psf = open(ppfile, 'r')
     lines = psf.readlines()
     psf.close()
     if len(lines) != 1:
         raise PassphraseError, ('ERROR, invalid passphrase file: %s@%s:%s' % (
                                 user, get_hostname(), ppfile))
     # chomp trailing whitespace and newline
     self.passphrase = lines[0].strip()
     return self.passphrase
Ejemplo n.º 12
0
    def mungeIdent(self, ident):
        """Receive (uuid, passphrase) from client. Encrypt the passphrase.

        Also pass client identification info to server for logging:
        (user, host, prog name).

        """
        hash_ = self._get_hash()
        uuid, passphrase = ident
        prog_name = os.path.basename(sys.argv[0])
        if passphrase is None:
            passphrase = NO_PASSPHRASE
        return (USER, get_hostname(), str(uuid), prog_name,
                hash_(passphrase).digest())
Ejemplo n.º 13
0
    def mungeIdent(self, ident):
        """Receive (uuid, passphrase) from client. Encrypt the passphrase.

        Also pass client identification info to server for logging:
        (user, host, prog name).

        """
        hash_ = self._get_hash()
        uuid, passphrase = ident
        prog_name = os.path.basename(sys.argv[0])
        if passphrase is None:
            passphrase = NO_PASSPHRASE
        return (USER, get_hostname(), str(uuid), prog_name,
                hash_(passphrase).digest())
Ejemplo n.º 14
0
 def call_server_func(self, category, fname, **fargs):
     """Call server_object.fname(*fargs, **fargs)."""
     if self.host is None and self.port is not None:
         self.host = get_hostname()
     if self.host is None or self.port is None:
         try:
             self._load_contact_info()
         except (IOError, ValueError, SuiteServiceFileError):
             raise ConnectionInfoError(self.suite)
     handle_proxies()
     payload = fargs.pop("payload", None)
     method = fargs.pop("method", self.METHOD)
     host = self.host
     if not self.host.split(".")[0].isdigit():
         host = self.host.split(".")[0]
     if host == "localhost":
         host = get_hostname().split(".")[0]
     url = 'https://%s:%s/%s/%s' % (host, self.port, category, fname)
     if fargs:
         import urllib
         params = urllib.urlencode(fargs, doseq=True)
         url += "?" + params
     return self._get_data_from_url(url, payload, method=method)
Ejemplo n.º 15
0
 def generate(self, dir):
     pfile = os.path.join(dir, 'passphrase')
     if os.path.isfile(pfile):
         try:
             self.get(pfile)
             return
         except PassphraseError:
             pass
     # Note: Perhaps a UUID might be better here?
     char_set = string.ascii_uppercase + string.ascii_lowercase + string.digits
     self.passphrase = ''.join(random.sample(char_set, 20))
     mkdir_p(dir)
     f = open(pfile, 'w')
     f.write(self.passphrase)
     f.close()
     # set passphrase file permissions to owner-only
     os.chmod(pfile, 0600)
     if cylc.flags.verbose:
         print 'Generated suite passphrase: %s@%s:%s' % (
             user, get_hostname(), pfile)
Ejemplo n.º 16
0
def load_passphrases(db):
    """Load all of the user's passphrases (back-compat for <= 6.4.1)."""
    global passphrases
    if passphrases:
        return passphrases

    # Find passphrases in all registered suite directories.
    reg = localdb(db)
    reg_suites = reg.get_list()
    for item in reg_suites:
        rg = item[0]
        di = item[1]
        try:
            p = passphrase(rg, user, get_hostname()).get(suitedir=di)
        except Exception, x:
            # Suite has no passphrase.
            if cylc.flags.debug:
                print >> sys.stderr, x
        else:
            passphrases.append(p)
Ejemplo n.º 17
0
def load_passphrases(db):
    """Load all of the user's passphrases (back-compat for <= 6.4.1)."""
    global passphrases
    if passphrases:
        return passphrases

    # Find passphrases in all registered suite directories.
    reg = localdb(db)
    reg_suites = reg.get_list()
    for item in reg_suites:
        rg = item[0]
        di = item[1]
        try:
            p = passphrase(rg, user, get_hostname()).get(suitedir=di)
        except Exception, x:
            # Suite has no passphrase.
            if cylc.flags.debug:
                print >> sys.stderr, x
        else:
            passphrases.append(p)
Ejemplo n.º 18
0
    def _create_ssl_pem_and_cert(self, path, reg):
        """Create ssl.pem and ssl.cert files for suite in path."""
        try:
            from OpenSSL import crypto
        except ImportError:
            # OpenSSL not installed, so we can't use HTTPS anyway.
            return
        host = get_hostname()
        altnames = [
            "DNS:*", "DNS:%s" % host,
            "IP:%s" % get_local_ip_address(host),
            # See https://github.com/kennethreitz/requests/issues/2621
            "DNS:%s" % get_local_ip_address(host)]

        # Use suite name as the 'common name', but no more than 64 chars.
        cert_common_name = reg
        if len(reg) > 64:
            cert_common_name = reg[:61] + "..."

        # Create a private key.
        pkey_obj = crypto.PKey()
        pkey_obj.generate_key(crypto.TYPE_RSA, 2048)
        self._dump_item(
            path, self.FILE_BASE_SSL_PEM,
            crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey_obj))

        # Create a self-signed certificate.
        cert_obj = crypto.X509()
        cert_obj.get_subject().O = "Cylc"
        cert_obj.get_subject().CN = cert_common_name
        cert_obj.gmtime_adj_notBefore(0)
        cert_obj.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60)  # 10 years.
        cert_obj.set_issuer(cert_obj.get_subject())
        cert_obj.set_pubkey(pkey_obj)
        cert_obj.add_extensions([crypto.X509Extension(
            "subjectAltName", False, ", ".join(altnames))])
        cert_obj.sign(pkey_obj, 'sha256')
        self._dump_item(
            path, self.FILE_BASE_SSL_CERT,
            crypto.dump_certificate(crypto.FILETYPE_PEM, cert_obj))
Ejemplo n.º 19
0
    def cache_passphrase(self, reg, owner, host, value):
        """Cache and dump passphrase for a remote suite in standard location.

        Save passphrase to ~/.cylc/auth/owner@host/reg if possible.
        This is normally called on a successful authentication, and will cache
        the remote passphrase in memory as well.
        """
        if owner is None:
            owner = USER
        if host is None:
            host = get_hostname()
        path = self._get_cache_dir(reg, owner, host)
        self.cache[self.FILE_BASE_PASSPHRASE][(reg, owner, host)] = value
        # Dump to a file only for remote suites loaded via SSH.
        if self.can_disk_cache_passphrases.get((reg, owner, host)):
            # Although not desirable, failing to dump the passphrase to a file
            # is not disastrous.
            try:
                self._dump_item(path, self.FILE_BASE_PASSPHRASE, value)
            except (IOError, OSError):
                if cylc.flags.debug:
                    import traceback
                    traceback.print_exc()
Ejemplo n.º 20
0
    def _is_local_auth_ok(self, reg, owner, host):
        """Return True if it is OK to use local passphrase, ssl.* files.

        Use values in ~/cylc-run/REG/.service/contact to make a judgement.
        Cache results in self.can_use_load_auths.
        """
        if (reg, owner, host) not in self.can_use_load_auths:
            if is_remote_user(owner) or is_remote_host(host):
                fname = os.path.join(
                    self.get_suite_srv_dir(reg), self.FILE_BASE_CONTACT)
                data = {}
                try:
                    for line in open(fname):
                        key, value = (
                            [item.strip() for item in line.split("=", 1)])
                        data[key] = value
                except IOError, ValueError:
                    # No contact file
                    self.can_use_load_auths[(reg, owner, host)] = False
                else:
                    # Contact file exists, check values match
                    if owner is None:
                        owner = USER
                    if host is None:
                        host = get_hostname()
                    host_value = data.get(self.KEY_HOST, "")
                    self.can_use_load_auths[(reg, owner, host)] = (
                        reg == data.get(self.KEY_NAME) and
                        owner == data.get(self.KEY_OWNER) and
                        (
                            host == host_value or
                            host == host_value.split(".", 1)[0]  # no domain
                        )
                    )
            else:
                self.can_use_load_auths[(reg, owner, host)] = True
Ejemplo n.º 21
0
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""In analogy with cylc.hostname.is_remote_host(), determine if a
username is "remote"."""

import os
import pwd
from cylc.suite_host import get_hostname

user = os.environ.get('USER', pwd.getpwuid(os.getuid()).pw_name)
host = get_hostname()
user_at_host = "%s@%s" % (user, host)


def is_remote_user(name):
    """Return True if name is different than the current username.
    Return False if name is None.
    """
    return name and name != user
Ejemplo n.º 22
0
    def get_passphrase_file(self, pfile=None, suitedir=None):
        """
Passphrase location, order of preference:

1/ The pfile argument - used for passphrase creation by "cylc register".

2/ The suite definition directory, because suites may be automatically
installed (e.g. by Rose) to remote task hosts, and remote tasks know
this location from their execution environment. Local user command
invocations can use the suite registration database to find the suite
definition directory.  HOWEVER, remote user command invocations cannot
do this even if the local and remote hosts share a common filesystem,
because we cannot be sure if finding the expected suite registration
implies a common filesystem or a different remote suite that happens to
be registered under the same name. User accounts used for remote control
must therefore install the passphrase in the secondary standard
locations (below) or use the command line option to explicitly reveal
the location. Remote tasks with 'ssh messaging = True' look first in the
suite definition directory of the suite host, which they know through
the variable CYLC_SUITE_DEF_PATH_ON_SUITE_HOST in the task execution
environment.

3/ Secondary locations:
    (i) $HOME/.cylc/SUITE_HOST/SUITE_OWNER/SUITE_NAME/passphrase
   (ii) $HOME/.cylc/SUITE_HOST/SUITE_NAME/passphrase
  (iii) $HOME/.cylc/SUITE_NAME/passphrase
These are more sensible locations for remote suite control from accounts
that do not actually need the suite definition directory to be installed.
"""
        # 1/ Explicit suite definition directory given on the command line.
        if pfile:
            if os.path.isdir(pfile):
                pfile = os.path.join(pfile, 'passphrase')
            if os.path.isfile(pfile):
                self.set_location(pfile)
            else:
                # If an explicit location is given, the file must exist.
                raise PassphraseError(
                    'ERROR, file not found on %s@%s: %s' % (
                        user, get_hostname(), pfile))

        # 2/ Cylc commands with suite definition directory from local reg.
        if not self.location and suitedir:
            pfile = os.path.join(suitedir, 'passphrase')
            if os.path.isfile(pfile):
                self.set_location(pfile)

        # (2 before 3 else sub-suites load their parent suite's
        # passphrase on start-up because the "cylc run" command runs in
        # a parent suite task execution environment).

        # 3/ Running tasks: suite def dir from the task execution environment.
        if not self.location:
            try:
                # Test for presence of task execution environment
                suite_host = os.environ['CYLC_SUITE_HOST']
                suite_owner = os.environ['CYLC_SUITE_OWNER']
            except KeyError:
                # not called by a task
                pass
            else:
                # called by a task
                if is_remote_host(suite_host) or is_remote_user(suite_owner):
                    # 2(i)/ Task messaging call on a remote account.

                    # First look in the remote suite definition
                    # directory ($CYLC_SUITE_DEF_PATH is modified for
                    # remote tasks):
                    try:
                        pfile = os.path.join(
                            os.environ['CYLC_SUITE_DEF_PATH'], 'passphrase')
                    except KeyError:
                        pass
                    else:
                        if os.path.isfile(pfile):
                            self.set_location(pfile)
                else:
                    # 2(ii)/ Task messaging call on the suite host account.

                    # Could be a local task or a remote task with 'ssh
                    # messaging = True'. In either case use
                    # $CYLC_SUITE_DEF_PATH_ON_SUITE_HOST which never
                    # changes, not $CYLC_SUITE_DEF_PATH which gets
                    # modified for remote tasks as described above.
                    try:
                        pfile = os.path.join(
                            os.environ['CYLC_SUITE_DEF_PATH_ON_SUITE_HOST'], 'passphrase')
                    except KeyError:
                        pass
                    else:
                        if os.path.isfile(pfile):
                            self.set_location(pfile)

        # 4/ Other allowed locations, as documented above.
        if not self.location:
            locations = []
            # For remote control commands, self.host here will be fully
            # qualified or not depending on what's given on the command line.
            short_host = re.sub('\..*', '', self.host)
            prefix = os.path.join(os.environ['HOME'], '.cylc')
            locations.append(
                os.path.join(
                    prefix, self.host, self.owner, self.suite, 'passphrase'))
            if short_host != self.host:
                locations.append(os.path.join(
                    prefix, short_host, self.owner, self.suite, 'passphrase'))
            locations.append(
                os.path.join(prefix, self.host, self.suite, 'passphrase'))
            if short_host != self.host:
                locations.append(os.path.join(
                    prefix, short_host, self.suite, 'passphrase'))
            locations.append(os.path.join(prefix, self.suite, 'passphrase'))
            for pfile in locations:
                if os.path.isfile(pfile):
                    self.set_location(pfile)
                    break

        if not self.location:
            raise PassphraseError(
                'ERROR: passphrase for suite %s not found on %s@%s' % (
                    self.suite, user, get_hostname()))
        return self.location
Ejemplo n.º 23
0
 def __init__(self, suite, owner=user, host=get_hostname()):
     self.suite = suite
     self.owner = owner
     self.host = host
     self.location = None
Ejemplo n.º 24
0
    def load_item(self, suite, owner, host, item="certificate",
                  create_ok=False, cache_ok=False):
        """Load or create a passphrase, SSL certificate or a private key.

        SSL files are searched from these locations in order:

        1/ For running task jobs:
           a/ $CYLC_SUITE_RUN_DIR then $CYLC_SUITE_DEF_PATH for remote jobs.
           b/ $CYLC_SUITE_DEF_PATH_ON_SUITE_HOST for local jobs or remote jobs
              with SSH messaging.

        2/ (Passphrases only) From memory cache, for remote suite passphrases.
           Don't use if cache_ok=False.

        3/ For suite on local user@host. The suite definition directory, as
           registered. (Note: Previously, this needs to be the 1st location,
           else sub-suites load their parent suite's passphrases, etc, on
           start-up because the "cylc run" command runs in a parent suite task
           execution environment. This problem no longer exists becase on suite
           start up, the "load_item_from_dir" method is called directly
           instead of through this method.)

        4/ Location under $HOME/.cylc/ for remote suite control from accounts
           that do not actually need the suite definition directory to be
           installed:
           $HOME/.cylc/passphrases/SUITE_OWNER@SUITE_HOST/SUITE_NAME/

        5/ (SSL files only) If create_ok is specified, create the SSL file and
           then return it.

        6/ For remote suites, try locating the file from the suite definition
           directory on remote owner@host via SSH.

        """
        item_is_passphrase = False
        if item == "certificate":
            item = self.SSL_CERTIFICATE_FILE_BASE
        elif item == "private_key":
            item = self.SSL_PRIVATE_KEY_FILE_BASE
        elif item == "passphrase":
            item_is_passphrase = True
            self.can_disk_cache_passphrases[(suite, owner, host)] = False

        suite_host = os.getenv('CYLC_SUITE_HOST')
        suite_owner = os.getenv('CYLC_SUITE_OWNER')
        if suite == os.getenv('CYLC_SUITE_NAME'):
            env_keys = []
            if is_remote_host(suite_host) or is_remote_user(suite_owner):
                # 1(a)/ Task messaging call on a remote account.
                # First look in the remote suite run directory than suite
                # definition directory ($CYLC_SUITE_DEF_PATH is modified
                # for remote tasks):
                env_keys = ['CYLC_SUITE_RUN_DIR', 'CYLC_SUITE_DEF_PATH']
            elif suite_host or suite_owner:
                # 1(b)/ Task messaging call on the suite host account.

                # Could be a local task or a remote task with 'ssh
                # messaging = True'. In either case use
                # $CYLC_SUITE_DEF_PATH_ON_SUITE_HOST which never
                # changes, not $CYLC_SUITE_DEF_PATH which gets
                # modified for remote tasks as described above.
                env_keys = ['CYLC_SUITE_DEF_PATH_ON_SUITE_HOST']
            for key in env_keys:
                try:
                    return self.load_item_from_dir(os.environ[key], item)
                except (KeyError, IOError, PassphraseError):
                    pass

        # 2/ From memory cache
        if cache_ok and item_is_passphrase:
            pass_owner = owner
            pass_host = host
            if pass_owner is None:
                pass_owner = USER
            if pass_host is None:
                pass_host = get_hostname()
            try:
                return self.cached_passphrases[(suite, pass_owner, pass_host)]
            except KeyError:
                pass

        # 3/ Cylc commands with suite definition directory from local reg.
        if cache_ok or not is_remote_user(owner) and not is_remote_host(host):
            try:
                return self.load_item_from_dir(self.get_suitedir(suite), item)
            except (IOError, PassphraseError, RegistrationError):
                pass

        # 4/ Other allowed locations, as documented above.
        prefix = os.path.expanduser(os.path.join('~', '.cylc'))
        if host is None:
            host = suite_host
        if (owner is not None and host is not None and
                (not item_is_passphrase or cache_ok)):
            prefix = os.path.expanduser(os.path.join('~', '.cylc'))
            paths = []
            path_types = [(prefix, self.PASSPHRASES_DIR_BASE,
                           owner + "@" + host, suite)]
            short_host = host.split('.', 1)[0]
            if short_host != host:
                path_types.append((prefix, self.PASSPHRASES_DIR_BASE,
                                  owner + "@" + short_host, suite))

            for names in path_types:
                try:
                    return self.load_item_from_dir(os.path.join(*names), item)
                except (IOError, PassphraseError):
                    pass

        if create_ok and not item_is_passphrase:
            # 5/ Create the SSL file if it doesn't exist.
            return self._dump_certificate_and_key_to_dir(
                self.get_suitedir(suite), suite)

        load_dest_root = None
        if not item_is_passphrase:
            load_dest_root = os.path.join(
                prefix, self.PASSPHRASES_DIR_BASE, owner + "@" + host, suite)
        try:
            # 6/ Try ssh-ing to grab the files directly.
            content = self._load_item_via_ssh(
                item, suite, owner, host, dest_dir=load_dest_root)
            if content and item_is_passphrase:
                self.can_disk_cache_passphrases[(suite, owner, host)] = True
            return content
        except Exception as exc:
            import traceback
            traceback.print_exc()
        raise PassphraseError("Couldn't get %s" % item)
Ejemplo n.º 25
0
    def _set_uri(self):
        """Set Pyro URI.

        Determine host and port using content in port file, unless already
        specified.

        """
        if ((self.host is None or self.port is None) and
                'CYLC_SUITE_RUN_DIR' in os.environ):
            # Looks like we are in a running task job, so we should be able to
            # use "cylc-suite-env" file under the suite running directory
            try:
                suite_env = CylcSuiteEnv.load(
                    self.suite, os.environ['CYLC_SUITE_RUN_DIR'])
            except CylcSuiteEnvLoadError:
                if cylc.flags.debug:
                    traceback.print_exc()
            else:
                self.host = suite_env.suite_host
                self.port = suite_env.suite_port
                self.owner = suite_env.suite_owner

        if self.host is None or self.port is None:
            port_file_path = os.path.join(
                GLOBAL_CFG.get(['pyro', 'ports directory']), self.suite)
            if is_remote_host(self.host) or is_remote_user(self.owner):
                ssh_tmpl = str(GLOBAL_CFG.get_host_item(
                    'remote shell template', self.host, self.owner))
                ssh_tmpl = ssh_tmpl.replace(' %s', '')
                user_at_host = ''
                if self.owner:
                    user_at_host = self.owner + '@'
                if self.host:
                    user_at_host += self.host
                else:
                    user_at_host += 'localhost'
                r_port_file_path = port_file_path.replace(
                    os.environ['HOME'], '$HOME')
                command = shlex.split(ssh_tmpl) + [
                    user_at_host, 'cat', r_port_file_path]
                proc = Popen(command, stdout=PIPE, stderr=PIPE)
                out, err = proc.communicate()
                ret_code = proc.wait()
                if ret_code:
                    if cylc.flags.debug:
                        print >> sys.stderr, {
                            "code": ret_code,
                            "command": command,
                            "stdout": out,
                            "stderr": err}
                    raise PortFileError(
                        "Port file '%s:%s' not found - suite not running?." %
                        (user_at_host, r_port_file_path))
            else:
                try:
                    out = open(port_file_path).read()
                except IOError:
                    raise PortFileError(
                        "Port file '%s' not found - suite not running?." %
                        (port_file_path))
            lines = out.splitlines()
            try:
                if self.port is None:
                    self.port = int(lines[0])
            except (IndexError, ValueError):
                raise PortFileError(
                    "ERROR, bad content in port file: %s" % port_file_path)
            if self.host is None:
                if len(lines) >= 2:
                    self.host = lines[1].strip()
                else:
                    self.host = get_hostname()

        # Qualify the obj name with user and suite name (unnecessary but
        # can't change it until we break back-compat with older daemons).
        self.uri = (
            'PYROLOC://%(host)s:%(port)s/%(owner)s.%(suite)s.%(target)s' % {
                "host": self.host,
                "port": self.port,
                "suite": self.suite,
                "owner": self.owner,
                "target": self.target_server_object})
Ejemplo n.º 26
0
 def set_location(self, pfile):
     if cylc.flags.debug:
         print '%s (%s@%s)' %(pfile, user, get_hostname())
     self.location = pfile
Ejemplo n.º 27
0
    def get_auth_item(self, item, reg, owner=None, host=None, content=False):
        """Locate/load passphrase, SSL private key, SSL certificate, etc.

        Return file name, or content of file if content=True is set.
        Files are searched from these locations in order:

        1/ For running task jobs, service directory under:
           a/ $CYLC_SUITE_RUN_DIR for remote jobs.
           b/ $CYLC_SUITE_RUN_DIR_ON_SUITE_HOST for local jobs or remote jobs
              with SSH messaging.

        2/ (Passphrases only) From memory cache, for remote suite passphrases.
           Don't use if content=False.

        3/ For suite on local user@host. The suite service directory.

        4/ Location under $HOME/.cylc/ for remote suite control from accounts
           that do not actually need the suite definition directory to be
           installed:
           $HOME/.cylc/auth/SUITE_OWNER@SUITE_HOST/SUITE_NAME/

        5/ For remote suites, try locating the file from the suite service
           directory on remote owner@host via SSH. If content=False, the value
           of the located file will be dumped under:
           $HOME/.cylc/auth/SUITE_OWNER@SUITE_HOST/SUITE_NAME/

        """
        if item not in [
                self.FILE_BASE_SSL_CERT, self.FILE_BASE_SSL_PEM,
                self.FILE_BASE_PASSPHRASE, self.FILE_BASE_CONTACT]:
            raise ValueError("%s: item not recognised" % item)
        if item == self.FILE_BASE_PASSPHRASE:
            self.can_disk_cache_passphrases[(reg, owner, host)] = False

        suite_host = os.getenv('CYLC_SUITE_HOST')
        suite_owner = os.getenv('CYLC_SUITE_OWNER')
        if reg == os.getenv('CYLC_SUITE_NAME'):
            env_keys = []
            if is_remote_host(suite_host) or is_remote_user(suite_owner):
                # 1(a)/ Task messaging call on a remote account.
                # Look in the remote suite run directory:
                env_keys = ['CYLC_SUITE_RUN_DIR']
            elif suite_host or suite_owner:
                # 1(b)/ Task messaging call on the suite host account.

                # Could be a local task or a remote task with 'ssh
                # messaging = True'. In either case use
                # $CYLC_SUITE_RUN_DIR_ON_SUITE_HOST which never changes.
                env_keys = ['CYLC_SUITE_RUN_DIR_ON_SUITE_HOST']
            for key in env_keys:
                path = os.path.join(os.environ[key], self.DIR_BASE_SRV)
                if content:
                    value = self._load_local_item(item, path)
                else:
                    value = self._locate_item(item, path)
                if value:
                    return value

        # 2/ From memory cache
        if item in self.cache:
            my_owner = owner
            my_host = host
            if my_owner is None:
                my_owner = USER
            if my_host is None:
                my_host = get_hostname()
            try:
                return self.cache[item][(reg, my_owner, my_host)]
            except KeyError:
                pass

        # 3/ Local suite service directory
        if self._is_local_auth_ok(reg, owner, host):
            path = self.get_suite_srv_dir(reg)
            if content:
                value = self._load_local_item(item, path)
            else:
                value = self._locate_item(item, path)
            if value:
                return value

        # 4/ Disk cache for remote suites
        if host is None:
            host = suite_host
        if owner is not None and host is not None:
            paths = [self._get_cache_dir(reg, owner, host)]
            short_host = host.split('.', 1)[0]
            if short_host != host:
                paths.append(self._get_cache_dir(reg, owner, short_host))
            for path in paths:
                if content:
                    value = self._load_local_item(item, path)
                else:
                    value = self._locate_item(item, path)
                if value:
                    return value

        # 5/ Use SSH to load content from remote owner@host
        value = self._load_remote_item(item, reg, owner, host)
        if value:
            if item == self.FILE_BASE_PASSPHRASE:
                self.can_disk_cache_passphrases[(reg, owner, host)] = True
            if not content:
                path = self._get_cache_dir(reg, owner, host)
                self._dump_item(path, item, value)
                value = os.path.join(path, item)
            return value

        raise SuiteServiceFileError("Couldn't get %s" % item)
Ejemplo n.º 28
0
        try:
            title = self.get_suite_title(name, path=dir)
        except Exception, x:
            print >> sys.stderr, 'WARNING: an error occurred parsing the suite definition:\n  ', x
            print >> sys.stderr, "Registering the suite with temporary title 'SUITE PARSE ERROR'."
            print >> sys.stderr, "You can update the title later with 'cylc db refresh'.\n"
            title = "SUITE PARSE ERROR"

        title = title.split('\n')[0]  # use the first of multiple lines
        print 'REGISTER', name + ':', dir
        with open(os.path.join(self.dbpath, name), 'w') as file:
            file.write('path=' + dir + '\n')
            file.write('title=' + title + '\n')

        # create a new passphrase for the suite if necessary
        passphrase(name, user, get_hostname()).generate(dir)

    def get_suite_data(self, suite):
        suite = RegPath(suite).get()
        fpath = os.path.join(self.dbpath, suite)
        if not os.path.isfile(fpath):
            raise RegistrationError, "ERROR: Suite not found " + suite
        data = {}
        with open(fpath, 'r') as file:
            lines = file.readlines()
        count = 0
        for line in lines:
            count += 1
            line = line.rstrip()
            try:
                key, val = line.split('=')
Ejemplo n.º 29
0
    def add_std_options( self ):
        """Add standard options if they have not been overridden."""
        try:
            self.add_option( "--user",
                help="Other user account name. This results in "
                "command reinvocation on the remote account.",
                metavar="USER", default=user,
                action="store", dest="owner" )
        except OptionConflictError:
            pass

        try:
            self.add_option( "--host",
                help="Other host name. This results in "
                "command reinvocation on the remote account.",
                metavar="HOST", action="store", default=get_hostname(),
                dest="host" )
        except OptionConflictError:
            pass

        try:
            self.add_option( "-v", "--verbose",
                help="Verbose output mode.",
                action="store_true", default=False, dest="verbose" )
        except OptionConflictError:
            pass


        try:
            self.add_option( "--debug",
                help="Run suites in non-daemon mode, and show exception tracebacks.",
                action="store_true", default=False, dest="debug" )
        except OptionConflictError:
            pass


        try:
            self.add_option( "--db",
                help="Alternative suite registration database location, "
                "defaults to $HOME/.cylc/REGDB.",
                metavar="PATH", action="store", default=None, dest="db" )
        except OptionConflictError:
            pass


        if self.pyro:
            try:
                self.add_option( "--port",
                    help="Suite port number on the suite host. NOTE: this is retrieved "
                    "automatically if passwordless ssh is configured to the suite host.",
                    metavar="INT", action="store", default=None, dest="port" )
            except OptionConflictError:
                pass

            try:
                self.add_option( "--use-ssh",
                    help="Use ssh to re-invoke the command on the suite host.",
                    action="store_true", default=False, dest="use_ssh" )
            except OptionConflictError:
                pass

            try:
                self.add_option( "--no-login",
                    help="Do not use a login shell to run remote ssh commands. "
                    "The default is to use a login shell.",
                    action="store_false", default=True, dest="ssh_login" )
            except OptionConflictError:
                pass

            try:
                self.add_option( "--pyro-timeout", metavar='SEC',
                    help="Set a timeout for network connections "
                    "to the running suite. The default is no timeout. "
                    "For task messaging connections see "
                    "site/user config file documentation.",
                    action="store", default=None, dest="pyro_timeout" )
            except OptionConflictError:
                pass

            try:
                self.add_option("--print-uuid",
                    help="Print the client UUID to stderr. This can be matched "
                    "to information logged by the receiving suite daemon.",
                    action="store_true", default=False, dest="print_uuid")
            except OptionConflictError:
                pass

            try:
                self.add_option("--set-uuid", metavar="UUID",
                    help="Set the client UUID manually (e.g. from prior use of "
                    "--print-uuid). This can be used to log multiple commands "
                    "under the same UUID (but note that only the first [info] "
                    "command from the same client ID will be logged unless the "
                    "suite is running in debug mode).",
                    action="store", default=None, dest="set_uuid")
            except OptionConflictError:
                pass

            if not self.noforce:
                try:
                    self.add_option( "-f", "--force",
                        help="Do not ask for confirmation before acting. Note that "
                        "it is not necessary to use this option if interactive command "
                        "prompts have been disabled in the site/user config files.",
                        action="store_true", default=False, dest="force" )
                except OptionConflictError:
                    pass

        if self.jset:
            try:
                self.add_option( "-s", "--set", metavar="NAME=VALUE",
                    help="Set the value of a Jinja2 template variable in the suite "
                    "definition. This option can be used multiple times on the command "
                    "line.  WARNING: these settings do not persist across suite restarts; "
                    "they need to be set again on the \"cylc restart\" command line.",
                    action="append", default=[], dest="templatevars" )
            except OptionConflictError:
                pass

            try:
                self.add_option( "--set-file", metavar="FILE",
                    help="Set the value of Jinja2 template variables in the suite "
                    "definition from a file containing NAME=VALUE pairs (one per line). "
                    "WARNING: these settings do not persist across suite restarts; "
                    "they need to be set again on the \"cylc restart\" command line.",
                    action="store", default=None, dest="templatevars_file" )
            except OptionConflictError:
                pass

        if self.multitask:
            try:
                self.add_option( "-m", "--family",
                    help="Match members of named families rather than tasks.",
                    action="store_true", default=False, dest="is_family" )
            except OptionConflictError:
                pass
Ejemplo n.º 30
0
Archivo: owner.py Proyecto: kaday/cylc
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""In analogy with cylc.hostname.is_remote_host(), determine if a
username is "remote"."""

import os
import pwd
from cylc.suite_host import get_hostname

user = os.environ.get('USER', pwd.getpwuid(os.getuid()).pw_name)
host = get_hostname()
user_at_host = "%s@%s" % (user, host)


def is_remote_user(name):
    """Return True if name is different than the current username.
    Return False if name is None.
    """
    return name and name != user
Ejemplo n.º 31
0
def scan(host=None, db=None, timeout=None):
    """Scan ports, return a list of suites found: [(port, suite.identify())].

    Note that we could easily scan for a given suite+owner and return its
    port instead of reading port files, but this may not always be fast enough.
    """
    if host is None:
        host = get_hostname()
    base_port = GLOBAL_CFG.get(
        ['communication', 'base port'])
    last_port = base_port + GLOBAL_CFG.get(
        ['communication', 'maximum number of ports'])
    if timeout:
        timeout = float(timeout)
    else:
        timeout = None

    reg_db = RegistrationDB(db)
    results = []
    my_uuid = uuid4()
    host_for_anon = host
    if is_remote_host(host):
        host_for_anon = get_host_ip_by_name(host)  # IP reduces DNS traffic.
    for port in range(base_port, last_port):
        client = SuiteIdClientAnon(None, host=host_for_anon, port=port,
                                   my_uuid=my_uuid, timeout=timeout)
        try:
            result = (port, client.identify())
        except ConnectionError as exc:
            if cylc.flags.debug:
                traceback.print_exc()
            continue
        except Exception as exc:
            if cylc.flags.debug:
                traceback.print_exc()
            raise
        else:
            owner = result[1].get('owner')
            name = result[1].get('name')
            states = result[1].get('states', None)
            if cylc.flags.debug:
                print '   suite:', name, owner
            if states is None:
                # This suite keeps its state info private.
                # Try again with the passphrase if I have it.
                pphrase = reg_db.load_passphrase(name, owner, host)
                if pphrase:
                    client = SuiteIdClient(name, owner=owner, host=host,
                                           port=port, my_uuid=my_uuid,
                                           timeout=timeout)
                    try:
                        result = (port, client.identify())
                    except Exception:
                        # Nope (private suite, wrong passphrase).
                        if cylc.flags.debug:
                            print '    (wrong passphrase)'
                    else:
                        reg_db.cache_passphrase(
                            name, owner, host, pphrase)
                        if cylc.flags.debug:
                            print '    (got states with passphrase)'
        results.append(result)
    return results
Ejemplo n.º 32
0
        try:
            title = self.get_suite_title( name, path=dir )
        except Exception, x:
            print >> sys.stderr, 'WARNING: an error occurred parsing the suite definition:\n  ', x
            print >> sys.stderr, "Registering the suite with temporary title 'SUITE PARSE ERROR'."
            print >> sys.stderr, "You can update the title later with 'cylc db refresh'.\n"
            title = "SUITE PARSE ERROR"

        title = title.split('\n')[0] # use the first of multiple lines
        print 'REGISTER', name + ':', dir
        with open( os.path.join( self.dbpath, name ), 'w' ) as file:
            file.write( 'path=' + dir + '\n' )
            file.write( 'title=' + title + '\n' )

        # create a new passphrase for the suite if necessary
        passphrase(name,user,get_hostname()).generate(dir)

    def get_suite_data( self, suite ):
        suite = RegPath(suite).get()
        fpath = os.path.join( self.dbpath, suite )
        if not os.path.isfile( fpath ):
            raise RegistrationError, "ERROR: Suite not found " + suite
        data = {}
        with open( fpath, 'r' ) as file:
            lines = file.readlines()
        count = 0
        for line in lines:
            count += 1
            line = line.rstrip()
            try:
                key,val = line.split('=')
Ejemplo n.º 33
0
    def load_passphrase(self, suite, owner, host, cache_ok=True):
        """Search for passphrase file for suite, load and return content.

        "passphrase" file is searched from these locations in order:

        1/ For running task jobs:
           a/ $CYLC_SUITE_RUN_DIR then $CYLC_SUITE_DEF_PATH for remote jobs.
           b/ $CYLC_SUITE_DEF_PATH_ON_SUITE_HOST for local jobs or remote jobs
              with SSH messaging.

        2/ From memory cache, for passphrases of remote suites.
           Don't use if cache_ok=False.

        3/ For suite on local user@host. The suite definition directory, as
           registered. (Note: Previously, this needs to be the 1st location,
           else sub-suites load their parent suite's passphrase on start-up
           because the "cylc run" command runs in a parent suite task execution
           environment. This problem no longer exists becase on suite start up,
           the "load_passphrase_from_dir" method is called directly instead of
           through this method.)

        4/ Locations under $HOME/.cylc/ for remote suite control from accounts
           that do not actually need the suite definition directory to be
           installed (a/ is now preferred. b/ c/ d/ are for back compat):
           a/ $HOME/.cylc/passphrases/SUITE_OWNER@SUITE_HOST/SUITE_NAME/
           b/ $HOME/.cylc/SUITE_HOST/SUITE_OWNER/SUITE_NAME/
           c/ $HOME/.cylc/SUITE_HOST/SUITE_NAME/
           d/ $HOME/.cylc/SUITE_NAME/
           Don't use if cache_ok=False.

        5/ For remote suites, try locating the passphrase file from suite
           definition directory on remote owner@host via SSH.

        """
        self.can_disk_cache_passphrases[(suite, owner, host)] = False
        # (1 before 2 else sub-suites load their parent suite's
        # passphrase on start-up because the "cylc run" command runs in
        # a parent suite task execution environment).

        # 1/ Running tasks: suite run/def dir from the task job environment.
        # Test for presence of task execution environment of requested suite.
        if suite == os.getenv('CYLC_SUITE_NAME'):
            suite_host = os.getenv('CYLC_SUITE_HOST')
            suite_owner = os.getenv('CYLC_SUITE_OWNER')
            env_keys = []
            if is_remote_host(suite_host) or is_remote_user(suite_owner):
                # 2(i)/ Task messaging call on a remote account.
                # First look in the remote suite run directory than suite
                # definition directory ($CYLC_SUITE_DEF_PATH is modified
                # for remote tasks):
                env_keys = ['CYLC_SUITE_RUN_DIR', 'CYLC_SUITE_DEF_PATH']
            elif suite_host or suite_owner:
                # 2(ii)/ Task messaging call on the suite host account.

                # Could be a local task or a remote task with 'ssh
                # messaging = True'. In either case use
                # $CYLC_SUITE_DEF_PATH_ON_SUITE_HOST which never
                # changes, not $CYLC_SUITE_DEF_PATH which gets
                # modified for remote tasks as described above.
                env_keys = ['CYLC_SUITE_DEF_PATH_ON_SUITE_HOST']
            for env_key in env_keys:
                try:
                    return self.load_passphrase_from_dir(os.environ[env_key])
                except (KeyError, IOError, PassphraseError):
                    pass

        # 2/ From memory cache
        if owner is None:
            owner = USER
        if host is None:
            host = get_hostname()

        if cache_ok:
            try:
                return self.cached_passphrases[(suite, owner, host)]
            except KeyError:
                pass

        # 3/ Cylc commands with suite definition directory from local reg.
        if cache_ok or not is_remote_user(owner) and not is_remote_host(host):
            try:
                return self.load_passphrase_from_dir(self.get_suitedir(suite))
            except (IOError, PassphraseError, RegistrationError):
                pass

        # 4/ Other allowed locations, as documented above.
        # For remote control commands, host here will be fully
        # qualified or not depending on what's given on the command line.
        if cache_ok:
            short_host = host.split('.', 1)[0]
            prefix = os.path.expanduser(os.path.join('~', '.cylc'))
            paths = []
            for names in [
                    (prefix, self.PASSPHRASES_DIR_BASE,
                     owner + "@" + host, suite),
                    (prefix, self.PASSPHRASES_DIR_BASE,
                     owner + "@" + short_host, suite),
                    (prefix, host, owner, suite),
                    (prefix, short_host, owner, suite),
                    (prefix, host, suite),
                    (prefix, short_host, suite),
                    (prefix, suite)]:
                path = os.path.join(*names)
                if path not in paths:
                    try:
                        return self.load_passphrase_from_dir(path)
                    except (IOError, PassphraseError):
                        pass
                    paths.append(path)

        # 5/ Try SSH to remote host
        passphrase = self._load_passphrase_via_ssh(suite, owner, host)
        if passphrase:
            self.can_disk_cache_passphrases[(suite, owner, host)] = True
            return passphrase

        if passphrase is None and cylc.flags.debug:
            print >> sys.stderr, (
                'ERROR: passphrase for suite %s not found for %s@%s' % (
                    suite, owner, host))
Ejemplo n.º 34
0
def scan(host=get_hostname(), db=None, pyro_timeout=None, owner=user):
    """Scan ports, return a list of suites found: [(port, suite.identify())].

    Note that we could easily scan for a given suite+owner and return its
    port instead of reading port files, but this may not always be fast enough.
    """
    base_port = GLOBAL_CFG.get(['pyro', 'base port'])
    last_port = base_port + GLOBAL_CFG.get(['pyro', 'maximum number of ports'])
    if pyro_timeout:
        pyro_timeout = float(pyro_timeout)
    else:
        pyro_timeout = None

    results = []
    for port in range(base_port, last_port):
        try:
            proxy = get_proxy(host, port, pyro_timeout)
            conn_val = ConnValidator()
            conn_val.set_default_hash(SCAN_HASH)
            proxy._setNewConnectionValidator(conn_val)
            proxy._setIdentification((user, NO_PASSPHRASE))
            result = (port, proxy.identify())
        except Pyro.errors.ConnectionDeniedError as exc:
            if cylc.flags.debug:
                print '%s:%s (connection denied)' % (host, port)
            # Back-compat <= 6.4.1
            msg = '  Old daemon at %s:%s?' % (host, port)
            for pphrase in load_passphrases(db):
                try:
                    proxy = get_proxy(host, port, pyro_timeout)
                    proxy._setIdentification(pphrase)
                    info = proxy.id()
                    result = (port, {'name': info[0], 'owner': info[1]})
                except Pyro.errors.ConnectionDeniedError:
                    connected = False
                else:
                    connected = True
                    break
            if not connected:
                if cylc.flags.verbose:
                    print >> sys.stderr, msg, "- connection denied (%s)" % exc
                continue
            else:
                if cylc.flags.verbose:
                    print >> sys.stderr, msg, "- connected with passphrase"
        except (Pyro.errors.ProtocolError, Pyro.errors.NamingError) as exc:
            # No suite at this port.
            if cylc.flags.debug:
                print str(exc)
                print '%s:%s (no suite)' % (host, port)
            continue
        except Pyro.errors.TimeoutError as exc:
            # E.g. Ctrl-Z suspended suite - holds up port scanning!
            if cylc.flags.debug:
                print '%s:%s (connection timed out)' % (host, port)
            print >> sys.stderr, (
                'suite? owner?@%s:%s - connection timed out (%s)' % (
                    host, port, exc))
        except Exception as exc:
            if cylc.flags.debug:
                print str(exc)
                break
            else:
                print >> sys.stderr, str(exc)
        else:
            name = result[1].get('name')
            owner = result[1].get('owner')
            states = result[1].get('states', None)
            if cylc.flags.debug:
                print '   suite:', name, owner
            if states is None:
                # This suite keeps its state info private.
                # Try again with the passphrase if I have it.
                try:
                    pphrase = get_passphrase(name, owner, host, localdb(db))
                except PassphraseError:
                    if cylc.flags.debug:
                        print '    (no passphrase)'
                else:
                    try:
                        proxy = get_proxy(host, port, pyro_timeout)
                        conn_val = ConnValidator()
                        conn_val.set_default_hash(SCAN_HASH)
                        proxy._setNewConnectionValidator(conn_val)
                        proxy._setIdentification((user, pphrase))
                        result = (port, proxy.identify())
                    except Exception:
                        # Nope (private suite, wrong passphrase).
                        if cylc.flags.debug:
                            print '    (wrong passphrase)'
                    else:
                        if cylc.flags.debug:
                            print '    (got states with passphrase)'
        results.append(result)
    return results
Ejemplo n.º 35
0
    def _dump_certificate_and_key_to_dir(self, path, suite):
        """Dump SSL certificate to "ssl.cert" file in "path"."""
        try:
            from OpenSSL import crypto
        except ImportError:
            # OpenSSL not installed, so we can't use HTTPS anyway.
            return
        host = get_hostname()
        altnames = ["DNS:*", "DNS:%s" % host,
                    "IP:%s" % get_local_ip_address(host)]
        # Workaround for https://github.com/kennethreitz/requests/issues/2621
        altnames.append("DNS:%s" % get_local_ip_address(host))

        # Use suite name as the 'common name', but no more than 64 chars.
        cert_common_name = suite
        if len(suite) > 64:
            cert_common_name = suite[:61] + "..."

        # Create a private key.
        pkey_obj = crypto.PKey()
        pkey_obj.generate_key(crypto.TYPE_RSA, 2048)

        # Create a self-signed certificate.
        cert_obj = crypto.X509()
        cert_obj.get_subject().O = "Cylc"
        cert_obj.get_subject().CN = cert_common_name
        cert_obj.gmtime_adj_notBefore(0)
        cert_obj.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60)  # 10 years.
        cert_obj.set_issuer(cert_obj.get_subject())
        cert_obj.set_pubkey(pkey_obj)
        cert_obj.add_extensions([
            crypto.X509Extension(
                "subjectAltName", False, ", ".join(altnames)
            )
        ])
        cert_obj.sign(pkey_obj, 'sha256')

        mkdir_p(path)

        # Work in a user-read-write-only directory for guaranteed safety.
        from tempfile import mkdtemp
        work_dir = mkdtemp()
        pkey_file = os.path.join(work_dir, self.SSL_PRIVATE_KEY_FILE_BASE)
        cert_file = os.path.join(work_dir, self.SSL_CERTIFICATE_FILE_BASE)

        with open(pkey_file, "w") as file_handle:
            file_handle.write(
                crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey_obj))
        with open(cert_file, "w") as file_handle:
            file_handle.write(
                crypto.dump_certificate(crypto.FILETYPE_PEM, cert_obj))

        import stat
        os.chmod(pkey_file, stat.S_IRUSR)
        os.chmod(cert_file, stat.S_IRUSR)
        pkey_dest_file = os.path.join(path, self.SSL_PRIVATE_KEY_FILE_BASE)
        cert_dest_file = os.path.join(path, self.SSL_CERTIFICATE_FILE_BASE)
        import shutil
        shutil.copy(pkey_file, pkey_dest_file)
        shutil.copy(cert_file, cert_dest_file)
        shutil.rmtree(work_dir)
        if cylc.flags.verbose:
            print 'Generated suite SSL certificate: %s' % cert_dest_file
            print 'Generated suite SSL private key: %s' % pkey_dest_file
Ejemplo n.º 36
0
def scan(host=get_hostname(), db=None, pyro_timeout=None, owner=user):
    """Scan ports, return a list of suites found: [(port, suite.identify())].

    Note that we could easily scan for a given suite+owner and return its
    port instead of reading port files, but this may not always be fast enough.
    """
    base_port = GLOBAL_CFG.get(['pyro', 'base port'])
    last_port = base_port + GLOBAL_CFG.get(['pyro', 'maximum number of ports'])
    if pyro_timeout:
        pyro_timeout = float(pyro_timeout)
    else:
        pyro_timeout = None

    results = []
    for port in range(base_port, last_port):
        try:
            proxy = get_proxy(host, port, pyro_timeout)
            conn_val = ConnValidator()
            conn_val.set_default_hash(SCAN_HASH)
            proxy._setNewConnectionValidator(conn_val)
            proxy._setIdentification((user, NO_PASSPHRASE))
            result = (port, proxy.identify())
        except Pyro.errors.ConnectionDeniedError as exc:
            if cylc.flags.debug:
                print '%s:%s (connection denied)' % (host, port)
            # Back-compat <= 6.4.1
            msg = '  Old daemon at %s:%s?' % (host, port)
            for pphrase in load_passphrases(db):
                try:
                    proxy = get_proxy(host, port, pyro_timeout)
                    proxy._setIdentification(pphrase)
                    info = proxy.id()
                    result = (port, {'name': info[0], 'owner': info[1]})
                except Pyro.errors.ConnectionDeniedError:
                    connected = False
                else:
                    connected = True
                    break
            if not connected:
                if cylc.flags.verbose:
                    print >> sys.stderr, msg, "- connection denied (%s)" % exc
                continue
            else:
                if cylc.flags.verbose:
                    print >> sys.stderr, msg, "- connected with passphrase"
        except (Pyro.errors.ProtocolError, Pyro.errors.NamingError) as exc:
            # No suite at this port.
            if cylc.flags.debug:
                print str(exc)
                print '%s:%s (no suite)' % (host, port)
            continue
        except Pyro.errors.TimeoutError as exc:
            # E.g. Ctrl-Z suspended suite - holds up port scanning!
            if cylc.flags.debug:
                print '%s:%s (connection timed out)' % (host, port)
            print >> sys.stderr, (
                'suite? owner?@%s:%s - connection timed out (%s)' %
                (host, port, exc))
        except Exception as exc:
            if cylc.flags.debug:
                print str(exc)
                break
            else:
                print >> sys.stderr, str(exc)
        else:
            name = result[1].get('name')
            owner = result[1].get('owner')
            states = result[1].get('states', None)
            if cylc.flags.debug:
                print '   suite:', name, owner
            if states is None:
                # This suite keeps its state info private.
                # Try again with the passphrase if I have it.
                try:
                    pphrase = get_passphrase(name, owner, host, localdb(db))
                except PassphraseError:
                    if cylc.flags.debug:
                        print '    (no passphrase)'
                else:
                    try:
                        proxy = get_proxy(host, port, pyro_timeout)
                        conn_val = ConnValidator()
                        conn_val.set_default_hash(SCAN_HASH)
                        proxy._setNewConnectionValidator(conn_val)
                        proxy._setIdentification((user, pphrase))
                        result = (port, proxy.identify())
                    except Exception:
                        # Nope (private suite, wrong passphrase).
                        if cylc.flags.debug:
                            print '    (wrong passphrase)'
                    else:
                        if cylc.flags.debug:
                            print '    (got states with passphrase)'
        results.append(result)
    return results
Ejemplo n.º 37
0
 def __init__(self, suite, task_id, owner=user,
              host=get_hostname(), pyro_timeout=None, port=None):
     self.target_server_object = task_id
     super(TaskMessageClient, self).__init__(
         suite, owner, host, pyro_timeout, port)
Ejemplo n.º 38
0
 def __init__(self, suite, task_id, owner=USER,
              host=get_hostname(), timeout=None, port=None):
     self.task_id = task_id
     super(TaskMessageClient, self).__init__(suite, owner, host,
                                             port=port, timeout=timeout)
Ejemplo n.º 39
0
    def _set_uri(self):
        """Set Pyro URI.

        Determine host and port using content in port file, unless already
        specified.

        """
        if ((self.host is None or self.port is None)
                and 'CYLC_SUITE_RUN_DIR' in os.environ):
            # Looks like we are in a running task job, so we should be able to
            # use "cylc-suite-env" file under the suite running directory
            try:
                suite_env = CylcSuiteEnv.load(self.suite,
                                              os.environ['CYLC_SUITE_RUN_DIR'])
            except CylcSuiteEnvLoadError:
                if cylc.flags.debug:
                    traceback.print_exc()
            else:
                self.host = suite_env.suite_host
                self.port = suite_env.suite_port
                self.owner = suite_env.suite_owner

        if self.host is None or self.port is None:
            port_file_path = os.path.join(
                GLOBAL_CFG.get(['pyro', 'ports directory']), self.suite)
            if is_remote_host(self.host) or is_remote_user(self.owner):
                ssh_tmpl = str(
                    GLOBAL_CFG.get_host_item('remote shell template',
                                             self.host, self.owner))
                ssh_tmpl = ssh_tmpl.replace(' %s', '')
                user_at_host = ''
                if self.owner:
                    user_at_host = self.owner + '@'
                if self.host:
                    user_at_host += self.host
                else:
                    user_at_host += 'localhost'
                r_port_file_path = port_file_path.replace(
                    os.environ['HOME'], '$HOME')
                command = shlex.split(ssh_tmpl) + [
                    user_at_host, 'cat', r_port_file_path
                ]
                proc = Popen(command, stdout=PIPE, stderr=PIPE)
                out, err = proc.communicate()
                ret_code = proc.wait()
                if ret_code:
                    if cylc.flags.debug:
                        print >> sys.stderr, {
                            "code": ret_code,
                            "command": command,
                            "stdout": out,
                            "stderr": err
                        }
                    raise PortFileError(
                        "Port file '%s:%s' not found - suite not running?." %
                        (user_at_host, r_port_file_path))
            else:
                try:
                    out = open(port_file_path).read()
                except IOError:
                    raise PortFileError(
                        "Port file '%s' not found - suite not running?." %
                        (port_file_path))
            lines = out.splitlines()
            try:
                if self.port is None:
                    self.port = int(lines[0])
            except (IndexError, ValueError):
                raise PortFileError("ERROR, bad content in port file: %s" %
                                    port_file_path)
            if self.host is None:
                if len(lines) >= 2:
                    self.host = lines[1].strip()
                else:
                    self.host = get_hostname()

        # Qualify the obj name with user and suite name (unnecessary but
        # can't change it until we break back-compat with older daemons).
        self.uri = (
            'PYROLOC://%(host)s:%(port)s/%(owner)s.%(suite)s.%(target)s' % {
                "host": self.host,
                "port": self.port,
                "suite": self.suite,
                "owner": self.owner,
                "target": self.target_server_object
            })