def register(self, name, dir): name = RegPath(name).get() for suite in self.list_all_suites(): if name == suite: raise RegistrationError("ERROR: " + name + " is already registered.") elif suite.startswith(name + RegPath.delimiter): raise RegistrationError("ERROR: " + name + " is a registered group.") elif name.startswith(suite + RegPath.delimiter): # suite starts with, to some level, an existing suite name raise RegistrationError("ERROR: " + suite + " is a registered suite.") dir = dir.rstrip('/') # strip trailing '/' dir = re.sub('^\./', '', dir) # strip leading './' if not dir.startswith('/'): # On AIX on GPFS os.path.abspath(dir) returns the path with # full 'fileset' prefix. Manual use of $PWD to absolutize a # relative path gives a cleaner result. dir = os.path.join(os.environ['PWD'], dir) title = self.get_suite_title(name, path=dir) title = title.split('\n')[0] # use the first of multiple lines print 'REGISTER', name + ':', dir with open(os.path.join(self.dbpath, name), 'w') as file: file.write('path=' + dir + '\n') file.write('title=' + title + '\n') # create a new passphrase for the suite if necessary passphrase(name, user, get_hostname()).generate(dir)
def cache_passphrase(self, suite, owner, host, passphrase): """Cache and dump passphrase for a remote suite in standard location. Save passphrase to ~/.cylc/passphrases/owner@host/suite if possible. This is normally called on a successful authentication, and will cache the remote passphrase in memory as well. """ if owner is None: owner = USER if host is None: host = get_hostname() path = os.path.expanduser(os.path.join( '~', '.cylc', self.PASSPHRASES_DIR_BASE, owner + "@" + host, suite )) self.cached_passphrases[(suite, owner, host)] = passphrase # Dump to a file only for remote suites loaded via SSH. if self.can_disk_cache_passphrases.get((suite, owner, host)): # Although not desirable, failing to dump the passphrase to a file # is not disastrous. try: self._dump_passphrase_to_dir(path, passphrase) except (IOError, OSError): if cylc.flags.debug: import traceback traceback.print_exc()
def register( self, name, dir ): name = RegPath(name).get() for suite in self.list_all_suites(): if name == suite: raise RegistrationError, "ERROR: " + name + " is already registered." elif suite.startswith( name + RegPath.delimiter ): raise RegistrationError, "ERROR: " + name + " is a registered group." elif name.startswith( suite + RegPath.delimiter ): # suite starts with, to some level, an existing suite name raise RegistrationError, "ERROR: " + suite + " is a registered suite." dir = dir.rstrip( '/' ) # strip trailing '/' dir = re.sub( '^\./', '', dir ) # strip leading './' if not dir.startswith( '/' ): # On AIX on GPFS os.path.abspath(dir) returns the path with # full 'fileset' prefix. Manual use of $PWD to absolutize a # relative path gives a cleaner result. dir = os.path.join( os.environ['PWD'], dir ) title = self.get_suite_title(name, path=dir) title = title.split('\n')[0] # use the first of multiple lines print 'REGISTER', name + ':', dir with open( os.path.join( self.dbpath, name ), 'w' ) as file: file.write( 'path=' + dir + '\n' ) file.write( 'title=' + title + '\n' ) # create a new passphrase for the suite if necessary passphrase(name,user,get_hostname()).generate(dir)
def _load_port_file(self): """Load port, host, etc from port file.""" # GLOBAL_CFG is expensive to import, so only load on demand from cylc.cfgspec.globalcfg import GLOBAL_CFG port_file_path = os.path.join( GLOBAL_CFG.get(['communication', 'ports directory']), self.suite) out = "" if is_remote_host(self.host) or is_remote_user(self.owner): # Only load these modules on demand, as they may be expensive import shlex from subprocess import Popen, PIPE ssh_tmpl = str(GLOBAL_CFG.get_host_item( 'remote shell template', self.host, self.owner)) ssh_tmpl = ssh_tmpl.replace(' %s', '') user_at_host = '' if self.owner: user_at_host = self.owner + '@' if self.host: user_at_host += self.host else: user_at_host += 'localhost' r_port_file_path = port_file_path.replace( os.environ['HOME'], '$HOME') command = shlex.split(ssh_tmpl) + [ user_at_host, 'cat', r_port_file_path] proc = Popen(command, stdout=PIPE, stderr=PIPE) out, err = proc.communicate() ret_code = proc.wait() if ret_code: if cylc.flags.debug: print >> sys.stderr, { "code": ret_code, "command": command, "stdout": out, "stderr": err} if self.port is None: raise PortFileError( "Port file '%s:%s' not found - suite not running?." % (user_at_host, r_port_file_path)) else: try: out = open(port_file_path).read() except IOError: if self.port is None: raise PortFileError( "Port file '%s' not found - suite not running?." % (port_file_path)) lines = out.splitlines() if self.port is None: try: self.port = int(lines[0]) except (IndexError, ValueError): raise PortFileError( "ERROR, bad content in port file: %s" % port_file_path) if self.host is None: if len(lines) >= 2: self.host = lines[1].strip() else: self.host = get_hostname()
def __init__(self, suite, owner=None, host=None): self.suite = suite self.owner = owner self.host = host if self.owner is None: self.owner = user if self.host is None: self.host = get_hostname() self.location = None
def __init__(self, suite, task_id, owner=user, host=get_hostname(), pyro_timeout=None, port=None): self.target_server_object = task_id super(TaskMessageClient, self).__init__(suite, owner, host, pyro_timeout, port)
def _get_headers(self): """Return HTTP headers identifying the client.""" user_agent_string = ( "cylc/%s prog_name/%s uuid/%s" % ( CYLC_VERSION, self.prog_name, self.my_uuid ) ) auth_info = "%s@%s" % (USER, get_hostname()) return {"User-Agent": user_agent_string, "From": auth_info}
def _ws_init(service_inst, *args, **kwargs): """Start quick web service.""" # cherrypy.config["tools.encode.on"] = True # cherrypy.config["tools.encode.encoding"] = "utf-8" cherrypy.config["server.socket_host"] = "0.0.0.0" cherrypy.config["engine.autoreload.on"] = False if can_ssl(): cherrypy.config["server.ssl_module"] = "pyopenSSL" cherrypy.config["server.ssl_certificate"] = service_inst.cert cherrypy.config["server.ssl_private_key"] = service_inst.pkey else: sys.stderr.write("WARNING: no HTTPS support: cannot import OpenSSL\n") cherrypy.config["log.screen"] = None key = binascii.hexlify(os.urandom(16)) cherrypy.config.update( { "tools.auth_digest.on": True, "tools.auth_digest.realm": service_inst.suite, "tools.auth_digest.get_ha1": service_inst.get_ha1, "tools.auth_digest.key": key, "tools.auth_digest.algorithm": service_inst.hash_algorithm, } ) cherrypy.tools.connect_log = cherrypy.Tool("on_end_resource", service_inst.report_connection_if_denied) cherrypy.config["tools.connect_log.on"] = True host = get_hostname() service_inst.engine = cherrypy.engine for port in service_inst.ok_ports: my_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: my_socket.bind((host, port)) except socket.error: # Host busy. my_socket.close() continue my_socket.close() cherrypy.config["server.socket_port"] = port try: cherrypy.engine.start() cherrypy.engine.wait(cherrypy.engine.states.STARTED) if cherrypy.engine.state != cherrypy.engine.states.STARTED: continue except (socket.error, IOError): pass except: import traceback traceback.print_exc() else: service_inst.port = port return # We need to reinitialise the httpserver for each port attempt. cherrypy.server.httpserver = None raise Exception("No available ports")
def call_server_func(self, category, fname, **fargs): """Call server_object.fname(*fargs, **fargs).""" if self.host is None and self.port is not None: self.host = get_hostname() try: self._load_contact_info() except (IOError, ValueError, SuiteServiceFileError): raise ConnectionInfoError(self.suite) handle_proxies() payload = fargs.pop("payload", None) method = fargs.pop("method", self.METHOD) host = self.host if host == "localhost": host = get_hostname().split(".")[0] url = 'https://%s:%s/%s/%s' % (host, self.port, category, fname) if fargs: import urllib params = urllib.urlencode(fargs, doseq=True) url += "?" + params return self._get_data_from_url(url, payload, method=method)
def get(self, pfile=None, suitedir=None): ppfile = self.get_passphrase_file(pfile, suitedir) psf = open(ppfile, 'r') lines = psf.readlines() psf.close() if len(lines) != 1: raise PassphraseError, ('ERROR, invalid passphrase file: %s@%s:%s' % ( user, get_hostname(), ppfile)) # chomp trailing whitespace and newline self.passphrase = lines[0].strip() return self.passphrase
def mungeIdent(self, ident): """Receive (uuid, passphrase) from client. Encrypt the passphrase. Also pass client identification info to server for logging: (user, host, prog name). """ hash_ = self._get_hash() uuid, passphrase = ident prog_name = os.path.basename(sys.argv[0]) if passphrase is None: passphrase = NO_PASSPHRASE return (USER, get_hostname(), str(uuid), prog_name, hash_(passphrase).digest())
def call_server_func(self, category, fname, **fargs): """Call server_object.fname(*fargs, **fargs).""" if self.host is None and self.port is not None: self.host = get_hostname() if self.host is None or self.port is None: try: self._load_contact_info() except (IOError, ValueError, SuiteServiceFileError): raise ConnectionInfoError(self.suite) handle_proxies() payload = fargs.pop("payload", None) method = fargs.pop("method", self.METHOD) host = self.host if not self.host.split(".")[0].isdigit(): host = self.host.split(".")[0] if host == "localhost": host = get_hostname().split(".")[0] url = 'https://%s:%s/%s/%s' % (host, self.port, category, fname) if fargs: import urllib params = urllib.urlencode(fargs, doseq=True) url += "?" + params return self._get_data_from_url(url, payload, method=method)
def generate(self, dir): pfile = os.path.join(dir, 'passphrase') if os.path.isfile(pfile): try: self.get(pfile) return except PassphraseError: pass # Note: Perhaps a UUID might be better here? char_set = string.ascii_uppercase + string.ascii_lowercase + string.digits self.passphrase = ''.join(random.sample(char_set, 20)) mkdir_p(dir) f = open(pfile, 'w') f.write(self.passphrase) f.close() # set passphrase file permissions to owner-only os.chmod(pfile, 0600) if cylc.flags.verbose: print 'Generated suite passphrase: %s@%s:%s' % ( user, get_hostname(), pfile)
def load_passphrases(db): """Load all of the user's passphrases (back-compat for <= 6.4.1).""" global passphrases if passphrases: return passphrases # Find passphrases in all registered suite directories. reg = localdb(db) reg_suites = reg.get_list() for item in reg_suites: rg = item[0] di = item[1] try: p = passphrase(rg, user, get_hostname()).get(suitedir=di) except Exception, x: # Suite has no passphrase. if cylc.flags.debug: print >> sys.stderr, x else: passphrases.append(p)
def _create_ssl_pem_and_cert(self, path, reg): """Create ssl.pem and ssl.cert files for suite in path.""" try: from OpenSSL import crypto except ImportError: # OpenSSL not installed, so we can't use HTTPS anyway. return host = get_hostname() altnames = [ "DNS:*", "DNS:%s" % host, "IP:%s" % get_local_ip_address(host), # See https://github.com/kennethreitz/requests/issues/2621 "DNS:%s" % get_local_ip_address(host)] # Use suite name as the 'common name', but no more than 64 chars. cert_common_name = reg if len(reg) > 64: cert_common_name = reg[:61] + "..." # Create a private key. pkey_obj = crypto.PKey() pkey_obj.generate_key(crypto.TYPE_RSA, 2048) self._dump_item( path, self.FILE_BASE_SSL_PEM, crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey_obj)) # Create a self-signed certificate. cert_obj = crypto.X509() cert_obj.get_subject().O = "Cylc" cert_obj.get_subject().CN = cert_common_name cert_obj.gmtime_adj_notBefore(0) cert_obj.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) # 10 years. cert_obj.set_issuer(cert_obj.get_subject()) cert_obj.set_pubkey(pkey_obj) cert_obj.add_extensions([crypto.X509Extension( "subjectAltName", False, ", ".join(altnames))]) cert_obj.sign(pkey_obj, 'sha256') self._dump_item( path, self.FILE_BASE_SSL_CERT, crypto.dump_certificate(crypto.FILETYPE_PEM, cert_obj))
def cache_passphrase(self, reg, owner, host, value): """Cache and dump passphrase for a remote suite in standard location. Save passphrase to ~/.cylc/auth/owner@host/reg if possible. This is normally called on a successful authentication, and will cache the remote passphrase in memory as well. """ if owner is None: owner = USER if host is None: host = get_hostname() path = self._get_cache_dir(reg, owner, host) self.cache[self.FILE_BASE_PASSPHRASE][(reg, owner, host)] = value # Dump to a file only for remote suites loaded via SSH. if self.can_disk_cache_passphrases.get((reg, owner, host)): # Although not desirable, failing to dump the passphrase to a file # is not disastrous. try: self._dump_item(path, self.FILE_BASE_PASSPHRASE, value) except (IOError, OSError): if cylc.flags.debug: import traceback traceback.print_exc()
def _is_local_auth_ok(self, reg, owner, host): """Return True if it is OK to use local passphrase, ssl.* files. Use values in ~/cylc-run/REG/.service/contact to make a judgement. Cache results in self.can_use_load_auths. """ if (reg, owner, host) not in self.can_use_load_auths: if is_remote_user(owner) or is_remote_host(host): fname = os.path.join( self.get_suite_srv_dir(reg), self.FILE_BASE_CONTACT) data = {} try: for line in open(fname): key, value = ( [item.strip() for item in line.split("=", 1)]) data[key] = value except IOError, ValueError: # No contact file self.can_use_load_auths[(reg, owner, host)] = False else: # Contact file exists, check values match if owner is None: owner = USER if host is None: host = get_hostname() host_value = data.get(self.KEY_HOST, "") self.can_use_load_auths[(reg, owner, host)] = ( reg == data.get(self.KEY_NAME) and owner == data.get(self.KEY_OWNER) and ( host == host_value or host == host_value.split(".", 1)[0] # no domain ) ) else: self.can_use_load_auths[(reg, owner, host)] = True
# This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """In analogy with cylc.hostname.is_remote_host(), determine if a username is "remote".""" import os import pwd from cylc.suite_host import get_hostname user = os.environ.get('USER', pwd.getpwuid(os.getuid()).pw_name) host = get_hostname() user_at_host = "%s@%s" % (user, host) def is_remote_user(name): """Return True if name is different than the current username. Return False if name is None. """ return name and name != user
def get_passphrase_file(self, pfile=None, suitedir=None): """ Passphrase location, order of preference: 1/ The pfile argument - used for passphrase creation by "cylc register". 2/ The suite definition directory, because suites may be automatically installed (e.g. by Rose) to remote task hosts, and remote tasks know this location from their execution environment. Local user command invocations can use the suite registration database to find the suite definition directory. HOWEVER, remote user command invocations cannot do this even if the local and remote hosts share a common filesystem, because we cannot be sure if finding the expected suite registration implies a common filesystem or a different remote suite that happens to be registered under the same name. User accounts used for remote control must therefore install the passphrase in the secondary standard locations (below) or use the command line option to explicitly reveal the location. Remote tasks with 'ssh messaging = True' look first in the suite definition directory of the suite host, which they know through the variable CYLC_SUITE_DEF_PATH_ON_SUITE_HOST in the task execution environment. 3/ Secondary locations: (i) $HOME/.cylc/SUITE_HOST/SUITE_OWNER/SUITE_NAME/passphrase (ii) $HOME/.cylc/SUITE_HOST/SUITE_NAME/passphrase (iii) $HOME/.cylc/SUITE_NAME/passphrase These are more sensible locations for remote suite control from accounts that do not actually need the suite definition directory to be installed. """ # 1/ Explicit suite definition directory given on the command line. if pfile: if os.path.isdir(pfile): pfile = os.path.join(pfile, 'passphrase') if os.path.isfile(pfile): self.set_location(pfile) else: # If an explicit location is given, the file must exist. raise PassphraseError( 'ERROR, file not found on %s@%s: %s' % ( user, get_hostname(), pfile)) # 2/ Cylc commands with suite definition directory from local reg. if not self.location and suitedir: pfile = os.path.join(suitedir, 'passphrase') if os.path.isfile(pfile): self.set_location(pfile) # (2 before 3 else sub-suites load their parent suite's # passphrase on start-up because the "cylc run" command runs in # a parent suite task execution environment). # 3/ Running tasks: suite def dir from the task execution environment. if not self.location: try: # Test for presence of task execution environment suite_host = os.environ['CYLC_SUITE_HOST'] suite_owner = os.environ['CYLC_SUITE_OWNER'] except KeyError: # not called by a task pass else: # called by a task if is_remote_host(suite_host) or is_remote_user(suite_owner): # 2(i)/ Task messaging call on a remote account. # First look in the remote suite definition # directory ($CYLC_SUITE_DEF_PATH is modified for # remote tasks): try: pfile = os.path.join( os.environ['CYLC_SUITE_DEF_PATH'], 'passphrase') except KeyError: pass else: if os.path.isfile(pfile): self.set_location(pfile) else: # 2(ii)/ Task messaging call on the suite host account. # Could be a local task or a remote task with 'ssh # messaging = True'. In either case use # $CYLC_SUITE_DEF_PATH_ON_SUITE_HOST which never # changes, not $CYLC_SUITE_DEF_PATH which gets # modified for remote tasks as described above. try: pfile = os.path.join( os.environ['CYLC_SUITE_DEF_PATH_ON_SUITE_HOST'], 'passphrase') except KeyError: pass else: if os.path.isfile(pfile): self.set_location(pfile) # 4/ Other allowed locations, as documented above. if not self.location: locations = [] # For remote control commands, self.host here will be fully # qualified or not depending on what's given on the command line. short_host = re.sub('\..*', '', self.host) prefix = os.path.join(os.environ['HOME'], '.cylc') locations.append( os.path.join( prefix, self.host, self.owner, self.suite, 'passphrase')) if short_host != self.host: locations.append(os.path.join( prefix, short_host, self.owner, self.suite, 'passphrase')) locations.append( os.path.join(prefix, self.host, self.suite, 'passphrase')) if short_host != self.host: locations.append(os.path.join( prefix, short_host, self.suite, 'passphrase')) locations.append(os.path.join(prefix, self.suite, 'passphrase')) for pfile in locations: if os.path.isfile(pfile): self.set_location(pfile) break if not self.location: raise PassphraseError( 'ERROR: passphrase for suite %s not found on %s@%s' % ( self.suite, user, get_hostname())) return self.location
def __init__(self, suite, owner=user, host=get_hostname()): self.suite = suite self.owner = owner self.host = host self.location = None
def load_item(self, suite, owner, host, item="certificate", create_ok=False, cache_ok=False): """Load or create a passphrase, SSL certificate or a private key. SSL files are searched from these locations in order: 1/ For running task jobs: a/ $CYLC_SUITE_RUN_DIR then $CYLC_SUITE_DEF_PATH for remote jobs. b/ $CYLC_SUITE_DEF_PATH_ON_SUITE_HOST for local jobs or remote jobs with SSH messaging. 2/ (Passphrases only) From memory cache, for remote suite passphrases. Don't use if cache_ok=False. 3/ For suite on local user@host. The suite definition directory, as registered. (Note: Previously, this needs to be the 1st location, else sub-suites load their parent suite's passphrases, etc, on start-up because the "cylc run" command runs in a parent suite task execution environment. This problem no longer exists becase on suite start up, the "load_item_from_dir" method is called directly instead of through this method.) 4/ Location under $HOME/.cylc/ for remote suite control from accounts that do not actually need the suite definition directory to be installed: $HOME/.cylc/passphrases/SUITE_OWNER@SUITE_HOST/SUITE_NAME/ 5/ (SSL files only) If create_ok is specified, create the SSL file and then return it. 6/ For remote suites, try locating the file from the suite definition directory on remote owner@host via SSH. """ item_is_passphrase = False if item == "certificate": item = self.SSL_CERTIFICATE_FILE_BASE elif item == "private_key": item = self.SSL_PRIVATE_KEY_FILE_BASE elif item == "passphrase": item_is_passphrase = True self.can_disk_cache_passphrases[(suite, owner, host)] = False suite_host = os.getenv('CYLC_SUITE_HOST') suite_owner = os.getenv('CYLC_SUITE_OWNER') if suite == os.getenv('CYLC_SUITE_NAME'): env_keys = [] if is_remote_host(suite_host) or is_remote_user(suite_owner): # 1(a)/ Task messaging call on a remote account. # First look in the remote suite run directory than suite # definition directory ($CYLC_SUITE_DEF_PATH is modified # for remote tasks): env_keys = ['CYLC_SUITE_RUN_DIR', 'CYLC_SUITE_DEF_PATH'] elif suite_host or suite_owner: # 1(b)/ Task messaging call on the suite host account. # Could be a local task or a remote task with 'ssh # messaging = True'. In either case use # $CYLC_SUITE_DEF_PATH_ON_SUITE_HOST which never # changes, not $CYLC_SUITE_DEF_PATH which gets # modified for remote tasks as described above. env_keys = ['CYLC_SUITE_DEF_PATH_ON_SUITE_HOST'] for key in env_keys: try: return self.load_item_from_dir(os.environ[key], item) except (KeyError, IOError, PassphraseError): pass # 2/ From memory cache if cache_ok and item_is_passphrase: pass_owner = owner pass_host = host if pass_owner is None: pass_owner = USER if pass_host is None: pass_host = get_hostname() try: return self.cached_passphrases[(suite, pass_owner, pass_host)] except KeyError: pass # 3/ Cylc commands with suite definition directory from local reg. if cache_ok or not is_remote_user(owner) and not is_remote_host(host): try: return self.load_item_from_dir(self.get_suitedir(suite), item) except (IOError, PassphraseError, RegistrationError): pass # 4/ Other allowed locations, as documented above. prefix = os.path.expanduser(os.path.join('~', '.cylc')) if host is None: host = suite_host if (owner is not None and host is not None and (not item_is_passphrase or cache_ok)): prefix = os.path.expanduser(os.path.join('~', '.cylc')) paths = [] path_types = [(prefix, self.PASSPHRASES_DIR_BASE, owner + "@" + host, suite)] short_host = host.split('.', 1)[0] if short_host != host: path_types.append((prefix, self.PASSPHRASES_DIR_BASE, owner + "@" + short_host, suite)) for names in path_types: try: return self.load_item_from_dir(os.path.join(*names), item) except (IOError, PassphraseError): pass if create_ok and not item_is_passphrase: # 5/ Create the SSL file if it doesn't exist. return self._dump_certificate_and_key_to_dir( self.get_suitedir(suite), suite) load_dest_root = None if not item_is_passphrase: load_dest_root = os.path.join( prefix, self.PASSPHRASES_DIR_BASE, owner + "@" + host, suite) try: # 6/ Try ssh-ing to grab the files directly. content = self._load_item_via_ssh( item, suite, owner, host, dest_dir=load_dest_root) if content and item_is_passphrase: self.can_disk_cache_passphrases[(suite, owner, host)] = True return content except Exception as exc: import traceback traceback.print_exc() raise PassphraseError("Couldn't get %s" % item)
def _set_uri(self): """Set Pyro URI. Determine host and port using content in port file, unless already specified. """ if ((self.host is None or self.port is None) and 'CYLC_SUITE_RUN_DIR' in os.environ): # Looks like we are in a running task job, so we should be able to # use "cylc-suite-env" file under the suite running directory try: suite_env = CylcSuiteEnv.load( self.suite, os.environ['CYLC_SUITE_RUN_DIR']) except CylcSuiteEnvLoadError: if cylc.flags.debug: traceback.print_exc() else: self.host = suite_env.suite_host self.port = suite_env.suite_port self.owner = suite_env.suite_owner if self.host is None or self.port is None: port_file_path = os.path.join( GLOBAL_CFG.get(['pyro', 'ports directory']), self.suite) if is_remote_host(self.host) or is_remote_user(self.owner): ssh_tmpl = str(GLOBAL_CFG.get_host_item( 'remote shell template', self.host, self.owner)) ssh_tmpl = ssh_tmpl.replace(' %s', '') user_at_host = '' if self.owner: user_at_host = self.owner + '@' if self.host: user_at_host += self.host else: user_at_host += 'localhost' r_port_file_path = port_file_path.replace( os.environ['HOME'], '$HOME') command = shlex.split(ssh_tmpl) + [ user_at_host, 'cat', r_port_file_path] proc = Popen(command, stdout=PIPE, stderr=PIPE) out, err = proc.communicate() ret_code = proc.wait() if ret_code: if cylc.flags.debug: print >> sys.stderr, { "code": ret_code, "command": command, "stdout": out, "stderr": err} raise PortFileError( "Port file '%s:%s' not found - suite not running?." % (user_at_host, r_port_file_path)) else: try: out = open(port_file_path).read() except IOError: raise PortFileError( "Port file '%s' not found - suite not running?." % (port_file_path)) lines = out.splitlines() try: if self.port is None: self.port = int(lines[0]) except (IndexError, ValueError): raise PortFileError( "ERROR, bad content in port file: %s" % port_file_path) if self.host is None: if len(lines) >= 2: self.host = lines[1].strip() else: self.host = get_hostname() # Qualify the obj name with user and suite name (unnecessary but # can't change it until we break back-compat with older daemons). self.uri = ( 'PYROLOC://%(host)s:%(port)s/%(owner)s.%(suite)s.%(target)s' % { "host": self.host, "port": self.port, "suite": self.suite, "owner": self.owner, "target": self.target_server_object})
def set_location(self, pfile): if cylc.flags.debug: print '%s (%s@%s)' %(pfile, user, get_hostname()) self.location = pfile
def get_auth_item(self, item, reg, owner=None, host=None, content=False): """Locate/load passphrase, SSL private key, SSL certificate, etc. Return file name, or content of file if content=True is set. Files are searched from these locations in order: 1/ For running task jobs, service directory under: a/ $CYLC_SUITE_RUN_DIR for remote jobs. b/ $CYLC_SUITE_RUN_DIR_ON_SUITE_HOST for local jobs or remote jobs with SSH messaging. 2/ (Passphrases only) From memory cache, for remote suite passphrases. Don't use if content=False. 3/ For suite on local user@host. The suite service directory. 4/ Location under $HOME/.cylc/ for remote suite control from accounts that do not actually need the suite definition directory to be installed: $HOME/.cylc/auth/SUITE_OWNER@SUITE_HOST/SUITE_NAME/ 5/ For remote suites, try locating the file from the suite service directory on remote owner@host via SSH. If content=False, the value of the located file will be dumped under: $HOME/.cylc/auth/SUITE_OWNER@SUITE_HOST/SUITE_NAME/ """ if item not in [ self.FILE_BASE_SSL_CERT, self.FILE_BASE_SSL_PEM, self.FILE_BASE_PASSPHRASE, self.FILE_BASE_CONTACT]: raise ValueError("%s: item not recognised" % item) if item == self.FILE_BASE_PASSPHRASE: self.can_disk_cache_passphrases[(reg, owner, host)] = False suite_host = os.getenv('CYLC_SUITE_HOST') suite_owner = os.getenv('CYLC_SUITE_OWNER') if reg == os.getenv('CYLC_SUITE_NAME'): env_keys = [] if is_remote_host(suite_host) or is_remote_user(suite_owner): # 1(a)/ Task messaging call on a remote account. # Look in the remote suite run directory: env_keys = ['CYLC_SUITE_RUN_DIR'] elif suite_host or suite_owner: # 1(b)/ Task messaging call on the suite host account. # Could be a local task or a remote task with 'ssh # messaging = True'. In either case use # $CYLC_SUITE_RUN_DIR_ON_SUITE_HOST which never changes. env_keys = ['CYLC_SUITE_RUN_DIR_ON_SUITE_HOST'] for key in env_keys: path = os.path.join(os.environ[key], self.DIR_BASE_SRV) if content: value = self._load_local_item(item, path) else: value = self._locate_item(item, path) if value: return value # 2/ From memory cache if item in self.cache: my_owner = owner my_host = host if my_owner is None: my_owner = USER if my_host is None: my_host = get_hostname() try: return self.cache[item][(reg, my_owner, my_host)] except KeyError: pass # 3/ Local suite service directory if self._is_local_auth_ok(reg, owner, host): path = self.get_suite_srv_dir(reg) if content: value = self._load_local_item(item, path) else: value = self._locate_item(item, path) if value: return value # 4/ Disk cache for remote suites if host is None: host = suite_host if owner is not None and host is not None: paths = [self._get_cache_dir(reg, owner, host)] short_host = host.split('.', 1)[0] if short_host != host: paths.append(self._get_cache_dir(reg, owner, short_host)) for path in paths: if content: value = self._load_local_item(item, path) else: value = self._locate_item(item, path) if value: return value # 5/ Use SSH to load content from remote owner@host value = self._load_remote_item(item, reg, owner, host) if value: if item == self.FILE_BASE_PASSPHRASE: self.can_disk_cache_passphrases[(reg, owner, host)] = True if not content: path = self._get_cache_dir(reg, owner, host) self._dump_item(path, item, value) value = os.path.join(path, item) return value raise SuiteServiceFileError("Couldn't get %s" % item)
try: title = self.get_suite_title(name, path=dir) except Exception, x: print >> sys.stderr, 'WARNING: an error occurred parsing the suite definition:\n ', x print >> sys.stderr, "Registering the suite with temporary title 'SUITE PARSE ERROR'." print >> sys.stderr, "You can update the title later with 'cylc db refresh'.\n" title = "SUITE PARSE ERROR" title = title.split('\n')[0] # use the first of multiple lines print 'REGISTER', name + ':', dir with open(os.path.join(self.dbpath, name), 'w') as file: file.write('path=' + dir + '\n') file.write('title=' + title + '\n') # create a new passphrase for the suite if necessary passphrase(name, user, get_hostname()).generate(dir) def get_suite_data(self, suite): suite = RegPath(suite).get() fpath = os.path.join(self.dbpath, suite) if not os.path.isfile(fpath): raise RegistrationError, "ERROR: Suite not found " + suite data = {} with open(fpath, 'r') as file: lines = file.readlines() count = 0 for line in lines: count += 1 line = line.rstrip() try: key, val = line.split('=')
def add_std_options( self ): """Add standard options if they have not been overridden.""" try: self.add_option( "--user", help="Other user account name. This results in " "command reinvocation on the remote account.", metavar="USER", default=user, action="store", dest="owner" ) except OptionConflictError: pass try: self.add_option( "--host", help="Other host name. This results in " "command reinvocation on the remote account.", metavar="HOST", action="store", default=get_hostname(), dest="host" ) except OptionConflictError: pass try: self.add_option( "-v", "--verbose", help="Verbose output mode.", action="store_true", default=False, dest="verbose" ) except OptionConflictError: pass try: self.add_option( "--debug", help="Run suites in non-daemon mode, and show exception tracebacks.", action="store_true", default=False, dest="debug" ) except OptionConflictError: pass try: self.add_option( "--db", help="Alternative suite registration database location, " "defaults to $HOME/.cylc/REGDB.", metavar="PATH", action="store", default=None, dest="db" ) except OptionConflictError: pass if self.pyro: try: self.add_option( "--port", help="Suite port number on the suite host. NOTE: this is retrieved " "automatically if passwordless ssh is configured to the suite host.", metavar="INT", action="store", default=None, dest="port" ) except OptionConflictError: pass try: self.add_option( "--use-ssh", help="Use ssh to re-invoke the command on the suite host.", action="store_true", default=False, dest="use_ssh" ) except OptionConflictError: pass try: self.add_option( "--no-login", help="Do not use a login shell to run remote ssh commands. " "The default is to use a login shell.", action="store_false", default=True, dest="ssh_login" ) except OptionConflictError: pass try: self.add_option( "--pyro-timeout", metavar='SEC', help="Set a timeout for network connections " "to the running suite. The default is no timeout. " "For task messaging connections see " "site/user config file documentation.", action="store", default=None, dest="pyro_timeout" ) except OptionConflictError: pass try: self.add_option("--print-uuid", help="Print the client UUID to stderr. This can be matched " "to information logged by the receiving suite daemon.", action="store_true", default=False, dest="print_uuid") except OptionConflictError: pass try: self.add_option("--set-uuid", metavar="UUID", help="Set the client UUID manually (e.g. from prior use of " "--print-uuid). This can be used to log multiple commands " "under the same UUID (but note that only the first [info] " "command from the same client ID will be logged unless the " "suite is running in debug mode).", action="store", default=None, dest="set_uuid") except OptionConflictError: pass if not self.noforce: try: self.add_option( "-f", "--force", help="Do not ask for confirmation before acting. Note that " "it is not necessary to use this option if interactive command " "prompts have been disabled in the site/user config files.", action="store_true", default=False, dest="force" ) except OptionConflictError: pass if self.jset: try: self.add_option( "-s", "--set", metavar="NAME=VALUE", help="Set the value of a Jinja2 template variable in the suite " "definition. This option can be used multiple times on the command " "line. WARNING: these settings do not persist across suite restarts; " "they need to be set again on the \"cylc restart\" command line.", action="append", default=[], dest="templatevars" ) except OptionConflictError: pass try: self.add_option( "--set-file", metavar="FILE", help="Set the value of Jinja2 template variables in the suite " "definition from a file containing NAME=VALUE pairs (one per line). " "WARNING: these settings do not persist across suite restarts; " "they need to be set again on the \"cylc restart\" command line.", action="store", default=None, dest="templatevars_file" ) except OptionConflictError: pass if self.multitask: try: self.add_option( "-m", "--family", help="Match members of named families rather than tasks.", action="store_true", default=False, dest="is_family" ) except OptionConflictError: pass
# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """In analogy with cylc.hostname.is_remote_host(), determine if a username is "remote".""" import os import pwd from cylc.suite_host import get_hostname user = os.environ.get('USER', pwd.getpwuid(os.getuid()).pw_name) host = get_hostname() user_at_host = "%s@%s" % (user, host) def is_remote_user(name): """Return True if name is different than the current username. Return False if name is None. """ return name and name != user
def scan(host=None, db=None, timeout=None): """Scan ports, return a list of suites found: [(port, suite.identify())]. Note that we could easily scan for a given suite+owner and return its port instead of reading port files, but this may not always be fast enough. """ if host is None: host = get_hostname() base_port = GLOBAL_CFG.get( ['communication', 'base port']) last_port = base_port + GLOBAL_CFG.get( ['communication', 'maximum number of ports']) if timeout: timeout = float(timeout) else: timeout = None reg_db = RegistrationDB(db) results = [] my_uuid = uuid4() host_for_anon = host if is_remote_host(host): host_for_anon = get_host_ip_by_name(host) # IP reduces DNS traffic. for port in range(base_port, last_port): client = SuiteIdClientAnon(None, host=host_for_anon, port=port, my_uuid=my_uuid, timeout=timeout) try: result = (port, client.identify()) except ConnectionError as exc: if cylc.flags.debug: traceback.print_exc() continue except Exception as exc: if cylc.flags.debug: traceback.print_exc() raise else: owner = result[1].get('owner') name = result[1].get('name') states = result[1].get('states', None) if cylc.flags.debug: print ' suite:', name, owner if states is None: # This suite keeps its state info private. # Try again with the passphrase if I have it. pphrase = reg_db.load_passphrase(name, owner, host) if pphrase: client = SuiteIdClient(name, owner=owner, host=host, port=port, my_uuid=my_uuid, timeout=timeout) try: result = (port, client.identify()) except Exception: # Nope (private suite, wrong passphrase). if cylc.flags.debug: print ' (wrong passphrase)' else: reg_db.cache_passphrase( name, owner, host, pphrase) if cylc.flags.debug: print ' (got states with passphrase)' results.append(result) return results
try: title = self.get_suite_title( name, path=dir ) except Exception, x: print >> sys.stderr, 'WARNING: an error occurred parsing the suite definition:\n ', x print >> sys.stderr, "Registering the suite with temporary title 'SUITE PARSE ERROR'." print >> sys.stderr, "You can update the title later with 'cylc db refresh'.\n" title = "SUITE PARSE ERROR" title = title.split('\n')[0] # use the first of multiple lines print 'REGISTER', name + ':', dir with open( os.path.join( self.dbpath, name ), 'w' ) as file: file.write( 'path=' + dir + '\n' ) file.write( 'title=' + title + '\n' ) # create a new passphrase for the suite if necessary passphrase(name,user,get_hostname()).generate(dir) def get_suite_data( self, suite ): suite = RegPath(suite).get() fpath = os.path.join( self.dbpath, suite ) if not os.path.isfile( fpath ): raise RegistrationError, "ERROR: Suite not found " + suite data = {} with open( fpath, 'r' ) as file: lines = file.readlines() count = 0 for line in lines: count += 1 line = line.rstrip() try: key,val = line.split('=')
def load_passphrase(self, suite, owner, host, cache_ok=True): """Search for passphrase file for suite, load and return content. "passphrase" file is searched from these locations in order: 1/ For running task jobs: a/ $CYLC_SUITE_RUN_DIR then $CYLC_SUITE_DEF_PATH for remote jobs. b/ $CYLC_SUITE_DEF_PATH_ON_SUITE_HOST for local jobs or remote jobs with SSH messaging. 2/ From memory cache, for passphrases of remote suites. Don't use if cache_ok=False. 3/ For suite on local user@host. The suite definition directory, as registered. (Note: Previously, this needs to be the 1st location, else sub-suites load their parent suite's passphrase on start-up because the "cylc run" command runs in a parent suite task execution environment. This problem no longer exists becase on suite start up, the "load_passphrase_from_dir" method is called directly instead of through this method.) 4/ Locations under $HOME/.cylc/ for remote suite control from accounts that do not actually need the suite definition directory to be installed (a/ is now preferred. b/ c/ d/ are for back compat): a/ $HOME/.cylc/passphrases/SUITE_OWNER@SUITE_HOST/SUITE_NAME/ b/ $HOME/.cylc/SUITE_HOST/SUITE_OWNER/SUITE_NAME/ c/ $HOME/.cylc/SUITE_HOST/SUITE_NAME/ d/ $HOME/.cylc/SUITE_NAME/ Don't use if cache_ok=False. 5/ For remote suites, try locating the passphrase file from suite definition directory on remote owner@host via SSH. """ self.can_disk_cache_passphrases[(suite, owner, host)] = False # (1 before 2 else sub-suites load their parent suite's # passphrase on start-up because the "cylc run" command runs in # a parent suite task execution environment). # 1/ Running tasks: suite run/def dir from the task job environment. # Test for presence of task execution environment of requested suite. if suite == os.getenv('CYLC_SUITE_NAME'): suite_host = os.getenv('CYLC_SUITE_HOST') suite_owner = os.getenv('CYLC_SUITE_OWNER') env_keys = [] if is_remote_host(suite_host) or is_remote_user(suite_owner): # 2(i)/ Task messaging call on a remote account. # First look in the remote suite run directory than suite # definition directory ($CYLC_SUITE_DEF_PATH is modified # for remote tasks): env_keys = ['CYLC_SUITE_RUN_DIR', 'CYLC_SUITE_DEF_PATH'] elif suite_host or suite_owner: # 2(ii)/ Task messaging call on the suite host account. # Could be a local task or a remote task with 'ssh # messaging = True'. In either case use # $CYLC_SUITE_DEF_PATH_ON_SUITE_HOST which never # changes, not $CYLC_SUITE_DEF_PATH which gets # modified for remote tasks as described above. env_keys = ['CYLC_SUITE_DEF_PATH_ON_SUITE_HOST'] for env_key in env_keys: try: return self.load_passphrase_from_dir(os.environ[env_key]) except (KeyError, IOError, PassphraseError): pass # 2/ From memory cache if owner is None: owner = USER if host is None: host = get_hostname() if cache_ok: try: return self.cached_passphrases[(suite, owner, host)] except KeyError: pass # 3/ Cylc commands with suite definition directory from local reg. if cache_ok or not is_remote_user(owner) and not is_remote_host(host): try: return self.load_passphrase_from_dir(self.get_suitedir(suite)) except (IOError, PassphraseError, RegistrationError): pass # 4/ Other allowed locations, as documented above. # For remote control commands, host here will be fully # qualified or not depending on what's given on the command line. if cache_ok: short_host = host.split('.', 1)[0] prefix = os.path.expanduser(os.path.join('~', '.cylc')) paths = [] for names in [ (prefix, self.PASSPHRASES_DIR_BASE, owner + "@" + host, suite), (prefix, self.PASSPHRASES_DIR_BASE, owner + "@" + short_host, suite), (prefix, host, owner, suite), (prefix, short_host, owner, suite), (prefix, host, suite), (prefix, short_host, suite), (prefix, suite)]: path = os.path.join(*names) if path not in paths: try: return self.load_passphrase_from_dir(path) except (IOError, PassphraseError): pass paths.append(path) # 5/ Try SSH to remote host passphrase = self._load_passphrase_via_ssh(suite, owner, host) if passphrase: self.can_disk_cache_passphrases[(suite, owner, host)] = True return passphrase if passphrase is None and cylc.flags.debug: print >> sys.stderr, ( 'ERROR: passphrase for suite %s not found for %s@%s' % ( suite, owner, host))
def scan(host=get_hostname(), db=None, pyro_timeout=None, owner=user): """Scan ports, return a list of suites found: [(port, suite.identify())]. Note that we could easily scan for a given suite+owner and return its port instead of reading port files, but this may not always be fast enough. """ base_port = GLOBAL_CFG.get(['pyro', 'base port']) last_port = base_port + GLOBAL_CFG.get(['pyro', 'maximum number of ports']) if pyro_timeout: pyro_timeout = float(pyro_timeout) else: pyro_timeout = None results = [] for port in range(base_port, last_port): try: proxy = get_proxy(host, port, pyro_timeout) conn_val = ConnValidator() conn_val.set_default_hash(SCAN_HASH) proxy._setNewConnectionValidator(conn_val) proxy._setIdentification((user, NO_PASSPHRASE)) result = (port, proxy.identify()) except Pyro.errors.ConnectionDeniedError as exc: if cylc.flags.debug: print '%s:%s (connection denied)' % (host, port) # Back-compat <= 6.4.1 msg = ' Old daemon at %s:%s?' % (host, port) for pphrase in load_passphrases(db): try: proxy = get_proxy(host, port, pyro_timeout) proxy._setIdentification(pphrase) info = proxy.id() result = (port, {'name': info[0], 'owner': info[1]}) except Pyro.errors.ConnectionDeniedError: connected = False else: connected = True break if not connected: if cylc.flags.verbose: print >> sys.stderr, msg, "- connection denied (%s)" % exc continue else: if cylc.flags.verbose: print >> sys.stderr, msg, "- connected with passphrase" except (Pyro.errors.ProtocolError, Pyro.errors.NamingError) as exc: # No suite at this port. if cylc.flags.debug: print str(exc) print '%s:%s (no suite)' % (host, port) continue except Pyro.errors.TimeoutError as exc: # E.g. Ctrl-Z suspended suite - holds up port scanning! if cylc.flags.debug: print '%s:%s (connection timed out)' % (host, port) print >> sys.stderr, ( 'suite? owner?@%s:%s - connection timed out (%s)' % ( host, port, exc)) except Exception as exc: if cylc.flags.debug: print str(exc) break else: print >> sys.stderr, str(exc) else: name = result[1].get('name') owner = result[1].get('owner') states = result[1].get('states', None) if cylc.flags.debug: print ' suite:', name, owner if states is None: # This suite keeps its state info private. # Try again with the passphrase if I have it. try: pphrase = get_passphrase(name, owner, host, localdb(db)) except PassphraseError: if cylc.flags.debug: print ' (no passphrase)' else: try: proxy = get_proxy(host, port, pyro_timeout) conn_val = ConnValidator() conn_val.set_default_hash(SCAN_HASH) proxy._setNewConnectionValidator(conn_val) proxy._setIdentification((user, pphrase)) result = (port, proxy.identify()) except Exception: # Nope (private suite, wrong passphrase). if cylc.flags.debug: print ' (wrong passphrase)' else: if cylc.flags.debug: print ' (got states with passphrase)' results.append(result) return results
def _dump_certificate_and_key_to_dir(self, path, suite): """Dump SSL certificate to "ssl.cert" file in "path".""" try: from OpenSSL import crypto except ImportError: # OpenSSL not installed, so we can't use HTTPS anyway. return host = get_hostname() altnames = ["DNS:*", "DNS:%s" % host, "IP:%s" % get_local_ip_address(host)] # Workaround for https://github.com/kennethreitz/requests/issues/2621 altnames.append("DNS:%s" % get_local_ip_address(host)) # Use suite name as the 'common name', but no more than 64 chars. cert_common_name = suite if len(suite) > 64: cert_common_name = suite[:61] + "..." # Create a private key. pkey_obj = crypto.PKey() pkey_obj.generate_key(crypto.TYPE_RSA, 2048) # Create a self-signed certificate. cert_obj = crypto.X509() cert_obj.get_subject().O = "Cylc" cert_obj.get_subject().CN = cert_common_name cert_obj.gmtime_adj_notBefore(0) cert_obj.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) # 10 years. cert_obj.set_issuer(cert_obj.get_subject()) cert_obj.set_pubkey(pkey_obj) cert_obj.add_extensions([ crypto.X509Extension( "subjectAltName", False, ", ".join(altnames) ) ]) cert_obj.sign(pkey_obj, 'sha256') mkdir_p(path) # Work in a user-read-write-only directory for guaranteed safety. from tempfile import mkdtemp work_dir = mkdtemp() pkey_file = os.path.join(work_dir, self.SSL_PRIVATE_KEY_FILE_BASE) cert_file = os.path.join(work_dir, self.SSL_CERTIFICATE_FILE_BASE) with open(pkey_file, "w") as file_handle: file_handle.write( crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey_obj)) with open(cert_file, "w") as file_handle: file_handle.write( crypto.dump_certificate(crypto.FILETYPE_PEM, cert_obj)) import stat os.chmod(pkey_file, stat.S_IRUSR) os.chmod(cert_file, stat.S_IRUSR) pkey_dest_file = os.path.join(path, self.SSL_PRIVATE_KEY_FILE_BASE) cert_dest_file = os.path.join(path, self.SSL_CERTIFICATE_FILE_BASE) import shutil shutil.copy(pkey_file, pkey_dest_file) shutil.copy(cert_file, cert_dest_file) shutil.rmtree(work_dir) if cylc.flags.verbose: print 'Generated suite SSL certificate: %s' % cert_dest_file print 'Generated suite SSL private key: %s' % pkey_dest_file
def scan(host=get_hostname(), db=None, pyro_timeout=None, owner=user): """Scan ports, return a list of suites found: [(port, suite.identify())]. Note that we could easily scan for a given suite+owner and return its port instead of reading port files, but this may not always be fast enough. """ base_port = GLOBAL_CFG.get(['pyro', 'base port']) last_port = base_port + GLOBAL_CFG.get(['pyro', 'maximum number of ports']) if pyro_timeout: pyro_timeout = float(pyro_timeout) else: pyro_timeout = None results = [] for port in range(base_port, last_port): try: proxy = get_proxy(host, port, pyro_timeout) conn_val = ConnValidator() conn_val.set_default_hash(SCAN_HASH) proxy._setNewConnectionValidator(conn_val) proxy._setIdentification((user, NO_PASSPHRASE)) result = (port, proxy.identify()) except Pyro.errors.ConnectionDeniedError as exc: if cylc.flags.debug: print '%s:%s (connection denied)' % (host, port) # Back-compat <= 6.4.1 msg = ' Old daemon at %s:%s?' % (host, port) for pphrase in load_passphrases(db): try: proxy = get_proxy(host, port, pyro_timeout) proxy._setIdentification(pphrase) info = proxy.id() result = (port, {'name': info[0], 'owner': info[1]}) except Pyro.errors.ConnectionDeniedError: connected = False else: connected = True break if not connected: if cylc.flags.verbose: print >> sys.stderr, msg, "- connection denied (%s)" % exc continue else: if cylc.flags.verbose: print >> sys.stderr, msg, "- connected with passphrase" except (Pyro.errors.ProtocolError, Pyro.errors.NamingError) as exc: # No suite at this port. if cylc.flags.debug: print str(exc) print '%s:%s (no suite)' % (host, port) continue except Pyro.errors.TimeoutError as exc: # E.g. Ctrl-Z suspended suite - holds up port scanning! if cylc.flags.debug: print '%s:%s (connection timed out)' % (host, port) print >> sys.stderr, ( 'suite? owner?@%s:%s - connection timed out (%s)' % (host, port, exc)) except Exception as exc: if cylc.flags.debug: print str(exc) break else: print >> sys.stderr, str(exc) else: name = result[1].get('name') owner = result[1].get('owner') states = result[1].get('states', None) if cylc.flags.debug: print ' suite:', name, owner if states is None: # This suite keeps its state info private. # Try again with the passphrase if I have it. try: pphrase = get_passphrase(name, owner, host, localdb(db)) except PassphraseError: if cylc.flags.debug: print ' (no passphrase)' else: try: proxy = get_proxy(host, port, pyro_timeout) conn_val = ConnValidator() conn_val.set_default_hash(SCAN_HASH) proxy._setNewConnectionValidator(conn_val) proxy._setIdentification((user, pphrase)) result = (port, proxy.identify()) except Exception: # Nope (private suite, wrong passphrase). if cylc.flags.debug: print ' (wrong passphrase)' else: if cylc.flags.debug: print ' (got states with passphrase)' results.append(result) return results
def __init__(self, suite, task_id, owner=user, host=get_hostname(), pyro_timeout=None, port=None): self.target_server_object = task_id super(TaskMessageClient, self).__init__( suite, owner, host, pyro_timeout, port)
def __init__(self, suite, task_id, owner=USER, host=get_hostname(), timeout=None, port=None): self.task_id = task_id super(TaskMessageClient, self).__init__(suite, owner, host, port=port, timeout=timeout)
def _set_uri(self): """Set Pyro URI. Determine host and port using content in port file, unless already specified. """ if ((self.host is None or self.port is None) and 'CYLC_SUITE_RUN_DIR' in os.environ): # Looks like we are in a running task job, so we should be able to # use "cylc-suite-env" file under the suite running directory try: suite_env = CylcSuiteEnv.load(self.suite, os.environ['CYLC_SUITE_RUN_DIR']) except CylcSuiteEnvLoadError: if cylc.flags.debug: traceback.print_exc() else: self.host = suite_env.suite_host self.port = suite_env.suite_port self.owner = suite_env.suite_owner if self.host is None or self.port is None: port_file_path = os.path.join( GLOBAL_CFG.get(['pyro', 'ports directory']), self.suite) if is_remote_host(self.host) or is_remote_user(self.owner): ssh_tmpl = str( GLOBAL_CFG.get_host_item('remote shell template', self.host, self.owner)) ssh_tmpl = ssh_tmpl.replace(' %s', '') user_at_host = '' if self.owner: user_at_host = self.owner + '@' if self.host: user_at_host += self.host else: user_at_host += 'localhost' r_port_file_path = port_file_path.replace( os.environ['HOME'], '$HOME') command = shlex.split(ssh_tmpl) + [ user_at_host, 'cat', r_port_file_path ] proc = Popen(command, stdout=PIPE, stderr=PIPE) out, err = proc.communicate() ret_code = proc.wait() if ret_code: if cylc.flags.debug: print >> sys.stderr, { "code": ret_code, "command": command, "stdout": out, "stderr": err } raise PortFileError( "Port file '%s:%s' not found - suite not running?." % (user_at_host, r_port_file_path)) else: try: out = open(port_file_path).read() except IOError: raise PortFileError( "Port file '%s' not found - suite not running?." % (port_file_path)) lines = out.splitlines() try: if self.port is None: self.port = int(lines[0]) except (IndexError, ValueError): raise PortFileError("ERROR, bad content in port file: %s" % port_file_path) if self.host is None: if len(lines) >= 2: self.host = lines[1].strip() else: self.host = get_hostname() # Qualify the obj name with user and suite name (unnecessary but # can't change it until we break back-compat with older daemons). self.uri = ( 'PYROLOC://%(host)s:%(port)s/%(owner)s.%(suite)s.%(target)s' % { "host": self.host, "port": self.port, "suite": self.suite, "owner": self.owner, "target": self.target_server_object })