def _scan1_impl(conn, reg_db_path, timeout, my_uuid): """Connect to host:port to get suite identify.""" while True: if not conn.poll(SLEEP_INTERVAL): continue item = conn.recv() if item == MSG_QUIT: break host, port = item host_anon = host if is_remote_host(host): host_anon = get_host_ip_by_name(host) # IP reduces DNS traffic client = SuiteIdClientAnon( None, host=host_anon, port=port, my_uuid=my_uuid, timeout=timeout) try: result = client.identify() except ConnectionTimeout as exc: conn.send((host, port, MSG_TIMEOUT)) except ConnectionError as exc: conn.send((host, port, None)) else: owner = result.get('owner') name = result.get('name') states = result.get('states', None) if cylc.flags.debug: print >> sys.stderr, ' suite:', name, owner if states is None: # This suite keeps its state info private. # Try again with the passphrase if I have it. reg_db = RegistrationDB(reg_db_path) pphrase = reg_db.load_passphrase(name, owner, host) if pphrase: client = SuiteIdClient( name, owner=owner, host=host, port=port, my_uuid=my_uuid, timeout=timeout) try: result = client.identify() except Exception: # Nope (private suite, wrong passphrase). if cylc.flags.debug: print >> sys.stderr, ' (wrong passphrase)' else: reg_db.cache_passphrase(name, owner, host, pphrase) if cylc.flags.debug: print >> sys.stderr, ( ' (got states with passphrase)') conn.send((host, port, result)) conn.close()
def __init__(self, suite, suite_dir): # Suite only needed for back-compat with old clients (see below): self.suite = suite # Figure out the ports we are allowed to use. base_port = GLOBAL_CFG.get(["communication", "base port"]) max_ports = GLOBAL_CFG.get(["communication", "maximum number of ports"]) self.ok_ports = range(int(base_port), int(base_port) + int(max_ports)) random.shuffle(self.ok_ports) comms_options = GLOBAL_CFG.get(["communication", "options"]) # HTTP Digest Auth uses MD5 - pretty secure in this use case. # Extending it with extra algorithms is allowed, but won't be # supported by most browsers. requests and urllib2 are OK though. self.hash_algorithm = "MD5" if "SHA1" in comms_options: # Note 'SHA' rather than 'SHA1'. self.hash_algorithm = "SHA" self.reg_db = RegistrationDB() try: self.cert = self.reg_db.load_item(suite, USER, None, "certificate", create_ok=True) self.pkey = self.reg_db.load_item(suite, USER, None, "private_key", create_ok=True) except PassphraseError: # No OpenSSL installed. self.cert = None self.pkey = None self.suite = suite passphrase = self.reg_db.load_passphrase(suite, USER, None) userpassdict = {"cylc": passphrase, "anon": NO_PASSPHRASE} get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(userpassdict, algorithm=self.hash_algorithm) self.get_ha1 = get_ha1 del passphrase del userpassdict self.client_reporter = CommsClientReporter.get_inst() self.start()
def start_updater(self, filtr=None): db = RegistrationDB(self.db) # self.db_button.set_label("_Local/Central DB") if self.updater: self.updater.quit = True # does this take effect? self.updater = db_updater(self.regd_treestore, db, filtr, self.pyro_timeout) self.updater.start()
def _getdef(self, arg, options): """Return (suite_name, suite_rc_path). If arg is a registered suite, suite name is the registered suite name. If arg is a directory, suite name is the name of the directory. If arg is a file, suite name is the name of its container directory. """ reg_db = RegistrationDB(options.db) try: path = reg_db.get_suiterc(arg) name = arg except (IllegalRegPathError, RegistrationError): arg = os.path.abspath(arg) if os.path.isdir(arg): path = os.path.join(arg, 'suite.rc') name = os.path.basename(arg) else: path = arg name = os.path.basename(os.path.dirname(arg)) return name, path
def _getdef(self, arg, options): """Return (suite_name, suite_rc_path). If arg is a registered suite, suite name is the registered suite name. If arg is a directory, suite name is the name of the directory. If arg is a file, suite name is the name of its container directory. """ reg_db = RegistrationDB(options.db) try: path = reg_db.get_suiterc(arg) name = arg except (IllegalRegPathError, RegistrationError): arg = os.path.abspath(arg) if os.path.isdir(arg): path = os.path.join(arg, "suite.rc") name = os.path.basename(arg) else: path = arg name = os.path.basename(os.path.dirname(arg)) return name, path
def __init__(self, suite, owner=USER, host=None, timeout=None, port=None, db=None, my_uuid=None, print_uuid=False): self.suite = suite self.host = host self.owner = owner if timeout is not None: timeout = float(timeout) self.timeout = timeout self.port = port self.my_uuid = my_uuid or uuid4() if print_uuid: print >> sys.stderr, '%s' % self.my_uuid self.reg_db = RegistrationDB(db) self.prog_name = os.path.basename(sys.argv[0])
def __init__(self, suite, owner=USER, host=None, pyro_timeout=None, port=None, db=None, my_uuid=None, print_uuid=False): self.suite = suite self.host = host self.owner = owner if pyro_timeout is not None: pyro_timeout = float(pyro_timeout) self.pyro_timeout = pyro_timeout self.port = port self.pyro_proxy = None self.my_uuid = my_uuid or uuid4() self.uri = None if print_uuid: print >> sys.stderr, '%s' % self.my_uuid self.reg_db = RegistrationDB(db) self.pphrase = None
def __init__(self, suite, suite_dir): # Suite only needed for back-compat with old clients (see below): self.suite = suite Pyro.config.PYRO_MULTITHREADED = 1 # Use dns names instead of fixed ip addresses from /etc/hosts # (see the Userguide "Networking Issues" section). Pyro.config.PYRO_DNS_URI = True # Base Pyro socket number. Pyro.config.PYRO_PORT = GLOBAL_CFG.get(['pyro', 'base port']) # Max number of sockets starting at base. Pyro.config.PYRO_PORT_RANGE = GLOBAL_CFG.get( ['pyro', 'maximum number of ports']) Pyro.core.initServer() self.daemon = Pyro.core.Daemon() cval = ConnValidator() self.daemon.setNewConnectionValidator(cval) cval.set_pphrase(RegistrationDB.load_passphrase_from_dir(suite_dir))
class BaseCommsClient(object): """Base class for client-side suite object interfaces.""" ACCESS_DESCRIPTION = 'private' METHOD = 'POST' METHOD_POST = 'POST' METHOD_GET = 'GET' def __init__(self, suite, owner=USER, host=None, timeout=None, port=None, db=None, my_uuid=None, print_uuid=False): self.suite = suite self.host = host self.owner = owner if timeout is not None: timeout = float(timeout) self.timeout = timeout self.port = port self.my_uuid = my_uuid or uuid4() if print_uuid: print >> sys.stderr, '%s' % self.my_uuid self.reg_db = RegistrationDB(db) self.prog_name = os.path.basename(sys.argv[0]) def call_server_func(self, category, fname, **fargs): """Call server_object.fname(*fargs, **fargs).""" if self.host is None or self.port is None: self._load_contact_info() handle_proxies() payload = fargs.pop("payload", None) method = fargs.pop("method", self.METHOD) host = self.host if not self.host.split(".")[0].isdigit(): host = self.host.split(".")[0] if host == "localhost": host = get_hostname().split(".")[0] url = 'https://%s:%s/%s/%s' % ( host, self.port, category, fname ) if fargs: import urllib params = urllib.urlencode(fargs, doseq=True) url += "?" + params return self.get_data_from_url(url, payload, method=method) def get_data_from_url(self, url, json_data, method=None): requests_ok = True try: import requests except ImportError: requests_ok = False else: version = [int(_) for _ in requests.__version__.split(".")] if version < [2, 4, 2]: requests_ok = False if requests_ok: return self.get_data_from_url_with_requests( url, json_data, method=method) return self.get_data_from_url_with_urllib2( url, json_data, method=method) def get_data_from_url_with_requests(self, url, json_data, method=None): import requests username, password = self._get_auth() auth = requests.auth.HTTPDigestAuth(username, password) if not hasattr(self, "session"): self.session = requests.Session() if method is None: method = self.METHOD if method == self.METHOD_POST: session_method = self.session.post else: session_method = self.session.get try: ret = session_method( url, json=json_data, verify=self._get_verify(), proxies={}, headers=self._get_headers(), auth=auth, timeout=self.timeout ) except requests.exceptions.SSLError as exc: if "unknown protocol" in str(exc) and url.startswith("https:"): # Server is using http rather than https, for some reason. sys.stderr.write(WARNING_NO_HTTPS_SUPPORT.format(exc)) return self.get_data_from_url_with_requests( url.replace("https:", "http:", 1), json_data) if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) except requests.exceptions.Timeout as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionTimeout(url, exc) except requests.exceptions.RequestException as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) if ret.status_code == 401: raise ConnectionDeniedError(url, self.prog_name, self.ACCESS_DESCRIPTION) if ret.status_code >= 400: from cylc.network.https.util import get_exception_from_html exception_text = get_exception_from_html(ret.text) if exception_text: sys.stderr.write(exception_text) else: sys.stderr.write(ret.text) try: ret.raise_for_status() except requests.exceptions.HTTPError as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) try: return ret.json() except ValueError: return ret.text def get_data_from_url_with_urllib2(self, url, json_data, method=None): import json import urllib2 import ssl if hasattr(ssl, '_create_unverified_context'): ssl._create_default_https_context = ssl._create_unverified_context if method is None: method = self.METHOD orig_json_data = json_data username, password = self._get_auth() auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() auth_manager.add_password(None, url, username, password) auth = urllib2.HTTPDigestAuthHandler(auth_manager) opener = urllib2.build_opener(auth, urllib2.HTTPSHandler()) headers_list = self._get_headers().items() if json_data: json_data = json.dumps(json_data) headers_list.append(('Accept', 'application/json')) json_headers = {'Content-Type': 'application/json', 'Content-Length': len(json_data)} else: json_data = None json_headers = {'Content-Length': 0} opener.addheaders = headers_list req = urllib2.Request(url, json_data, json_headers) # This is an unpleasant monkey patch, but there isn't an alternative. # urllib2 uses POST iff there is a data payload, but that is not the # correct criterion. The difference is basically that POST changes # server state and GET doesn't. req.get_method = lambda: method try: response = opener.open(req, timeout=self.timeout) except urllib2.URLError as exc: if "unknown protocol" in str(exc) and url.startswith("https:"): # Server is using http rather than https, for some reason. sys.stderr.write(WARNING_NO_HTTPS_SUPPORT.format(exc)) return self.get_data_from_url_with_urllib2( url.replace("https:", "http:", 1), orig_json_data) if cylc.flags.debug: import traceback traceback.print_exc() if "timed out" in str(exc): raise ConnectionTimeout(url, exc) else: raise ConnectionError(url, exc) except Exception as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) if response.getcode() == 401: raise ConnectionDeniedError(url, self.prog_name, self.ACCESS_DESCRIPTION) response_text = response.read() if response.getcode() >= 400: from cylc.network.https.util import get_exception_from_html exception_text = get_exception_from_html(response_text) if exception_text: sys.stderr.write(exception_text) else: sys.stderr.write(response_text) raise ConnectionError(url, "%s HTTP return code" % response.getcode()) try: return json.loads(response_text) except ValueError: return response_text def _get_auth(self): """Return a user/password Digest Auth.""" self.pphrase = self.reg_db.load_passphrase( self.suite, self.owner, self.host) if self.pphrase: self.reg_db.cache_passphrase( self.suite, self.owner, self.host, self.pphrase) if self.pphrase is None: return 'anon', NO_PASSPHRASE return 'cylc', self.pphrase def _get_headers(self): """Return HTTP headers identifying the client.""" user_agent_string = ( "cylc/%s prog_name/%s uuid/%s" % ( CYLC_VERSION, self.prog_name, self.my_uuid ) ) auth_info = "%s@%s" % (USER, get_hostname()) return {"User-Agent": user_agent_string, "From": auth_info} def _get_verify(self): """Return the server certificate if possible.""" if not hasattr(self, "server_cert"): try: self.server_cert = self.reg_db.load_item( self.suite, self.owner, self.host, "certificate") except PassphraseError: return False return self.server_cert def _load_contact_info(self): """Obtain URL info. Determine host and port using content in port file, unless already specified. """ if self.host and self.port: return if 'CYLC_SUITE_RUN_DIR' in os.environ: # Looks like we are in a running task job, so we should be able to # use "cylc-suite-env" file under the suite running directory try: suite_env = CylcSuiteEnv.load( self.suite, os.environ['CYLC_SUITE_RUN_DIR']) except CylcSuiteEnvLoadError: if cylc.flags.debug: import traceback traceback.print_exc() else: self.host = suite_env.suite_host self.port = suite_env.suite_port self.owner = suite_env.suite_owner if self.host is None or self.port is None: self._load_port_file() def _load_port_file(self): """Load port, host, etc from port file.""" # GLOBAL_CFG is expensive to import, so only load on demand from cylc.cfgspec.globalcfg import GLOBAL_CFG port_file_path = os.path.join( GLOBAL_CFG.get(['communication', 'ports directory']), self.suite) out = "" if is_remote_host(self.host) or is_remote_user(self.owner): # Only load these modules on demand, as they may be expensive import shlex from subprocess import Popen, PIPE ssh_tmpl = str(GLOBAL_CFG.get_host_item( 'remote shell template', self.host, self.owner)) ssh_tmpl = ssh_tmpl.replace(' %s', '') user_at_host = '' if self.owner: user_at_host = self.owner + '@' if self.host: user_at_host += self.host else: user_at_host += 'localhost' r_port_file_path = port_file_path.replace( os.environ['HOME'], '$HOME') command = shlex.split(ssh_tmpl) + [ user_at_host, 'cat', r_port_file_path] proc = Popen(command, stdout=PIPE, stderr=PIPE) out, err = proc.communicate() ret_code = proc.wait() if ret_code: if cylc.flags.debug: print >> sys.stderr, { "code": ret_code, "command": command, "stdout": out, "stderr": err} if self.port is None: raise PortFileError( "Port file '%s:%s' not found - suite not running?." % (user_at_host, r_port_file_path)) else: try: out = open(port_file_path).read() except IOError: if self.port is None: raise PortFileError( "Port file '%s' not found - suite not running?." % (port_file_path)) lines = out.splitlines() if self.port is None: try: self.port = int(lines[0]) except (IndexError, ValueError): raise PortFileError( "ERROR, bad content in port file: %s" % port_file_path) if self.host is None: if len(lines) >= 2: self.host = lines[1].strip() else: self.host = get_hostname() def reset(self, *args, **kwargs): pass def signout(self, *args, **kwargs): pass
def scan(host=None, db=None, pyro_timeout=None): """Scan ports, return a list of suites found: [(port, suite.identify())]. Note that we could easily scan for a given suite+owner and return its port instead of reading port files, but this may not always be fast enough. """ if host is None: host = get_hostname() base_port = GLOBAL_CFG.get(['pyro', 'base port']) last_port = base_port + GLOBAL_CFG.get(['pyro', 'maximum number of ports']) if pyro_timeout: pyro_timeout = float(pyro_timeout) else: pyro_timeout = None reg_db = RegistrationDB(db) results = [] for port in range(base_port, last_port): try: proxy = get_proxy(host, port, pyro_timeout) conn_val = ConnValidator() conn_val.set_default_hash(SCAN_HASH) proxy._setNewConnectionValidator(conn_val) proxy._setIdentification((USER, NO_PASSPHRASE)) result = (port, proxy.identify()) except Pyro.errors.ConnectionDeniedError as exc: if cylc.flags.debug: print '%s:%s (connection denied)' % (host, port) # Back-compat <= 6.4.1 msg = ' Old daemon at %s:%s?' % (host, port) for pphrase in reg_db.load_all_passphrases(): try: proxy = get_proxy(host, port, pyro_timeout) proxy._setIdentification(pphrase) info = proxy.id() result = (port, {'name': info[0], 'owner': info[1]}) except Pyro.errors.ConnectionDeniedError: connected = False else: connected = True break if not connected: if cylc.flags.verbose: print >> sys.stderr, msg, "- connection denied (%s)" % exc continue else: if cylc.flags.verbose: print >> sys.stderr, msg, "- connected with passphrase" except Pyro.errors.TimeoutError as exc: # E.g. Ctrl-Z suspended suite - holds up port scanning! if cylc.flags.debug: print '%s:%s (connection timed out)' % (host, port) print >> sys.stderr, ( 'suite? owner?@%s:%s - connection timed out (%s)' % (host, port, exc)) continue except (Pyro.errors.ProtocolError, Pyro.errors.NamingError) as exc: # No suite at this port. if cylc.flags.debug: print str(exc) print '%s:%s (no suite)' % (host, port) continue except SuiteStillInitialisingError: continue except Exception as exc: if cylc.flags.debug: traceback.print_exc() raise else: owner = result[1].get('owner') name = result[1].get('name') states = result[1].get('states', None) if cylc.flags.debug: print ' suite:', name, owner if states is None: # This suite keeps its state info private. # Try again with the passphrase if I have it. pphrase = reg_db.load_passphrase(name, owner, host) if pphrase: try: proxy = get_proxy(host, port, pyro_timeout) conn_val = ConnValidator() conn_val.set_default_hash(SCAN_HASH) proxy._setNewConnectionValidator(conn_val) proxy._setIdentification((USER, pphrase)) result = (port, proxy.identify()) except Exception: # Nope (private suite, wrong passphrase). if cylc.flags.debug: print ' (wrong passphrase)' else: reg_db.cache_passphrase(name, owner, host, pphrase) if cylc.flags.debug: print ' (got states with passphrase)' results.append(result) return results
class CommsDaemon(object): """Wrap HTTPS daemon for a suite.""" def __init__(self, suite, suite_dir): # Suite only needed for back-compat with old clients (see below): self.suite = suite # Figure out the ports we are allowed to use. base_port = GLOBAL_CFG.get(["communication", "base port"]) max_ports = GLOBAL_CFG.get(["communication", "maximum number of ports"]) self.ok_ports = range(int(base_port), int(base_port) + int(max_ports)) random.shuffle(self.ok_ports) comms_options = GLOBAL_CFG.get(["communication", "options"]) # HTTP Digest Auth uses MD5 - pretty secure in this use case. # Extending it with extra algorithms is allowed, but won't be # supported by most browsers. requests and urllib2 are OK though. self.hash_algorithm = "MD5" if "SHA1" in comms_options: # Note 'SHA' rather than 'SHA1'. self.hash_algorithm = "SHA" self.reg_db = RegistrationDB() try: self.cert = self.reg_db.load_item(suite, USER, None, "certificate", create_ok=True) self.pkey = self.reg_db.load_item(suite, USER, None, "private_key", create_ok=True) except PassphraseError: # No OpenSSL installed. self.cert = None self.pkey = None self.suite = suite passphrase = self.reg_db.load_passphrase(suite, USER, None) userpassdict = {"cylc": passphrase, "anon": NO_PASSPHRASE} get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(userpassdict, algorithm=self.hash_algorithm) self.get_ha1 = get_ha1 del passphrase del userpassdict self.client_reporter = CommsClientReporter.get_inst() self.start() def start(self): _ws_init(self) def shutdown(self): """Shutdown the daemon.""" if hasattr(self, "engine"): self.engine.exit() self.engine.block() def connect(self, obj, name): """Connect obj and name to the daemon.""" import cherrypy cherrypy.tree.mount(obj, "/" + name) def disconnect(self, obj): """Disconnect obj from the daemon.""" pass def get_port(self): """Return the daemon port.""" return self.port def report_connection_if_denied(self): self.client_reporter.report_connection_if_denied()
def scan(host=None, db=None, pyro_timeout=None): """Scan ports, return a list of suites found: [(port, suite.identify())]. Note that we could easily scan for a given suite+owner and return its port instead of reading port files, but this may not always be fast enough. """ if host is None: host = get_hostname() base_port = GLOBAL_CFG.get(['pyro', 'base port']) last_port = base_port + GLOBAL_CFG.get(['pyro', 'maximum number of ports']) if pyro_timeout: pyro_timeout = float(pyro_timeout) else: pyro_timeout = None reg_db = RegistrationDB(db) results = [] for port in range(base_port, last_port): try: proxy = get_proxy(host, port, pyro_timeout) conn_val = ConnValidator() conn_val.set_default_hash(SCAN_HASH) proxy._setNewConnectionValidator(conn_val) proxy._setIdentification((USER, NO_PASSPHRASE)) result = (port, proxy.identify()) except Pyro.errors.ConnectionDeniedError as exc: if cylc.flags.debug: print '%s:%s (connection denied)' % (host, port) # Back-compat <= 6.4.1 msg = ' Old daemon at %s:%s?' % (host, port) for pphrase in reg_db.load_all_passphrases(): try: proxy = get_proxy(host, port, pyro_timeout) proxy._setIdentification(pphrase) info = proxy.id() result = (port, {'name': info[0], 'owner': info[1]}) except Pyro.errors.ConnectionDeniedError: connected = False else: connected = True break if not connected: if cylc.flags.verbose: print >> sys.stderr, msg, "- connection denied (%s)" % exc continue else: if cylc.flags.verbose: print >> sys.stderr, msg, "- connected with passphrase" except Pyro.errors.TimeoutError as exc: # E.g. Ctrl-Z suspended suite - holds up port scanning! if cylc.flags.debug: print '%s:%s (connection timed out)' % (host, port) print >> sys.stderr, ( 'suite? owner?@%s:%s - connection timed out (%s)' % ( host, port, exc)) continue except (Pyro.errors.ProtocolError, Pyro.errors.NamingError) as exc: # No suite at this port. if cylc.flags.debug: print str(exc) print '%s:%s (no suite)' % (host, port) continue except SuiteStillInitialisingError: continue except Exception as exc: if cylc.flags.debug: traceback.print_exc() raise else: owner = result[1].get('owner') name = result[1].get('name') states = result[1].get('states', None) if cylc.flags.debug: print ' suite:', name, owner if states is None: # This suite keeps its state info private. # Try again with the passphrase if I have it. pphrase = reg_db.load_passphrase(name, owner, host) if pphrase: try: proxy = get_proxy(host, port, pyro_timeout) conn_val = ConnValidator() conn_val.set_default_hash(SCAN_HASH) proxy._setNewConnectionValidator(conn_val) proxy._setIdentification((USER, pphrase)) result = (port, proxy.identify()) except Exception: # Nope (private suite, wrong passphrase). if cylc.flags.debug: print ' (wrong passphrase)' else: reg_db.cache_passphrase( name, owner, host, pphrase) if cylc.flags.debug: print ' (got states with passphrase)' results.append(result) return results
class PyroClient(object): """Base class for client-side suite object interfaces.""" target_server_object = None def __init__(self, suite, owner=USER, host=None, pyro_timeout=None, port=None, db=None, my_uuid=None, print_uuid=False): self.suite = suite self.host = host self.owner = owner if pyro_timeout is not None: pyro_timeout = float(pyro_timeout) self.pyro_timeout = pyro_timeout self.port = port self.pyro_proxy = None self.my_uuid = my_uuid or uuid4() self.uri = None if print_uuid: print >> sys.stderr, '%s' % self.my_uuid self.reg_db = RegistrationDB(db) self.pphrase = None def call_server_func(self, fname, *fargs): """Call server_object.fname(*fargs) Get a Pyro proxy for the server object if we don't already have it, and handle back compat retry for older daemons. """ items = [ {}, {"reset": True, "cache_ok": False}, {"reset": True, "cache_ok": False, "old": True}, ] for hash_name in OK_HASHES[1:]: items.append( {"reset": True, "cache_ok": False, "hash_name": hash_name}) for i, proxy_kwargs in enumerate(items): func = getattr(self._get_proxy(**proxy_kwargs), fname) try: ret = func(*fargs) break except Pyro.errors.ProtocolError: if i + 1 == len(items): # final attempt raise self.reg_db.cache_passphrase( self.suite, self.owner, self.host, self.pphrase) return ret def _set_uri(self): """Set Pyro URI. Determine host and port using content in port file, unless already specified. """ if ((self.host is None or self.port is None) and 'CYLC_SUITE_RUN_DIR' in os.environ): # Looks like we are in a running task job, so we should be able to # use "cylc-suite-env" file under the suite running directory try: suite_env = CylcSuiteEnv.load( self.suite, os.environ['CYLC_SUITE_RUN_DIR']) except CylcSuiteEnvLoadError: if cylc.flags.debug: traceback.print_exc() else: self.host = suite_env.suite_host self.port = suite_env.suite_port self.owner = suite_env.suite_owner if self.host is None or self.port is None: port_file_path = os.path.join( GLOBAL_CFG.get(['pyro', 'ports directory']), self.suite) if is_remote_host(self.host) or is_remote_user(self.owner): ssh_tmpl = str(GLOBAL_CFG.get_host_item( 'remote shell template', self.host, self.owner)) ssh_tmpl = ssh_tmpl.replace(' %s', '') user_at_host = '' if self.owner: user_at_host = self.owner + '@' if self.host: user_at_host += self.host else: user_at_host += 'localhost' r_port_file_path = port_file_path.replace( os.environ['HOME'], '$HOME') command = shlex.split(ssh_tmpl) + [ user_at_host, 'cat', r_port_file_path] proc = Popen(command, stdout=PIPE, stderr=PIPE) out, err = proc.communicate() ret_code = proc.wait() if ret_code: if cylc.flags.debug: print >> sys.stderr, { "code": ret_code, "command": command, "stdout": out, "stderr": err} raise PortFileError( "Port file '%s:%s' not found - suite not running?." % (user_at_host, r_port_file_path)) else: try: out = open(port_file_path).read() except IOError: raise PortFileError( "Port file '%s' not found - suite not running?." % (port_file_path)) lines = out.splitlines() try: if self.port is None: self.port = int(lines[0]) except (IndexError, ValueError): raise PortFileError( "ERROR, bad content in port file: %s" % port_file_path) if self.host is None: if len(lines) >= 2: self.host = lines[1].strip() else: self.host = get_hostname() # Qualify the obj name with user and suite name (unnecessary but # can't change it until we break back-compat with older daemons). self.uri = ( 'PYROLOC://%(host)s:%(port)s/%(owner)s.%(suite)s.%(target)s' % { "host": self.host, "port": self.port, "suite": self.suite, "owner": self.owner, "target": self.target_server_object}) def _get_proxy(self, reset=True, hash_name=None, cache_ok=True, old=False): """Get a Pyro proxy.""" if reset or self.pyro_proxy is None: self._set_uri() self.pphrase = self.reg_db.load_passphrase( self.suite, self.owner, self.host, cache_ok) # Fails only for unknown hosts (no connection till RPC call). self.pyro_proxy = Pyro.core.getProxyForURI(self.uri) self.pyro_proxy._setTimeout(self.pyro_timeout) if old: self.pyro_proxy._setIdentification(self.pphrase) else: conn_val = ConnValidator() if hash_name is None: hash_name = getattr(self, "_hash_name", None) if hash_name is not None and hash_name in OK_HASHES: conn_val.set_default_hash(hash_name) self.pyro_proxy._setNewConnectionValidator(conn_val) self.pyro_proxy._setIdentification( (self.my_uuid, self.pphrase)) return self.pyro_proxy def reset(self): """Reset pyro_proxy.""" self.pyro_proxy = None def signout(self): """Multi-connect clients should call this on exit.""" try: self._get_proxy().signout() except Exception: # Suite may have stopped before the client exits. pass
def scan(host=None, db=None, timeout=None): """Scan ports, return a list of suites found: [(port, suite.identify())]. Note that we could easily scan for a given suite+owner and return its port instead of reading port files, but this may not always be fast enough. """ if host is None: host = get_hostname() base_port = GLOBAL_CFG.get( ['communication', 'base port']) last_port = base_port + GLOBAL_CFG.get( ['communication', 'maximum number of ports']) if timeout: timeout = float(timeout) else: timeout = None reg_db = RegistrationDB(db) results = [] my_uuid = uuid4() host_for_anon = host if is_remote_host(host): host_for_anon = get_host_ip_by_name(host) # IP reduces DNS traffic. for port in range(base_port, last_port): client = SuiteIdClientAnon(None, host=host_for_anon, port=port, my_uuid=my_uuid, timeout=timeout) try: result = (port, client.identify()) except ConnectionError as exc: if cylc.flags.debug: traceback.print_exc() continue except Exception as exc: if cylc.flags.debug: traceback.print_exc() raise else: owner = result[1].get('owner') name = result[1].get('name') states = result[1].get('states', None) if cylc.flags.debug: print ' suite:', name, owner if states is None: # This suite keeps its state info private. # Try again with the passphrase if I have it. pphrase = reg_db.load_passphrase(name, owner, host) if pphrase: client = SuiteIdClient(name, owner=owner, host=host, port=port, my_uuid=my_uuid, timeout=timeout) try: result = (port, client.identify()) except Exception: # Nope (private suite, wrong passphrase). if cylc.flags.debug: print ' (wrong passphrase)' else: reg_db.cache_passphrase( name, owner, host, pphrase) if cylc.flags.debug: print ' (got states with passphrase)' results.append(result) return results
class PyroClient(object): """Base class for client-side suite object interfaces.""" target_server_object = None def __init__(self, suite, owner=USER, host=None, pyro_timeout=None, port=None, db=None, my_uuid=None, print_uuid=False): self.suite = suite self.host = host self.owner = owner if pyro_timeout is not None: pyro_timeout = float(pyro_timeout) self.pyro_timeout = pyro_timeout self.port = port self.pyro_proxy = None self.my_uuid = my_uuid or uuid4() self.uri = None if print_uuid: print >> sys.stderr, '%s' % self.my_uuid self.reg_db = RegistrationDB(db) self.pphrase = None def call_server_func(self, fname, *fargs): """Call server_object.fname(*fargs) Get a Pyro proxy for the server object if we don't already have it, and handle back compat retry for older daemons. """ items = [ {}, { "reset": True, "cache_ok": False }, { "reset": True, "cache_ok": False, "old": True }, ] for hash_name in OK_HASHES[1:]: items.append({ "reset": True, "cache_ok": False, "hash_name": hash_name }) for i, proxy_kwargs in enumerate(items): func = getattr(self._get_proxy(**proxy_kwargs), fname) try: ret = func(*fargs) break except Pyro.errors.ProtocolError: if i + 1 == len(items): # final attempt raise self.reg_db.cache_passphrase(self.suite, self.owner, self.host, self.pphrase) return ret def _set_uri(self): """Set Pyro URI. Determine host and port using content in port file, unless already specified. """ if ((self.host is None or self.port is None) and 'CYLC_SUITE_RUN_DIR' in os.environ): # Looks like we are in a running task job, so we should be able to # use "cylc-suite-env" file under the suite running directory try: suite_env = CylcSuiteEnv.load(self.suite, os.environ['CYLC_SUITE_RUN_DIR']) except CylcSuiteEnvLoadError: if cylc.flags.debug: traceback.print_exc() else: self.host = suite_env.suite_host self.port = suite_env.suite_port self.owner = suite_env.suite_owner if self.host is None or self.port is None: port_file_path = os.path.join( GLOBAL_CFG.get(['pyro', 'ports directory']), self.suite) if is_remote_host(self.host) or is_remote_user(self.owner): ssh_tmpl = str( GLOBAL_CFG.get_host_item('remote shell template', self.host, self.owner)) ssh_tmpl = ssh_tmpl.replace(' %s', '') user_at_host = '' if self.owner: user_at_host = self.owner + '@' if self.host: user_at_host += self.host else: user_at_host += 'localhost' r_port_file_path = port_file_path.replace( os.environ['HOME'], '$HOME') command = shlex.split(ssh_tmpl) + [ user_at_host, 'cat', r_port_file_path ] proc = Popen(command, stdout=PIPE, stderr=PIPE) out, err = proc.communicate() ret_code = proc.wait() if ret_code: if cylc.flags.debug: print >> sys.stderr, { "code": ret_code, "command": command, "stdout": out, "stderr": err } raise PortFileError( "Port file '%s:%s' not found - suite not running?." % (user_at_host, r_port_file_path)) else: try: out = open(port_file_path).read() except IOError: raise PortFileError( "Port file '%s' not found - suite not running?." % (port_file_path)) lines = out.splitlines() try: if self.port is None: self.port = int(lines[0]) except (IndexError, ValueError): raise PortFileError("ERROR, bad content in port file: %s" % port_file_path) if self.host is None: if len(lines) >= 2: self.host = lines[1].strip() else: self.host = get_hostname() # Qualify the obj name with user and suite name (unnecessary but # can't change it until we break back-compat with older daemons). self.uri = ( 'PYROLOC://%(host)s:%(port)s/%(owner)s.%(suite)s.%(target)s' % { "host": self.host, "port": self.port, "suite": self.suite, "owner": self.owner, "target": self.target_server_object }) def _get_proxy(self, reset=True, hash_name=None, cache_ok=True, old=False): """Get a Pyro proxy.""" if reset or self.pyro_proxy is None: self._set_uri() self.pphrase = self.reg_db.load_passphrase(self.suite, self.owner, self.host, cache_ok) # Fails only for unknown hosts (no connection till RPC call). self.pyro_proxy = Pyro.core.getProxyForURI(self.uri) self.pyro_proxy._setTimeout(self.pyro_timeout) if old: self.pyro_proxy._setIdentification(self.pphrase) else: conn_val = ConnValidator() if hash_name is None: hash_name = getattr(self, "_hash_name", None) if hash_name is not None and hash_name in OK_HASHES: conn_val.set_default_hash(hash_name) self.pyro_proxy._setNewConnectionValidator(conn_val) self.pyro_proxy._setIdentification( (self.my_uuid, self.pphrase)) return self.pyro_proxy def reset(self): """Reset pyro_proxy.""" self.pyro_proxy = None def signout(self): """Multi-connect clients should call this on exit.""" try: self._get_proxy().signout() except Exception: # Suite may have stopped before the client exits. pass