def __init__(self, suite, owner=None, host=None, port=None, timeout=None, my_uuid=None, print_uuid=False, comms_protocol=None, auth=None): self.suite = suite if not owner: owner = get_user() self.owner = owner self.host = host if self.host and self.host.split('.')[0] == 'localhost': self.host = get_host() elif self.host and '.' not in self.host: # Not IP and no domain self.host = get_fqdn_by_host(self.host) self.port = port self.srv_files_mgr = SuiteSrvFilesManager() self.comms_protocol = comms_protocol if timeout is not None: timeout = float(timeout) self.timeout = timeout self.my_uuid = my_uuid or uuid4() if print_uuid: print >> sys.stderr, '%s' % self.my_uuid self.prog_name = os.path.basename(sys.argv[0]) self.auth = auth
def __init__( self, suite, owner=None, host=None, port=None, timeout=None, my_uuid=None, print_uuid=False, auth=None): self.suite = suite if not owner: owner = get_user() self.owner = owner self.host = host if self.host and self.host.split('.')[0] == 'localhost': self.host = get_host() elif self.host and '.' not in self.host: # Not IP and no domain self.host = get_fqdn_by_host(self.host) self.port = port self.srv_files_mgr = SuiteSrvFilesManager() if timeout is not None: timeout = float(timeout) self.timeout = timeout self.my_uuid = my_uuid or uuid4() if print_uuid: sys.stderr.write('%s\n' % self.my_uuid) self.prog_name = os.path.basename(sys.argv[0]) self.auth = auth self.session = None self.comms1 = {} # content in primary contact file self.comms2 = {} # content in extra contact file, e.g. contact via ssh
def get_scan_items_from_fs(owner_pattern=None, reg_pattern=None): """Scrape list of suites from the filesystem. Walk users' "~/cylc-run/" to get (host, port) from ".service/contact" for active suites. Yields: tuple - (reg, host, port) """ srv_files_mgr = SuiteSrvFilesManager() if owner_pattern is None: # Run directory of current user only run_dirs = [(glbl_cfg().get_host_item('run directory'), None)] else: # Run directory of all users matching "owner_pattern". # But skip those with /nologin or /false shells run_dirs = [] skips = ('/false', '/nologin') for pwent in getpwall(): if any(pwent.pw_shell.endswith(s) for s in (skips)): continue if owner_pattern.match(pwent.pw_name): run_dirs.append(( glbl_cfg().get_host_item( 'run directory', owner=pwent.pw_name, owner_home=pwent.pw_dir), pwent.pw_name)) if cylc.flags.debug: sys.stderr.write('Listing suites:%s%s\n' % ( DEBUG_DELIM, DEBUG_DELIM.join(item[1] for item in run_dirs if item[1] is not None))) for run_d, owner in run_dirs: for dirpath, dnames, _ in os.walk(run_d, followlinks=True): # Always descend for top directory, but # don't descend further if it has a .service/ or log/ dir if dirpath != run_d and ( srv_files_mgr.DIR_BASE_SRV in dnames or 'log' in dnames): dnames[:] = [] # Filter suites by name reg = os.path.relpath(dirpath, run_d) if reg_pattern and not reg_pattern.match(reg): continue # Choose only suites with .service and matching filter try: contact_data = srv_files_mgr.load_contact_file(reg, owner) except (SuiteServiceFileError, IOError, TypeError, ValueError): continue else: yield ( reg, contact_data[srv_files_mgr.KEY_HOST], contact_data[srv_files_mgr.KEY_PORT] )
def get_scan_items_from_fs(owner_pattern=None, updater=None): """Get list of host:port available to scan using the file system. Walk users' "~/cylc-run/" to get (host, port) from ".service/contact" for active suites. Return (list): List of (host, port) available for scan. """ srv_files_mgr = SuiteSrvFilesManager() if owner_pattern is None: # Run directory of current user only run_dirs = [(glbl_cfg().get_host_item('run directory'), None)] else: # Run directory of all users matching "owner_pattern". # But skip those with /nologin or /false shells run_dirs = [] skips = ('/false', '/nologin') for pwent in getpwall(): if any(pwent.pw_shell.endswith(s) for s in (skips)): continue if owner_pattern.match(pwent.pw_name): run_dirs.append(( glbl_cfg().get_host_item( 'run directory', owner=pwent.pw_name, owner_home=pwent.pw_dir), pwent.pw_name)) if cylc.flags.debug: sys.stderr.write('Listing suites:%s%s\n' % ( DEBUG_DELIM, DEBUG_DELIM.join(item[1] for item in run_dirs if item[1] is not None))) items = [] for run_d, owner in run_dirs: for dirpath, dnames, fnames in os.walk(run_d, followlinks=True): if updater and updater.quit: return # Always descend for top directory, but # don't descend further if it has a: # * .service/ or log/ # * cylc-suite.db: (pre-cylc-7 suites don't have ".service/"). if dirpath != run_d and ( srv_files_mgr.DIR_BASE_SRV in dnames or 'log' in dnames or 'cylc-suite.db' in fnames): dnames[:] = [] # Choose only suites with .service and matching filter reg = os.path.relpath(dirpath, run_d) try: contact_data = srv_files_mgr.load_contact_file(reg, owner) except (SuiteServiceFileError, IOError, TypeError, ValueError): continue else: items.append(( contact_data[srv_files_mgr.KEY_HOST], contact_data[srv_files_mgr.KEY_PORT])) return items
def get_scan_items_from_fs(owner_pattern=None, reg_pattern=None): """Scrape list of suites from the filesystem. Walk users' "~/cylc-run/" to get (host, port) from ".service/contact" for active suites. Yields: tuple - (reg, host, port) """ srv_files_mgr = SuiteSrvFilesManager() if owner_pattern is None: # Run directory of current user only run_dirs = [(glbl_cfg().get_host_item('run directory'), None)] else: # Run directory of all users matching "owner_pattern". # But skip those with /nologin or /false shells run_dirs = [] skips = ('/false', '/nologin') for pwent in getpwall(): if any(pwent.pw_shell.endswith(s) for s in (skips)): continue if owner_pattern.match(pwent.pw_name): run_dirs.append( (glbl_cfg().get_host_item('run directory', owner=pwent.pw_name, owner_home=pwent.pw_dir), pwent.pw_name)) if cylc.flags.debug: sys.stderr.write( 'Listing suites:%s%s\n' % (DEBUG_DELIM, DEBUG_DELIM.join(item[1] for item in run_dirs if item[1] is not None))) for run_d, owner in run_dirs: for dirpath, dnames, _ in os.walk(run_d, followlinks=True): # Always descend for top directory, but # don't descend further if it has a .service/ or log/ dir if dirpath != run_d and (srv_files_mgr.DIR_BASE_SRV in dnames or 'log' in dnames): dnames[:] = [] # Filter suites by name reg = os.path.relpath(dirpath, run_d) if reg_pattern and not reg_pattern.match(reg): continue # Choose only suites with .service and matching filter try: contact_data = srv_files_mgr.load_contact_file(reg, owner) except (SuiteServiceFileError, IOError, TypeError, ValueError): continue else: yield (reg, contact_data[srv_files_mgr.KEY_HOST], contact_data[srv_files_mgr.KEY_PORT])
def main(is_restart=False): """CLI main.""" options, args = parse_commandline(is_restart) if not args: # Auto-registration: "cylc run" (no args) in source dir. try: reg = SuiteSrvFilesManager().register() except SuiteServiceFileError as exc: sys.exit(exc) # Replace this process with "cylc run REG ..." for 'ps -f'. os.execv(sys.argv[0], [sys.argv[0]] + [reg] + sys.argv[1:]) # Check suite is not already running before start of host selection. try: SuiteSrvFilesManager().detect_old_contact_file(args[0]) except SuiteServiceFileError as exc: sys.exit(exc) # Create auth files if needed. On a shared FS if the suite host changes # this may (will?) renew the ssl.cert to reflect the change in host name. SuiteSrvFilesManager().create_auth_files(args[0]) # Check whether a run host is explicitly specified, else select one. if not options.host: try: host = HostAppointer().appoint_host() except EmptyHostList as exc: if cylc.flags.debug: raise else: sys.exit(str(exc)) if is_remote_host(host): if is_restart: base_cmd = ["restart"] + sys.argv[1:] else: base_cmd = ["run"] + sys.argv[1:] # Prevent recursive host selection base_cmd.append("--host=localhost") return remote_cylc_cmd(base_cmd, host=host) if remrun(set_rel_local=True): # State localhost as above. sys.exit() try: SuiteSrvFilesManager().get_suite_source_dir(args[0], options.owner) except SuiteServiceFileError: # Source path is assumed to be the run directory SuiteSrvFilesManager().register( args[0], glbl_cfg().get_derived_host_item(args[0], 'suite run directory')) try: scheduler = Scheduler(is_restart, options, args) except SuiteServiceFileError as exc: sys.exit(exc) scheduler.start()
def get_scan_items_from_fs(owner_pattern=None, updater=None): """Get list of host:port available to scan using the file system. Walk users' "~/cylc-run/" to get (host, port) from ".service/contact" for active suites. Return (list): List of (host, port) available for scan. """ srv_files_mgr = SuiteSrvFilesManager() if owner_pattern is None: # Run directory of current user only run_dirs = [(GLOBAL_CFG.get_host_item('run directory'), None)] else: # Run directory of all users matching "owner_pattern". # But skip those with /nologin or /false shells run_dirs = [] skips = ('/false', '/nologin') for pwent in getpwall(): if any(pwent.pw_shell.endswith(s) for s in (skips)): continue if owner_pattern.match(pwent.pw_name): run_dirs.append( (GLOBAL_CFG.get_host_item('run directory', owner=pwent.pw_name, owner_home=pwent.pw_dir), pwent.pw_name)) if cylc.flags.debug: sys.stderr.write( 'Listing suites:%s%s\n' % (DEBUG_DELIM, DEBUG_DELIM.join(item[1] for item in run_dirs if item[1] is not None))) items = [] for run_d, owner in run_dirs: for dirpath, dnames, fnames in os.walk(run_d, followlinks=True): if updater and updater.quit: return # Always descend for top directory, but # don't descend further if it has a: # * .service/ or log/ # * cylc-suite.db: (pre-cylc-7 suites don't have ".service/"). if dirpath != run_d and (srv_files_mgr.DIR_BASE_SRV in dnames or 'log' in dnames or 'cylc-suite.db' in fnames): dnames[:] = [] # Choose only suites with .service and matching filter reg = os.path.relpath(dirpath, run_d) try: contact_data = srv_files_mgr.load_contact_file(reg, owner) except (SuiteServiceFileError, IOError, TypeError, ValueError): continue else: items.append((contact_data[srv_files_mgr.KEY_HOST], contact_data[srv_files_mgr.KEY_PORT])) return items
def _scan1_impl(conn, timeout, my_uuid): """Connect to host:port to get suite identify.""" srv_files_mgr = SuiteSrvFilesManager() while True: if not conn.poll(SLEEP_INTERVAL): continue item = conn.recv() if item == MSG_QUIT: break host, port = item host_anon = host if is_remote_host(host): host_anon = get_host_ip_by_name(host) # IP reduces DNS traffic client = SuiteIdClientAnon( None, host=host_anon, port=port, my_uuid=my_uuid, timeout=timeout) try: result = client.identify() except ConnectionTimeout as exc: conn.send((host, port, MSG_TIMEOUT)) except ConnectionError as exc: conn.send((host, port, None)) else: owner = result.get('owner') name = result.get('name') states = result.get('states', None) if cylc.flags.debug: print >> sys.stderr, ' suite:', name, owner if states is None: # This suite keeps its state info private. # Try again with the passphrase if I have it. try: pphrase = srv_files_mgr.get_auth_item( srv_files_mgr.FILE_BASE_PASSPHRASE, name, owner, host, content=True) except SuiteServiceFileError: pass else: if pphrase: client = SuiteIdClient( name, owner=owner, host=host, port=port, my_uuid=my_uuid, timeout=timeout) try: result = client.identify() except ConnectionError as exc: # Nope (private suite, wrong passphrase). if cylc.flags.debug: print >> sys.stderr, ' (wrong passphrase)' else: if cylc.flags.debug: print >> sys.stderr, ( ' (got states with passphrase)') conn.send((host, port, result)) conn.close()
def _load_suite_contact_file(self): """Load contact environment file.""" from cylc.suite_srv_files_mgr import SuiteSrvFilesManager srv_files_mgr = SuiteSrvFilesManager() contact_data = srv_files_mgr.load_contact_file(self.suite) for attr, key in ( ('owner', srv_files_mgr.KEY_OWNER), ('host', srv_files_mgr.KEY_HOST), ('port', srv_files_mgr.KEY_PORT)): try: self.env_map[key] = contact_data[key] setattr(self, attr, contact_data[key]) except KeyError: pass
def __init__(self, suite): # Suite only needed for back-compat with old clients (see below): self.suite = suite self.engine = None self.port = None # Figure out the ports we are allowed to use. base_port = glbl_cfg().get(['communication', 'base port']) max_ports = glbl_cfg().get( ['communication', 'maximum number of ports']) self.ok_ports = range(int(base_port), int(base_port) + int(max_ports)) random.shuffle(self.ok_ports) comms_options = glbl_cfg().get(['communication', 'options']) # HTTP Digest Auth uses MD5 - pretty secure in this use case. # Extending it with extra algorithms is allowed, but won't be # supported by most browsers. requests and urllib2 are OK though. self.hash_algorithm = "MD5" if "SHA1" in comms_options: # Note 'SHA' rather than 'SHA1'. self.hash_algorithm = "SHA" self.srv_files_mgr = SuiteSrvFilesManager() self.comms_method = glbl_cfg().get(['communication', 'method']) self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain( { 'cylc': self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_PASSPHRASE, suite, content=True), 'anon': NO_PASSPHRASE }, algorithm=self.hash_algorithm) if self.comms_method == 'http': self.cert = None self.pkey = None else: # if self.comms_method in [None, 'https']: try: self.cert = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_CERT, suite) self.pkey = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_PEM, suite) except SuiteServiceFileError: ERR.error("no HTTPS/OpenSSL support. Aborting...") raise CylcError("No HTTPS support. " "Configure user's global.rc to use HTTP.") self.start()
class TestSuiteSrvFilesManager(unittest.TestCase): def setUp(self): self.suite_srv_files_mgr = SuiteSrvFilesManager() @mock.patch('cylc.suite_srv_files_mgr.os') def test_register(self, mocked_os): """Test the SuiteSrvFilesManager register function.""" def mkdirs_standin(_, exist_ok=False): return True # we do not need to mock these functions mocked_os.path.basename.side_effect = os.path.basename mocked_os.path.join = os.path.join mocked_os.path.normpath = os.path.normpath mocked_os.path.dirname = os.path.dirname mocked_os.makedirs.side_effect = mkdirs_standin mocked_os.path.abspath.side_effect = lambda x: x for reg, source, redirect, cwd, isabs, isfile, \ suite_srv_dir, readlink, expected_symlink, \ expected, e_expected, e_message \ in get_register_test_cases(): mocked_os.getcwd.side_effect = lambda: cwd mocked_os.path.isabs.side_effect = lambda x: isabs mocked_os.path.isfile = lambda x: isfile self.suite_srv_files_mgr.get_suite_srv_dir = mock.MagicMock( return_value=suite_srv_dir ) if readlink == OSError: mocked_os.readlink.side_effect = readlink else: mocked_os.readlink.side_effect = lambda x: readlink if e_expected is None: reg = self.suite_srv_files_mgr.register(reg, source, redirect) self.assertEqual(expected, reg) if mocked_os.symlink.call_count > 0: # first argument, of the first call arg0 = mocked_os.symlink.call_args[0][0] self.assertEqual(expected_symlink, arg0) else: with self.assertRaises(e_expected) as cm: self.suite_srv_files_mgr.register(reg, source, redirect) if e_message is not None: the_exception = cm.exception self.assertTrue(e_message in str(the_exception), str(the_exception))
def __init__(self, suite, owner=USER, host=None, port=None, timeout=None, my_uuid=None, print_uuid=False): self.suite = suite self.owner = owner self.host = host self.port = port if timeout is not None: timeout = float(timeout) self.timeout = timeout self.my_uuid = my_uuid or uuid4() if print_uuid: print >> sys.stderr, '%s' % self.my_uuid self.srv_files_mgr = SuiteSrvFilesManager() self.prog_name = os.path.basename(sys.argv[0]) self.server_cert = None self.auth = None
def __init__( self, suite, owner=None, host=None, port=None, timeout=None, my_uuid=None, print_uuid=False, auth=None): self.suite = suite if not owner: owner = get_user() self.owner = owner self.host = host if self.host and self.host.split('.')[0] == 'localhost': self.host = get_host() elif self.host and '.' not in self.host: # Not IP and no domain self.host = get_fqdn_by_host(self.host) self.port = port self.srv_files_mgr = SuiteSrvFilesManager() if timeout is not None: timeout = float(timeout) self.timeout = timeout self.my_uuid = my_uuid or uuid4() if print_uuid: sys.stderr.write('%s\n' % self.my_uuid) self.prog_name = os.path.basename(sys.argv[0]) self.auth = auth self.session = None self.comms1 = {} # content in primary contact file self.comms2 = {} # content in extra contact file, e.g. contact via ssh
def __init__(self, regd_treestore, filtr=None, timeout=None): self.db = SuiteSrvFilesManager() self.quit = False if timeout: self.timeout = float(timeout) else: self.timeout = None self.regd_treestore = regd_treestore super(db_updater, self).__init__() self.next_scan_time = None self.running_choices = None self.newtree = {} self.regd_choices = self.db.list_suites(filtr)
class TestSuiteSrvFilesManager(unittest.TestCase): def setUp(self): self.suite_srv_files_mgr = SuiteSrvFilesManager() @mock.patch('cylc.suite_srv_files_mgr.os') def test_register(self, mocked_os): """Test the SuiteSrvFilesManager register function.""" def mkdirs_standin(_, exist_ok=False): return True # we do not need to mock these functions mocked_os.path.basename.side_effect = os.path.basename mocked_os.path.join = os.path.join mocked_os.path.normpath = os.path.normpath mocked_os.path.dirname = os.path.dirname mocked_os.makedirs.side_effect = mkdirs_standin mocked_os.path.abspath.side_effect = lambda x: x for reg, source, redirect, cwd, isabs, isfile, \ suite_srv_dir, readlink, expected_symlink, \ expected, e_expected, e_message \ in get_register_test_cases(): mocked_os.getcwd.side_effect = lambda: cwd mocked_os.path.isabs.side_effect = lambda x: isabs mocked_os.path.isfile = lambda x: isfile self.suite_srv_files_mgr.get_suite_srv_dir = mock.MagicMock( return_value=suite_srv_dir) if readlink == OSError: mocked_os.readlink.side_effect = readlink else: mocked_os.readlink.side_effect = lambda x: readlink if e_expected is None: reg = self.suite_srv_files_mgr.register(reg, source, redirect) self.assertEqual(expected, reg) if mocked_os.symlink.call_count > 0: # first argument, of the first call arg0 = mocked_os.symlink.call_args[0][0] self.assertEqual(expected_symlink, arg0) else: with self.assertRaises(e_expected) as cm: self.suite_srv_files_mgr.register(reg, source, redirect) if e_message is not None: the_exception = cm.exception self.assertTrue(e_message in str(the_exception), str(the_exception))
def __init__(self, suite): # Suite only needed for back-compat with old clients (see below): self.suite = suite # Figure out the ports we are allowed to use. base_port = GLOBAL_CFG.get(['communication', 'base port']) max_ports = GLOBAL_CFG.get( ['communication', 'maximum number of ports']) self.ok_ports = range(int(base_port), int(base_port) + int(max_ports)) random.shuffle(self.ok_ports) comms_options = GLOBAL_CFG.get(['communication', 'options']) # HTTP Digest Auth uses MD5 - pretty secure in this use case. # Extending it with extra algorithms is allowed, but won't be # supported by most browsers. requests and urllib2 are OK though. self.hash_algorithm = "MD5" if "SHA1" in comms_options: # Note 'SHA' rather than 'SHA1'. self.hash_algorithm = "SHA" self.srv_files_mgr = SuiteSrvFilesManager() self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain( { 'cylc': self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_PASSPHRASE, suite, content=True), 'anon': NO_PASSPHRASE }, algorithm=self.hash_algorithm) try: self.cert = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_CERT, suite) self.pkey = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_PEM, suite) except SuiteServiceFileError: self.cert = None self.pkey = None self.client_reporter = CommsClientReporter.get_inst() self.start()
def get_secret(suite): """Return the secret used for encrypting messages. Currently this is the suite passphrase. This means we are sending many messages all encrypted with the same hash which isn't great. TODO: Upgrade the secret to add foreword security. """ return SuiteSrvFilesManager().get_auth_item( SuiteSrvFilesManager.FILE_BASE_PASSPHRASE, suite, content=True )
def _scan_worker(conn, timeout, my_uuid): """Port scan worker.""" srv_files_mgr = SuiteSrvFilesManager() while True: try: if not conn.poll(SLEEP_INTERVAL): continue item = conn.recv() if item == MSG_QUIT: break conn.send(_scan_item(timeout, my_uuid, srv_files_mgr, item)) except KeyboardInterrupt: break conn.close()
def __init__(self, regd_treestore, filtr=None, timeout=None): self.db = SuiteSrvFilesManager() self.quit = False if timeout: self.timeout = float(timeout) else: self.timeout = None self.regd_treestore = regd_treestore super(db_updater, self).__init__() self.next_scan_time = None self.running_choices = None self.newtree = {} self.regd_choices = self.db.list_suites(filtr)
def _timeout_handler(suite, host, port): """Handle the eventuality of a communication timeout with the suite.""" if suite is None: return # Cannot connect, perhaps suite is no longer running and is leaving # behind a contact file? try: SuiteSrvFilesManager().detect_old_contact_file(suite, (host, port)) except (AssertionError, SuiteServiceFileError): # * contact file not have matching (host, port) to suite proc # * old contact file exists and the suite process still alive return else: # the suite has stopped raise ClientError( # TODO: SuiteStoppedError? {'message': 'Suite "%s" already stopped' % suite})
def __init__(self, suite, owner=USER, host=None, port=None, timeout=None, my_uuid=None, print_uuid=False): self.suite = suite self.owner = owner self.host = host self.port = port if timeout is not None: timeout = float(timeout) self.timeout = timeout self.my_uuid = my_uuid or uuid4() if print_uuid: print >> sys.stderr, '%s' % self.my_uuid self.srv_files_mgr = SuiteSrvFilesManager() self.prog_name = os.path.basename(sys.argv[0]) self.server_cert = None self.auth = None
def send(self, messages): """Send messages back to the suite.""" self._print_messages(messages) self._update_job_status_file(messages) try: self.env_map.update(SuiteSrvFilesManager().load_contact_file( self.suite)) except (IOError, ValueError, SuiteServiceFileError) as exc: # No suite to communicate with, just print to stdout. if cylc.flags.debug: print >> sys.stderr, exc return if (self.env_map.get('CYLC_TASK_COMMS_METHOD') == 'ssh' and self._send_by_ssh()): return self._send_by_remote_port(messages)
def __init__(self, suite): # Suite only needed for back-compat with old clients (see below): self.suite = suite self.engine = None self.port = None # Figure out the ports we are allowed to use. base_port = glbl_cfg().get(['communication', 'base port']) max_ports = glbl_cfg().get( ['communication', 'maximum number of ports']) self.ok_ports = range(int(base_port), int(base_port) + int(max_ports)) random.shuffle(self.ok_ports) comms_options = glbl_cfg().get(['communication', 'options']) # HTTP Digest Auth uses MD5 - pretty secure in this use case. # Extending it with extra algorithms is allowed, but won't be # supported by most browsers. requests and urllib2 are OK though. self.hash_algorithm = "MD5" if "SHA1" in comms_options: # Note 'SHA' rather than 'SHA1'. self.hash_algorithm = "SHA" self.srv_files_mgr = SuiteSrvFilesManager() self.comms_method = glbl_cfg().get(['communication', 'method']) self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain( { 'cylc': self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_PASSPHRASE, suite, content=True), 'anon': NO_PASSPHRASE }, algorithm=self.hash_algorithm) if self.comms_method == 'http': self.cert = None self.pkey = None else: # if self.comms_method in [None, 'https']: try: self.cert = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_CERT, suite) self.pkey = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_PEM, suite) except SuiteServiceFileError: ERR.error("no HTTPS/OpenSSL support. Aborting...") raise CylcError("No HTTPS support. " "Configure user's global.rc to use HTTP.") self.start()
def get_location(cls, suite, owner, host): """Extract host and port from a suite's contact file.""" try: contact = SuiteSrvFilesManager().load_contact_file( suite, owner, host) except SuiteServiceFileError: sys.exit(cls.NOT_RUNNING % suite) # monkey-patch the error message to make it more informative. # exc.args = (cls.NOT_RUNNING % suite,) # raise if not host: host = contact[SuiteSrvFilesManager.KEY_HOST] host = get_fqdn_by_host(host) port = int(contact[SuiteSrvFilesManager.KEY_PORT]) return host, port
def send(self, messages): """Send messages back to the suite.""" self._print_messages(messages) self._update_job_status_file(messages) messages = [msg + ' at ' + self.true_event_time for msg in messages] try: self.env_map.update(SuiteSrvFilesManager().load_contact_file( self.suite)) except (IOError, ValueError, SuiteServiceFileError): # No suite to communicate with, just print to stdout. if cylc.flags.debug: import traceback traceback.print_exc() return if self.env_map.get('CYLC_TASK_COMMS_METHOD') == 'ssh': self._send_by_ssh() else: self._send_by_remote_port(messages)
def __init__(self, suite): # Suite only needed for back-compat with old clients (see below): self.suite = suite # Figure out the ports we are allowed to use. base_port = GLOBAL_CFG.get(['communication', 'base port']) max_ports = GLOBAL_CFG.get( ['communication', 'maximum number of ports']) self.ok_ports = range(int(base_port), int(base_port) + int(max_ports)) random.shuffle(self.ok_ports) comms_options = GLOBAL_CFG.get(['communication', 'options']) # HTTP Digest Auth uses MD5 - pretty secure in this use case. # Extending it with extra algorithms is allowed, but won't be # supported by most browsers. requests and urllib2 are OK though. self.hash_algorithm = "MD5" if "SHA1" in comms_options: # Note 'SHA' rather than 'SHA1'. self.hash_algorithm = "SHA" self.srv_files_mgr = SuiteSrvFilesManager() self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain( { 'cylc': self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_PASSPHRASE, suite, content=True), 'anon': NO_PASSPHRASE }, algorithm=self.hash_algorithm) try: self.cert = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_CERT, suite) self.pkey = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_PEM, suite) except SuiteServiceFileError: self.cert = None self.pkey = None self.client_reporter = CommsClientReporter.get_inst() self.start()
class HTTPServer(object): """HTTP(S) server by cherrypy, for serving suite runtime API.""" API = 1 LOG_CONNECT_DENIED_TMPL = "[client-connect] DENIED %s@%s:%s %s" def __init__(self, suite): # Suite only needed for back-compat with old clients (see below): self.suite = suite self.engine = None self.port = None # Figure out the ports we are allowed to use. base_port = GLOBAL_CFG.get(['communication', 'base port']) max_ports = GLOBAL_CFG.get( ['communication', 'maximum number of ports']) self.ok_ports = range(int(base_port), int(base_port) + int(max_ports)) random.shuffle(self.ok_ports) comms_options = GLOBAL_CFG.get(['communication', 'options']) # HTTP Digest Auth uses MD5 - pretty secure in this use case. # Extending it with extra algorithms is allowed, but won't be # supported by most browsers. requests and urllib2 are OK though. self.hash_algorithm = "MD5" if "SHA1" in comms_options: # Note 'SHA' rather than 'SHA1'. self.hash_algorithm = "SHA" self.srv_files_mgr = SuiteSrvFilesManager() self.comms_method = GLOBAL_CFG.get(['communication', 'method']) self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain( { 'cylc': self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_PASSPHRASE, suite, content=True), 'anon': NO_PASSPHRASE }, algorithm=self.hash_algorithm) if self.comms_method == 'http': self.cert = None self.pkey = None else: # if self.comms_method in [None, 'https']: try: self.cert = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_CERT, suite) self.pkey = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_PEM, suite) except SuiteServiceFileError: ERR.error("no HTTPS/OpenSSL support. Aborting...") raise CylcError("No HTTPS support. " "Configure user's global.rc to use HTTP.") self.start() @cherrypy.expose def apiversion(self): """Return API version.""" return str(self.API) @staticmethod def connect(schd): """Mount suite schedular object to the web server.""" cherrypy.tree.mount(SuiteRuntimeService(schd), '/') # For back-compat with "scan" cherrypy.tree.mount(SuiteRuntimeService(schd), '/id') @staticmethod def disconnect(schd): """Disconnect obj from the web server.""" del cherrypy.tree.apps['/%s/%s' % (schd.owner, schd.suite)] def get_port(self): """Return the web server port.""" return self.port def shutdown(self): """Shutdown the web server.""" if hasattr(self, "engine"): self.engine.exit() self.engine.block() def start(self): """Start quick web service.""" # cherrypy.config["tools.encode.on"] = True # cherrypy.config["tools.encode.encoding"] = "utf-8" cherrypy.config["server.socket_host"] = get_host() cherrypy.config["engine.autoreload.on"] = False if self.comms_method == "https": # Setup SSL etc. Otherwise fail and exit. # Require connection method to be the same e.g HTTP/HTTPS matching. cherrypy.config['server.ssl_module'] = 'pyopenSSL' cherrypy.config['server.ssl_certificate'] = self.cert cherrypy.config['server.ssl_private_key'] = self.pkey cherrypy.config['log.screen'] = None key = binascii.hexlify(os.urandom(16)) cherrypy.config.update({ 'tools.auth_digest.on': True, 'tools.auth_digest.realm': self.suite, 'tools.auth_digest.get_ha1': self.get_ha1, 'tools.auth_digest.key': key, 'tools.auth_digest.algorithm': self.hash_algorithm }) cherrypy.tools.connect_log = cherrypy.Tool( 'on_end_resource', self._report_connection_if_denied) cherrypy.config['tools.connect_log.on'] = True self.engine = cherrypy.engine for port in self.ok_ports: cherrypy.config["server.socket_port"] = port try: cherrypy.engine.start() cherrypy.engine.wait(cherrypy.engine.states.STARTED) except cherrypy.process.wspbus.ChannelFailures: if cylc.flags.debug: traceback.print_exc() # We need to reinitialise the httpserver for each port attempt. cherrypy.server.httpserver = None else: if cherrypy.engine.state == cherrypy.engine.states.STARTED: self.port = port return raise Exception("No available ports") @staticmethod def _get_client_connection_denied(): """Return whether a connection was denied.""" if "Authorization" not in cherrypy.request.headers: # Probably just the initial HTTPS handshake. return False status = cherrypy.response.status if isinstance(status, basestring): return cherrypy.response.status.split()[0] in ["401", "403"] return cherrypy.response.status in [401, 403] def _report_connection_if_denied(self): """Log an (un?)successful connection attempt.""" prog_name, user, host, uuid = _get_client_info()[1:] connection_denied = self._get_client_connection_denied() if connection_denied: LOG.warning(self.__class__.LOG_CONNECT_DENIED_TMPL % ( user, host, prog_name, uuid))
class CommsDaemon(object): """Wrap HTTPS daemon for a suite.""" def __init__(self, suite): # Suite only needed for back-compat with old clients (see below): self.suite = suite # Figure out the ports we are allowed to use. base_port = GLOBAL_CFG.get(['communication', 'base port']) max_ports = GLOBAL_CFG.get( ['communication', 'maximum number of ports']) self.ok_ports = range(int(base_port), int(base_port) + int(max_ports)) random.shuffle(self.ok_ports) comms_options = GLOBAL_CFG.get(['communication', 'options']) # HTTP Digest Auth uses MD5 - pretty secure in this use case. # Extending it with extra algorithms is allowed, but won't be # supported by most browsers. requests and urllib2 are OK though. self.hash_algorithm = "MD5" if "SHA1" in comms_options: # Note 'SHA' rather than 'SHA1'. self.hash_algorithm = "SHA" self.srv_files_mgr = SuiteSrvFilesManager() self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain( { 'cylc': self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_PASSPHRASE, suite, content=True), 'anon': NO_PASSPHRASE }, algorithm=self.hash_algorithm) try: self.cert = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_CERT, suite) self.pkey = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_PEM, suite) except SuiteServiceFileError: self.cert = None self.pkey = None self.client_reporter = CommsClientReporter.get_inst() self.start() def shutdown(self): """Shutdown the daemon.""" if hasattr(self, "engine"): self.engine.exit() self.engine.block() def connect(self, obj, name): """Connect obj and name to the daemon.""" cherrypy.tree.mount(obj, "/" + name) def disconnect(self, obj): """Disconnect obj from the daemon.""" pass def get_port(self): """Return the daemon port.""" return self.port def report_connection_if_denied(self): """Report connection if denied.""" self.client_reporter.report_connection_if_denied() def start(self): """Start quick web service.""" # cherrypy.config["tools.encode.on"] = True # cherrypy.config["tools.encode.encoding"] = "utf-8" cherrypy.config["server.socket_host"] = '0.0.0.0' cherrypy.config["engine.autoreload.on"] = False try: from OpenSSL import SSL, crypto cherrypy.config['server.ssl_module'] = 'pyopenSSL' cherrypy.config['server.ssl_certificate'] = self.cert cherrypy.config['server.ssl_private_key'] = self.pkey except ImportError: ERR.warning("no HTTPS/OpenSSL support") cherrypy.config['log.screen'] = None key = binascii.hexlify(os.urandom(16)) cherrypy.config.update({ 'tools.auth_digest.on': True, 'tools.auth_digest.realm': self.suite, 'tools.auth_digest.get_ha1': self.get_ha1, 'tools.auth_digest.key': key, 'tools.auth_digest.algorithm': self.hash_algorithm }) cherrypy.tools.connect_log = cherrypy.Tool( 'on_end_resource', self.report_connection_if_denied) cherrypy.config['tools.connect_log.on'] = True self.engine = cherrypy.engine for port in self.ok_ports: cherrypy.config["server.socket_port"] = port try: cherrypy.engine.start() cherrypy.engine.wait(cherrypy.engine.states.STARTED) except Exception: if cylc.flags.debug: traceback.print_exc() # We need to reinitialise the httpserver for each port attempt. cherrypy.server.httpserver = None else: if cherrypy.engine.state == cherrypy.engine.states.STARTED: self.port = port return raise Exception("No available ports")
class db_updater(threading.Thread): SCAN_INTERVAL = 60.0 def __init__(self, regd_treestore, filtr=None, timeout=None): self.db = SuiteSrvFilesManager() self.quit = False if timeout: self.timeout = float(timeout) else: self.timeout = None self.regd_treestore = regd_treestore super(db_updater, self).__init__() self.next_scan_time = None self.running_choices = None self.newtree = {} self.regd_choices = self.db.list_suites(filtr) def construct_newtree(self): """construct self.newtree[one][two]...[nnn] = [auth, descr, dir ]""" regd_choices = {} for suite, suite_dir, descr in sorted(self.regd_choices): regd_choices[suite] = (suite, suite_dir, descr) self.newtree = {} for suite, auth in self.running_choices: if suite in regd_choices: if is_remote_host(auth.split(':', 1)[0]): descr, suite_dir = (None, None) else: # local suite _, suite_dir, descr = regd_choices[suite] del regd_choices[suite] nest2 = self.newtree regp = suite.split(SuiteSrvFilesManager.DELIM) for key in regp[:-1]: if key not in nest2: nest2[key] = {} nest2 = nest2[key] nest2[(regp[-1], suite, auth)] = [auth, descr, suite_dir] for suite, suite_dir, descr in regd_choices.values(): suite_dir = re.sub('^' + os.environ['HOME'], '~', suite_dir) nest2 = self.newtree regp = suite.split(SuiteSrvFilesManager.DELIM) for key in regp[:-1]: if key not in nest2: nest2[key] = {} nest2 = nest2[key] nest2[(regp[-1], suite, '-')] = ['-', descr, suite_dir] def build_treestore(self, data, piter=None): for key, value in sorted(data.items()): if isinstance(key, tuple): item = key[0] else: item = key if isinstance(value, dict): # final three items are colours iter_ = self.regd_treestore.append( piter, [item, None, None, None, None, None, None]) self.build_treestore(value, iter_) else: state, descr, suite_dir = value self.regd_treestore.append( piter, [item, state, descr, suite_dir, None, None, None]) def update(self): """Update tree, if necessary.""" if self.next_scan_time is not None and self.next_scan_time > time(): return # Scan for running suites choices = [] for host, port, suite_identity in scan_all(timeout=self.timeout): name = suite_identity['name'] owner = suite_identity['owner'] if is_remote_user(owner): continue # current user only auth = "%s:%d" % (host, port) choices.append((name, auth)) choices.sort() self.next_scan_time = time() + self.SCAN_INTERVAL if choices == self.running_choices: return # Update tree if running suites changed self.running_choices = choices self.construct_newtree() self.update_treestore( self.newtree, self.regd_treestore.get_iter_first()) def update_treestore(self, new, iter_): # iter_ is None for an empty treestore (no suites registered) ts = self.regd_treestore if iter_: opath = ts.get_path(iter_) # get parent iter_ before pruning in case we prune last item at # this level piter = ts.iter_parent(iter_) else: opath = None piter = None def my_get_iter(item): # find the TreeIter pointing at item at this level if not opath: return None iter_ = ts.get_iter(opath) while iter_: val, = ts.get(iter_, 0) if val == item: return iter_ iter_ = ts.iter_next(iter_) return None # new items at this level new_items = new.keys() old_items = [] prune = [] while iter_: # iterate through old items at this level item, state, descr, dir = ts.get(iter_, 0, 1, 2, 3) if item not in new_items: # old item is not in new - prune it res = ts.remove(iter_) if not res: # Nec? iter_ = None else: # old item is in new - update it in case it changed old_items.append(item) # update old items that do appear in new chiter = ts.iter_children(iter_) if not isinstance(new[item], dict): # new item is not a group - update title etc. state, descr, dir = new[item] sc = self.statecol(state) ni = new[item] ts.set(iter_, 0, item, 1, ni[0], 2, ni[1], 3, ni[2], 4, sc[0], 5, sc[1], 6, sc[2]) if chiter: # old item was a group - kill its children while chiter: res = ts.remove(chiter) if not res: chiter = None else: # new item is a group if not chiter: # old item was not a group ts.set( iter_, 0, item, 1, None, 2, None, 3, None, 4, None, 5, None, 6, None) self.build_treestore(new[item], iter_) # continue iter_ = ts.iter_next(iter_) # return to original iter_ if opath: try: iter_ = ts.get_iter(opath) except ValueError: # removed the item pointed to # TODO - NEED TO WORRY ABOUT OTHERS AT THIS LEVEL? iter_ = None else: iter_ = None # add new items at this level for key in sorted(new_items): if isinstance(key, tuple): item = key[0] else: item = key if item not in old_items: # new data wasn't in old - add it if isinstance(new[key], dict): xiter = ts.append( piter, [item] + [None, None, None, None, None, None]) self.build_treestore(new[key], xiter) else: state, descr, dir = new[key] yiter = ts.append( piter, [item] + new[key] + list(self.statecol(state))) else: # new data was already in old if isinstance(new[key], dict): # check lower levels niter = my_get_iter(key) if niter: chiter = ts.iter_children(niter) if chiter: self.update_treestore(new[key], chiter) def run(self): """Main loop.""" while not self.quit: gobject.idle_add(self.update) sleep(0.1) def statecol(self, state): bg = '#19ae0a' fg = '#030' if state == '-': return (None, None, None) else: return (fg, bg, bg) def search_level(self, model, iter_, func, data): while iter_: if func(model, iter_, data): return iter_ iter_ = model.iter_next(iter_) return None def search_treemodel(self, model, iter_, func, data): while iter_: if func(model, iter_, data): return iter_ result = self.search_treemodel( model, model.iter_children(iter_), func, data) if result: return result iter_ = model.iter_next(iter_) return None def match_func(self, model, iter_, data): column, key = data value = model.get_value(iter_, column) return value == key
class SuiteRuntimeServiceClient(object): """Client for calling the HTTP(S) API of running suites.""" ANON_AUTH = ('anon', NO_PASSPHRASE, False) COMPAT_MAP = { # Limited pre-7.5.0 API compat mapping 'clear_broadcast': {0: 'broadcast/clear'}, 'expire_broadcast': {0: 'broadcast/expire'}, 'get_broadcast': {0: 'broadcast/get'}, 'get_info': {0: 'info/'}, 'get_suite_state_summary': {0: 'state/get_state_summary'}, 'put_broadcast': {0: 'broadcast/put'}, 'put_command': {0: 'command/'}, 'put_ext_trigger': {0: 'ext-trigger/put'}, 'put_messages': {0: 'message/put', 1: 'put_message'}, } ERROR_NO_HTTPS_SUPPORT = ( "ERROR: server has no HTTPS support," + " configure your global.rc file to use HTTP : {0}\n" ) METHOD = 'POST' METHOD_POST = 'POST' METHOD_GET = 'GET' MSG_RETRY_INTVL = 5.0 MSG_MAX_TRIES = 7 MSG_TIMEOUT = 30.0 def __init__( self, suite, owner=None, host=None, port=None, timeout=None, my_uuid=None, print_uuid=False, auth=None): self.suite = suite if not owner: owner = get_user() self.owner = owner self.host = host if self.host and self.host.split('.')[0] == 'localhost': self.host = get_host() elif self.host and '.' not in self.host: # Not IP and no domain self.host = get_fqdn_by_host(self.host) self.port = port self.srv_files_mgr = SuiteSrvFilesManager() if timeout is not None: timeout = float(timeout) self.timeout = timeout self.my_uuid = my_uuid or uuid4() if print_uuid: sys.stderr.write('%s\n' % self.my_uuid) self.prog_name = os.path.basename(sys.argv[0]) self.auth = auth self.session = None self.comms1 = {} # content in primary contact file self.comms2 = {} # content in extra contact file, e.g. contact via ssh def _compat(self, name, default=None): """Return server function name. Handle back-compat for pre-7.5.0 if relevant. """ # Need to load contact info here to get API version. self._load_contact_info() if default is None: default = name return self.COMPAT_MAP[name].get( self.comms1.get(self.srv_files_mgr.KEY_API), default) def clear_broadcast(self, payload): """Clear broadcast runtime task settings.""" return self._call_server( self._compat('clear_broadcast'), payload=payload) def expire_broadcast(self, **kwargs): """Expire broadcast runtime task settings.""" return self._call_server(self._compat('expire_broadcast'), **kwargs) def get_broadcast(self, **kwargs): """Return broadcast settings.""" return self._call_server( self._compat('get_broadcast'), method=self.METHOD_GET, **kwargs) def get_info(self, command, **kwargs): """Return suite info.""" return self._call_server( self._compat('get_info', default='') + command, method=self.METHOD_GET, **kwargs) def get_latest_state(self, full_mode=False): """Return latest state of the suite (for the GUI).""" self._load_contact_info() if self.comms1.get(self.srv_files_mgr.KEY_API) == 0: # Basic compat for pre-7.5.0 suites # Full mode only. # Error content/size not supported. # Report made-up main loop interval of 5.0 seconds. return { 'cylc_version': self.get_info('get_cylc_version'), 'full_mode': full_mode, 'summary': self.get_suite_state_summary(), 'ancestors': self.get_info('get_first_parent_ancestors'), 'ancestors_pruned': self.get_info( 'get_first_parent_ancestors', pruned=True), 'descendants': self.get_info('get_first_parent_descendants'), 'err_content': '', 'err_size': 0, 'mean_main_loop_interval': 5.0} else: return self._call_server( 'get_latest_state', method=self.METHOD_GET, full_mode=full_mode) def get_suite_state_summary(self): """Return the global, task, and family summary data structures.""" return utf8_enforce(self._call_server( self._compat('get_suite_state_summary'), method=self.METHOD_GET)) def identify(self): """Return suite identity.""" # Note on compat: Suites on 7.6.0 or above can just call "identify", # but has compat for "id/identity". return self._call_server('id/identify', method=self.METHOD_GET) def put_broadcast(self, payload): """Put/set broadcast runtime task settings.""" return self._call_server( self._compat('put_broadcast'), payload=payload) def put_command(self, command, **kwargs): """Invoke suite command.""" return self._call_server( self._compat('put_command', default='') + command, **kwargs) def put_ext_trigger(self, event_message, event_id): """Put external trigger.""" return self._call_server( self._compat('put_ext_trigger'), event_message=event_message, event_id=event_id) def put_messages(self, payload): """Send task messages to suite server program. Arguments: payload (dict): task_job (str): Task job as "CYCLE/TASK_NAME/SUBMIT_NUM". event_time (str): Event time as string. messages (list): List in the form [[severity, message], ...]. """ retry_intvl = float(self.comms1.get( self.srv_files_mgr.KEY_TASK_MSG_RETRY_INTVL, self.MSG_RETRY_INTVL)) max_tries = int(self.comms1.get( self.srv_files_mgr.KEY_TASK_MSG_MAX_TRIES, self.MSG_MAX_TRIES)) for i in range(1, max_tries + 1): # 1..max_tries inclusive orig_timeout = self.timeout if self.timeout is None: self.timeout = self.MSG_TIMEOUT try: func_name = self._compat('put_messages') if func_name == 'put_messages': results = self._call_server(func_name, payload=payload) elif func_name == 'put_message': # API 1, 7.5.0 compat cycle, name = payload['task_job'].split('/')[0:2] for severity, message in payload['messages']: results.append(self._call_server( func_name, task_id='%s.%s' % (name, cycle), severity=severity, message=message)) else: # API 0, pre-7.5.0 compat, priority instead of severity cycle, name = payload['task_job'].split('/')[0:2] for severity, message in payload['messages']: results.append(self._call_server( func_name, task_id='%s.%s' % (name, cycle), priority=severity, message=message)) except ClientInfoError: # Contact info file not found, suite probably not running. # Don't bother with retry, suite restart will poll any way. raise except ClientError as exc: now = get_current_time_string() sys.stderr.write( "%s WARNING - Message send failed, try %s of %s: %s\n" % ( now, i, max_tries, exc)) if i < max_tries: sys.stderr.write( " retry in %s seconds, timeout is %s\n" % ( retry_intvl, self.timeout)) sleep(retry_intvl) # Reset in case contact info or passphrase change self.comms1 = {} self.host = None self.port = None self.auth = None else: if i > 1: # Continue to write to STDERR, so users can easily see that # it has recovered from previous failures. sys.stderr.write( "%s INFO - Send message: try %s of %s succeeded\n" % ( get_current_time_string(), i, max_tries)) return results finally: self.timeout = orig_timeout def reset(self): """Compat method, does nothing.""" pass def signout(self): """Tell server to forget this client.""" return self._call_server('signout') def _call_server(self, function, method=METHOD, payload=None, **kwargs): """Build server URL + call it""" if self.comms2: return self._call_server_via_comms2(function, payload, **kwargs) url = self._call_server_get_url(function, **kwargs) # Remove proxy settings from environment for now environ = {} for key in ("http_proxy", "https_proxy"): val = os.environ.pop(key, None) if val: environ[key] = val try: return self.call_server_impl(url, method, payload) finally: os.environ.update(environ) def _call_server_get_url(self, function, **kwargs): """Build request URL.""" scheme = self.comms1.get(self.srv_files_mgr.KEY_COMMS_PROTOCOL) if scheme is None: # Use standard setting from global configuration scheme = glbl_cfg().get(['communication', 'method']) url = '%s://%s:%s/%s' % ( scheme, self.host, self.port, function) # If there are any parameters left in the dict after popping, # append them to the url. if kwargs: import urllib params = urllib.urlencode(kwargs, doseq=True) url += "?" + params return url def call_server_impl(self, url, method, payload): """Determine whether to use requests or urllib2 to call suite API.""" impl = self._call_server_impl_urllib2 try: import requests except ImportError: pass else: if [int(_) for _ in requests.__version__.split(".")] >= [2, 4, 2]: impl = self._call_server_impl_requests try: return impl(url, method, payload) except ClientConnectError as exc: if self.suite is None: raise # Cannot connect, perhaps suite is no longer running and is leaving # behind a contact file? try: self.srv_files_mgr.detect_old_contact_file( self.suite, (self.host, self.port)) except (AssertionError, SuiteServiceFileError): raise exc else: # self.srv_files_mgr.detect_old_contact_file should delete left # behind contact file if the old suite process no longer # exists. Should be safe to report that the suite has stopped. raise ClientConnectError(exc.args[0], exc.STOPPED % self.suite) def _call_server_impl_requests(self, url, method, payload): """Call server with "requests" library.""" import requests from requests.packages.urllib3.exceptions import InsecureRequestWarning warnings.simplefilter("ignore", InsecureRequestWarning) if self.session is None: self.session = requests.Session() if method == self.METHOD_POST: session_method = self.session.post else: session_method = self.session.get scheme = url.split(':', 1)[0] # Can use urlparse? username, password, verify = self._get_auth(scheme) try: ret = session_method( url, json=payload, verify=verify, proxies={}, headers=self._get_headers(), auth=requests.auth.HTTPDigestAuth(username, password), timeout=self.timeout ) except requests.exceptions.SSLError as exc: if "unknown protocol" in str(exc) and url.startswith("https:"): # Server is using http rather than https, for some reason. sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc)) raise CylcError( "Cannot issue commands over unsecured http.") if cylc.flags.debug: traceback.print_exc() raise ClientConnectError(url, exc) except requests.exceptions.Timeout as exc: if cylc.flags.debug: traceback.print_exc() raise ClientTimeout(url, exc) except requests.exceptions.RequestException as exc: if cylc.flags.debug: traceback.print_exc() raise ClientConnectError(url, exc) if ret.status_code == 401: access_desc = 'private' if self.auth == self.ANON_AUTH: access_desc = 'public' raise ClientDeniedError(url, self.prog_name, access_desc) if ret.status_code >= 400: exception_text = get_exception_from_html(ret.text) if exception_text: sys.stderr.write(exception_text) else: sys.stderr.write(ret.text) try: ret.raise_for_status() except requests.exceptions.HTTPError as exc: if cylc.flags.debug: traceback.print_exc() raise ClientConnectedError(url, exc) if self.auth and self.auth[1] != NO_PASSPHRASE: self.srv_files_mgr.cache_passphrase( self.suite, self.owner, self.host, self.auth[1]) try: return ret.json() except ValueError: return ret.text def _call_server_impl_urllib2(self, url, method, payload): """Call server with "urllib2" library.""" import json import urllib2 import ssl unverified_context = getattr(ssl, '_create_unverified_context', None) if unverified_context is not None: ssl._create_default_https_context = unverified_context scheme = url.split(':', 1)[0] # Can use urlparse? username, password = self._get_auth(scheme)[0:2] auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() auth_manager.add_password(None, url, username, password) auth = urllib2.HTTPDigestAuthHandler(auth_manager) opener = urllib2.build_opener(auth, urllib2.HTTPSHandler()) headers_list = self._get_headers().items() if payload: payload = json.dumps(payload) headers_list.append(('Accept', 'application/json')) json_headers = {'Content-Type': 'application/json', 'Content-Length': len(payload)} else: payload = None json_headers = {'Content-Length': 0} opener.addheaders = headers_list req = urllib2.Request(url, payload, json_headers) # This is an unpleasant monkey patch, but there isn't an # alternative. urllib2 uses POST if there is a data payload # but that is not the correct criterion. # The difference is basically that POST changes # server state and GET doesn't. req.get_method = lambda: method try: response = opener.open(req, timeout=self.timeout) except urllib2.URLError as exc: if "unknown protocol" in str(exc) and url.startswith("https:"): # Server is using http rather than https, for some reason. sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc)) raise CylcError( "Cannot issue commands over unsecured http.") if cylc.flags.debug: traceback.print_exc() if "timed out" in str(exc): raise ClientTimeout(url, exc) else: raise ClientConnectError(url, exc) except Exception as exc: if cylc.flags.debug: traceback.print_exc() raise ClientError(url, exc) if response.getcode() == 401: access_desc = 'private' if self.auth == self.ANON_AUTH: access_desc = 'public' raise ClientDeniedError(url, self.prog_name, access_desc) response_text = response.read() if response.getcode() >= 400: exception_text = get_exception_from_html(response_text) if exception_text: sys.stderr.write(exception_text) else: sys.stderr.write(response_text) raise ClientConnectedError( url, "%s HTTP return code" % response.getcode()) if self.auth and self.auth[1] != NO_PASSPHRASE: self.srv_files_mgr.cache_passphrase( self.suite, self.owner, self.host, self.auth[1]) try: return json.loads(response_text) except ValueError: return response_text def _call_server_via_comms2(self, function, payload, **kwargs): """Call server via "cylc client --use-ssh". Call "cylc client --use-ssh" using `subprocess.Popen`. Payload and arguments of the API method are serialized as JSON and are written to a temporary file, which is then used as the STDIN of the "cylc client" command. The external call here should be even safer than a direct HTTP(S) call, as it can be blocked by SSH before it even gets a chance to make the subsequent HTTP(S) call. Arguments: function (str): name of API method, argument 1 of "cylc client". payload (str): extra data or information for the API method. **kwargs (dict): arguments for the API method. """ import json from cylc.remote import remote_cylc_cmd command = ["client", function, self.suite] if payload: kwargs["payload"] = payload if kwargs: from tempfile import TemporaryFile stdin = TemporaryFile() json.dump(kwargs, stdin) stdin.seek(0) else: # With stdin=None, `remote_cylc_cmd` will: # * Set stdin to open(os.devnull) # * Add `-n` to the SSH command stdin = None proc = remote_cylc_cmd( command, self.owner, self.host, capture=True, ssh_login_shell=(self.comms1.get( self.srv_files_mgr.KEY_SSH_USE_LOGIN_SHELL ) in ['True', 'true']), ssh_cylc=(r'%s/bin/cylc' % self.comms1.get( self.srv_files_mgr.KEY_DIR_ON_SUITE_HOST) ), stdin=stdin, ) out = proc.communicate()[0] return_code = proc.wait() if return_code: from pipes import quote command_str = " ".join(quote(item) for item in command) raise ClientError(command_str, "return-code=%d" % return_code) return json.loads(out) def _get_auth(self, protocol): """Return a user/password Digest Auth.""" if self.auth is None: self.auth = self.ANON_AUTH try: pphrase = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_PASSPHRASE, self.suite, self.owner, self.host, content=True) if protocol == 'https': verify = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_CERT, self.suite, self.owner, self.host) else: verify = False except SuiteServiceFileError: pass else: if pphrase and pphrase != NO_PASSPHRASE: self.auth = ('cylc', pphrase, verify) return self.auth def _get_headers(self): """Return HTTP headers identifying the client.""" user_agent_string = ( "cylc/%s prog_name/%s uuid/%s" % ( CYLC_VERSION, self.prog_name, self.my_uuid ) ) auth_info = "%s@%s" % (get_user(), get_host()) return {"User-Agent": user_agent_string, "From": auth_info} def _load_contact_info(self): """Obtain suite owner, host, port info. Determine host and port using content in port file, unless already specified. """ if self.host and self.port: return if self.port: # In case the contact file is corrupted, user can specify the port. self.host = get_host() return try: # Always trust the values in the contact file otherwise. self.comms1 = self.srv_files_mgr.load_contact_file( self.suite, self.owner, self.host) # Port inside "try" block, as it needs a type conversion self.port = int(self.comms1.get(self.srv_files_mgr.KEY_PORT)) except (IOError, ValueError, SuiteServiceFileError): raise ClientInfoError(self.suite) else: # Check mismatch suite UUID env_suite = os.getenv(self.srv_files_mgr.KEY_NAME) env_uuid = os.getenv(self.srv_files_mgr.KEY_UUID) if (self.suite and env_suite and env_suite == self.suite and env_uuid and env_uuid != self.comms1.get(self.srv_files_mgr.KEY_UUID)): raise ClientInfoUUIDError( env_uuid, self.comms1[self.srv_files_mgr.KEY_UUID]) # All good self.host = self.comms1.get(self.srv_files_mgr.KEY_HOST) self.owner = self.comms1.get(self.srv_files_mgr.KEY_OWNER) if self.srv_files_mgr.KEY_API not in self.comms1: self.comms1[self.srv_files_mgr.KEY_API] = 0 # <=7.5.0 compat # Indirect comms settings self.comms2.clear() try: self.comms2.update(self.srv_files_mgr.load_contact_file( self.suite, self.owner, self.host, SuiteSrvFilesManager.FILE_BASE_CONTACT2)) except SuiteServiceFileError: pass
def _scan1_impl(conn, timeout, my_uuid): """Connect to host:port to get suite identify.""" srv_files_mgr = SuiteSrvFilesManager() while True: if not conn.poll(SLEEP_INTERVAL): continue item = conn.recv() if item == MSG_QUIT: break host, port = item host_anon = host if is_remote_host(host): host_anon = get_host_ip_by_name(host) # IP reduces DNS traffic client = SuiteIdClientAnon(None, host=host_anon, port=port, my_uuid=my_uuid, timeout=timeout) try: result = client.identify() except ConnectionTimeout as exc: conn.send((host, port, MSG_TIMEOUT)) except (ConnectionError, SuiteStillInitialisingError) as exc: conn.send((host, port, None)) else: owner = result.get('owner') name = result.get('name') states = result.get('states', None) if cylc.flags.debug: print >> sys.stderr, ' suite:', name, owner if states is None: # This suite keeps its state info private. # Try again with the passphrase if I have it. try: pphrase = srv_files_mgr.get_auth_item( srv_files_mgr.FILE_BASE_PASSPHRASE, name, owner, host, content=True) except SuiteServiceFileError: pass else: if pphrase: client = SuiteIdClient(name, owner=owner, host=host, port=port, my_uuid=my_uuid, timeout=timeout) try: result = client.identify() except SuiteStillInitialisingError as exc: if cylc.flags.debug: print >> sys.stderr, ( ' (connected with passphrase,' + ' suite initialising)') except ConnectionError as exc: # Nope (private suite, wrong passphrase). if cylc.flags.debug: print >> sys.stderr, ' (wrong passphrase)' else: if cylc.flags.debug: print >> sys.stderr, ( ' (got states with passphrase)') conn.send((host, port, result)) conn.close()
class SuiteRuntimeServiceClient(object): """Client for calling the HTTP(S) API of running suites.""" ANON_AUTH = ('anon', NO_PASSPHRASE, False) ERROR_NO_HTTPS_SUPPORT = ( "ERROR: server has no HTTPS support," + " configure your global.rc file to use HTTP : {0}\n") METHOD = 'POST' METHOD_POST = 'POST' METHOD_GET = 'GET' def __init__(self, suite, owner=None, host=None, port=None, timeout=None, my_uuid=None, print_uuid=False, comms_protocol=None, auth=None): self.suite = suite if not owner: owner = get_user() self.owner = owner self.host = host if self.host and self.host.split('.')[0] == 'localhost': self.host = get_host() elif self.host and '.' not in self.host: # Not IP and no domain self.host = get_fqdn_by_host(self.host) self.port = port self.srv_files_mgr = SuiteSrvFilesManager() self.comms_protocol = comms_protocol if timeout is not None: timeout = float(timeout) self.timeout = timeout self.my_uuid = my_uuid or uuid4() if print_uuid: print >> sys.stderr, '%s' % self.my_uuid self.prog_name = os.path.basename(sys.argv[0]) self.auth = auth def clear_broadcast(self, **kwargs): """Clear broadcast runtime task settings.""" return self._call_server('clear_broadcast', payload=kwargs) def expire_broadcast(self, **kwargs): """Expire broadcast runtime task settings.""" return self._call_server('expire_broadcast', **kwargs) def get_broadcast(self, **kwargs): """Return broadcast settings.""" return self._call_server('get_broadcast', method=self.METHOD_GET, **kwargs) def get_info(self, command, *args, **kwargs): """Return suite info.""" kwargs['method'] = self.METHOD_GET return self._call_server(command, *args, **kwargs) def get_latest_state(self, full_mode): """Return latest state of the suite (for the GUI).""" return self._call_server('get_latest_state', method=self.METHOD_GET, full_mode=full_mode) def get_suite_state_summary(self): """Return the global, task, and family summary data structures.""" return utf8_enforce( self._call_server('get_suite_state_summary', method=self.METHOD_GET)) def get_tasks_by_state(self): """Returns a dict containing lists of tasks by state. Result in the form: {state: [(most_recent_time_string, task_name, point_string), ...]} """ return self._call_server('get_tasks_by_state', method=self.METHOD_GET) def identify(self): """Return suite identity.""" return self._call_server('identify', method=self.METHOD_GET) def put_broadcast(self, **kwargs): """Put/set broadcast runtime task settings.""" return self._call_server('put_broadcast', payload=kwargs) def put_command(self, command, **kwargs): """Invoke suite command.""" return self._call_server(command, **kwargs) def put_ext_trigger(self, event_message, event_id): """Put external trigger.""" return self._call_server('put_ext_trigger', event_message=event_message, event_id=event_id) def put_message(self, task_id, priority, message): """Send task message.""" return self._call_server('put_message', task_id=task_id, priority=priority, message=message) def reset(self, *args, **kwargs): """Compat method, does nothing.""" pass def signout(self, *args, **kwargs): """Tell server to forget this client.""" return self._call_server('signout') def _compile_request(self, func_dict, host, comms_protocol=None): """Build request URL.""" payload = func_dict.pop("payload", None) method = func_dict.pop("method", self.METHOD) function = func_dict.pop("function", None) if comms_protocol is None: # Use standard setting from global configuration from cylc.cfgspec.globalcfg import GLOBAL_CFG comms_protocol = GLOBAL_CFG.get(['communication', 'method']) url = '%s://%s:%s/%s' % (comms_protocol, host, self.port, function) # If there are any parameters left in the dict after popping, # append them to the url. if func_dict: import urllib params = urllib.urlencode(func_dict, doseq=True) url += "?" + params return (method, url, payload, comms_protocol) def _call_server(self, *func_dicts, **fargs): """func_dict is a dictionary of command names (fnames) and arguments to that command""" # Deal with the case of one func_dict/function name passed # by converting them to the generic case: a dictionary of # a single function and its function arguments. if isinstance(func_dicts[0], str): func_dict = {"function": func_dicts[0]} func_dict.update(fargs) else: func_dict = None try: self._load_contact_info() except (IOError, ValueError, SuiteServiceFileError): raise ClientInfoError(self.suite) http_request_items = [] try: # dictionary containing: url, payload, method http_request_items.append( self._compile_request(func_dict, self.host, self.comms_protocol)) except (IndexError, ValueError, AttributeError): for f_dict in func_dicts: http_request_items.append( self._compile_request(f_dict, self.host, self.comms_protocol)) # Remove proxy settings from environment for now environ = {} for key in ("http_proxy", "https_proxy"): val = os.environ.pop(key, None) if val: environ[key] = val # Returns a list of http returns from the requests try: return self.call_server_impl(http_request_items) finally: os.environ.update(environ) def call_server_impl(self, http_request_items): """Determine whether to use requests or urllib2 to call suite API.""" method = self._call_server_impl_urllib2 try: import requests except ImportError: pass else: if [int(_) for _ in requests.__version__.split(".")] >= [2, 4, 2]: method = self._call_server_impl_requests try: return method(http_request_items) except ClientConnectError as exc: if self.suite is None: raise # Cannot connect, perhaps suite is no longer running and is leaving # behind a contact file? try: self.srv_files_mgr.detect_old_contact_file( self.suite, (self.host, self.port)) except (AssertionError, SuiteServiceFileError): raise exc else: # self.srv_files_mgr.detect_old_contact_file should delete left # behind contact file if the old suite process no longer # exists. Should be safe to report that the suite has stopped. raise ClientConnectError(exc.args[0], exc.STOPPED % self.suite) def _call_server_impl_requests(self, http_request_items): """Call server with "requests" library.""" import requests from requests.packages.urllib3.exceptions import InsecureRequestWarning warnings.simplefilter("ignore", InsecureRequestWarning) if not hasattr(self, "session"): self.session = requests.Session() http_return_items = [] for method, url, payload, comms_protocol in http_request_items: if method is None: method = self.METHOD if method == self.METHOD_POST: session_method = self.session.post else: session_method = self.session.get username, password, verify = self._get_auth(comms_protocol) try: ret = session_method(url, json=payload, verify=verify, proxies={}, headers=self._get_headers(), auth=requests.auth.HTTPDigestAuth( username, password), timeout=self.timeout) except requests.exceptions.SSLError as exc: if "unknown protocol" in str(exc) and url.startswith("https:"): # Server is using http rather than https, for some reason. sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc)) raise CylcError( "Cannot issue commands over unsecured http.") if cylc.flags.debug: traceback.print_exc() raise ClientConnectError(url, exc) except requests.exceptions.Timeout as exc: if cylc.flags.debug: traceback.print_exc() raise ClientTimeout(url, exc) except requests.exceptions.RequestException as exc: if cylc.flags.debug: traceback.print_exc() raise ClientConnectError(url, exc) if ret.status_code == 401: access_desc = 'private' if self.auth == self.ANON_AUTH: access_desc = 'public' raise ClientDeniedError(url, self.prog_name, access_desc) if ret.status_code >= 400: exception_text = get_exception_from_html(ret.text) if exception_text: sys.stderr.write(exception_text) else: sys.stderr.write(ret.text) try: ret.raise_for_status() except requests.exceptions.HTTPError as exc: if cylc.flags.debug: traceback.print_exc() raise ClientConnectedError(url, exc) if self.auth and self.auth[1] != NO_PASSPHRASE: self.srv_files_mgr.cache_passphrase(self.suite, self.owner, self.host, self.auth[1]) try: ret = ret.json() http_return_items.append(ret) except ValueError: ret = ret.text http_return_items.append(ret) # Return a single http return or a list of them if multiple return (http_return_items if len(http_return_items) > 1 else http_return_items[0]) def _call_server_impl_urllib2(self, http_request_items): """Call server with "urllib2" library.""" import json import urllib2 import ssl if hasattr(ssl, '_create_unverified_context'): ssl._create_default_https_context = ssl._create_unverified_context http_return_items = [] for method, url, payload, comms_protocol in http_request_items: if method is None: method = self.METHOD username, password = self._get_auth(comms_protocol)[0:2] auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() auth_manager.add_password(None, url, username, password) auth = urllib2.HTTPDigestAuthHandler(auth_manager) opener = urllib2.build_opener(auth, urllib2.HTTPSHandler()) headers_list = self._get_headers().items() if payload: payload = json.dumps(payload) headers_list.append(('Accept', 'application/json')) json_headers = { 'Content-Type': 'application/json', 'Content-Length': len(payload) } else: payload = None json_headers = {'Content-Length': 0} opener.addheaders = headers_list req = urllib2.Request(url, payload, json_headers) # This is an unpleasant monkey patch, but there isn't an # alternative. urllib2 uses POST if there is a data payload # but that is not the correct criterion. # The difference is basically that POST changes # server state and GET doesn't. req.get_method = lambda: method try: response = opener.open(req, timeout=self.timeout) except urllib2.URLError as exc: if "unknown protocol" in str(exc) and url.startswith("https:"): # Server is using http rather than https, for some reason. sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc)) raise CylcError( "Cannot issue commands over unsecured http.") if cylc.flags.debug: traceback.print_exc() if "timed out" in str(exc): raise ClientTimeout(url, exc) else: raise ClientConnectError(url, exc) except Exception as exc: if cylc.flags.debug: traceback.print_exc() raise ClientError(url, exc) if response.getcode() == 401: access_desc = 'private' if self.auth == self.ANON_AUTH: access_desc = 'public' raise ClientDeniedError(url, self.prog_name, access_desc) response_text = response.read() if response.getcode() >= 400: exception_text = get_exception_from_html(response_text) if exception_text: sys.stderr.write(exception_text) else: sys.stderr.write(response_text) raise ClientConnectedError( url, "%s HTTP return code" % response.getcode()) if self.auth and self.auth[1] != NO_PASSPHRASE: self.srv_files_mgr.cache_passphrase(self.suite, self.owner, self.host, self.auth[1]) try: http_return_items.append(json.loads(response_text)) except ValueError: http_return_items.append(response_text) # Return a single http return or a list of them if multiple return (http_return_items if len(http_return_items) > 1 else http_return_items[0]) def _get_auth(self, protocol): """Return a user/password Digest Auth.""" if self.auth is None: self.auth = self.ANON_AUTH try: pphrase = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_PASSPHRASE, self.suite, self.owner, self.host, content=True) if protocol == 'https': verify = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_CERT, self.suite, self.owner, self.host) else: verify = False except SuiteServiceFileError: pass else: if pphrase and pphrase != NO_PASSPHRASE: self.auth = ('cylc', pphrase, verify) return self.auth def _get_headers(self): """Return HTTP headers identifying the client.""" user_agent_string = ("cylc/%s prog_name/%s uuid/%s" % (CYLC_VERSION, self.prog_name, self.my_uuid)) auth_info = "%s@%s" % (get_user(), get_host()) return {"User-Agent": user_agent_string, "From": auth_info} def _load_contact_info(self): """Obtain suite owner, host, port info. Determine host and port using content in port file, unless already specified. """ if self.host and self.port: return if self.port: # In case the contact file is corrupted, user can specify the port. self.host = get_host() return # Always trust the values in the contact file otherwise. data = self.srv_files_mgr.load_contact_file(self.suite, self.owner, self.host) self.host = data.get(self.srv_files_mgr.KEY_HOST) self.port = int(data.get(self.srv_files_mgr.KEY_PORT)) self.owner = data.get(self.srv_files_mgr.KEY_OWNER) self.comms_protocol = data.get(self.srv_files_mgr.KEY_COMMS_PROTOCOL)
class HTTPServer(object): """HTTP(S) server by cherrypy, for serving suite runtime API.""" API = 2 LOG_CONNECT_DENIED_TMPL = "[client-connect] DENIED %s@%s:%s %s" RE_MESSAGE_TIME = re.compile( r'\A(.+) at (' + RE_DATE_TIME_FORMAT_EXTENDED + r')\Z', re.DOTALL) def __init__(self, suite): # Suite only needed for back-compat with old clients (see below): self.suite = suite self.engine = None self.port = None # Figure out the ports we are allowed to use. base_port = glbl_cfg().get(['communication', 'base port']) max_ports = glbl_cfg().get( ['communication', 'maximum number of ports']) self.ok_ports = range(int(base_port), int(base_port) + int(max_ports)) random.shuffle(self.ok_ports) comms_options = glbl_cfg().get(['communication', 'options']) # HTTP Digest Auth uses MD5 - pretty secure in this use case. # Extending it with extra algorithms is allowed, but won't be # supported by most browsers. requests and urllib2 are OK though. self.hash_algorithm = "MD5" if "SHA1" in comms_options: # Note 'SHA' rather than 'SHA1'. self.hash_algorithm = "SHA" self.srv_files_mgr = SuiteSrvFilesManager() self.comms_method = glbl_cfg().get(['communication', 'method']) self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain( { 'cylc': self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_PASSPHRASE, suite, content=True), 'anon': NO_PASSPHRASE }, algorithm=self.hash_algorithm) if self.comms_method == 'http': self.cert = None self.pkey = None else: # if self.comms_method in [None, 'https']: try: self.cert = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_CERT, suite) self.pkey = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_PEM, suite) except SuiteServiceFileError: ERR.error("no HTTPS/OpenSSL support. Aborting...") raise CylcError("No HTTPS support. " "Configure user's global.rc to use HTTP.") self.start() @cherrypy.expose def apiversion(self): """Return API version.""" return str(self.API) @staticmethod def connect(schd): """Mount suite schedular object to the web server.""" cherrypy.tree.mount(SuiteRuntimeService(schd), '/') # For back-compat with "scan" cherrypy.tree.mount(SuiteRuntimeService(schd), '/id') @staticmethod def disconnect(schd): """Disconnect obj from the web server.""" del cherrypy.tree.apps['/%s/%s' % (schd.owner, schd.suite)] def get_port(self): """Return the web server port.""" return self.port def shutdown(self): """Shutdown the web server.""" if hasattr(self, "engine"): self.engine.exit() self.engine.block() def start(self): """Start quick web service.""" # cherrypy.config["tools.encode.on"] = True # cherrypy.config["tools.encode.encoding"] = "utf-8" cherrypy.config["server.socket_host"] = get_host() cherrypy.config["engine.autoreload.on"] = False if self.comms_method == "https": # Setup SSL etc. Otherwise fail and exit. # Require connection method to be the same e.g HTTP/HTTPS matching. cherrypy.config['server.ssl_module'] = 'pyopenSSL' cherrypy.config['server.ssl_certificate'] = self.cert cherrypy.config['server.ssl_private_key'] = self.pkey cherrypy.config['log.screen'] = None key = binascii.hexlify(os.urandom(16)) cherrypy.config.update({ 'tools.auth_digest.on': True, 'tools.auth_digest.realm': self.suite, 'tools.auth_digest.get_ha1': self.get_ha1, 'tools.auth_digest.key': key, 'tools.auth_digest.algorithm': self.hash_algorithm }) cherrypy.tools.connect_log = cherrypy.Tool( 'on_end_resource', self._report_connection_if_denied) cherrypy.config['tools.connect_log.on'] = True self.engine = cherrypy.engine for port in self.ok_ports: cherrypy.config["server.socket_port"] = port try: cherrypy.engine.start() cherrypy.engine.wait(cherrypy.engine.states.STARTED) except cherrypy.process.wspbus.ChannelFailures: if cylc.flags.debug: traceback.print_exc() # We need to reinitialise the httpserver for each port attempt. cherrypy.server.httpserver = None else: if cherrypy.engine.state == cherrypy.engine.states.STARTED: self.port = port return raise Exception("No available ports") @staticmethod def _get_client_connection_denied(): """Return whether a connection was denied.""" if "Authorization" not in cherrypy.request.headers: # Probably just the initial HTTPS handshake. return False status = cherrypy.response.status if isinstance(status, basestring): return cherrypy.response.status.split()[0] in ["401", "403"] return cherrypy.response.status in [401, 403] def _report_connection_if_denied(self): """Log an (un?)successful connection attempt.""" prog_name, user, host, uuid = _get_client_info()[1:] connection_denied = self._get_client_connection_denied() if connection_denied: LOG.warning(self.__class__.LOG_CONNECT_DENIED_TMPL % ( user, host, prog_name, uuid))
def __init__(self): self.initialised = {} # {(user, host): should_unlink, ...} self.single_task_mode = False self.suite_srv_files_mgr = SuiteSrvFilesManager()
class BaseCommsClient(object): """Base class for client-side suite object interfaces.""" ACCESS_DESCRIPTION = 'private' METHOD = 'POST' METHOD_POST = 'POST' METHOD_GET = 'GET' def __init__(self, suite, owner=USER, host=None, port=None, timeout=None, my_uuid=None, print_uuid=False): self.suite = suite self.owner = owner self.host = host self.port = port if timeout is not None: timeout = float(timeout) self.timeout = timeout self.my_uuid = my_uuid or uuid4() if print_uuid: print >> sys.stderr, '%s' % self.my_uuid self.srv_files_mgr = SuiteSrvFilesManager() self.prog_name = os.path.basename(sys.argv[0]) self.server_cert = None self.auth = None def call_server_func(self, category, fname, **fargs): """Call server_object.fname(*fargs, **fargs).""" if self.host is None and self.port is not None: self.host = get_hostname() try: self._load_contact_info() except (IOError, ValueError, SuiteServiceFileError): raise ConnectionInfoError(self.suite) handle_proxies() payload = fargs.pop("payload", None) method = fargs.pop("method", self.METHOD) host = self.host if host == "localhost": host = get_hostname().split(".")[0] url = 'https://%s:%s/%s/%s' % (host, self.port, category, fname) if fargs: import urllib params = urllib.urlencode(fargs, doseq=True) url += "?" + params return self._get_data_from_url(url, payload, method=method) def _get_data_from_url(self, url, json_data, method=None): requests_ok = True try: import requests except ImportError: requests_ok = False else: version = [int(_) for _ in requests.__version__.split(".")] if version < [2, 4, 2]: requests_ok = False if requests_ok: return self._get_data_from_url_with_requests( url, json_data, method=method) return self._get_data_from_url_with_urllib2( url, json_data, method=method) def _get_data_from_url_with_requests(self, url, json_data, method=None): import requests username, password = self._get_auth() auth = requests.auth.HTTPDigestAuth(username, password) if not hasattr(self, "session"): self.session = requests.Session() if method is None: method = self.METHOD if method == self.METHOD_POST: session_method = self.session.post else: session_method = self.session.get try: ret = session_method( url, json=json_data, verify=self._get_verify(), proxies={}, headers=self._get_headers(), auth=auth, timeout=self.timeout ) except requests.exceptions.SSLError as exc: if "unknown protocol" in str(exc) and url.startswith("https:"): # Server is using http rather than https, for some reason. sys.stderr.write(WARNING_NO_HTTPS_SUPPORT.format(exc)) return self._get_data_from_url_with_requests( url.replace("https:", "http:", 1), json_data) if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) except requests.exceptions.Timeout as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionTimeout(url, exc) except requests.exceptions.RequestException as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) if ret.status_code == 401: raise ConnectionDeniedError(url, self.prog_name, self.ACCESS_DESCRIPTION) if ret.status_code >= 400: from cylc.network.https.util import get_exception_from_html exception_text = get_exception_from_html(ret.text) if exception_text: sys.stderr.write(exception_text) else: sys.stderr.write(ret.text) try: ret.raise_for_status() except requests.exceptions.HTTPError as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) if self.auth and self.auth[1] != NO_PASSPHRASE: self.srv_files_mgr.cache_passphrase( self.suite, self.owner, self.host, self.auth[1]) try: return ret.json() except ValueError: return ret.text def _get_data_from_url_with_urllib2(self, url, json_data, method=None): import json import urllib2 import ssl if hasattr(ssl, '_create_unverified_context'): ssl._create_default_https_context = ssl._create_unverified_context if method is None: method = self.METHOD orig_json_data = json_data username, password = self._get_auth() auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() auth_manager.add_password(None, url, username, password) auth = urllib2.HTTPDigestAuthHandler(auth_manager) opener = urllib2.build_opener(auth, urllib2.HTTPSHandler()) headers_list = self._get_headers().items() if json_data: json_data = json.dumps(json_data) headers_list.append(('Accept', 'application/json')) json_headers = {'Content-Type': 'application/json', 'Content-Length': len(json_data)} else: json_data = None json_headers = {'Content-Length': 0} opener.addheaders = headers_list req = urllib2.Request(url, json_data, json_headers) # This is an unpleasant monkey patch, but there isn't an alternative. # urllib2 uses POST if there is a data payload, but that is not the # correct criterion. The difference is basically that POST changes # server state and GET doesn't. req.get_method = lambda: method try: response = opener.open(req, timeout=self.timeout) except urllib2.URLError as exc: if "unknown protocol" in str(exc) and url.startswith("https:"): # Server is using http rather than https, for some reason. sys.stderr.write(WARNING_NO_HTTPS_SUPPORT.format(exc)) return self._get_data_from_url_with_urllib2( url.replace("https:", "http:", 1), orig_json_data) if cylc.flags.debug: import traceback traceback.print_exc() if "timed out" in str(exc): raise ConnectionTimeout(url, exc) else: raise ConnectionError(url, exc) except Exception as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) if response.getcode() == 401: raise ConnectionDeniedError(url, self.prog_name, self.ACCESS_DESCRIPTION) response_text = response.read() if response.getcode() >= 400: from cylc.network.https.util import get_exception_from_html exception_text = get_exception_from_html(response_text) if exception_text: sys.stderr.write(exception_text) else: sys.stderr.write(response_text) raise ConnectionError(url, "%s HTTP return code" % response.getcode()) if self.auth and self.auth[1] != NO_PASSPHRASE: self.srv_files_mgr.cache_passphrase( self.suite, self.owner, self.host, self.auth[1]) try: return json.loads(response_text) except ValueError: return response_text def _get_auth(self): """Return a user/password Digest Auth.""" if self.auth is None: self.auth = ('anon', NO_PASSPHRASE) try: pphrase = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_PASSPHRASE, self.suite, self.owner, self.host, content=True) except SuiteServiceFileError: pass else: if pphrase and pphrase != NO_PASSPHRASE: self.auth = ('cylc', pphrase) return self.auth def _get_headers(self): """Return HTTP headers identifying the client.""" user_agent_string = ( "cylc/%s prog_name/%s uuid/%s" % ( CYLC_VERSION, self.prog_name, self.my_uuid ) ) auth_info = "%s@%s" % (USER, get_hostname()) return {"User-Agent": user_agent_string, "From": auth_info} def _get_verify(self): """Return the server certificate if possible.""" if self.server_cert is None: try: self.server_cert = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_CERT, self.suite, self.owner, self.host) except SuiteServiceFileError: self.server_cert = False return self.server_cert def _load_contact_info(self): """Obtain suite owner, host, port info. Determine host and port using content in port file, unless already specified. """ if self.host and self.port: return data = self.srv_files_mgr.load_contact_file( self.suite, self.owner, self.host) if not self.host: self.host = data.get(self.srv_files_mgr.KEY_HOST) if not self.port: self.port = int(data.get(self.srv_files_mgr.KEY_PORT)) if not self.owner: self.owner = data.get(self.srv_files_mgr.KEY_OWNER) def reset(self, *args, **kwargs): pass def signout(self, *args, **kwargs): pass
class BaseCommsClient(object): """Base class for client-side suite object interfaces.""" ACCESS_DESCRIPTION = 'private' METHOD = 'POST' METHOD_POST = 'POST' METHOD_GET = 'GET' def __init__(self, suite, owner=USER, host=None, port=None, timeout=None, my_uuid=None, print_uuid=False): self.suite = suite self.owner = owner self.host = host self.port = port if timeout is not None: timeout = float(timeout) self.timeout = timeout self.my_uuid = my_uuid or uuid4() if print_uuid: print >> sys.stderr, '%s' % self.my_uuid self.srv_files_mgr = SuiteSrvFilesManager() self.prog_name = os.path.basename(sys.argv[0]) self.server_cert = None self.auth = None def call_server_func(self, category, fname, **fargs): """Call server_object.fname(*fargs, **fargs).""" if self.host is None and self.port is not None: self.host = get_hostname() if self.host is None or self.port is None: try: self._load_contact_info() except (IOError, ValueError, SuiteServiceFileError): raise ConnectionInfoError(self.suite) handle_proxies() payload = fargs.pop("payload", None) method = fargs.pop("method", self.METHOD) host = self.host if not self.host.split(".")[0].isdigit(): host = self.host.split(".")[0] if host == "localhost": host = get_hostname().split(".")[0] url = 'https://%s:%s/%s/%s' % (host, self.port, category, fname) if fargs: import urllib params = urllib.urlencode(fargs, doseq=True) url += "?" + params return self._get_data_from_url(url, payload, method=method) def _get_data_from_url(self, url, json_data, method=None): requests_ok = True try: import requests except ImportError: requests_ok = False else: version = [int(_) for _ in requests.__version__.split(".")] if version < [2, 4, 2]: requests_ok = False if requests_ok: return self._get_data_from_url_with_requests( url, json_data, method=method) return self._get_data_from_url_with_urllib2( url, json_data, method=method) def _get_data_from_url_with_requests(self, url, json_data, method=None): import requests username, password = self._get_auth() auth = requests.auth.HTTPDigestAuth(username, password) if not hasattr(self, "session"): self.session = requests.Session() if method is None: method = self.METHOD if method == self.METHOD_POST: session_method = self.session.post else: session_method = self.session.get try: ret = session_method( url, json=json_data, verify=self._get_verify(), proxies={}, headers=self._get_headers(), auth=auth, timeout=self.timeout ) except requests.exceptions.SSLError as exc: if "unknown protocol" in str(exc) and url.startswith("https:"): # Server is using http rather than https, for some reason. sys.stderr.write(WARNING_NO_HTTPS_SUPPORT.format(exc)) return self._get_data_from_url_with_requests( url.replace("https:", "http:", 1), json_data) if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) except requests.exceptions.Timeout as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionTimeout(url, exc) except requests.exceptions.RequestException as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) if ret.status_code == 401: raise ConnectionDeniedError(url, self.prog_name, self.ACCESS_DESCRIPTION) if ret.status_code >= 400: from cylc.network.https.util import get_exception_from_html exception_text = get_exception_from_html(ret.text) if exception_text: sys.stderr.write(exception_text) else: sys.stderr.write(ret.text) try: ret.raise_for_status() except requests.exceptions.HTTPError as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) if self.auth and self.auth[1] != NO_PASSPHRASE: self.srv_files_mgr.cache_passphrase( self.suite, self.owner, self.host, self.auth[1]) try: return ret.json() except ValueError: return ret.text def _get_data_from_url_with_urllib2(self, url, json_data, method=None): import json import urllib2 import ssl if hasattr(ssl, '_create_unverified_context'): ssl._create_default_https_context = ssl._create_unverified_context if method is None: method = self.METHOD orig_json_data = json_data username, password = self._get_auth() auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() auth_manager.add_password(None, url, username, password) auth = urllib2.HTTPDigestAuthHandler(auth_manager) opener = urllib2.build_opener(auth, urllib2.HTTPSHandler()) headers_list = self._get_headers().items() if json_data: json_data = json.dumps(json_data) headers_list.append(('Accept', 'application/json')) json_headers = {'Content-Type': 'application/json', 'Content-Length': len(json_data)} else: json_data = None json_headers = {'Content-Length': 0} opener.addheaders = headers_list req = urllib2.Request(url, json_data, json_headers) # This is an unpleasant monkey patch, but there isn't an alternative. # urllib2 uses POST iff there is a data payload, but that is not the # correct criterion. The difference is basically that POST changes # server state and GET doesn't. req.get_method = lambda: method try: response = opener.open(req, timeout=self.timeout) except urllib2.URLError as exc: if "unknown protocol" in str(exc) and url.startswith("https:"): # Server is using http rather than https, for some reason. sys.stderr.write(WARNING_NO_HTTPS_SUPPORT.format(exc)) return self._get_data_from_url_with_urllib2( url.replace("https:", "http:", 1), orig_json_data) if cylc.flags.debug: import traceback traceback.print_exc() if "timed out" in str(exc): raise ConnectionTimeout(url, exc) else: raise ConnectionError(url, exc) except Exception as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) if response.getcode() == 401: raise ConnectionDeniedError(url, self.prog_name, self.ACCESS_DESCRIPTION) response_text = response.read() if response.getcode() >= 400: from cylc.network.https.util import get_exception_from_html exception_text = get_exception_from_html(response_text) if exception_text: sys.stderr.write(exception_text) else: sys.stderr.write(response_text) raise ConnectionError(url, "%s HTTP return code" % response.getcode()) if self.auth and self.auth[1] != NO_PASSPHRASE: self.srv_files_mgr.cache_passphrase( self.suite, self.owner, self.host, self.auth[1]) try: return json.loads(response_text) except ValueError: return response_text def _get_auth(self): """Return a user/password Digest Auth.""" if self.auth is None: self.auth = ('anon', NO_PASSPHRASE) try: pphrase = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_PASSPHRASE, self.suite, self.owner, self.host, content=True) except SuiteServiceFileError: pass else: if pphrase and pphrase != NO_PASSPHRASE: self.auth = ('cylc', pphrase) return self.auth def _get_headers(self): """Return HTTP headers identifying the client.""" user_agent_string = ( "cylc/%s prog_name/%s uuid/%s" % ( CYLC_VERSION, self.prog_name, self.my_uuid ) ) auth_info = "%s@%s" % (USER, get_hostname()) return {"User-Agent": user_agent_string, "From": auth_info} def _get_verify(self): """Return the server certificate if possible.""" if self.server_cert is None: try: self.server_cert = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_CERT, self.suite, self.owner, self.host) except SuiteServiceFileError: self.server_cert = False return self.server_cert def _load_contact_info(self): """Obtain suite owner, host, port info. Determine host and port using content in port file, unless already specified. """ if self.host and self.port: return data = self.srv_files_mgr.load_contact_file( self.suite, self.owner, self.host) if not self.host: self.host = data.get(self.srv_files_mgr.KEY_HOST) if not self.port: self.port = int(data.get(self.srv_files_mgr.KEY_PORT)) if not self.owner: self.owner = data.get(self.srv_files_mgr.KEY_OWNER) def reset(self, *args, **kwargs): pass def signout(self, *args, **kwargs): pass
class db_updater(threading.Thread): SCAN_INTERVAL = 60.0 def __init__(self, regd_treestore, filtr=None, timeout=None): self.db = SuiteSrvFilesManager() self.quit = False if timeout: self.timeout = float(timeout) else: self.timeout = None self.regd_treestore = regd_treestore super(db_updater, self).__init__() self.next_scan_time = None self.running_choices = None self.newtree = {} self.regd_choices = self.db.list_suites(filtr) def construct_newtree(self): """construct self.newtree[one][two]...[nnn] = [auth, descr, dir ]""" regd_choices = {} for suite, suite_dir, descr in sorted(self.regd_choices): regd_choices[suite] = (suite, suite_dir, descr) self.newtree = {} for suite, auth in self.running_choices: if suite in regd_choices: if is_remote_host(auth.split(':', 1)[0]): descr, suite_dir = (None, None) else: # local suite suite_dir, descr = regd_choices[suite][1:3] del regd_choices[suite] nest2 = self.newtree regp = suite.split(SuiteSrvFilesManager.DELIM) for key in regp[:-1]: if key not in nest2: nest2[key] = {} nest2 = nest2[key] nest2[(regp[-1], suite, auth)] = [auth, descr, suite_dir] for suite, suite_dir, descr in regd_choices.values(): suite_dir = re.sub('^' + os.environ['HOME'], '~', suite_dir) nest2 = self.newtree regp = suite.split(SuiteSrvFilesManager.DELIM) for key in regp[:-1]: if key not in nest2: nest2[key] = {} nest2 = nest2[key] nest2[(regp[-1], suite, '-')] = ['-', descr, suite_dir] def build_treestore(self, data, piter=None): for key, value in sorted(data.items()): if isinstance(key, tuple): item = key[0] else: item = key if isinstance(value, dict): # final three items are colours iter_ = self.regd_treestore.append( piter, [item, None, None, None, None, None, None]) self.build_treestore(value, iter_) else: state, descr, suite_dir = value self.regd_treestore.append( piter, [item, state, descr, suite_dir, None, None, None]) def update(self): """Update tree, if necessary.""" if self.next_scan_time is not None and self.next_scan_time > time(): return # Scan for running suites choices = [] for host, port, suite_identity in scan_many(get_scan_items_from_fs(), timeout=self.timeout): name = suite_identity[KEY_NAME] owner = suite_identity[KEY_OWNER] if is_remote_user(owner): continue # current user only auth = "%s:%s" % (host, port) choices.append((name, auth)) choices.sort() self.next_scan_time = time() + self.SCAN_INTERVAL if choices == self.running_choices: return # Update tree if running suites changed self.running_choices = choices self.construct_newtree() self.update_treestore(self.newtree, self.regd_treestore.get_iter_first()) def update_treestore(self, new, iter_): # iter_ is None for an empty treestore (no suites registered) ts = self.regd_treestore if iter_: opath = ts.get_path(iter_) # get parent iter_ before pruning in case we prune last item at # this level piter = ts.iter_parent(iter_) else: opath = None piter = None def my_get_iter(item): # find the TreeIter pointing at item at this level if not opath: return None iter_ = ts.get_iter(opath) while iter_: val, = ts.get(iter_, 0) if val == item: return iter_ iter_ = ts.iter_next(iter_) return None # new items at this level new_items = new.keys() old_items = [] while iter_: # iterate through old items at this level item, state = ts.get(iter_, 0, 1, 2, 3)[0:2] if item not in new_items: # old item is not in new - prune it res = ts.remove(iter_) if not res: # Nec? iter_ = None else: # old item is in new - update it in case it changed old_items.append(item) # update old items that do appear in new chiter = ts.iter_children(iter_) if not isinstance(new[item], dict): # new item is not a group - update title etc. state = new[item][0] sc = self.statecol(state) ni = new[item] ts.set(iter_, 0, item, 1, ni[0], 2, ni[1], 3, ni[2], 4, sc[0], 5, sc[1], 6, sc[2]) if chiter: # old item was a group - kill its children while chiter: res = ts.remove(chiter) if not res: chiter = None else: # new item is a group if not chiter: # old item was not a group ts.set(iter_, 0, item, 1, None, 2, None, 3, None, 4, None, 5, None, 6, None) self.build_treestore(new[item], iter_) # continue iter_ = ts.iter_next(iter_) # return to original iter_ if opath: try: iter_ = ts.get_iter(opath) except ValueError: # removed the item pointed to # TODO - NEED TO WORRY ABOUT OTHERS AT THIS LEVEL? iter_ = None else: iter_ = None # add new items at this level for key in sorted(new_items): if isinstance(key, tuple): item = key[0] else: item = key if item not in old_items: # new data wasn't in old - add it if isinstance(new[key], dict): xiter = ts.append(piter, [item] + [None, None, None, None, None, None]) self.build_treestore(new[key], xiter) else: state = new[key][0] ts.append(piter, [item] + new[key] + list(self.statecol(state))) else: # new data was already in old if isinstance(new[key], dict): # check lower levels niter = my_get_iter(key) if niter: chiter = ts.iter_children(niter) if chiter: self.update_treestore(new[key], chiter) def run(self): """Main loop.""" while not self.quit: gobject.idle_add(self.update) sleep(0.1) @staticmethod def statecol(state): bg_ = '#19ae0a' fg_ = '#030' if state == '-': return (None, None, None) else: return (fg_, bg_, bg_) @staticmethod def search_level(model, iter_, func, data): while iter_: if func(model, iter_, data): return iter_ iter_ = model.iter_next(iter_) return None def search_treemodel(self, model, iter_, func, data): while iter_: if func(model, iter_, data): return iter_ result = self.search_treemodel(model, model.iter_children(iter_), func, data) if result: return result iter_ = model.iter_next(iter_) return None @staticmethod def match_func(model, iter_, data): column, key = data value = model.get_value(iter_, column) return value == key
class SuiteRuntimeServiceClient(object): """Client for calling the HTTP(S) API of running suites.""" ANON_AUTH = ('anon', NO_PASSPHRASE, False) COMPAT_MAP = { # Limited pre-7.5.0 API compat mapping 'clear_broadcast': {0: 'broadcast/clear'}, 'expire_broadcast': {0: 'broadcast/expire'}, 'get_broadcast': {0: 'broadcast/get'}, 'get_info': {0: 'info/'}, 'get_suite_state_summary': {0: 'state/get_state_summary'}, 'get_tasks_by_state': {0: 'state/get_tasks_by_state'}, 'put_broadcast': {0: 'broadcast/put'}, 'put_command': {0: 'command/'}, 'put_ext_trigger': {0: 'ext-trigger/put'}, 'put_message': {0: 'message/put'}, } ERROR_NO_HTTPS_SUPPORT = ( "ERROR: server has no HTTPS support," + " configure your global.rc file to use HTTP : {0}\n") METHOD = 'POST' METHOD_POST = 'POST' METHOD_GET = 'GET' def __init__(self, suite, owner=None, host=None, port=None, timeout=None, my_uuid=None, print_uuid=False, comms_protocol=None, auth=None): self.suite = suite if not owner: owner = get_user() self.owner = owner self.host = host if self.host and self.host.split('.')[0] == 'localhost': self.host = get_host() elif self.host and '.' not in self.host: # Not IP and no domain self.host = get_fqdn_by_host(self.host) self.port = port self.srv_files_mgr = SuiteSrvFilesManager() self.comms_protocol = comms_protocol if timeout is not None: timeout = float(timeout) self.timeout = timeout self.my_uuid = my_uuid or uuid4() if print_uuid: sys.stderr.write('%s\n' % self.my_uuid) self.prog_name = os.path.basename(sys.argv[0]) self.auth = auth self.session = None self.api = None def _compat(self, name, default=None): """Return server function name. Handle back-compat for pre-7.5.0 if relevant. """ # Need to load contact info here to get API version. self._load_contact_info() if default is None: default = name return self.COMPAT_MAP[name].get(self.api, default) def clear_broadcast(self, **kwargs): """Clear broadcast runtime task settings.""" return self._call_server(self._compat('clear_broadcast'), payload=kwargs) def expire_broadcast(self, **kwargs): """Expire broadcast runtime task settings.""" return self._call_server(self._compat('expire_broadcast'), **kwargs) def get_broadcast(self, **kwargs): """Return broadcast settings.""" return self._call_server(self._compat('get_broadcast'), method=self.METHOD_GET, **kwargs) def get_info(self, command, **kwargs): """Return suite info.""" return self._call_server(self._compat('get_info', default='') + command, method=self.METHOD_GET, **kwargs) def get_latest_state(self, full_mode): """Return latest state of the suite (for the GUI).""" self._load_contact_info() if self.api == 0: # Basic compat for pre-7.5.0 suites # Full mode only. # Error content/size not supported. # Report made-up main loop interval of 5.0 seconds. return { 'cylc_version': self.get_info('get_cylc_version'), 'full_mode': full_mode, 'summary': self.get_suite_state_summary(), 'ancestors': self.get_info('get_first_parent_ancestors'), 'ancestors_pruned': self.get_info('get_first_parent_ancestors', pruned=True), 'descendants': self.get_info('get_first_parent_descendants'), 'err_content': '', 'err_size': 0, 'mean_main_loop_interval': 5.0 } else: return self._call_server('get_latest_state', method=self.METHOD_GET, full_mode=full_mode) def get_suite_state_summary(self): """Return the global, task, and family summary data structures.""" return utf8_enforce( self._call_server(self._compat('get_suite_state_summary'), method=self.METHOD_GET)) def get_tasks_by_state(self): """Returns a dict containing lists of tasks by state. Result in the form: {state: [(most_recent_time_string, task_name, point_string), ...]} """ return self._call_server(self._compat('get_tasks_by_state'), method=self.METHOD_GET) def identify(self): """Return suite identity.""" # Note on compat: Suites on 7.6.0 or above can just call "identify", # but has compat for "id/identity". return self._call_server('id/identify', method=self.METHOD_GET) def put_broadcast(self, **kwargs): """Put/set broadcast runtime task settings.""" return self._call_server(self._compat('put_broadcast'), payload=kwargs) def put_command(self, command, **kwargs): """Invoke suite command.""" return self._call_server( self._compat('put_command', default='') + command, **kwargs) def put_ext_trigger(self, event_message, event_id): """Put external trigger.""" return self._call_server(self._compat('put_ext_trigger'), event_message=event_message, event_id=event_id) def put_message(self, task_id, severity, message): """Send task message.""" func_name = self._compat('put_message') if func_name == 'put_message': return self._call_server(func_name, task_id=task_id, severity=severity, message=message) else: # pre-7.5.0 API compat return self._call_server(func_name, task_id=task_id, priority=severity, message=message) def reset(self): """Compat method, does nothing.""" pass def signout(self): """Tell server to forget this client.""" return self._call_server('signout') def _call_server(self, function, method=METHOD, payload=None, **kwargs): """Build server URL + call it""" url = self._call_server_get_url(function, **kwargs) # Remove proxy settings from environment for now environ = {} for key in ("http_proxy", "https_proxy"): val = os.environ.pop(key, None) if val: environ[key] = val try: return self.call_server_impl(url, method, payload) finally: os.environ.update(environ) def _call_server_get_url(self, function, **kwargs): """Build request URL.""" comms_protocol = self.comms_protocol if comms_protocol is None: # Use standard setting from global configuration comms_protocol = glbl_cfg().get(['communication', 'method']) url = '%s://%s:%s/%s' % (comms_protocol, self.host, self.port, function) # If there are any parameters left in the dict after popping, # append them to the url. if kwargs: import urllib params = urllib.urlencode(kwargs, doseq=True) url += "?" + params return url def call_server_impl(self, url, method, payload): """Determine whether to use requests or urllib2 to call suite API.""" impl = self._call_server_impl_urllib2 try: import requests except ImportError: pass else: if [int(_) for _ in requests.__version__.split(".")] >= [2, 4, 2]: impl = self._call_server_impl_requests try: return impl(url, method, payload) except ClientConnectError as exc: if self.suite is None: raise # Cannot connect, perhaps suite is no longer running and is leaving # behind a contact file? try: self.srv_files_mgr.detect_old_contact_file( self.suite, (self.host, self.port)) except (AssertionError, SuiteServiceFileError): raise exc else: # self.srv_files_mgr.detect_old_contact_file should delete left # behind contact file if the old suite process no longer # exists. Should be safe to report that the suite has stopped. raise ClientConnectError(exc.args[0], exc.STOPPED % self.suite) def _call_server_impl_requests(self, url, method, payload): """Call server with "requests" library.""" import requests from requests.packages.urllib3.exceptions import InsecureRequestWarning warnings.simplefilter("ignore", InsecureRequestWarning) if self.session is None: self.session = requests.Session() if method == self.METHOD_POST: session_method = self.session.post else: session_method = self.session.get comms_protocol = url.split(':', 1)[0] # Can use urlparse? username, password, verify = self._get_auth(comms_protocol) try: ret = session_method(url, json=payload, verify=verify, proxies={}, headers=self._get_headers(), auth=requests.auth.HTTPDigestAuth( username, password), timeout=self.timeout) except requests.exceptions.SSLError as exc: if "unknown protocol" in str(exc) and url.startswith("https:"): # Server is using http rather than https, for some reason. sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc)) raise CylcError("Cannot issue commands over unsecured http.") if cylc.flags.debug: traceback.print_exc() raise ClientConnectError(url, exc) except requests.exceptions.Timeout as exc: if cylc.flags.debug: traceback.print_exc() raise ClientTimeout(url, exc) except requests.exceptions.RequestException as exc: if cylc.flags.debug: traceback.print_exc() raise ClientConnectError(url, exc) if ret.status_code == 401: access_desc = 'private' if self.auth == self.ANON_AUTH: access_desc = 'public' raise ClientDeniedError(url, self.prog_name, access_desc) if ret.status_code >= 400: exception_text = get_exception_from_html(ret.text) if exception_text: sys.stderr.write(exception_text) else: sys.stderr.write(ret.text) try: ret.raise_for_status() except requests.exceptions.HTTPError as exc: if cylc.flags.debug: traceback.print_exc() raise ClientConnectedError(url, exc) if self.auth and self.auth[1] != NO_PASSPHRASE: self.srv_files_mgr.cache_passphrase(self.suite, self.owner, self.host, self.auth[1]) try: return ret.json() except ValueError: return ret.text def _call_server_impl_urllib2(self, url, method, payload): """Call server with "urllib2" library.""" import json import urllib2 import ssl unverified_context = getattr(ssl, '_create_unverified_context', None) if unverified_context is not None: ssl._create_default_https_context = unverified_context comms_protocol = url.split(':', 1)[0] # Can use urlparse? username, password = self._get_auth(comms_protocol)[0:2] auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() auth_manager.add_password(None, url, username, password) auth = urllib2.HTTPDigestAuthHandler(auth_manager) opener = urllib2.build_opener(auth, urllib2.HTTPSHandler()) headers_list = self._get_headers().items() if payload: payload = json.dumps(payload) headers_list.append(('Accept', 'application/json')) json_headers = { 'Content-Type': 'application/json', 'Content-Length': len(payload) } else: payload = None json_headers = {'Content-Length': 0} opener.addheaders = headers_list req = urllib2.Request(url, payload, json_headers) # This is an unpleasant monkey patch, but there isn't an # alternative. urllib2 uses POST if there is a data payload # but that is not the correct criterion. # The difference is basically that POST changes # server state and GET doesn't. req.get_method = lambda: method try: response = opener.open(req, timeout=self.timeout) except urllib2.URLError as exc: if "unknown protocol" in str(exc) and url.startswith("https:"): # Server is using http rather than https, for some reason. sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc)) raise CylcError("Cannot issue commands over unsecured http.") if cylc.flags.debug: traceback.print_exc() if "timed out" in str(exc): raise ClientTimeout(url, exc) else: raise ClientConnectError(url, exc) except Exception as exc: if cylc.flags.debug: traceback.print_exc() raise ClientError(url, exc) if response.getcode() == 401: access_desc = 'private' if self.auth == self.ANON_AUTH: access_desc = 'public' raise ClientDeniedError(url, self.prog_name, access_desc) response_text = response.read() if response.getcode() >= 400: exception_text = get_exception_from_html(response_text) if exception_text: sys.stderr.write(exception_text) else: sys.stderr.write(response_text) raise ClientConnectedError( url, "%s HTTP return code" % response.getcode()) if self.auth and self.auth[1] != NO_PASSPHRASE: self.srv_files_mgr.cache_passphrase(self.suite, self.owner, self.host, self.auth[1]) try: return json.loads(response_text) except ValueError: return response_text def _get_auth(self, protocol): """Return a user/password Digest Auth.""" if self.auth is None: self.auth = self.ANON_AUTH try: pphrase = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_PASSPHRASE, self.suite, self.owner, self.host, content=True) if protocol == 'https': verify = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_CERT, self.suite, self.owner, self.host) else: verify = False except SuiteServiceFileError: pass else: if pphrase and pphrase != NO_PASSPHRASE: self.auth = ('cylc', pphrase, verify) return self.auth def _get_headers(self): """Return HTTP headers identifying the client.""" user_agent_string = ("cylc/%s prog_name/%s uuid/%s" % (CYLC_VERSION, self.prog_name, self.my_uuid)) auth_info = "%s@%s" % (get_user(), get_host()) return {"User-Agent": user_agent_string, "From": auth_info} def _load_contact_info(self): """Obtain suite owner, host, port info. Determine host and port using content in port file, unless already specified. """ if self.host and self.port: return if self.port: # In case the contact file is corrupted, user can specify the port. self.host = get_host() return try: # Always trust the values in the contact file otherwise. data = self.srv_files_mgr.load_contact_file( self.suite, self.owner, self.host) # Port inside "try" block, as it needs a type conversion self.port = int(data.get(self.srv_files_mgr.KEY_PORT)) except (IOError, ValueError, SuiteServiceFileError): raise ClientInfoError(self.suite) self.host = data.get(self.srv_files_mgr.KEY_HOST) self.owner = data.get(self.srv_files_mgr.KEY_OWNER) self.comms_protocol = data.get(self.srv_files_mgr.KEY_COMMS_PROTOCOL) try: self.api = int(data.get(self.srv_files_mgr.KEY_API)) except (TypeError, ValueError): self.api = 0 # Assume cylc-7.5.0 or before
def update_suites_info(updater, full_mode=False): """Return mapping of suite info by host, owner and suite name. Args: updater (object): gscan or gpanel updater: Compulsory attributes from updater: hosts: hosts to scan owner_pattern: re to filter results by owners suite_info_map: previous results returned by this function Optional attributes from updater: timeout: communication timeout full_mode (boolean): update in full mode? Return: dict: {(host, owner, name): suite_info, ...} where each "suite_info" is a dict with keys: KEY_GROUP: group name of suite KEY_OWNER: suite owner name KEY_PORT: suite port, for running suites only KEY_STATES: suite state KEY_TASKS_BY_STATE: tasks by state KEY_TITLE: suite title KEY_UPDATE_TIME: last update time of suite """ # Compulsory attributes from updater # hosts - hosts to scan, or the default set in the site/user global.rc # owner_pattern - return only suites with owners matching this compiled re # suite_info_map - previous results returned by this function # Optional attributes from updater # timeout - communication timeout owner_pattern = updater.owner_pattern timeout = getattr(updater, "comms_timeout", None) # name_pattern - return only suites with names matching this compiled re name_pattern = getattr(updater, "name_pattern", None) # Determine items to scan results = {} items = [] if full_mode and not updater.hosts: # Scan users suites. Walk "~/cylc-run/" to get (host, port) from # ".service/contact" for active suites suite_srv_files_mgr = SuiteSrvFilesManager() if owner_pattern is None: # Run directory of current user only run_dirs = [GLOBAL_CFG.get_host_item('run directory')] else: # Run directory of all users matching "owner_pattern". # But skip those with /nologin or /false shells run_dirs = [] skips = ('/false', '/nologin') for pwent in getpwall(): if any(pwent.pw_shell.endswith(s) for s in (skips)): continue if owner_pattern.match(pwent.pw_name): run_dirs.append( GLOBAL_CFG.get_host_item('run directory', owner=pwent.pw_name, owner_home=pwent.pw_dir)) if cylc.flags.debug: sys.stderr.write( 'Listing suites:%s%s\n' % (_UPDATE_DEBUG_DELIM, _UPDATE_DEBUG_DELIM.join(run_dirs))) for run_d in run_dirs: for dirpath, dnames, fnames in os.walk(run_d, followlinks=True): if updater.quit: return # Always descend for top directory, but # don't descend further if it has a: # * .service/ # * cylc-suite.db: (pre-cylc-7 suites don't have ".service/"). if dirpath != run_d and (suite_srv_files_mgr.DIR_BASE_SRV in dnames or 'cylc-suite.db' in fnames): dnames[:] = [] # Choose only suites with .service and matching filter reg = os.path.relpath(dirpath, run_d) try: contact_data = suite_srv_files_mgr.load_contact_file(reg) except (SuiteServiceFileError, IOError, TypeError, ValueError): continue else: items.append((contact_data[suite_srv_files_mgr.KEY_HOST], contact_data[suite_srv_files_mgr.KEY_PORT])) elif full_mode: # Scan full port range on all hosts items.extend(updater.hosts) else: # Scan suites in previous results only for (host, owner, name), prev_result in updater.suite_info_map.items(): port = prev_result.get(KEY_PORT) if port: items.append((host, port)) else: results[(host, owner, name)] = prev_result if not items: return results if cylc.flags.debug: sys.stderr.write( 'Scan items:%s%s\n' % (_UPDATE_DEBUG_DELIM, _UPDATE_DEBUG_DELIM.join(str(item) for item in items))) # Scan for host, port, result in scan_many(items, timeout=timeout, updater=updater): if updater.quit: return if (name_pattern and not name_pattern.match(result[KEY_NAME]) or owner_pattern and not owner_pattern.match(result[KEY_OWNER])): continue try: result[KEY_PORT] = port results[(host, result[KEY_OWNER], result[KEY_NAME])] = result result[KEY_UPDATE_TIME] = int(float(result[KEY_UPDATE_TIME])) except (KeyError, TypeError, ValueError): pass expire_threshold = time() - DURATION_EXPIRE_STOPPED for (host, owner, name), prev_result in updater.suite_info_map.items(): if updater.quit: return if ((host, owner, name) in results or owner_pattern and not owner_pattern.match(owner) or name_pattern and not name_pattern.match(name)): # OK if suite already in current results set. # Don't bother if: # * previous owner does not match current owner pattern # * previous suite name does not match current name pattern continue if prev_result.get(KEY_PORT): # A previously running suite is no longer running. # Get suite info with "cat-state", if possible, and include in the # results set. try: prev_result = _update_stopped_suite_info((host, owner, name)) except (IndexError, TypeError, ValueError): continue if prev_result.get(KEY_UPDATE_TIME, 0) > expire_threshold: results[(host, owner, name)] = prev_result return results
class RemoteJobHostManager(object): """Manage a remote job host.""" _INSTANCE = None @classmethod def get_inst(cls): """Return a singleton instance of this class.""" if cls._INSTANCE is None: cls._INSTANCE = cls() return cls._INSTANCE def __init__(self): self.initialised = {} # {(user, host): should_unlink, ...} self.single_task_mode = False self.suite_srv_files_mgr = SuiteSrvFilesManager() def init_suite_run_dir(self, reg, host, owner): """Initialise suite run dir on a user@host. Create SUITE_RUN_DIR/log/job/ if necessary. Install suite contact environment file. Install suite python modules. Raise RemoteJobHostInitError if initialisation cannot complete. """ if host is None: host = 'localhost' if ((host, owner) in [('localhost', None), ('localhost', USER)] or (host, owner) in self.initialised or self.single_task_mode): return user_at_host = host if owner: user_at_host = owner + '@' + host r_suite_run_dir = GLOBAL_CFG.get_derived_host_item( reg, 'suite run directory', host, owner) r_log_job_dir = GLOBAL_CFG.get_derived_host_item( reg, 'suite job log directory', host, owner) r_suite_srv_dir = os.path.join(r_suite_run_dir, self.suite_srv_files_mgr.DIR_BASE_SRV) # Create a UUID file in the service directory. # If remote host has the file in its service directory, we can assume # that the remote host has a shared file system with the suite host. ssh_tmpl = GLOBAL_CFG.get_host_item('remote shell template', host, owner) uuid_str = str(uuid4()) uuid_fname = os.path.join( self.suite_srv_files_mgr.get_suite_srv_dir(reg), uuid_str) try: open(uuid_fname, 'wb').close() proc = Popen(shlex.split(ssh_tmpl) + [ '-n', user_at_host, 'test', '-e', os.path.join(r_suite_srv_dir, uuid_str) ], stdout=PIPE, stderr=PIPE) if proc.wait() == 0: # Initialised, but no need to tidy up self.initialised[(host, owner)] = False return finally: try: os.unlink(uuid_fname) except OSError: pass cmds = [] # Command to create suite directory structure on remote host. cmds.append( shlex.split(ssh_tmpl) + [ '-n', user_at_host, 'mkdir', '-p', r_suite_run_dir, r_log_job_dir, r_suite_srv_dir ]) # Command to copy contact and authentication files to remote host. # Note: no need to do this if task communication method is "poll". should_unlink = GLOBAL_CFG.get_host_item('task communication method', host, owner) != "poll" if should_unlink: scp_tmpl = GLOBAL_CFG.get_host_item('remote copy template', host, owner) cmds.append( shlex.split(scp_tmpl) + [ '-p', self.suite_srv_files_mgr.get_contact_file(reg), self.suite_srv_files_mgr.get_auth_item( self.suite_srv_files_mgr.FILE_BASE_PASSPHRASE, reg), self.suite_srv_files_mgr.get_auth_item( self.suite_srv_files_mgr.FILE_BASE_SSL_CERT, reg), user_at_host + ':' + r_suite_srv_dir + '/' ]) # Command to copy python library to remote host. suite_run_py = os.path.join( GLOBAL_CFG.get_derived_host_item(reg, 'suite run directory'), 'python') if os.path.isdir(suite_run_py): cmds.append( shlex.split(scp_tmpl) + [ '-pr', suite_run_py, user_at_host + ':' + r_suite_run_dir + '/' ]) # Run commands in sequence. for cmd in cmds: proc = Popen(cmd, stdout=PIPE, stderr=PIPE) out, err = proc.communicate() if proc.wait(): raise RemoteJobHostInitError( RemoteJobHostInitError.MSG_INIT, user_at_host, ' '.join([quote(item) for item in cmd]), proc.returncode, out, err) self.initialised[(host, owner)] = should_unlink LOG.info('Initialised %s:%s' % (user_at_host, r_suite_run_dir)) def unlink_suite_contact_files(self, reg): """Remove suite contact files from initialised hosts. This is called on shutdown, so we don't want anything to hang. Terminate any incomplete SSH commands after 10 seconds. """ # Issue all SSH commands in parallel procs = {} for (host, owner), should_unlink in self.initialised.items(): if not should_unlink: continue user_at_host = host if owner: user_at_host = owner + '@' + host ssh_tmpl = GLOBAL_CFG.get_host_item('remote shell template', host, owner) r_suite_contact_file = os.path.join( GLOBAL_CFG.get_derived_host_item(reg, 'suite run directory', host, owner), SuiteSrvFilesManager.DIR_BASE_SRV, SuiteSrvFilesManager.FILE_BASE_CONTACT) cmd = shlex.split(ssh_tmpl) + [ '-n', user_at_host, 'rm', '-f', r_suite_contact_file ] procs[user_at_host] = (cmd, Popen(cmd, stdout=PIPE, stderr=PIPE)) # Wait for commands to complete for a max of 10 seconds timeout = time() + 10.0 while procs and time() < timeout: for user_at_host, (cmd, proc) in procs.copy().items(): if proc.poll() is None: continue del procs[user_at_host] out, err = proc.communicate() if proc.wait(): ERR.warning( RemoteJobHostInitError( RemoteJobHostInitError.MSG_TIDY, user_at_host, ' '.join([quote(item) for item in cmd]), proc.returncode, out, err)) # Terminate any remaining commands for user_at_host, (cmd, proc) in procs.items(): try: proc.terminate() except OSError: pass out, err = proc.communicate() if proc.wait(): ERR.warning( RemoteJobHostInitError( RemoteJobHostInitError.MSG_TIDY, user_at_host, ' '.join([quote(item) for item in cmd]), proc.returncode, out, err))
class SuiteRuntimeServiceClient(object): """Client for calling the HTTP(S) API of running suites.""" ANON_AUTH = ('anon', NO_PASSPHRASE, False) COMPAT_MAP = { # Limited pre-7.5.0 API compat mapping 'clear_broadcast': {0: 'broadcast/clear'}, 'expire_broadcast': {0: 'broadcast/expire'}, 'get_broadcast': {0: 'broadcast/get'}, 'get_info': {0: 'info/'}, 'get_suite_state_summary': {0: 'state/get_state_summary'}, 'put_broadcast': {0: 'broadcast/put'}, 'put_command': {0: 'command/'}, 'put_ext_trigger': {0: 'ext-trigger/put'}, 'put_messages': {0: 'message/put', 1: 'put_message'}, } ERROR_NO_HTTPS_SUPPORT = ( "ERROR: server has no HTTPS support," + " configure your global.rc file to use HTTP : {0}\n" ) METHOD = 'POST' METHOD_POST = 'POST' METHOD_GET = 'GET' MSG_RETRY_INTVL = 5.0 MSG_MAX_TRIES = 7 MSG_TIMEOUT = 30.0 def __init__( self, suite, owner=None, host=None, port=None, timeout=None, my_uuid=None, print_uuid=False, auth=None): self.suite = suite if not owner: owner = get_user() self.owner = owner self.host = host if self.host and self.host.split('.')[0] == 'localhost': self.host = get_host() elif self.host and '.' not in self.host: # Not IP and no domain self.host = get_fqdn_by_host(self.host) self.port = port self.srv_files_mgr = SuiteSrvFilesManager() if timeout is not None: timeout = float(timeout) self.timeout = timeout self.my_uuid = my_uuid or uuid4() if print_uuid: sys.stderr.write('%s\n' % self.my_uuid) self.prog_name = os.path.basename(sys.argv[0]) self.auth = auth self.session = None self.comms1 = {} # content in primary contact file self.comms2 = {} # content in extra contact file, e.g. contact via ssh def _compat(self, name, default=None): """Return server function name. Handle back-compat for pre-7.5.0 if relevant. """ # Need to load contact info here to get API version. self._load_contact_info() if default is None: default = name return self.COMPAT_MAP[name].get( self.comms1.get(self.srv_files_mgr.KEY_API), default) def clear_broadcast(self, payload): """Clear broadcast runtime task settings.""" return self._call_server( self._compat('clear_broadcast'), payload=payload) def expire_broadcast(self, **kwargs): """Expire broadcast runtime task settings.""" return self._call_server(self._compat('expire_broadcast'), **kwargs) def get_broadcast(self, **kwargs): """Return broadcast settings.""" return self._call_server( self._compat('get_broadcast'), method=self.METHOD_GET, **kwargs) def get_info(self, command, **kwargs): """Return suite info.""" return self._call_server( self._compat('get_info', default='') + command, method=self.METHOD_GET, **kwargs) def get_latest_state(self, full_mode=False): """Return latest state of the suite (for the GUI).""" self._load_contact_info() if self.comms1.get(self.srv_files_mgr.KEY_API) == 0: # Basic compat for pre-7.5.0 suites # Full mode only. # Error content/size not supported. # Report made-up main loop interval of 5.0 seconds. return { 'cylc_version': self.get_info('get_cylc_version'), 'full_mode': full_mode, 'summary': self.get_suite_state_summary(), 'ancestors': self.get_info('get_first_parent_ancestors'), 'ancestors_pruned': self.get_info( 'get_first_parent_ancestors', pruned=True), 'descendants': self.get_info('get_first_parent_descendants'), 'err_content': '', 'err_size': 0, 'mean_main_loop_interval': 5.0} else: return self._call_server( 'get_latest_state', method=self.METHOD_GET, full_mode=full_mode) def get_suite_state_summary(self): """Return the global, task, and family summary data structures.""" return utf8_enforce(self._call_server( self._compat('get_suite_state_summary'), method=self.METHOD_GET)) def identify(self): """Return suite identity.""" # Note on compat: Suites on 7.6.0 or above can just call "identify", # but has compat for "id/identity". return self._call_server('id/identify', method=self.METHOD_GET) def put_broadcast(self, payload): """Put/set broadcast runtime task settings.""" return self._call_server( self._compat('put_broadcast'), payload=payload) def put_command(self, command, **kwargs): """Invoke suite command.""" return self._call_server( self._compat('put_command', default='') + command, **kwargs) def put_ext_trigger(self, event_message, event_id): """Put external trigger.""" return self._call_server( self._compat('put_ext_trigger'), event_message=event_message, event_id=event_id) def put_messages(self, payload): """Send task messages to suite server program. Arguments: payload (dict): task_job (str): Task job as "CYCLE/TASK_NAME/SUBMIT_NUM". event_time (str): Event time as string. messages (list): List in the form [[severity, message], ...]. """ retry_intvl = float(self.comms1.get( self.srv_files_mgr.KEY_TASK_MSG_RETRY_INTVL, self.MSG_RETRY_INTVL)) max_tries = int(self.comms1.get( self.srv_files_mgr.KEY_TASK_MSG_MAX_TRIES, self.MSG_MAX_TRIES)) for i in range(1, max_tries + 1): # 1..max_tries inclusive orig_timeout = self.timeout if self.timeout is None: self.timeout = self.MSG_TIMEOUT try: func_name = self._compat('put_messages') if func_name == 'put_messages': results = self._call_server(func_name, payload=payload) elif func_name == 'put_message': # API 1, 7.5.0 compat cycle, name = payload['task_job'].split('/')[0:2] for severity, message in payload['messages']: results.append(self._call_server( func_name, task_id='%s.%s' % (name, cycle), severity=severity, message=message)) else: # API 0, pre-7.5.0 compat, priority instead of severity cycle, name = payload['task_job'].split('/')[0:2] for severity, message in payload['messages']: results.append(self._call_server( func_name, task_id='%s.%s' % (name, cycle), priority=severity, message=message)) except ClientInfoError: # Contact info file not found, suite probably not running. # Don't bother with retry, suite restart will poll any way. raise except ClientError as exc: now = get_current_time_string() sys.stderr.write( "%s WARNING - Message send failed, try %s of %s: %s\n" % ( now, i, max_tries, exc)) if i < max_tries: sys.stderr.write( " retry in %s seconds, timeout is %s\n" % ( retry_intvl, self.timeout)) sleep(retry_intvl) # Reset in case contact info or passphrase change self.comms1 = {} self.host = None self.port = None self.auth = None else: if i > 1: # Continue to write to STDERR, so users can easily see that # it has recovered from previous failures. sys.stderr.write( "%s INFO - Send message: try %s of %s succeeded\n" % ( get_current_time_string(), i, max_tries)) return results finally: self.timeout = orig_timeout def reset(self): """Compat method, does nothing.""" pass def signout(self): """Tell server to forget this client.""" return self._call_server('signout') def _call_server(self, function, method=METHOD, payload=None, **kwargs): """Build server URL + call it""" if self.comms2: return self._call_server_via_comms2(function, payload, **kwargs) url = self._call_server_get_url(function, **kwargs) # Remove proxy settings from environment for now environ = {} for key in ("http_proxy", "https_proxy"): val = os.environ.pop(key, None) if val: environ[key] = val try: return self.call_server_impl(url, method, payload) finally: os.environ.update(environ) def _call_server_get_url(self, function, **kwargs): """Build request URL.""" scheme = self.comms1.get(self.srv_files_mgr.KEY_COMMS_PROTOCOL) if scheme is None: # Use standard setting from global configuration scheme = glbl_cfg().get(['communication', 'method']) url = '%s://%s:%s/%s' % ( scheme, self.host, self.port, function) # If there are any parameters left in the dict after popping, # append them to the url. if kwargs: import urllib params = urllib.urlencode(kwargs, doseq=True) url += "?" + params return url def call_server_impl(self, url, method, payload): """Determine whether to use requests or urllib2 to call suite API.""" impl = self._call_server_impl_urllib2 try: import requests except ImportError: pass else: if [int(_) for _ in requests.__version__.split(".")] >= [2, 4, 2]: impl = self._call_server_impl_requests try: return impl(url, method, payload) except ClientConnectError as exc: if self.suite is None: raise # Cannot connect, perhaps suite is no longer running and is leaving # behind a contact file? try: self.srv_files_mgr.detect_old_contact_file( self.suite, (self.host, self.port)) except (AssertionError, SuiteServiceFileError): raise exc else: # self.srv_files_mgr.detect_old_contact_file should delete left # behind contact file if the old suite process no longer # exists. Should be safe to report that the suite has stopped. raise ClientConnectError(exc.args[0], exc.STOPPED % self.suite) def _call_server_impl_requests(self, url, method, payload): """Call server with "requests" library.""" import requests from requests.packages.urllib3.exceptions import InsecureRequestWarning warnings.simplefilter("ignore", InsecureRequestWarning) if self.session is None: self.session = requests.Session() if method == self.METHOD_POST: session_method = self.session.post else: session_method = self.session.get scheme = url.split(':', 1)[0] # Can use urlparse? username, password, verify = self._get_auth(scheme) try: ret = session_method( url, json=payload, verify=verify, proxies={}, headers=self._get_headers(), auth=requests.auth.HTTPDigestAuth(username, password), timeout=self.timeout ) except requests.exceptions.SSLError as exc: if "unknown protocol" in str(exc) and url.startswith("https:"): # Server is using http rather than https, for some reason. sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc)) raise CylcError( "Cannot issue commands over unsecured http.") if cylc.flags.debug: traceback.print_exc() raise ClientConnectError(url, exc) except requests.exceptions.Timeout as exc: if cylc.flags.debug: traceback.print_exc() raise ClientTimeout(url, exc) except requests.exceptions.RequestException as exc: if cylc.flags.debug: traceback.print_exc() raise ClientConnectError(url, exc) if ret.status_code == 401: access_desc = 'private' if self.auth == self.ANON_AUTH: access_desc = 'public' raise ClientDeniedError(url, self.prog_name, access_desc) if ret.status_code >= 400: exception_text = get_exception_from_html(ret.text) if exception_text: sys.stderr.write(exception_text) else: sys.stderr.write(ret.text) try: ret.raise_for_status() except requests.exceptions.HTTPError as exc: if cylc.flags.debug: traceback.print_exc() raise ClientConnectedError(url, exc) if self.auth and self.auth[1] != NO_PASSPHRASE: self.srv_files_mgr.cache_passphrase( self.suite, self.owner, self.host, self.auth[1]) try: return ret.json() except ValueError: return ret.text def _call_server_impl_urllib2(self, url, method, payload): """Call server with "urllib2" library.""" import json import urllib2 import ssl unverified_context = getattr(ssl, '_create_unverified_context', None) if unverified_context is not None: ssl._create_default_https_context = unverified_context scheme = url.split(':', 1)[0] # Can use urlparse? username, password = self._get_auth(scheme)[0:2] auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() auth_manager.add_password(None, url, username, password) auth = urllib2.HTTPDigestAuthHandler(auth_manager) opener = urllib2.build_opener(auth, urllib2.HTTPSHandler()) headers_list = self._get_headers().items() if payload: payload = json.dumps(payload) headers_list.append(('Accept', 'application/json')) json_headers = {'Content-Type': 'application/json', 'Content-Length': len(payload)} else: payload = None json_headers = {'Content-Length': 0} opener.addheaders = headers_list req = urllib2.Request(url, payload, json_headers) # This is an unpleasant monkey patch, but there isn't an # alternative. urllib2 uses POST if there is a data payload # but that is not the correct criterion. # The difference is basically that POST changes # server state and GET doesn't. req.get_method = lambda: method try: response = opener.open(req, timeout=self.timeout) except urllib2.URLError as exc: if "unknown protocol" in str(exc) and url.startswith("https:"): # Server is using http rather than https, for some reason. sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc)) raise CylcError( "Cannot issue commands over unsecured http.") if cylc.flags.debug: traceback.print_exc() if "timed out" in str(exc): raise ClientTimeout(url, exc) else: raise ClientConnectError(url, exc) except Exception as exc: if cylc.flags.debug: traceback.print_exc() raise ClientError(url, exc) if response.getcode() == 401: access_desc = 'private' if self.auth == self.ANON_AUTH: access_desc = 'public' raise ClientDeniedError(url, self.prog_name, access_desc) response_text = response.read() if response.getcode() >= 400: exception_text = get_exception_from_html(response_text) if exception_text: sys.stderr.write(exception_text) else: sys.stderr.write(response_text) raise ClientConnectedError( url, "%s HTTP return code" % response.getcode()) if self.auth and self.auth[1] != NO_PASSPHRASE: self.srv_files_mgr.cache_passphrase( self.suite, self.owner, self.host, self.auth[1]) try: return json.loads(response_text) except ValueError: return response_text def _call_server_via_comms2(self, function, payload, **kwargs): """Call server via "cylc client --use-ssh". Call "cylc client --use-ssh" using `subprocess.Popen`. Payload and arguments of the API method are serialized as JSON and are written to a temporary file, which is then used as the STDIN of the "cylc client" command. The external call here should be even safer than a direct HTTP(S) call, as it can be blocked by SSH before it even gets a chance to make the subsequent HTTP(S) call. Arguments: function (str): name of API method, argument 1 of "cylc client". payload (str): extra data or information for the API method. **kwargs (dict): arguments for the API method. """ import json from cylc.remote import remote_cylc_cmd command = ["client", function, self.suite] if payload: kwargs["payload"] = payload if kwargs: from tempfile import TemporaryFile stdin = TemporaryFile() json.dump(kwargs, stdin) stdin.seek(0) else: # With stdin=None, `remote_cylc_cmd` will: # * Set stdin to open(os.devnull) # * Add `-n` to the SSH command stdin = None proc = remote_cylc_cmd( command, self.owner, self.host, capture=True, ssh_login_shell=(self.comms1.get( self.srv_files_mgr.KEY_SSH_USE_LOGIN_SHELL ) in ['True', 'true']), ssh_cylc=(r'%s/bin/cylc' % self.comms1.get( self.srv_files_mgr.KEY_DIR_ON_SUITE_HOST) ), stdin=stdin, ) out = proc.communicate()[0] return_code = proc.wait() if return_code: from pipes import quote command_str = " ".join(quote(item) for item in command) raise ClientError(command_str, "return-code=%d" % return_code) return json.loads(out) def _get_auth(self, protocol): """Return a user/password Digest Auth.""" if self.auth is None: self.auth = self.ANON_AUTH try: pphrase = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_PASSPHRASE, self.suite, self.owner, self.host, content=True) if protocol == 'https': verify = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_CERT, self.suite, self.owner, self.host) else: verify = False except SuiteServiceFileError: pass else: if pphrase and pphrase != NO_PASSPHRASE: self.auth = ('cylc', pphrase, verify) return self.auth def _get_headers(self): """Return HTTP headers identifying the client.""" user_agent_string = ( "cylc/%s prog_name/%s uuid/%s" % ( CYLC_VERSION, self.prog_name, self.my_uuid ) ) auth_info = "%s@%s" % (get_user(), get_host()) return {"User-Agent": user_agent_string, "From": auth_info} def _load_contact_info(self): """Obtain suite owner, host, port info. Determine host and port using content in port file, unless already specified. """ if self.host and self.port: return if self.port: # In case the contact file is corrupted, user can specify the port. self.host = get_host() return try: # Always trust the values in the contact file otherwise. self.comms1 = self.srv_files_mgr.load_contact_file( self.suite, self.owner, self.host) # Port inside "try" block, as it needs a type conversion self.port = int(self.comms1.get(self.srv_files_mgr.KEY_PORT)) except (IOError, ValueError, SuiteServiceFileError): raise ClientInfoError(self.suite) else: # Check mismatch suite UUID env_suite = os.getenv(self.srv_files_mgr.KEY_NAME) env_uuid = os.getenv(self.srv_files_mgr.KEY_UUID) if (self.suite and env_suite and env_suite == self.suite and env_uuid and env_uuid != self.comms1.get(self.srv_files_mgr.KEY_UUID)): raise ClientInfoUUIDError( env_uuid, self.comms1[self.srv_files_mgr.KEY_UUID]) # All good self.host = self.comms1.get(self.srv_files_mgr.KEY_HOST) self.owner = self.comms1.get(self.srv_files_mgr.KEY_OWNER) if self.srv_files_mgr.KEY_API not in self.comms1: self.comms1[self.srv_files_mgr.KEY_API] = 0 # <=7.5.0 compat # Indirect comms settings self.comms2.clear() try: self.comms2.update(self.srv_files_mgr.load_contact_file( self.suite, self.owner, self.host, SuiteSrvFilesManager.FILE_BASE_CONTACT2)) except SuiteServiceFileError: pass
def __init__(self): self.initialised_hosts = {} # user_at_host: should_unlink self.single_task_mode = False self.suite_srv_files_mgr = SuiteSrvFilesManager()
def setUp(self): self.suite_srv_files_mgr = SuiteSrvFilesManager()
def setUp(self): self.suite_srv_files_mgr = SuiteSrvFilesManager()
class CommsDaemon(object): """Wrap HTTPS daemon for a suite.""" def __init__(self, suite): # Suite only needed for back-compat with old clients (see below): self.suite = suite # Figure out the ports we are allowed to use. base_port = GLOBAL_CFG.get(['communication', 'base port']) max_ports = GLOBAL_CFG.get( ['communication', 'maximum number of ports']) self.ok_ports = range(int(base_port), int(base_port) + int(max_ports)) random.shuffle(self.ok_ports) comms_options = GLOBAL_CFG.get(['communication', 'options']) # HTTP Digest Auth uses MD5 - pretty secure in this use case. # Extending it with extra algorithms is allowed, but won't be # supported by most browsers. requests and urllib2 are OK though. self.hash_algorithm = "MD5" if "SHA1" in comms_options: # Note 'SHA' rather than 'SHA1'. self.hash_algorithm = "SHA" self.srv_files_mgr = SuiteSrvFilesManager() self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain( { 'cylc': self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_PASSPHRASE, suite, content=True), 'anon': NO_PASSPHRASE }, algorithm=self.hash_algorithm) try: self.cert = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_CERT, suite) self.pkey = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_PEM, suite) except SuiteServiceFileError: self.cert = None self.pkey = None self.client_reporter = CommsClientReporter.get_inst() self.start() def shutdown(self): """Shutdown the daemon.""" if hasattr(self, "engine"): self.engine.exit() self.engine.block() def connect(self, obj, name): """Connect obj and name to the daemon.""" cherrypy.tree.mount(obj, "/" + name) def disconnect(self, obj): """Disconnect obj from the daemon.""" pass def get_port(self): """Return the daemon port.""" return self.port def report_connection_if_denied(self): """Report connection if denied.""" self.client_reporter.report_connection_if_denied() def start(self): """Start quick web service.""" # cherrypy.config["tools.encode.on"] = True # cherrypy.config["tools.encode.encoding"] = "utf-8" cherrypy.config["server.socket_host"] = '0.0.0.0' cherrypy.config["engine.autoreload.on"] = False try: from OpenSSL import SSL, crypto cherrypy.config['server.ssl_module'] = 'pyopenSSL' cherrypy.config['server.ssl_certificate'] = self.cert cherrypy.config['server.ssl_private_key'] = self.pkey except ImportError: ERR.warning("no HTTPS/OpenSSL support") cherrypy.config['log.screen'] = None key = binascii.hexlify(os.urandom(16)) cherrypy.config.update({ 'tools.auth_digest.on': True, 'tools.auth_digest.realm': self.suite, 'tools.auth_digest.get_ha1': self.get_ha1, 'tools.auth_digest.key': key, 'tools.auth_digest.algorithm': self.hash_algorithm }) cherrypy.tools.connect_log = cherrypy.Tool( 'on_end_resource', self.report_connection_if_denied) cherrypy.config['tools.connect_log.on'] = True self.engine = cherrypy.engine for port in self.ok_ports: cherrypy.config["server.socket_port"] = port try: cherrypy.engine.start() cherrypy.engine.wait(cherrypy.engine.states.STARTED) except Exception: if cylc.flags.debug: traceback.print_exc() # We need to reinitialise the httpserver for each port attempt. cherrypy.server.httpserver = None else: if cherrypy.engine.state == cherrypy.engine.states.STARTED: self.port = port return raise Exception("No available ports")
class RemoteJobHostManager(object): """Manage a remote job host.""" _INSTANCE = None @classmethod def get_inst(cls): """Return a singleton instance of this class.""" if cls._INSTANCE is None: cls._INSTANCE = cls() return cls._INSTANCE def __init__(self): self.initialised_hosts = {} # user_at_host: should_unlink self.single_task_mode = False self.suite_srv_files_mgr = SuiteSrvFilesManager() def init_suite_run_dir(self, reg, user_at_host): """Initialise suite run dir on a user@host. Create SUITE_RUN_DIR/log/job/ if necessary. Install suite contact environment file. Install suite python modules. Raise RemoteJobHostInitError if initialisation cannot complete. """ if "@" in user_at_host: owner, host = user_at_host.split("@", 1) else: owner, host = None, user_at_host if ( (owner, host) in [(None, "localhost"), (USER, "localhost")] or host in self.initialised_hosts or self.single_task_mode ): return r_suite_run_dir = GLOBAL_CFG.get_derived_host_item(reg, "suite run directory", host, owner) r_log_job_dir = GLOBAL_CFG.get_derived_host_item(reg, "suite job log directory", host, owner) r_suite_srv_dir = os.path.join(r_suite_run_dir, self.suite_srv_files_mgr.DIR_BASE_SRV) # Create a UUID file in the service directory. # If remote host has the file in its service directory, we can assume # that the remote host has a shared file system with the suite host. ssh_tmpl = GLOBAL_CFG.get_host_item("remote shell template", host, owner) uuid_str = str(uuid4()) uuid_fname = os.path.join(self.suite_srv_files_mgr.get_suite_srv_dir(reg), uuid_str) try: open(uuid_fname, "wb").close() proc = Popen( shlex.split(ssh_tmpl) + ["-n", user_at_host, "test", "-e", os.path.join(r_suite_srv_dir, uuid_str)], stdout=PIPE, stderr=PIPE, ) if proc.wait() == 0: # Initialised, but no need to tidy up self.initialised_hosts[user_at_host] = False return finally: try: os.unlink(uuid_fname) except OSError: pass cmds = [] # Command to create suite directory structure on remote host. cmds.append( shlex.split(ssh_tmpl) + ["-n", user_at_host, "mkdir", "-p", r_suite_run_dir, r_log_job_dir, r_suite_srv_dir] ) # Command to copy contact and authentication files to remote host. # Note: no need to do this if task communication method is "poll". should_unlink = GLOBAL_CFG.get_host_item("task communication method", host, owner) != "poll" if should_unlink: scp_tmpl = GLOBAL_CFG.get_host_item("remote copy template", host, owner) cmds.append( shlex.split(scp_tmpl) + [ "-p", self.suite_srv_files_mgr.get_contact_file(reg), self.suite_srv_files_mgr.get_auth_item(self.suite_srv_files_mgr.FILE_BASE_PASSPHRASE, reg), self.suite_srv_files_mgr.get_auth_item(self.suite_srv_files_mgr.FILE_BASE_SSL_CERT, reg), user_at_host + ":" + r_suite_srv_dir + "/", ] ) # Command to copy python library to remote host. suite_run_py = os.path.join(GLOBAL_CFG.get_derived_host_item(reg, "suite run directory"), "python") if os.path.isdir(suite_run_py): cmds.append(shlex.split(scp_tmpl) + ["-pr", suite_run_py, user_at_host + ":" + r_suite_run_dir + "/"]) # Run commands in sequence. for cmd in cmds: proc = Popen(cmd, stdout=PIPE, stderr=PIPE) out, err = proc.communicate() if proc.wait(): raise RemoteJobHostInitError( RemoteJobHostInitError.MSG_INIT, user_at_host, " ".join([quote(item) for item in cmd]), proc.returncode, out, err, ) self.initialised_hosts[user_at_host] = should_unlink LOG.info("Initialised %s:%s" % (user_at_host, r_suite_run_dir)) def unlink_suite_contact_files(self, reg): """Remove suite contact files from initialised hosts. This is called on shutdown, so we don't want anything to hang. Terminate any incomplete SSH commands after 10 seconds. """ # Issue all SSH commands in parallel procs = {} for user_at_host, should_unlink in self.initialised_hosts.items(): if not should_unlink: continue if "@" in user_at_host: owner, host = user_at_host.split("@", 1) else: owner, host = None, user_at_host ssh_tmpl = GLOBAL_CFG.get_host_item("remote shell template", host, owner) r_suite_contact_file = os.path.join( GLOBAL_CFG.get_derived_host_item(reg, "suite run directory", host, owner), SuiteSrvFilesManager.DIR_BASE_SRV, SuiteSrvFilesManager.FILE_BASE_CONTACT, ) cmd = shlex.split(ssh_tmpl) + ["-n", user_at_host, "rm", "-f", r_suite_contact_file] procs[user_at_host] = (cmd, Popen(cmd, stdout=PIPE, stderr=PIPE)) # Wait for commands to complete for a max of 10 seconds timeout = time() + 10.0 while procs and time() < timeout: for user_at_host, (cmd, proc) in procs.items(): if not proc.poll(): continue del procs[user_at_host] out, err = proc.communicate() if proc.wait(): ERR.warning( RemoteJobHostInitError( RemoteJobHostInitError.MSG_TIDY, user_at_host, " ".join([quote(item) for item in cmd]), proc.returncode, out, err, ) ) # Terminate any remaining commands for user_at_host, (cmd, proc) in procs.items(): try: proc.terminate() except OSError: pass out, err = proc.communicate() proc.wait() ERR.warning( RemoteJobHostInitError( RemoteJobHostInitError.MSG_TIDY, user_at_host, " ".join([quote(item) for item in cmd]), proc.returncode, out, err, ) )