def coordinator(): """ Deploy the coordinator configuration to the coordinator node """ if env.host in util.get_coordinator_role(): _LOGGER.info("Setting coordinator configuration for " + env.host) configure_presto(coord.get_conf(), constants.REMOTE_CONF_DIR)
def coordinator(): """ Deploy the coordinator configuration to the coordinator node """ if env.host in util.get_coordinator_role(): _LOGGER.info("Setting coordinator configuration for " + env.host) configure_presto(coord.get_conf(), constants.REMOTE_CONF_DIR)
def query_info(query_id): """ Gather information about the query identified by the given query_id and store that in a JSON file. Parameters: query_id - id of the query for which info has to be gathered """ if env.host not in fabricapi.get_coordinator_role(): return err_msg = 'Unable to retrieve information. Please check that the ' \ 'query_id is correct, or check that server is up with ' \ 'command: server status' req = get_request(QUERY_REQUEST_URL + query_id, err_msg) query_info_file_name = os.path.join(TMP_PRESTO_DEBUG, 'query_info_' + query_id + '.json') if not os.path.exists(TMP_PRESTO_DEBUG): os.mkdir(TMP_PRESTO_DEBUG) with open(query_info_file_name, 'w') as out_file: out_file.write(json.dumps(req.json(), indent=4)) print('Gathered query information in file: ' + query_info_file_name)
def query_info(query_id): """ Gather information about the query identified by the given query_id and store that in a JSON file. Parameters: query_id - id of the query for which info has to be gathered """ if env.host not in fabricapi.get_coordinator_role(): return err_msg = 'Unable to retrieve information. Please check that the ' \ 'query_id is correct, or check that server is up with ' \ 'command: server status' req = get_request(request_url(QUERY_REQUEST_EXT + query_id), err_msg) query_info_file_name = os.path.join(TMP_PRESTO_DEBUG, 'query_info_' + query_id + '.json') try: os.makedirs(TMP_PRESTO_DEBUG) except OSError: if not os.path.isdir(TMP_PRESTO_DEBUG): raise with open(query_info_file_name, 'w') as out_file: out_file.write(json.dumps(req.json(), indent=4)) print('Gathered query information in file: ' + query_info_file_name)
def workers(): """ Deploy workers configuration to the worker nodes. This will not deploy configuration for a coordinator that is also a worker """ if env.host in util.get_worker_role() and env.host not in util.get_coordinator_role(): _LOGGER.info("Setting worker configuration for " + env.host) configure_presto(w.get_conf(), constants.REMOTE_CONF_DIR)
def workers(): """ Deploy workers configuration to the worker nodes. This will not deploy configuration for a coordinator that is also a worker """ if env.host in util.get_worker_role() and env.host \ not in util.get_coordinator_role(): _LOGGER.info("Setting worker configuration for " + env.host) configure_presto(w.get_conf(), constants.REMOTE_CONF_DIR)
def default_config(self, filename): try: conf = copy.deepcopy(self.DEFAULT_PROPERTIES[filename]) except KeyError: raise ConfigurationError('Invalid configuration file name: %s' % filename) if filename == 'config.properties': coordinator = util.get_coordinator_role()[0] conf['discovery.uri'] = 'http://%s:8080' % coordinator return conf
def default_config(self, filename): try: conf = copy.deepcopy(self.DEFAULT_PROPERTIES[filename]) except KeyError: raise ConfigurationError('Invalid configuration file name: %s' % filename) if filename == 'config.properties': coordinator = util.get_coordinator_role()[0] conf['discovery.uri'] = 'http://%s:8080' % coordinator return conf
def check_server_status(): """ Checks if server is running for env.host. Retries connecting to server until server is up or till RETRY_TIMEOUT is reached Parameters: client - client that executes the query Returns: True or False """ if len(get_coordinator_role()) < 1: warn('No coordinator defined. Cannot verify server status.') client = PrestoClient(get_coordinator_role()[0], env.user) node_id = lookup_string_config('node.id', os.path.join(constants.REMOTE_CONF_DIR, 'node.properties'), env.host) try: return query_server_for_status(client, node_id) except RetryError: return False
def set_env_from_conf(): conf = get_conf() env.user = conf['username'] env.port = conf['port'] env.roledefs['coordinator'] = [conf['coordinator']] env.roledefs['worker'] = conf['workers'] env.roledefs['all'] = dedup_list(util.get_coordinator_role() + util.get_worker_role()) if not env.hosts: env.hosts = env.roledefs['all'][:]
def set_env_from_conf(): conf = get_conf() env.user = conf['username'] env.port = conf['port'] env.roledefs['coordinator'] = [conf['coordinator']] env.roledefs['worker'] = conf['workers'] env.roledefs['all'] = dedup_list(util.get_coordinator_role() + util.get_worker_role()) # This ensures that we honor a hosts list passed on the command line. if not env.hosts: env.hosts = env.roledefs['all'][:]
def set_env_from_conf(self, conf): env.user = conf['username'] env.port = conf['port'] try: env.java8_home = conf['java8_home'] except KeyError: env.java8_home = None env.roledefs['coordinator'] = [conf['coordinator']] env.roledefs['worker'] = conf['workers'] env.roledefs['all'] = self._dedup_list(util.get_coordinator_role() + util.get_worker_role()) env.hosts = env.roledefs['all'][:]
def set_env_from_conf(self, conf): env.user = conf['username'] env.port = conf['port'] try: env.java8_home = conf['java8_home'] except KeyError: env.java8_home = None env.roledefs['coordinator'] = [conf['coordinator']] env.roledefs['worker'] = conf['workers'] env.roledefs['all'] = self._dedup_list(util.get_coordinator_role() + util.get_worker_role()) env.hosts = env.roledefs['all'][:]
def check_server_status(): """ Checks if server is running for env.host. Retries connecting to server until server is up or till RETRY_TIMEOUT is reached Parameters: client - client that executes the query Returns: True or False """ if len(get_coordinator_role()) < 1: warn('No coordinator defined. Cannot verify server status.') client = PrestoClient(get_coordinator_role()[0], env.user) node_id = lookup_string_config( 'node.id', os.path.join(constants.REMOTE_CONF_DIR, 'node.properties'), env.host) try: return query_server_for_status(client, node_id) except RetryError: return False
def get_status_from_coordinator(): with closing(PrestoClient(get_coordinator_role()[0], env.user)) as client: try: coordinator_status = run_sql(client, SYSTEM_RUNTIME_NODES) catalog_status = get_catalog_info_from(client) except BaseException as e: # Just log errors that come from a missing port or anything else; if # we can't connect to the coordinator, we just want to print out a # minimal status anyway. _LOGGER.warn(e.message) coordinator_status = [] catalog_status = [] with settings(hide('running')): node_information = execute(collect_node_information, hosts=get_host_list()) for host in get_host_list(): if isinstance(node_information[host], Exception): external_ip = 'Unknown' is_running = False error_message = node_information[host].message else: (external_ip, is_running, error_message) = node_information[host] print_status_header(external_ip, is_running, host) if error_message: print('\t' + error_message) elif not coordinator_status: print( '\tNo information available: unable to query coordinator') elif not is_running: print('\tNo information available') else: version_string = get_presto_version() version = strip_tag(split_version(version_string)) query, processor = NODE_INFO_PER_URI_SQL.for_version(version) # just get the node_info row for the host if server is up node_info_row = run_sql(client, query % external_ip) node_status = processor(node_info_row) if node_status: print_node_info(node_status, catalog_status) else: print( '\tNo information available: the coordinator has not yet' ' discovered this node')
def collect_node_information(): client = PrestoClient(get_coordinator_role()[0], env.user) with settings(hide('warnings')): error_message = check_presto_version() if error_message: external_ip = 'Unknown' is_running = False else: with settings(hide('warnings', 'aborts', 'stdout')): try: external_ip = get_ext_ip_of_node(client) except: external_ip = 'Unknown' try: is_running = service('status') except: is_running = False return external_ip, is_running, error_message
def collect_node_information(): with closing(PrestoClient(get_coordinator_role()[0], env.user)) as client: with settings(hide('warnings')): error_message = check_presto_version() if error_message: external_ip = 'Unknown' is_running = False else: with settings(hide('warnings', 'aborts', 'stdout')): try: external_ip = get_ext_ip_of_node(client) except: external_ip = 'Unknown' try: is_running = service('status') except: is_running = False return external_ip, is_running, error_message
def get_status_from_coordinator(): client = PrestoClient(get_coordinator_role()[0], env.user) try: coordinator_status = run_sql(client, SYSTEM_RUNTIME_NODES) connector_status = get_connector_info_from(client) except BaseException as e: # Just log errors that come from a missing port or anything else; if # we can't connect to the coordinator, we just want to print out a # minimal status anyway. _LOGGER.warn(e.message) coordinator_status = [] connector_status = [] with settings(hide('running')): node_information = execute(collect_node_information, hosts=get_host_list()) for host in get_host_list(): if isinstance(node_information[host], Exception): external_ip = 'Unknown' is_running = False error_message = node_information[host].message else: (external_ip, is_running, error_message) = node_information[host] print_status_header(external_ip, is_running, host) if error_message: print('\t' + error_message) elif not coordinator_status: print('\tNo information available: unable to query coordinator') elif not is_running: print('\tNo information available') else: version_string = get_presto_version() version = strip_tag(split_version(version_string)) query, processor = NODE_INFO_PER_URI_SQL.for_version(version) # just get the node_info row for the host if server is up node_info_row = run_sql(client, query % external_ip) node_status = processor(node_info_row) if node_status: print_node_info(node_status, connector_status) else: print('\tNo information available: the coordinator has not yet' ' discovered this node')
def system_info(): """ Gather system information like nodes in the system, presto version, presto-admin version, os version etc. """ if env.host not in fabricapi.get_coordinator_role(): return err_msg = 'Unable to access node information. ' \ 'Please check that server is up with command: server status' req = get_request(request_url(NODES_REQUEST_EXT), err_msg) if not os.path.exists(TMP_PRESTO_DEBUG): os.mkdir(TMP_PRESTO_DEBUG) downloaded_sys_info_loc = os.path.join(TMP_PRESTO_DEBUG, "sysinfo") node_info_file_name = os.path.join(downloaded_sys_info_loc, 'node_info.json') if not os.path.exists(downloaded_sys_info_loc): os.mkdir(downloaded_sys_info_loc) with open(node_info_file_name, 'w') as out_file: out_file.write(json.dumps(req.json(), indent=4)) _LOGGER.debug('Gathered node information in file: ' + node_info_file_name) conn_file_name = os.path.join(downloaded_sys_info_loc, 'connector_info.txt') client = PrestoClient(env.host, env.user) conn_info = get_connector_info_from(client) with open(conn_file_name, 'w') as out_file: out_file.write(conn_info + '\n') _LOGGER.debug('Gathered connector information in file: ' + conn_file_name) execute(get_system_info, downloaded_sys_info_loc, roles=env.roles) make_tarfile(OUTPUT_FILENAME_FOR_SYS_INFO, downloaded_sys_info_loc) print 'System info archive created: ' + OUTPUT_FILENAME_FOR_SYS_INFO
def system_info(): """ Gather system information like nodes in the system, presto version, presto-admin version, os version etc. """ if env.host not in fabricapi.get_coordinator_role(): return err_msg = 'Unable to access node information. ' \ 'Please check that server is up with command: server status' req = get_request(request_url(NODES_REQUEST_EXT), err_msg) if not os.path.exists(TMP_PRESTO_DEBUG): os.mkdir(TMP_PRESTO_DEBUG) downloaded_sys_info_loc = os.path.join(TMP_PRESTO_DEBUG, "sysinfo") node_info_file_name = os.path.join(downloaded_sys_info_loc, 'node_info.json') if not os.path.exists(downloaded_sys_info_loc): os.mkdir(downloaded_sys_info_loc) with open(node_info_file_name, 'w') as out_file: out_file.write(json.dumps(req.json(), indent=4)) _LOGGER.debug('Gathered node information in file: ' + node_info_file_name) conn_file_name = os.path.join(downloaded_sys_info_loc, 'connector_info.txt') client = PrestoClient(env.host, env.user) conn_info = get_connector_info_from(client) with open(conn_file_name, 'w') as out_file: out_file.write(conn_info + '\n') _LOGGER.debug('Gathered connector information in file: ' + conn_file_name) execute(get_system_info, downloaded_sys_info_loc, roles=env.roles) make_tarfile(OUTPUT_FILENAME_FOR_SYS_INFO, downloaded_sys_info_loc) print 'System info archive created: ' + OUTPUT_FILENAME_FOR_SYS_INFO
def get_conf_from_fabric(): return {'coordinator': util.get_coordinator_role()[0], 'workers': util.get_worker_role(), 'port': env.port, 'username': env.user}
def build_defaults(): conf = copy.deepcopy(DEFAULT_PROPERTIES) coordinator = util.get_coordinator_role()[0] conf['config.properties']['discovery.uri'] = 'http://' + coordinator \ + ':8080' return conf
def get_conf_from_fabric(): return {'coordinator': util.get_coordinator_role()[0], 'workers': util.get_worker_role(), 'port': env.port, 'username': env.user}
def build_defaults(): conf = copy.deepcopy(DEFAULT_PROPERTIES) coordinator = util.get_coordinator_role()[0] conf["config.properties"]["discovery.uri"] = "http://" + coordinator + ":8080" return conf