def coordinator():
"""
Deploy the coordinator configuration to the coordinator node
"""
if env.host in util.get_coordinator_role():
_LOGGER.info("Setting coordinator configuration for " + env.host)
configure_presto(coord.get_conf(), constants.REMOTE_CONF_DIR)
def coordinator():
"""
Deploy the coordinator configuration to the coordinator node
"""
if env.host in util.get_coordinator_role():
_LOGGER.info("Setting coordinator configuration for " + env.host)
configure_presto(coord.get_conf(), constants.REMOTE_CONF_DIR)
def query_info(query_id):
"""
Gather information about the query identified by the given
query_id and store that in a JSON file.
Parameters:
query_id - id of the query for which info has to be gathered
"""
if env.host not in fabricapi.get_coordinator_role():
return
err_msg = 'Unable to retrieve information. Please check that the ' \
'query_id is correct, or check that server is up with ' \
'command: server status'
req = get_request(QUERY_REQUEST_URL + query_id, err_msg)
query_info_file_name = os.path.join(TMP_PRESTO_DEBUG,
'query_info_' + query_id + '.json')
if not os.path.exists(TMP_PRESTO_DEBUG):
os.mkdir(TMP_PRESTO_DEBUG)
with open(query_info_file_name, 'w') as out_file:
out_file.write(json.dumps(req.json(), indent=4))
print('Gathered query information in file: ' + query_info_file_name)
def query_info(query_id):
"""
Gather information about the query identified by the given
query_id and store that in a JSON file.
Parameters:
query_id - id of the query for which info has to be gathered
"""
if env.host not in fabricapi.get_coordinator_role():
return
err_msg = 'Unable to retrieve information. Please check that the ' \
'query_id is correct, or check that server is up with ' \
'command: server status'
req = get_request(request_url(QUERY_REQUEST_EXT + query_id), err_msg)
query_info_file_name = os.path.join(TMP_PRESTO_DEBUG, 'query_info_' + query_id + '.json')
try:
os.makedirs(TMP_PRESTO_DEBUG)
except OSError:
if not os.path.isdir(TMP_PRESTO_DEBUG):
raise
with open(query_info_file_name, 'w') as out_file:
out_file.write(json.dumps(req.json(), indent=4))
print('Gathered query information in file: ' + query_info_file_name)
def workers():
"""
Deploy workers configuration to the worker nodes.
This will not deploy configuration for a coordinator that is also a worker
"""
if env.host in util.get_worker_role() and env.host not in util.get_coordinator_role():
_LOGGER.info("Setting worker configuration for " + env.host)
configure_presto(w.get_conf(), constants.REMOTE_CONF_DIR)
def workers():
"""
Deploy workers configuration to the worker nodes.
This will not deploy configuration for a coordinator that is also a worker
"""
if env.host in util.get_worker_role() and env.host \
not in util.get_coordinator_role():
_LOGGER.info("Setting worker configuration for " + env.host)
configure_presto(w.get_conf(), constants.REMOTE_CONF_DIR)
def default_config(self, filename):
try:
conf = copy.deepcopy(self.DEFAULT_PROPERTIES[filename])
except KeyError:
raise ConfigurationError('Invalid configuration file name: %s' %
filename)
if filename == 'config.properties':
coordinator = util.get_coordinator_role()[0]
conf['discovery.uri'] = 'http://%s:8080' % coordinator
return conf
def default_config(self, filename):
try:
conf = copy.deepcopy(self.DEFAULT_PROPERTIES[filename])
except KeyError:
raise ConfigurationError('Invalid configuration file name: %s' %
filename)
if filename == 'config.properties':
coordinator = util.get_coordinator_role()[0]
conf['discovery.uri'] = 'http://%s:8080' % coordinator
return conf
def check_server_status():
"""
Checks if server is running for env.host. Retries connecting to server
until server is up or till RETRY_TIMEOUT is reached
Parameters:
client - client that executes the query
Returns:
True or False
"""
if len(get_coordinator_role()) < 1:
warn('No coordinator defined. Cannot verify server status.')
client = PrestoClient(get_coordinator_role()[0], env.user)
node_id = lookup_string_config('node.id', os.path.join(constants.REMOTE_CONF_DIR, 'node.properties'), env.host)
try:
return query_server_for_status(client, node_id)
except RetryError:
return False
def set_env_from_conf():
conf = get_conf()
env.user = conf['username']
env.port = conf['port']
env.roledefs['coordinator'] = [conf['coordinator']]
env.roledefs['worker'] = conf['workers']
env.roledefs['all'] = dedup_list(util.get_coordinator_role()
+ util.get_worker_role())
if not env.hosts:
env.hosts = env.roledefs['all'][:]
def set_env_from_conf():
conf = get_conf()
env.user = conf['username']
env.port = conf['port']
env.roledefs['coordinator'] = [conf['coordinator']]
env.roledefs['worker'] = conf['workers']
env.roledefs['all'] = dedup_list(util.get_coordinator_role() +
util.get_worker_role())
# This ensures that we honor a hosts list passed on the command line.
if not env.hosts:
env.hosts = env.roledefs['all'][:]
def set_env_from_conf(self, conf):
env.user = conf['username']
env.port = conf['port']
try:
env.java8_home = conf['java8_home']
except KeyError:
env.java8_home = None
env.roledefs['coordinator'] = [conf['coordinator']]
env.roledefs['worker'] = conf['workers']
env.roledefs['all'] = self._dedup_list(util.get_coordinator_role() +
util.get_worker_role())
env.hosts = env.roledefs['all'][:]
def set_env_from_conf(self, conf):
env.user = conf['username']
env.port = conf['port']
try:
env.java8_home = conf['java8_home']
except KeyError:
env.java8_home = None
env.roledefs['coordinator'] = [conf['coordinator']]
env.roledefs['worker'] = conf['workers']
env.roledefs['all'] = self._dedup_list(util.get_coordinator_role() +
util.get_worker_role())
env.hosts = env.roledefs['all'][:]
def check_server_status():
"""
Checks if server is running for env.host. Retries connecting to server
until server is up or till RETRY_TIMEOUT is reached
Parameters:
client - client that executes the query
Returns:
True or False
"""
if len(get_coordinator_role()) < 1:
warn('No coordinator defined. Cannot verify server status.')
client = PrestoClient(get_coordinator_role()[0], env.user)
node_id = lookup_string_config(
'node.id', os.path.join(constants.REMOTE_CONF_DIR, 'node.properties'),
env.host)
try:
return query_server_for_status(client, node_id)
except RetryError:
return False
def get_status_from_coordinator():
with closing(PrestoClient(get_coordinator_role()[0], env.user)) as client:
try:
coordinator_status = run_sql(client, SYSTEM_RUNTIME_NODES)
catalog_status = get_catalog_info_from(client)
except BaseException as e:
# Just log errors that come from a missing port or anything else; if
# we can't connect to the coordinator, we just want to print out a
# minimal status anyway.
_LOGGER.warn(e.message)
coordinator_status = []
catalog_status = []
with settings(hide('running')):
node_information = execute(collect_node_information,
hosts=get_host_list())
for host in get_host_list():
if isinstance(node_information[host], Exception):
external_ip = 'Unknown'
is_running = False
error_message = node_information[host].message
else:
(external_ip, is_running,
error_message) = node_information[host]
print_status_header(external_ip, is_running, host)
if error_message:
print('\t' + error_message)
elif not coordinator_status:
print(
'\tNo information available: unable to query coordinator')
elif not is_running:
print('\tNo information available')
else:
version_string = get_presto_version()
version = strip_tag(split_version(version_string))
query, processor = NODE_INFO_PER_URI_SQL.for_version(version)
# just get the node_info row for the host if server is up
node_info_row = run_sql(client, query % external_ip)
node_status = processor(node_info_row)
if node_status:
print_node_info(node_status, catalog_status)
else:
print(
'\tNo information available: the coordinator has not yet'
' discovered this node')
def collect_node_information():
client = PrestoClient(get_coordinator_role()[0], env.user)
with settings(hide('warnings')):
error_message = check_presto_version()
if error_message:
external_ip = 'Unknown'
is_running = False
else:
with settings(hide('warnings', 'aborts', 'stdout')):
try:
external_ip = get_ext_ip_of_node(client)
except:
external_ip = 'Unknown'
try:
is_running = service('status')
except:
is_running = False
return external_ip, is_running, error_message
def collect_node_information():
with closing(PrestoClient(get_coordinator_role()[0], env.user)) as client:
with settings(hide('warnings')):
error_message = check_presto_version()
if error_message:
external_ip = 'Unknown'
is_running = False
else:
with settings(hide('warnings', 'aborts', 'stdout')):
try:
external_ip = get_ext_ip_of_node(client)
except:
external_ip = 'Unknown'
try:
is_running = service('status')
except:
is_running = False
return external_ip, is_running, error_message
def get_status_from_coordinator():
client = PrestoClient(get_coordinator_role()[0], env.user)
try:
coordinator_status = run_sql(client, SYSTEM_RUNTIME_NODES)
connector_status = get_connector_info_from(client)
except BaseException as e:
# Just log errors that come from a missing port or anything else; if
# we can't connect to the coordinator, we just want to print out a
# minimal status anyway.
_LOGGER.warn(e.message)
coordinator_status = []
connector_status = []
with settings(hide('running')):
node_information = execute(collect_node_information,
hosts=get_host_list())
for host in get_host_list():
if isinstance(node_information[host], Exception):
external_ip = 'Unknown'
is_running = False
error_message = node_information[host].message
else:
(external_ip, is_running, error_message) = node_information[host]
print_status_header(external_ip, is_running, host)
if error_message:
print('\t' + error_message)
elif not coordinator_status:
print('\tNo information available: unable to query coordinator')
elif not is_running:
print('\tNo information available')
else:
version_string = get_presto_version()
version = strip_tag(split_version(version_string))
query, processor = NODE_INFO_PER_URI_SQL.for_version(version)
# just get the node_info row for the host if server is up
node_info_row = run_sql(client, query % external_ip)
node_status = processor(node_info_row)
if node_status:
print_node_info(node_status, connector_status)
else:
print('\tNo information available: the coordinator has not yet'
' discovered this node')
def system_info():
"""
Gather system information like nodes in the system, presto
version, presto-admin version, os version etc.
"""
if env.host not in fabricapi.get_coordinator_role():
return
err_msg = 'Unable to access node information. ' \
'Please check that server is up with command: server status'
req = get_request(request_url(NODES_REQUEST_EXT), err_msg)
if not os.path.exists(TMP_PRESTO_DEBUG):
os.mkdir(TMP_PRESTO_DEBUG)
downloaded_sys_info_loc = os.path.join(TMP_PRESTO_DEBUG, "sysinfo")
node_info_file_name = os.path.join(downloaded_sys_info_loc,
'node_info.json')
if not os.path.exists(downloaded_sys_info_loc):
os.mkdir(downloaded_sys_info_loc)
with open(node_info_file_name, 'w') as out_file:
out_file.write(json.dumps(req.json(), indent=4))
_LOGGER.debug('Gathered node information in file: ' + node_info_file_name)
conn_file_name = os.path.join(downloaded_sys_info_loc,
'connector_info.txt')
client = PrestoClient(env.host, env.user)
conn_info = get_connector_info_from(client)
with open(conn_file_name, 'w') as out_file:
out_file.write(conn_info + '\n')
_LOGGER.debug('Gathered connector information in file: ' + conn_file_name)
execute(get_system_info, downloaded_sys_info_loc, roles=env.roles)
make_tarfile(OUTPUT_FILENAME_FOR_SYS_INFO, downloaded_sys_info_loc)
print 'System info archive created: ' + OUTPUT_FILENAME_FOR_SYS_INFO
def system_info():
"""
Gather system information like nodes in the system, presto
version, presto-admin version, os version etc.
"""
if env.host not in fabricapi.get_coordinator_role():
return
err_msg = 'Unable to access node information. ' \
'Please check that server is up with command: server status'
req = get_request(request_url(NODES_REQUEST_EXT), err_msg)
if not os.path.exists(TMP_PRESTO_DEBUG):
os.mkdir(TMP_PRESTO_DEBUG)
downloaded_sys_info_loc = os.path.join(TMP_PRESTO_DEBUG, "sysinfo")
node_info_file_name = os.path.join(downloaded_sys_info_loc,
'node_info.json')
if not os.path.exists(downloaded_sys_info_loc):
os.mkdir(downloaded_sys_info_loc)
with open(node_info_file_name, 'w') as out_file:
out_file.write(json.dumps(req.json(), indent=4))
_LOGGER.debug('Gathered node information in file: ' + node_info_file_name)
conn_file_name = os.path.join(downloaded_sys_info_loc,
'connector_info.txt')
client = PrestoClient(env.host, env.user)
conn_info = get_connector_info_from(client)
with open(conn_file_name, 'w') as out_file:
out_file.write(conn_info + '\n')
_LOGGER.debug('Gathered connector information in file: ' + conn_file_name)
execute(get_system_info, downloaded_sys_info_loc, roles=env.roles)
make_tarfile(OUTPUT_FILENAME_FOR_SYS_INFO, downloaded_sys_info_loc)
print 'System info archive created: ' + OUTPUT_FILENAME_FOR_SYS_INFO
def get_conf_from_fabric():
return {'coordinator': util.get_coordinator_role()[0],
'workers': util.get_worker_role(),
'port': env.port,
'username': env.user}
def build_defaults():
conf = copy.deepcopy(DEFAULT_PROPERTIES)
coordinator = util.get_coordinator_role()[0]
conf['config.properties']['discovery.uri'] = 'http://' + coordinator \
+ ':8080'
return conf
def get_conf_from_fabric():
return {'coordinator': util.get_coordinator_role()[0],
'workers': util.get_worker_role(),
'port': env.port,
'username': env.user}
def build_defaults():
conf = copy.deepcopy(DEFAULT_PROPERTIES)
coordinator = util.get_coordinator_role()[0]
conf["config.properties"]["discovery.uri"] = "http://" + coordinator + ":8080"
return conf
