def configure_airflow(self, env):
import params
env.set_params(params)
File("/etc/systemd/system/airflow_webserver.service",
content=Template("webserver.j2", configurations=params),
owner='root',
group='root',
mode=0o0600)
File("/etc/systemd/system/airflow_scheduler.service",
content=Template("scheduler.j2", configurations=params),
owner='root',
group='root',
mode=0o0600)
Execute('sudo systemctl daemon-reload')
def configure_ac(self, env):
import params
env.set_params(params)
print("!!!!!!!!!!!!!!!!!!!!!!!creating file!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
File("/etc/systemd/system/airflow_portforward.service",
content=Template("portforward_service.j2", configurations=params),
owner='root',
group='root',
mode=0o0600
)
Execute('sudo systemctl daemon-reload')
def configure_ac(self, env):
import params
env.set_params(params)
File("/etc/systemd/system/jupyter_portforward.service",
content=Template("portforward_service_jupyter.j2",
configurations=params),
owner=params.anaconda_user,
group=params.anaconda_group,
mode=0o0600)
Execute('sudo systemctl daemon-reload')
def do_ams_collector_post(metric_collector_host, params):
ams_metrics_post_url = "/ws/v1/timeline/metrics/"
random_value1 = random.random()
headers = {"Content-type": "application/json"}
ca_certs = os.path.join(params.ams_grafana_conf_dir,
params.metric_truststore_ca_certs)
current_time = int(time.time()) * 1000
metric_json = Template('smoketest_metrics.json.j2', hostname=params.hostname, random1=random_value1,
current_time=current_time).get_content()
post_metrics_to_collector(ams_metrics_post_url, metric_collector_host, params.metric_collector_port, params.metric_collector_https_enabled,
metric_json, headers, ca_certs)
def configure_ac(self, env):
import params
env.set_params(params)
conf = dict()
for key in params.config['configurations']['jupyter-env']:
conf[key] = params.config['configurations']['jupyter-env'][key]
conf['jupyter_password'] = params.hashText(conf['jupyter_password'])
print("----------------------------\n", conf, "\n------------------------------------------------")
File("{0}jupyter_notebook_config.py".format(params.config_dir),
content=Template("jupyter_notebook_config.py.j2",
configurations=conf),
owner=params.anaconda_user,
group=params.anaconda_group,
mode=0o0600
)
def create_ams_datasource():
import params
server = Server(protocol = params.ams_grafana_protocol.strip(),
host = params.ams_grafana_host.strip(),
port = params.ams_grafana_port,
user = params.ams_grafana_admin_user,
password = params.ams_grafana_admin_pwd)
"""
Create AMS datasource in Grafana, if exsists make sure the collector url is accurate
"""
ams_datasource_json = Template('metrics_grafana_datasource.json.j2',
ams_datasource_name=METRICS_GRAFANA_DATASOURCE_NAME).get_content()
Logger.info("Checking if AMS Grafana datasource already exists")
response = perform_grafana_get_call(GRAFANA_DATASOURCE_URL, server)
create_datasource = True
if response and response.status == 200:
datasources = response.read()
datasources_json = json.loads(datasources)
for i in xrange(0, len(datasources_json)):
datasource_name = datasources_json[i]["name"]
if datasource_name == METRICS_GRAFANA_DATASOURCE_NAME:
create_datasource = False # datasource already exists
Logger.info("Ambari Metrics Grafana datasource already present. Checking Metrics Collector URL")
datasource_url = datasources_json[i]["url"]
if is_unchanged_datasource_url(datasource_url):
Logger.info("Metrics Collector URL validation succeeded.")
return
else: # Metrics datasource present, but collector host is wrong.
datasource_id = datasources_json[i]["id"]
Logger.info("Metrics Collector URL validation failed. Updating "
"datasource, id = %s" % datasource_id)
(response, data) = perform_grafana_put_call(GRAFANA_DATASOURCE_URL, datasource_id,
ams_datasource_json, server)
if response.status == 200:
Logger.info("Ambari Metrics Grafana data source updated.")
elif response.status == 500:
Logger.info("Ambari Metrics Grafana data source update failed. Not retrying.")
raise Fail("Ambari Metrics Grafana data source update failed. PUT request status: %s %s \n%s" %
(response.status, response.reason, data))
else:
raise Fail("Ambari Metrics Grafana data source creation failed. "
"PUT request status: %s %s \n%s" % (response.status, response.reason, data))
pass
pass
pass
else:
Logger.info("Error checking for Ambari Metrics Grafana datasource. Will attempt to create.")
if not create_datasource:
return
else:
Logger.info("Generating datasource:\n%s" % ams_datasource_json)
(response, data) = perform_grafana_post_call(GRAFANA_DATASOURCE_URL, ams_datasource_json, server)
if response.status == 200:
Logger.info("Ambari Metrics Grafana data source created.")
elif response.status == 500:
Logger.info("Ambari Metrics Grafana data source creation failed. Not retrying.")
raise Fail("Ambari Metrics Grafana data source creation failed. POST request status: %s %s \n%s" %
(response.status, response.reason, data))
else:
Logger.info("Ambari Metrics Grafana data source creation failed.")
raise Fail("Ambari Metrics Grafana data source creation failed. POST request status: %s %s \n%s" %
(response.status, response.reason, data))
pass
def setup_livy(env, type, upgrade_type=None, action=None):
import params
Directory([params.livy2_pid_dir, params.livy2_log_dir],
owner=params.livy2_user,
group=params.user_group,
mode=0775,
cd_access='a',
create_parents=True)
if type == 'server' and action == 'config':
params.HdfsResource(params.livy2_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.livy2_user,
mode=0775)
params.HdfsResource(None, action="execute")
params.HdfsResource(params.livy2_recovery_dir,
type="directory",
action="create_on_execute",
owner=params.livy2_user,
mode=0700)
params.HdfsResource(None, action="execute")
generate_logfeeder_input_config(
'spark2',
Template("input.config-spark2.json.j2", extra_imports=[default]))
# create livy-env.sh in etc/conf dir
File(
os.path.join(params.livy2_conf, 'livy-env.sh'),
owner=params.livy2_user,
group=params.livy2_group,
content=InlineTemplate(params.livy2_env_sh),
mode=0644,
)
# create livy.conf in etc/conf dir
PropertiesFile(
format("{livy2_conf}/livy.conf"),
properties=params.config['configurations']['livy2-conf'],
key_value_delimiter=" ",
owner=params.livy2_user,
group=params.livy2_group,
)
# create log4j.properties in etc/conf dir
File(
os.path.join(params.livy2_conf, 'log4j.properties'),
owner=params.livy2_user,
group=params.livy2_group,
content=params.livy2_log4j_properties,
mode=0644,
)
# create spark-blacklist.properties in etc/conf dir
File(
os.path.join(params.livy2_conf, 'spark-blacklist.conf'),
owner=params.livy2_user,
group=params.livy2_group,
content=params.livy2_spark_blacklist_properties,
mode=0644,
)
Directory(
params.livy2_logs_dir,
owner=params.livy2_user,
group=params.livy2_group,
mode=0755,
)
def create_ams_datasource():
import params
server = Server(protocol = params.ams_grafana_protocol.strip(),
host = params.ams_grafana_host.strip(),
port = params.ams_grafana_port,
user = params.ams_grafana_admin_user,
password = params.ams_grafana_admin_pwd)
"""
Create AMS datasource in Grafana, if exsists make sure the collector url is accurate
"""
Logger.info("Trying to find working metric collector")
results = execute_in_parallel(do_ams_collector_post, params.ams_collector_hosts.split(','), params)
new_datasource_host = ""
for host in params.ams_collector_hosts.split(','):
if host in results:
if results[host].status == SUCCESS:
new_datasource_host = host
Logger.info("Found working collector on host %s" % new_datasource_host)
break
else:
Logger.warning(results[host].result)
if new_datasource_host == "":
Logger.warning("All metric collectors are unavailable. Will use random collector as datasource host.")
new_datasource_host = params.metric_collector_host
Logger.info("New datasource host will be %s" % new_datasource_host)
ams_datasource_json = Template('metrics_grafana_datasource.json.j2',
ams_datasource_name=METRICS_GRAFANA_DATASOURCE_NAME, ams_datasource_host=new_datasource_host).get_content()
Logger.info("Checking if AMS Grafana datasource already exists")
response = perform_grafana_get_call(GRAFANA_DATASOURCE_URL, server)
create_datasource = True
if response and response.status == 200:
datasources = response.read()
datasources_json = json.loads(datasources)
for i in xrange(0, len(datasources_json)):
datasource_name = datasources_json[i]["name"]
if datasource_name == METRICS_GRAFANA_DATASOURCE_NAME:
create_datasource = False # datasource already exists
Logger.info("Ambari Metrics Grafana datasource already present. Checking Metrics Collector URL")
datasource_url = datasources_json[i]["url"]
update_datasource = False
if is_unchanged_datasource_url(datasource_url, new_datasource_host):
Logger.info("Metrics Collector URL validation succeeded.")
else:
Logger.info("Metrics Collector URL validation failed.")
update_datasource = True
datasource_type = datasources_json[i]["type"]
new_datasource_def = json.loads(ams_datasource_json)
new_datasource_type = new_datasource_def["type"]
if datasource_type == new_datasource_type:
Logger.info("Grafana datasource type validation succeeded.")
else:
Logger.info("Grafana datasource type validation failed. Old type = %s, New type = %s" % (datasource_type, new_datasource_type))
update_datasource = True
if update_datasource: # Metrics datasource present, but collector host is wrong or the datasource type is outdated.
datasource_id = datasources_json[i]["id"]
Logger.info("Updating datasource, id = %s" % datasource_id)
(response, data) = perform_grafana_put_call(GRAFANA_DATASOURCE_URL, datasource_id,
ams_datasource_json, server)
if response.status == 200:
Logger.info("Ambari Metrics Grafana data source updated.")
elif response.status == 500:
Logger.info("Ambari Metrics Grafana data source update failed. Not retrying.")
raise Fail("Ambari Metrics Grafana data source update failed. PUT request status: %s %s \n%s" %
(response.status, response.reason, data))
else:
raise Fail("Ambari Metrics Grafana data source creation failed. "
"PUT request status: %s %s \n%s" % (response.status, response.reason, data))
pass
pass
pass
else:
Logger.info("Error checking for Ambari Metrics Grafana datasource. Will attempt to create.")
if not create_datasource:
return
else:
Logger.info("Generating datasource:\n%s" % ams_datasource_json)
(response, data) = perform_grafana_post_call(GRAFANA_DATASOURCE_URL, ams_datasource_json, server)
if response.status == 200:
Logger.info("Ambari Metrics Grafana data source created.")
elif response.status == 500:
Logger.info("Ambari Metrics Grafana data source creation failed. Not retrying.")
raise Fail("Ambari Metrics Grafana data source creation failed. POST request status: %s %s \n%s" %
(response.status, response.reason, data))
else:
Logger.info("Ambari Metrics Grafana data source creation failed.")
raise Fail("Ambari Metrics Grafana data source creation failed. POST request status: %s %s \n%s" %
(response.status, response.reason, data))
pass
def service_check_for_single_host(self, metric_collector_host, params):
random_value1 = random.random()
headers = {"Content-type": "application/json"}
ca_certs = os.path.join(params.ams_monitor_conf_dir,
params.metric_truststore_ca_certs)
current_time = int(time.time()) * 1000
metric_json = Template('smoketest_metrics.json.j2',
hostname=params.hostname,
random1=random_value1,
current_time=current_time).get_content()
try:
post_metrics_to_collector(
self.AMS_METRICS_POST_URL, metric_collector_host,
params.metric_collector_port,
params.metric_collector_https_enabled, metric_json, headers,
ca_certs, self.AMS_CONNECT_TRIES, self.AMS_CONNECT_TIMEOUT)
get_metrics_parameters = {
"metricNames": "AMBARI_METRICS.SmokeTest.FakeMetric",
"appId": "amssmoketestfake",
"hostname": params.hostname,
"startTime": current_time - 60000,
"endTime": current_time + 61000,
"precision": "seconds",
"grouped": "false",
}
encoded_get_metrics_parameters = urllib.urlencode(
get_metrics_parameters)
Logger.info(
"Connecting (GET) to %s:%s%s" %
(metric_collector_host, params.metric_collector_port,
self.AMS_METRICS_GET_URL % encoded_get_metrics_parameters))
for i in xrange(0, self.AMS_READ_TRIES):
conn = network.get_http_connection(
metric_collector_host,
int(params.metric_collector_port),
params.metric_collector_https_enabled,
ca_certs,
ssl_version=Script.get_force_https_protocol_value())
conn.request(
"GET",
self.AMS_METRICS_GET_URL % encoded_get_metrics_parameters)
response = conn.getresponse()
Logger.info(
"Http response for host %s : %s %s" %
(metric_collector_host, response.status, response.reason))
data = response.read()
Logger.info("Http data: %s" % data)
conn.close()
if response.status == 200:
Logger.info("Metrics were retrieved from host %s" %
metric_collector_host)
else:
raise Fail(
"Metrics were not retrieved from host %s. GET request status: %s %s \n%s"
% (metric_collector_host, response.status,
response.reason, data))
data_json = json.loads(data)
def floats_eq(f1, f2, delta):
return abs(f1 - f2) < delta
values_are_present = False
for metrics_data in data_json["metrics"]:
if (str(current_time) in metrics_data["metrics"] and
str(current_time + 1000) in metrics_data["metrics"]
and floats_eq(
metrics_data["metrics"][str(current_time)],
random_value1, 0.0000001) and floats_eq(
metrics_data["metrics"][str(current_time +
1000)],
current_time, 1)):
Logger.info(
"Values %s and %s were found in the response from host %s."
% (metric_collector_host, random_value1,
current_time))
values_are_present = True
break
pass
if not values_are_present:
if i < self.AMS_READ_TRIES - 1: #range/xrange returns items from start to end-1
Logger.info(
"Values weren't stored yet. Retrying in %s seconds."
% (self.AMS_READ_TIMEOUT))
time.sleep(self.AMS_READ_TIMEOUT)
else:
raise Fail(
"Values %s and %s were not found in the response."
% (random_value1, current_time))
else:
break
pass
except Fail as ex:
Logger.warning(
"Ambari Metrics service check failed on collector host %s. Reason : %s"
% (metric_collector_host, str(ex)))
raise Fail(
"Ambari Metrics service check failed on collector host %s. Reason : %s"
% (metric_collector_host, str(ex)))
def service_check(self, env):
import params
Logger.info("Ambari Metrics service check was started.")
env.set_params(params)
random_value1 = random.random()
headers = {"Content-type": "application/json"}
for i in xrange(0, self.AMS_CONNECT_TRIES):
try:
current_time = int(time.time()) * 1000
metric_json = Template(
'smoketest_metrics.json.j2',
hostname=params.hostname,
random1=random_value1,
current_time=current_time).get_content()
Logger.info("Generated metrics:\n%s" % metric_json)
Logger.info(
"Connecting (POST) to %s:%s%s" %
(params.metric_collector_host,
params.metric_collector_port, self.AMS_METRICS_POST_URL))
conn = self.get_http_connection(
params.metric_collector_host,
int(params.metric_collector_port),
params.metric_collector_https_enabled)
conn.request("POST", self.AMS_METRICS_POST_URL, metric_json,
headers)
response = conn.getresponse()
Logger.info("Http response: %s %s" %
(response.status, response.reason))
except (httplib.HTTPException, socket.error) as ex:
if i < self.AMS_CONNECT_TRIES - 1: #range/xrange returns items from start to end-1
time.sleep(self.AMS_CONNECT_TIMEOUT)
Logger.info(
"Connection failed. Next retry in %s seconds." %
(self.AMS_CONNECT_TIMEOUT))
continue
else:
raise Fail(
"Metrics were not saved. Service check has failed. "
"\nConnection failed.")
data = response.read()
Logger.info("Http data: %s" % data)
conn.close()
if response.status == 200:
Logger.info("Metrics were saved.")
break
else:
Logger.info(
"Metrics were not saved. Service check has failed.")
if i < self.AMS_CONNECT_TRIES - 1: #range/xrange returns items from start to end-1
time.sleep(self.AMS_CONNECT_TIMEOUT)
Logger.info("Next retry in %s seconds." %
(self.AMS_CONNECT_TIMEOUT))
else:
raise Fail(
"Metrics were not saved. Service check has failed. POST request status: %s %s \n%s"
% (response.status, response.reason, data))
get_metrics_parameters = {
"metricNames": "AMBARI_METRICS.SmokeTest.FakeMetric",
"appId": "amssmoketestfake",
"hostname": params.hostname,
"startTime": current_time - 60000,
"endTime": current_time + 61000,
"precision": "seconds",
"grouped": "false",
}
encoded_get_metrics_parameters = urllib.urlencode(
get_metrics_parameters)
Logger.info(
"Connecting (GET) to %s:%s%s" %
(params.metric_collector_host, params.metric_collector_port,
self.AMS_METRICS_GET_URL % encoded_get_metrics_parameters))
conn = self.get_http_connection(params.metric_collector_host,
int(params.metric_collector_port),
params.metric_collector_https_enabled)
conn.request("GET",
self.AMS_METRICS_GET_URL % encoded_get_metrics_parameters)
response = conn.getresponse()
Logger.info("Http response: %s %s" %
(response.status, response.reason))
data = response.read()
Logger.info("Http data: %s" % data)
conn.close()
if response.status == 200:
Logger.info("Metrics were retrieved.")
else:
Logger.info(
"Metrics were not retrieved. Service check has failed.")
raise Fail(
"Metrics were not retrieved. Service check has failed. GET request status: %s %s \n%s"
% (response.status, response.reason, data))
data_json = json.loads(data)
def floats_eq(f1, f2, delta):
return abs(f1 - f2) < delta
for metrics_data in data_json["metrics"]:
if (str(current_time) in metrics_data["metrics"]
and str(current_time + 1000) in metrics_data["metrics"]
and floats_eq(metrics_data["metrics"][str(current_time)],
random_value1, 0.0000001)
and floats_eq(
metrics_data["metrics"][str(current_time + 1000)],
current_time, 1)):
Logger.info("Values %s and %s were found in the response." %
(random_value1, current_time))
break
pass
else:
Logger.info("Values %s and %s were not found in the response." %
(random_value1, current_time))
raise Fail("Values %s and %s were not found in the response." %
(random_value1, current_time))
Logger.info("Ambari Metrics service check is finished.")
def create_ams_datasource():
import params
GRAFANA_CONNECT_TRIES = 5
GRAFANA_CONNECT_TIMEOUT = 15
GRAFANA_URL = "/api/datasources"
METRICS_GRAFANA_DATASOURCE_NAME = "AMBARI_METRICS"
headers = {"Content-type": "application/json"}
Logger.info("Checking if AMS Grafana datasource already exists")
Logger.info("Connecting (GET) to %s:%s%s" %
(params.hostname, params.ams_grafana_port, GRAFANA_URL))
conn = httplib.HTTPConnection(params.hostname,
int(params.ams_grafana_port))
conn.request("GET", GRAFANA_URL)
response = conn.getresponse()
Logger.info("Http response: %s %s" % (response.status, response.reason))
if (response.status == 200):
datasources = response.read()
datasources_json = json.loads(datasources)
for i in xrange(0, len(datasources_json)):
datasource_name = datasources_json[i]["name"]
if (datasource_name == METRICS_GRAFANA_DATASOURCE_NAME):
Logger.info(
"Ambari Metrics Grafana datasource already present. Checking Metrics Collector URL"
)
datasource_url = datasources_json[i]["url"]
if datasource_url == (params.ams_grafana_protocol + "://" +
params.metric_collector_host + ":" +
params.metric_collector_port):
Logger.info(
"Metrics Collector URL validation succeeded. Skipping datasource creation"
)
GRAFANA_CONNECT_TRIES = 0 # No need to create datasource again
else: # Metrics datasource present, but collector host is wrong.
Logger.info("Metrics Collector URL validation failed.")
datasource_id = datasources_json[i]["id"]
Logger.info("Deleting obselete Metrics datasource.")
conn = httplib.HTTPConnection(params.hostname,
int(params.ams_grafana_port))
conn.request("DELETE",
GRAFANA_URL + "/" + str(datasource_id))
response = conn.getresponse()
Logger.info("Http response: %s %s" %
(response.status, response.reason))
break
else:
Logger.info(
"Error checking for Ambari Metrics Grafana datasource. Will attempt to create."
)
if GRAFANA_CONNECT_TRIES > 0:
Logger.info("Attempting to create Ambari Metrics Grafana datasource")
for i in xrange(0, GRAFANA_CONNECT_TRIES):
try:
ams_datasource_json = Template(
'metrics_grafana_datasource.json.j2',
ams_datasource_name=METRICS_GRAFANA_DATASOURCE_NAME,
ams_grafana_protocol=params.ams_grafana_protocol,
ams_collector_host=params.metric_collector_host,
ams_collector_port=params.metric_collector_port).get_content()
Logger.info("Generated datasource:\n%s" % ams_datasource_json)
Logger.info(
"Connecting (POST) to %s:%s%s" %
(params.hostname, params.ams_grafana_port, GRAFANA_URL))
conn = httplib.HTTPConnection(params.hostname,
int(params.ams_grafana_port))
conn.request("POST", GRAFANA_URL, ams_datasource_json, headers)
response = conn.getresponse()
Logger.info("Http response: %s %s" %
(response.status, response.reason))
except (httplib.HTTPException, socket.error) as ex:
if i < GRAFANA_CONNECT_TRIES - 1:
time.sleep(GRAFANA_CONNECT_TIMEOUT)
Logger.info(
"Connection to Grafana failed. Next retry in %s seconds." %
(GRAFANA_CONNECT_TIMEOUT))
continue
else:
raise Fail("Ambari Metrics Grafana datasource not created")
data = response.read()
Logger.info("Http data: %s" % data)
conn.close()
if response.status == 200:
Logger.info("Ambari Metrics Grafana data source created.")
break
elif response.status == 500:
Logger.info(
"Ambari Metrics Grafana data source creation failed. Not retrying."
)
raise Fail(
"Ambari Metrics Grafana data source creation failed. POST request status: %s %s \n%s"
% (response.status, response.reason, data))
else:
Logger.info("Ambari Metrics Grafana data source creation failed.")
if i < GRAFANA_CONNECT_TRIES - 1:
time.sleep(GRAFANA_CONNECT_TIMEOUT)
Logger.info("Next retry in %s seconds." %
(GRAFANA_CONNECT_TIMEOUT))
else:
raise Fail(
"Ambari Metrics Grafana data source creation failed. POST request status: %s %s \n%s"
% (response.status, response.reason, data))