def es_all_stop(request):
obj = helper(request, Elastic_search)
cursor = connection.cursor()
nodes_sql = "select e.ip,n.port from elastic_search_elastic_search as e join administer_nodes as n on " \
"e.ip=n.ip where n.node_cluster_id='" + obj.cluster_id + "'"
url_list = []
cursor.execute(nodes_sql)
active_nodes_without_client = cursor.fetchall()
colnames = [desc[0] for desc in cursor.description]
for node in active_nodes_without_client:
c = dict(zip(colnames, node))
url_list.append('http://%s:%s/es/stop/' % (c["ip"], c["port"]))
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
future_to_url = {
executor.submit(
requests.post(url,
headers={"API-KEY": helper.get_api_key()},
data=json.dumps(
{"cluster_id": int(obj.cluster_id)})), url,
10): url
for url in url_list
}
for future in concurrent.futures.as_completed(future_to_url):
future_to_url[future]
try:
future.result()
except Exception as e:
print(e)
data = {'success': 1, 'msg': ["successfully restarted all ES nodes"]}
return JsonResponse(data)
def hb_all_restart(request):
if request.POST['node_ip'] is not '':
node_ip = request.POST['node_ip']
else:
data = {'success': False,
'msg': "We are unable to get the IP of the active namenode. Please hard refresh the page and try again"}
return JsonResponse(data)
obj = helper(request=request, model=Hbase)
op_status = obj.restart_all("hbase/restart/", master_ip=node_ip)
print(op_status)
if op_status["success"]:
Restart_after_configuration.objects.filter(service_id=obj.get_service_id("hbase")).update(status=0)
return JsonResponse(op_status)
def edit_configure_service(request, service):
context = context_processors.base_variables_all(request)
service_object = Services.objects.get(name=service)
nodes_configuration = helper(request).get_all_nodes()
key_configurations = User_preferred_configuration.objects.filter(
service_id=service_object.id)
context["key_configurations"] = key_configurations
context["id"] = service_object.id
context["service_name"] = service
context["nodes_configuration"] = nodes_configuration
return render(request, 'configuraion/edit_configuration.html', context)
def h_hm_restart(request):
obj = helper(request=request, model=Hbase)
if request.POST['node_ip'] is not '':
node_ip = request.POST['node_ip']
else:
data = {'success': 0,
'msg': "We are unable to get the IP of the active hmaster. Please hard refresh the page and try again"}
return JsonResponse(data)
try:
node = obj.get_node_data(node_ip)
# node_ip = request.POST['node_ip']
url = 'http://%s:%s/hbase/master/restart/' % (node_ip, node["port"])
return JsonResponse(obj.restart_service(url))
except Exception as e:
data = {'success': False, 'msg': e}
return JsonResponse(data)
def add_configure_service(request, service):
service_object = Services.objects.get(name=service)
key_configurations_users = User_preferred_configuration.objects.filter(
service_id=service_object.id)
key_configurations = Default_configuration.objects.exclude(
name__in=[x.key_name for x in key_configurations_users]).filter(
service_id=service_object.id)
nodes_configuration = helper(request).get_all_nodes()
context = context_processors.base_variables_all(request)
context["key_configurations"] = key_configurations
context["id"] = service_object.id
context["service_name"] = service
context["nodes_configuration"] = nodes_configuration
return render(request, 'configuraion/configure_service.html', context)
def add_configure_nodes_save(request):
nodes = helper(request).get_all_nodes()
service_id = request.POST['service_id']
for node in nodes:
configuration = {}
row_user_types = User_preferred_configuration.objects.order_by().values('key_type').distinct() \
.filter(value__contains=node, service_id=service_id)
for row_user_type in row_user_types:
configuration_inside = {}
row_users = User_preferred_configuration.objects.filter(
key_type=row_user_type['key_type'], value__contains=node)
for row_user in row_users:
value = ast.literal_eval(row_user.value)
configuration_inside[row_user.key_name] = value[str(node)]
configuration[row_user_type['key_type']] = configuration_inside
url = "http://%s:11605/config/" % node
response = requests.post(url,
data=json.dumps(configuration),
headers={"API-KEY": helper.get_api_key()})
response_dict = json.loads(response.content.decode())
if response_dict["success"] == 0:
data = {'success': False}
return JsonResponse(data)
data = {'success': True}
restart_service_check = Restart_after_configuration.objects.filter(
service_id=service_id).exists()
if restart_service_check:
restart_service = Restart_after_configuration.objects.get(
service_id=service_id)
restart_service.status = 1
restart_service.save()
else:
restart_service = Restart_after_configuration(service_id=service_id,
status=1)
restart_service.save()
return JsonResponse(data)
def h_hm_stop(request):
obj = helper(request=request, model=Hbase)
if request.POST['node_ip'] is not '':
node_ip = request.POST['node_ip']
else:
data = {'success': 0,
'msg': "We are unable to get the IP of the active namenode. Please hard refresh the page and try again"}
return JsonResponse(data)
try:
node = obj.get_node_data(node_ip)
url_stop = 'http://%s:%s/hbase/master/stop/' % (node_ip, node["port"])
resp = obj.stop_service(url_stop)
print(resp)
return JsonResponse(resp)
except Exception as e:
data = {'success': False, 'msg': e}
return JsonResponse(data)
def sw_restart(request):
obj = helper(request=request, model=Spark)
if request.POST['node_ip'] is not '':
node_ip = request.POST['node_ip']
else:
data = {
'success':
0,
'msg':
"We are unable to get the IP of the active namenode. Please hard refresh the page and try again"
}
return JsonResponse(data)
try:
node = obj.get_node_data(node_ip)
url = 'http://%s:%s/spark/slave/restart/' % (node_ip, node["port"])
return JsonResponse(obj.restart_special_service(url))
except Exception as e:
data = {'success': False, 'msg': e}
return JsonResponse(data)
def es_restart(request, node_ip=None):
obj = helper(request)
if request.POST['node_ip'] is not '':
node_ip = request.POST['node_ip']
else:
data = {
'success':
0,
'msg':
"We are unable to get the IP of the active namenode. Please hard refresh the page and try again"
}
return JsonResponse(data)
try:
node = obj.get_node_data(node_ip)
url = 'http://%s:%s/es/restart/' % (node_ip, node["port"])
data = obj.restart_service(url)
print(data)
return JsonResponse(data)
except Exception as e:
data = {'success': False, 'msg': e}
return JsonResponse(data)
def s_all_stop(request):
obj = helper(request=request, model=Spark)
return JsonResponse(obj.stop_all("spark/stop/"))
def index(request):
obj = helper(request, Spark)
master_ip = ""
client = True
master = ""
context = context_processors.base_variables_all(request)
if obj.atleast_one_client_is_installed():
if obj.clientIsInstalledOnMaster():
master = obj.get_active_master()
if master:
master_ip = master["ip"]
context["client"] = client
else:
messages.error(
request,
"Sorry !! due to some problem we are unable to fetch the information from server."
" You can perform following steps to find the problem and then restart the services."
"<ul>"
"<li> Reload after 10 seconds</li>"
"<li> Restart again</li>"
"<li> Check the log of spark master and spark worker</li>"
"<li> make there is no problem in configuration file </li> "
)
context["error_in_conf_file"] = True
s_master = obj.get_service_master()
if s_master:
context["master_ip"] = s_master["ip"]
context["client"] = client
return render(request, 'spark/spark.html', context)
else:
messages.error(
request, "We have encountered some problems."
"Please make sure following conditions are met"
"<ul>"
"<li> Client is installed on master node</li>"
"<li> Environment variables for all services are set properly</li>"
"<li> Restart agent on master node [url here]</li>")
context["client"] = False
return render(request, 'spark/spark.html', context)
else:
messages.error(request, "Seems like no client is installed")
context["client"] = False
return render(request, 'spark/spark.html', context)
all_nodes = obj.get_all_nodes()
cursor = connection.cursor()
node_with_client = "select s.ip from spark_spark as s join administer_nodes " \
"as n on s.ip=n.ip"
masters_sql = "select s.*,n.hostname,n.fqdn,n.name from spark_spark as s join administer_nodes " \
"as n on s.ip=n.ip where s.type=1"
slave_with_client = "select s.*,sm.*,n.hostname,n.fqdn,n.name from spark_spark as s join administer_nodes as n on " \
"s.ip=n.ip join spark_metrics as sm on s.id=sm.node_id " \
" where s.type=0 and sm.updated_at in (select max(updated_at) " \
"from spark_metrics limit 1)"
slave_without_client = "select s.*,sm.* from spark_spark as s join spark_metrics as sm on s.id=sm.node_id " \
"where s.type=0 and s.ip not in (" + node_with_client + ") and sm.updated_at in" \
" (select max(updated_at) from spark_metrics limit 1)"
alive_workers_list = []
dead_workers_list = []
cursor.execute(masters_sql)
masters = cursor.fetchall()
colnames = [desc[0] for desc in cursor.description]
for node in masters:
c = dict(zip(colnames, node))
client_installed = True
if c["ip"] not in all_nodes:
client_installed = False
if c["type"] == 1 and c["state"] == 1:
master = c
if c["type"] == 1 == 1 and c["state"] == 0:
c["client_installed"] = client_installed
standby_data = c
cursor.execute(slave_with_client)
nodes_with_client = cursor.fetchall()
colnames = [desc[0] for desc in cursor.description]
for node in nodes_with_client:
c = dict(zip(colnames, node))
client_installed = True
if c["ip"] not in all_nodes:
client_installed = False
c["client_installed"] = client_installed
if c["status"] == "RUNNING":
alive_workers_list.append(c)
else:
dead_workers_list.append(c)
cursor.execute(slave_without_client)
nodes_without_client = cursor.fetchall()
for node in nodes_without_client:
c = dict(zip(colnames, node))
client_installed = True
if c["ip"] not in all_nodes:
client_installed = False
c["client_installed"] = client_installed
if c["status"] == "RUNNING":
alive_workers_list.append(c)
else:
dead_workers_list.append(c)
service_object = Services.objects.get(name='spark')
restart_status_checks = Restart_after_configuration.objects.filter(
service_id=service_object.id).exists()
if restart_status_checks:
restart_status_check = Restart_after_configuration.objects.get(
service_id=service_object.id)
restart_status = restart_status_check.status
else:
restart_status = 0
context["alive_spark_workers"] = alive_workers_list
context["dead_spark_workers"] = dead_workers_list
context["restart_status"] = restart_status
context["spark_master"] = master
context["master_ip"] = master_ip
context["service_id"] = service_object.id
return render(request, 'spark/spark.html', context)
def h_kill(request):
obj = helper(request=request, model=Hbase)
if request.POST['node_ip'] is not '':
node_ip = request.POST['node_ip']
else:
data = {'success': 0,
'msg': "We are unable to get the IP of the active namenode. Please hard refresh the page and try again"}
return JsonResponse(data)
node = obj.get_node_data(node_ip)
if request.POST['server_type'] is not '':
server_type = request.POST['server_type']
if server_type == 'HMaster':
url_start = 'http://%s:%s/hbase/master/start/' % (node_ip, node["port"])
else:
url_start = 'http://%s:%s/hbase/regionserver/start/' % (node_ip, node["port"])
else:
data = {'success': 0,
'msg': "Sorry we didnot receive some of the required data. Please hard refresh the page and try again"}
return JsonResponse(data)
if request.POST['node_id'] is not '':
node_id = request.POST['node_id']
else:
data = {'success': 0,
'msg': "We are unable to get the IP of the active namenode. Please hard refresh the page and try again"}
return JsonResponse(data)
if request.POST['action_type'] is not '':
action_type = request.POST['action_type']
print(" 208 action type = %s " % action_type)
else:
data = {'success': 0,
'msg': "Sorry we did not receive some of the required data. Please hard refresh the page and try again"}
return JsonResponse(data)
url = "http://%s:%s/command/kill/" % (node_ip, node["port"])
try:
payload = {"service_name": server_type, "node_id": node_id, "table_name": "hbase_hbase"}
r = requests.post(url, headers={"API-KEY": helper.get_api_key()},
data=json.dumps(payload))
if r.status_code != 200:
return JsonResponse({'success': 0, 'msg': 'server threw status code ' + r.status_code})
data = ast.literal_eval(r.content.decode())
# data = r.json()
if data["success"] == 1:
if action_type == "1":
try:
r_start = requests.post(url_start, headers={"API-KEY": helper.get_api_key()},
data=json.dumps({"cluster_id": int(obj.cluster_id)}))
print(r_start.status_code)
if r_start.status_code != 200:
return JsonResponse(r_start.json())
if r_start.json()['success'] != 1:
return JsonResponse(r_start.json())
except Exception as e:
data = {'success': 0, 'msg': e.args}
return JsonResponse(data)
else:
return JsonResponse(data, False)
except ConnectionError as e:
data = {'success': 0, 'msg': e}
return JsonResponse(data)
def hb_all_stop(request):
obj = helper(request=request, model=Hbase)
return JsonResponse(obj.stop_all("hbase/stop/"))
def index(request):
obj = helper(request, Hbase)
master_ip = ""
client = True
context = context_processors.base_variables_all(request)
if obj.atleast_one_client_is_installed():
if obj.clientIsInstalledOnMaster():
master = obj.get_active_master()
if master:
master_ip = master["ip"]
context["master_ip"] = master_ip
context["master_id"] = master["id"]
context["client"] = client
else:
messages.error(request,
"Sorry !! due to some problem we are unable to fetch the information from server."
" You can perform following steps to find the problem and then restart the services."
"<ul>"
"<li> Reload after 10 seconds</li>"
"<li> Restart again</li>"
"<li> Check the log of Hmaster and RegionServers</li>"
"<li> Check the log of Namenode and Datanode</li>"
"<li> make there is no problem in configuration file </li> "
"</ul>")
service_master = obj.get_service_master()
if service_master:
context["error_in_conf_file"] = True
context["master_ip"] = service_master["ip"]
context["master_id"] = service_master["id"]
context["client"] = client
return render(request, 'hbase/hbase.html', context)
else:
messages.error(request, "We have encountered some problems."
"Please make sure following conditions are met"
"<ul>"
"<li> Client is installed on master node</li>"
"<li> Environment variables for all services are set properly</li>"
"<li> Restart agent on master node [url here]</li>")
context["client"] = False
return render(request, 'hbase/hbase.html', context)
else:
messages.error(request, "Seems like no client is installed")
context["client"] = False
return render(request, 'hbase/hbase.html', context)
all_nodes = obj.get_all_nodes()
cursor = connection.cursor()
node_with_client = "select hb.ip from hbase_hbase as hb join administer_nodes " \
"as n on hb.ip=n.ip"
masters_sql = "select hb.*,n.hostname,n.fqdn,n.name from hbase_hbase as hb join administer_nodes " \
"as n on hb.ip=n.ip where hb.type=1"
slave_with_client = "select h.*,hbm.*,n.hostname,n.fqdn,n.name from hbase_hbase as h join administer_nodes as n on " \
"h.ip=n.ip join hbase_metrics as hbm on h.id=hbm.node_id " \
"where h.type=0 and hbm.updated_at in (select max(updated_at) " \
"from hbase_metrics limit 1)"
slave_without_client = "select h.*,hbm.* from hbase_hbase as h join hbase_metrics as hbm on h.id=hbm.node_id " \
"where h.type=0 and h.ip not in (" + node_with_client + ") and hbm.updated_at in (select max(updated_at) from hbase_metrics limit 1)"
live_regionservers_list = []
dead_regionservers_list = []
active_data = ""
backup_data = ""
cursor.execute(masters_sql)
masters = cursor.fetchall()
colnames = [desc[0] for desc in cursor.description]
for node in masters:
print(node)
c = dict(zip(colnames, node))
client_installed = True
if c["ip"] not in all_nodes:
client_installed = False
if c["type"] == 1 and c["state"] == 1:
active_data = c
if c["type"] == 1 and c["state"] == 0:
c["client_installed"] = client_installed
backup_data = c
cursor.execute(slave_with_client)
nodes_with_client = cursor.fetchall()
colnames = [desc[0] for desc in cursor.description]
for node in nodes_with_client:
c = dict(zip(colnames, node))
client_installed = True
if c["ip"] not in all_nodes:
client_installed = False
c["client_installed"] = client_installed
if c["status"] == "RUNNING":
live_regionservers_list.append(c)
else:
dead_regionservers_list.append(c)
cursor.execute(slave_without_client)
nodes_without_client = cursor.fetchall()
for node in nodes_without_client:
c = dict(zip(colnames, node))
client_installed = True
if c["ip"] not in all_nodes:
client_installed = False
c["client_installed"] = client_installed
if c["status"] == "RUNNING":
live_regionservers_list.append(c)
else:
dead_regionservers_list.append(c)
tst = {"k1": "v1", "k2": "v2", "k3": "v3"}
# tst = 'bibek'
service_object = Services.objects.get(name='hbase')
restart_status_checks = Restart_after_configuration.objects.filter(service_id=service_object.id).exists()
if restart_status_checks:
restart_status_check = Restart_after_configuration.objects.get(service_id=service_object.id)
restart_status = restart_status_check.status
else:
restart_status = 0
context["live_regionservers"] = live_regionservers_list
context["dead_regionservers"] = dead_regionservers_list
context["active_master"] = active_data
context["backup_master"] = backup_data
context["restart_status"] = restart_status
context["service_id"] = service_object.id
return render(request, 'hbase/hbase.html', context)
def index(request):
obj = helper(request, Elastic_search)
client = True
context = context_processors.base_variables_all(request)
if obj.atleast_one_client_is_installed():
if obj.clientIsInstalledOnMaster():
master = obj.get_active_master()
if master:
master_ip = master["ip"]
port = master["web_port"]
context["master_ip"] = master_ip
context["client"] = client
else:
messages.error(
request,
"Sorry !! due to some problem we are unable to fetch the information from server."
" You can perform following steps to find the problem and then restart the services."
"<ul>"
"<li> Reload after 10 seconds</li>"
"<li> Restart again</li>"
"<li> Check the log of master node</li>"
"<li> make there is no problem in configuration file </li> "
)
s_master = obj.get_service_master()
if s_master:
context["error_in_conf_file"] = True
context["master_ip"] = s_master["ip"]
context["client"] = client
return render(request, 'elasticsearch/elasticsearch.html',
context)
else:
messages.error(
request, "We have encountered some problems."
"Please make sure following conditions are met"
"<ul>"
"<li> Client is installed on master node</li>"
"<li> Environment variables for all services are set properly</li>"
"<li> Restart agent on master node [url here]</li>")
context["client"] = False
return render(request, 'elasticsearch/elasticsearch.html', context)
else:
messages.error(request, "Seems like no client is installed")
context["client"] = False
return render(request, 'elasticsearch/elasticsearch.html', context)
all_nodes = obj.get_all_nodes()
cursor = connection.cursor()
node_with_client = "select e.ip from elastic_search_elastic_search as e join administer_nodes " \
"as n on e.ip=n.ip"
masters_sql = "select e.*,n.hostname,n.fqdn,n.name from elastic_search_elastic_search as e join administer_nodes " \
"as n on e.ip=n.ip where e.type=1"
active_slave_with_client = "select e.*,em.*,n.hostname,n.fqdn,n.name from elastic_search_elastic_search as e join administer_nodes as n on " \
"e.ip=n.ip join elastic_search_metrics as em on e.id=em.node_id " \
"where e.type=0 and em.updated_at in (select max(updated_at) from elastic_search_metrics limit 1)"
inactive_slave_with_client = "select e.*,n.hostname,n.fqdn,n.name from elastic_search_elastic_search as e join administer_nodes as n on " \
"e.ip=n.ip where e.type=0"
active_slave_without_client = "select e.*,em.* from elastic_search_elastic_search as e join elastic_search_metrics " \
"as em on e.id=em.node_id where e.ip not in (" + node_with_client + ") " \
"and e.type=0 and e.status='RUNNING' and em.updated_at in (select max(updated_at) from elastic_search_metrics limit 1)"
inactive_slave_without_client = "select * from elastic_search_elastic_search where ip not in (" + node_with_client + ") " \
"and type=0 and status='SHUTDOWN'"
backup_masters_list = []
alive_datanode_list = []
dead_datanode_list = []
cursor.execute(masters_sql)
masters = cursor.fetchall()
colnames = [desc[0] for desc in cursor.description]
for node in masters:
c = dict(zip(colnames, node))
client_installed = True
if c["ip"] not in all_nodes:
client_installed = False
if c["type"] == 1 and c["state"] == 1 and c["status"] == "RUNNING":
master = c
if c["type"] == 1 and c["state"] == 0:
c["client_installed"] = client_installed
backup_masters_list.append(c)
cursor.execute(active_slave_with_client)
nodes_with_client = cursor.fetchall()
colnames = [desc[0] for desc in cursor.description]
for node in nodes_with_client:
print(node)
c = dict(zip(colnames, node))
client_installed = True
if c["ip"] not in all_nodes:
client_installed = False
c["client_installed"] = client_installed
alive_datanode_list.append(c)
# cursor.execute(inactive_slave_with_client)
# inactive_nodes_with_client = cursor.fetchall()
# colnames = [desc[0] for desc in cursor.description]
# for node in inactive_nodes_with_client:
# c = dict(zip(colnames, node))
# client_installed = True
# if c["ip"] not in all_nodes:
# client_installed = False
# c["client_installed"] = client_installed
# dead_datanode_list.append(c)
cursor.execute(active_slave_without_client)
active_nodes_without_client = cursor.fetchall()
colnames = [desc[0] for desc in cursor.description]
for node in active_nodes_without_client:
# print(node)
c = dict(zip(colnames, node))
client_installed = True
if c["ip"] not in all_nodes:
client_installed = False
c["client_installed"] = client_installed
alive_datanode_list.append(c)
# cursor.execute(inactive_slave_without_client)
# inactive_nodes_without_client = cursor.fetchall()
# colnames = [desc[0] for desc in cursor.description]
# for node in inactive_nodes_without_client:
# c = dict(zip(colnames, node))
# client_installed = True
# if c["ip"] not in all_nodes:
# client_installed = False
#
# c["client_installed"] = client_installed
# dead_datanode_list.append(c)
service_object = Services.objects.get(name='elasticsearch')
restart_status_checks = Restart_after_configuration.objects.filter(
service_id=service_object.id).exists()
if restart_status_checks:
restart_status_check = Restart_after_configuration.objects.get(
service_id=service_object.id)
restart_status = restart_status_check.status
else:
restart_status = 0
context["master"] = master
context["backup_masters"] = backup_masters_list
context["alive_datanode"] = alive_datanode_list
context["dead_datanode"] = dead_datanode_list
context["restart_status"] = restart_status
context["service_id"] = service_object.id
return render(request, 'elasticsearch/elasticsearch.html', context)