def back_with_ajax(request):
obj = helper.helper(request, Hdfs)
node_id = request.GET['node_id']
directory = request.GET['directory_name']
directory_list = directory.split('/')
directory_list.pop(-1)
directory_list.pop(-1)
directory_str = "/".join(directory_list)
if directory_str == '':
directory = '/'
else:
directory = directory_str
node = Nodes.objects.get(id=node_id)
url = "http://%s:%s/command/ls/" % (node.ip, node.port)
data = {}
data["path"] = directory
headers = {"API-KEY": obj.get_api_key()}
response = requests.post(url, data=json.dumps(data), headers=headers)
data = response.json()
data['back_directory'] = data["path"]
return JsonResponse(data)
def extract_files_local(request):
obj = helper.helper(request, Hdfs)
file_name = request.POST['file_name']
root_folder = request.POST['root_folder']
override = request.POST['override']
node_id = request.POST['node_id']
node = Nodes.objects.get(id=node_id)
if (root_folder == ''):
root_folder = 'user'
else:
root_folder = root_folder + '/'
to_extract = {}
to_extract["file_name"] = file_name
to_extract["root_folder"] = root_folder
to_extract["override"] = override
url = "http://%s:%s/command/extract/" % (node.ip, node.port)
headers = {"API-KEY": obj.get_api_key()}
response = requests.post(url, data=json.dumps(to_extract), headers=headers)
response_json = response.json()
if response_json['success'] is 0:
data = {'success': 0, 'msg': response_json['msg']}
else:
data = {
'success': 1,
}
return JsonResponse(data)
def back_eachnode_with_ajax(request):
obj = helper.helper(request, Hdfs)
node_ip = request.POST['node_ip']
node_port = Nodes.objects.filter(ip=node_ip).values("port").first()["port"]
directory = request.POST['directory_name']
directory_list = directory.split('/')
directory_list.pop(-1)
directory_list.pop(-1)
directory_str = "/".join(directory_list)
if directory_str == '':
directory = '/'
else:
directory = directory_str
url = "http://%s:%s/command/ls/" % (node_ip, node_port)
data = {}
data["path"] = directory
headers = {"API-KEY": obj.get_api_key()}
response = requests.post(url, data=json.dumps(data), headers=headers)
data = response.json()
data['back_directory'] = directory_str
return JsonResponse(data)
def files_to_local_ajax(request):
obj = helper.helper(request, Hdfs)
file_name = request.POST['file_name']
root_folder = request.POST['root_folder']
destination_folder = request.POST['destination_folder']
node_ip = request.POST['node_ip']
node_port = Nodes.objects.filter(ip=node_ip).values("port").first()["port"]
nfs = request.POST['nfs']
if (root_folder == ''):
root_folder = '/'
else:
root_folder = root_folder + '/'
if (destination_folder == ''):
destination_folder = '/'
to_local = {}
to_local['file_name'] = file_name
to_local['root_folder'] = root_folder
to_local['destination'] = destination_folder
to_local['nfs'] = nfs
url = "http://%s:%s/hdfs/get/" % (node_ip, node_port)
headers = {"API-KEY": obj.get_api_key()}
response = requests.post(url, data=json.dumps(to_local), headers=headers)
response_json = response.json()
if response_json['success'] is 0:
data = {'success': 0, 'msg': response_json['msg']}
else:
data = {
'success': 1,
}
return JsonResponse(data)
def tail_file(request):
obj = helper.helper(request)
file_name = request.POST['file_name']
source = request.POST['source'] + "/" + file_name
node_id = request.POST['node_id']
node = Nodes.objects.get(id=node_id)
data_copy = {}
data_copy['file_path'] = source
url = "http://%s:%s/command/tail/" % (node.ip, node.port)
headers = {"API-KEY": obj.get_api_key()}
response = requests.post(url, data=json.dumps(data_copy), headers=headers)
response_json = None
try:
response_json = json.loads(response.content)
except TypeError:
response_json = response.json()
except Exception as e:
print(e)
# response_json = json.loads(response.content)
if response_json['success'] is 0:
data = {'success': 0, 'msg': response_json['msg']}
else:
result = response_json['tail_result'].replace('\t', ' ').split('\n')
data = {'success': 1, 'result': result}
return JsonResponse(data)
def browse_hdfs_ajax(request):
directory = request.GET['directory_name']
client = True
context = context_processors.base_variables_all(request)
if directory == "":
directory = "/"
obj = helper.helper(request, Hdfs)
master = obj.get_active_master()
if master:
master_ip = master["ip"]
master_port = master["web_port"]
url = "https://%s:%s/webhdfs/v1%s?op=LISTSTATUS" % (
master_ip, master_port, directory)
r = requests.get(url, verify=False)
response_json = r.json()
data = response_json['FileStatuses']
return JsonResponse(data)
else:
data = ''
messages.error(request, "client is not installed on the master node ")
client = False
context["client"] = client
return JsonResponse(data)
def files_to_hdfs_ajax(request):
obj = helper.helper(request, Hdfs)
file_name = request.POST['file_name']
root_folder = request.POST['root_folder']
destination_folder = request.POST['destination_folder']
node_id = request.POST['node_id']
node = Nodes.objects.get(id=node_id)
if (root_folder == ''):
root_folder = '/'
else:
root_folder = root_folder + '/'
if (destination_folder == ''):
destination_folder = '/'
to_hdfs = {}
to_hdfs['file_name'] = file_name
to_hdfs['root_folder'] = root_folder
to_hdfs['destination'] = destination_folder
url = "http://%s:%s/hdfs/write/" % (node.ip, node.port)
headers = {"API-KEY": obj.get_api_key()}
response = requests.post(url, data=json.dumps(to_hdfs), headers=headers)
response_json = response.json()
if response_json['success'] is 0:
data = {'success': 0, 'msg': response_json['msg']}
else:
data = {
'success': 1,
}
return JsonResponse(data)
def browse_hdfs(request):
if 'user' not in request.session:
return redirect('login')
obj = helper.helper(request, Hdfs)
master_ip = ""
client = True
master = ""
context = context_processors.base_variables_all(request)
if obj.atleast_one_client_is_installed():
if obj.clientIsInstalledOnMaster():
master = obj.get_active_master()
if master:
master_ip = master["ip"]
master_port = master["web_port"]
context["client"] = client
else:
messages.error(
request,
"Sorry there is some problem in your configuration file <h3>namenode is down</h3> "
)
context["error_in_conf_file"] = True
return render(request, 'browse/browse_hdfs.html', context)
else:
messages.error(request, "Client is not installed on master node")
context["client"] = False
return render(request, 'browse/browse_hdfs.html', context)
else:
messages.error(request, "Seems like no client is installed")
return render(request, 'browse/browse_hdfs.html', context)
cluster_id = str(request.session[str(request.session['user'])])
nodes_hdfs = Nodes.objects.all()
try:
url = "https://%s:%s/webhdfs/v1/?op=LISTSTATUS" % (master_ip,
master_port)
r = requests.get(url, verify=False)
response_json = r.json()
rows = response_json['FileStatuses']['FileStatus']
except ConnectionError as e:
messages.error(request, "There is some problem with the connection.Please make sure your internet connection" \
" is working and client is up and running on " + master_ip)
return render(request, "error/500.html", context)
context["rows"] = rows
context["nodes_hdfs"] = nodes_hdfs
context["master_ip"] = master_ip
return render(request, 'browse/browse_hdfs.html', context)
def h_all_restart(request):
if request.POST['node_ip'] is not '':
node_ip = request.POST['node_ip']
else:
data = {'success': 0,
'msg': "We are unable to get the IP of the active namenode. Please hard refresh the page and try again"}
return JsonResponse(data)
obj = helper(request=request, model=Hdfs)
op_status = obj.restart_all("hadoop/dfs/restart/", master_ip=node_ip)
if op_status["success"]:
Restart_after_configuration.objects.filter(service_id=obj.get_service_id("hdfs")).update(status=0)
return JsonResponse(op_status)
def dn_stop(request):
obj = helper(request)
if request.POST['node_ip'] is not '':
node_ip = request.POST['node_ip']
else:
data = {'success': 0,
'msg': "We are unable to get the IP of the datanode. Please hard refresh the page and try again"}
return JsonResponse(data)
try:
node = obj.get_node_data(node_ip)
url_stop = 'http://%s:%s/hadoop/datanode/stop/' % (node_ip,node["port"])
return JsonResponse(obj.stop_service(url_stop))
except Exception as e:
data = {'success': 0, 'msg': e}
return JsonResponse(data, safe=False)
def nn_restart(request):
obj = helper(request)
if request.POST['node_ip'] is not '':
node_ip = request.POST['node_ip']
else:
data = {'success': 0,
'msg': "We are unable to get the IP of the active namenode. Please hard refresh the page and try again"}
return JsonResponse(data)
try:
node = obj.get_node_data(node_ip)
url = 'http://%s:%s/hadoop/namenode/restart/' % (node_ip,node["port"])
data = obj.restart_service(url)
return JsonResponse(data)
except Exception as e:
data = {'success': 0, 'msg': e.args}
return JsonResponse(data)
def browse_eachnode_directory_ajax(request):
obj = helper.helper(request, Hdfs)
node_ip = request.POST['node_ip']
node_port = Nodes.objects.filter(ip=node_ip).values("port").first()["port"]
directory = request.POST['directory_name']
if directory == '':
directory = 'user'
url = "http://%s:%s/command/ls/" % (node_ip, node_port)
data = {}
data["path"] = directory
headers = {"API-KEY": obj.get_api_key()}
response = requests.post(url, data=json.dumps(data), headers=headers)
data = response.json()
return JsonResponse(data)
def copy_on_nodes(request):
if 'user' not in request.session:
return redirect('login')
obj = helper.helper(request)
source = request.POST['source']
if source == "":
source = "/"
destination = request.POST['destination']
if destination == "":
destination = "/"
files = request.POST['files']
node_id = request.POST['node_id']
node = Nodes.objects.get(id=node_id)
url = "http://%s:%s/command/copy/" % (node.ip, node.port)
data_copy = {}
data_copy['source'] = source
data_copy['file_list'] = files
data_copy['destination'] = destination
headers = {"API-KEY": obj.get_api_key()}
response = requests.post(url, data=json.dumps(data_copy), headers=headers)
response_json = None
try:
response_json = json.loads(response.content)
except TypeError:
response_json = response.json()
except Exception as e:
print(e)
# response_json = response.json()
if response_json['success'] == 0:
# print(response_json['msg'])
data = {'success': 0, 'msg': response_json['msg']}
else:
data = {
'success': 1,
}
return JsonResponse(data)
def browse_directory_ajax(request):
obj = helper.helper(request, Hdfs)
node_id = request.GET['node_id']
node = Nodes.objects.get(id=node_id)
url = "http://%s:%s/command/ls/" % (node.ip, node.port)
directory = request.GET['directory_name']
if directory == "true":
directory = "/"
if directory == '':
directory = 'user'
data = {}
data["path"] = directory
headers = {"API-KEY": obj.get_api_key()}
response = requests.post(url, data=json.dumps(data), headers=headers)
data = response.json()
return JsonResponse(data)
def browse_nodes(request, id):
if 'user' not in request.session:
return redirect('login')
global cluster_id
obj = helper.helper(request, Hdfs)
context = context_processors.base_variables_all(request)
cluster_id = str(request.session[str(request.session['user'])])
node = Nodes.objects.get(id=id)
url = "http://%s:%s/command/ls/" % (node.ip, node.port)
data = {}
if "browse" in request.GET:
data["path"] = request.GET["browse"]
else:
data["path"] = "user"
headers = {"API-KEY": obj.get_api_key()}
try:
response = requests.post(url, data=json.dumps(data), headers=headers)
response_jsons = response.json()
context["path"] = response_jsons["path"]
print(response_jsons["path"])
back_dir = back_browse(response_jsons["path"]) + '/'
except ConnectionError as e:
messages.error(request, "There is some problem with the connection.Please make sure your internet connection" \
" is working and client is up and running on " + node.ip)
return render(request, "error/500.html", context)
rows = directory_information(response_jsons)
context["rows"] = rows
context["node_ip"] = node.ip
context["node_name"] = node.name
context["id"] = id
context["back_path"] = back_dir
return render(request, 'browse/browse.html', context)
def rename_file(request):
obj = helper.helper(request)
source = request.POST['source']
if source == "":
source = "/"
files = request.POST['files']
new_name = request.POST['new_name']
node_id = request.POST['node_id']
node = Nodes.objects.get(id=node_id)
url = "http://%s:%s/command/move/" % (node.ip, node.port)
data_copy = {}
data_copy['source'] = source
data_copy['file_list'] = files
data_copy['destination'] = source + "/" + new_name
headers = {"API-KEY": obj.get_api_key()}
response = requests.post(url, data=json.dumps(data_copy), headers=headers)
response_json = None
try:
response_json = json.loads(response.content)
except TypeError:
response_json = response.json()
except Exception as e:
print(e)
# response_json = response.json()
if response_json['success'] is 0:
# print(response_json['msg'])
data = {'success': 0, 'msg': response_json['msg']}
else:
data = {
'success': 1,
}
return JsonResponse(data)
def back_with_hdfs_ajax(request):
directory = request.GET['directory_name']
context = context_processors.base_variables_all(request)
directory_list = directory.split('/')
directory_list.pop(-1)
directory_str = "/".join(directory_list)
obj = helper.helper(request, Hdfs)
master = obj.get_active_master()
if master:
master_ip = master["ip"]
master_port = master["web_port"]
if directory_str == "":
url = "https://%s:%s/webhdfs/v1/%s?op=LISTSTATUS" % (
master_ip, master_port, directory_str)
else:
url = "https://%s:%s/webhdfs/v1%s?op=LISTSTATUS" % (
master_ip, master_port, directory_str)
r = requests.get(url, verify=False)
response_json = r.json()
data = response_json['FileStatuses']
data['back_directory'] = directory_str
return JsonResponse(data)
else:
data = ''
messages.error(request, "client is not installed on the master node ")
client = False
context["client"] = client
return JsonResponse(data)
def create_dir_local_ajax(request):
obj = helper.helper(request, Hdfs)
root_folder = request.GET['root_folder']
folder_name = request.GET['folder_name']
node_ip = request.GET['node_ip']
node_port = Nodes.objects.filter(ip=node_ip).values("port").first()["port"]
if root_folder == "":
root_folder = "user"
else:
root_folder = root_folder + '/'
create_dir = {}
create_dir["folder_name"] = folder_name
create_dir["root_folder"] = root_folder
url = "http://%s:%s/command/mkdir/" % (node_ip, node_port)
headers = {"API-KEY": obj.get_api_key()}
response = requests.post(url, data=json.dumps(create_dir), headers=headers)
response_json = None
try:
response_json = json.loads(response.content)
except TypeError:
response_json = response.json()
except Exception as e:
print(e)
# response_json = response.json()
if response_json['success'] is 0:
result = {'success': False, 'msg': response_json['msg']}
else:
result = {
'success': True,
}
return JsonResponse(result)
def create_dir_hdfs_ajax(request):
root_folder = request.GET['root_folder']
folder_name = request.GET['folder_name']
context = context_processors.base_variables_all(request)
cluster_id = str(request.session[str(request.session['user'])])
obj = helper.helper(request, Hdfs)
master = obj.get_active_master()
if master:
master_ip = master["ip"]
master_port = master["web_port"]
url = "https://%s:%s/webhdfs/v1%s/%s?op=MKDIRS&permission=711" % (
master_ip, master_port, root_folder, folder_name)
requests.put(url, verify=False)
response = {'success': True}
return JsonResponse(response)
else:
messages.error(request, "client is not installed on the master node ")
client = False
context["client"] = client
response = {'success': False}
return JsonResponse(response)
def get_node_datas(request):
if 'user' not in request.session:
return redirect('login')
obj = helper.helper(request, Hdfs)
node_ip = request.POST['node_ip']
node_port = Nodes.objects.filter(ip=node_ip).values("port").first()["port"]
url = "http://%s:%s/command/ls/" % (node_ip, node_port)
data = {}
data["path"] = "user"
headers = {"API-KEY": obj.get_api_key()}
response = requests.post(url, data=json.dumps(data), headers=headers)
response_jsons = response.json()
row_external = []
for response_json in response_jsons['folder_details']:
owner = response_json["owner"]
group = response_json["group"]
name = response_json["name"]
type = response_json["type"]
size = response_json["size"]
permission = response_json["permission"]
row_external.append({
'permission': permission,
'owner': owner,
'group': group,
'size': size,
'name': name,
'type': type
})
data = {'row_external': row_external}
return JsonResponse(data)
def h_kill(request):
obj = helper(request=request, model=Hdfs)
if request.POST['node_ip'] is not '':
node_ip = request.POST['node_ip']
else:
data = {'success': 0,
'msg': "We are unable to get the IP of the active namenode. Please hard refresh the page and try again"}
return JsonResponse(data)
if request.POST['node_id'] is not '':
node_id = request.POST['node_id']
else:
data = {'success': 0,
'msg': "We are unable to get the IP of the active namenode. Please hard refresh the page and try again"}
return JsonResponse(data)
node = obj.get_node_data(node_ip)
if request.POST['server_type'] is not '':
server_type = request.POST['server_type']
if server_type == 'NameNode':
url_start = 'http://%s:%s/hadoop/namenode/start/' % (node_ip,node["port"])
else:
url_start = 'http://%s:%s/hadoop/datanode/start/' % (node_ip,node["port"])
else:
data = {'success': 0,
'msg': "Sorry we didnot receive some of the required data. Please hard refresh the page and try again"}
return JsonResponse(data)
if request.POST['action_type'] is not '':
action_type = request.POST['action_type']
else:
data = {'success': 0,
'msg': "Sorry we didnot receive some of the required data. Please hard refresh the page and try again"}
return JsonResponse(data)
url = "http://%s:%s/command/kill/" % (node_ip,node["port"])
try:
payload = {"service_name": server_type, "node_id": node_id, "table_name": "hdfs_hdfs"}
r = requests.post(url, headers={"API-KEY": helper.get_api_key()}, data=json.dumps(payload))
if r.status_code != 200:
return JsonResponse({'success': 0, 'msg': 'server threw status code ' + r.status_code})
data = r.json()
if data["success"] == 1:
if action_type == "1":
try:
r_start = requests.get(url_start, headers={"API-KEY": helper.get_api_key()},
data=json.dumps({"cluster_id": int(obj.cluster_id)}))
if r_start.status_code != 200:
return r_start.json()
if r_start.json()['success'] != 1:
return r_start.json()
except Exception as e:
data = {'success': 0, 'msg': e.args}
return JsonResponse(data)
else:
return JsonResponse(data)
except ConnectionError as e:
data = {'success': 0, 'msg': e}
return JsonResponse(data)
def h_all_stop(request):
obj = helper(request=request, model=Hdfs)
return JsonResponse(obj.stop_all("hadoop/dfs/stop/"))
def index(request):
obj = helper(request, Hdfs)
client = True
context = context_processors.base_variables_all(request)
if obj.atleast_one_client_is_installed():
if obj.clientIsInstalledOnMaster():
master = obj.get_active_master()
if master:
master_ip = master["ip"]
context["master_ip"] = master_ip
context["master_id"] = master["id"]
port = master["web_port"]
context["client"] = client
else:
messages.error(request,
"Sorry !! due to some problem we are unable to fetch the information from server."
" You can perform following steps to find the problem and then restart the services."
"<ul>"
"<li> Reload after 10 seconds</li>"
"<li> Restart again</li>"
"<li> Check the log of Namenode and Datanode</li>"
"<li> make there is no problem in configuration file </li> "
"</ul>")
s_master = obj.get_service_master()
if s_master:
context["master_ip"] = s_master["ip"]
context["master_id"] = s_master["id"]
context["error_in_conf_file"] = True
context["client"] = client
return render(request, 'hdfs/hdfs.html', context)
else:
messages.error(request, "We have encountered some problems."
"Please make sure following conditions are met"
"<ul>"
"<li> Client is installed on master node</li>"
"<li> Environment variables for all services are set properly</li>"
"<li> Restart agent on master node [url here]</li>")
context["client"] = False
return render(request, 'hdfs/hdfs.html', context)
else:
messages.error(request, "Seems like no client is installed")
context["client"] = False
return render(request, 'hdfs/hdfs.html', context)
all_nodes = obj.get_all_nodes()
cursor = connection.cursor()
node_with_client = "select h.ip from hdfs_hdfs as h join administer_nodes " \
"as n on h.ip=n.ip "
masters_sql = "select h.*,n.hostname,n.fqdn,n.name from hdfs_hdfs as h left outer join administer_nodes " \
"as n on h.ip=n.ip where h.type=1"
slave_with_client = "select h.*,hm.*,n.hostname,n.fqdn,n.name from hdfs_hdfs as h join administer_nodes as n on " \
"h.ip=n.ip join hdfs_metrics as hm on h.id=hm.node_id " \
"where h.type=0 and hm.updated_at in (select max(updated_at) " \
"from hdfs_metrics limit 1)"
slave_without_client = "select h.*,hm.* from hdfs_hdfs as h join hdfs_metrics as hm on h.id=hm.node_id " \
"where h.type=0 and h.ip not in (" + node_with_client + ") and hm.updated_at in " \
"(select max(updated_at) from hdfs_metrics limit 1)"
name_livenodes = []
name_deadnodes = []
standby_data = {}
cursor.execute(masters_sql)
masters = cursor.fetchall()
colnames = [desc[0] for desc in cursor.description]
for node in masters:
c = dict(zip(colnames, node))
print()
print()
print(c)
client_installed = True
if c["ip"] not in all_nodes:
client_installed = False
if c["type"] == 1 and c["state"] == 1:
safemode = c["safemode"]
active_data = c
if c["type"] == 1 and c["state"] == 0:
c["client_installed"] = client_installed
standby_data = c
if c["type"] == 1 and c["state"] == 2:
c["client_installed"] = client_installed
standby_data = c
cursor.execute(slave_with_client)
nodes_with_client = cursor.fetchall()
colnames = [desc[0] for desc in cursor.description]
for node in nodes_with_client:
c = dict(zip(colnames, node))
client_installed = True
if c["ip"] not in all_nodes:
client_installed = False
c["client_installed"] = client_installed
if c["status"] == "RUNNING":
name_livenodes.append(c)
else:
name_deadnodes.append(c)
cursor.execute(slave_without_client)
nodes_without_client = cursor.fetchall()
for node in nodes_without_client:
c = dict(zip(colnames, node))
client_installed = True
if c["ip"] not in all_nodes:
client_installed = False
c["client_installed"] = client_installed
if c["status"] == "RUNNING":
name_livenodes.append(c)
else:
name_deadnodes.append(c)
service_object = Services.objects.get(name='hdfs')
restart_status_checks = Restart_after_configuration.objects.filter(service_id=service_object.id).exists()
if restart_status_checks:
restart_status_check = Restart_after_configuration.objects.get(service_id=service_object.id)
restart_status = restart_status_check.status
else:
restart_status = 0
context['name_livenodes'] = name_livenodes
context['name_deadnodes'] = name_deadnodes
context['restart_status'] = restart_status
context['active_node'] = active_data
context['standby_node'] = standby_data
context["safemode"] = safemode
context["service_id"] = service_object.id
return render(request, 'hdfs/hdfs.html', context)
