def back_with_ajax(request): obj = helper.helper(request, Hdfs) node_id = request.GET['node_id'] directory = request.GET['directory_name'] directory_list = directory.split('/') directory_list.pop(-1) directory_list.pop(-1) directory_str = "/".join(directory_list) if directory_str == '': directory = '/' else: directory = directory_str node = Nodes.objects.get(id=node_id) url = "http://%s:%s/command/ls/" % (node.ip, node.port) data = {} data["path"] = directory headers = {"API-KEY": obj.get_api_key()} response = requests.post(url, data=json.dumps(data), headers=headers) data = response.json() data['back_directory'] = data["path"] return JsonResponse(data)
def extract_files_local(request): obj = helper.helper(request, Hdfs) file_name = request.POST['file_name'] root_folder = request.POST['root_folder'] override = request.POST['override'] node_id = request.POST['node_id'] node = Nodes.objects.get(id=node_id) if (root_folder == ''): root_folder = 'user' else: root_folder = root_folder + '/' to_extract = {} to_extract["file_name"] = file_name to_extract["root_folder"] = root_folder to_extract["override"] = override url = "http://%s:%s/command/extract/" % (node.ip, node.port) headers = {"API-KEY": obj.get_api_key()} response = requests.post(url, data=json.dumps(to_extract), headers=headers) response_json = response.json() if response_json['success'] is 0: data = {'success': 0, 'msg': response_json['msg']} else: data = { 'success': 1, } return JsonResponse(data)
def back_eachnode_with_ajax(request): obj = helper.helper(request, Hdfs) node_ip = request.POST['node_ip'] node_port = Nodes.objects.filter(ip=node_ip).values("port").first()["port"] directory = request.POST['directory_name'] directory_list = directory.split('/') directory_list.pop(-1) directory_list.pop(-1) directory_str = "/".join(directory_list) if directory_str == '': directory = '/' else: directory = directory_str url = "http://%s:%s/command/ls/" % (node_ip, node_port) data = {} data["path"] = directory headers = {"API-KEY": obj.get_api_key()} response = requests.post(url, data=json.dumps(data), headers=headers) data = response.json() data['back_directory'] = directory_str return JsonResponse(data)
def files_to_local_ajax(request): obj = helper.helper(request, Hdfs) file_name = request.POST['file_name'] root_folder = request.POST['root_folder'] destination_folder = request.POST['destination_folder'] node_ip = request.POST['node_ip'] node_port = Nodes.objects.filter(ip=node_ip).values("port").first()["port"] nfs = request.POST['nfs'] if (root_folder == ''): root_folder = '/' else: root_folder = root_folder + '/' if (destination_folder == ''): destination_folder = '/' to_local = {} to_local['file_name'] = file_name to_local['root_folder'] = root_folder to_local['destination'] = destination_folder to_local['nfs'] = nfs url = "http://%s:%s/hdfs/get/" % (node_ip, node_port) headers = {"API-KEY": obj.get_api_key()} response = requests.post(url, data=json.dumps(to_local), headers=headers) response_json = response.json() if response_json['success'] is 0: data = {'success': 0, 'msg': response_json['msg']} else: data = { 'success': 1, } return JsonResponse(data)
def tail_file(request): obj = helper.helper(request) file_name = request.POST['file_name'] source = request.POST['source'] + "/" + file_name node_id = request.POST['node_id'] node = Nodes.objects.get(id=node_id) data_copy = {} data_copy['file_path'] = source url = "http://%s:%s/command/tail/" % (node.ip, node.port) headers = {"API-KEY": obj.get_api_key()} response = requests.post(url, data=json.dumps(data_copy), headers=headers) response_json = None try: response_json = json.loads(response.content) except TypeError: response_json = response.json() except Exception as e: print(e) # response_json = json.loads(response.content) if response_json['success'] is 0: data = {'success': 0, 'msg': response_json['msg']} else: result = response_json['tail_result'].replace('\t', ' ').split('\n') data = {'success': 1, 'result': result} return JsonResponse(data)
def browse_hdfs_ajax(request): directory = request.GET['directory_name'] client = True context = context_processors.base_variables_all(request) if directory == "": directory = "/" obj = helper.helper(request, Hdfs) master = obj.get_active_master() if master: master_ip = master["ip"] master_port = master["web_port"] url = "https://%s:%s/webhdfs/v1%s?op=LISTSTATUS" % ( master_ip, master_port, directory) r = requests.get(url, verify=False) response_json = r.json() data = response_json['FileStatuses'] return JsonResponse(data) else: data = '' messages.error(request, "client is not installed on the master node ") client = False context["client"] = client return JsonResponse(data)
def files_to_hdfs_ajax(request): obj = helper.helper(request, Hdfs) file_name = request.POST['file_name'] root_folder = request.POST['root_folder'] destination_folder = request.POST['destination_folder'] node_id = request.POST['node_id'] node = Nodes.objects.get(id=node_id) if (root_folder == ''): root_folder = '/' else: root_folder = root_folder + '/' if (destination_folder == ''): destination_folder = '/' to_hdfs = {} to_hdfs['file_name'] = file_name to_hdfs['root_folder'] = root_folder to_hdfs['destination'] = destination_folder url = "http://%s:%s/hdfs/write/" % (node.ip, node.port) headers = {"API-KEY": obj.get_api_key()} response = requests.post(url, data=json.dumps(to_hdfs), headers=headers) response_json = response.json() if response_json['success'] is 0: data = {'success': 0, 'msg': response_json['msg']} else: data = { 'success': 1, } return JsonResponse(data)
def browse_hdfs(request): if 'user' not in request.session: return redirect('login') obj = helper.helper(request, Hdfs) master_ip = "" client = True master = "" context = context_processors.base_variables_all(request) if obj.atleast_one_client_is_installed(): if obj.clientIsInstalledOnMaster(): master = obj.get_active_master() if master: master_ip = master["ip"] master_port = master["web_port"] context["client"] = client else: messages.error( request, "Sorry there is some problem in your configuration file <h3>namenode is down</h3> " ) context["error_in_conf_file"] = True return render(request, 'browse/browse_hdfs.html', context) else: messages.error(request, "Client is not installed on master node") context["client"] = False return render(request, 'browse/browse_hdfs.html', context) else: messages.error(request, "Seems like no client is installed") return render(request, 'browse/browse_hdfs.html', context) cluster_id = str(request.session[str(request.session['user'])]) nodes_hdfs = Nodes.objects.all() try: url = "https://%s:%s/webhdfs/v1/?op=LISTSTATUS" % (master_ip, master_port) r = requests.get(url, verify=False) response_json = r.json() rows = response_json['FileStatuses']['FileStatus'] except ConnectionError as e: messages.error(request, "There is some problem with the connection.Please make sure your internet connection" \ " is working and client is up and running on " + master_ip) return render(request, "error/500.html", context) context["rows"] = rows context["nodes_hdfs"] = nodes_hdfs context["master_ip"] = master_ip return render(request, 'browse/browse_hdfs.html', context)
def h_all_restart(request): if request.POST['node_ip'] is not '': node_ip = request.POST['node_ip'] else: data = {'success': 0, 'msg': "We are unable to get the IP of the active namenode. Please hard refresh the page and try again"} return JsonResponse(data) obj = helper(request=request, model=Hdfs) op_status = obj.restart_all("hadoop/dfs/restart/", master_ip=node_ip) if op_status["success"]: Restart_after_configuration.objects.filter(service_id=obj.get_service_id("hdfs")).update(status=0) return JsonResponse(op_status)
def dn_stop(request): obj = helper(request) if request.POST['node_ip'] is not '': node_ip = request.POST['node_ip'] else: data = {'success': 0, 'msg': "We are unable to get the IP of the datanode. Please hard refresh the page and try again"} return JsonResponse(data) try: node = obj.get_node_data(node_ip) url_stop = 'http://%s:%s/hadoop/datanode/stop/' % (node_ip,node["port"]) return JsonResponse(obj.stop_service(url_stop)) except Exception as e: data = {'success': 0, 'msg': e} return JsonResponse(data, safe=False)
def nn_restart(request): obj = helper(request) if request.POST['node_ip'] is not '': node_ip = request.POST['node_ip'] else: data = {'success': 0, 'msg': "We are unable to get the IP of the active namenode. Please hard refresh the page and try again"} return JsonResponse(data) try: node = obj.get_node_data(node_ip) url = 'http://%s:%s/hadoop/namenode/restart/' % (node_ip,node["port"]) data = obj.restart_service(url) return JsonResponse(data) except Exception as e: data = {'success': 0, 'msg': e.args} return JsonResponse(data)
def browse_eachnode_directory_ajax(request): obj = helper.helper(request, Hdfs) node_ip = request.POST['node_ip'] node_port = Nodes.objects.filter(ip=node_ip).values("port").first()["port"] directory = request.POST['directory_name'] if directory == '': directory = 'user' url = "http://%s:%s/command/ls/" % (node_ip, node_port) data = {} data["path"] = directory headers = {"API-KEY": obj.get_api_key()} response = requests.post(url, data=json.dumps(data), headers=headers) data = response.json() return JsonResponse(data)
def copy_on_nodes(request): if 'user' not in request.session: return redirect('login') obj = helper.helper(request) source = request.POST['source'] if source == "": source = "/" destination = request.POST['destination'] if destination == "": destination = "/" files = request.POST['files'] node_id = request.POST['node_id'] node = Nodes.objects.get(id=node_id) url = "http://%s:%s/command/copy/" % (node.ip, node.port) data_copy = {} data_copy['source'] = source data_copy['file_list'] = files data_copy['destination'] = destination headers = {"API-KEY": obj.get_api_key()} response = requests.post(url, data=json.dumps(data_copy), headers=headers) response_json = None try: response_json = json.loads(response.content) except TypeError: response_json = response.json() except Exception as e: print(e) # response_json = response.json() if response_json['success'] == 0: # print(response_json['msg']) data = {'success': 0, 'msg': response_json['msg']} else: data = { 'success': 1, } return JsonResponse(data)
def browse_directory_ajax(request): obj = helper.helper(request, Hdfs) node_id = request.GET['node_id'] node = Nodes.objects.get(id=node_id) url = "http://%s:%s/command/ls/" % (node.ip, node.port) directory = request.GET['directory_name'] if directory == "true": directory = "/" if directory == '': directory = 'user' data = {} data["path"] = directory headers = {"API-KEY": obj.get_api_key()} response = requests.post(url, data=json.dumps(data), headers=headers) data = response.json() return JsonResponse(data)
def browse_nodes(request, id): if 'user' not in request.session: return redirect('login') global cluster_id obj = helper.helper(request, Hdfs) context = context_processors.base_variables_all(request) cluster_id = str(request.session[str(request.session['user'])]) node = Nodes.objects.get(id=id) url = "http://%s:%s/command/ls/" % (node.ip, node.port) data = {} if "browse" in request.GET: data["path"] = request.GET["browse"] else: data["path"] = "user" headers = {"API-KEY": obj.get_api_key()} try: response = requests.post(url, data=json.dumps(data), headers=headers) response_jsons = response.json() context["path"] = response_jsons["path"] print(response_jsons["path"]) back_dir = back_browse(response_jsons["path"]) + '/' except ConnectionError as e: messages.error(request, "There is some problem with the connection.Please make sure your internet connection" \ " is working and client is up and running on " + node.ip) return render(request, "error/500.html", context) rows = directory_information(response_jsons) context["rows"] = rows context["node_ip"] = node.ip context["node_name"] = node.name context["id"] = id context["back_path"] = back_dir return render(request, 'browse/browse.html', context)
def rename_file(request): obj = helper.helper(request) source = request.POST['source'] if source == "": source = "/" files = request.POST['files'] new_name = request.POST['new_name'] node_id = request.POST['node_id'] node = Nodes.objects.get(id=node_id) url = "http://%s:%s/command/move/" % (node.ip, node.port) data_copy = {} data_copy['source'] = source data_copy['file_list'] = files data_copy['destination'] = source + "/" + new_name headers = {"API-KEY": obj.get_api_key()} response = requests.post(url, data=json.dumps(data_copy), headers=headers) response_json = None try: response_json = json.loads(response.content) except TypeError: response_json = response.json() except Exception as e: print(e) # response_json = response.json() if response_json['success'] is 0: # print(response_json['msg']) data = {'success': 0, 'msg': response_json['msg']} else: data = { 'success': 1, } return JsonResponse(data)
def back_with_hdfs_ajax(request): directory = request.GET['directory_name'] context = context_processors.base_variables_all(request) directory_list = directory.split('/') directory_list.pop(-1) directory_str = "/".join(directory_list) obj = helper.helper(request, Hdfs) master = obj.get_active_master() if master: master_ip = master["ip"] master_port = master["web_port"] if directory_str == "": url = "https://%s:%s/webhdfs/v1/%s?op=LISTSTATUS" % ( master_ip, master_port, directory_str) else: url = "https://%s:%s/webhdfs/v1%s?op=LISTSTATUS" % ( master_ip, master_port, directory_str) r = requests.get(url, verify=False) response_json = r.json() data = response_json['FileStatuses'] data['back_directory'] = directory_str return JsonResponse(data) else: data = '' messages.error(request, "client is not installed on the master node ") client = False context["client"] = client return JsonResponse(data)
def create_dir_local_ajax(request): obj = helper.helper(request, Hdfs) root_folder = request.GET['root_folder'] folder_name = request.GET['folder_name'] node_ip = request.GET['node_ip'] node_port = Nodes.objects.filter(ip=node_ip).values("port").first()["port"] if root_folder == "": root_folder = "user" else: root_folder = root_folder + '/' create_dir = {} create_dir["folder_name"] = folder_name create_dir["root_folder"] = root_folder url = "http://%s:%s/command/mkdir/" % (node_ip, node_port) headers = {"API-KEY": obj.get_api_key()} response = requests.post(url, data=json.dumps(create_dir), headers=headers) response_json = None try: response_json = json.loads(response.content) except TypeError: response_json = response.json() except Exception as e: print(e) # response_json = response.json() if response_json['success'] is 0: result = {'success': False, 'msg': response_json['msg']} else: result = { 'success': True, } return JsonResponse(result)
def create_dir_hdfs_ajax(request): root_folder = request.GET['root_folder'] folder_name = request.GET['folder_name'] context = context_processors.base_variables_all(request) cluster_id = str(request.session[str(request.session['user'])]) obj = helper.helper(request, Hdfs) master = obj.get_active_master() if master: master_ip = master["ip"] master_port = master["web_port"] url = "https://%s:%s/webhdfs/v1%s/%s?op=MKDIRS&permission=711" % ( master_ip, master_port, root_folder, folder_name) requests.put(url, verify=False) response = {'success': True} return JsonResponse(response) else: messages.error(request, "client is not installed on the master node ") client = False context["client"] = client response = {'success': False} return JsonResponse(response)
def get_node_datas(request): if 'user' not in request.session: return redirect('login') obj = helper.helper(request, Hdfs) node_ip = request.POST['node_ip'] node_port = Nodes.objects.filter(ip=node_ip).values("port").first()["port"] url = "http://%s:%s/command/ls/" % (node_ip, node_port) data = {} data["path"] = "user" headers = {"API-KEY": obj.get_api_key()} response = requests.post(url, data=json.dumps(data), headers=headers) response_jsons = response.json() row_external = [] for response_json in response_jsons['folder_details']: owner = response_json["owner"] group = response_json["group"] name = response_json["name"] type = response_json["type"] size = response_json["size"] permission = response_json["permission"] row_external.append({ 'permission': permission, 'owner': owner, 'group': group, 'size': size, 'name': name, 'type': type }) data = {'row_external': row_external} return JsonResponse(data)
def h_kill(request): obj = helper(request=request, model=Hdfs) if request.POST['node_ip'] is not '': node_ip = request.POST['node_ip'] else: data = {'success': 0, 'msg': "We are unable to get the IP of the active namenode. Please hard refresh the page and try again"} return JsonResponse(data) if request.POST['node_id'] is not '': node_id = request.POST['node_id'] else: data = {'success': 0, 'msg': "We are unable to get the IP of the active namenode. Please hard refresh the page and try again"} return JsonResponse(data) node = obj.get_node_data(node_ip) if request.POST['server_type'] is not '': server_type = request.POST['server_type'] if server_type == 'NameNode': url_start = 'http://%s:%s/hadoop/namenode/start/' % (node_ip,node["port"]) else: url_start = 'http://%s:%s/hadoop/datanode/start/' % (node_ip,node["port"]) else: data = {'success': 0, 'msg': "Sorry we didnot receive some of the required data. Please hard refresh the page and try again"} return JsonResponse(data) if request.POST['action_type'] is not '': action_type = request.POST['action_type'] else: data = {'success': 0, 'msg': "Sorry we didnot receive some of the required data. Please hard refresh the page and try again"} return JsonResponse(data) url = "http://%s:%s/command/kill/" % (node_ip,node["port"]) try: payload = {"service_name": server_type, "node_id": node_id, "table_name": "hdfs_hdfs"} r = requests.post(url, headers={"API-KEY": helper.get_api_key()}, data=json.dumps(payload)) if r.status_code != 200: return JsonResponse({'success': 0, 'msg': 'server threw status code ' + r.status_code}) data = r.json() if data["success"] == 1: if action_type == "1": try: r_start = requests.get(url_start, headers={"API-KEY": helper.get_api_key()}, data=json.dumps({"cluster_id": int(obj.cluster_id)})) if r_start.status_code != 200: return r_start.json() if r_start.json()['success'] != 1: return r_start.json() except Exception as e: data = {'success': 0, 'msg': e.args} return JsonResponse(data) else: return JsonResponse(data) except ConnectionError as e: data = {'success': 0, 'msg': e} return JsonResponse(data)
def h_all_stop(request): obj = helper(request=request, model=Hdfs) return JsonResponse(obj.stop_all("hadoop/dfs/stop/"))
def index(request): obj = helper(request, Hdfs) client = True context = context_processors.base_variables_all(request) if obj.atleast_one_client_is_installed(): if obj.clientIsInstalledOnMaster(): master = obj.get_active_master() if master: master_ip = master["ip"] context["master_ip"] = master_ip context["master_id"] = master["id"] port = master["web_port"] context["client"] = client else: messages.error(request, "Sorry !! due to some problem we are unable to fetch the information from server." " You can perform following steps to find the problem and then restart the services." "<ul>" "<li> Reload after 10 seconds</li>" "<li> Restart again</li>" "<li> Check the log of Namenode and Datanode</li>" "<li> make there is no problem in configuration file </li> " "</ul>") s_master = obj.get_service_master() if s_master: context["master_ip"] = s_master["ip"] context["master_id"] = s_master["id"] context["error_in_conf_file"] = True context["client"] = client return render(request, 'hdfs/hdfs.html', context) else: messages.error(request, "We have encountered some problems." "Please make sure following conditions are met" "<ul>" "<li> Client is installed on master node</li>" "<li> Environment variables for all services are set properly</li>" "<li> Restart agent on master node [url here]</li>") context["client"] = False return render(request, 'hdfs/hdfs.html', context) else: messages.error(request, "Seems like no client is installed") context["client"] = False return render(request, 'hdfs/hdfs.html', context) all_nodes = obj.get_all_nodes() cursor = connection.cursor() node_with_client = "select h.ip from hdfs_hdfs as h join administer_nodes " \ "as n on h.ip=n.ip " masters_sql = "select h.*,n.hostname,n.fqdn,n.name from hdfs_hdfs as h left outer join administer_nodes " \ "as n on h.ip=n.ip where h.type=1" slave_with_client = "select h.*,hm.*,n.hostname,n.fqdn,n.name from hdfs_hdfs as h join administer_nodes as n on " \ "h.ip=n.ip join hdfs_metrics as hm on h.id=hm.node_id " \ "where h.type=0 and hm.updated_at in (select max(updated_at) " \ "from hdfs_metrics limit 1)" slave_without_client = "select h.*,hm.* from hdfs_hdfs as h join hdfs_metrics as hm on h.id=hm.node_id " \ "where h.type=0 and h.ip not in (" + node_with_client + ") and hm.updated_at in " \ "(select max(updated_at) from hdfs_metrics limit 1)" name_livenodes = [] name_deadnodes = [] standby_data = {} cursor.execute(masters_sql) masters = cursor.fetchall() colnames = [desc[0] for desc in cursor.description] for node in masters: c = dict(zip(colnames, node)) print() print() print(c) client_installed = True if c["ip"] not in all_nodes: client_installed = False if c["type"] == 1 and c["state"] == 1: safemode = c["safemode"] active_data = c if c["type"] == 1 and c["state"] == 0: c["client_installed"] = client_installed standby_data = c if c["type"] == 1 and c["state"] == 2: c["client_installed"] = client_installed standby_data = c cursor.execute(slave_with_client) nodes_with_client = cursor.fetchall() colnames = [desc[0] for desc in cursor.description] for node in nodes_with_client: c = dict(zip(colnames, node)) client_installed = True if c["ip"] not in all_nodes: client_installed = False c["client_installed"] = client_installed if c["status"] == "RUNNING": name_livenodes.append(c) else: name_deadnodes.append(c) cursor.execute(slave_without_client) nodes_without_client = cursor.fetchall() for node in nodes_without_client: c = dict(zip(colnames, node)) client_installed = True if c["ip"] not in all_nodes: client_installed = False c["client_installed"] = client_installed if c["status"] == "RUNNING": name_livenodes.append(c) else: name_deadnodes.append(c) service_object = Services.objects.get(name='hdfs') restart_status_checks = Restart_after_configuration.objects.filter(service_id=service_object.id).exists() if restart_status_checks: restart_status_check = Restart_after_configuration.objects.get(service_id=service_object.id) restart_status = restart_status_check.status else: restart_status = 0 context['name_livenodes'] = name_livenodes context['name_deadnodes'] = name_deadnodes context['restart_status'] = restart_status context['active_node'] = active_data context['standby_node'] = standby_data context["safemode"] = safemode context["service_id"] = service_object.id return render(request, 'hdfs/hdfs.html', context)