def config_home(): ''' Get configuration for hadoop services, and change it in required config locations :return: Returns success 1 if successful or success 0 with error message ''' log.info("\nChanging Configuration\n") header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': try: loaded_json = json.loads(request.data.decode()) for x in loaded_json: filename = x final_content = loaded_json[x] write_to_file(filename, final_content) except Exception as e: log.error("\nError while changing configuration\n") log.error(e) return '{"success": 0, "msg": ["%s"]}' % e log.info("\nSuccessfully changed configuration\n") return '{"success": 1}' else: return api_status
def dfs_stop(): ''' Stops dfs services (namenode, datanode, zkfc, journalnode) :return: success if dfs_stop_cmd runs successfully else error message is shown ''' log.info("\nStopping DFS Services\n") header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': conn = get_postgres_connection() cur = conn.cursor() sql = "select status from hdfs_hdfs where type=1" cur.execute(sql) rows = cur.fetchall() cur.close() conn.close() status_list = [' '.join(item) for item in rows] if "RUNNING" in status_list: result = run_services.run_bigdata_services(dfs_stop_cmd, 'Stopped DFS Services', 'Error Stopping DFS Services') if json.loads(result)["success"]: update_namenode_info() return result else: update_namenode_info() return '{"success": 1}' else: return api_status
def stop_all_hbase(): header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': conn = get_postgres_connection() cur = conn.cursor() sql = "select status from hbase_hbase where type=1" cur.execute(sql) rows = cur.fetchall() status_list = [' '.join(item) for item in rows] if "RUNNING" in status_list: result = run_services.run_bigdata_services( hbase_stop_cmd, 'HBase Service Stopped', 'Error Stopping HBase Services') update_hbase_master() if json.loads(result)["success"]: cur.execute(sql) rows = cur.fetchall() status_list = [' '.join(item) for item in rows] if "RUNNING" in status_list: cur.close() conn.close() return '{"success": 2, "msg": ["HBase Service not Stopped!!!"]}' else: cur.close() conn.close() return result else: cur.close() conn.close() return result else: return '{"success": 1}' else: return api_status
def kill(): try: updated_at = int(str(time.time()).split(".")[0]) header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': conn = get_postgres_connection() cur = conn.cursor() loaded_json = json.loads(request.data.decode()) service_name = loaded_json['service_name'] node_id = loaded_json['node_id'] table_name = loaded_json['table_name'] kill_result = kill_service(service_name, user_pass) kill_result = json.loads(kill_result) if kill_result["success"]: sql = """UPDATE %s set status='SHUTDOWN', updated_at=%d where id=%d;""" % (table_name, updated_at, int(node_id)) cur.execute(sql) conn.commit() cur.close() conn.close() return '{"success": 1, "msg": ["%s killed forcefully"]}' % service_name else: sql = """UPDATE %s set status='RUNNING' where id=%d;""" % (table_name, int(node_id)) cur.execute(sql) conn.commit() cur.close() conn.close() return '{"success": 0, "msg": ["Error killing %s forcefully"]}' % service_name else: return api_status except Exception as e: log.error("Error in kill()") log.error(e) return '{"success": 0, "msg": ["%s"]}' % e
def hregionserver_restart(): ''' Restarts hbase regionserver ''' log.info("\nRestarting Hbase Regionserver\n") header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': with Pool(processes=1) as pool: loaded_json = json.loads(request.data.decode()) cluster_id = loaded_json['cluster_id'] mul_pool = pool.apply_async(regionserver_stop, args=[cluster_id]) try: result = mul_pool.get(timeout=20) stop = json.loads(result) if stop["success"] == 1: start = json.loads(regionserver_start(cluster_id)) if start["success"]: return '{"success": 1, "msg": ["Successfully Restarted HBase Regionserver"]}' else: return start elif stop["success"] == 2: stop["success"] = 2 return stop else: return stop except Exception as e: pool.terminate() return '{"success": 2, "msg": ["Took more than 20 seconds to restart regionserver!!!"]}' else: return api_status
def stop_spark(): ''' Stops spark :return: Returns success if spark_stop command executes successfully or error message is shown ''' log.info("\nStopping spark\n") header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': conn = get_postgres_connection() cur = conn.cursor() sql = "select status from spark_spark where type=1" cur.execute(sql) rows = cur.fetchall() status_list = [' '.join(item) for item in rows] if "RUNNING" in status_list: result = run_services.run_bigdata_services(spark_stop_cmd, 'spark Stopped', 'Error Stopping spark') if json.loads(result)["success"]: sql = """UPDATE spark_spark set status='SHUTDOWN' where type=1 and state=1;""" cur.execute(sql) conn.commit() cur.close() conn.close() return result else: cur.close() conn.close() return '{"success": 1}' else: return api_status
def get_system_statistics_history(): try: header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': if not parent_dir.endswith('/'): history_dir = parent_dir + '/system_statistics_history/' else: history_dir = parent_dir + 'system_statistics_history/' system_data = [] loaded_json = json.loads(request.data.decode()) key = list(loaded_json.keys())[0] current_time = datetime.utcfromtimestamp(time.time()) if key == 'time': requested_hour = int(loaded_json["time"]) requested_time = current_time - timedelta(hours=requested_hour) elif key == 'days': days = int(loaded_json["days"]) requested_time = current_time - timedelta(days=days) else: return 'No such key "%s" found!!!' % key while requested_time <= current_time: requested_date = requested_time.strftime('%Y-%m-%d') if not os.path.exists("%s.bin" % (history_dir + requested_date)): if key == 'time': requested_time = requested_time + timedelta( hours=requested_hour) elif key == 'days': requested_time = requested_time + timedelta(days=1) continue with open("%s.bin" % (history_dir + requested_date), "r") as history_file: contents = history_file.read() contents = contents.strip().split("\n") for content in contents: content = int(content, 2) content = content.to_bytes( (content.bit_length() + 7) // 8, 'big').decode() content_dict = ast.literal_eval(content) system_data.append(content_dict) requested_time = requested_time + timedelta(days=1) return '{\'final_data\': %s}' % system_data else: return api_status except Exception as e: log.error("Error in list_dir()") log.error(e) return '{"success": 0, "msg": ["%s"]}' % e
def get_system_statistics(): log.info("\nGetting System Statistics\n") try: header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': return system_stats() else: return api_status except Exception as e: log.error("Error in list_dir()") log.error(e) return '{"success": 0, "msg": ["%s"]}' % e
def stop_spark_slave(): ''' Stops spark slave :return: Returns success if stop_spark_slave command executes successfully or error message is shown ''' log.info("\nStopping spark slave\n") header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': loaded_json = json.loads(request.data.decode()) cluster_id = loaded_json['cluster_id'] return sps_stop(cluster_id) else: return api_status
def hregionserver_start(): ''' Starts hbase regionserver :return: success if hregionserver_start_cmd runs successfully else error message is shown ''' log.info("\nStarting HRegionserver\n") header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': loaded_json = json.loads(request.data.decode()) cluster_id = loaded_json['cluster_id'] return regionserver_start(cluster_id) else: return api_status
def datanode_stop(): ''' Stops datanode service :return: success if datanode_stop_cmd runs successfully else error message is shown ''' log.info("\nStopping Datanode\n") header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': loaded_json = json.loads(request.data.decode()) cluster_id = loaded_json['cluster_id'] return dn_stop(cluster_id) else: return api_status
def nodemanager_stop(): ''' Stops node manager :return: Returns success if nm_start_cmd command executes successfully or error message is shown ''' log.info("\nStopping Nodemanager\n") header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': loaded_json = json.loads(request.data.decode()) cluster_id = loaded_json['cluster_id'] return nm_stop(cluster_id) else: return api_status
def elasticsearch_start(): ''' Starts :return: Returns success if es_start_cmd command executes successfully or error message is shown ''' log.info("\nStarting elasticsearch\n") header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': loaded_json = json.loads(request.data.decode()) cluster_id = loaded_json['cluster_id'] return es_start(cluster_id) else: return api_status
def dfs_start(): ''' Starts dfs services (namenode, datanode, zkfc, journalnode) :return: success if dfs_start_cmd runs successfully else error message is shown ''' log.info("\nStarting DFS Services\n") header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': result = run_services.run_bigdata_services(dfs_start_cmd, 'Started DFS Service', 'Error Starting DFS Services') update_namenode_info() return result else: return api_status
def tail(): try: header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': loaded_json = json.loads(request.data.decode()) file_path = loaded_json['file_path'] return file_read_from_tail(file_path, nlines, user_pass) else: return api_status except Exception as e: log.error("Error in tail_view()") log.error(e) return '{"success": 0, "msg": ["%s"]}' % e
def yarn_start(): ''' Starts yarn services (nodemanager, resourcemanager) :return: Returns success if yarn_start_cmd command executes successfully or error message is shown ''' log.info("\nStarting Yarn\n") header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': result = run_services.run_bigdata_services( yarn_start_cmd, 'Started Yarn Services', 'Error Starting Yarn Services') update_rm_info() return result else: return api_status
def kill_running_service(): header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': loaded_json = json.loads(request.data.decode()) table_name = loaded_json['table_name'] if any(table_name in a for a in my_service_process): service_tuple = next((i, d) for i, d in enumerate(my_service_process) if '%s' % table_name in d) service_index = service_tuple[0] value = service_tuple[1][table_name] result = kill_service_process(service_index, value) return result else: return api_status
def remove_files(): try: header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': loaded_json = json.loads(request.data.decode()) src = loaded_json['source'] files = ast.literal_eval(loaded_json['files']) return delete(src, files, user_pass) else: return api_status except Exception as e: log.error("Error in remove_files()") log.error(e) return '{"success": 0, "msg": ["%s"]}' % e
def change_mode(): try: header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': loaded_json = json.loads(request.data.decode()) file_path = loaded_json['file_path'] mode = loaded_json['mode'] return chmod(file_path, mode, user_pass) else: return api_status except Exception as e: log.error("Error in change_mode()") log.error(e) return '{"success": 0, "msg": ["%s"]}' % e
def get_total_space(): log.info("\nGetting Total Memory and Disk space\n") try: header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': total_memory = get_total_memory() total_disk = get_total_disk() # log.info("Total Memory: %d GB && Total Disk: %d GB" % (total_memory, total_disk)) return '{"total_memory": %d, "total_disk": %d}' % (total_memory, total_disk) else: return api_status except Exception as e: log.error("Error in list_dir()") log.error(e) return '{"success": 0, "msg": ["%s"]}' % e
def copy_files(): try: header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': loaded_json = json.loads(request.data.decode()) src = loaded_json['source'] file_list = ast.literal_eval(loaded_json['file_list']) dest = loaded_json['destination'] return copy(src, file_list, dest, user_pass) else: return api_status except Exception as e: log.error("Error in copy_files()") log.error(e) return '{"success": 0, "msg": ["%s"]}' % e
def list_dir(): try: header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': loaded_json = json.loads(request.data.decode()) path = loaded_json['path'] if path == 'user': path = '/home/%s' % username return browse_local(path) else: return api_status except Exception as e: log.error("Error in list_dir()") log.error(e) return '{"success": 0, "msg": ["%s"]}' % e
def hbase_start(): ''' Starts hbase services (hbase master, hbase regionserver) :return: success if hbase_start_cmd runs successfully else error message is shown ''' log.info("\nStarting HBase\n") header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': result = run_services.run_bigdata_services( hbase_start_cmd, 'HBase Services Started', 'Error Starting HBase Services') up_status = update_hbase_master() if up_status is not None: if up_status["success"] == 0: return json.dumps(up_status) return result else: return api_status
def extract(): try: header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': loaded_json = json.loads(request.data.decode()) root_folder = loaded_json['root_folder'] file_name = loaded_json['file_name'] override = loaded_json['override'] if not root_folder.endswith('/'): root_folder = root_folder + '/' return extract_archive(file_name, root_folder, override, user_pass) else: return api_status except Exception as e: log.error("Error in extract()") log.error(e) return '{"success": 0, "msg": ["%s"]}' % e
def make_dir(): try: header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': loaded_json = json.loads(request.data.decode()) root_folder = loaded_json['root_folder'] folder_name = loaded_json['folder_name'] if not root_folder.endswith('/'): root_folder = root_folder + '/' folder_to_create = '%s%s' % (root_folder, folder_name) return make_directory(folder_to_create, user_pass) else: return api_status except Exception as e: log.error("Error in mkdir_dir()") log.error(e) return '{"success": 0, "msg": ["%s"]}' % e
def dfs_restart(): ''' Stops dfs services and starts them again ''' log.info("\nRestarting DFS Services\n") header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': stop = json.loads(dfs_stop()) if stop["success"]: start = json.loads(dfs_start()) if start["success"]: return '{"success": 1, "msg": ["Successfully restarted dfs service"]}' else: return '{"success": 0, "msg": ["Error restarting dfs services!!!"]}' else: return '{"success": 0, "msg": ["Error restarting dfs services!!!"]}' else: return api_status
def elasticsearch_restart(): ''' Restarts elasticsearch ''' log.info("\nRestarting ES\n") header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': loaded_json = json.loads(request.data.decode()) cluster_id = loaded_json['cluster_id'] stop = json.loads(es_stop(cluster_id)) if stop["success"]: start = json.loads(es_start(cluster_id)) if start["success"]: return '{"success": 1, "msg": ["Successfully Restarted Elasticsearch"]}' else: return '{"success": 0, "msg": ["Error Restarting Elasticsearch!!!"]}' else: return '{"success": 0, "msg": ["Error Restarting Elasticsearch!!!"]}' else: return api_status
def restart_spark(): ''' Restarts spark :return: Returns success if spark_restart command executes successfully or error message is shown ''' log.info("\nRestarting spark\n") header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': stop = json.loads(stop_spark()) if stop["success"]: start = json.loads(start_spark()) if start["success"]: return '{"success": 1, "msg": ["Successfully restarted spark service"]}' else: return '{"success": 0, "msg": ["Error restarting spark services!!!"]}' else: return '{"success": 0, "msg": ["Error restarting spark services!!!"]}' else: return api_status
def spark_master_restart(): ''' Restarts spark master ''' log.info("\nRestarting Spark Master\n") header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': loaded_json = json.loads(request.data.decode()) cluster_id = loaded_json['cluster_id'] stop = json.loads(spm_stop(cluster_id)) if stop["success"]: start = json.loads(spm_start(cluster_id)) if start["success"]: return '{"success": 1, "msg": ["Successfully Restarted Spark Master"]}' else: return '{"success": 0, "msg": ["Error Restarting Spark Master!!!"]}' else: return '{"success": 0, "msg": ["Error Restarting Spark Master!!!"]}' else: return api_status
def hregionserver_stop(): ''' Stops hbase regionserver :return: success if hregionserver_stop_cmd runs successfully else error message is shown ''' log.info("\nStopping HRegionserver\n") header_key = request.headers.get('API-KEY') api_status = check_apiKey(header_key) if api_status == 'success': with Pool(processes=1) as pool: loaded_json = json.loads(request.data.decode()) cluster_id = loaded_json['cluster_id'] mul_pool = pool.apply_async(regionserver_stop, args=[cluster_id]) try: result = mul_pool.get(timeout=20) return result except Exception as e: pool.terminate() return '{"success": 2, "msg": ["Took more than 20 seconds to stop regionserver!!!"]}' else: return api_status