def update_exercise(exercise_id: str, title: str, html_file: str, adoc_file: str, public: bool, grader: str, grading_file: str, container_image: str, max_time_sec: int, max_mem_mb: int, asset_files: list, executors: list): html = util.get_optional_file_content(html_file) adoc = util.get_optional_file_content(adoc_file) body = { 'title': title, 'text_html': html, 'text_adoc': adoc, 'public': public, 'grader_type': grader.upper(), 'container_image': container_image, 'max_time_sec': max_time_sec, 'max_mem_mb': max_mem_mb, } if grading_file is not None: grading_script = util.get_file_content(grading_file) body["grading_script"] = grading_script if len(asset_files) > 0: assets = list(map(lambda filename: {'file_name': os.path.basename(filename), 'file_content': util.get_file_content(filename)}, asset_files) ) body["assets"] = assets if len(executors) > 0: executors = list(map(lambda e: {'executor_id': e}, executors)) body["executors"] = executors resp = requests.put(conf.EMS_BASE_URL + '/exercises/' + exercise_id, json=body, headers=util.get_token_header()) print(resp.status_code)
def removeDecommissionedMachine(slaveName=None): ''' Destroy decommissioned machines from the cluster and removes traces from excludes and slaves file INPUT: String slaveName (optional) OUTPUT: boolean (True if successful, False otherwise) ''' util.debug_print('calling downsize removeDecommissionedMachine()') # if not set, then get from excludes list if slaveName is None: util.debug_print('not slaveName passed as parameter') # get the excludes file from master excludes_file_content = util.get_file_content( config.DEFAULT_DESTINATION_EXCLUDES_FILENAME) # no magic, just get last one if len(excludes_file_content) > 0: slaveName = excludes_file_content[-1].strip() else: util.debug_print( 'no slavename passed in as argument AND we got empty slaves file!' ) return False # remove that slavename from excludes remove_line = slaveName + "\n" util.debug_print('removing from excludes file the line: ' + remove_line) update_excludes = util.updateFile('excludes', remove_line, addLine=False) # remove that name from slaves file util.debug_print('removing from slaves file the line: ' + str(remove_line)) update_slaves = util.updateFile('slaves', remove_line, addLine=False) # get vmid from slaveName vmid = util.get_vm_id_by_name(slaveName) # NOW deestroy vm util.debug_print('Now we will be trying to destroy the machine with ID: ' + str(vmid)) result = api.destroyVirtualMachine({'id': vmid}) util.debug_print( 'waiting for the destroyed machine to be finished being destroyed') waitResult = util.waitForAsync(result.get('jobid')) # since we destroyed the vm, we can remove from master's /etc/hosts file hosts = util.get_file_content(config.DEFAULT_DESTINATION_HOSTS_FILENAME) checker = re.compile('.*' + slaveName + '\n') to_be_removed_hosts_line = [ line for line in hosts if checker.match(line) is not None ] util.debug_print('remove line:' + str(to_be_removed_hosts_line) + ' from /etc/hosts file') util.updateFile('hosts', to_be_removed_hosts_line[0], addLine=False) util.debug_print('Done destroying VM.') return True
def decommission(also_stop_vm=True): ''' This function basically copies slave names from slaves list to excludes list and run refresh scripts Input: None Output: None ''' util.debug_print('Trying to decommission') # get all slave names in slaves file all_slave_names = map( str.strip, util.get_file_content(config.DEFAULT_DESTINATION_SLAVES_FILENAME)) util.debug_print('all_slave_names:') util.debug_print(all_slave_names) # get excludes content from master excludes_list = map( str.strip, util.get_file_content(config.DEFAULT_DESTINATION_EXCLUDES_FILENAME)) util.debug_print('current excludes list:') util.debug_print(excludes_list) # basic sanity check to see if we should try to decommission remaining_slaves = len(all_slave_names) - len(excludes_list) if remaining_slaves <= config.MINIMUM_DATANODE_SIZE: util.debug_print('We have reached the minimum cluster size of ' + str(remaining_slaves) + ', skipping decomissioning.') return False # ok, now we know we can remove some removable_slaves = list(set(all_slave_names) - set(excludes_list)) max_name = get_max_slavename(removable_slaves, return_all=False) util.debug_print('next slavename to remove is: ' + max_name) # ok, now we have the slave we want to decommission, update the excludes file newLine = max_name + "\n" util.updateFile('excludes', newLine) # run commands on the master that will output make decommission happen ssh = SSHWrapper.SSHWrapper(config.MASTER_IP) util.debug_print('trying to hdfs dfsadmin -refreshNodes') outmsg, errmsg = ssh.sudo_command( 'sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -refreshNodes"' ) util.debug_print('trying to yarn rmadmin -refreshNodes') outmsg, errmsg = ssh.sudo_command( 'sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/yarn rmadmin -refreshNodes"' ) if also_stop_vm: stopDecommissionedMachine(max_name)
def removeDecommissionedMachine(slaveName = None): ''' Destroy decommissioned machines from the cluster and removes traces from excludes and slaves file INPUT: String slaveName (optional) OUTPUT: boolean (True if successful, False otherwise) ''' util.debug_print('calling downsize removeDecommissionedMachine()') # if not set, then get from excludes list if slaveName is None: util.debug_print('not slaveName passed as parameter') # get the excludes file from master excludes_file_content = util.get_file_content(config.DEFAULT_DESTINATION_EXCLUDES_FILENAME) # no magic, just get last one if len(excludes_file_content) > 0: slaveName = excludes_file_content[-1].strip() else: util.debug_print('no slavename passed in as argument AND we got empty slaves file!') return False # remove that slavename from excludes remove_line = slaveName + "\n" util.debug_print('removing from excludes file the line: ' + remove_line) update_excludes = util.updateFile('excludes', remove_line, addLine = False) # remove that name from slaves file util.debug_print('removing from slaves file the line: ' + str(remove_line)) update_slaves = util.updateFile('slaves', remove_line, addLine = False) # get vmid from slaveName vmid = util.get_vm_id_by_name(slaveName) # NOW deestroy vm util.debug_print('Now we will be trying to destroy the machine with ID: ' + str(vmid)) result = api.destroyVirtualMachine({'id': vmid}) util.debug_print('waiting for the destroyed machine to be finished being destroyed') waitResult = util.waitForAsync(result.get('jobid')) # since we destroyed the vm, we can remove from master's /etc/hosts file hosts = util.get_file_content(config.DEFAULT_DESTINATION_HOSTS_FILENAME) checker = re.compile('.*' + slaveName + '\n') to_be_removed_hosts_line = [line for line in hosts if checker.match(line) is not None] util.debug_print('remove line:' + str(to_be_removed_hosts_line) + ' from /etc/hosts file') util.updateFile('hosts', to_be_removed_hosts_line[0], addLine = False) util.debug_print('Done destroying VM.') return True
def generate_codeforces(self, code, file, out_name): m = datamap.DataMap(util.get_file_content(util.get_map(file))) self.m = m g = Digraph('stones', encoding='utf-8') for n in m.nodes: if n.is_root: count = self.get_module_problem_count(m) label = "%s(%s)" % (n.name, str(count)) # 根节点 g.node(name=n.name, label=label, style='filled', target="_parent", href="https://codeforces.com/problemset", fontsize='14', fillcolor="orangered", color='lightgrey', fontcolor="white", fontname="Microsoft YaHei", shape='box') else: # 普通模块节点 label = "%s(%s)" % (n.name, str(len(n.problems))) g.node(name=n.name, label=label, style='filled', fillcolor="lightslategray", color='lightgrey', fontsize='12', fontcolor="white", fontname="Microsoft YaHei", shape='box') g.edge(n.parent, n.name, color=theme.color_arrow) # add problem last = "" for p in n.problems: problem = code.get_db_problem(p.id, False) if not problem: print("problem not exist "+p.id) continue title = problem['name'] level = code.get_level(problem) idstr = str(p.id) title = idstr+". "+title color = "lightgrey" if level == "Easy": color = "greenyellow" elif level == "Medium": color = "orange" elif level == "Hard": color = "red" else: print("unknown level:", level) # 题目节点 href = "https://codeforces.com/problemset/problem/%d/%s" % (problem['contestId'], problem['index']) g.node(name=idstr, label=title, target="_parent", href=href, color=color, fontname="Microsoft YaHei", fontsize='12', shape='box') if len(last) > 0: g.edge(last, idstr, color=theme.color_arrow) else: g.edge(n.name, idstr, color=theme.color_arrow) last = idstr g.format = 'svg' g.render(filename=util.get_images(out_name)) os.remove(util.get_images(out_name)) self.post_process(util.get_images(out_name)+".svg")
def show_file(file_name, error="", conf=""): file_dict = util.get_file_dict(file_name) if file_dict["type"] in ("img", "pdf"): util.copy_file_to_tmp(file_name) template_data = {"file": file_dict} else: template_data = {"file": file_dict, "file_content": util.get_file_content(file_name)} return render_template("file.html", error=error, conf=conf, **template_data)
def generate_leetcode(self, leet, file, slug, out_name): c = util.get_file_content(util.get_map(file)) m = datamap.DataMap(c) self.m = m g = Digraph('stones', encoding='utf-8') for n in m.nodes: if n.is_root: count = self.get_module_problem_count(m) label = "%s(%s)" % (n.name, str(count)) # 根节点 g.node(name=n.name, label=label, style='filled', target="_parent", href="https://leetcode-cn.com/tag/"+slug, fontsize='14', fillcolor="orangered", color='lightgrey', fontcolor="white", fontname="Microsoft YaHei", shape='box') else: # 普通模块节点 label = "%s(%s)" % (n.name, str(len(n.problems))) g.node(name=n.name, label=label, style='filled', fillcolor="lightslategray", color='lightgrey', fontsize='12', fontcolor="white", fontname="Microsoft YaHei", shape='box') g.edge(n.parent, n.name, color=theme.color_arrow) # add problem last = "" for p in n.problems: title = leet.get_title(p.id) level = leet.get_level(p.id) problem = leet.get_problem(p.id) idstr = str(p.id) title = idstr+". "+title color = "lightgrey" if level == "Easy": color = "greenyellow" elif level == "Medium": color = "orange" elif level == "Hard": color = "red" else: print("unknown level:", level) continue slug = problem['data']['question']['questionTitleSlug'] # 题目节点 g.node(name=idstr, label=title, target="_parent", href="https://leetcode-cn.com/problems/"+slug, color=color, fontname="Microsoft YaHei", fontsize='12', shape='box') if len(last) > 0: g.edge(last, idstr, color=theme.color_arrow) else: g.edge(n.name, idstr, color=theme.color_arrow) last = idstr g.format = 'svg' g.render(filename=util.get_images(out_name)) os.remove(util.get_images(out_name)) self.post_process(util.get_images(out_name)+".svg")
def decommission(also_stop_vm = True): ''' This function basically copies slave names from slaves list to excludes list and run refresh scripts Input: None Output: None ''' util.debug_print('Trying to decommission') # get all slave names in slaves file all_slave_names = map(str.strip, util.get_file_content(config.DEFAULT_DESTINATION_SLAVES_FILENAME)) util.debug_print('all_slave_names:') util.debug_print(all_slave_names) # get excludes content from master excludes_list = map(str.strip, util.get_file_content(config.DEFAULT_DESTINATION_EXCLUDES_FILENAME)) util.debug_print('current excludes list:') util.debug_print(excludes_list) # basic sanity check to see if we should try to decommission remaining_slaves = len(all_slave_names) - len(excludes_list) if remaining_slaves <= config.MINIMUM_DATANODE_SIZE: util.debug_print('We have reached the minimum cluster size of ' + str(remaining_slaves) + ', skipping decomissioning.') return False # ok, now we know we can remove some removable_slaves = list(set(all_slave_names) - set(excludes_list)) max_name = get_max_slavename(removable_slaves, return_all=False) util.debug_print('next slavename to remove is: ' + max_name) # ok, now we have the slave we want to decommission, update the excludes file newLine = max_name + "\n" util.updateFile('excludes', newLine) # run commands on the master that will output make decommission happen ssh = SSHWrapper.SSHWrapper(config.MASTER_IP) util.debug_print('trying to hdfs dfsadmin -refreshNodes') outmsg, errmsg = ssh.sudo_command('sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -refreshNodes"') util.debug_print('trying to yarn rmadmin -refreshNodes') outmsg, errmsg = ssh.sudo_command('sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/yarn rmadmin -refreshNodes"') if also_stop_vm: stopDecommissionedMachine(max_name)
def autoass(course_id: str, exercise_id: str, solution_file: str): solution = util.get_file_content(solution_file) body = { 'solution': solution } resp = requests.post( conf.EMS_BASE_URL + '/teacher/courses/' + course_id + '/exercises/' + exercise_id + '/autoassess', json=body, headers=util.get_token_header()) print(resp.status_code) print(resp.json())
def leetcode_add_finish_icon(self, path): c = util.get_file_content(path) b = BeautifulSoup(c, "xml") nodes = b.select("g.node") graph = b.select_one("g.graph") for n in nodes: title = n.title.get_text() if not title.isdigit(): continue self.post_process_problem_node(graph, n) content = b.prettify() util.save_file_content(path, content)
def add_finish_icon(self, path): c = util.get_file_content(path) b = BeautifulSoup(c, "xml") svg = b.select_one("svg") self.resize_svg(svg) nodes = b.select("g.node") graph = b.select_one("g.graph") for n in nodes: title = n.title.get_text() if not self.is_valid_title(title): continue self.post_process_problem_node(graph, n) content = b.prettify() util.save_file_content(path, content)
def stopDecommissionedMachine(slaveName=None): ''' Checks whether decommissioning completed, then stops the vm Input String slavename Output: None ''' util.debug_print('calling on downsize stopDecommissionedMachine()') if slaveName is None: util.debug_print('not slaveName passed as parameter') # get the excludes file from master excludes_file_content = util.get_file_content( config.DEFAULT_DESTINATION_EXCLUDES_FILENAME) # no magic, just get last one if len(excludes_file_content) > 0: slaveName = excludes_file_content[-1].strip() else: util.debug_print( 'no slavename passed in as arument AND we got empty slaves file!' ) return False vmid = util.get_vm_id_by_name(slaveName) # connect to master ssh = SSHWrapper.SSHWrapper(config.MASTER_IP) # get status util.debug_print('Trying to get report on status of machines... ') outmsg, errmsg = ssh.sudo_command( 'sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -report"' ) # find the section for the slave we are interested in # eg line is "Name: 199.60.17.186:50010 (dlw-Slave71)" checker = re.compile(config.REPORT_DATANODE_STATUS_STARTING_REGEX + str(slaveName) + '\)') util.debug_print( 'starting while loop to check for status of VM is done decommissioning' ) while True: # get line for checking machine status line = '' commission_line = 0 # ok, I admin it might not be the best to put this here as the report MIGHT not chagne, but who knows... slower but safer for now for line in outmsg: matchobj = checker.match(line) if matchobj: util.debug_print('found the line! it is: ' + str(line)) commission_line = outmsg.index(line) + 2 break line = outmsg[commission_line] util.debug_print('on line: ' + str(commission_line) + ' status of decommissioning machine is: ' + str(line)) if line.find('Decommissioned') > -1: util.debug_print( 'VM is finally decommissioned...., trying to stop VM now') result = api.stopVirtualMachine({'id': vmid}) waitResult = util.waitForAsync(result.get('jobid')) if waitResult != True: # whoops something went wrong! return waitResult # let's get out of here! util.debug_print( 'DONE, we waited for it to finish decomissioning, then we stopped the VM! ' ) break # ok, not decommissioned yet, so callthe ssh command again! util.debug_print( 'checking again inside forever WHileTrue loop, as it is not in decomissioned state.' ) outmsg, errmsg = ssh.sudo_command( 'sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -report"' ) return True
def recommission(slaveName = None): ''' This function recommissions, aka removes from excludes list and runs correct script to add back to node Input: None Output: None ''' util.debug_print('calling upsize recommission()') # if not set, then get from excludes list if slaveName is None: util.debug_print('not slaveName passed as parameter') # get the excludes file from master excludes_file_content = util.get_file_content(config.DEFAULT_DESTINATION_EXCLUDES_FILENAME) # no magic, just get last one if len(excludes_file_content) > 0: slaveName = excludes_file_content[-1].strip() else: util.debug_print('no slavename passed in as argument AND we got empty slaves file!') return False # remove that slavename from excludes remove_line = slaveName + "\n" util.debug_print('removing from excludes file the line: ' + remove_line) update_excludes = util.updateFile('excludes', remove_line, addLine = False) # confirm if VM is running or stopped or whatever vmid = util.get_vm_id_by_name(slaveName) raw_result = api.listVirtualMachines({'id':vmid}) result = raw_result.get('virtualmachine')[0] ipaddr = result.get('nic')[0].get('ipaddress') while True: current_state = result.get('state') if current_state == 'Running': util.debug_print('Machine is currently running') util.debug_print('trying to hdfs dfsadmin -refreshNodes') ssh = SSHWrapper.SSHWrapper(config.MASTER_IP) outmsg, errmsg = ssh.sudo_command('sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -refreshNodes"') util.debug_print('trying to yarn rmadmin -refreshNodes') outmsg, errmsg = ssh.sudo_command('sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/yarn rmadmin -refreshNodes"') break elif current_state == 'Stopped': util.debug_print('Machine is currently Stopped') # start up machine and wait till it finishes starting up util.debug_print('Trying to start VM') result = api.startVirtualMachine({'id': vmid}) # now we wait for the async deployVirtualMachine() to finsih waitResult = util.waitForAsync(result.get('jobid')) if waitResult != True: # whoops something went wrong! return waitResult # SSHWrapper checks and loops and waits for connection so let's just use it for testing that ''' WHY DOES RESULT NOT GET ANYTHING! INEED THE IP OF THE DEVICE! MAYBE GET IT EARLIER? ''' sshWaiting = SSHWrapper.SSHWrapper(ipaddr) util.debug_print('trying to start-dfs.sh') ssh = SSHWrapper.SSHWrapper(config.MASTER_IP) ssh.sudo_command('sudo -S su hduser -c "bash /home/hduser/hadoop-2.7.0/sbin/start-dfs.sh"') util.debug_print('trying to run start-yarn.sh') ssh.sudo_command('sudo -S su hduser -c "bash /home/hduser/hadoop-2.7.0/sbin/start-yarn.sh"') util.debug_print('trying to hdfs dfsadmin -refreshNodes') ssh.sudo_command('sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -refreshNodes"') util.debug_print('trying to yarn rmadmin -refreshNodes') ssh.sudo_command('sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/yarn rmadmin -refreshNodes"') break elif current_state == 'Stopping' or current_state == 'Starting': util.debug_print('OK, currently changing state, let us just call listVirtualMachines again and see.') raw_result = api.listVirtualMachines({'id':vmid}) result = raw_result.get('virtualmachine')[0] else: # something went wrong here! util.debug_print('ok, it is in an unexpected state: ' + str(current_state)) return False
} util.debug(body) resp = requests.put(util.AAS_BASE_URL + '/exercises/' + ex_id, json=body, headers=util.get_required_headers()) print(resp.status_code) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--id', required=True) parser.add_argument('--grading-file', required=True) parser.add_argument('--image-name', required=True) parser.add_argument('--max-time', required=True, type=int) parser.add_argument('--max-mem', required=True, type=int) parser.add_argument('--asset-files', nargs='*', default=[]) parser.add_argument('--executors', nargs='+', required=True) args = parser.parse_args() grading_script = util.get_file_content(args.grading_file) assets = list( map( lambda filename: { 'file_name': os.path.basename(filename), 'file_content': util.get_file_content(filename) }, args.asset_files)) update_exercise(args.id, grading_script, args.image_name, args.max_time, args.max_mem, assets, args.executors)
def stopDecommissionedMachine(slaveName = None): ''' Checks whether decommissioning completed, then stops the vm Input String slavename Output: None ''' util.debug_print('calling on downsize stopDecommissionedMachine()') if slaveName is None: util.debug_print('not slaveName passed as parameter') # get the excludes file from master excludes_file_content = util.get_file_content(config.DEFAULT_DESTINATION_EXCLUDES_FILENAME) # no magic, just get last one if len(excludes_file_content) > 0: slaveName = excludes_file_content[-1].strip() else: util.debug_print('no slavename passed in as arument AND we got empty slaves file!') return False vmid = util.get_vm_id_by_name(slaveName) # connect to master ssh = SSHWrapper.SSHWrapper(config.MASTER_IP) # get status util.debug_print('Trying to get report on status of machines... ') outmsg, errmsg = ssh.sudo_command('sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -report"') # find the section for the slave we are interested in # eg line is "Name: 199.60.17.186:50010 (dlw-Slave71)" checker = re.compile(config.REPORT_DATANODE_STATUS_STARTING_REGEX + str(slaveName) + '\)') util.debug_print('starting while loop to check for status of VM is done decommissioning') while True: # get line for checking machine status line = '' commission_line = 0 # ok, I admin it might not be the best to put this here as the report MIGHT not chagne, but who knows... slower but safer for now for line in outmsg: matchobj = checker.match(line) if matchobj: util.debug_print('found the line! it is: ' + str(line)) commission_line = outmsg.index(line) + 2 break; line = outmsg[commission_line] util.debug_print('on line: ' + str(commission_line) + ' status of decommissioning machine is: ' + str(line)) if line.find('Decommissioned') > -1: util.debug_print('VM is finally decommissioned...., trying to stop VM now') result = api.stopVirtualMachine({'id': vmid}) waitResult = util.waitForAsync(result.get('jobid')) if waitResult != True: # whoops something went wrong! return waitResult # let's get out of here! util.debug_print('DONE, we waited for it to finish decomissioning, then we stopped the VM! ') break; # ok, not decommissioned yet, so callthe ssh command again! util.debug_print('checking again inside forever WHileTrue loop, as it is not in decomissioned state.') outmsg, errmsg = ssh.sudo_command('sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -report"') return True
import argparse import requests import util def update_exercise(ex_id, ex_title, ex_text, ex_public): body = {'title': ex_title, 'text_html': ex_text, 'public': ex_public} util.debug(body) resp = requests.put(util.EMS_BASE_URL + '/teacher/exercises/' + ex_id, json=body, headers=util.get_required_headers()) print(resp.status_code) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--id', required=True) parser.add_argument('--title', required=True) parser.add_argument('--text-file') parser.add_argument('--public', action='store_true') args = parser.parse_args() if args.text_file is None: exercise_text = None else: exercise_text = util.get_file_content(args.text_file) update_exercise(args.id, args.title, exercise_text, args.public)
def recommission(slaveName=None): ''' This function recommissions, aka removes from excludes list and runs correct script to add back to node Input: None Output: None ''' util.debug_print('calling upsize recommission()') # if not set, then get from excludes list if slaveName is None: util.debug_print('not slaveName passed as parameter') # get the excludes file from master excludes_file_content = util.get_file_content( config.DEFAULT_DESTINATION_EXCLUDES_FILENAME) # no magic, just get last one if len(excludes_file_content) > 0: slaveName = excludes_file_content[-1].strip() else: util.debug_print( 'no slavename passed in as argument AND we got empty slaves file!' ) return False # remove that slavename from excludes remove_line = slaveName + "\n" util.debug_print('removing from excludes file the line: ' + remove_line) update_excludes = util.updateFile('excludes', remove_line, addLine=False) # confirm if VM is running or stopped or whatever vmid = util.get_vm_id_by_name(slaveName) raw_result = api.listVirtualMachines({'id': vmid}) result = raw_result.get('virtualmachine')[0] ipaddr = result.get('nic')[0].get('ipaddress') while True: current_state = result.get('state') if current_state == 'Running': util.debug_print('Machine is currently running') util.debug_print('trying to hdfs dfsadmin -refreshNodes') ssh = SSHWrapper.SSHWrapper(config.MASTER_IP) outmsg, errmsg = ssh.sudo_command( 'sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -refreshNodes"' ) util.debug_print('trying to yarn rmadmin -refreshNodes') outmsg, errmsg = ssh.sudo_command( 'sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/yarn rmadmin -refreshNodes"' ) break elif current_state == 'Stopped': util.debug_print('Machine is currently Stopped') # start up machine and wait till it finishes starting up util.debug_print('Trying to start VM') result = api.startVirtualMachine({'id': vmid}) # now we wait for the async deployVirtualMachine() to finsih waitResult = util.waitForAsync(result.get('jobid')) if waitResult != True: # whoops something went wrong! return waitResult # SSHWrapper checks and loops and waits for connection so let's just use it for testing that ''' WHY DOES RESULT NOT GET ANYTHING! INEED THE IP OF THE DEVICE! MAYBE GET IT EARLIER? ''' sshWaiting = SSHWrapper.SSHWrapper(ipaddr) util.debug_print('trying to start-dfs.sh') ssh = SSHWrapper.SSHWrapper(config.MASTER_IP) ssh.sudo_command( 'sudo -S su hduser -c "bash /home/hduser/hadoop-2.7.0/sbin/start-dfs.sh"' ) util.debug_print('trying to run start-yarn.sh') ssh.sudo_command( 'sudo -S su hduser -c "bash /home/hduser/hadoop-2.7.0/sbin/start-yarn.sh"' ) util.debug_print('trying to hdfs dfsadmin -refreshNodes') ssh.sudo_command( 'sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -refreshNodes"' ) util.debug_print('trying to yarn rmadmin -refreshNodes') ssh.sudo_command( 'sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/yarn rmadmin -refreshNodes"' ) break elif current_state == 'Stopping' or current_state == 'Starting': util.debug_print( 'OK, currently changing state, let us just call listVirtualMachines again and see.' ) raw_result = api.listVirtualMachines({'id': vmid}) result = raw_result.get('virtualmachine')[0] else: # something went wrong here! util.debug_print('ok, it is in an unexpected state: ' + str(current_state)) return False