def update_exercise(exercise_id: str, title: str, html_file: str, adoc_file: str, public: bool, grader: str,
grading_file: str, container_image: str, max_time_sec: int, max_mem_mb: int,
asset_files: list, executors: list):
html = util.get_optional_file_content(html_file)
adoc = util.get_optional_file_content(adoc_file)
body = {
'title': title,
'text_html': html,
'text_adoc': adoc,
'public': public,
'grader_type': grader.upper(),
'container_image': container_image,
'max_time_sec': max_time_sec,
'max_mem_mb': max_mem_mb,
}
if grading_file is not None:
grading_script = util.get_file_content(grading_file)
body["grading_script"] = grading_script
if len(asset_files) > 0:
assets = list(map(lambda filename: {'file_name': os.path.basename(filename),
'file_content': util.get_file_content(filename)},
asset_files)
)
body["assets"] = assets
if len(executors) > 0:
executors = list(map(lambda e: {'executor_id': e}, executors))
body["executors"] = executors
resp = requests.put(conf.EMS_BASE_URL + '/exercises/' + exercise_id, json=body, headers=util.get_token_header())
print(resp.status_code)
def removeDecommissionedMachine(slaveName=None):
'''
Destroy decommissioned machines from the cluster and removes traces from excludes and slaves file
INPUT: String slaveName (optional)
OUTPUT: boolean (True if successful, False otherwise)
'''
util.debug_print('calling downsize removeDecommissionedMachine()')
# if not set, then get from excludes list
if slaveName is None:
util.debug_print('not slaveName passed as parameter')
# get the excludes file from master
excludes_file_content = util.get_file_content(
config.DEFAULT_DESTINATION_EXCLUDES_FILENAME)
# no magic, just get last one
if len(excludes_file_content) > 0:
slaveName = excludes_file_content[-1].strip()
else:
util.debug_print(
'no slavename passed in as argument AND we got empty slaves file!'
)
return False
# remove that slavename from excludes
remove_line = slaveName + "\n"
util.debug_print('removing from excludes file the line: ' + remove_line)
update_excludes = util.updateFile('excludes', remove_line, addLine=False)
# remove that name from slaves file
util.debug_print('removing from slaves file the line: ' + str(remove_line))
update_slaves = util.updateFile('slaves', remove_line, addLine=False)
# get vmid from slaveName
vmid = util.get_vm_id_by_name(slaveName)
# NOW deestroy vm
util.debug_print('Now we will be trying to destroy the machine with ID: ' +
str(vmid))
result = api.destroyVirtualMachine({'id': vmid})
util.debug_print(
'waiting for the destroyed machine to be finished being destroyed')
waitResult = util.waitForAsync(result.get('jobid'))
# since we destroyed the vm, we can remove from master's /etc/hosts file
hosts = util.get_file_content(config.DEFAULT_DESTINATION_HOSTS_FILENAME)
checker = re.compile('.*' + slaveName + '\n')
to_be_removed_hosts_line = [
line for line in hosts if checker.match(line) is not None
]
util.debug_print('remove line:' + str(to_be_removed_hosts_line) +
' from /etc/hosts file')
util.updateFile('hosts', to_be_removed_hosts_line[0], addLine=False)
util.debug_print('Done destroying VM.')
return True
def decommission(also_stop_vm=True):
'''
This function basically copies slave names from slaves list to excludes list and run refresh scripts
Input: None
Output: None
'''
util.debug_print('Trying to decommission')
# get all slave names in slaves file
all_slave_names = map(
str.strip,
util.get_file_content(config.DEFAULT_DESTINATION_SLAVES_FILENAME))
util.debug_print('all_slave_names:')
util.debug_print(all_slave_names)
# get excludes content from master
excludes_list = map(
str.strip,
util.get_file_content(config.DEFAULT_DESTINATION_EXCLUDES_FILENAME))
util.debug_print('current excludes list:')
util.debug_print(excludes_list)
# basic sanity check to see if we should try to decommission
remaining_slaves = len(all_slave_names) - len(excludes_list)
if remaining_slaves <= config.MINIMUM_DATANODE_SIZE:
util.debug_print('We have reached the minimum cluster size of ' +
str(remaining_slaves) + ', skipping decomissioning.')
return False
# ok, now we know we can remove some
removable_slaves = list(set(all_slave_names) - set(excludes_list))
max_name = get_max_slavename(removable_slaves, return_all=False)
util.debug_print('next slavename to remove is: ' + max_name)
# ok, now we have the slave we want to decommission, update the excludes file
newLine = max_name + "\n"
util.updateFile('excludes', newLine)
# run commands on the master that will output make decommission happen
ssh = SSHWrapper.SSHWrapper(config.MASTER_IP)
util.debug_print('trying to hdfs dfsadmin -refreshNodes')
outmsg, errmsg = ssh.sudo_command(
'sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -refreshNodes"'
)
util.debug_print('trying to yarn rmadmin -refreshNodes')
outmsg, errmsg = ssh.sudo_command(
'sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/yarn rmadmin -refreshNodes"'
)
if also_stop_vm:
stopDecommissionedMachine(max_name)
def removeDecommissionedMachine(slaveName = None):
'''
Destroy decommissioned machines from the cluster and removes traces from excludes and slaves file
INPUT: String slaveName (optional)
OUTPUT: boolean (True if successful, False otherwise)
'''
util.debug_print('calling downsize removeDecommissionedMachine()')
# if not set, then get from excludes list
if slaveName is None:
util.debug_print('not slaveName passed as parameter')
# get the excludes file from master
excludes_file_content = util.get_file_content(config.DEFAULT_DESTINATION_EXCLUDES_FILENAME)
# no magic, just get last one
if len(excludes_file_content) > 0:
slaveName = excludes_file_content[-1].strip()
else:
util.debug_print('no slavename passed in as argument AND we got empty slaves file!')
return False
# remove that slavename from excludes
remove_line = slaveName + "\n"
util.debug_print('removing from excludes file the line: ' + remove_line)
update_excludes = util.updateFile('excludes', remove_line, addLine = False)
# remove that name from slaves file
util.debug_print('removing from slaves file the line: ' + str(remove_line))
update_slaves = util.updateFile('slaves', remove_line, addLine = False)
# get vmid from slaveName
vmid = util.get_vm_id_by_name(slaveName)
# NOW deestroy vm
util.debug_print('Now we will be trying to destroy the machine with ID: ' + str(vmid))
result = api.destroyVirtualMachine({'id': vmid})
util.debug_print('waiting for the destroyed machine to be finished being destroyed')
waitResult = util.waitForAsync(result.get('jobid'))
# since we destroyed the vm, we can remove from master's /etc/hosts file
hosts = util.get_file_content(config.DEFAULT_DESTINATION_HOSTS_FILENAME)
checker = re.compile('.*' + slaveName + '\n')
to_be_removed_hosts_line = [line for line in hosts if checker.match(line) is not None]
util.debug_print('remove line:' + str(to_be_removed_hosts_line) + ' from /etc/hosts file')
util.updateFile('hosts', to_be_removed_hosts_line[0], addLine = False)
util.debug_print('Done destroying VM.')
return True
def generate_codeforces(self, code, file, out_name):
m = datamap.DataMap(util.get_file_content(util.get_map(file)))
self.m = m
g = Digraph('stones', encoding='utf-8')
for n in m.nodes:
if n.is_root:
count = self.get_module_problem_count(m)
label = "%s(%s)" % (n.name, str(count))
# 根节点
g.node(name=n.name, label=label, style='filled', target="_parent", href="https://codeforces.com/problemset",
fontsize='14',
fillcolor="orangered", color='lightgrey', fontcolor="white", fontname="Microsoft YaHei", shape='box')
else:
# 普通模块节点
label = "%s(%s)" % (n.name, str(len(n.problems)))
g.node(name=n.name, label=label, style='filled', fillcolor="lightslategray", color='lightgrey',
fontsize='12',
fontcolor="white", fontname="Microsoft YaHei", shape='box')
g.edge(n.parent, n.name, color=theme.color_arrow)
# add problem
last = ""
for p in n.problems:
problem = code.get_db_problem(p.id, False)
if not problem:
print("problem not exist "+p.id)
continue
title = problem['name']
level = code.get_level(problem)
idstr = str(p.id)
title = idstr+". "+title
color = "lightgrey"
if level == "Easy":
color = "greenyellow"
elif level == "Medium":
color = "orange"
elif level == "Hard":
color = "red"
else:
print("unknown level:", level)
# 题目节点
href = "https://codeforces.com/problemset/problem/%d/%s" % (problem['contestId'], problem['index'])
g.node(name=idstr, label=title, target="_parent", href=href,
color=color, fontname="Microsoft YaHei", fontsize='12', shape='box')
if len(last) > 0:
g.edge(last, idstr, color=theme.color_arrow)
else:
g.edge(n.name, idstr, color=theme.color_arrow)
last = idstr
g.format = 'svg'
g.render(filename=util.get_images(out_name))
os.remove(util.get_images(out_name))
self.post_process(util.get_images(out_name)+".svg")
def show_file(file_name, error="", conf=""):
file_dict = util.get_file_dict(file_name)
if file_dict["type"] in ("img", "pdf"):
util.copy_file_to_tmp(file_name)
template_data = {"file": file_dict}
else:
template_data = {"file": file_dict, "file_content": util.get_file_content(file_name)}
return render_template("file.html", error=error, conf=conf, **template_data)
def generate_leetcode(self, leet, file, slug, out_name):
c = util.get_file_content(util.get_map(file))
m = datamap.DataMap(c)
self.m = m
g = Digraph('stones', encoding='utf-8')
for n in m.nodes:
if n.is_root:
count = self.get_module_problem_count(m)
label = "%s(%s)" % (n.name, str(count))
# 根节点
g.node(name=n.name, label=label, style='filled', target="_parent", href="https://leetcode-cn.com/tag/"+slug,
fontsize='14',
fillcolor="orangered", color='lightgrey', fontcolor="white", fontname="Microsoft YaHei", shape='box')
else:
# 普通模块节点
label = "%s(%s)" % (n.name, str(len(n.problems)))
g.node(name=n.name, label=label, style='filled', fillcolor="lightslategray", color='lightgrey',
fontsize='12',
fontcolor="white", fontname="Microsoft YaHei", shape='box')
g.edge(n.parent, n.name, color=theme.color_arrow)
# add problem
last = ""
for p in n.problems:
title = leet.get_title(p.id)
level = leet.get_level(p.id)
problem = leet.get_problem(p.id)
idstr = str(p.id)
title = idstr+". "+title
color = "lightgrey"
if level == "Easy":
color = "greenyellow"
elif level == "Medium":
color = "orange"
elif level == "Hard":
color = "red"
else:
print("unknown level:", level)
continue
slug = problem['data']['question']['questionTitleSlug']
# 题目节点
g.node(name=idstr, label=title, target="_parent", href="https://leetcode-cn.com/problems/"+slug,
color=color, fontname="Microsoft YaHei", fontsize='12', shape='box')
if len(last) > 0:
g.edge(last, idstr, color=theme.color_arrow)
else:
g.edge(n.name, idstr, color=theme.color_arrow)
last = idstr
g.format = 'svg'
g.render(filename=util.get_images(out_name))
os.remove(util.get_images(out_name))
self.post_process(util.get_images(out_name)+".svg")
def decommission(also_stop_vm = True):
'''
This function basically copies slave names from slaves list to excludes list and run refresh scripts
Input: None
Output: None
'''
util.debug_print('Trying to decommission')
# get all slave names in slaves file
all_slave_names = map(str.strip, util.get_file_content(config.DEFAULT_DESTINATION_SLAVES_FILENAME))
util.debug_print('all_slave_names:')
util.debug_print(all_slave_names)
# get excludes content from master
excludes_list = map(str.strip, util.get_file_content(config.DEFAULT_DESTINATION_EXCLUDES_FILENAME))
util.debug_print('current excludes list:')
util.debug_print(excludes_list)
# basic sanity check to see if we should try to decommission
remaining_slaves = len(all_slave_names) - len(excludes_list)
if remaining_slaves <= config.MINIMUM_DATANODE_SIZE:
util.debug_print('We have reached the minimum cluster size of ' + str(remaining_slaves) + ', skipping decomissioning.')
return False
# ok, now we know we can remove some
removable_slaves = list(set(all_slave_names) - set(excludes_list))
max_name = get_max_slavename(removable_slaves, return_all=False)
util.debug_print('next slavename to remove is: ' + max_name)
# ok, now we have the slave we want to decommission, update the excludes file
newLine = max_name + "\n"
util.updateFile('excludes', newLine)
# run commands on the master that will output make decommission happen
ssh = SSHWrapper.SSHWrapper(config.MASTER_IP)
util.debug_print('trying to hdfs dfsadmin -refreshNodes')
outmsg, errmsg = ssh.sudo_command('sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -refreshNodes"')
util.debug_print('trying to yarn rmadmin -refreshNodes')
outmsg, errmsg = ssh.sudo_command('sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/yarn rmadmin -refreshNodes"')
if also_stop_vm:
stopDecommissionedMachine(max_name)
def autoass(course_id: str, exercise_id: str, solution_file: str):
solution = util.get_file_content(solution_file)
body = {
'solution': solution
}
resp = requests.post(
conf.EMS_BASE_URL + '/teacher/courses/' + course_id + '/exercises/' + exercise_id + '/autoassess', json=body,
headers=util.get_token_header())
print(resp.status_code)
print(resp.json())
def leetcode_add_finish_icon(self, path):
c = util.get_file_content(path)
b = BeautifulSoup(c, "xml")
nodes = b.select("g.node")
graph = b.select_one("g.graph")
for n in nodes:
title = n.title.get_text()
if not title.isdigit():
continue
self.post_process_problem_node(graph, n)
content = b.prettify()
util.save_file_content(path, content)
def add_finish_icon(self, path):
c = util.get_file_content(path)
b = BeautifulSoup(c, "xml")
svg = b.select_one("svg")
self.resize_svg(svg)
nodes = b.select("g.node")
graph = b.select_one("g.graph")
for n in nodes:
title = n.title.get_text()
if not self.is_valid_title(title):
continue
self.post_process_problem_node(graph, n)
content = b.prettify()
util.save_file_content(path, content)
def stopDecommissionedMachine(slaveName=None):
'''
Checks whether decommissioning completed, then stops the vm
Input String slavename
Output: None
'''
util.debug_print('calling on downsize stopDecommissionedMachine()')
if slaveName is None:
util.debug_print('not slaveName passed as parameter')
# get the excludes file from master
excludes_file_content = util.get_file_content(
config.DEFAULT_DESTINATION_EXCLUDES_FILENAME)
# no magic, just get last one
if len(excludes_file_content) > 0:
slaveName = excludes_file_content[-1].strip()
else:
util.debug_print(
'no slavename passed in as arument AND we got empty slaves file!'
)
return False
vmid = util.get_vm_id_by_name(slaveName)
# connect to master
ssh = SSHWrapper.SSHWrapper(config.MASTER_IP)
# get status
util.debug_print('Trying to get report on status of machines... ')
outmsg, errmsg = ssh.sudo_command(
'sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -report"'
)
# find the section for the slave we are interested in
# eg line is "Name: 199.60.17.186:50010 (dlw-Slave71)"
checker = re.compile(config.REPORT_DATANODE_STATUS_STARTING_REGEX +
str(slaveName) + '\)')
util.debug_print(
'starting while loop to check for status of VM is done decommissioning'
)
while True:
# get line for checking machine status
line = ''
commission_line = 0
# ok, I admin it might not be the best to put this here as the report MIGHT not chagne, but who knows... slower but safer for now
for line in outmsg:
matchobj = checker.match(line)
if matchobj:
util.debug_print('found the line! it is: ' + str(line))
commission_line = outmsg.index(line) + 2
break
line = outmsg[commission_line]
util.debug_print('on line: ' + str(commission_line) +
' status of decommissioning machine is: ' + str(line))
if line.find('Decommissioned') > -1:
util.debug_print(
'VM is finally decommissioned...., trying to stop VM now')
result = api.stopVirtualMachine({'id': vmid})
waitResult = util.waitForAsync(result.get('jobid'))
if waitResult != True: # whoops something went wrong!
return waitResult
# let's get out of here!
util.debug_print(
'DONE, we waited for it to finish decomissioning, then we stopped the VM! '
)
break
# ok, not decommissioned yet, so callthe ssh command again!
util.debug_print(
'checking again inside forever WHileTrue loop, as it is not in decomissioned state.'
)
outmsg, errmsg = ssh.sudo_command(
'sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -report"'
)
return True
def recommission(slaveName = None):
'''
This function recommissions, aka removes from excludes list and runs correct script to add back to node
Input: None
Output: None
'''
util.debug_print('calling upsize recommission()')
# if not set, then get from excludes list
if slaveName is None:
util.debug_print('not slaveName passed as parameter')
# get the excludes file from master
excludes_file_content = util.get_file_content(config.DEFAULT_DESTINATION_EXCLUDES_FILENAME)
# no magic, just get last one
if len(excludes_file_content) > 0:
slaveName = excludes_file_content[-1].strip()
else:
util.debug_print('no slavename passed in as argument AND we got empty slaves file!')
return False
# remove that slavename from excludes
remove_line = slaveName + "\n"
util.debug_print('removing from excludes file the line: ' + remove_line)
update_excludes = util.updateFile('excludes', remove_line, addLine = False)
# confirm if VM is running or stopped or whatever
vmid = util.get_vm_id_by_name(slaveName)
raw_result = api.listVirtualMachines({'id':vmid})
result = raw_result.get('virtualmachine')[0]
ipaddr = result.get('nic')[0].get('ipaddress')
while True:
current_state = result.get('state')
if current_state == 'Running':
util.debug_print('Machine is currently running')
util.debug_print('trying to hdfs dfsadmin -refreshNodes')
ssh = SSHWrapper.SSHWrapper(config.MASTER_IP)
outmsg, errmsg = ssh.sudo_command('sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -refreshNodes"')
util.debug_print('trying to yarn rmadmin -refreshNodes')
outmsg, errmsg = ssh.sudo_command('sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/yarn rmadmin -refreshNodes"')
break
elif current_state == 'Stopped':
util.debug_print('Machine is currently Stopped')
# start up machine and wait till it finishes starting up
util.debug_print('Trying to start VM')
result = api.startVirtualMachine({'id': vmid})
# now we wait for the async deployVirtualMachine() to finsih
waitResult = util.waitForAsync(result.get('jobid'))
if waitResult != True:
# whoops something went wrong!
return waitResult
# SSHWrapper checks and loops and waits for connection so let's just use it for testing that
'''
WHY DOES RESULT NOT GET ANYTHING! INEED THE IP OF THE DEVICE! MAYBE GET IT EARLIER?
'''
sshWaiting = SSHWrapper.SSHWrapper(ipaddr)
util.debug_print('trying to start-dfs.sh')
ssh = SSHWrapper.SSHWrapper(config.MASTER_IP)
ssh.sudo_command('sudo -S su hduser -c "bash /home/hduser/hadoop-2.7.0/sbin/start-dfs.sh"')
util.debug_print('trying to run start-yarn.sh')
ssh.sudo_command('sudo -S su hduser -c "bash /home/hduser/hadoop-2.7.0/sbin/start-yarn.sh"')
util.debug_print('trying to hdfs dfsadmin -refreshNodes')
ssh.sudo_command('sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -refreshNodes"')
util.debug_print('trying to yarn rmadmin -refreshNodes')
ssh.sudo_command('sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/yarn rmadmin -refreshNodes"')
break
elif current_state == 'Stopping' or current_state == 'Starting':
util.debug_print('OK, currently changing state, let us just call listVirtualMachines again and see.')
raw_result = api.listVirtualMachines({'id':vmid})
result = raw_result.get('virtualmachine')[0]
else:
# something went wrong here!
util.debug_print('ok, it is in an unexpected state: ' + str(current_state))
return False
}
util.debug(body)
resp = requests.put(util.AAS_BASE_URL + '/exercises/' + ex_id,
json=body,
headers=util.get_required_headers())
print(resp.status_code)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--id', required=True)
parser.add_argument('--grading-file', required=True)
parser.add_argument('--image-name', required=True)
parser.add_argument('--max-time', required=True, type=int)
parser.add_argument('--max-mem', required=True, type=int)
parser.add_argument('--asset-files', nargs='*', default=[])
parser.add_argument('--executors', nargs='+', required=True)
args = parser.parse_args()
grading_script = util.get_file_content(args.grading_file)
assets = list(
map(
lambda filename: {
'file_name': os.path.basename(filename),
'file_content': util.get_file_content(filename)
}, args.asset_files))
update_exercise(args.id, grading_script, args.image_name, args.max_time,
args.max_mem, assets, args.executors)
def stopDecommissionedMachine(slaveName = None):
'''
Checks whether decommissioning completed, then stops the vm
Input String slavename
Output: None
'''
util.debug_print('calling on downsize stopDecommissionedMachine()')
if slaveName is None:
util.debug_print('not slaveName passed as parameter')
# get the excludes file from master
excludes_file_content = util.get_file_content(config.DEFAULT_DESTINATION_EXCLUDES_FILENAME)
# no magic, just get last one
if len(excludes_file_content) > 0:
slaveName = excludes_file_content[-1].strip()
else:
util.debug_print('no slavename passed in as arument AND we got empty slaves file!')
return False
vmid = util.get_vm_id_by_name(slaveName)
# connect to master
ssh = SSHWrapper.SSHWrapper(config.MASTER_IP)
# get status
util.debug_print('Trying to get report on status of machines... ')
outmsg, errmsg = ssh.sudo_command('sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -report"')
# find the section for the slave we are interested in
# eg line is "Name: 199.60.17.186:50010 (dlw-Slave71)"
checker = re.compile(config.REPORT_DATANODE_STATUS_STARTING_REGEX + str(slaveName) + '\)')
util.debug_print('starting while loop to check for status of VM is done decommissioning')
while True:
# get line for checking machine status
line = ''
commission_line = 0
# ok, I admin it might not be the best to put this here as the report MIGHT not chagne, but who knows... slower but safer for now
for line in outmsg:
matchobj = checker.match(line)
if matchobj:
util.debug_print('found the line! it is: ' + str(line))
commission_line = outmsg.index(line) + 2
break;
line = outmsg[commission_line]
util.debug_print('on line: ' + str(commission_line) + ' status of decommissioning machine is: ' + str(line))
if line.find('Decommissioned') > -1:
util.debug_print('VM is finally decommissioned...., trying to stop VM now')
result = api.stopVirtualMachine({'id': vmid})
waitResult = util.waitForAsync(result.get('jobid'))
if waitResult != True: # whoops something went wrong!
return waitResult
# let's get out of here!
util.debug_print('DONE, we waited for it to finish decomissioning, then we stopped the VM! ')
break;
# ok, not decommissioned yet, so callthe ssh command again!
util.debug_print('checking again inside forever WHileTrue loop, as it is not in decomissioned state.')
outmsg, errmsg = ssh.sudo_command('sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -report"')
return True
import argparse
import requests
import util
def update_exercise(ex_id, ex_title, ex_text, ex_public):
body = {'title': ex_title, 'text_html': ex_text, 'public': ex_public}
util.debug(body)
resp = requests.put(util.EMS_BASE_URL + '/teacher/exercises/' + ex_id,
json=body,
headers=util.get_required_headers())
print(resp.status_code)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--id', required=True)
parser.add_argument('--title', required=True)
parser.add_argument('--text-file')
parser.add_argument('--public', action='store_true')
args = parser.parse_args()
if args.text_file is None:
exercise_text = None
else:
exercise_text = util.get_file_content(args.text_file)
update_exercise(args.id, args.title, exercise_text, args.public)
def recommission(slaveName=None):
'''
This function recommissions, aka removes from excludes list and runs correct script to add back to node
Input: None
Output: None
'''
util.debug_print('calling upsize recommission()')
# if not set, then get from excludes list
if slaveName is None:
util.debug_print('not slaveName passed as parameter')
# get the excludes file from master
excludes_file_content = util.get_file_content(
config.DEFAULT_DESTINATION_EXCLUDES_FILENAME)
# no magic, just get last one
if len(excludes_file_content) > 0:
slaveName = excludes_file_content[-1].strip()
else:
util.debug_print(
'no slavename passed in as argument AND we got empty slaves file!'
)
return False
# remove that slavename from excludes
remove_line = slaveName + "\n"
util.debug_print('removing from excludes file the line: ' + remove_line)
update_excludes = util.updateFile('excludes', remove_line, addLine=False)
# confirm if VM is running or stopped or whatever
vmid = util.get_vm_id_by_name(slaveName)
raw_result = api.listVirtualMachines({'id': vmid})
result = raw_result.get('virtualmachine')[0]
ipaddr = result.get('nic')[0].get('ipaddress')
while True:
current_state = result.get('state')
if current_state == 'Running':
util.debug_print('Machine is currently running')
util.debug_print('trying to hdfs dfsadmin -refreshNodes')
ssh = SSHWrapper.SSHWrapper(config.MASTER_IP)
outmsg, errmsg = ssh.sudo_command(
'sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -refreshNodes"'
)
util.debug_print('trying to yarn rmadmin -refreshNodes')
outmsg, errmsg = ssh.sudo_command(
'sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/yarn rmadmin -refreshNodes"'
)
break
elif current_state == 'Stopped':
util.debug_print('Machine is currently Stopped')
# start up machine and wait till it finishes starting up
util.debug_print('Trying to start VM')
result = api.startVirtualMachine({'id': vmid})
# now we wait for the async deployVirtualMachine() to finsih
waitResult = util.waitForAsync(result.get('jobid'))
if waitResult != True:
# whoops something went wrong!
return waitResult
# SSHWrapper checks and loops and waits for connection so let's just use it for testing that
'''
WHY DOES RESULT NOT GET ANYTHING! INEED THE IP OF THE DEVICE! MAYBE GET IT EARLIER?
'''
sshWaiting = SSHWrapper.SSHWrapper(ipaddr)
util.debug_print('trying to start-dfs.sh')
ssh = SSHWrapper.SSHWrapper(config.MASTER_IP)
ssh.sudo_command(
'sudo -S su hduser -c "bash /home/hduser/hadoop-2.7.0/sbin/start-dfs.sh"'
)
util.debug_print('trying to run start-yarn.sh')
ssh.sudo_command(
'sudo -S su hduser -c "bash /home/hduser/hadoop-2.7.0/sbin/start-yarn.sh"'
)
util.debug_print('trying to hdfs dfsadmin -refreshNodes')
ssh.sudo_command(
'sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/hdfs dfsadmin -refreshNodes"'
)
util.debug_print('trying to yarn rmadmin -refreshNodes')
ssh.sudo_command(
'sudo -S su hduser -c "/home/hduser/hadoop-2.7.0/bin/yarn rmadmin -refreshNodes"'
)
break
elif current_state == 'Stopping' or current_state == 'Starting':
util.debug_print(
'OK, currently changing state, let us just call listVirtualMachines again and see.'
)
raw_result = api.listVirtualMachines({'id': vmid})
result = raw_result.get('virtualmachine')[0]
else:
# something went wrong here!
util.debug_print('ok, it is in an unexpected state: ' +
str(current_state))
return False