def image_cluster(self, cluster, commit_id, build_url, user_email, hyp_type,
hyp_version, svm_version, build_type):
if build_url:
request = ClientPhoenixRequest(
user_email=user_email, cluster=cluster, commit_id=commit_id,
svm_installer_url=build_url,
hyp_type=hyp_type, hyp_version=hyp_version,
svm_version=svm_version, build_type=build_type)
else:
build_url = self.get_svm_installer(commit_id, svm_version, build_type)
if not build_url:
INFO("Could not get build url to phoenix")
return False
request = ClientPhoenixRequest(
user_email=user_email, cluster=cluster, hyp_type=hyp_type,
hyp_version=hyp_version, svm_version=svm_version,
build_type=build_type, svm_installer_url=build_url)
phoenix = PhoenixClient()
INFO("Imaging cluster %s to build %s" % (cluster, svm_version))
if not phoenix.image_cluster(request):
return False
return True
def get_svm_installer(self, commit_id, branch, build_type):
"""
Get the build location for th given commit id and branch.
"""
INFO("Getting installer for commit %s" % commit_id)
filer = FLAGS.lab
jita=JitaRest()
build_url = jita.get_build_url(commit_id, build_type, filer)
if build_url:
INFO("Found build %s" % build_url)
return build_url
INFO("Checking if build is in progress")
build_task_id = jita.check_build_in_progress(commit_id, branch, build_type,
filer)
if build_task_id:
status = jita.get_build_task(build_task_id).get_status()
if not build_task_id or status in ["failed", "aborted", "skipped"]:
INFO("Could not find an existing build. Hence requesting build")
res = jita.request_build(commit_id, branch, build_type, filer)
if res.get_status():
build_task_id = res.get_build_task_id()
else:
ERROR("Build request for commit id %s failed" % commit_id)
return False
if self.wait_for_build_to_complete(build_task_id):
return jita.get_build_url(commit_id, build_type, filer)
return False
def execute_command(uvm, cmd, fatal_on_error=False, username="******",
password="******", timeout_secs=180):
"""
Execute command 'cmd' on the Windows VM with the supplied credentials
'username' and 'password' (which is needed to access the Windows VM). If
'fatal_on_error' is true, then we fatal if the out returns a non-zero
return value or contains anything in the stderr stream. Returns a tuple
consisting of the return value, stdout and stderr.
"""
remote_shell = BasicWsmanRemoteShell(uvm.ip(), username, password)
try:
ret, stdout, stderr = remote_shell.posh_execute(
cmd, timeout_secs=timeout_secs)
INFO("cmd returned %s and %s and %s" %(ret,stdout,stderr))
stdout = stdout.strip()
stderr = stderr.strip()
if FLAGS.agave_verbose:
INFO("Result of executing command %s: ret %s, stdout %s, stderr %s"
% (cmd, ret, stdout, stderr))
if fatal_on_error and (ret != 0 or stderr.strip()):
FATAL("Execution of %s on VM with IP %s failed with ret %s, "
"stdout %s and stderr %s"
% (cmd, uvm.ip(), ret, stdout, stderr))
return ret, stdout, stderr
except WsmanRemoteShellException as wsre:
if FLAGS.agave_verbose:
INFO("Unable to execute command %s due to %s" % (cmd, str(wsre)))
if fatal_on_error:
FATAL("Execution of %s on VM %s failed due to %s"
% (cmd, uvm.ip(), str(wsre)))
return 255, '', str(wsre)
def run_test_on_jita(
self, cluster, commit_id=None, build_url=None, job_ids_list=None,
index=None, hypervisor=None, hypervisor_version=None, branch=None,
input_type="build_url", build_type="opt", test=None, username=None,
test_upgrade=None, base_commit=None):
# NEEDS CHANGE
if not hypervisor:
hypervisor = FLAGS.hypervisor
if not hypervisor_version:
hypervisor_version = FLAGS.hypervisor_version
if not branch:
branch = FLAGS.branch
if not test:
test = FLAGS.test_name
if not username:
username = USERNAME
test_framework = FLAGS.test_framework
if test_upgrade is None:
test_upgrade = FLAGS.upgrade_test
base_commit=FLAGS.upgrade_base_commit
if test_upgrade:
phoenix_commit = base_commit
else:
phoenix_commit = commit_id
if not self.image_cluster( cluster, phoenix_commit, build_url, USERNAME,
hypervisor, hypervisor_version, branch,
build_type):
job_ids_list[index] = False
ERROR("Imaging failed on cluster %s" % cluster)
return False
jita = JitaRest()
jita_test = jita.get_test(test,test_framework)
if test_upgrade:
svm_installer_url = self.get_svm_installer(commit_id, branch, build_type)
INFO("Found build %s " % svm_installer_url)
if not svm_installer_url:
return False
args_map = {"target_rel" : svm_installer_url}
else:
args_map = None
INFO("Going to submit task for commit id: %s" % commit_id)
response_obj = jita.submit_task(
cluster, commit_id, branch, [jita_test.get_id()],
test_framework=test_framework, label="auto_bisector", args_map=args_map)
if response_obj.is_success():
job_ids_list[index] = response_obj.get_task_id()
return True
else:
ERROR("Task submit failed: %s" % response_obj.get_message())
return False
def wait_for_jita_test_results(self, job_info, timeout=18000,
poll_period=600):
# Sleep for 10 minutes before starting polling for results.
expired_time = 0
unfinished_jobs = job_info.values()
jita = JitaRest()
while len(unfinished_jobs) and expired_time < timeout:
for job_id in job_info.values():
if self.did_jita_task_finish(job_id):
if job_id in unfinished_jobs:
unfinished_jobs.remove(job_id)
time.sleep(poll_period)
expired_time += poll_period
for commit, job_id in job_info.iteritems():
response = jita.get_task(job_id)
if response.get_status() not in ["passed", "completed"]:
self.global_results_dict[commit] = False
else:
test_results = response.get_test_results()
#test_results = response.get_test_result_dict()
for test, result in test_results.test_result_dict.iteritems():
if result.get_status() != "Succeeded":
INFO("Test %s failed with status %s" % (test, result.get_status()))
self.global_results_dict[commit] = False
break
self.global_results_dict[commit] = True
return True
def wait_for_test_result(job_id, timeout=18000, poll_period=600):
"""
Wait for results from nucloud and then populate the results.
"""
expired_time = 0
nucloud = NuCloudREST()
while expired_time < timeout:
if did_nucloud_job_finish(job_id):
break
time.sleep(poll_period)
expired_time += poll_period
response = nucloud.get_jobs(job_id)
if response.get_status() != "passed" or response.get_status(
) != "completed":
return False
else:
response = response.get_test_results(job_id)
test_results = response.get_test_result_dict()
for test, result in test_results.iteritems():
if result.get_status() != "Succeeded":
INFO("Test %s failed with status %s" %
(test, result.get_status()))
return False
return True
def find_offending_commit(self, commits_file=None):
input_type = FLAGS.input_type
self.get_commit_search_list(commits_file, input_type)
if FLAGS.use_nucloud:
INFO("Using nucloud to run the tests")
offending_commit = self.binary_search_commit(
self.commits_dict.keys(), input_type)
else:
clusters = FLAGS.clusters.split(",")
INFO("Using Jita to run the tests on clusters %s" % clusters)
offending_commit = self.binary_search_commit_using_jita(
self.commits_dict.keys(), input_type, clusters)
if offending_commit == None:
INFO("The commit cannot be None! BUG.")
return offending_commit
def wait_for_test_results(self, job_info, timeout=18000, poll_period=600):
"""
Wait for results from nucloud and then populate the results.
"""
expired_time = 0
unfinished_jobs = job_info.values()
nucloud = NuCloudREST()
while len(unfinished_jobs) and expired_time < timeout:
for job_id in job_info.values():
if self.did_nucloud_job_finish(job_id):
if job_id in unfinished_jobs:
unfinished_jobs.remove(job_id)
time.sleep(poll_period)
expired_time += poll_period
for commit, job_id in job_info.iteritems():
response = nucloud.get_jobs(job_id)
if response.get_status() not in ["passed", "completed"]:
self.global_results_dict[commit] = False
else:
response = nucloud.get_test_results(job_id)
test_results = response.get_test_result_dict()
for test, result in test_results.iteritems():
if test != "send_test_results_email" and \
result.get_status() != "Succeeded":
INFO("Test %s failed with status %s" % (test, result.get_status()))
self.global_results_dict[commit] = False
break
self.global_results_dict[commit] = True
return True
def did_nucloud_job_finish(self, job_id):
"""
Helper function to check if a particular job has finished execution in
nucloud.
"""
nucloud = NuCloudREST()
response = nucloud.get_jobs(job_id)
status = response.get_status()
INFO("Response from nucloud for job id %s is %s" % (job_id, status))
if status not in ["passed", "killed", "failed", "completed"]:
return False
else:
return True
def run_test_on_nucloud(test=None,
username=None,
node_pool=None,
branch=None,
build_url=None,
build_type="opt",
preferred_hypervisor=None,
commit_id=None):
"""
Run the given test on nucloud and return the job id if submit succeeds.
"""
if not test:
test = FLAGS.test_name
if not username:
username = USERNAME
if not node_pool:
node_pool = NODE_POOL
if not branch:
branch = FLAGS.branch
if not preferred_hypervisor:
preferred_hypervisor = FLAGS.hypervisor
nucloud = NuCloudREST()
INFO("Submitting job to Nucloud for commit id %s" % commit_id)
response_obj = nucloud.submit_job(
[test],
username,
node_pool,
branch=branch,
build_url=build_url,
build_type=build_type,
preferred_hypervisor=preferred_hypervisor,
commit_id=commit_id,
skip_email_report=True)
print response_obj.get_message()
if response_obj.is_success():
return response_obj.get_job_id()
else:
ERROR("Failed to trigger job in nucloud %s " %
response_obj.get_message())
return False
def run_test_on_nucloud(
self, test=None, username=None, node_pool=None, branch=None, build_url=None,
build_type="opt", preferred_hypervisor=None, commit_id=None,
input_type="build_url", test_upgrade=None, image_commit=None):
"""
Run the given test on nucloud and return the job id if submit succeeds.
"""
if not test:
test=FLAGS.test_name
if not username:
username=USERNAME
if not node_pool:
node_pool=NODE_POOL
if not branch:
branch = FLAGS.branch
if not preferred_hypervisor:
preferred_hypervisor = FLAGS.hypervisor
if not test_upgrade:
test_upgrade = FLAGS.upgrade_test
image_commit = FLAGS.upgrade_base_commit
if input_type=="change_id":
change_id = commit_id
commit_id = None
else:
change_id = None
nucloud = NuCloudREST()
INFO("Submitting job to Nucloud for commit id %s" % commit_id)
response_obj = nucloud.submit_job(
[test], username, node_pool, branch=branch, build_url=build_url,
build_type=build_type, preferred_hypervisor=preferred_hypervisor,
commit_id=commit_id, gerrit_change_id=change_id,
test_upgrade=test_upgrade, image_commit=image_commit,
skip_email_report=True)
if response_obj.is_success():
return response_obj.get_job_id()
else:
return False
def get_commit_search_list(self, commits_file=None, input_type=None):
"""
Get the commits list from the file and create a dict with commit as the key
and build_url as the value. If the build_url isnt specified then the value
would be null
"""
if self.good_commit:
commits = self.get_commits_in_range(self.good_commit,
self.last_bad_commit)
for commit in commits:
self.commits_dict[commit] = None
else:
with open(commits_file) as f:
lines = f.read().splitlines()
for line in lines:
if not line:
continue
commit = line.split()[0]
if input_type == "build_url":
self.commits_dict[commit] = line.split()[1]
else:
self.commits_dict[commit] = None
INFO("Commits map: \n%s " % self.commits_dict)
def log_info(self, msg): """ See superclass for documentation. """ INFO(msg)
def transfer_powershell_module(uvm, out_dir, port,
username="******",
password="******"):
"""
Transfer Powershell modules to the VM at
${Env:ProgramFiles}\WindowsPowershell\Modules.
uvm: Windows VM for which the Powershell module needs to be transferred.
out_dir: Test runner output directory from where the files should be
copied.
port: Http server port from which the Windows VM will be able to pull the
modules.
username: Username of the account to access the Windows VM
password: Password for the above account.
Returns True on success, False on failure.
"""
hyperv_ps_modules_dir = (
"%s/%s" % (top_dir(), WindowsVMCommandUtil.WINDOWS_VM_MODULE_PATH))
agave_posh_modules = []
for (root, _, filenames) in os.walk(hyperv_ps_modules_dir):
for filename in filenames:
if filename.endswith(".psm1"):
module_path = os.path.join(root, filename)
agave_posh_modules.append(module_path)
for agave_posh_module in agave_posh_modules:
try:
module_basename = os.path.basename(agave_posh_module)
os.symlink(agave_posh_module, "%s/%s" % (out_dir, module_basename))
except OSError as ose:
if ose.errno != errno.EEXIST:
ERROR("Failed to create a symlink to the Hyper-V Agave PS module "
"in %s: %s" % (out_dir, str(ose)))
succeeded = True
for agave_posh_module in agave_posh_modules:
module_basename = os.path.basename(agave_posh_module)
module_name_no_extension = module_basename.rsplit(".psm1")[0]
remote_path = ("${Env:ProgramFiles}\WindowsPowershell\Modules\%s" %
module_name_no_extension)
INFO("Transferring the NutanixWindows Powershell modules to %s"
% uvm.ip())
# Determine the local IP that has reachability to this host.
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.connect((uvm.ip(), 80)) # Port number doesn't matter.
local_ip = sock.getsockname()[0]
sock.close()
install_posh_module_cmd = """
$ErrorActionPreference = "Stop"
$dest = New-Item -ItemType Directory -Path {remote_path} -Force
Invoke-WebRequest http://{ip}:{port}/output/{module_name} `
-OutFile $dest\\{module_name}""".format(remote_path=remote_path,
ip=local_ip, port=port,
module_name=module_basename)
if FLAGS.agave_verbose:
INFO("Transferring the %s Powershell module to %s using command: %s" %
(module_name_no_extension, uvm.ip(), install_posh_module_cmd))
ret, _, stderr = WindowsVMCommandUtil.execute_command(
uvm, install_posh_module_cmd, username=username, password=password)
if ret or stderr.strip():
ERROR("Failed transferring %s Powershell module to %s: %s" %
(module_name_no_extension, uvm.ip(), stderr))
succeeded = succeeded and False
return succeeded
def binary_search_commit(self, commit_search_list, input_type):
"""
This is the main function that implements the binary search flow through the
given commit_search_list.
Params:
commit_search_list: The list of gerrit change ids or commits to be
searched
test: test set name
node_pool: The node_pool to use to trigger the tests
input_type: String to indicate type of input for the commits
branch: The branch on which this commit appears
hypervisor: The hypervisor that needs to be used
"""
INFO("Below are the list of commits being searched:\n%s" %
pprint.pformat(commit_search_list))
if len(commit_search_list) == 0:
ERROR ("Length of commit_search_list is 0, this is probably a bug")
return None
elif len(commit_search_list) == 1:
INFO("Returning %s " % commit_search_list[0])
return commit_search_list[0]
elif len(commit_search_list) == 2:
if commit_search_list[0] in self.global_results_dict.keys():
if self.global_results_dict[commit_search_list[0]]:
return commit_search_list[1]
else:
return commit_search_list[0]
if commit_search_list[1] in self.global_results_dict.keys():
if self.global_results_dict[commit_search_list[1]]:
return commit_search_list[0]
# We reach here if we don't already have the test results for at least the
# first commit
response_map = {}
response_map[commit_search_list[0]] = self.run_test_on_nucloud(
build_url=self.commits_dict[commit_search_list[0]],
commit_id=commit_search_list[0], input_type=input_type)
self.wait_for_test_results(response_map)
if self.global_results_dict[commit_search_list[0]]:
return commit_search_list[1]
else:
return commit_search_list[0]
# Get the number of free clusters from the pool.
free_nodes = self.get_free_nodes_from_jarvis()
num_free_clusters = len(free_nodes)/3
INFO("free clusters in nucloud: %s " % num_free_clusters)
search_list_len = len(commit_search_list)
# If the number of free clusters is less than the number of commits,
# let us do a binary search.
if num_free_clusters < search_list_len:
mid_commit_index = search_list_len / 2
first_half_mid_commit_index = mid_commit_index / 2
second_half_mid_commit_index = (
mid_commit_index + 1 + search_list_len) / 2
index_list = [mid_commit_index, first_half_mid_commit_index,
second_half_mid_commit_index]
INFO("Commits selected for verification are: %s, %s and %s" % (
commit_search_list[mid_commit_index], commit_search_list[
first_half_mid_commit_index], commit_search_list[
second_half_mid_commit_index]))
response_map = {}
for index in index_list:
# If we already have the test result for this commit don't run the test
# again.
if commit_search_list[index] in self.global_results_dict.keys():
continue
response_map[commit_search_list[index]] = self.run_test_on_nucloud(
build_url=self.commits_dict[commit_search_list[index]],
commit_id=commit_search_list[index], input_type=input_type)
self.wait_for_test_results(response_map)
INFO("Results from the run are: %s" % self.global_results_dict)
# Based on the test result, call the function again.
if not self.global_results_dict[commit_search_list[
first_half_mid_commit_index]]:
INFO("Narrowing the search based on the results to commits between "
"%s and %s" % (commit_search_list[0], commit_search_list[
first_half_mid_commit_index]))
return self.binary_search_commit(commit_search_list[
0:(first_half_mid_commit_index+1)], input_type)
elif not self.global_results_dict[commit_search_list[mid_commit_index]]:
INFO("Narrowing the search based on the results to commits between "
"%s and %s" % (commit_search_list[first_half_mid_commit_index],
commit_search_list[mid_commit_index]))
return self.binary_search_commit(
commit_search_list[first_half_mid_commit_index:(mid_commit_index+1)],
input_type)
elif not self.global_results_dict[commit_search_list[
second_half_mid_commit_index]]:
INFO("Narrowing the search based on the results to commits between "
"%s and %s" % (commit_search_list[mid_commit_index],
commit_search_list[second_half_mid_commit_index]))
return self.binary_search_commit(
commit_search_list[mid_commit_index:(second_half_mid_commit_index+1)],
input_type)
else:
INFO("Narrowing the search based on the results to commits between "
"%s and %s" % (commit_search_list[second_half_mid_commit_index],
commit_search_list[-1]))
return self.binary_search_commit(
commit_search_list[second_half_mid_commit_index:], input_type)
else:
# We have enough clusters. Trigger all the runs in parallel.
response_map = {}
for commit in commit_search_list:
if commit in self.global_results_dict.keys():
continue
response_map[commit] = self.run_test_on_nucloud(
build_url=self.commits_dict[commit],
commit_id=commit, input_type=input_type)
self.wait_for_test_results(response_map)
INFO("Results from the run are: %s" % self.global_results_dict)
for commit in commit_search_list:
if not self.global_results_dict[commit]:
INFO("Returning the offending commit %s" % commit)
return commit
if flag:
commit_range.append(commit['commit_id'])
commit_range.reverse()
return commit_range
def main(argv):
try:
argv = FLAGS(argv)
except gflags.FlagsError, err:
print "%s\nUsage: %s ARGS\n%s" % (err, sys.argv[0], FLAGS)
sys.exit(1)
if not FLAGS.commits_file and not (
FLAGS.last_good_commit and FLAGS.latest_bad_commit):
print "Exiting! One of commits_file or last_good_commit and "
"latest_bad_commit is a required parameter."
sys.exit(1)
file_template = "auto_bisector_log_%d" % (int(time.time()))
bisect = Auto_Bisect()
bisect.setup_logging(file_template)
commits_file = FLAGS.commits_file
offending_commit = bisect.find_offending_commit(commits_file)
INFO("Found the offending commit: %s" %
offending_commit)
if __name__ == "__main__":
main(sys.argv)
def binary_search_commit_using_jita(self, commit_search_list, input_type,
clusters):
"""
This is the main function that implements the binary search flow through the
given commit_search_list.
Params:
commit_search_list: The list of gerrit change ids or commits to be
searched
input_type: String to indicate type of input for the commits
clusters: list of clusters to run the test on
"""
INFO("Below are the list of commits being searched:\n%s" %
pprint.pformat(commit_search_list))
if len(commit_search_list) == 0:
print "Length of commit_search_list is 0, this is probably a bug"
return None
elif len(commit_search_list) == 1:
INFO ("returning the only commit passed to the binary_search is %s " %
commit_search_list[0])
return commit_search_list[0]
elif len(commit_search_list) == 2:
if commit_search_list[0] in self.global_results_dict.keys():
if self.global_results_dict[commit_search_list[0]]:
return commit_search_list[1]
else:
return commit_search_list[0]
if commit_search_list[1] in self.global_results_dict.keys():
if self.global_results_dict[commit_search_list[1]]:
return commit_search_list[0]
else:
return commit_search_list[1]
# We reach here if we don't already have the test results for either of
# the commits.Run the test on jita.
response_map = {}
job_ids_list = [None]
self.run_test_on_jita(clusters[0], commit_search_list[0],
build_url=self.commits_dict[commit_search_list[0]],
job_ids_list=job_ids_list, index=0)
response_map[commit_search_list[0]] = job_ids_list[0]
self.wait_for_jita_test_results(response_map)
if self.global_results_dict[commit_search_list[0]]:
return commit_search_list[1]
else:
return commit_search_list[0]
search_list_len = len(commit_search_list)
# If there are less free clusters, let us do a binary search.
if len(clusters) < search_list_len and len(clusters) >= 3:
mid_commit_index = search_list_len / 2
first_half_mid_commit_index = mid_commit_index / 2
second_half_mid_commit_index = (
mid_commit_index + 1 + search_list_len) / 2
index_list = [mid_commit_index, first_half_mid_commit_index,
second_half_mid_commit_index]
response_map = {}
threads = [None] * search_list_len
job_ids_list = [None] * search_list_len
for index in index_list:
# If we already have the test result for this commit don't run the test
# again.
if commit_search_list[index] in self.global_results_dict.keys():
continue
threads[index] = threading.Thread(
target=self.run_test_on_jita, name="jita_thread",
args=(clusters.pop(), commit_search_list[index],
self.commits_dict[commit_search_list[index]], job_ids_list,
index))
threads[index].start()
for index in index_list:
threads[index].join()
if job_ids_list[index]:
return None
response_map[commit_search_list[index]] = job_ids_list[index]
self.wait_for_jita_test_results(response_map)
# Based on the test result, call the function again.
if not self.global_results_dict[commit_search_list[
first_half_mid_commit_index]]:
return self.binary_search_commit_using_jita(
commit_search_list[0:(first_half_mid_commit_index+1)],
input_type, clusters)
elif not self.global_results_dict[commit_search_list[mid_commit_index]]:
return self.binary_search_commit_using_jita(
commit_search_list[first_half_mid_commit_index:(mid_commit_index+1)],
test, username, node_pool, branch, hypervisor, input_type)
elif not self.global_results_dict[commit_search_list[
second_half_mid_commit_index]]:
return self.binary_search_commit_using_jita(
commit_search_list[mid_commit_index:(second_half_mid_commit_index+1)],
input_type, clusters)
else:
return self.binary_search_commit_using_jita(
commit_search_list[second_half_mid_commit_index:], input_type,
clusters)
elif len(clusters) >= search_list_len:
# We have enough clusters. Trigger all the runs in parallel.
response_map = {}
threads = []
job_ids_list = [None] * search_list_len
for index in xrange(len(commit_search_list)):
threads[index] = threading.Thread(
target=self.run_test_on_jita, name="jita_thread",
args=(clusters.pop(), commit_search_list[index],
self.commits_dict[commit_search_list[index]], job_ids_list,
index))
threads[index].start()
for index in index_list:
threads[index].join()
response_map[commit_search_list[index]] = job_ids_list[index]
self.wait_for_jita_test_results(response_map)
for commit in commit_search_list:
if not self.global_results_dict[commit]:
return commit
else:
# We just have one cluster. This is going to take forever.
mid_commit_index = search_list_len / 2
response_map = {}
job_ids_list = [None]
self.run_test_on_jita(
clusters[0], commit_search_list[mid_commit_index],
build_url=self.commits_dict[commit_search_list[mid_commit_index]],
job_ids_list=job_ids_list, index=0)
response_map[commit_search_list[mid_commit_index]] = job_ids_list[0]
self.wait_for_jita_test_results(response_map)
if not self.global_results_dict[commit_search_list[mid_commit_index]]:
self.binary_search_commit_using_jita(
commit_search_list[0:(mid_commit_index+1)], input_type, clusters)
else:
self.binary_search_commit_using_jita(
commit_search_list[mid_commit_index:], input_type, clusters)