def test_hs_builds(self, axis):
debug('enter')
job = HSNightlyBuild(jobname="trigger-nightly-builds",
jenkinsserver=self.servers["hs-jenkins"])
builds = job.bind_lastbuild(axis[0])
tests = {}
for label in builds:
if builds[label].result == 'FAILURE':
continue
test_nodes = self.get_nodes_by_label(label)
if test_nodes:
test_node = test_nodes[0]
#test_node = None
#builds[label].wait_for_replication(test_node.region)
test = TestJob(test_node, builds[label], label)
if test.start_hstest():
tests[label] = test
while True:
tests_running = False
for label in tests:
if tests[label].building:
print label + " test still running"
tests_running = True
if not tests_running:
break
time.sleep(2)
print "Tests executed on:"
for label in tests:
print label
self.builds = builds
self.tests = tests
def wait_for_replication(self, region):
if not region:
return
if self.replication_hash is None:
return
debug("Waiting for build replication")
self.api.wait('build-image', self.replication_hash)
def trigger_test(self, **params):
try:
super(TestJob, self).start(**params)
return True
except EnvironmentError, e:
debug("Failed triggering build: %r" % (e, ))
return False
def building(self):
debug("enter")
if self.joburl is None:
self.joburl = self.api.get_build_url(self.jobname, self.hash)
debug(self.joburl)
self.refresh(self.api.json_api_request(self.joburl, depth=0))
return super(JenkinsJob, self).building
def get_nodes_by_label(self, label):
nodes = []
for server in self.servers:
debug('Search in ' + server)
for node in self.servers[server].get_nodes_by_label(label):
nodes.append(node)
return nodes
def copy_and_overwrite(from_path, to_path, ignore_files=[]):
if os.path.exists(to_path):
shutil.rmtree(to_path)
try:
shutil.copytree(from_path, to_path, symlinks=True, ignore=shutil.ignore_patterns(*ignore_files))
except:
debug("Errors occured while copying artifact %s" % (from_path))
def clean_directory_by_age(element, extremity_date, exclude_list, lockfile='never-delete'):
filename = os.path.basename(element)
if filename in exclude_list:
debug(element + " excluded")
return
try:
if os.path.isdir(element):
files = os.listdir(element)
if lockfile in files:
debug("%s build is locked for deletion, lockfile:%s" % (element, lockfile))
return
for el in files:
element_abs = os.path.join(element, el)
clean_directory_by_age(element_abs, extremity_date, exclude_list, lockfile=lockfile)
if os.listdir(element) == []:
created_date = time.strftime("%Y/%m/%d", time.localtime(os.path.getmtime(element)))
if extremity_date > created_date:
debug(element + " is empty and being deleted")
shutil.rmtree(element)
else:
if os.path.exists(element):
created_date = time.strftime("%Y/%m/%d", time.localtime(os.path.getmtime(element)))
elif os.path.islink(element):
parent = os.path.dirname(element)
created_date = time.strftime("%Y/%m/%d", time.localtime(os.path.getmtime(parent)))
if extremity_date > created_date:
debug("File %s created on %s is being deleted" % (element, created_date))
if os.path.isfile(element):
os.remove(element)
elif os.path.islink(element):
os.unlink(element)
except Exception, e:
debug('Failed delete: ' + element + " Error:" + str(e))
def start_hstest(self):
try:
debug('enter')
super(TestJob, self).start(NODE_LABEL=self.label,
NIGHTLY_BUILD_NUMBER=self.build.number)
return True
except EnvironmentError, e:
debug("Failed triggering build: %r" % (e, ))
return False
def bind_lastbuild(self, matrixlabel=None):
debug('enter')
if matrixlabel:
matrixlabel = 'BASELINE=%s/' % matrixlabel
job = self.api.job('%s/%s' % (self.jobname, matrixlabel))
self.joburl = job['lastBuild']['url'].replace(matrixlabel, '')
else:
job = self.api.job(self.jobname)
self.joburl = job['lastBuild']['url']
print self.joburl
def pull_completed_jobs(self, axislist=[]):
debug("enter")
matrixbuilds = self.api.matrix_job_status(self.joburl)
self.refresh(self.api.json_api_request(self.joburl, depth=0))
newly_completed_builds = {}
for axis in matrixbuilds:
if axis not in axislist:
continue
if matrixbuilds[axis] is not None:
if not matrixbuilds[
axis].building and axis not in self.completed_builds:
buildjob = BuildJob(jenkinsserver=self.server,
builddata=matrixbuilds[axis])
buildjob.load_json_data()
self.completed_builds[axis] = buildjob
newly_completed_builds[axis] = buildjob
return newly_completed_builds
def test_matrix_job(self, axis):
debug('enter')
job = MatrixJob(jobname="build-nightly-system",
jenkinsserver=self.servers["hs-jenkins"])
job.bind_lastbuild(axis[0])
tests = {}
building = True
self.all_builds = {}
while True:
debug('Still building')
building = job.building
builds = job.pull_completed_jobs(axis)
for label in builds:
if builds[label].result == 'FAILURE':
continue
test_nodes = self.get_nodes_by_label(label)
if test_nodes:
test_node = test_nodes[0]
#test_node = None
#builds[label].wait_for_replication(test_node.region)
test = TestJob(test_node, builds[label], label)
if test.start_hstest():
tests[label] = test
if not building:
break
time.sleep(30)
while True:
tests_running = False
for label in tests:
if tests[label].building:
print label + " test still running"
tests_running = True
if not tests_running:
break
time.sleep(2)
print "Builds executed on:"
for label in job.completed_builds:
print label
print "Tests executed on:"
for label in tests:
print label
self.builds = job.completed_builds
self.tests = tests
def command(cmd, shell=False, env=None):
call_env = os.environ.copy()
if env is not None:
call_env.update(env)
debug("executing")
retval = 0
if os.environ.get('INVADE_BUILD_LOG',
'false') == 'true' and 'build' in " ".join(cmd):
file = open("build.log", "w")
retval = subprocess.call(cmd,
shell=shell,
env=call_env,
stdout=file,
stderr=file)
file.close()
else:
retval = subprocess.call(cmd, shell=shell, env=call_env)
debug("done, retval=%d" % (retval, ))
return retval
def bind_lastbuild(self, baseline=None):
builds = {}
debug('enter')
if baseline:
job = self.api.job('%s' % (self.jobname))
buildnumber = job['lastCompletedBuild']['number']
while True:
debug('%sjob/%s/%s/artifact/builds.json' %
(self.server.url, self.jobname, buildnumber))
r = self.api.request(
'%sjob/%s/%s/artifact/builds.json' %
(self.server.url, self.jobname, buildnumber))
if r.status_code == httplib.OK:
if baseline in r.text:
records = r.json()
for record in records:
jenkinsbuild = JenkinsJobBuild(record)
baseline = jenkinsbuild.params()['BASELINE']
build = BuildJob(jenkinsserver=self.server,
builddata=jenkinsbuild)
build.load_json_data()
builds[baseline] = build
return builds
buildnumber = buildnumber - 1
def test_binaries(self, builds):
debug('enter')
tests = {}
for label in builds:
test_nodes = self.get_nodes_by_label(label)
if test_nodes:
test_node = test_nodes[0]
#test_node = None
#builds[label].wait_for_replication(test_node.region)
test = self.clsTest(test_node, builds[label], label)
if test.trigger_test():
tests[label] = test
while True:
tests_running = False
for label in tests:
if tests[label].building:
print label + " test still running"
tests_running = True
if not tests_running:
break
time.sleep(2)
self.builds = builds
self.tests = tests
def check_output(args, shell=False):
debug("executing")
output = subprocess.check_output(args, stderr=1, shell=shell)
if hasattr(sys.stderr, 'flush'):
sys.stderr.flush()
return output
def check_command(cmd, shell=True):
debug("executing")
subprocess.check_call(cmd, shell=shell, env=os.environ)
