def get_wf_build_info(self, name, number):
"""
Get build information dictionary.
:param name: Job name, ``str``
:param name: Build number, ``int``
:returns: dictionary of build information, ``dict``
"""
folder_url, short_name = self.client._get_job_folder(name)
number = int(number)
try:
response = self.client.jenkins_open(
Request(self.client._build_url(WF_BUILD_INFO, locals())))
if response:
return json.loads(response)
else:
raise jenkins.JenkinsException(
'job[%s] number[%d] does not exist' % (name, number))
except HTTPError:
raise jenkins.JenkinsException(
'job[%s] number[%d] does not exist' % (name, number))
except ValueError:
raise jenkins.JenkinsException(
'Could not parse JSON info for job[%s] number[%d]' %
(name, number))
def get_build_env_vars(self, name, number, depth=0):
'''Get build environment variables.
:param name: Job name, ``str``
:param name: Build number, ``int``
:param depth: JSON depth, ``int``
:returns: dictionary of build env vars, ``dict``
'''
folder_url, short_name = self._get_job_folder(name)
try:
response = self.jenkins_open(
requests.Request('GET', self._build_url(BUILD_ENV_VARS, locals())))
if response:
return json.loads(response)
else:
raise jenkins.JenkinsException(
'job[%s] number[%d] does not exist' % (name, number))
except requests.exceptions.HTTPError:
raise jenkins.JenkinsException('job[%s] number[%d] does not exist' %
(name, number))
except ValueError:
raise jenkins.JenkinsException(
'Could not parse JSON info for job[%s] number[%d]' %
(name, number))
except jenkins.NotFoundException as e:
# This can happen on workflow jobs, or if InjectEnvVars plugin not installed
print("Not retrieving env_vars: %s" % e)
return None
def get_build_test_report(self, name, number, depth=0):
'''Get test results report.
:param name: Job name, ``str``
:param name: Build number, ``int``
:returns: dictionary of test report results, ``dict``
'''
folder_url, short_name = self._get_job_folder(name)
try:
response = self.jenkins_open(
requests.Request('GET', self._build_url(BUILD_TEST_REPORT,
locals())))
if response:
return json.loads(response)
else:
raise jenkins.JenkinsException(
'job[%s] number[%d] does not exist' % (name, number))
except requests.exceptions.HTTPError:
raise jenkins.JenkinsException('job[%s] number[%d] does not exist' %
(name, number))
except ValueError:
raise jenkins.JenkinsException(
'Could not parse JSON info for job[%s] number[%d]' %
(name, number))
except jenkins.NotFoundException as e:
# This can happen if the test report wasn't generated for any reason
print("Not retrieving test report: %s" % e)
return None
def set_build_config(
self, name, number, display_name,
description): # need to allow anonymous user to update build
try:
# print '{{ "displayName": "{}", "description": "{}" }}'.format(display_name, description)
response = self.jenkins_open(
urllib2.Request(
self.server + BUILD_SET_CONFIG % locals(),
urllib.urlencode({
'json':
'{{ "displayName": "{}", "description": "{}" }}'.
format(display_name, description)
})))
if response:
return response
else:
raise jenkins.JenkinsException(
'job[%s] number[%d] does not exist' % (name, number))
except urllib2.HTTPError:
raise jenkins.JenkinsException(
'job[%s] number[%d] does not exist' % (name, number))
except ValueError:
raise jenkins.JenkinsException(
'Could not parse JSON info for job[%s] number[%d]' %
(name, number))
def get_build_artifact(self, name, number, artifact):
"""Get artifacts from job
:param name: Job name, ``str``
:param number: Build number, ``int``
:param artifact: Artifact relative path, ``str``
:returns: artifact to download, ``dict``
"""
folder_url, short_name = self._get_job_folder(name)
try:
response = self.jenkins_open(
requests.Request('GET', self._build_url(BUILD_ARTIFACT, locals())))
if response:
return json.loads(response)
else:
raise jenkins.JenkinsException(
'job[%s] number[%d] does not exist' % (name, number))
except requests.exceptions.HTTPError:
raise jenkins.JenkinsException('job[%s] number[%d] does not exist' %
(name, number))
except ValueError:
raise jenkins.JenkinsException(
'Could not parse JSON info for job[%s] number[%d]' %
(name, number))
except jenkins.NotFoundException as e:
# This can happen if the artifact is not found
print("Not retrieving artifact: %s" % e)
return None
def get_node_data(self, retry=10):
'''Get a list of nodes connected to the Master
Each node is a dict with keys 'name' and 'offline'
:returns: List of nodes, ``[ { str: str, str: bool} ]``
'''
try:
data = self.jenkins_open(Request(self._build_url(NODE_LIST_INFO)))
#print("data:"+str(data))
nodes_data = json.loads(data)
return nodes_data['computer']
except (HTTPError, BadStatusLine):
raise BadHTTPException("Error communicating with server[%s]" %
self.getServer())
except (jenkins.JenkinsException):
print(
"Authentication Exception? use the '-u <username>' and '-p <password>' options to reserve/release"
)
raise jenkins.JenkinsException("Authentication Exception:")
except (ValueError, httplib.IncompleteRead):
# retry
if (retry > 0):
time.sleep(3)
return self.get_node_data(retry=retry - 1)
else:
raise jenkins.JenkinsException(
"Could not parse JSON info for server[%s]" %
str(self.getServer()))
def get_build_stages(self, name, number):
"""Get stages info from job
:param name: Job name, ``str``
:param number: Build number, ``int``
:returns: dictionary of stages in the job, ``dict``
"""
folder_url, short_name = self._get_job_folder(name)
try:
response = self.jenkins_open(
requests.Request('GET', self._build_url(BUILD_STAGES, locals())))
if response:
return json.loads(response)
else:
raise jenkins.JenkinsException(
'job[%s] number[%d] does not exist' % (name, number))
except requests.exceptions.HTTPError:
raise jenkins.JenkinsException('job[%s] number[%d] does not exist' %
(name, number))
except ValueError:
raise jenkins.JenkinsException(
'Could not parse JSON info for job[%s] number[%d]' %
(name, number))
except jenkins.NotFoundException as e:
# This can happen if this isn't a stages/pipeline job
print("Not retrieving stages: %s" % e)
return None
def side_effect(*args, **kwargs):
if 'depth' in kwargs and kwargs['depth'] > 0:
raise jenkins.JenkinsException(
"Error in request. Possibly authentication failed"
"[500]: Server Error")
else:
return {"success": True}
def test_trigger_job_exception(self, mock_jenkins, _):
mock_jenkins.side_effect = jenkins.JenkinsException()
param = {"node": "test"}
with self.assertRaises(jenkins.JenkinsException) as exception:
jxcore = JxCore()
jxcore.trigger_job("test_job", param)
self.assertEqual(str(exception.exception), "Job not found")
def test_jxcore_invalid_context(self, mock_jenkins, mock_ctxcore,
mock_exit):
mock_jenkins.side_effect = jenkins.JenkinsException()
mock_ctxcore_obj = mock_ctxcore.return_value
mock_ctxcore_obj.validate_context.return_value = False
jxcore = JxCore()
mock_exit.assert_called_once()
def test_console_output_jenkins_connection_failure_warning(
self, get_plugins_mock):
"""
Run test mode and verify that failed Jenkins connection attempt
exception does not bubble out of cmd.main. Ideally, we would also test
that an appropriate message is logged to stderr but it's somewhat
difficult to figure out how to actually enable stderr in this test
suite.
"""
get_plugins_mock.side_effect = jenkins.JenkinsException(
"Connection refused")
with mock.patch("sys.stdout"):
try:
args = [
"--conf",
self.default_config_file,
"test",
os.path.join(self.fixtures_path, "cmd-001.yaml"),
]
self.execute_jenkins_jobs_with_args(args)
except jenkins.JenkinsException:
self.fail("jenkins.JenkinsException propagated to main")
except Exception:
pass # only care about jenkins.JenkinsException for now
def is_building(self, servicename, product):
'''
get service is_building status:
return True or False
'''
if 'install-deploy' in servicename:
install_job = servicename
else:
#sql = "SELECT install_job_name from service_jobs_manager WHERE service_name like '%s' and product='%s'" % (servicename,product)
#data = jenkinsMysql(sql)
data = ServiceJobsManager.objects.filter(
Q(service_name__contains=servicename))
if not data:
raise jenkins.JenkinsException('service name is not exists')
else:
install_job = data[0].install_job_name
logger.info("install_job:%s" % install_job)
try:
time.sleep(5)
install_build_number = self.jenkins_server.get_job_info(
install_job)['lastBuild']['number']
build_status = self.jenkins_server.get_build_info(
install_job, install_build_number)['building']
#print "build_status:%s"%build_status
return build_status
except Exception, error:
logger.error(error)
raise Exception(error)
def create_servicejob(self, servicename, env='uat', servicetype='service'):
'''
create a new job with demo config.
'''
democonf = self.jenkins_server.get_job_config(
'uat-install-service-demo')
jobconfig = democonf.replace('service-demo', servicename)
if servicetype == 'war':
jobconfig = jobconfig.replace('target/*.zip', 'target/*.war')
jobname = 'uat-install-' + servicename
if self.jenkins_server.job_exists(jobname):
raise jenkins.JenkinsException('jenkins job[%s] already exist' %
jobname)
try:
self.jenkins_server.create_job(jobname, jobconfig)
except jenkins.JenkinsException, e:
raise jenkins.JenkinsException(e)
def assert_credential_exists(self, id, exc_msg='credential[%s] not exist'):
'''Raise an exception if a job does not exist
:param name: Name of Jenkins job, ``str``
:param exception_message: Message to use for the exception. Formatted
with ``name``
:throws: :class:`JenkinsException` whenever the job does not exist
'''
if not self.credential_exists(id):
raise jenkins.JenkinsException(exc_msg % (id))
def get_info_at_path(self, path):
try:
return json.loads(self.jenkins_open(Request(path + jenkins.INFO)))
except (HTTPError, BadStatusLine):
raise jenkins.BadHTTPException(
"Error communicating with server[%s]" % self.server)
except ValueError:
raise jenkins.JenkinsException(
"Could not parse JSON info for server[%s]" % self.server)
def build_job(jenkins_instance, job_name, parameters=None):
#return jenkins_instance.build_job(job_name)
# replicate internal implementation of Jenkins.build_job()
import urllib2
if not jenkins_instance.job_exists(job_name):
raise jenkins.JenkinsException('no such job[%s]' % (job_name))
# pass parameters to create a POST request instead of GET
return jenkins_instance.jenkins_open(
urllib2.Request(jenkins_instance.build_job_url(job_name, parameters),
'foo=bar'))
def test_in_another_folder_failed(self, jenkins_mock):
jenkins_mock.side_effect = [jenkins.JenkinsException()]
with self.assertRaises(jenkins.JenkinsException) as context_manager:
self.j.copy_job(u'a Folder/TestJob', u'another Folder/TestJob_2')
self.assertEqual(
str(context_manager.exception),
('copy[a Folder/TestJob to another Folder/TestJob_2] failed, '
'source and destination folder must be the same'))
self._check_requests(jenkins_mock.call_args_list)
def delete_job(self, job_name):
"""
Delete a job from Jenkins
:param `job_name`: job name `str`
"""
try:
self.server.delete_job(job_name)
print("jxctl >> \"%s\" Job deleted" % job_name)
except jenkins.JenkinsException as jenkins_error:
print(jenkins_error)
raise jenkins.JenkinsException("Jenkins Exception")
def test_http_fail(self, jenkins_mock):
jenkins_mock.side_effect = jenkins.JenkinsException(
'Error in request. Possibly authentication failed [401]: '
'basic auth failed')
with self.assertRaises(jenkins.JenkinsException):
self.j.quiet_down()
self.assertEqual(jenkins_mock.call_args[0][0].url,
self.make_url('quietDown'))
self._check_requests(jenkins_mock.call_args_list)
def jenkins_open(self, req, add_crumb=True):
'''Utility routine for opening an HTTP request to a Jenkins server.
Extended over the original method to allow connections to SF with a
cookie.
'''
self.opener.addheaders = [('Cookie', 'auth_pubtkt=%s' % self.cookie)]
try:
if add_crumb:
self.maybe_add_crumb(req)
response = self.opener.open(req, timeout=self.timeout).read()
if response is None:
raise jenkins.EmptyResponseException(
"Error communicating with server[%s]: "
"empty response" % self.server)
return response.decode('utf-8')
except HTTPError as e:
# Jenkins's funky authentication means its nigh impossible to
# distinguish errors.
if e.code in [401, 403, 500]:
# six.moves.urllib.error.HTTPError provides a 'reason'
# attribute for all python version except for ver 2.6
# Falling back to HTTPError.msg since it contains the
# same info as reason
raise jenkins.JenkinsException(
'Error in request. ' +
'Possibly authentication failed [%s]: %s' %
(e.code, e.msg))
elif e.code == 404:
raise jenkins.NotFoundException(
'Requested item could not be found')
else:
raise
except socket.timeout as e:
raise jenkins.TimeoutException('Error in request: %s' % (e))
except URLError as e:
# python 2.6 compatibility to ensure same exception raised
# since URLError wraps a socket timeout on python 2.6.
if str(e.reason) == "timed out":
raise jenkins.TimeoutException('Error in request: %s' %
(e.reason))
raise jenkins.JenkinsException('Error in request: %s' % (e.reason))
def abort_job(self, job_name, build_number):
"""
Abort the job build
:param `job_name`: job name `str`
:param `build_number: build number of the job `int`
"""
try:
self.server.stop_build(job_name, build_number)
print("jxctl >> {0}-{1} aborted sucessfully".format(job_name, build_number))
except jenkins.JenkinsException as jenkins_error:
print(jenkins_error)
raise jenkins.JenkinsException("Job/Build not found")
def trigger_job(self, job_name, params=None):
"""
Build a Job
:param `job_name`: job name `str`
:param `params`: parameters for job `dict`
"""
try:
self.server.build_job(job_name, params)
print("jxctl >> {0} triggered sucessfully".format(job_name))
except jenkins.JenkinsException as jenkins_error:
print(jenkins_error)
raise jenkins.JenkinsException("Job not found")
def get_wf_node_log(self, name, number, node):
"""
Get build log for execution node.
:param name: Job name, ``str``
:param name: Build number, ``int``
:param name: Execution node number, ``int``
:returns: Execution node build log, ``dict``
"""
folder_url, short_name = self.client._get_job_folder(name)
try:
response = self.client.jenkins_open(
Request(self.client._build_url(WF_NODE_LOG, locals())))
if response:
return json.loads(response)
else:
raise jenkins.JenkinsException(
'job[%s] number[%d] does not exist' % (name, number))
except HTTPError:
raise jenkins.JenkinsException(
'job[%s] number[%d] does not exist' % (name, number))
def test_return_empty_response(self, jenkins_mock):
jenkins_mock.side_effect = jenkins.JenkinsException(
"Error communicating with server[{0}/]: empty response".format(
self.base_url))
with self.assertRaises(jenkins.JenkinsException) as context_manager:
self.j.get_info()
self.assertEqual(jenkins_mock.call_args[0][0].get_full_url(),
self.make_url('api/json'))
self.assertEqual(
str(context_manager.exception),
'Error communicating with server[{0}/]: '
'empty response'.format(self.base_url))
self._check_requests(jenkins_mock.call_args_list)
def release(self,nodeNames) :
if nodeNames == None :
print("you must enter a device to release");
return;
nodes = self.getData();
for nodeName in str.split(nodeNames,",") :
print("releasing "+nodeName);
try :
self.getServer().enable_node(nodeName)
except (jenkins.JenkinsException):
print("Authentication Exception? use the '-u <username>' and '-p <password>' options to reserve/release");
raise jenkins.JenkinsException("Authentication Exception")
except jenkins.NotFoundException:
print("Can't find device '"+nodeName+"', use 'resv list' to view available devices");
def set_build_config(
self, name, number, display_name,
description): # need to allow anonymous user to update build
try:
parameters = json.dumps({
'displayName': display_name,
'description': description
})
response = self.jenkins_open(
requests.Request('POST',
self._build_url(BUILD_SET_CONFIG, locals()),
data={'json': parameters}))
if response:
return response
else:
raise jenkins.JenkinsException(
'job[%s] number[%d] does not exist' % (name, number))
except urllib2.HTTPError:
raise jenkins.JenkinsException(
'job[%s] number[%d] does not exist' % (name, number))
except ValueError:
raise jenkins.JenkinsException(
'Could not parse JSON info for job[%s] number[%d]' %
(name, number))
def trigger_if_necessary(da, pkg, rosdistro, jenkins_instance,
missing_by_arch):
if da != 'source' and 'source' in missing_by_arch and \
pkg in missing_by_arch['source']:
print(
" Skipping trigger of binarydeb job for package '%s' on arch '%s' as the sourcedeb job will trigger them automatically"
% (pkg, da))
return False
if da == 'source':
job_name = '%s_sourcedeb' % (debianize_package_name(rosdistro, pkg))
else:
job_name = '%s_binarydeb_%s' % (debianize_package_name(rosdistro,
pkg), da)
job_info = jenkins_instance.get_job_info(job_name)
if 'color' in job_info and 'anime' in job_info['color']:
print(" Skipping trigger of job %s because it's already running" %
job_name)
return False
if 'inQueue' in job_info and job_info['inQueue']:
print(" Skipping trigger of job '%s' because it's already queued" %
job_name)
return False
if da != 'source' and 'upstreamProjects' in job_info:
upstream = job_info['upstreamProjects']
for p in missing_by_arch[da]:
p_name = '%s_binarydeb_%s' % (debianize_package_name(rosdistro,
p), da)
for u in upstream:
if u['name'] == p_name:
print(
" Skipping trigger of job '%s' because the upstream job '%s' is also triggered"
% (job_name, p_name))
return False
print("Triggering '%s'" % (job_name))
#return jenkins_instance.build_job(job_name)
# replicate internal implementation of Jenkins.build_job()
import urllib2
if not jenkins_instance.job_exists(job_name):
raise jenkins.JenkinsException('no such job[%s]' % (job_name))
# pass parameters to create a POST request instead of GET
return jenkins_instance.jenkins_open(
urllib2.Request(jenkins_instance.build_job_url(job_name), 'foo=bar'))
def test_console_output_jenkins_connection_failure_warning(
self, get_plugins_info_mock):
"""
Run test mode and verify that failed Jenkins connection attempt
exception does not bubble out of cmd.main. Ideally, we would also test
that an appropriate message is logged to stderr but it's somewhat
difficult to figure out how to actually enable stderr in this test
suite.
"""
get_plugins_info_mock.side_effect = \
jenkins.JenkinsException("Connection refused")
with mock.patch('sys.stdout'):
try:
cmd.main(['test', os.path.join(self.fixtures_path,
'cmd-001.yaml')])
except jenkins.JenkinsException:
self.fail("jenkins.JenkinsException propagated to main")
except:
pass # only care about jenkins.JenkinsException for now
def trigger_hudson_build_debs(name, distro_name, os_platform):
from buildfarm import jenkins_support
import jenkins
import urllib
import urllib2
jenkins_instance = jenkins_support.JenkinsConfig_to_handle(
jenkins_support.load_server_config_file(
jenkins_support.get_default_catkin_debs_config()))
parameters = {
'DISTRO_NAME': distro_name,
'STACK_NAME': name,
'OS_PLATFORM': os_platform,
}
for arch in ['i386', 'amd64']:
parameters['ARCH'] = arch
job_name = 'ros-%s-%s_binarydeb_%s_%s' % (
distro_name, name.replace('_', '-'), os_platform, arch)
print('triggering job: %s' % job_name)
if not jenkins_instance.job_exists(job_name):
raise jenkins.JenkinsException('no such job[%s]' % (job_name))
# pass parameters to create a POST request instead of GET
jenkins_instance.jenkins_open(
urllib2.Request(jenkins_instance.build_job_url(job_name),
urllib.urlencode(parameters)))
def test_delete_job_exception(self, mock_jenkins, _):
mock_jenkins.side_effect = jenkins.JenkinsException()
with self.assertRaises(jenkins.JenkinsException) as exception:
jxcore = JxCore()
jxcore.delete_job("test_job")
self.assertEqual(str(exception.exception), "Jenkins Exception")
