def test_fb_rm(self, jenkins_settings, request, tmp_job_name,
global_config_file, branch):
'''
test "fb.rm" command
:param branch:
parametrized to test removing passing a branch name in the command line and without
(which means "use current branch as branch name")
'''
jenkins = Jenkins(*jenkins_settings)
new_job_name = tmp_job_name + '-new-feature'
jenkins.copy_job(tmp_job_name, new_job_name)
jenkins = Jenkins(*jenkins_settings)
assert jenkins.has_job(
new_job_name), "no job %s found. available: %s" % (
new_job_name, jenkins.get_jobs_list())
with mock.patch('cit.get_git_branch',
autospec=True) as mock_get_git_branch:
mock_get_git_branch.return_value = 'new-feature'
with mock.patch('cit.get_global_config_file',
autospec=True) as mock_get_global_config_file:
mock_get_global_config_file.return_value = str(
global_config_file)
argv = ['fb.rm']
if branch:
argv.append(branch)
assert cit.app.main(argv) is None
jenkins = Jenkins(*jenkins_settings)
assert not jenkins.has_job(
new_job_name), "job %s found! available: %s" % (
new_job_name, jenkins.get_jobs_list())
def test_fb_rm(self, tmp_job_name, global_config_file, branch):
'''
test "fb.rm" command
:param branch:
parametrized to test removing passing a branch name in the command line and without
(which means "use current branch as branch name")
'''
jenkins = Jenkins(JENKINS_URL)
new_job_name = tmp_job_name + '-new-feature'
jenkins.copy_job(tmp_job_name, new_job_name)
jenkins = Jenkins(JENKINS_URL)
assert jenkins.has_job(new_job_name), "no job %s found. available: %s" % (new_job_name, jenkins.get_jobs_list())
with mock.patch('cit.get_git_branch', autospec=True) as mock_get_git_branch:
mock_get_git_branch.return_value = 'new-feature'
with mock.patch('cit.get_global_config_file', autospec=True) as mock_get_global_config_file:
mock_get_global_config_file.return_value = str(global_config_file)
argv = ['fb.rm']
if branch:
argv.append(branch)
assert cit.app.main(argv) is None
jenkins = Jenkins(JENKINS_URL)
assert not jenkins.has_job(new_job_name), "job %s found! available: %s" % (new_job_name, jenkins.get_jobs_list())
class JenkinsTools:
def __init__(self, host='http://localhost:8080/'):
self.host = host
self.jenkins = Jenkins(host)
self.exclude_view = [
"自动化测试", "调试用", "静态代码检查", "SQL注入日志配置检查", "Preview", "All"
]
def get_all_jobs(self):
return self.jenkins.get_jobs()
def get_job(self, job_name):
return self.jenkins.get_job(job_name)
def is_not_exclude_view(self, view_name):
flag = True
for view in self.exclude_view:
flag = False if (view_name.find(view) >= 0) else True
if not flag:
return flag
return flag
def trigger_all_jobs(self):
for name, view in self.jenkins.views.iteritems():
if self.is_not_exclude_view(name):
for job_name, job_url in view.get_job_dict().items():
# start invoke jobs
print("start trigger job ", job_name)
self.jenkins.get_job(job_name).invoke()
def copy(self):
self.jenkins.copy_job()
class BaseJenkinsAPI:
def __init__(self, server_url, username=None, password=None):
self.url = server_url
self.username = username
self.password = password
if self.username and self.password:
self.instance = Jenkins(self.url, self.username, self.password)
else:
self.instance = Jenkins(self.url)
# JJ group
def copy_job(self, job_name, new_job_name):
self.instance.copy_job(job_name, new_job_name)
def get_jobs(self):
return self.instance.get_jobs()
def has_job(self, job_name):
return self.instance.has_job(job_name)
def get_number_jobs(self):
return len(self.instance.keys())
def create_job(self, job_name, config_xml):
"""
Create a job
:param str config_xml: XML configuration of new job
"""
if not config_xml:
raise JenkinsAPIException('Job XML config cannot be empty')
params = {'name': job_name}
self.instance.requester.post_xml_and_confirm_status(
self.instance.get_create_url(), data=config_xml, params=params)
print("The job {} has been successfully created".format(job_name))
def build_job(self, job_name):
self.instance.build_job(job_name)
# JJ /group
def get_name(self):
return self.url
def get_version(self):
return self.instance.version
def get_api_version(self):
return jenkinsapi.__version__
def jenkins_create_job(project):
for url in jenkins_url:
j = Jenkins(url, username=jenkins_username, password=jenkins_password)
if not j.has_job(project):
job = j.copy_job('compile_template', project)
job.disable()
job.enable()
else:
print 'The project is exist in jenkins.'
return True
def jenkins_create_job(project, uat_env):
try:
if uat_env in ['uat', 'uat_aws']:
for url in [jenkins_url[uat_env], jenkins_url['deploy']]:
j = Jenkins(url,
username=jenkins_username,
password=jenkins_password)
if not j.has_job(project):
job = j.copy_job('compile_template', project)
job.disable()
job.enable()
else:
print 'The project is exist in jenkins.'
else:
print 'The uat env is not found.'
return false
except Exception, e:
print e
return false
else:
New_Job="Random_Job_Copy"
Jenkins_Server=Jenkins('http://localhost:8080')
#This will give you the list of all jobs in Jenkins
Jobs=Jenkins_Server.keys()
print "\n Printing all the Jobs in Jenkins \n"
for job in range(len(Jobs)):
print Jobs[job]
print ""
if len(Jobs) is 0:
print "No Jobs Exist to copy"
raise Exception("No Jobs Exist to copy")
sample_job_to_copy=Jobs[0]
if New_Job is "":
New_Job='Created_Automatically'
print ""
print "The Following Job will be copied"
print sample_job_to_copy
print "\n New Job Created will be"
print New_Job+"\n"
#This command will copy jobs in Jenkins
Jenkins_Server.copy_job(sample_job_to_copy,New_Job)
}
r = requests.post("http://git.anvil8.com/api/v3/projects?private_token={}".format(config.get('gitlab_token'),), data=create_project_data)
response = r.json()
print("Create git repo on http://git.anvil8.com")
config['git_url'] = '[email protected]:{}.git'.format(response.get('path_with_namespace'),)
"""
Add jenkins user as reporter to repo
"""
add_member_data = {
'id': response.get('id'),
'user_id': 28,
'access_level': 20
}
r = requests.post("http://git.anvil8.com/api/v3/projects/{}/members?private_token={}".format(response.get('id'), config.get('gitlab_token')), data=add_member_data)
"""
Init git repo and push project
"""
subprocess.call(['git', 'init'])
subprocess.call(['git', 'remote', 'add', 'origin', config.get('git_url')])
subprocess.call(['git', 'add', '.'])
subprocess.call(['git', 'commit', '-am', '"Initial commit"'])
subprocess.call(['git', 'push', 'origin', 'master'])
print('Create project on Jenkins')
new_job = J.copy_job('template_project_python3', config.get('repo_name'))
new_job.modify_scm_url(config.get('git_url'))
J.build_job(config.get('repo_name'))
shutil.rmtree('./config')
print "\n New_Job_Name : "+New_Job_Name
Jenkins_Server=Jenkins('http://localhost:8080')
#This will give you the list of all jobs in Jenkins
Jobs=Jenkins_Server.keys()
print "\n Printing all the Jobs in Jenkins \n"
for job in range(len(Jobs)):
print Jobs[job]
print ""
Job_Exist=Jenkins_Server.has_job(Existing_Job_Name)
if Job_Exist is False:
print "The Job " + Existing_Job_Name + " does not Exist in Jenkins \n"
raise Exception("The Job"+ Existing_Job_Name + " does not Exist in Jenkins")
elif Job_Exist is True:
print "The Job " + Existing_Job_Name + " does Exist in Jenkins \n"
print ""
print "The Following Job will be copied"
print Existing_Job_Name
print "\n New Job Created will be"
print New_Job_Name
#This command will copy the existing job to a new job in jenkins
Jenkins_Server.copy_job(Existing_Job_Name,New_Job_Name)
class JenkinsCI:
def __init__(self, jenkins_url, username, password):
self.jenkins_url = jenkins_url
self.jenkins = Jenkins(self.jenkins_url, username=username, password=password)
self.build_info = []
__user_data = "Basic " + (username + ":" + password).encode("base64").rstrip()
self.__headers = {"Content-Type": "application/x-www-form-urlencoded", "Authorization": __user_data}
def __invoke_method(self, command, method="GET", parameters=None, silent=False):
if parameters is None:
parameters = {}
method_url = urljoin(self.jenkins_url, command)
request_data = None
if method == "GET":
query_string = urlencode(parameters)
method_url = method_url + "?" + query_string
else:
request_data = urlencode(parameters)
req = Request(method_url, data=request_data, headers=self.__headers)
req.get_method = lambda: method
try:
response = urlopen(req).read()
except HTTPError as e:
if not silent:
print "{0} : {1}".format(method_url, e)
return None
try:
return loads(response.decode("utf-8"))
except ValueError:
return response.decode("utf-8")
def create_job(self, job_name, xml_config):
try:
self.jenkins.create_job(job_name, xml_config)
except Exception as e:
print e
return False
else:
print "[+] : job {0} created".format(job_name)
return True
def format_path_folder(self, path_folder):
return "job/" + path_folder.replace("/", "/job/")
def rename_folder(self, path_folder, new_path_folder):
new_folder_name = new_path_folder.split("/")[-1]
command = "{0}/doRename".format(self.format_path_folder(path_folder))
json_parameters = dumps({"newName": new_folder_name})
parameters = {"newName": new_folder_name, "Submit": "Yes", "json": json_parameters}
result = self.__invoke_method(command, "POST", parameters)
if not result:
raise JenkinsException("Cannot rename folder {0} to {1}".format(path_folder, new_path_folder), 1)
print "[+] : folder {0} renamed to {1}".format(path_folder, new_path_folder)
def create_folder(self, path_folder, recursive=False):
if recursive:
return self.recursive_create_folders(path_folder)
split_path = path_folder.split("/")
folder_name = split_path[-1]
command = "createItem"
if len(split_path) > 1:
new_path = "/".join(f for f in split_path[:-1])
command = self.format_path_folder(new_path) + "/" + command
json_parameters = dumps({"name": folder_name, "mode": "com.cloudbees.hudson.plugins.folder.Folder",
"from": "", "Submit": "OK"})
parameters = {"name": folder_name, "mode": "com.cloudbees.hudson.plugins.folder.Folder",
"from": "", "Submit": "OK", "json": json_parameters}
result = self.__invoke_method(command, "GET", parameters)
if not result:
raise JenkinsException("Cannot create folder {0}".format(path_folder), 1)
print "[+] : folder {0} created".format(path_folder)
def recursive_create_folders(self, path_folder):
split_path = path_folder.split("/")
root = split_path[0]
for folder in split_path[1:]:
if not self.check_folder_exist(root):
self.create_folder(root)
root += "/{0}".format(folder)
def check_folder_exist(self, path_folder):
result = self.__invoke_method(self.format_path_folder(path_folder), "GET", silent=True)
if result:
return True
return False
def move_job_on_folder(self, job_name, path_folder):
command = "{0}/move/move".format(self.format_path_folder(job_name))
json_parameters = {"destination": "/{0}".format(path_folder)}
parameters = {"destination": "/{0}".format(path_folder), "json": json_parameters, "Submit": "Move"}
result = self.__invoke_method(command, "POST", parameters)
if not result:
raise JenkinsException("Cannot move job {0} to {1} folder".format(job_name, path_folder), 2)
print "[+] : Job {0} moved to {1} folder".format(job_name, path_folder)
def get_plugins_version_dict(self, plugins_name_list):
plugins_version_dict = {}
all_plugins = self.jenkins.get_plugins().get_plugins_dict()
for plugin in plugins_name_list:
plugins_version_dict[plugin] = all_plugins[plugin].version
return plugins_version_dict
def copy_job(self, template_job, new_job):
try:
self.jenkins.copy_job(template_job, new_job)
except Exception as e:
print e
return False
else:
print "[+] : Job {0} created".format(new_job)
return True
def plugins_checker(self, plugins_list):
all_installed = True
for plugin in plugins_list:
if not self.jenkins.has_plugin(plugin):
all_installed = False
print "[-] : Plugin {0} is not installed".format(plugin)
if not all_installed:
return False
return True
def check_job_exist(self, job_name):
if self.jenkins.has_job(job_name):
return True
return False
def check_jobs_exist(self, jobs_list):
status_exist = False
for job in jobs_list:
if self.check_job_exist(job):
print "[-] : Job {0} already exist".format(job)
status_exist = True
return status_exist
def create_view(self, view_name):
views = self.jenkins.views
if view_name not in views.keys():
views.create(view_name)
print "[+] : view {0} created".format(view_name)
def move_job_on_view(self, view_name, job_name):
view_url = "{0}/view/{1}".format(self.jenkins_url, view_name)
view = self.jenkins.get_view_by_url(view_url)
view.add_job(job_name)
def delete_job(self, job_name):
if self.check_job_exist(job_name):
self.jenkins.delete_job(job_name)
print "[+] : job {0} deleted".format(job_name)
def build_job(self, job_name, arguments):
job = self.jenkins.get_job(job_name)
build_number = job.get_next_build_number()
job.invoke(build_params=arguments)
return build_number
def get_git_info_build(self, job_name, build_number):
job = self.jenkins.get_job(job_name)
build = job.get_build(build_number)
return build.get_revision()
def get_url_build(self, job_name, build_number):
return "{0}/job/{1}/{2}".format(self.jenkins_url, job_name, build_number)
def print_info_build(self, job_name, build_number):
url = self.get_url_build(job_name, build_number)
git_info = self.get_git_info_build(job_name, build_number)
print "Job: {0} #{1}".format(job_name, build_number)
print "Url: {0}".format(url)
for info in git_info:
print "Branch : {0} | Revision : {1}".format(info["name"], info["SHA1"])
custom_configxml_content = xmlhandle.toxml('UTF-8')
# j.create_job(jobname="senyang", xml=custom_configxml_content)
# j.build_job(jobname="senyang")
print j.get_job("senyang").get_last_build()
#copy job
for i in range(5):
newjob = "longrun_%d" % i
print j.copy_job("longrun", newjob)
print j.build_job(newjob)
#---------------------------------------------------------------------------
# !!!!!! create slave node using JNLP (java web start) !!!!!!
# slave connection command:
# >yum install -y wget curl screen;
# >wget http://******:9999/jnlpJars/slave.jar ;
# >screen java -jar slave.jar -noReconnect -jnlpUrl http://*****:9999/computer/${slave_node_name}/slave-agent.jnlp
#---------------------------------------------------------------------------
# print j.create_node(name="slave41", num_executors=2, node_description="docker container node", remote_fs='/var/lib/jenkins', labels="testing webui", exclusive=False)
# print j.create_node(name="slave42", num_executors=2, node_description="docker container node", remote_fs='/var/lib/jenkins', labels="testing webui", exclusive=True)
# print j.create_node(name="slave43", num_executors=2, node_description="docker container node", remote_fs='/var/lib/jenkins1', labels="testing webui", exclusive=False)
def jenkins_create_job(issue, jira):
"""Creates jenkins job, if it already exists only performs the svn diff.
"""
env = str(issue.fields.customfield_13128)
app_product = str(issue.fields.customfield_13120)
change_type = issue.fields.customfield_13122
version = str(issue.fields.customfield_13132)
version_folder = str(issue.fields.customfield_13133)
description = issue.fields.description
env = env.replace('-', '')
non_jenkins_apps = ['Services', 'OMS', 'JEDI', 'Customer Profile', 'MDS']
jenkins_repository = {'Events & Notifications': 'Event_Notification', 'Routing Guide': 'Routing_Guide'}
non_repository = {'Dock Door': 'DockDoor', 'Mobile': 'MOBILE_SERVICES' }
jenkins_message = 'Jenkins job created {}'
jenkins_url = 'http://hq-build3:8080'
new_version = '%s_%s_%s' % (app_product, env, version)
#regex - finds any jenkins job with application and environment in it with a version number
#example: SMART_BUILD_CREA_2015.03.1
#example: SMART_CREA_2015.03.2
regex = re.compile(r'(%s.*%s.*[\d.]+)'% (app_product, env))
job_list = list()
apps = list()
server = Jenkins(jenkins_url, username='******', password=decrypt_key('jenkins'))
#if new version not found, create a jenkins job
#else, use one in use.
print("new version ", new_version)
#print("server.keys ", server.keys())
if new_version not in server.keys():
logging_message('attempting to create jenkins job %s' % new_version)
for jenkins_job in server.keys():
if (regex.match(jenkins_job)) and (server.get_job(jenkins_job).is_enabled()):
job_list.append(jenkins_job)
recent_job = max(job_list)
logging_message('creating new job from "recent job" = %s' % recent_job)
job = server.get_job(recent_job)
server.copy_job(recent_job, new_version)
new_job = server.get_job(new_version)
#for some reason you have to disable it first to get it enabled correctly
new_job.disable()
new_job.enable()
logging_message('Created Jenkins job %s' % new_job )
else:
new_job = server.get_job(new_version)
if change_type == None:
change_type = 'Other'
for app in change_type:
apps.append(str(app))
if app_product in jenkins_repository:
app_product = jenkins_repository[app_product]
#grabs config from job
config = new_job.get_config()
end = config[1:].find('<')+1
job_xml = etree.fromstring(config[end:])
if is_debug:
print("would have added a jira comment", issue, jenkins_message.format(new_job))
else:
jira.add_comment(issue, jenkins_message.format(new_job))
#navigates the config xml and changes the values based on the new jenkins job
#remote = Repository URL
#command = Command (execute shell)
#subject = email subject
#body = email body
#recipientList = list of people to email from pre-build
script_env = env.lower()
logging_message('searching through Jenkins %s xml ...' % new_job)
print("job xml ", job_xml)
split_text = job_xml.xpath('.//command')[0].text.split(' ')
old_branch_name = (job_xml.xpath('.//remote')[0].text)
if old_branch_name:
last_folder = find_occurences(old_branch_name, '/')[-1]
print("updated xml path remote ", old_branch_name[:last_folder]+'/%s' % version_folder)
job_xml.xpath('.//remote')[0].text = old_branch_name[:last_folder]+'/%s' % version_folder
if len(change_type) == 1:
if str(change_type[0]) == 'SMART - MT':
print("updated xml path command ", '%s %s %s %s -no_ui -test' % (split_text[0], version, version_folder, script_env))
job_xml.xpath('.//command')[0].text = '%s %s %s %s -no_ui -test' % (split_text[0], version, version_folder, script_env)
elif str(change_type[0]) in jenkins_repository:
print("updated xml path command ",'%s %s %s %s' % (split_text[0], version, version_folder, script_env))
job_xml.xpath('.//command')[0].text = '%s %s %s %s' % (split_text[0], version, version_folder, script_env)
elif app_product in non_repository:
print("updated xml path command ", '%s %s %s' % (split_text[0], version, script_env))
job_xml.xpath('.//command')[0].text = '%s %s %s' % (split_text[0], version, script_env)
elif apps == ['SMART - MT', 'SMART - UI']:
print("updated xml path command ",'%s %s %s %s -test' % (split_text[0], version, version_folder, script_env))
job_xml.xpath('.//command')[0].text = '%s %s %s %s -test' % (split_text[0], version, version_folder, script_env)
print("updated xml path subject ", '%s %s-%s Build and Deployment for %s has begun in %s' % (issue.key, app_product, version, app_product, env))
job_xml.xpath('.//subject')[0].text = '%s %s-%s Build and Deployment for %s has begun in %s' % (issue.key, app_product, version, app_product, env)
#[-1] is for success email
print("updated xml path subject ",job_xml.xpath('.//recipientList')[1].text)
job_xml.xpath('.//recipientList')[-1].text = job_xml.xpath('.//recipientList')[1].text
print("updated xml path subject ", '%s %s-%s Build and Deployment for %s has COMPLETED in %s' % (issue.key, app_product, version, app_product, env))
job_xml.xpath('.//subject')[-1].text = '%s %s-%s Build and Deployment for %s has COMPLETED in %s' % (issue.key, app_product, version, app_product, env)
#cannot have as attachment since it will delete the workspace first
#Adds svn log to email
diff_log = svn_change_log(str(change_type[0]), new_version, version_folder)
print("updated xml path subject ",'%s\n\n%s' % (issue.fields.description, diff_log))
job_xml.xpath('.//body')[0].text = '%s\n\n%s' % (issue.fields.description, diff_log)
print("updated xml path subject ", '%s\n\n%s' % (issue.fields.description, diff_log))
job_xml.xpath('.//body')[-1].text = '%s\n\n%s' % (issue.fields.description, diff_log)
config_str = etree.tostring(job_xml).decode('utf-8')
print("updated the job config ", config_str)
new_job.update_config(config_str)
#change sonar release to whatever the new version is.
change_sonar_release(issue, script_env, version, version_folder)
return new_version