def __init__(self, file_path, git_credentials=None):
"""
__build_name - Refer to the field "build-name" in manifest file.
__repositories - Refer to the field "repositories" in manifest file.
__downstream_jobs - Refer to the field "downstream-jobs" in manifest file.
__file_path - The file path of the manifest file
__name - The file name of the manifest file
__manifest -The content of the manifest file
__changed - If manifest is changed, be True; The default value is False
__git_credentials - URL, credentials pair for the access to github repos
gitbit - Class instance of gitbit
"""
self._build_name = None
self._build_requirements = None
self._repositories = []
self._downstream_jobs = []
self._file_path = file_path
self._name = file_path.split('/')[-1]
self._manifest = None
self._changed = False
self._git_credentials = None
self.gitbit = GitBit(verbose=True)
if git_credentials:
self._git_credentials = git_credentials
self.setup_gitbit()
self.read_manifest_file(self._file_path)
self.parse_manifest()
def __init__(self, git_credentials=None):
"""
Create a repository interface object
:return:
"""
self._git_credentials = git_credentials
self.git = GitBit(verbose=True)
if self._git_credentials:
self.setup_gitbit()
def __init__(self, file_path, git_credentials = None):
"""
__build_name - Refer to the field "build-name" in manifest file.
__repositories - Refer to the field "repositories" in manifest file.
__downstream_jobs - Refer to the field "downstream-jobs" in manifest file.
__file_path - The file path of the manifest file
__name - The file name of the manifest file
__manifest -The content of the manifest file
__git_credentials - URL, credentials pair for the access to github repos
gitbit - Class instance of gitbit
"""
self._build_name = None
self._build_requirements = None
self._repositories = []
self._downstream_jobs = []
self._file_path = file_path
self._name = file_path.split('/')[-1]
self._manifest = None
self._git_credentials = None
self.gitbit = GitBit(verbose=True)
if git_credentials:
self._git_credentials = git_credentials
self.setup_gitbit()
self.read_manifest_file(self._file_path)
self.parse_manifest()
def __init__(self, git_credentials=None):
"""
Create a repository interface object
:return:
"""
self._git_credentials = git_credentials
self.git = GitBit(verbose=True)
if self._git_credentials:
self.setup_gitbit()
def do_one_task(self, name, data, results):
"""
Perform the actual work of checking out a repository. This portion of the
task is performed in a subprocess, and may be performed in parallel with other
instances.
name and data will come from the values passed in to add_task()
:param name:
:param data:
data should contain:
'credentials': a list of Git credentials in URL:VARIABLE_NAME format
'repo': a repository entry from a manifest file
'builddir': the location to check out the repository into
:param results: a shared dictionary for storing results and sharing them to the
parent process
:return: None (all output data stored in results)
"""
# make sure we have all of the right data that we need to start the build
if name is None or data is None:
raise ValueError("name or data not present")
for key in ['repo', 'builddir']:
if key not in data:
raise ValueError("{0} key missing from data: {1}".format(
key, data))
repo = data['repo']
if 'repository' not in repo:
raise ValueError("no repository in work {0}".format(repo))
# data validation okay, so start the work
print "Starting checkout of {0}".format(name)
# someplace to start storing results of the commands that will be run
results['commands'] = []
git = GitBit(verbose=False)
if 'credentials' in data and data['credentials'] is not None:
for credential in data['credentials']:
url, cred = credential.split(',', 2)
git.add_credential_from_variable(url, cred)
repo_url = repo['repository']
destination_directory_name = strip_suffix(os.path.basename(repo_url),
".git")
# build up a git clone command line
# clone [ -b branchname ] repository_url [ destination_name ]
command = ['clone']
#clone big files with git-lfs is much faster
if repo.has_key('lfs') and repo['lfs']:
command = ['lfs', 'clone']
if 'branch' in repo and repo['branch'] != "":
command.extend(['-b', repo['branch']])
command.append(repo_url)
if 'checked-out-directory-name' in repo:
# this specifies what the directory name of the checked out repository
# should be, as opposed to using Git's default (the basename of the repository URL)
# note to self: do not combine the following two lines again
destination_directory_name = repo['checked-out-directory-name']
command.append(destination_directory_name)
destination_directory = os.path.abspath(
data['builddir']) + "/" + destination_directory_name
if os.path.isdir(destination_directory):
shutil.rmtree(destination_directory)
self.run_git_command(git, command, data['builddir'], results)
# the clone has been performed -- now check to see if we need to move the HEAD
# to point to a specific location within the tree history. That will be true
# if there is a commit-id or tag value specified in the repository (which will
# be the case most of the time).
reset_id = self._get_reset_value(repo)
if reset_id is not None:
working_directory = os.path.join(data['builddir'],
destination_directory_name)
command = [
"fetch", "origin", "refs/pull/*:refs/remotes/origin/pr/*"
]
self.run_git_command(git, command, working_directory, results)
command = ["reset", "--hard", reset_id]
self.run_git_command(git, command, working_directory, results)
results['status'] = "success"
class RepoOperator(object):
def __init__(self, git_credentials=None):
"""
Create a repository interface object
:return:
"""
self._git_credentials = git_credentials
self.git = GitBit(verbose=True)
if self._git_credentials:
self.setup_gitbit()
def setup_gitbit(self, credentials=None):
"""
Set gitbit credentials.
:return:
"""
if credentials is None:
if self._git_credentials is None:
return
else:
credentials = self._git_credentials
else:
self._git_credentials = credentials
for url_cred_pair in credentials:
url, cred = url_cred_pair.split(',')
self.git.add_credential_from_variable(url, cred)
def set_git_dryrun(self, dryrun):
self.git.set_dryrun(dryrun)
def set_git_verbose(self, verbose):
self.git.set_verbose(verbose)
def set_git_executable(self, executable):
self.git.set_excutable(excutable)
@staticmethod
def print_command_summary(name, results):
"""
Print the results of running commands.
first the command line itself
and the error code if it's non-zero
then the stdout & stderr values from running that command
:param name:
:param results:
:return: True if any command exited with an error condition
"""
error_found = False
print "============================"
print "Command output for {0}".format(name)
if 'commands' in results[name]:
commands = results[name]['commands']
for command in commands:
for key in ['command', 'stdout', 'stderr']:
if key in command:
if command[key] != '':
print command[key]
if key == 'command':
if command['return_code'] != 0:
error_found = True
print "EXITED: {0}".format(
command['return_code'])
else:
print "SUCCEED"
return error_found
def clone_repo_list(self, repo_list, dest_dir, jobs=1):
"""
check out repository to dest dir based on repo list
:param repo_list: a list of repository entry which should contain:
'repository': the url of repository, it is required
'branch': the branch to be check out, it is optional
'commit-id': the commit id to be reset, it is optional
:param dest_dir: the directory where repository will be check out
:param jobs: Number of parallel jobs to run
:return:
"""
cloner = RepoCloner(jobs)
if cloner is not None:
for repo in repo_list:
data = {
'repo': repo,
'builddir': dest_dir,
'credentials': self._git_credentials
}
cloner.add_task(data)
cloner.finish()
results = cloner.get_results()
error = False
for name in results.keys():
error |= self.print_command_summary(name, results)
if error:
raise RuntimeError("Failed to clone repositories")
def clone_repo(self, repo_url, dest_dir, repo_commit="HEAD"):
"""
check out a repository to dest directory from the repository url
:param repo_url: the url of repository, it is required
:param dest_dir: the directory where repository will be check out
:return: the directory of the repository
"""
repo = {}
repo["repository"] = repo_url
repo["commit-id"] = repo_commit
repo_list = [repo]
self.clone_repo_list(repo_list, dest_dir)
repo_directory_name = strip_suffix(os.path.basename(repo_url), ".git")
return os.path.join(dest_dir, repo_directory_name)
def get_latest_commit_date(self, repo_dir):
"""
:param repo_dir: path of the repository
:return: commit-date
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError(
"The repository directory {0} is not specified. or its format is wrong"
.format(repo_dir))
return_code, output, error = self.git.run(
['show', '-s', '--pretty=format:%ct'], directory=repo_dir)
if return_code == 0:
return output.strip()
else:
raise RuntimeError(
"Unable to get commit date in directory {0}".format(repo_dir))
def get_latest_commit_id(self, repo_dir):
"""
:param repo_dir: path of the repository
:return: commit-id
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError(
"The repository directory {0} is not specified. or its format is wrong"
.format(repo_dir))
return_code, output, error = self.git.run(
['log', '--format=format:%H', '-n', '1'], directory=repo_dir)
if return_code == 0:
return output.strip()
else:
raise RuntimeError(
"Unable to get commit id in directory {0}".format(repo_dir))
def get_latest_merge_commit_before_date(self, repo_dir, date):
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError(
"The repository directory {0} is not specified. or its format is wrong"
.format(repo_dir))
return_code, output, error = self.git.run([
'log', '--merges', '--format=format:%H', '--before=' + date, '-n',
'1'
],
directory=repo_dir)
if return_code == 0:
return output.strip()
else:
raise RuntimeError("Unable to get commit id before {date} in directory {repo_dir}"\
.format(date=date, repo_dir=repo_dir))
def get_latest_author_commit_before_date(self, repo_dir, date, author):
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError(
"The repository directory {0} is not specified. or its format is wrong"
.format(repo_dir))
return_code, output, error = self.git.run([
'log', '--author=' + author, '--format=format:%H',
'--before=' + date, '-n', '1'
],
directory=repo_dir)
if return_code == 0:
return output.strip()
else:
raise RuntimeError("Unable to get commit id of {author} before {date} in directory {repo_dir}"\
.format(author=author, date=date, repo_dir=repo_dir))
def get_newer_commit(self, repo_dir, commit1, commit2):
'''
if commit1 is newer than commit2, return True
else, return False
'''
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError(
"The repository directory {0} is not specified. or its format is wrong"
.format(repo_dir))
return_code, output, error = self.git.run(
['rev-list', commit1 + '..' + commit2, '--count'],
directory=repo_dir)
if return_code == 0:
if output.strip() == "0":
return commit1
else:
return commit2
else:
raise RuntimeError("Unable to get any commit between {commit1} and {commit2} in directory {repo_dir}"\
.format(commit1=commit1, commit2=commit2, repo_dir=repo_dir))
def get_commit_message(self, repo_dir, commit):
"""
:param repo_dir: path of the repository
:param commit: the commit id of the repository
:return: commit-message
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError(
"The repository directory {0} is not specified. or its format is wrong"
.format(repo_dir))
return_code, output, error = self.git.run(
['log', '--format=format:%B', '-n', '1', commit],
directory=repo_dir)
if return_code == 0:
return output.strip()
else:
raise RuntimeError("Unable to get commit message of {commit_id} in directory {repo_dir}"\
.format(commit_id=commit, repo_dir=repo_dir))
def get_repo_url(self, repo_dir):
"""
get the remote url of the repository
:param repo_dir: the directory of the repository
:return: repository url
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError(
"The repository directory {0} is not specified. or its format is wrong"
.format(repo_dir))
return_code, output, error = self.git.run(['ls-remote', '--get-url'],
directory=repo_dir)
if return_code == 0:
return output.strip()
else:
raise RuntimeError(
"Unable to find the repository url in directory {0}".format(
repo_dir))
def get_current_branch(self, repo_dir):
"""
get the current branch name of the repository
:param repo_dir: the directory of the repository
:return: branch name
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError(
"The repository directory {0} is not specified. or its format is wrong"
.format(repo_dir))
return_code, output, error = self.git.run(
['symbolic-ref', '--short', 'HEAD'], directory=repo_dir)
if return_code == 0:
return output.strip()
else:
raise RuntimeError(
"Unable to find the current branch in directory {0}".format(
repo_dir))
def check_branch(self, repo_url, branch):
"""
Checks if the specified branch name exists for the provided repository. Leave only the characters
following the final "/". This is to handle remote repositories.
Raise RuntimeError if it is not found.
:return: None
"""
if "/" in branch:
sliced_branch = branch.split("/")[-1]
else:
sliced_branch = branch
return_code, output, error = self.git.run(
['ls-remote', repo_url, 'heads/*{0}'.format(sliced_branch)])
if return_code is not 0 or output is '':
raise RuntimeError(
"The branch, '{0}', provided for '{1}', does not exist.".
format(branch, repo_url))
def set_repo_tagname(self, repo_url, repo_dir, tag_name):
"""
Sets tagname on the repo
:param repo_url: the url of the repository
:param repo_dir: the directory of the repository
:param tag_name: the tag name to be set
:return: None
"""
# See if that tag exists for the repo
return_code, output, error = self.git.run(["tag", "-l", tag_name],
repo_dir)
# Return if tag already exists, otherwise create it
if return_code == 0 and output != '':
print "Tag {0} already exists in {1}".format(output, repo_url)
return
else:
print "Creating tag {0} for repo {1}".format(tag_name, repo_url)
return_code, output, error = self.git.run(
["tag", "-a", tag_name, "-m", "\"Creating new tag\""],
repo_dir)
if return_code != 0:
raise RuntimeError(
"Failed to create tag {0} for {1}.\nError: {2}".format(
tag_name, repo_url, error))
return_code, output, error = self.git.run(
["push", "origin", "--tags"], repo_dir)
if return_code != 0:
raise RuntimeError(
"Failed to push tag {0} for {1}.\nError: {2}".format(
tag_name, repo_url, error))
def create_repo_branch(self, repo_url, repo_dir, branch_name):
"""
Creates branch on the repo
:param repo_url: the url of the repository
:param repo_dir: the directory of the repository
:param branch_name: the branch name to be set
:return: None
"""
# See if that branch exists for the repo
return_code, output, error = self.git.run(
["ls-remote", "--exit-code", "--heads", repo_url, branch_name],
repo_dir)
# Raise RuntimeError if branch already exists, otherwise create it
if return_code == 0 and output != '':
raise RuntimeError(
"Error: Branch {0} already exists - exiting now...".format(
output))
else:
print "Creating branch {0} for repo {1}".format(
branch_name, repo_url)
return_code, output, error = self.git.run(["branch", branch_name],
repo_dir)
if return_code != 0:
print output
raise RuntimeError(
"Error: Failed to create local branch {0} with error: {1}."
.format(branch_name, error))
return_code, output, error = self.git.run(
["push", "-u", "origin", branch_name], repo_dir)
if return_code != 0:
print output
raise RuntimeError(
"Error: Failed to publish local branch {0} with error: {1}"
.format(branch_name, error))
def checkout_repo_branch(self, repo_dir, branch_name):
"""
Check out to specify branch on the repo
:param repo_dir: the directory of the repository
:param branch_name: the branch name to be checked
:return: None
"""
return_code, output, error = self.git.run(["checkout", branch_name],
repo_dir)
if return_code != 0:
raise RuntimeError(
"Error: Failed to checkout branch {0}".format(output))
def push_repo_changes(self, repo_dir, commit_message, push_all=False):
"""
publish changes of reposioty
:param repo_dir: the directory of the repository
:param commit_message: the message to be added to commit
:return: None
"""
run_add = False
run_commit = False
status_code, status_out, status_error = self.git.run(['status'],
repo_dir)
if status_code == 0:
if "nothing to commit, working directory clean" in status_out and \
"Your branch is up-to-date with" in status_out :
print status_out
return
if "Changes not staged for commit" in status_out:
run_add = True
run_commit = True
if "Changes to be committed" in status_out:
run_commit = True
if run_add:
if push_all:
add_code, add_out, add_error = self.git.run(['add', '-A'],
repo_dir)
else:
add_code, add_out, add_error = self.git.run(['add', '-u'],
repo_dir)
if add_code != 0:
raise RuntimeError('Unable to add files for commiting.\n{0}\n{1}\n{2}'.format\
(add_code, add_out, add_error))
if run_commit:
commit_code, commit_out, commit_error = self.git.run(
['commit', '-m', commit_message], repo_dir)
if commit_code != 0:
raise RuntimeError('Unable to commit changes for pushing.\n{0}\n{1}\n{2}'.format\
(commit_code, commit_out, commit_error))
push_code, push_out, push_error = self.git.run(['push'], repo_dir)
if push_code != 0:
raise RuntimeError('Unable to push changes.\n{0}\n{1}\n{2}'.format(
push_code, push_out, push_error))
return
class Manifest(object):
def __init__(self, file_path, git_credentials=None):
"""
__build_name - Refer to the field "build-name" in manifest file.
__repositories - Refer to the field "repositories" in manifest file.
__downstream_jobs - Refer to the field "downstream-jobs" in manifest file.
__file_path - The file path of the manifest file
__name - The file name of the manifest file
__manifest -The content of the manifest file
__changed - If manifest is changed, be True; The default value is False
__git_credentials - URL, credentials pair for the access to github repos
gitbit - Class instance of gitbit
"""
self._build_name = None
self._build_requirements = None
self._repositories = []
self._downstream_jobs = []
self._file_path = file_path
self._name = file_path.split('/')[-1]
self._manifest = None
self._changed = False
self._git_credentials = None
self.gitbit = GitBit(verbose=True)
if git_credentials:
self._git_credentials = git_credentials
self.setup_gitbit()
self.read_manifest_file(self._file_path)
self.parse_manifest()
@staticmethod
def instance_of_sample(manifest_sample="manifest.json"):
repo_dir = os.path.dirname(sys.path[0])
for subdir, dirs, files in os.walk(repo_dir):
for file in files:
if file == manifest_sample:
manifest = Manifest(os.path.join(subdir, file))
return manifest
return None
def set_git_credentials(self, git_credentials):
self._git_credentials = git_credentials
self.setup_gitbit()
@property
def downstream_jobs(self):
return self._downstream_jobs
@property
def repositories(self):
return self._repositories
@property
def manifest(self):
return self._manifest
@property
def name(self):
return self._name
@property
def file_path(self):
return self._file_path
@property
def build_name(self):
return self._build_name
@build_name.setter
def build_name(self, build_name):
self._build_name = build_name
self._manifest['build-name'] = build_name
@property
def build_requirements(self):
return self._build_requirements
@build_requirements.setter
def build_requirements(self, requirements):
self._manifest['build-requirements'] = requirements
self._build_requirements = requirements
@property
def changed(self):
return self._changed
def setup_gitbit(self):
"""
Set gitbit credentials.
:return: None
"""
if self._git_credentials:
for url_cred_pair in self._git_credentials:
url, cred = url_cred_pair.split(',')
self.gitbit.add_credential_from_variable(url, cred)
def read_manifest_file(self, filename):
"""
Reads the manifest file json data to class member _manifest.
:param filename: where to read the manifest data from
:return: None
"""
if not os.path.isfile(filename):
raise KeyError(
"No file found for manifest at {0}".format(filename))
with open(filename, "r") as manifest_file:
self._manifest = json.load(manifest_file)
def parse_manifest(self):
"""
parse manifest and assign properties
:return: None
"""
if 'build-name' in self._manifest:
self._build_name = self._manifest['build-name']
if 'build-requirements' in self._manifest:
self._build_requirements = self._manifest['build-requirements']
if 'repositories' in self._manifest:
for repo in self._manifest['repositories']:
self._repositories.append(repo)
if 'downstream-jobs' in self._manifest:
for job in self._manifest['downstream-jobs']:
self._downstream_jobs.append(job)
@staticmethod
def validate_repositories(repositories):
"""
validate whether the entry 'repositories' contains useful information
return: True if the entry is valid,
False if the entry is unusable
and message including omission imformation
"""
result = True
message = []
for repo in repositories:
valid = True
# repository url is required
if 'repository' not in repo or repo['repository'] == "":
valid = False
message.append("entry without tag repository")
# either branch or commit-id should be set.
if ('branch' not in repo or repo['branch'] == "") and \
('commit-id' not in repo or repo['commit-id'] == ""):
valid = False
message.append(
"Either branch or commit-id should be set for entry")
if not valid:
result = False
message.append("entry content:")
message.append("{0}".format(json.dumps(repo, indent=True)))
return result, message
@staticmethod
def validate_downstream_jobs(downstream_jobs):
"""
validate whether the entry 'downstream-jobs' contains useful information
return: True if the entry is valid,
False if the entry is unusable
and message including omission imformation
"""
result = True
message = []
for job in downstream_jobs:
valid = True
# repository url is required
if 'repository' not in job or job['repository'] == "":
valid = False
message.append("entry without tag repository")
#command is required
if 'command' not in job:
valid = False
message.append("entry without tag command")
#working-directory is required
if 'working-directory' not in job:
valid = False
message.append("entry without tag working-directory")
#running-label is required
if 'running-label' not in job:
valid = False
message.append("entry without tag running-label")
# either branch or commit-id should be set.
if ('branch' not in job or job['branch'] == "") and \
('commit-id' not in job or job['commit-id'] == ""):
valid = False
message.append(
"Either commit-id or branch should be set for job repository"
)
# downstream-jobs is optional
# if it is specified, the value should be validate
if 'downstream-jobs' in job:
downstream_result, downstream_message = Manifest.validate_downstream_jobs(
job['downstream-jobs'])
if not downstream_result:
valid = False
message.extend(downstream_message)
if not valid:
result = False
message.append("entry content:")
message.append("{0}".format(json.dumps(job, indent=True)))
return result, message
def validate_manifest(self):
"""
Identify whether the manifest contains useful information (as we understand it)
raise error if manifest is unusable
:return: None
"""
result = True
messages = ["Validate manifest file: {0}".format(self._name)]
if self._manifest is None:
result = False
messages.append("No manifest contents")
#build-name is required
if 'build-name' not in self._manifest:
result = False
messages.append("No build-name in manifest file")
#repositories is required
if 'repositories' not in self._manifest:
result = False
messages.append("No repositories in manifest file")
else:
r, m = self.validate_repositories(self._repositories)
if not r:
result = False
messages.extend(m)
#downstream-jobs is required
if 'downstream-jobs' not in self._manifest:
result = False
messages.append("No downstream-jobs in manifest file")
else:
r, m = Manifest.validate_downstream_jobs(self._downstream_jobs)
if not r:
result = False
messages.extend(m)
#build-requirements is required
if 'build-requirements' not in self._manifest:
result = False
messages.append("No build-requirements in manifest file")
if not result:
messages.append("manifest file {0} is not valid".format(
self._name))
error = '\n'.join(messages)
raise KeyError(error)
@staticmethod
def check_commit_changed(repo, repo_url, branch, commit):
"""
Check whether the repository is changed based on its url, branch ,commit-id and arguments repo_url, branch, commit
:param repo: an repository entry of member _repositories or _downstream_jobs
:param repo_url: the url of the repository
:param branch: the branch of the repository
:param commit: the commit id of the repository
:return: True when the commit-id is different with argument commit
and the url and branch is the same with the arguments repo_url, branch;
otherwise, False
"""
if (repo['repository'] == repo_url):
# If repo has "branch", compare "commit-id" in repo with the argument commit
# only when "branch" is the same with argument branch.
if 'branch' in repo:
sliced_repo_branch = repo['branch'].split("/")[-1]
sliced_branch = branch.split("/")[-1]
if (repo['branch'] == branch or repo['branch'] == sliced_branch
or sliced_repo_branch == sliced_branch):
if 'commit-id' in repo:
print "checking the commit-id for {0} with branch {1} from {2} to {3}".format\
(repo_url, branch, repo['commit-id'], commit)
if repo['commit-id'] != commit:
print " commit-id updated!"
return True
else:
print " commit-id unchanged"
return False
else:
print "add commit-id{0} for {1} with branch {2} ".format\
(commit, repo_url, branch)
return True
# If repo doesn't have "branch", compare "commit-id" in repo with argument commit
# Exits with 1 if repo doesn't have "commit-id"
else:
if 'commit-id' not in repo:
raise KeyError(
"Neither commit-id nor branch is set for repository {0}"
.format(repo['repository']))
else:
if repo['commit-id'] != commit:
print " commit-id updated!"
return True
else:
print " commit-id unchanged"
return False
return False
@staticmethod
def check_branch_changed(repo, repo_url, branch, commit):
"""
Check whether the repository is changed based on its url, branch ,commit-id and arguments repo_url, branch, commit
:param repo: an repository entry of member _repositories or _downstream_jobs
:param repo_url: the url of the repository
:param branch: the branch of the repository
:param commit: the commit id of the repository
:return: True when the branch is different with argument branch
and the url and commit is the same with the arguments repo_url, commit;
otherwise, False
"""
if (repo['repository'] == repo_url):
# If repo has "branch", compare "commit-id" in repo with the argument commit
# only when "branch" is the same with argument branch.
if 'commit-id' in repo:
if repo['commit-id'] != commit:
return False
# Exits with 1 if repo doesn't have "branch" and "commit-id"
elif 'branch' not in repo:
raise KeyError(
"Neither commit-id nor branch is set for repository {0}".
format(repo['repository']))
if 'branch' in repo:
print "checking the branch for {0} with commit {1} from {2} to {3}".format\
(repo_url, commit, repo['branch'], branch )
sliced_repo_branch = repo['branch'].split("/")[-1]
sliced_branch = branch.split("/")[-1]
if (repo['branch'] != branch
and repo['branch'] != sliced_branch
and sliced_repo_branch != sliced_branch):
print " branch updated!"
return True
else:
print " branch unchanged"
return False
else:
print "add branch{0} for {1} with commit {2} ".format\
(branch, repo_url, commit)
return True
@staticmethod
def check_under_test_changed(repo, repo_url, branch, commit, under_test):
"""
Check whether the under_test is changed based on its url, branch ,commit-id and arguments repo_url, branch, commit
:param repo: an repository entry of member _repositories or _downstream_jobs
:param repo_url: the url of the repository
:param branch: the branch of the repository
:param commit: the commit id of the repository
:param under_test: the if under test of the repository
:return: True when the under-test is different with argument under_test
and the url and commit, branch are the same with the arguments repo_url, commit, branch;
otherwise, False
"""
if (repo['repository'] == repo_url):
# If repo has "commit-id", and "branch", compare "under-test" in repo with the argument commit
# only when "commit-id" is the same with argument commit.
if 'commit-id' in repo:
if repo['commit-id'] != commit:
return False
if 'branch' in repo:
sliced_repo_branch = repo['branch'].split("/")[-1]
sliced_branch = branch.split("/")[-1]
if (repo['branch'] != branch
and repo['branch'] != sliced_branch
and sliced_repo_branch != sliced_branch):
return False
if 'commit-id' not in repo and 'branch' not in repo:
if 'under-test' not in repo:
raise KeyError(
"Neither commit-id nor branch nor under test is set for repository {0}"
.format(repo['repository']))
if 'under-test' not in repo:
print "add under-test {0} for {1} with commit {2} ".format\
(under_test, repo_url, commit)
return True
else:
if repo['under-test'] != under_test:
print " under_test updated!"
return True
else:
print " under_test unchanged"
return False
@staticmethod
def update_downstream_jobs(downstream_jobs, repo_url, branch, commit):
"""
update the instance of the class based on member:
_downstream_jobs and provided arguments.
:param downstream_jobs: the entry downstream_jobs
:param repo_url: the url of the repository
:param branch: the branch of the repository
:param commit: the commit id of the repository
:return: True if any job in downstream_jobs is updated
False if none of jobs in downstream_jobs is updated
"""
updated = False
for job in downstream_jobs:
if Manifest.check_commit_changed(job, repo_url, branch, commit):
job['commit-id'] = commit
updated = True
if 'downstream-jobs' in job:
nested_downstream_jobs = job['downstream-jobs']
if Manifest.update_downstream_jobs(nested_downstream_jobs,
repo_url, branch, commit):
updated = True
return updated
@staticmethod
def update_repositories(repositories,
repo_url,
branch,
commit,
under_test=False):
"""
update the instance of the class based on member:
_repositories and provided arguments.
:param repositories: the entry repositories
:param repo_url: the url of the repository
:param branch: the branch of the repository
:param commit: the commit id of the repository
:return:
"""
updated = False
for repo in repositories:
if Manifest.check_commit_changed(repo, repo_url, branch, commit):
repo['commit-id'] = commit
updated = True
if Manifest.check_branch_changed(repo, repo_url, branch, commit):
repo['branch'] = branch
updated = True
if under_test:
if Manifest.check_under_test_changed(repo, repo_url, branch,
commit, under_test):
repo['under-test'] = under_test
updated = True
return updated
def update_manifest(self, repo_url, branch, commit, under_test=False):
"""
update the instance of the class based on members
_repositories , _downstream_jobs and provided arguments.
:param repo_url: the url of the repository
:param branch: the branch of the repository
:param commit: the commit id of the repository
:return:
"""
print "start updating manifest file {0}".format(self._name)
if self.update_repositories(self._repositories, repo_url, branch,
commit, under_test):
self._changed = True
if self.update_downstream_jobs(self._downstream_jobs, repo_url, branch,
commit):
self._changed = True
def write_manifest_file(self, file_path=None, dryrun=False):
"""
Add, commit, and push the manifest changes to the manifest repo.
:param file_path: String, The path to the temporary file.
If it is not set, the default value is self._file_path where manifest come from
:param dry_run: If true, would not push changes
:return:
"""
self.dump_to_json_file(file_path)
return
def dump_to_json_file(self, file_path=None):
"""
dump manifest json to a file
"""
if file_path is None:
file_path = self._file_path
with open(file_path, 'w') as fp:
json.dump(self._manifest, fp, indent=4, sort_keys=True)
def do_one_task(self, name, data, results):
"""
Perform the actual work of checking out a repository. This portion of the
task is performed in a subprocess, and may be performed in parallel with other
instances.
name and data will come from the values passed in to add_task()
:param name:
:param data:
data should contain:
'credentials': a list of Git credentials in URL:VARIABLE_NAME format
'repo': a repository entry from a manifest file
'builddir': the location to check out the repository into
:param results: a shared dictionary for storing results and sharing them to the
parent process
:return: None (all output data stored in results)
"""
# make sure we have all of the right data that we need to start the build
if name is None or data is None:
raise ValueError("name or data not present")
for key in ['repo', 'builddir']:
if key not in data:
raise ValueError("{0} key missing from data: {1}".format(key, data))
repo = data['repo']
if 'repository' not in repo:
raise ValueError("no repository in work {0}".format(repo))
# data validation okay, so start the work
print "Starting checkout of {0}".format(name)
# someplace to start storing results of the commands that will be run
results['commands'] = []
git = GitBit(verbose=False)
if 'credentials' in data and data['credentials'] is not None:
for credential in data['credentials']:
url, cred = credential.split(',', 2)
git.add_credential_from_variable(url, cred)
repo_url = repo['repository']
destination_directory_name = strip_suffix(os.path.basename(repo_url), ".git")
# build up a git clone command line
# clone [ -b branchname ] repository_url [ destination_name ]
command = ['clone']
#clone big files with git-lfs is much faster
if repo.has_key('lfs') and repo['lfs']:
command = ['lfs', 'clone']
if 'branch' in repo and repo['branch'] != "":
command.extend(['-b', repo['branch']])
command.append(repo_url)
if 'checked-out-directory-name' in repo:
# this specifies what the directory name of the checked out repository
# should be, as opposed to using Git's default (the basename of the repository URL)
# note to self: do not combine the following two lines again
destination_directory_name = repo['checked-out-directory-name']
command.append(destination_directory_name)
destination_directory = os.path.abspath(data['builddir']) + "/" + destination_directory_name
if os.path.isdir(destination_directory):
shutil.rmtree(destination_directory)
self.run_git_command(git,command,data['builddir'],results)
# the clone has been performed -- now check to see if we need to move the HEAD
# to point to a specific location within the tree history. That will be true
# if there is a commit-id or tag value specified in the repository (which will
# be the case most of the time).
reset_id = self._get_reset_value(repo)
if reset_id is not None:
working_directory = os.path.join(data['builddir'], destination_directory_name)
command = ["fetch", "origin", "refs/pull/*:refs/remotes/origin/pr/*"]
self.run_git_command(git,command,working_directory,results)
command = ["reset", "--hard", reset_id]
self.run_git_command(git,command,working_directory,results)
results['status'] = "success"
class RepoOperator(object):
def __init__(self, git_credentials=None):
"""
Create a repository interface object
:return:
"""
self._git_credentials = git_credentials
self.git = GitBit(verbose=True)
if self._git_credentials:
self.setup_gitbit()
def setup_gitbit(self, credentials=None):
"""
Set gitbit credentials.
:return:
"""
if credentials is None:
if self._git_credentials is None:
return
else:
credentials = self._git_credentials
else:
self._git_credentials = credentials
for url_cred_pair in credentials:
url, cred = url_cred_pair.split(',')
self.git.add_credential_from_variable(url, cred)
def set_git_dryrun(self, dryrun):
self.git.set_dryrun(dryrun)
def set_git_verbose(self, verbose):
self.git.set_verbose(verbose)
def set_git_executable(self, executable):
self.git.set_excutable(excutable)
@staticmethod
def print_command_summary(name, results):
"""
Print the results of running commands.
first the command line itself
and the error code if it's non-zero
then the stdout & stderr values from running that command
:param name:
:param results:
:return: True if any command exited with an error condition
"""
error_found = False
print "============================"
print "Command output for {0}".format(name)
if 'commands' in results[name]:
commands = results[name]['commands']
for command in commands:
for key in ['command', 'stdout', 'stderr']:
if key in command:
if command[key] != '':
print command[key]
if key == 'command':
if command['return_code'] != 0:
error_found = True
print "EXITED: {0}".format(command['return_code'])
else:
print "SUCCEED"
return error_found
def clone_repo_list(self, repo_list, dest_dir, jobs=1):
"""
check out repository to dest dir based on repo list
:param repo_list: a list of repository entry which should contain:
'repository': the url of repository, it is required
'branch': the branch to be check out, it is optional
'commit-id': the commit id to be reset, it is optional
:param dest_dir: the directory where repository will be check out
:param jobs: Number of parallel jobs to run
:return:
"""
cloner = RepoCloner(jobs)
if cloner is not None:
for repo in repo_list:
data = {'repo': repo,
'builddir': dest_dir,
'credentials': self._git_credentials
}
cloner.add_task(data)
cloner.finish()
results = cloner.get_results()
error = False
for name in results.keys():
error |= self.print_command_summary(name, results)
if error:
raise RuntimeError("Failed to clone repositories")
def clone_repo(self, repo_url, dest_dir, repo_commit="HEAD"):
"""
check out a repository to dest directory from the repository url
:param repo_url: the url of repository, it is required
:param dest_dir: the directory where repository will be check out
:return: the directory of the repository
"""
repo = {}
repo["repository"] = repo_url
repo["commit-id"] = repo_commit
repo_list = [repo]
self.clone_repo_list(repo_list, dest_dir)
repo_directory_name = strip_suffix(os.path.basename(repo_url), ".git")
return os.path.join(dest_dir, repo_directory_name)
def get_latest_commit_date(self, repo_dir):
"""
:param repo_dir: path of the repository
:return: commit-date
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError("The repository directory {0} is not specified. or its format is wrong".format(repo_dir))
return_code, output, error = self.git.run(['show', '-s', '--pretty=format:%ct'], directory=repo_dir)
if return_code == 0:
return output.strip()
else:
raise RuntimeError("Unable to get commit date in directory {0}".format(repo_dir))
def get_latest_commit_id(self, repo_dir):
"""
:param repo_dir: path of the repository
:return: commit-id
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError("The repository directory {0} is not specified. or its format is wrong".format(repo_dir))
return_code, output, error = self.git.run(['log', '--format=format:%H', '-n', '1'], directory=repo_dir)
if return_code == 0:
return output.strip()
else:
raise RuntimeError("Unable to get commit id in directory {0}".format(repo_dir))
def get_latest_merge_commit_before_date(self, repo_dir, date):
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError("The repository directory {0} is not specified. or its format is wrong".format(repo_dir))
return_code, output, error = self.git.run(['log', '--merges', '--format=format:%H', '--before='+date, '-n', '1'], directory=repo_dir)
if return_code == 0:
return output.strip()
else:
raise RuntimeError("Unable to get commit id before {date} in directory {repo_dir}"\
.format(date=date, repo_dir=repo_dir))
def get_latest_author_commit_before_date(self, repo_dir, date, author):
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError("The repository directory {0} is not specified. or its format is wrong".format(repo_dir))
return_code, output, error = self.git.run(['log', '--author='+author, '--format=format:%H', '--before='+date, '-n', '1'], directory=repo_dir)
if return_code == 0:
return output.strip()
else:
raise RuntimeError("Unable to get commit id of {author} before {date} in directory {repo_dir}"\
.format(author=author, date=date, repo_dir=repo_dir))
def get_newer_commit(self, repo_dir, commit1, commit2):
'''
if commit1 is newer than commit2, return True
else, return False
'''
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError("The repository directory {0} is not specified. or its format is wrong".format(repo_dir))
return_code, output, error = self.git.run(['rev-list', commit1+'..'+commit2, '--count'], directory=repo_dir)
if return_code == 0:
if output.strip() == "0":
return commit1
else:
return commit2
else:
raise RuntimeError("Unable to get any commit between {commit1} and {commit2} in directory {repo_dir}"\
.format(commit1=commit1, commit2=commit2, repo_dir=repo_dir))
def get_commit_message(self, repo_dir, commit):
"""
:param repo_dir: path of the repository
:param commit: the commit id of the repository
:return: commit-message
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError("The repository directory {0} is not specified. or its format is wrong".format(repo_dir))
return_code, output, error = self.git.run(['log', '--format=format:%B', '-n', '1', commit], directory=repo_dir)
if return_code == 0:
return output.strip()
else:
raise RuntimeError("Unable to get commit message of {commit_id} in directory {repo_dir}"\
.format(commit_id=commit, repo_dir=repo_dir))
def get_repo_url(self, repo_dir):
"""
get the remote url of the repository
:param repo_dir: the directory of the repository
:return: repository url
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError("The repository directory {0} is not specified. or its format is wrong".format(repo_dir))
return_code, output, error = self.git.run(['ls-remote', '--get-url'], directory=repo_dir)
if return_code == 0:
return output.strip()
else:
raise RuntimeError("Unable to find the repository url in directory {0}".format(repo_dir))
def get_current_branch(self, repo_dir):
"""
get the current branch name of the repository
:param repo_dir: the directory of the repository
:return: branch name
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError("The repository directory {0} is not specified. or its format is wrong".format(repo_dir))
return_code, output, error = self.git.run(['symbolic-ref', '--short', 'HEAD'], directory=repo_dir)
if return_code == 0:
return output.strip()
else:
raise RuntimeError("Unable to find the current branch in directory {0}".format(repo_dir))
def check_branch(self, repo_url, branch):
"""
Checks if the specified branch name exists for the provided repository. Leave only the characters
following the final "/". This is to handle remote repositories.
Raise RuntimeError if it is not found.
:return: None
"""
if "/" in branch:
sliced_branch = branch.split("/")[-1]
else:
sliced_branch = branch
return_code, output, error = self.git.run(['ls-remote', repo_url, 'heads/*{0}'.format(sliced_branch)])
if return_code is not 0 or output is '':
raise RuntimeError("The branch, '{0}', provided for '{1}', does not exist."
.format(branch, repo_url))
def set_repo_tagname(self, repo_url, repo_dir, tag_name):
"""
Sets tagname on the repo
:param repo_url: the url of the repository
:param repo_dir: the directory of the repository
:param tag_name: the tag name to be set
:return: None
"""
# See if that tag exists for the repo
return_code, output, error = self.git.run(["tag", "-l", tag_name], repo_dir)
# Return if tag already exists, otherwise create it
if return_code == 0 and output != '':
print "Tag {0} already exists in {1}".format(output, repo_url)
return
else:
print "Creating tag {0} for repo {1}".format(tag_name, repo_url)
return_code, output, error = self.git.run(["tag", "-a", tag_name, "-m", "\"Creating new tag\""], repo_dir)
if return_code != 0:
raise RuntimeError("Failed to create tag {0} for {1}.\nError: {2}".format(tag_name, repo_url, error))
return_code, output, error = self.git.run(["push", "origin", "--tags"], repo_dir)
if return_code != 0:
raise RuntimeError("Failed to push tag {0} for {1}.\nError: {2}".format(tag_name, repo_url, error))
def create_repo_branch(self, repo_url, repo_dir, branch_name):
"""
Creates branch on the repo
:param repo_url: the url of the repository
:param repo_dir: the directory of the repository
:param branch_name: the branch name to be set
:return: None
"""
# See if that branch exists for the repo
return_code, output, error = self.git.run(["ls-remote", "--exit-code", "--heads", repo_url, branch_name], repo_dir)
# Raise RuntimeError if branch already exists, otherwise create it
if return_code == 0 and output != '':
raise RuntimeError("Error: Branch {0} already exists - exiting now...".format(output))
else:
print "Creating branch {0} for repo {1}".format(branch_name, repo_url)
return_code, output, error = self.git.run(["branch", branch_name], repo_dir)
if return_code != 0:
print output
raise RuntimeError("Error: Failed to create local branch {0} with error: {1}.".format(branch_name, error))
return_code, output, error = self.git.run(["push", "-u", "origin", branch_name], repo_dir)
if return_code != 0:
print output
raise RuntimeError("Error: Failed to publish local branch {0} with error: {1}".format(branch_name, error))
def checkout_repo_branch(self, repo_dir, branch_name):
"""
Check out to specify branch on the repo
:param repo_dir: the directory of the repository
:param branch_name: the branch name to be checked
:return: None
"""
return_code, output, error = self.git.run(["checkout", branch_name], repo_dir)
if return_code != 0:
raise RuntimeError("Error: Failed to checkout branch {0}".format(output))
def push_repo_changes(self, repo_dir, commit_message, push_all=False):
"""
publish changes of reposioty
:param repo_dir: the directory of the repository
:param commit_message: the message to be added to commit
:return: None
"""
run_add = False
run_commit = False
status_code, status_out, status_error = self.git.run(['status'], repo_dir)
if status_code == 0:
if "nothing to commit, working directory clean" in status_out and \
"Your branch is up-to-date with" in status_out :
print status_out
return
if "Changes not staged for commit" in status_out:
run_add = True
run_commit = True
if "Changes to be committed" in status_out:
run_commit = True
if run_add:
if push_all:
add_code, add_out, add_error = self.git.run(['add', '-A'], repo_dir)
else:
add_code, add_out, add_error = self.git.run(['add', '-u'], repo_dir)
if add_code != 0:
raise RuntimeError('Unable to add files for commiting.\n{0}\n{1}\n{2}'.format\
(add_code, add_out, add_error))
if run_commit:
commit_code, commit_out, commit_error = self.git.run(['commit', '-m', commit_message], repo_dir)
if commit_code != 0:
raise RuntimeError('Unable to commit changes for pushing.\n{0}\n{1}\n{2}'.format\
(commit_code, commit_out, commit_error))
push_code, push_out, push_error = self.git.run(['push'], repo_dir)
if push_code !=0:
raise RuntimeError('Unable to push changes.\n{0}\n{1}\n{2}'.format(push_code, push_out, push_error))
return
class Manifest(object):
def __init__(self, file_path, git_credentials = None):
"""
__build_name - Refer to the field "build-name" in manifest file.
__repositories - Refer to the field "repositories" in manifest file.
__downstream_jobs - Refer to the field "downstream-jobs" in manifest file.
__file_path - The file path of the manifest file
__name - The file name of the manifest file
__manifest -The content of the manifest file
__git_credentials - URL, credentials pair for the access to github repos
gitbit - Class instance of gitbit
"""
self._build_name = None
self._build_requirements = None
self._repositories = []
self._downstream_jobs = []
self._file_path = file_path
self._name = file_path.split('/')[-1]
self._manifest = None
self._git_credentials = None
self.gitbit = GitBit(verbose=True)
if git_credentials:
self._git_credentials = git_credentials
self.setup_gitbit()
self.read_manifest_file(self._file_path)
self.parse_manifest()
def set_git_credentials(self, git_credentials):
self._git_credentials.append = git_credentials
self.setup_gitbit()
def get_downstream_jobs(self):
return self._downstream_jobs
def get_repositories(self):
return self._repositories
def get_manifest(self):
return self._manifest
def get_name(self):
return self._name
def get_file_path(self):
return self._file_path
def get_build_name(self):
return self._build_name
def get_build_requirements(self):
return self._build_requirements
def setup_gitbit(self):
"""
Set gitbit credentials.
:return:
"""
self.gitbit.set_identity(config.gitbit_identity['username'], config.gitbit_identity['email'])
if self._git_credentials:
for url_cred_pair in self._git_credentials:
url, cred = url_cred_pair.split(',')
self.gitbit.add_credential_from_variable(url, cred)
def read_manifest_file(self, filename):
"""
Reads the manifest file json data to class member _manifest.
:param filename: where to read the manifest data from
:return: None
"""
if not os.path.isfile(filename):
raise KeyError("No file found for manifest at {0}".format(filename))
with open(filename, "r") as manifest_file:
self._manifest = json.load(manifest_file)
def parse_manifest(self):
"""
parse manifest and assign properties
"""
if 'build-name' in self._manifest:
self._build_name = self._manifest['build-name']
if 'build-requirements' in self._manifest:
self._build_requirements = self._manifest['build-requirements']
if 'repositories' in self._manifest:
for repo in self._manifest['repositories']:
self._repositories.append(repo)
if 'downstream-jobs' in self._manifest:
for job in self._manifest['downstream-jobs']:
self._downstream_jobs.append(job)
@staticmethod
def validate_repositories(repositories):
"""
validate whether the entry 'repositories' contains useful information
return: True if the entry is valid,
False if the entry is unusable
and message including omission imformation
"""
result = True
message = []
for repo in repositories:
valid = True
# repository url is required
if 'repository' not in repo:
valid = False
message.append("entry without tag repository")
# either branch or commit-id should be set.
if 'branch' not in repo and \
'commit-id' not in repo:
valid = False
message.append("Either branch or commit-id should be set for entry")
if not valid:
result = False
message.append("entry content:")
message.append("{0}".format(json.dumps(repo, indent=True)))
return result, message
@staticmethod
def validate_downstream_jobs(downstream_jobs):
"""
validate whether the entry 'downstream-jobs' contains useful information
return: True if the entry is valid,
False if the entry is unusable
and message including omission imformation
"""
result = True
message = []
for job in downstream_jobs:
valid = True
# repository url is required
if 'repository' not in job:
valid = False
message.append("entry without tag repository")
#command is required
if 'command' not in job:
valid = False
message.append("entry without tag command")
#working-directory is required
if 'working-directory' not in job:
valid = False
message.append("entry without tag working-directory")
#running-label is required
if 'running-label' not in job:
valid = False
message.append("entry without tag running-label")
# either branch or commit-id should be set.
if 'branch' not in job and \
'commit-id' not in job:
valid = False
message.append("Either commit-id or branch should be set for job repository")
# downstream-jobs is optional
# if it is specified, the value should be validate
if 'downstream-jobs' in job:
downstream_result, downstream_message = Manifest.validate_downstream_jobs(job['downstream-jobs'])
if not downstream_result:
valid = False
message.extend(downstream_message)
if not valid:
result = False
message.append("entry content:")
message.append("{0}".format(json.dumps(job, indent=True)))
return result, message
def validate_manifest(self):
"""
Identify whether the manifest contains useful information (as we understand it)
raise error if manifest is unusable
:return:
"""
result = True
message = ["Validate manifest file: {0}".format(self._name)]
if self._manifest is None:
result = False
message.append("No manifest contents")
#build-name is required
if 'build-name' not in self._manifest:
result = False
message.append("No build-name in manifest file")
#repositories is required
if 'repositories' not in self._manifest:
result = False
message.append("No repositories in manifest file")
else:
r, m = self.validate_repositories(self._repositories)
if not r:
result = False
message.extend(m)
#downstream-jobs is required
if 'downstream-jobs' not in self._manifest:
result = False
message.append("No downstream-jobs in manifest file")
else:
r, m = self.validate_downstream_jobs(self._downstream_jobs)
if not r:
result = False
message.extend(m)
#build-requirements is required
if 'build-requirements' not in self._manifest:
result = False
message.append("No build-requirements in manifest file")
if not result:
message.append("manifest file {0} is not valid".format(self._name))
error = '\n'.join(message)
raise KeyError(error)
class Manifest(object):
def __init__(self, file_path, git_credentials = None):
"""
__build_name - Refer to the field "build-name" in manifest file.
__repositories - Refer to the field "repositories" in manifest file.
__downstream_jobs - Refer to the field "downstream-jobs" in manifest file.
__file_path - The file path of the manifest file
__name - The file name of the manifest file
__manifest -The content of the manifest file
__changed - If manifest is changed, be True; The default value is False
__git_credentials - URL, credentials pair for the access to github repos
gitbit - Class instance of gitbit
"""
self._build_name = None
self._build_requirements = None
self._repositories = []
self._downstream_jobs = []
self._file_path = file_path
self._name = file_path.split('/')[-1]
self._manifest = None
self._changed = False
self._git_credentials = None
self.gitbit = GitBit(verbose=True)
if git_credentials:
self._git_credentials = git_credentials
self.setup_gitbit()
self.read_manifest_file(self._file_path)
self.parse_manifest()
@staticmethod
def instance_of_sample():
repo_dir = os.path.dirname(sys.path[0])
for subdir, dirs, files in os.walk(repo_dir):
for file in files:
if file == manifest_sample:
manifest = Manifest(os.path.join(subdir, file))
return manifest
return None
def set_git_credentials(self, git_credentials):
self._git_credentials = git_credentials
self.setup_gitbit()
@property
def downstream_jobs(self):
return self._downstream_jobs
@property
def repositories(self):
return self._repositories
@property
def manifest(self):
return self._manifest
@property
def name(self):
return self._name
@property
def file_path(self):
return self._file_path
@property
def build_name(self):
return self._build_name
@build_name.setter
def build_name(self, build_name):
self._build_name = build_name
self._manifest['build-name'] = build_name
@property
def build_requirements(self):
return self._build_requirements
@build_requirements.setter
def build_requirements(self, requirements):
self._manifest['build-requirements'] = requirements
self._build_requirements = requirements
@property
def changed(self):
return self._changed
def setup_gitbit(self):
"""
Set gitbit credentials.
:return: None
"""
self.gitbit.set_identity(config.gitbit_identity['username'], config.gitbit_identity['email'])
if self._git_credentials:
for url_cred_pair in self._git_credentials:
url, cred = url_cred_pair.split(',')
self.gitbit.add_credential_from_variable(url, cred)
def read_manifest_file(self, filename):
"""
Reads the manifest file json data to class member _manifest.
:param filename: where to read the manifest data from
:return: None
"""
if not os.path.isfile(filename):
raise KeyError("No file found for manifest at {0}".format(filename))
with open(filename, "r") as manifest_file:
self._manifest = json.load(manifest_file)
def parse_manifest(self):
"""
parse manifest and assign properties
:return: None
"""
if 'build-name' in self._manifest:
self._build_name = self._manifest['build-name']
if 'build-requirements' in self._manifest:
self._build_requirements = self._manifest['build-requirements']
if 'repositories' in self._manifest:
for repo in self._manifest['repositories']:
self._repositories.append(repo)
if 'downstream-jobs' in self._manifest:
for job in self._manifest['downstream-jobs']:
self._downstream_jobs.append(job)
@staticmethod
def validate_repositories(repositories):
"""
validate whether the entry 'repositories' contains useful information
return: True if the entry is valid,
False if the entry is unusable
and message including omission imformation
"""
result = True
message = []
for repo in repositories:
valid = True
# repository url is required
if 'repository' not in repo or repo['repository'] == "":
valid = False
message.append("entry without tag repository")
# either branch or commit-id should be set.
if ('branch' not in repo or repo['branch'] == "") and \
('commit-id' not in repo or repo['commit-id'] == ""):
valid = False
message.append("Either branch or commit-id should be set for entry")
if not valid:
result = False
message.append("entry content:")
message.append("{0}".format(json.dumps(repo, indent=True)))
return result, message
@staticmethod
def validate_downstream_jobs(downstream_jobs):
"""
validate whether the entry 'downstream-jobs' contains useful information
return: True if the entry is valid,
False if the entry is unusable
and message including omission imformation
"""
result = True
message = []
for job in downstream_jobs:
valid = True
# repository url is required
if 'repository' not in job or job['repository'] == "":
valid = False
message.append("entry without tag repository")
#command is required
if 'command' not in job:
valid = False
message.append("entry without tag command")
#working-directory is required
if 'working-directory' not in job:
valid = False
message.append("entry without tag working-directory")
#running-label is required
if 'running-label' not in job:
valid = False
message.append("entry without tag running-label")
# either branch or commit-id should be set.
if ('branch' not in job or job['branch'] == "") and \
('commit-id' not in job or job['commit-id'] == ""):
valid = False
message.append("Either commit-id or branch should be set for job repository")
# downstream-jobs is optional
# if it is specified, the value should be validate
if 'downstream-jobs' in job:
downstream_result, downstream_message = Manifest.validate_downstream_jobs(job['downstream-jobs'])
if not downstream_result:
valid = False
message.extend(downstream_message)
if not valid:
result = False
message.append("entry content:")
message.append("{0}".format(json.dumps(job, indent=True)))
return result, message
def validate_manifest(self):
"""
Identify whether the manifest contains useful information (as we understand it)
raise error if manifest is unusable
:return: None
"""
result = True
messages = ["Validate manifest file: {0}".format(self._name)]
if self._manifest is None:
result = False
messages.append("No manifest contents")
#build-name is required
if 'build-name' not in self._manifest:
result = False
messages.append("No build-name in manifest file")
#repositories is required
if 'repositories' not in self._manifest:
result = False
messages.append("No repositories in manifest file")
else:
r, m = self.validate_repositories(self._repositories)
if not r:
result = False
messages.extend(m)
#downstream-jobs is required
if 'downstream-jobs' not in self._manifest:
result = False
messages.append("No downstream-jobs in manifest file")
else:
r, m = Manifest.validate_downstream_jobs(self._downstream_jobs)
if not r:
result = False
messages.extend(m)
#build-requirements is required
if 'build-requirements' not in self._manifest:
result = False
messages.append("No build-requirements in manifest file")
if not result:
messages.append("manifest file {0} is not valid".format(self._name))
error = '\n'.join(messages)
raise KeyError(error)
@staticmethod
def check_commit_changed(repo, repo_url, branch, commit):
"""
Check whether the repository is changed based on its url, branch ,commit-id and arguments repo_url, branch, commit
:param repo: an repository entry of member _repositories or _downstream_jobs
:param repo_url: the url of the repository
:param branch: the branch of the repository
:param commit: the commit id of the repository
:return: True when the commit-id is different with argument commit
and the url and branch is the same with the arguments repo_url, branch;
otherwise, False
"""
if (repo['repository'] == repo_url):
# If repo has "branch", compare "commit-id" in repo with the argument commit
# only when "branch" is the same with argument branch.
if 'branch' in repo:
sliced_repo_branch = repo['branch'].split("/")[-1]
sliced_branch = branch.split("/")[-1]
if (repo['branch'] == branch or
repo['branch'] == sliced_branch or
sliced_repo_branch == sliced_branch):
if 'commit-id' in repo:
print "checking the commit-id for {0} with branch {1} from {2} to {3}".format\
(repo_url, branch, repo['commit-id'], commit)
if repo['commit-id'] != commit:
print " commit-id updated!"
return True
else:
print " commit-id unchanged"
return False
else:
print "add commit-id{0} for {1} with branch {2} ".format\
(commit, repo_url, branch)
return True
# If repo doesn't have "branch", compare "commit-id" in repo with argument commit
# Exits with 1 if repo doesn't have "commit-id"
else:
if 'commit-id' not in repo:
raise KeyError("Neither commit-id nor branch is set for repository {0}".format(repo['repository']))
else:
if repo['commit-id'] != commit:
print " commit-id updated!"
return True
else:
print " commit-id unchanged"
return False
return False
@staticmethod
def update_downstream_jobs(downstream_jobs, repo_url, branch, commit):
"""
update the instance of the class based on member:
_downstream_jobs and provided arguments.
:param downstream_jobs: the entry downstream_jobs
:param repo_url: the url of the repository
:param branch: the branch of the repository
:param commit: the commit id of the repository
:return: True if any job in downstream_jobs is updated
False if none of jobs in downstream_jobs is updated
"""
updated = False
for job in downstream_jobs:
if Manifest.check_commit_changed(job, repo_url, branch, commit):
job['commit-id'] = commit
updated = True
if 'downstream-jobs' in job:
nested_downstream_jobs = job['downstream-jobs']
if Manifest.update_downstream_jobs(nested_downstream_jobs, repo_url, branch, commit):
updated = True
return updated
@staticmethod
def update_repositories(repositories, repo_url, branch, commit):
"""
update the instance of the class based on member:
_repositories and provided arguments.
:param repositories: the entry repositories
:param repo_url: the url of the repository
:param branch: the branch of the repository
:param commit: the commit id of the repository
:return:
"""
updated = False
for repo in repositories:
if Manifest.check_commit_changed(repo, repo_url, branch, commit):
repo['commit-id'] = commit
updated = True
return updated
def update_manifest(self, repo_url, branch, commit):
"""
update the instance of the class based on members
_repositories , _downstream_jobs and provided arguments.
:param repo_url: the url of the repository
:param branch: the branch of the repository
:param commit: the commit id of the repository
:return:
"""
print "start updating manifest file {0}".format(self._name)
if self.update_repositories(self._repositories, repo_url, branch, commit):
self._changed = True
if self.update_downstream_jobs(self._downstream_jobs, repo_url, branch, commit):
self._changed = True
def write_manifest_file(self, file_path=None, dryrun=False):
"""
Add, commit, and push the manifest changes to the manifest repo.
:param file_path: String, The path to the temporary file.
If it is not set, the default value is self._file_path where manifest come from
:param dry_run: If true, would not push changes
:return:
"""
self.dump_to_json_file(file_path)
return
def dump_to_json_file(self, file_path=None):
"""
dump manifest json to a file
"""
if file_path is None:
file_path = self._file_path
with open(file_path, 'w') as fp:
json.dump(self._manifest, fp, indent=4, sort_keys=True)
class RepoOperator(object):
def __init__(self, git_credentials=None):
"""
Create a repository interface object
:return:
"""
self._git_credentials = git_credentials
self.git = GitBit(verbose=True)
if self._git_credentials:
self.setup_gitbit()
def setup_gitbit(self, credentials=None):
"""
Set gitbit credentials.
:return:
"""
self.git.set_identity(config.gitbit_identity['username'],
config.gitbit_identity['email'])
if credentials is None:
if self._git_credentials is None:
return
else:
credentials = self._git_credentials
else:
self._git_credentials = credentials
for url_cred_pair in credentials:
url, cred = url_cred_pair.split(',')
self.git.add_credential_from_variable(url, cred)
def set_git_dryrun(self, dryrun):
self.git.set_dryrun(dryrun)
def set_git_verbose(self, verbose):
self.git.set_verbose(verbose)
def set_git_executable(self, executable):
self.git.set_excutable(excutable)
@staticmethod
def print_command_summary(name, results):
"""
Print the results of running commands.
first the command line itself
and the error code if it's non-zero
then the stdout & stderr values from running that command
:param name:
:param results:
:return: True if any command exited with an error condition
"""
error_found = False
print "============================"
print "Command output for {0}".format(name)
if 'commands' in results[name]:
commands = results[name]['commands']
for command in commands:
for key in ['command', 'stdout', 'stderr']:
if key in command:
if command[key] != '':
print command[key]
if key == 'command':
if command['return_code'] != 0:
error_found = True
print "EXITED: {0}".format(
command['return_code'])
return error_found
def clone_repo_list(self, repo_list, dest_dir, jobs=1):
"""
check out repository to dest dir based on repo list
:param repo_list: a list of repository entry which should contain:
'repository': the url of repository, it is required
'branch': the branch to be check out, it is optional
'commit-id': the commit id to be reset, it is optional
:param dest_dir: the directory where repository will be check out
:param jobs: Number of parallel jobs to run
:return:
"""
cloner = RepoCloner(jobs)
if cloner is not None:
for repo in repo_list:
data = {
'repo': repo,
'builddir': dest_dir,
'credentials': self._git_credentials
}
cloner.add_task(data)
cloner.finish()
results = cloner.get_results()
error = False
for name in results.keys():
error |= self.print_command_summary(name, results)
if error:
raise RuntimeError("Failed to clone repositories")
def clone_repo(self, repo_url, dest_dir, repo_commit="HEAD"):
"""
check out a repository to dest directory from the repository url
:param repo_url: the url of repository, it is required
:param dest_dir: the directory where repository will be check out
:return: the directory of the repository
"""
repo = {}
repo["repository"] = repo_url
repo["commit-id"] = repo_commit
repo_list = [repo]
self.clone_repo_list(repo_list, dest_dir)
repo_directory_name = strip_suffix(os.path.basename(repo_url), ".git")
return os.path.join(dest_dir, repo_directory_name)
def get_lastest_commit_date(self, repo_dir):
"""
:param repo_dir: path of the repository
:return: commit-date
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError("The repository directory is not a directory")
code, output, error = self.git.run(
['show', '-s', '--pretty=format:%ct'], directory=repo_dir)
if code == 0:
return output.strip()
else:
raise RuntimeError(
"Unable to get commit date in directory {0}".format(repo_dir))
def get_lastest_commit_id(self, repo_dir):
"""
:param repo_dir: path of the repository
:return: commit-id
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError("The repository directory is not a directory")
code, output, error = self.git.run(
['log', '--format=format:%H', '-n', '1'], directory=repo_dir)
if code == 0:
return output.strip()
else:
raise RuntimeError(
"Unable to get commit id in directory {0}".format(repo_dir))
def get_commit_message(self, repo_dir, commit):
"""
:param repo_dir: path of the repository
:param commit: the commit id of the repository
:return: commit-message
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError("The repository directory is not a directory")
code, output, error = self.git.run(
['log', '--format=format:%B', '-n', '1', commit],
directory=repo_dir)
if code == 0:
return output.strip()
else:
raise RuntimeError("Unable to get commit message of {commit_id} in directory {repo_dir}"\
.format(commit_id=commit, repo_dir=repo_dir))
def get_repo_url(self, repo_dir):
"""
get the remote url of the repository
:param repo_dir: the directory of the repository
:return: repository url
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError("The repository directory is not a directory")
code, output, error = self.git.run(['ls-remote', '--get-url'],
directory=repo_dir)
if code == 0:
return output.strip()
else:
raise RuntimeError(
"Unable to find the repository url in directory {0}".format(
repo_dir))
def get_current_branch(self, repo_dir):
"""
get the current branch name of the repository
:param repo_dir: the directory of the repository
:return: branch name
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError("The repository directory is not a directory")
code, output, error = self.git.run(['symbolic-ref', '--short', 'HEAD'],
directory=repo_dir)
if code == 0:
return output.strip()
else:
raise RuntimeError(
"Unable to find the current branch in directory {0}".format(
repo_dir))
def check_branch(self, repo_url, branch):
"""
Checks if the specified branch name exists for the provided repository. Leave only the characters
following the final "/". This is to handle remote repositories.
Raise RuntimeError if it is not found.
:return: None
"""
if "/" in branch:
sliced_branch = branch.split("/")[-1]
else:
sliced_branch = branch
cmd_returncode, cmd_value, cmd_error = self.git.run(
['ls-remote', repo_url, 'heads/*{0}'.format(sliced_branch)])
if cmd_returncode is not 0 or cmd_value is '':
raise RuntimeError(
"The branch, '{0}', provided for '{1}', does not exist.".
format(branch, repo_url))
def set_repo_tagname(self, repo_url, repo_dir, tag_name):
"""
Sets tagname on the repo
:param repo_url: the url of the repository
:param repo_dir: the directory of the repository
:param tag_name: the tag name to be set
:return: None
"""
# See if that tag exists for the repo
cmd_returncode, cmd_value, cmd_error = self.git.run(
["tag", "-l", tag_name], repo_dir)
# Raise RuntimeError if tag already exists, otherwise create it
if cmd_returncode == 0 and cmd_value != '':
raise RuntimeError(
"Error: Tag {0} already exists - exiting now...".format(
cmd_value))
else:
print "Creating tag {0} for repo {1}".format(tag_name, repo_url)
self.git.run(["tag", "-a", tag_name, "-m", "\"Creating new tag\""],
repo_dir)
self.git.run(["push", "origin", "--tags"], repo_dir)
class RepoOperator(object):
def __init__(self, git_credentials=None):
"""
Create a repository interface object
:return:
"""
self._git_credentials = git_credentials
self.git = GitBit(verbose=True)
if self._git_credentials:
self.setup_gitbit()
def setup_gitbit(self, credentials=None):
"""
Set gitbit credentials.
:return:
"""
self.git.set_identity(config.gitbit_identity['username'], config.gitbit_identity['email'])
if credentials is None:
if self._git_credentials is None:
return
else:
credentials = self._git_credentials
else:
self._git_credentials = credentials
for url_cred_pair in credentials:
url, cred = url_cred_pair.split(',')
self.git.add_credential_from_variable(url, cred)
def set_git_dryrun(self, dryrun):
self.git.set_dryrun(dryrun)
def set_git_verbose(self, verbose):
self.git.set_verbose(verbose)
def set_git_executable(self, executable):
self.git.set_excutable(excutable)
@staticmethod
def print_command_summary(name, results):
"""
Print the results of running commands.
first the command line itself
and the error code if it's non-zero
then the stdout & stderr values from running that command
:param name:
:param results:
:return: True if any command exited with an error condition
"""
error_found = False
print "============================"
print "Command output for {0}".format(name)
if 'commands' in results[name]:
commands = results[name]['commands']
for command in commands:
for key in ['command', 'stdout', 'stderr']:
if key in command:
if command[key] != '':
print command[key]
if key == 'command':
if command['return_code'] != 0:
error_found = True
print "EXITED: {0}".format(command['return_code'])
return error_found
def clone_repo_list(self, repo_list, dest_dir, jobs=1):
"""
check out repository to dest dir based on repo list
:param repo_list: a list of repository entry which should contain:
'repository': the url of repository, it is required
'branch': the branch to be check out, it is optional
'commit-id': the commit id to be reset, it is optional
:param dest_dir: the directory where repository will be check out
:param jobs: Number of parallel jobs to run
:return:
"""
cloner = RepoCloner(jobs)
if cloner is not None:
for repo in repo_list:
data = {'repo': repo,
'builddir': dest_dir,
'credentials': self._git_credentials
}
cloner.add_task(data)
cloner.finish()
results = cloner.get_results()
error = False
for name in results.keys():
error |= self.print_command_summary(name, results)
if error:
raise RuntimeError("Failed to clone repositories")
def clone_repo(self, repo_url, dest_dir, repo_commit="HEAD"):
"""
check out a repository to dest directory from the repository url
:param repo_url: the url of repository, it is required
:param dest_dir: the directory where repository will be check out
:return: the directory of the repository
"""
repo = {}
repo["repository"] = repo_url
repo["commit-id"] = repo_commit
repo_list = [repo]
self.clone_repo_list(repo_list, dest_dir)
repo_directory_name = strip_suffix(os.path.basename(repo_url), ".git")
return os.path.join(dest_dir, repo_directory_name)
def get_lastest_commit_date(self, repo_dir):
"""
:param repo_dir: path of the repository
:return: commit-date
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError("The repository directory is not a directory")
code, output, error = self.git.run(['show', '-s', '--pretty=format:%ct'], directory=repo_dir)
if code == 0:
return output.strip()
else:
raise RuntimeError("Unable to get commit date in directory {0}".format(repo_dir))
def get_lastest_commit_id(self, repo_dir):
"""
:param repo_dir: path of the repository
:return: commit-id
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError("The repository directory is not a directory")
code, output, error = self.git.run(['log', '--format=format:%H', '-n', '1'], directory=repo_dir)
if code == 0:
return output.strip()
else:
raise RuntimeError("Unable to get commit id in directory {0}".format(repo_dir))
def get_commit_message(self, repo_dir, commit):
"""
:param repo_dir: path of the repository
:param commit: the commit id of the repository
:return: commit-message
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError("The repository directory is not a directory")
code, output, error = self.git.run(['log', '--format=format:%B', '-n', '1', commit], directory=repo_dir)
if code == 0:
return output.strip()
else:
raise RuntimeError("Unable to get commit message of {commit_id} in directory {repo_dir}"\
.format(commit_id=commit, repo_dir=repo_dir))
def get_repo_url(self, repo_dir):
"""
get the remote url of the repository
:param repo_dir: the directory of the repository
:return: repository url
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError("The repository directory is not a directory")
code, output, error = self.git.run(['ls-remote', '--get-url'], directory=repo_dir)
if code == 0:
return output.strip()
else:
raise RuntimeError("Unable to find the repository url in directory {0}".format(repo_dir))
def get_current_branch(self, repo_dir):
"""
get the current branch name of the repository
:param repo_dir: the directory of the repository
:return: branch name
"""
if repo_dir is None or not os.path.isdir(repo_dir):
raise RuntimeError("The repository directory is not a directory")
code, output, error = self.git.run(['symbolic-ref', '--short', 'HEAD'], directory=repo_dir)
if code == 0:
return output.strip()
else:
raise RuntimeError("Unable to find the current branch in directory {0}".format(repo_dir))
def check_branch(self, repo_url, branch):
"""
Checks if the specified branch name exists for the provided repository. Leave only the characters
following the final "/". This is to handle remote repositories.
Raise RuntimeError if it is not found.
:return: None
"""
if "/" in branch:
sliced_branch = branch.split("/")[-1]
else:
sliced_branch = branch
cmd_returncode, cmd_value, cmd_error = self.git.run(['ls-remote', repo_url, 'heads/*{0}'
.format(sliced_branch)])
if cmd_returncode is not 0 or cmd_value is '':
raise RuntimeError("The branch, '{0}', provided for '{1}', does not exist."
.format(branch, repo_url))
def set_repo_tagname(self, repo_url, repo_dir, tag_name):
"""
Sets tagname on the repo
:param repo_url: the url of the repository
:param repo_dir: the directory of the repository
:param tag_name: the tag name to be set
:return: None
"""
# See if that tag exists for the repo
cmd_returncode, cmd_value, cmd_error = self.git.run(["tag", "-l", tag_name], repo_dir)
# Raise RuntimeError if tag already exists, otherwise create it
if cmd_returncode == 0 and cmd_value != '':
raise RuntimeError("Error: Tag {0} already exists - exiting now...".format(cmd_value))
else:
print "Creating tag {0} for repo {1}".format(tag_name, repo_url)
self.git.run(["tag", "-a", tag_name, "-m", "\"Creating new tag\""], repo_dir)
self.git.run(["push", "origin", "--tags"], repo_dir)