def __init__(self):
"""
__repo - the repository url being updated in the manifest
__branch - the branch name associated with __repo
__sliced_branch - the branch name sliced at any forward slashes
__commit - The commit id associated with the __repo and __branch that we want to update
__manifest_repository_url - the repository url the points to the collection of manifest files
__manifest_file - the desired manifest file to update which resides in __manifest_repository_url
__cleanup_directories - the path/name of directories created are appended here for cleanup in task_cleanup
__git_credentials - url, credentials pair for the access to github repos
quiet - used for testing to minimize text written to a terminal
repo_operator - Class instance of RepoOperator
:return: None
"""
self.__repo = None
self.__branch = None
self.__sliced_branch = None
self.__commit = None
self.__manifest_repository_url = None
self.__manifest_file = None
self.__cleanup_directories = []
self.__git_credentials = None
self.__updated_manifest = None
self.__dryrun = False
self.quiet = False
self.repo_operator = RepoOperator()
def main():
# parse arguments
args = parse_command_line(sys.argv[1:])
if args.publish:
if args.git_credential:
repo_operator = RepoOperator(args.git_credential)
else:
print "Error occurs when get crendtail in update submodule"
sys.exit(1)
else:
repo_operator = RepoOperator(args.git_credential)
if os.path.isdir(args.build_dir):
for filename in os.listdir(args.build_dir):
try:
repo_dir = os.path.join(args.build_dir, filename)
submodules_list= subModulesExist(repo_dir,repo_operator)
if submodules_list is None:
continue
#git pull is used to get the lastest code to make sure push successfully
test_info = repo_operator.git_pull(repo_dir,"origin","master")
revert_commit_for_submodules_update(repo_dir,repo_operator,args.version)
if args.publish:
print "start to publish revert update submodule in {0}".format(repo_dir)
commit_message = "revert update submodule for new commit {}".format(args.version)
repo_operator.push_repo_changes(repo_dir, commit_message)
except Exception,e:
print "Failed to revert update submodule of {0} due to {1}".format(filename, e)
sys.exit(1)
def main():
# parse arguments
args = parse_command_line(sys.argv[1:])
if args.publish:
if args.git_credential:
repo_operator = RepoOperator(args.git_credential)
else:
print "If you want to publish the updated changelog, please specify the git-credential. Exiting now..."
sys.exit(1)
if os.path.isdir(args.build_dir):
for filename in os.listdir(args.build_dir):
try:
repo_dir = os.path.join(args.build_dir, filename)
updater = ChangelogUpdater(repo_dir, args.version)
if updater.update_changelog(message=args.message):
if args.publish:
commit_message = "update changelog for new release {0}".format(
args.version)
repo_operator.push_repo_changes(
repo_dir, commit_message)
except Exception, e:
print "Failed to update changelog of {0} due to {1}".format(
filename, e)
sys.exit(1)
def __init__(self,
manifest_path,
builddir,
force=False,
git_credentials=None,
jobs=1,
actions=[],
branch_name=None,
tag_name=None):
"""
__force - Overwrite a directory if it exists
__git_credential - url, credentials pair for the access to github repos
__manifest - Repository manifest contents
__builddir - Destination for checked out repositories
__jobs - Number of parallel jobs to run
__actions -Supported actions
:return:
"""
self._force = force
self._git_credentials = git_credentials
self._builddir = builddir
self._manifest = None
self.handle_manifest(manifest_path)
self._jobs = jobs
self.actions = []
for action in actions:
self.add_action(action)
self._branch_name = branch_name
self._tag_name = tag_name
self.repo_operator = RepoOperator(self._git_credentials)
def __init__(self,
dest,
branch,
builddir,
git_credential=None,
force=False,
jobs=1):
"""
Generate a new manifest according to the manifest sample: manifest.json
_dest_manifest_file: the path of new manifest
_branch: the branch name
_force: overwrite the destination if it exists.
_builddir: the destination for checked out repositories.
_jobs: number of parallel jobs to run. The number is related to the compute architecture, multi-core processors...
:return: None
"""
self._dest_manifest_file = dest
self._branch = branch
self._builddir = builddir
self._force = force
self._jobs = jobs
self._manifest = Manifest.instance_of_sample()
self.repo_operator = RepoOperator(git_credential)
self.check_builddir()
def __init__(self,
source,
dest,
branch,
builddir,
git_credentials,
force=False,
jobs=1):
"""
Generate a new manifest for new branch according to a source manifest file.
_source_manifest_file: the path of source manifest
_dest_manifest_file: the path of new manifest
_new_branch: the new branch name
_force: overwrite the destination if it exists.
_git_credentials: url, credentials pair for the access to github repos.
_builddir: the destination for checked out repositories.
_jobs: number of parallel jobs to run. The number is related to the compute architecture, multi-core processors...
:return: None
"""
self._source_manifest_file = source
self._dest_manifest_file = dest
self._new_branch = branch
self._git_credentials = git_credentials
self._manifest = None
self._force = force
self._builddir = builddir
self._jobs = jobs
self.initiate_manifest()
self.repo_operator = RepoOperator(git_credentials)
self.check_builddir()
def main():
# parse arguments
args = parse_command_line(sys.argv[1:])
if args.publish:
if args.git_credential:
repo_operator = RepoOperator(args.git_credential)
else:
print "If you want to publish the updated changelog, please specify the git-credential. Exiting now..."
sys.exit(1)
if os.path.isdir(args.build_dir):
for filename in os.listdir(args.build_dir):
try:
repo_dir = os.path.join(args.build_dir, filename)
changelog_updater = ChangelogUpdater(repo_dir, args.version)
npm_package_updater = NPMVersionUpdater(repo_dir, args.version)
npm_package_updater.update_package_json()
changelog_updater.update_changelog(message = args.message)
if args.publish:
print "start to push changes in {0}".format(repo_dir)
commit_message = "jump version for new release {0}".format(args.version)
repo_operator.push_repo_changes(repo_dir, commit_message)
except Exception,e:
print "Failed to jump version of {0} due to {1}".format(filename, e)
sys.exit(1)
def main():
# parse arguments
args = parse_command_line(sys.argv[1:])
if args.publish:
if args.git_credential:
repo_operator = RepoOperator(args.git_credential)
else:
print "If you want to publish the updated changelog, please specify the git-credential. Exiting now..."
sys.exit(1)
if os.path.isdir(args.build_dir):
for filename in os.listdir(args.build_dir):
try:
repo_dir = os.path.join(args.build_dir, filename)
changelog_updater = ChangelogUpdater(repo_dir, args.version)
npm_package_updater = NPMVersionUpdater(repo_dir, args.version)
package_updated = npm_package_updater.update_package_json()
changelog_updated = changelog_updater.update_changelog(message = args.message)
if changelog_updated or package_updated:
if args.publish:
print "start to push changes in {0}".format(repo_dir)
commit_message = "jump version for new release {0}".format(args.version)
repo_operator.push_repo_changes(repo_dir, commit_message)
except Exception,e:
print "Failed to jump version of {0} due to {1}".format(filename, e)
sys.exit(1)
def __init__(self, repo_dir, version):
"""
The module updates debian/changelog under the directory of repository
_repo_dir: the directory of the repository
_version: the new version which is going to be updated to changelog
"""
self._repo_dir = repo_dir
self._version = version
self.repo_operator = RepoOperator()
def __init__(self, dest, builddir, git_credential=None, force=False, jobs=1):
self._jenkins_author = config.gitbit_identity["username"]
self._dest_manifest_file = dest
self._builddir = builddir
self._force = force
self._jobs = jobs
self._manifest = Manifest.instance_of_sample()
self.repo_operator = RepoOperator(git_credential)
self.check_builddir()
def __init__(self, repo_dir):
"""
This module compute the version of a repository
The version for candidate release: {release_version}-{build_version}
The release version is parsed from debian/changelog
The samll version is consist of the commit hash and commit date of manifest repository
:return:None
"""
self._repo_dir = repo_dir
self.repo_operator = RepoOperator()
self._repo_name = self.get_repo_name()
def __init__(self, repo_dir):
"""
This module compute the version of a repository
The version for candidate release: {big-version}~{version-stage}-{small-version}
The big version is parsed from debian/changelog
The version-stage is devel if branch is master; or rc if branch if not master
The samll version is consist of the commit hash and commit date of manifest repository
:return:None
"""
self._repo_dir = repo_dir
self.repo_operator = RepoOperator()
self._repo_name = self.get_repo_name()
def __init__(
self, manifest_path, builddir, force=False, git_credentials=None, jobs=1, actions=[], branch_name=None
):
"""
__force - Overwrite a directory if it exists
__git_credential - url, credentials pair for the access to github repos
__manifest - Repository manifest contents
__builddir - Destination for checked out repositories
__jobs - Number of parallel jobs to run
__actions -Supported actions
:return:
"""
self._force = force
self._git_credentials = git_credentials
self._builddir = builddir
self._manifest = None
self.handle_manifest(manifest_path)
self._jobs = jobs
self.actions = []
for action in actions:
self.add_action(action)
self._branch_name = branch_name
self.repo_operator = RepoOperator(self._git_credentials)
def __init__(self):
"""
__repo - the repository url being updated in the manifest
__branch - the branch name associated with __repo
__sliced_branch - the branch name sliced at any forward slashes
__commit - The commit id associated with the __repo and __branch that we want to update
__manifest_base_url - the base download url points to the collection of manifest files
__manifest_file - the desired manifest file to update which resides in __manifest_repository_url
__cleanup_directories - the path/name of directories created are appended here for cleanup in task_cleanup
__git_credentials - url, credentials pair for the access to github repos
quiet - used for testing to minimize text written to a terminal
repo_operator - Class instance of RepoOperator
:return: None
"""
self.__repo = None
self.__branch = None
self.__sliced_branch = None
self.__commit = None
self.__manifest_download_url = None
self.__manifest_file = None
self.__cleanup_directories = []
self.__git_credentials = None
self.__updated_manifest = None
self.__dryrun = False
self.quiet = False
self.repo_operator = RepoOperator()
def main():
# parse arguments
args = parse_command_line(sys.argv[1:])
if args.publish:
if args.git_credential:
repo_operator = RepoOperator(args.git_credential)
else:
print "Error occurs when get crendtail in update submodule"
sys.exit(1)
else:
repo_operator = RepoOperator(args.git_credential)
if os.path.isdir(args.build_dir):
for filename in os.listdir(args.build_dir):
try:
repo_dir = os.path.join(args.build_dir, filename)
submodules_list = subModulesExist(repo_dir, repo_operator)
if submodules_list is None:
continue
#git pull is used to get the lastest code to make sure push successfully
test_info = repo_operator.git_pull(repo_dir, "origin",
"master")
revert_commit_for_submodules_update(repo_dir, repo_operator,
args.version)
if args.publish:
print "start to publish revert update submodule in {0}".format(
repo_dir)
commit_message = "revert update submodule for new commit {}".format(
args.version)
repo_operator.push_repo_changes(repo_dir, commit_message)
except Exception, e:
print "Failed to revert update submodule of {0} due to {1}".format(
filename, e)
sys.exit(1)
def __init__(self, repo_dir, version):
"""
The module updates debian/changelog under the directory of repository
_repo_dir: the directory of the repository
_version: the new version which is going to be updated to changelog
"""
self._repo_dir = repo_dir
self._version = version
self.repo_operator = RepoOperator()
def __init__(self, manifest_path, builddir):
"""
__force - Overwrite a directory if it exists
__git_credential - url, credentials pair for the access to github repos
__manifest - Repository manifest contents
__builddir - Destination for checked out repositories
__jobs - Number of parallel jobs to run
__actions -Supported actions
:return:
"""
self._force = False
self._git_credentials = None
self._builddir = builddir
self._manifest = None
self.handle_manifest(manifest_path)
self._jobs = 1
self.actions = []
self.repo_operator = RepoOperator()
def __init__(self, repo_dir):
"""
This module compute the version of a repository
The version for candidate release: {big-version}~{version-stage}-{small-version}
The big version is parsed from debian/changelog
The version-stage is devel if branch is master; or rc if branch if not master
The samll version is consist of the commit hash and commit date of manifest repository
:return:None
"""
self._repo_dir = repo_dir
self.repo_operator = RepoOperator()
self._repo_name = self.get_repo_name()
def __init__(self, manifest_dir, builddir, is_official_release=False):
"""
Compute the version of each repository under builddir
and update the debian/control with these versions
__manifest_repo_dir - The directory of Repository manifest
__builddir - Destination for checked out repositories
__is_official_release - True if the official is official release
:return: None
"""
self._builddir = builddir
self._manifest_repo_dir = manifest_dir
self._is_official_release = is_official_release
self.repo_operator = RepoOperator()
def __init__(self, dest, branch, builddir, git_credential, force=False, jobs=1):
"""
Generate a new manifest according to the manifest sample: manifest.json
_dest_manifest_file: the path of new manifest
_branch: the branch name
_force: overwrite the destination if it exists.
_builddir: the destination for checked out repositories.
_jobs: number of parallel jobs to run. The number is related to the compute architecture, multi-core processors...
:return: None
"""
self._dest_manifest_file = dest
self._branch = branch
self._builddir = builddir
self._force = force
self._jobs = jobs
self._manifest = Manifest.instance_of_sample()
self.repo_operator = RepoOperator(git_credential)
self.check_builddir()
class ExistDirManifestGenerator(ManifestGenerator):
def __init__(self, dest, builddir, git_credential=None, force=False, jobs=1):
self._jenkins_author = config.gitbit_identity["username"]
self._dest_manifest_file = dest
self._builddir = builddir
self._force = force
self._jobs = jobs
self._manifest = Manifest.instance_of_sample()
self.repo_operator = RepoOperator(git_credential)
self.check_builddir()
def check_builddir(self):
"""
Checks the given builddir name and force flag.
Deletes exists directory if one already exists and --force is set
:return: None
"""
if not os.path.exists(self._builddir):
print "The {0} doesn't exist".format(self._builddir)
sys.exit(1)
def update_manifest(self):
"""
update the manifest with branch name
:return: None
"""
repositories = self._manifest.repositories
downstream_jobs = self._manifest.downstream_jobs
self.update_repositories_commit(repositories)
self.update_repositories_branch(repositories)
self.update_repositories_commit(downstream_jobs)
self.update_repositories_branch(downstream_jobs)
self._manifest.validate_manifest()
def update_repositories_branch(self, repositories):
"""
update the commit-id of repository with the latest commit id
:param repositories: a list of repository directory
:return: None
"""
for repo in repositories:
repo_dir = self.directory_for_repo(repo)
repo["branch"] = self.repo_operator.get_current_branch(repo_dir)
def main():
# parse arguments
args = parse_command_line(sys.argv[1:])
generator = ManifestGenerator(args.source_manifest,
args.dest_manifest,
args.branch,
args.builddir,
args.git_credentials,
jobs=args.jobs,
force=args.force)
try:
generator.update_manifest()
if args.force:
generator.set_force(args.force)
if args.publish:
if args.git_credentials and args.publish_branch:
repo_operator = RepoOperator(args.git_credentials)
commit_message = "add a manifest file for new branch {0}".format(
args.branch)
repo_dir = os.path.dirname(args.dest_manifest)
repo_operator.checkout_repo_branch(repo_dir,
args.publish_branch)
generator.generate_manifest()
repo_operator.push_repo_changes(repo_dir,
commit_message,
push_all=True)
else:
print "Please specify the git-credential and publish-branch if you want to push the new manifest"
sys.exit(1)
else:
generator.generate_manifest()
except Exception, e:
traceback.print_exc()
print "Failed to generate new manifest for {0} due to \n{1}\nExiting now".format(
args.branch, e)
sys.exit(1)
class ManifestActions(object):
"""
valid actions:
checkout: check out a set of repositories to match the manifest file
"""
valid_actions = ["checkout"]
def __init__(self, manifest_path, builddir):
"""
__force - Overwrite a directory if it exists
__git_credential - url, credentials pair for the access to github repos
__manifest - Repository manifest contents
__builddir - Destination for checked out repositories
__jobs - Number of parallel jobs to run
__actions -Supported actions
:return:
"""
self._force = False
self._git_credentials = None
self._builddir = builddir
self._manifest = None
self.handle_manifest(manifest_path)
self._jobs = 1
self.actions = []
self.repo_operator = RepoOperator()
def set_force(self, force):
"""
Standard setter for force
:param force: if true, overwrite a directory if it exists
:return: None
"""
self._force = force
def get_force(self):
"""
Standard getter for git_credentials
:return: force
"""
return force
def set_git_credentials(self, git_credential):
"""
Standard setter for git_credentials
:param git_credential: url, credentials pair for the access to github repos
:return: None
"""
self._git_credentials = git_credential
self.repo_operator.setup_gitbit(credentials=self._git_credentials)
def get_manifest(self):
"""
Standard getter for manifest
:return: an instance of Manifest
"""
return self._manifest
def add_action(self, action):
"""
Add action to actions
:param action: a string, just like: checkout
:return: None
"""
if action not in self.valid_actions:
print "Unknown action '{0}' requested".format(action)
print "Valid actions are:"
for op in self.valid_actions:
print " {0}".format(op)
sys.exit(1)
else:
self.actions.append(action)
def set_jobs(self, jobs):
"""
Standard setter for jobs
:param jobs: number of parallel jobs to run
:return: None
"""
self._jobs = jobs
if self._jobs < 1:
print "--jobs value must be an integer >=1"
sys.exit(1)
def handle_manifest(self, manifest_path):
"""
initial manifest and validate it
:param manifest_path: the path of manifest file
:return: None
"""
try:
self._manifest = Manifest(manifest_path)
self._manifest.validate_manifest()
except KeyError as error:
print "Failed to create a Manifest instance for the manifest file {0} \nERROR:\n{1}".format(
manifest_path, error.message
)
sys.exit(1)
for repo in self._manifest.get_repositories():
repo["directory-name"] = self.directory_for_repo(repo)
def check_builddir(self):
"""
Checks the given builddir name and force flag.
Deletes exists directory if one already exists and --force is set
:return: None
"""
if os.path.exists(self._builddir):
if self._force:
shutil.rmtree(self._builddir)
print "Removing existing data at {0}".format(self._builddir)
else:
print "Unwilling to overwrite destination builddir of {0}".format(self._builddir)
sys.exit(1)
os.makedirs(self._builddir)
def get_repositories(self):
"""
Issues checkout commands to dictionaries within a provided manifest
:return: None
"""
repo_list = self._manifest.get_repositories()
try:
self.repo_operator.clone_repo_list(repo_list, self._builddir, jobs=self._jobs)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
def directory_for_repo(self, repo):
"""
Get the directory of a repository
:param repo: a dictionary
:return: the directary of repository
"""
if "checked-out-directory-name" in repo:
repo_directory = repo["checked-out-directory-name"]
else:
if "repository" in repo:
repo_url = repo["repository"]
repo_directory = strip_suffix(os.path.basename(repo_url), ".git")
else:
raise ValueError("no way to find basename")
repo_directory = os.path.join(self._builddir, repo_directory)
return repo_directory
class VersionGenerator(object):
def __init__(self, repo_dir):
"""
This module compute the version of a repository
The version for candidate release: {big-version}~{version-stage}-{small-version}
The big version is parsed from debian/changelog
The version-stage is devel if branch is master; or rc if branch if not master
The samll version is consist of the commit hash and commit date of manifest repository
:return:None
"""
self._repo_dir = repo_dir
self.repo_operator = RepoOperator()
self._repo_name = self.get_repo_name()
def get_repo_name(self):
repo_url = self.repo_operator.get_repo_url(self._repo_dir)
repo_name = common.strip_suffix(os.path.basename(repo_url), ".git")
return repo_name
def generate_small_version(self):
"""
Generate the small version which consists of commit date and commit hash of manifest repository
According to small version, users can track the commit of all repositories in manifest file
return: small version
"""
if self._repo_name == "RackHD":
utc_now = datetime.utcnow()
utc_yesterday = utc_now + timedelta(days=-1)
version = utc_yesterday.strftime('%Y%m%dUTC')
return version
else:
commit_timestamp_str = self.repo_operator.get_lastest_commit_date(self._repo_dir)
date = datetime.utcfromtimestamp(int(commit_timestamp_str)).strftime('%Y%m%dUTC')
commit_id = self.repo_operator.get_lastest_commit_id(self._repo_dir)
version = "{date}-{commit}".format(date=date, commit=commit_id[0:7])
return version
def debian_exist(self):
"""
check whether debian or debianstatic directory under the repository
return: True if debian or debianstatic exist
False
"""
if os.path.isdir(self._repo_dir):
for filename in os.listdir(self._repo_dir):
if filename == "debian":
return True
return False
def generate_big_version(self):
"""
Generate the big version according to changelog
The big version is the latest version of debian/changelog
return: big version
"""
# If the repository has the debianstatic/repository name/,
# create a soft link to debian before compute version
debian_exist = self.debian_exist()
linked = False
if not debian_exist:
for filename in os.listdir(self._repo_dir):
if filename == "debianstatic":
debianstatic_dir = os.path.join(self._repo_dir, "debianstatic")
for debianstatic_filename in os.listdir(debianstatic_dir):
if debianstatic_filename == self._repo_name:
debianstatic_repo_dir = "debianstatic/{0}".format(self._repo_name)
common.link_dir(debianstatic_repo_dir, "debian", self._repo_dir)
linked = True
if not debian_exist and not linked:
return None
cmd_args = ["dpkg-parsechangelog", "--show-field", "Version"]
version = common.run_command(cmd_args, directory=self._repo_dir)
if linked:
os.remove(os.path.join(self._repo_dir, "debian"))
return version
def generate_version_stage(self):
"""
Generate the version stage according to the stage of deveplopment
return: devel ,if the branch is master
rc, if the branch is not master
"""
current_branch = self.repo_operator.get_current_branch(self._repo_dir)
version_stage = ""
if "master" in current_branch:
version_stage = "devel"
else:
version_stage = "rc"
return version_stage
def generate_package_version(self, is_official_release):
"""
generate the version of package, just like:
1.1-1-devel-20160809150908-7396d91 or 1.1-1
:return: package version
"""
big_version = self.generate_big_version()
if big_version is None:
common.logging.warning("Failed to generate big version, maybe the {0} doesn't contain debian directory".format(self._repo_dir))
return None
if is_official_release:
version = big_version
else:
small_version = self.generate_small_version()
if small_version is None:
raise RuntimeError("Failed to generate version for {0}, due to the small version is None".format(self._repo_dir))
version = "{0}-{1}".format(big_version, small_version)
return version
class UpdateManifest(object):
def __init__(self):
"""
__repo - the repository url being updated in the manifest
__branch - the branch name associated with __repo
__sliced_branch - the branch name sliced at any forward slashes
__commit - The commit id associated with the __repo and __branch that we want to update
__manifest_base_url - the base download url points to the collection of manifest files
__manifest_file - the desired manifest file to update which resides in __manifest_repository_url
__cleanup_directories - the path/name of directories created are appended here for cleanup in task_cleanup
__git_credentials - url, credentials pair for the access to github repos
quiet - used for testing to minimize text written to a terminal
repo_operator - Class instance of RepoOperator
:return: None
"""
self.__repo = None
self.__branch = None
self.__sliced_branch = None
self.__commit = None
self.__manifest_download_url = None
self.__manifest_file = None
self.__cleanup_directories = []
self.__git_credentials = None
self.__updated_manifest = None
self.__dryrun = False
self.quiet = False
self.repo_operator = RepoOperator()
#pylint: disable=no-self-use
def parse_args(self, args):
"""
Take in values from the user.
Repo, branch, and manifest_repo are required. This exits if they are not given.
:return: Parsed args for assignment
"""
parser = argparse.ArgumentParser()
parser.add_argument('-n', '--dryrun',
help="Do not commit any changes, just print what would be done",
action="store_true")
parser.add_argument("--repo",
help="Git url to match for updating the commit-id",
action="store")
parser.add_argument("--branch",
help="The target branch for the named repo",
action="store")
parser.add_argument("--manifest_download_url",
help="The manifest derectory URL in bintray.",
action="store")
parser.add_argument("--commit",
help="OPTIONAL: The commit id to target an exact version",
action="store")
parser.add_argument("--manifest_file",
help="The target manifest file to be updated. Searches through all if left empty.",
action="store")
parser.add_argument("--git-credential",
help="Git URL and credentials comma separated",
action="append")
parser.add_argument('--updated-manifest',
help="Output file containing the name of the updated manifest, the download url, branch, commit of manifest repository. The key, value pairs will be passed to downstream jobs as parameters",
action="store")
parser = parser.parse_args(args)
return parser
def assign_args(self, args):
"""
Assign args to member variables and perform checks on branch and commit-id.
:param args: Parsed args from the user
:return:
"""
if args.repo:
self.__repo = args.repo
else:
print "\nMust specify repository url for cloning (--repo <git_url>)\n"
if args.branch:
self.__branch = args.branch
if "/" in self.__branch:
self.__sliced_branch = self.__branch.split("/")[-1]
else:
self.__sliced_branch = self.__branch
else:
print "\nMust specify a branch name (--branch <branch_name>)\n"
# if args.manifest_repo:
# self.__manifest_repository_url = args.manifest_repo
# else:
# print "\n Must specify a full repository url for retrieving <manifest>.json files\n"
if args.manifest_download_url:
self.__manifest_download_url = args.manifest_download_url
else:
print "\n Must specify a manifest base url for download <manifest>.json files\n"
if args.dryrun:
self.__dryrun = True
self.repo_operator.setup_git_dryrun(self.__dryrun)
if args.commit:
self.__commit = args.commit
if args.manifest_file:
self.__manifest_file = args.manifest_file
if args.git_credential:
self.__git_credentials = args.git_credential
self.repo_operator.setup_gitbit(credentials=self.__git_credentials)
if args.updated_manifest:
self.__updated_manifest = args.updated_manifest
def check_args(self):
"""
Check the values given for branch and commit-id
"""
if self.__branch:
try:
self.repo_operator.check_branch(self.__repo, self.__sliced_branch)
except RuntimeError as error:
self.cleanup_and_exit(error, 1)
if self.__commit:
self.__check_commit()
def __check_commit(self):
"""
Check the format of the commit-id. It must be 40 hex characters.
Exits if it is not.
:return: None
"""
commit_match = re.match('^[0-9a-fA-F]{40}$', self.__commit)
if not commit_match:
self.cleanup_and_exit("Id, '{0}' is not valid. It must be a 40 character hex string.".format(self.__commit), 1)
def download_manifest_file(self):
"""
Download the manifest json files. Return that directory name which stores manifest. The directory
is temporary and deleted in the cleanup_and_exit function
:return: A string containing the name of the folder where the manifest file was download.
"""
directory_name = tempfile.mkdtemp()
if os.path.isdir(directory_name):
pass
# For now script of 'upload to bintray' is seperated from this script.
# So before uploading this directory shouldn't be deleted'
# The blow code snippet will be unfolded when involved bintray upload functions into this script.
# self.__cleanup_directories.append(directory_name)
else:
self.cleanup_and_exit("Failed to make temporary directory for the repository: {0}".format(url), 1)
try:
url = "/".join([self.__manifest_download_url, self.__manifest_file])
dest_dir = "/".join([directory_name, self.__manifest_file])
if os.environ['BINTRAY_USERNAME'] and os.environ['BINTRAY_API_KEY']:
print "Requests bintray with token"
auth = (os.environ['BINTRAY_USERNAME'].strip(), os.environ['BINTRAY_API_KEY'].strip())
resp = requests.get(url, auth=auth)
else:
print "Requests without token"
resp = requests.get(url)
if resp.ok:
with open(dest_dir, "wb") as file_handle:
file_handle.write(resp.content)
elif resp.status_code==404:
# If there's no manifest file in bintray server, init an empty one
print "can't find manifest in remote server, will use template manifest"
Manifest.instance_of_sample().dump_to_json_file(dest_dir)
else:
print "Unknown error, {0}".format(resp.status_code)
return directory_name
except RuntimeError as error:
self.cleanup_and_exit(error, 1)
def validate_manifest_files(self, *args):
"""
validate several manifest files
For example: validate_manifest_files(file1, file2) or
validate_manifest_files(file1, file2, file3)
"""
validate_result = True
for filename in args:
try:
manifest = Manifest(filename)
manifest.validate_manifest()
print "manifest file {0} is valid".format(filename)
except KeyError as error:
print "Failed to validate manifest file {0}".format(filename)
print "\nERROR: \n{0}".format(error.message)
validate_result = False
return validate_result
def cleanup_and_exit(self, message=None, code=0):
"""
Delete all files and folders made during this job which are named in self.cleanup_directories
:return: None
"""
if not self.quiet:
if message is not None:
print "\nERROR: {0}".format(message)
print "\nCleaning environment!\n"
for item in self.__cleanup_directories:
subprocess.check_output(["rm", "-rf", item])
sys.exit(code)
def get_updated_commit_message(self):
"""
get the updated repository commit message
"""
# get commit message based on the arguments repo, branch and commit
directory_name = tempfile.mkdtemp()
if os.path.isdir(directory_name):
self.__cleanup_directories.append(directory_name)
else:
self.cleanup_and_exit("Failed to make temporary directory for the repository: {0}".format(url), 1)
try:
updated_repo_dir = self.repo_operator.clone_repo(self.__repo, directory_name)
repo_commit_message = self.repo_operator.get_commit_message(updated_repo_dir, self.__commit)
return repo_commit_message
except RuntimeError as error:
self.cleanup_and_exit(error, 1)
def upload_manifest_to_bintray(self, dir_name, bintray):
"""
Update manifest to bintray based on its contents and user arguments.
:param dir_name: The directory of the repository
:return: if repo is updated, return updated manifest file path and the manifest object
otherwise, return None, None
"""
def update_manifest_repo(self, dir_name, repo_commit_message):
"""
Update manifest repository based on its contents and user arguments.
:param dir_name: The directory of the repository
:return: if repo is updated, return updated manifest file path and the manifest object
otherwise, return None, None
"""
if self.__manifest_file is not None:
path_name = os.path.join(dir_name, self.__manifest_file)
if os.path.isfile(path_name):
try:
manifest = Manifest(path_name, self.__git_credentials)
manifest.update_manifest(self.__repo, self.__branch, self.__commit)
if manifest.changed:
manifest.write_manifest_file(path_name, self.__dryrun)
return path_name, manifest
else:
print "No changes to {0}".format(manifest.name)
except KeyError as error:
self.cleanup_and_exit("Failed to create an Manifest instance for the manifest file {0}\nError:{1}"\
.format(self.__manifest_file, error.message),1)
except RuntimeError as error:
self.cleanup_and_exit("Failed to update manifest repo\nError:{0}".format(error.message),1)
else:
for item in os.listdir(dir_name):
path_name = os.path.join(dir_name, item)
if os.path.isfile(path_name):
try:
manifest = Manifest(path_name, self.__git_credentials)
manifest.update_manifest(self.__repo, self.__branch, self.__commit)
if manifest.changed:
manifest.write_manifest_file(path_name, self.__dryrun)
return path_name, manifest
else:
print "No changes to {0}".format(manifest.name)
except KeyError as error:
self.cleanup_and_exit("Failed to create an Manifest instance for the manifest file {0}\nError:{1}"\
.format(path_name, error.message),1)
except RuntimeError as error:
self.cleanup_and_exit("Failed to update manifest repo\nError:{0}".format(error.message),1)
return None, None
def write_downstream_parameters(self, filename, params):
"""
Add/append downstream parameter (java variable value pair) to the given
parameter file. If the file does not exist, then create the file.
:param filename: The parameter file that will be used for making environment
variable for downstream job.
:param params: the parameters dictionary
:return:
None on success
Raise any error if there is any
"""
if filename is None:
return
with open(filename, 'w') as fp:
try:
for key in params:
entry = "{key}={value}\n".format(key=key, value=params[key])
fp.write(entry)
except IOError as error:
print "Unable to write parameter(s) for next step(s), exit"
self.cleanup_and_exit(error, 1)
def downstream_manifest_to_use(self, manifest_folder, file_with_path, manifest, validate_result):
"""
Write file which contains the name of the manifest file most recently updated.
:param manifest_folder: the path of the manifest repository
:param file_with_path: The path to be split to claim the filename
:param manifest: the Manifest object of manifest file
"""
file_name = file_with_path.split('/')[-1]
downstream_parameters = {}
# If not validate_result, do not trigger downstream job
downstream_parameters['MANIFEST_FILE_VALID'] = validate_result
downstream_parameters['BUILD_REQUIREMENTS'] = manifest.build_requirements
downstream_parameters['MANIFEST_FILE_PATH'] = file_with_path
downstream_parameters['MANIFEST_FILE_URL'] = "".join([self.__manifest_download_url, file_name])
self.write_downstream_parameters(self.__updated_manifest, downstream_parameters)
return
def main():
# parse arguments
args = parse_command_line(sys.argv[1:])
try:
manifest = Manifest(args.manifest)
manifest.validate_manifest()
except KeyError as error:
print "Failed to create a Manifest instance for the manifest file {0} \nERROR:\n{1}"\
.format(args.manifest, error.message)
sys.exit(1)
if args.publish:
if args.git_credential:
repo_operator = RepoOperator(args.git_credential)
else:
print "Error occurs when get crendtail in update submodule"
sys.exit(1)
else:
repo_operator = RepoOperator(args.git_credential)
if os.path.isdir(args.build_dir):
print args.build_dir
for filename in os.listdir(args.build_dir):
try:
repo_dir = os.path.join(args.build_dir, filename)
repo_operator.submodule_init(repo_dir)
repo_operator.submodule_update(repo_dir)
submodules_list = repo_operator.get_current_submodule(repo_dir)
if len(submodules_list)==0:
continue;
for key in submodules_list:
commit_id = get_manifest_commit_id(key,manifest)
if commit_id != None:
sub_dir = repo_dir+"/"+key
repo_operator.checkout_to_commit(sub_dir,commit_id)
if args.publish:
print "start to publish update submodule in {0}".format(repo_dir)
commit_message = "update submodule for new commit {0}".format(args.version)
repo_operator.push_repo_changes(repo_dir, commit_message)
except Exception,e:
print "Failed to update submodule of {0} due to {1}".format(filename, e)
sys.exit(1)
class ManifestActions(object):
"""
valid actions:
checkout: check out a set of repositories to match the manifest file
"""
valid_actions = ['checkout']
def __init__(self, manifest_path, builddir):
"""
__force - Overwrite a directory if it exists
__git_credential - url, credentials pair for the access to github repos
__manifest - Repository manifest contents
__builddir - Destination for checked out repositories
__jobs - Number of parallel jobs to run
__actions -Supported actions
:return:
"""
self._force = False
self._git_credentials = None
self._builddir = builddir
self._manifest = None
self.handle_manifest(manifest_path)
self._jobs = 1
self.actions = []
self.repo_operator = RepoOperator()
def set_force(self, force):
"""
Standard setter for force
:param force: if true, overwrite a directory if it exists
:return: None
"""
self._force = force
def get_force(self):
"""
Standard getter for git_credentials
:return: force
"""
return force
def set_git_credentials(self, git_credential):
"""
Standard setter for git_credentials
:param git_credential: url, credentials pair for the access to github repos
:return: None
"""
self._git_credentials = git_credential
self.repo_operator.setup_gitbit(credentials=self._git_credentials)
def get_manifest(self):
"""
Standard getter for manifest
:return: an instance of Manifest
"""
return self._manifest
def add_action(self, action):
"""
Add action to actions
:param action: a string, just like: checkout
:return: None
"""
if action not in self.valid_actions:
print "Unknown action '{0}' requested".format(action)
print "Valid actions are:"
for op in self.valid_actions:
print " {0}".format(op)
sys.exit(1)
else:
self.actions.append(action)
def set_jobs(self, jobs):
"""
Standard setter for jobs
:param jobs: number of parallel jobs to run
:return: None
"""
self._jobs = jobs
if self._jobs < 1:
print "--jobs value must be an integer >=1"
sys.exit(1)
def handle_manifest(self, manifest_path):
"""
initial manifest and validate it
:param manifest_path: the path of manifest file
:return: None
"""
try:
self._manifest = Manifest(manifest_path)
self._manifest.validate_manifest()
except KeyError as error:
print "Failed to create a Manifest instance for the manifest file {0} \nERROR:\n{1}"\
.format(manifest_path, error.message)
sys.exit(1)
for repo in self._manifest.get_repositories():
repo['directory-name'] = self.directory_for_repo(repo)
def check_builddir(self):
"""
Checks the given builddir name and force flag.
Deletes exists directory if one already exists and --force is set
:return: None
"""
if os.path.exists(self._builddir):
if self._force:
shutil.rmtree(self._builddir)
print "Removing existing data at {0}".format(self._builddir)
else:
print "Unwilling to overwrite destination builddir of {0}".format(
self._builddir)
sys.exit(1)
os.makedirs(self._builddir)
def get_repositories(self):
"""
Issues checkout commands to dictionaries within a provided manifest
:return: None
"""
repo_list = self._manifest.get_repositories()
try:
self.repo_operator.clone_repo_list(repo_list,
self._builddir,
jobs=self._jobs)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
def directory_for_repo(self, repo):
"""
Get the directory of a repository
:param repo: a dictionary
:return: the directary of repository
"""
if 'checked-out-directory-name' in repo:
repo_directory = repo['checked-out-directory-name']
else:
if 'repository' in repo:
repo_url = repo['repository']
repo_directory = strip_suffix(os.path.basename(repo_url),
".git")
else:
raise ValueError("no way to find basename")
repo_directory = os.path.join(self._builddir, repo_directory)
return repo_directory
class ManifestGenerator(object):
def __init__(self,
source,
dest,
branch,
builddir,
git_credentials,
force=False,
jobs=1):
"""
Generate a new manifest for new branch according to a source manifest file.
_source_manifest_file: the path of source manifest
_dest_manifest_file: the path of new manifest
_new_branch: the new branch name
_force: overwrite the destination if it exists.
_git_credentials: url, credentials pair for the access to github repos.
_builddir: the destination for checked out repositories.
_jobs: number of parallel jobs to run. The number is related to the compute architecture, multi-core processors...
:return: None
"""
self._source_manifest_file = source
self._dest_manifest_file = dest
self._new_branch = branch
self._git_credentials = git_credentials
self._manifest = None
self._force = force
self._builddir = builddir
self._jobs = jobs
self.initiate_manifest()
self.repo_operator = RepoOperator(git_credentials)
self.check_builddir()
def set_force(self, force):
"""
Standard setter for force
:param force: if true, overwrite a directory file if it exists
:return: None
"""
self._force = force
def initiate_manifest(self):
"""
initial manifest and validate it
:return: None
"""
self._manifest = Manifest(self._source_manifest_file)
self._manifest.validate_manifest()
def directory_for_repo(self, repo):
"""
Get the directory of a repository
:param repo: a dictionary
:return: the directary of repository
"""
if 'checked-out-directory-name' in repo:
repo_directory = repo['checked-out-directory-name']
else:
if 'repository' in repo:
repo_url = repo['repository']
repo_directory = strip_suffix(os.path.basename(repo_url),
".git")
else:
raise ValueError("no way to find basename")
repo_directory = os.path.join(self._builddir, repo_directory)
return repo_directory
def check_builddir(self):
"""
Checks the given builddir name and force flag.
Deletes exists directory if one already exists and --force is set
:return: None
"""
if os.path.exists(self._builddir):
if self._force:
shutil.rmtree(self._builddir)
print "Removing existing data at {0}".format(self._builddir)
else:
print "Unwilling to overwrite destination builddir of {0}".format(
self._builddir)
sys.exit(1)
os.makedirs(self._builddir)
def update_repositories_with_lastest_commit(self, repositories):
"""
update the commit-id of repository with the lastest commit id
:param repositories: a list of repository directory
:return: None
"""
self.repo_operator.clone_repo_list(repositories,
self._builddir,
jobs=self._jobs)
for repo in repositories:
repo_dir = self.directory_for_repo(repo)
repo["commit-id"] = self.repo_operator.get_lastest_commit_id(
repo_dir)
def update_manifest(self):
"""
update the manifest with new branch
:return: None
"""
repositories = self._manifest.repositories
downstream_jobs = self._manifest.downstream_jobs
build_name = os.path.basename(self._dest_manifest_file)
for repo in repositories:
repo["branch"] = self._new_branch
repo["commit-id"] = ""
self.update_repositories_with_lastest_commit(repositories)
for job in downstream_jobs:
job["branch"] = self._new_branch
repo["commit-id"] = ""
self.update_repositories_with_lastest_commit(downstream_jobs)
self._manifest.build_name = build_name
self._manifest.validate_manifest()
def generate_manifest(self):
"""
generate a new manifest
:return: None
"""
dest_dir = os.path.dirname(self._dest_manifest_file)
dest_file = os.path.basename(self._dest_manifest_file)
for filename in os.listdir(dest_dir):
if filename == dest_file and self._force == False:
raise RuntimeError("The file {0} already exist under {1}. \n \
If you want to overrite the file, please specify --force."
.format(dest_file, dest_dir))
with open(self._dest_manifest_file, 'w') as fp:
json.dump(self._manifest.manifest, fp, indent=4, sort_keys=True)
class ChangelogUpdater(object):
def __init__(self, repo_dir, version):
"""
The module updates debian/changelog under the directory of repository
_repo_dir: the directory of the repository
_version: the new version which is going to be updated to changelog
"""
self._repo_dir = repo_dir
self._version = version
self.repo_operator = RepoOperator()
def debian_exist(self):
"""
check whether debian directory under the repository
return: True if debian exist
False
"""
if os.path.isdir(self._repo_dir):
for filename in os.listdir(self._repo_dir):
if filename == "debian":
return True
return False
def get_repo_name(self):
"""
get the name of the repository
:return: the name of the repository
"""
repo_url = self.repo_operator.get_repo_url(self._repo_dir)
repo_name = strip_suffix(os.path.basename(repo_url), ".git")
return repo_name
def update_changelog(self, message=None):
"""
add an entry to changelog
:param message: the message which is going to be added to changelog
return: Ture if changelog is updated
False, otherwise
"""
repo_name = self.get_repo_name()
debian_exist = self.debian_exist()
linked = False
if not debian_exist:
# Handle repository which contains debianstatic/repository_name folder,
# for example: debianstatic/on-http
for filename in os.listdir(self._repo_dir):
if filename == "debianstatic":
debianstatic_dir = os.path.join(self._repo_dir,
"debianstatic")
for debianstatic_filename in os.listdir(debianstatic_dir):
if debianstatic_filename == repo_name:
debianstatic_repo_dir = "debianstatic/{0}".format(
repo_name)
link_dir(debianstatic_repo_dir, "debian",
self._repo_dir)
linked = True
if not debian_exist and not linked:
return False
print "start to update changelog of {0}".format(self._repo_dir)
# -v: Add a new changelog entry with version number specified
# -b: Force a version to be less than the current one
# -m: Don't change (maintain) the trailer line in the changelog entry; i.e.
# maintain the maintainer and date/time details
cmd_args = ["dch", "-v", self._version, "-b", "-m"]
if message is None:
message = "new release {0}".format(self._version)
cmd_args += ["-p", message]
proc = subprocess.Popen(cmd_args,
cwd=self._repo_dir,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
shell=False)
(out, err) = proc.communicate()
if linked:
os.remove(os.path.join(self._repo_dir, "debian"))
if proc.returncode != 0:
raise RuntimeError(
"Failed to add an entry for {0} in debian/changelog due to {1}"
.format(self._version, err))
return True
def main():
# parse arguments
args = parse_command_line(sys.argv[1:])
try:
manifest = Manifest(args.manifest)
manifest.validate_manifest()
except KeyError as error:
print "Failed to create a Manifest instance for the manifest file {0} \nERROR:\n{1}"\
.format(args.manifest, error.message)
sys.exit(1)
if args.publish:
if args.git_credential:
repo_operator = RepoOperator(args.git_credential)
else:
print "Error occurs when get crendtail in update submodule"
sys.exit(1)
else:
repo_operator = RepoOperator(args.git_credential)
if os.path.isdir(args.build_dir):
print args.build_dir
for filename in os.listdir(args.build_dir):
try:
repo_dir = os.path.join(args.build_dir, filename)
repo_operator.submodule_init(repo_dir)
repo_operator.submodule_update(repo_dir)
submodules_list = repo_operator.get_current_submodule(repo_dir)
if len(submodules_list) == 0:
continue
for key in submodules_list:
commit_id = get_manifest_commit_id(key, manifest)
if commit_id != None:
sub_dir = repo_dir + "/" + key
repo_operator.checkout_to_commit(sub_dir, commit_id)
if args.publish:
print "start to publish update submodule in {0}".format(
repo_dir)
commit_message = "update submodule for new commit {0}".format(
args.version)
repo_operator.push_repo_changes(repo_dir, commit_message)
except Exception, e:
print "Failed to update submodule of {0} due to {1}".format(
filename, e)
sys.exit(1)
class ManifestActions(object):
"""
valid actions:
checkout: check out a set of repositories to match the manifest file
"""
valid_actions = ['checkout', 'branch', 'packagerefs', 'tag']
def __init__(self,
manifest_path,
builddir,
force=False,
git_credentials=None,
jobs=1,
actions=[],
branch_name=None,
tag_name=None):
"""
__force - Overwrite a directory if it exists
__git_credential - url, credentials pair for the access to github repos
__manifest - Repository manifest contents
__builddir - Destination for checked out repositories
__jobs - Number of parallel jobs to run
__actions -Supported actions
:return:
"""
self._force = force
self._git_credentials = git_credentials
self._builddir = builddir
self._manifest = None
self.handle_manifest(manifest_path)
self._jobs = jobs
self.actions = []
for action in actions:
self.add_action(action)
self._branch_name = branch_name
self._tag_name = tag_name
self.repo_operator = RepoOperator(self._git_credentials)
def set_force(self, force):
"""
Standard setter for force
:param force: if true, overwrite a directory if it exists
:return: None
"""
self._force = force
def get_force(self):
"""
Standard getter for force
:return: force
"""
return self._force
def set_branch_name(self, branch):
"""
Standard setter for branch_name
:param force: if true, overwrite a directory if it exists
:return: None
"""
self._branch_name = branch
def get_branch_name(self):
"""
Standard getter for branch_name
:return: force
"""
return self._branch_name
def set_git_credentials(self, git_credential):
"""
Standard setter for git_credentials
:param git_credential: url, credentials pair for the access to github repos
:return: None
"""
self._git_credentials = git_credential
self.repo_operator.setup_gitbit(credentials=self._git_credentials)
def get_manifest(self):
"""
Standard getter for manifest
:return: an instance of Manifest
"""
return self._manifest
def add_action(self, action):
"""
Add action to actions
:param action: a string, just like: checkout
:return: None
"""
if action not in self.valid_actions:
print "Unknown action '{0}' requested".format(action)
print "Valid actions are:"
for op in self.valid_actions:
print " {0}".format(op)
sys.exit(1)
else:
if action in ['branch', 'tag'] and self._git_credentials == None:
print "Must Specify git_credentials when try to write repository"
sys.exit(1)
self.actions.append(action)
def set_jobs(self, jobs):
"""
Standard setter for jobs
:param jobs: number of parallel jobs to run
:return: None
"""
self._jobs = jobs
if self._jobs < 1:
print "--jobs value must be an integer >=1"
sys.exit(1)
def handle_manifest(self, manifest_path):
"""
initial manifest and validate it
:param manifest_path: the path of manifest file
:return: None
"""
try:
self._manifest = Manifest(manifest_path)
self._manifest.validate_manifest()
except KeyError as error:
print "Failed to create a Manifest instance for the manifest file {0} \nERROR:\n{1}"\
.format(manifest_path, error.message)
sys.exit(1)
for repo in self._manifest.repositories:
repo['directory-name'] = self.directory_for_repo(repo)
def check_builddir(self):
"""
Checks the given builddir name and force flag.
Deletes exists directory if one already exists and --force is set
:return: None
"""
if os.path.exists(self._builddir):
if self._force:
shutil.rmtree(self._builddir)
print "Removing existing data at {0}".format(self._builddir)
else:
print "Unwilling to overwrite destination builddir of {0}".format(
self._builddir)
sys.exit(1)
os.makedirs(self._builddir)
def get_repositories(self):
"""
Issues checkout commands to dictionaries within a provided manifest
:return: None
"""
repo_list = self._manifest.repositories
try:
self.repo_operator.clone_repo_list(repo_list,
self._builddir,
jobs=self._jobs)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
def directory_for_repo(self, repo):
"""
Get the directory of a repository
:param repo: a dictionary
:return: the directary of repository
"""
if 'checked-out-directory-name' in repo:
repo_directory = repo['checked-out-directory-name']
else:
if 'repository' in repo:
repo_url = repo['repository']
repo_directory = strip_suffix(os.path.basename(repo_url),
".git")
else:
raise ValueError("no way to find basename")
repo_directory = os.path.join(self._builddir, repo_directory)
return repo_directory
def execute_actions(self):
"""
start to execute actions
:return: None
"""
# Start to check out a set of repositories within a manifest file
if 'checkout' in self.actions:
self.check_builddir()
self.get_repositories()
# Start to create branch and update package.json
if 'branch' in self.actions:
self.execute_branch_action()
if 'tag' in self.actions:
self.create_tag()
# Start to update the packge.json, for example:
# - git+https://github.com/RackHD/on-core.git
# +
if 'packagerefs' in self.actions:
self.update_package_references()
def execute_branch_action(self):
"""
execute the action "branch"
:return: None
"""
if self._branch_name is None:
raise ValueError("No setting for branch-name")
else:
print "create branch and update package.json for the repos..."
self.branch_existing_repositories()
self.checkout_branch_repositories(self._branch_name)
self.update_package_references(version=self._branch_name)
commit_message = "update the dependencies version to {0}".format(
self._branch_name)
self.push_changed_repositories(commit_message)
def branch_existing_repositories(self):
"""
Issues create branch commands to repos in a provided manifest
:return: None
"""
if self._branch_name is None:
print "Please provide the new branch name"
sys.exit(2)
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and create specified branch on each
for repo in repo_list:
self.create_repo_branch(repo, self._branch_name)
def create_repo_branch(self, repo, branch):
"""
create branch on the repos in the manifest file
:param repo: A dictionary
:return: None
"""
try:
repo_directory = self.directory_for_repo(repo)
repo_url = repo["repository"]
self.repo_operator.create_repo_branch(repo_url, repo_directory,
branch)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
def checkout_branch_repositories(self, branch):
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and checkout specified branch on each
for repo in repo_list:
self.checkout_repo_branch(repo, branch)
def checkout_repo_branch(self, repo, branch):
"""
checkout to a specify branch on repository
:param repo: A dictionary
:param branch: the specify branch name
:return: None
"""
try:
repo_directory = self.directory_for_repo(repo)
self.repo_operator.checkout_repo_branch(repo_directory, branch)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
def update_package_references(self, version=None):
print "Update internal package lists"
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and update package.json on each
for repo in repo_list:
self.update_repo_package_list(repo, pkg_version=version)
def update_repo_package_list(self, repo, pkg_version=None):
"""
Update the package.json of repository to point to new version
:param repo: a manifest repository entry
:param pkg_version: the version of package.json to point to
:return:
"""
repo_dir = repo['directory-name']
package_json_file = os.path.join(repo_dir, "package.json")
if not os.path.exists(package_json_file):
# if there's no package.json file, there is nothing more for us to do here
return
changes = False
log = ""
with open(package_json_file, "r") as fp:
package_data = json.load(fp)
if 'dependencies' in package_data:
for package, version in package_data['dependencies'].items():
new_version = self._update_dependency(
version, pkg_version=pkg_version)
if new_version != version:
log += " {0}:\n WAS {1}\n NOW {2}\n".format(
package, version, new_version)
package_data['dependencies'][package] = new_version
changes = True
if changes:
print "There are changes to dependencies for {0}\n{1}".format(
package_json_file, log)
os.remove(package_json_file)
new_file = package_json_file
with open(new_file, "w") as newfile:
json.dump(package_data, newfile, indent=4, sort_keys=True)
else:
print "There are NO changes to data for {0}".format(
package_json_file)
def _update_dependency(self, version, pkg_version=None):
"""
Check the specified package & version, and return a new package version if
the package is listed in the manifest.
:param version:
:return:
"""
if not version.startswith("git+"):
return version
url = strip_prefix(version, "git+")
url = url.split('#')[0]
new_url = url
if pkg_version is None:
for repo in self._manifest.repositories:
if new_url == repo['repository']:
if 'directory-name' in repo:
new_url = os.path.abspath(repo['directory-name'])
return new_url
else:
for repo in self._manifest.repositories:
if new_url == repo['repository']:
new_url = "git+{url}#{pkg_version}".format(
url=new_url, pkg_version=pkg_version)
return new_url
return version
def create_tag(self):
if self._tag_name is None:
raise ValueError("No setting for tag-name")
else:
print "create tag for the repos..."
self.create_tag_for_repositories()
def create_tag_for_repositories(self):
"""
Issues set_tagname commands to repos in a provided manifest
:return: None
"""
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and create specified tag on each
for repo in repo_list:
self.set_repo_tagname(repo)
def set_repo_tagname(self, repo):
"""
Add a tag to a repository
:param repo: A dictionary
:return: None
"""
try:
repo_url = repo["repository"]
repo_directory = self.directory_for_repo(repo)
self.repo_operator.set_repo_tagname(repo_url, repo_directory,
self._tag_name)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
def push_changed_repositories(self, commit_message):
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and publish changes on each
for repo in repo_list:
self.push_changed_repo(repo, commit_message)
def push_changed_repo(self, repo, commit_message):
"""
publish changes in the repository
:param repo: A dictionary
:param commit_message: the message to be added to the commit
:return: None
"""
repo_dir = repo['directory-name']
try:
self.repo_operator.push_repo_changes(repo_dir, commit_message)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
class ManifestActions(object):
"""
valid actions:
checkout: check out a set of repositories to match the manifest file
"""
valid_actions = ['checkout', 'branch', 'packagerefs', 'packagerefs-commit', 'tag']
def __init__(self, manifest_path, builddir, force=False, git_credentials=None, jobs=1, actions=[], branch_name=None, tag_name=None):
"""
__force - Overwrite a directory if it exists
__git_credential - url, credentials pair for the access to github repos
__manifest - Repository manifest contents
__builddir - Destination for checked out repositories
__jobs - Number of parallel jobs to run
__actions -Supported actions
:return:
"""
self._force = force
self._git_credentials = git_credentials
self._builddir = builddir
self._manifest = None
self.handle_manifest(manifest_path)
self._jobs = jobs
self.actions = []
for action in actions:
self.add_action(action)
self._branch_name = branch_name
self._tag_name = tag_name
self.repo_operator = RepoOperator(self._git_credentials)
def set_force(self, force):
"""
Standard setter for force
:param force: if true, overwrite a directory if it exists
:return: None
"""
self._force = force
def get_force(self):
"""
Standard getter for force
:return: force
"""
return self._force
def set_branch_name(self, branch):
"""
Standard setter for branch_name
:param force: if true, overwrite a directory if it exists
:return: None
"""
self._branch_name = branch
def get_branch_name(self):
"""
Standard getter for branch_name
:return: force
"""
return self._branch_name
def set_git_credentials(self, git_credential):
"""
Standard setter for git_credentials
:param git_credential: url, credentials pair for the access to github repos
:return: None
"""
self._git_credentials = git_credential
self.repo_operator.setup_gitbit(credentials=self._git_credentials)
def get_manifest(self):
"""
Standard getter for manifest
:return: an instance of Manifest
"""
return self._manifest
def add_action(self, action):
"""
Add action to actions
:param action: a string, just like: checkout
:return: None
"""
if action not in self.valid_actions:
print "Unknown action '{0}' requested".format(action)
print "Valid actions are:"
for op in self.valid_actions:
print " {0}".format(op)
sys.exit(1)
else:
if action in ['branch', 'tag'] and self._git_credentials == None:
print "Must Specify git_credentials when try to write repository"
sys.exit(1)
self.actions.append(action)
def set_jobs(self, jobs):
"""
Standard setter for jobs
:param jobs: number of parallel jobs to run
:return: None
"""
self._jobs = jobs
if self._jobs < 1:
print "--jobs value must be an integer >=1"
sys.exit(1)
def handle_manifest(self, manifest_path):
"""
initial manifest and validate it
:param manifest_path: the path of manifest file
:return: None
"""
try:
self._manifest = Manifest(manifest_path)
self._manifest.validate_manifest()
except KeyError as error:
print "Failed to create a Manifest instance for the manifest file {0} \nERROR:\n{1}"\
.format(manifest_path, error.message)
sys.exit(1)
for repo in self._manifest.repositories:
repo['directory-name'] = self.directory_for_repo(repo)
def check_builddir(self):
"""
Checks the given builddir name and force flag.
Deletes exists directory if one already exists and --force is set
:return: None
"""
if os.path.exists(self._builddir):
if self._force:
shutil.rmtree(self._builddir)
print "Removing existing data at {0}".format(self._builddir)
else:
print "Unwilling to overwrite destination builddir of {0}".format(self._builddir)
sys.exit(1)
os.makedirs(self._builddir)
def get_repositories(self):
"""
Issues checkout commands to dictionaries within a provided manifest
:return: None
"""
repo_list = self._manifest.repositories
try:
self.repo_operator.clone_repo_list(repo_list, self._builddir, jobs=self._jobs)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
def directory_for_repo(self, repo):
"""
Get the directory of a repository
:param repo: a dictionary
:return: the directary of repository
"""
if 'checked-out-directory-name' in repo:
repo_directory = repo['checked-out-directory-name']
else:
if 'repository' in repo:
repo_url = repo['repository']
repo_directory = strip_suffix(os.path.basename(repo_url), ".git")
else:
raise ValueError("no way to find basename")
repo_directory = os.path.join(self._builddir, repo_directory)
return repo_directory
def execute_actions(self):
"""
start to execute actions
:return: None
"""
# Start to check out a set of repositories within a manifest file
if 'checkout' in self.actions:
self.check_builddir()
self.get_repositories()
# Start to create branch and update package.json
if 'branch' in self.actions:
self.execute_branch_action()
if 'tag' in self.actions:
self.create_tag()
# Start to update the packge.json, for example:
# - git+https://github.com/RackHD/on-core.git
# +
if 'packagerefs' in self.actions:
self.update_package_references()
# Update package dependency with repo commit-id
if 'packagerefs-commit' in self.actions:
self.update_package_references(ref_type="commit")
def execute_branch_action(self):
"""
execute the action "branch"
:return: None
"""
if self._branch_name is None:
raise ValueError("No setting for branch-name")
else:
print "create branch and update package.json for the repos..."
self.branch_existing_repositories()
self.checkout_branch_repositories(self._branch_name)
self.update_package_references(version=self._branch_name)
commit_message = "update the dependencies version to {0}".format(self._branch_name)
self.push_changed_repositories(commit_message)
def branch_existing_repositories(self):
"""
Issues create branch commands to repos in a provided manifest
:return: None
"""
if self._branch_name is None:
print "Please provide the new branch name"
sys.exit(2)
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and create specified branch on each
for repo in repo_list:
self.create_repo_branch(repo, self._branch_name)
def create_repo_branch(self, repo, branch):
"""
create branch on the repos in the manifest file
:param repo: A dictionary
:return: None
"""
try:
repo_directory = self.directory_for_repo(repo)
repo_url = repo["repository"]
self.repo_operator.create_repo_branch(repo_url, repo_directory, branch)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
def checkout_branch_repositories(self, branch):
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and checkout specified branch on each
for repo in repo_list:
self.checkout_repo_branch(repo, branch)
def checkout_repo_branch(self, repo, branch):
"""
checkout to a specify branch on repository
:param repo: A dictionary
:param branch: the specify branch name
:return: None
"""
try:
repo_directory = self.directory_for_repo(repo)
self.repo_operator.checkout_repo_branch(repo_directory, branch)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
def update_package_references(self, version=None, ref_type="path"):
print "Update internal package lists"
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and update package.json on each
for repo in repo_list:
self.update_repo_package_list(repo, pkg_version=version, ref_type=ref_type)
def update_repo_package_list(self, repo, pkg_version=None, ref_type="path"):
"""
Update the package.json of repository to point to new version
:param repo: a manifest repository entry
:param pkg_version: the version of package.json to point to
:return:
"""
repo_dir = repo['directory-name']
package_json_file = os.path.join(repo_dir, "package.json")
if not os.path.exists(package_json_file):
# if there's no package.json file, there is nothing more for us to do here
return
changes = False
log = ""
with open(package_json_file, "r") as fp:
package_data = json.load(fp)
if 'dependencies' in package_data:
for package, version in package_data['dependencies'].items():
new_version = self._update_dependency(version, pkg_version=pkg_version, ref_type=ref_type)
if new_version != version:
log += " {0}:\n WAS {1}\n NOW {2}\n".format(package,
version,
new_version)
package_data['dependencies'][package] = new_version
changes = True
if changes:
print "There are changes to dependencies for {0}\n{1}".format(package_json_file, log)
os.remove(package_json_file)
new_file = package_json_file
with open(new_file, "w") as newfile:
json.dump(package_data, newfile, indent=4, sort_keys=True)
else:
print "There are NO changes to data for {0}".format(package_json_file)
def _update_dependency(self, version, pkg_version=None, ref_type="path"):
"""
Check the specified package & version, and return a new package version if
the package is listed in the manifest.
:param version:
:return:
"""
if not version.startswith("git+"):
return version
url = strip_prefix(version, "git+")
url = url.split('#')[0]
new_url = url
if pkg_version is None:
for repo in self._manifest.repositories:
if new_url == repo['repository']:
if ref_type == "path":
if 'directory-name' in repo:
new_url = os.path.abspath(repo['directory-name'])
return new_url
elif ref_type == "commit":
new_url = "git+{url}#{pkg_version}".format(url=new_url, pkg_version=repo['commit-id'])
return new_url
else:
for repo in self._manifest.repositories:
if new_url == repo['repository']:
new_url = "git+{url}#{pkg_version}".format(url=new_url, pkg_version=pkg_version)
return new_url
return version
def create_tag(self):
if self._tag_name is None:
raise ValueError("No setting for tag-name")
else:
print "create tag for the repos..."
self.create_tag_for_repositories()
def create_tag_for_repositories(self):
"""
Issues set_tagname commands to repos in a provided manifest
:return: None
"""
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and create specified tag on each
for repo in repo_list:
self.set_repo_tagname(repo)
def set_repo_tagname(self, repo):
"""
Add a tag to a repository
:param repo: A dictionary
:return: None
"""
try:
repo_url = repo["repository"]
repo_directory = self.directory_for_repo(repo)
self.repo_operator.set_repo_tagname(repo_url, repo_directory, self._tag_name)
except RuntimeError as error:
raise RuntimeError("Failed to create tag {0} for {1} \ndue to error: {2}".format(self._tag_name, repo_url, error))
def push_changed_repositories(self, commit_message):
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and publish changes on each
for repo in repo_list:
self.push_changed_repo(repo, commit_message)
def push_changed_repo(self, repo, commit_message):
"""
publish changes in the repository
:param repo: A dictionary
:param commit_message: the message to be added to the commit
:return: None
"""
repo_dir = repo['directory-name']
try:
self.repo_operator.push_repo_changes(repo_dir, commit_message)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
class UpdateManifest(object):
def __init__(self):
"""
__repo - the repository url being updated in the manifest
__branch - the branch name associated with __repo
__sliced_branch - the branch name sliced at any forward slashes
__commit - The commit id associated with the __repo and __branch that we want to update
__manifest_repository_url - the repository url the points to the collection of manifest files
__manifest_file - the desired manifest file to update which resides in __manifest_repository_url
__cleanup_directories - the path/name of directories created are appended here for cleanup in task_cleanup
__git_credentials - url, credentials pair for the access to github repos
quiet - used for testing to minimize text written to a terminal
repo_operator - Class instance of RepoOperator
:return: None
"""
self.__repo = None
self.__branch = None
self.__sliced_branch = None
self.__commit = None
self.__manifest_repository_url = None
self.__manifest_file = None
self.__cleanup_directories = []
self.__git_credentials = None
self.__updated_manifest = None
self.__dryrun = False
self.quiet = False
self.repo_operator = RepoOperator()
#pylint: disable=no-self-use
def parse_args(self, args):
"""
Take in values from the user.
Repo, branch, and manifest_repo are required. This exits if they are not given.
:return: Parsed args for assignment
"""
parser = argparse.ArgumentParser()
parser.add_argument(
'-n',
'--dryrun',
help="Do not commit any changes, just print what would be done",
action="store_true")
parser.add_argument("--repo",
help="Git url to match for updating the commit-id",
action="store")
parser.add_argument("--branch",
help="The target branch for the named repo",
action="store")
parser.add_argument("--manifest_repo",
help="The manifest repository URL.",
action="store")
parser.add_argument(
"--commit",
help="OPTIONAL: The commit id to target an exact version",
action="store")
parser.add_argument(
"--manifest_file",
help=
"The target manifest file to update. Searches through all if left empty.",
action="store")
parser.add_argument("--git-credential",
help="Git URL and credentials comma separated",
action="append")
parser.add_argument(
'--updated_manifest',
help="file containing the name of the updated manifest",
action="store")
parser = parser.parse_args(args)
return parser
def assign_args(self, args):
"""
Assign args to member variables and perform checks on branch and commit-id.
:param args: Parsed args from the user
:return:
"""
if args.repo:
self.__repo = args.repo
else:
print "\nMust specify repository url for cloning (--repo <git_url>)\n"
if args.branch:
self.__branch = args.branch
if "/" in self.__branch:
self.__sliced_branch = self.__branch.split("/")[-1]
else:
self.__sliced_branch = self.__branch
else:
print "\nMust specify a branch name (--branch <branch_name>)\n"
if args.manifest_repo:
self.__manifest_repository_url = args.manifest_repo
else:
print "\n Must specify a full repository url for retrieving <manifest>.json files\n"
if args.dryrun:
self.__dryrun = True
self.repo_operator.setup_git_dryrun(self.__dryrun)
if args.commit:
self.__commit = args.commit
if args.manifest_file:
self.__manifest_file = args.manifest_file
if args.git_credential:
self.__git_credentials = args.git_credential
self.repo_operator.setup_gitbit(credentials=self.__git_credentials)
if args.updated_manifest:
self.__updated_manifest = args.updated_manifest
def check_args(self):
"""
Check the values given for branch and commit-id
"""
if self.__branch:
try:
self.repo_operator.check_branch(self.__repo,
self.__sliced_branch)
except RuntimeError as error:
self.cleanup_and_exit(error, 1)
if self.__commit:
self.__check_commit()
def __check_commit(self):
"""
Check the format of the commit-id. It must be 40 hex characters.
Exits if it is not.
:return: None
"""
commit_match = re.match('^[0-9a-fA-F]{40}$', self.__commit)
if not commit_match:
self.cleanup_and_exit(
"Id, '{0}' is not valid. It must be a 40 character hex string."
.format(self.__commit), 1)
def clone_manifest_repo(self):
"""
Clone the repository which holds the manifest json files. Return that directory name. The directory
is temporary and deleted in the cleanup_and_exit function
:return: A string containing the name of the folder where the manifest repo was cloned
"""
directory_name = tempfile.mkdtemp()
if os.path.isdir(directory_name):
self.__cleanup_directories.append(directory_name)
else:
self.cleanup_and_exit(
"Failed to make temporary directory for the repository: {0}".
format(url), 1)
try:
manifest_repo = self.repo_operator.clone_repo(
self.__manifest_repository_url, directory_name)
return manifest_repo
except RuntimeError as error:
self.cleanup_and_exit(error, 1)
def validate_manifest_files(self, *args):
"""
validate several manifest files
For example: validate_manifest_files(file1, file2) or
validate_manifest_files(file1, file2, file3)
"""
validate_result = True
for filename in args:
try:
manifest = Manifest(filename)
manifest.validate_manifest()
print "manifest file {0} is valid".format(filename)
except KeyError as error:
print "Failed to validate manifest file {0}".format(filename)
print "\nERROR: \n{0}".format(error.message)
validate_result = False
return validate_result
def cleanup_and_exit(self, message=None, code=0):
"""
Delete all files and folders made during this job which are named in self.cleanup_directories
:return: None
"""
if not self.quiet:
if message is not None:
print "\nERROR: {0}".format(message)
print "\nCleaning environment!\n"
for item in self.__cleanup_directories:
subprocess.check_output(["rm", "-rf", item])
sys.exit(code)
def get_updated_commit_message(self):
"""
get the updated repository commit message
"""
# get commit message based on the arguments repo, branch and commit
directory_name = tempfile.mkdtemp()
if os.path.isdir(directory_name):
self.__cleanup_directories.append(directory_name)
else:
self.cleanup_and_exit(
"Failed to make temporary directory for the repository: {0}".
format(url), 1)
try:
updated_repo_dir = self.repo_operator.clone_repo(
self.__repo, directory_name)
repo_commit_message = self.repo_operator.get_commit_message(
updated_repo_dir, self.__commit)
return repo_commit_message
except RuntimeError as error:
self.cleanup_and_exit(error, 1)
def update_manifest_repo(self, dir_name, repo_commit_message):
"""
Update manifest repository based on its contents and user arguments.
:param dir_name: The directory of the repository
:return: if repo is updated, return updated manifest file path and the manifest object
otherwise, return None, None
"""
url_short_name = self.__repo.split('/')[-1][:-4]
commit_message = "update to {0} */{1} commit #{2}\n{3}"\
.format(url_short_name, self.__branch, self.__commit, repo_commit_message)
if self.__manifest_file is not None:
path_name = os.path.join(dir_name, self.__manifest_file)
if os.path.isfile(path_name):
try:
manifest = Manifest(path_name, self.__git_credentials)
manifest.update_manifest(self.__repo, self.__branch,
self.__commit)
if manifest.changed:
manifest.write_manifest_file(dir_name, commit_message,
path_name, self.__dryrun)
return path_name, manifest
else:
print "No changes to {0}".format(manifest.name)
except KeyError as error:
self.cleanup_and_exit("Failed to create an Manifest instance for the manifest file {0}\nError:{1}"\
.format(self.__manifest_file, error.message),1)
except RuntimeError as error:
self.cleanup_and_exit(
"Failed to update manifest repo\nError:{0}".format(
error.message), 1)
else:
for item in os.listdir(dir_name):
path_name = os.path.join(dir_name, item)
if os.path.isfile(path_name):
try:
manifest = Manifest(path_name, self.__git_credentials)
manifest.update_manifest(self.__repo, self.__branch,
self.__commit)
if manifest.changed:
manifest.write_manifest_file(
dir_name, commit_message, path_name,
self.__dryrun)
return path_name, manifest
else:
print "No changes to {0}".format(manifest.name)
except KeyError as error:
self.cleanup_and_exit("Failed to create an Manifest instance for the manifest file {0}\nError:{1}"\
.format(path_name, error.message),1)
except RuntimeError as error:
self.cleanup_and_exit(
"Failed to update manifest repo\nError:{0}".format(
error.message), 1)
return None, None
def write_downstream_parameters(self, filename, params):
"""
Add/append downstream parameter (java variable value pair) to the given
parameter file. If the file does not exist, then create the file.
:param filename: The parameter file that will be used for making environment
variable for downstream job.
:param params: the parameters dictionary
:return:
None on success
Raise any error if there is any
"""
if filename is None:
return
with open(filename, 'w') as fp:
try:
for key in params:
entry = "{key}={value}\n".format(key=key,
value=params[key])
fp.write(entry)
except IOError as error:
print "Unable to write parameter(s) for next step(s), exit"
self.cleanup_and_exit(error, 1)
def downstream_manifest_to_use(self, manifest_folder, file_with_path,
manifest):
"""
Write file which contains the name of the manifest file most recently updated.
:param manifest_folder: the path of the manifest repository
:param file_with_path: The path to be split to claim the filename
:param manifest: the Manifest object of manifest file
"""
file_name = file_with_path.split('/')[-1]
downstream_parameters = {}
downstream_parameters['MANIFEST_FILE_NAME'] = file_name
downstream_parameters[
'BUILD_REQUIREMENTS'] = manifest.build_requirements
downstream_parameters[
'MANIFEST_FILE_REPO'] = self.__manifest_repository_url
try:
downstream_parameters[
'MANIFEST_FILE_BRANCH'] = self.repo_operator.get_current_branch(
manifest_folder)
downstream_parameters[
'MANIFEST_FILE_COMMIT'] = self.repo_operator.get_lastest_commit_id(
manifest_folder)
except RuntimeError as error:
self.cleanup_and_exit(error, 1)
self.write_downstream_parameters(self.__updated_manifest,
downstream_parameters)
return
def create_inject_properties_file(self):
"""
Create inject.properties file for env vars injection after excute shell.
If upstream_manifest_name exists then copy it to inject.properties, else
create an empty inject.properties file to avoid exceptions throw by plugin.
"""
if os.path.isfile(self.__updated_manifest):
try:
shutil.copyfile(self.__updated_manifest, "inject.properties")
except IOError as error:
print "ERROR: copy file {0} to inject.properties error.\n{1}".format(
self.__updated_manifest, error)
sys.exit(1)
else:
try:
open('inject.properties', 'a').close()
except IOError as error:
print "ERROR: create empty file inject.properties error.\n{0}".format(
error)
sys.exit(1)
return
class ManifestGenerator(object):
def __init__(self, dest, branch, builddir, git_credential, force=False, jobs=1):
"""
Generate a new manifest according to the manifest sample: manifest.json
_dest_manifest_file: the path of new manifest
_branch: the branch name
_force: overwrite the destination if it exists.
_builddir: the destination for checked out repositories.
_jobs: number of parallel jobs to run. The number is related to the compute architecture, multi-core processors...
:return: None
"""
self._dest_manifest_file = dest
self._branch = branch
self._builddir = builddir
self._force = force
self._jobs = jobs
self._manifest = Manifest.instance_of_sample()
self.repo_operator = RepoOperator(git_credential)
self.check_builddir()
def directory_for_repo(self, repo):
"""
Get the directory of a repository
:param repo: a dictionary
:return: the directary of repository
"""
if 'checked-out-directory-name' in repo:
repo_directory = repo['checked-out-directory-name']
else:
if 'repository' in repo:
repo_url = repo['repository']
repo_directory = common.strip_suffix(os.path.basename(repo_url), ".git")
else:
raise ValueError("no way to find basename")
repo_directory = os.path.join(self._builddir, repo_directory)
return repo_directory
def check_builddir(self):
"""
Checks the given builddir name and force flag.
Deletes exists directory if one already exists and --force is set
:return: None
"""
if os.path.exists(self._builddir):
if self._force:
shutil.rmtree(self._builddir)
print "Removing existing data at {0}".format(self._builddir)
else:
print "Unwilling to overwrite destination builddir of {0}".format(self._builddir)
sys.exit(1)
os.makedirs(self._builddir)
def update_repositories_commit(self, repositories):
"""
update the commit-id of repository with the lastest commit id
:param repositories: a list of repository directory
:return: None
"""
for repo in repositories:
repo_dir = self.directory_for_repo(repo)
repo["commit-id"] = self.repo_operator.get_lastest_commit_id(repo_dir)
def update_manifest(self):
"""
update the manifest with branch name
:return: None
"""
repositories = self._manifest.repositories
downstream_jobs = self._manifest.downstream_jobs
for repo in repositories:
repo["branch"] = self._branch
repo["commit-id"] = ""
self.repo_operator.clone_repo_list(repositories, self._builddir, jobs=self._jobs)
self.update_repositories_commit(repositories)
for job in downstream_jobs:
job["branch"] = self._branch
repo["commit-id"] = ""
self.repo_operator.clone_repo_list(downstream_jobs, self._builddir, jobs=self._jobs)
self.update_repositories_commit(downstream_jobs)
self._manifest.validate_manifest()
def generate_manifest(self):
"""
generate a new manifest
:return: None
"""
if os.path.isfile(self._dest_manifest_file):
if self._force == False:
raise RuntimeError("The file {0} already exist . \n \
If you want to overrite the file, please specify --force."
.format(dest_file))
with open(self._dest_manifest_file, 'w') as fp:
json.dump(self._manifest.manifest, fp, indent=4, sort_keys=True)
class VersionGenerator(object):
def __init__(self, repo_dir):
"""
This module compute the version of a repository
The version for candidate release: {release_version}-{build_version}
The release version is parsed from debian/changelog
The samll version is consist of the commit hash and commit date of manifest repository
:return:None
"""
self._repo_dir = repo_dir
self.repo_operator = RepoOperator()
self._repo_name = self.get_repo_name()
def get_repo_name(self):
repo_url = self.repo_operator.get_repo_url(self._repo_dir)
repo_name = common.strip_suffix(os.path.basename(repo_url), ".git")
return repo_name
def generate_build_version(self):
"""
Generate the build version which consists of commit date and commit hash of manifest repository
According to build version, users can track the commit of all repositories in manifest file
return: build version
"""
if self._repo_name == "RackHD":
utc_now = datetime.utcnow()
utc_yesterday = utc_now + timedelta(days=-1)
version = utc_yesterday.strftime('%Y%m%dUTC')
return version
else:
commit_timestamp_str = self.repo_operator.get_lastest_commit_date(
self._repo_dir)
date = datetime.utcfromtimestamp(
int(commit_timestamp_str)).strftime('%Y%m%dUTC')
commit_id = self.repo_operator.get_lastest_commit_id(
self._repo_dir)
version = "{date}-{commit}".format(date=date,
commit=commit_id[0:7])
return version
def debian_exist(self):
"""
check whether debian or debianstatic directory under the repository
return: True if debian or debianstatic exist
False
"""
if os.path.isdir(self._repo_dir):
for filename in os.listdir(self._repo_dir):
if filename == "debian":
return True
return False
def generate_debian_release_version(self):
"""
Generate the release version according to changelog
The release version is the latest version of debian/changelog
return: release version
"""
# If the repository has the debianstatic/repository name/,
# create a soft link to debian before compute version
debian_exist = self.debian_exist()
linked = False
if not debian_exist:
for filename in os.listdir(self._repo_dir):
if filename == "debianstatic":
debianstatic_dir = os.path.join(self._repo_dir,
"debianstatic")
for debianstatic_filename in os.listdir(debianstatic_dir):
if debianstatic_filename == self._repo_name:
debianstatic_repo_dir = "debianstatic/{0}".format(
self._repo_name)
common.link_dir(debianstatic_repo_dir, "debian",
self._repo_dir)
linked = True
if not debian_exist and not linked:
return None
cmd_args = ["dpkg-parsechangelog", "--show-field", "Version"]
version = common.run_command(cmd_args, directory=self._repo_dir)
if linked:
os.remove(os.path.join(self._repo_dir, "debian"))
return version
def generate_npm_release_version(self):
"""
Generate the release version according to the version field in package.json
"""
package_json_file = os.path.join(self._repo_dir, "package.json")
if not os.path.exists(package_json_file):
# if there's no package.json file, there is nothing more for us to do here
return None
with open(package_json_file, "r") as fp:
package_data = json.load(fp)
fp.close()
version = package_data["version"]
return version
def generate_package_version(self,
is_official_release,
version_type="debian"):
"""
Generate the version of package, just like:
1.1.1-20160809150908UTC-7396d91 or 1.1.1
:return: package version
"""
if version_type == "debian":
release_version = self.generate_debian_release_version()
elif version_type == "npm":
release_version = self.generate_npm_release_version()
else:
common.logging.error(
"The parameter version_type {0} is not valid".format(
version_type))
common.logging.error(
"The parameter version_type can only be debian or npm")
return None
if release_version is None:
common.logging.warning(
"Failed to generate release version, maybe the {0} doesn't contain debian directory or package.json"
.format(self._repo_dir))
return None
if is_official_release:
version = release_version
else:
build_version = self.generate_build_version()
if build_version is None:
raise RuntimeError(
"Failed to generate version for {0}, due to the build version is None"
.format(self._repo_dir))
version = "{0}-{1}".format(release_version, build_version)
return version
class VersionGenerator(object):
def __init__(self, repo_dir):
"""
This module compute the version of a repository
The version for candidate release: {big-version}~{version-stage}-{small-version}
The big version is parsed from debian/changelog
The version-stage is devel if branch is master; or rc if branch if not master
The samll version is consist of the commit hash and commit date of manifest repository
:return:None
"""
self._repo_dir = repo_dir
self.repo_operator = RepoOperator()
self._repo_name = self.get_repo_name()
def get_repo_name(self):
repo_url = self.repo_operator.get_repo_url(self._repo_dir)
repo_name = common.strip_suffix(os.path.basename(repo_url), ".git")
return repo_name
def generate_small_version(self):
"""
Generate the small version which consists of commit date and commit hash of manifest repository
According to small version, users can track the commit of all repositories in manifest file
return: small version
"""
if self._repo_name == "RackHD":
utc_now = datetime.utcnow()
utc_yesterday = utc_now + timedelta(days=-1)
version = utc_yesterday.strftime('%Y%m%dUTC')
return version
else:
commit_timestamp_str = self.repo_operator.get_lastest_commit_date(
self._repo_dir)
date = datetime.utcfromtimestamp(
int(commit_timestamp_str)).strftime('%Y%m%dUTC')
commit_id = self.repo_operator.get_lastest_commit_id(
self._repo_dir)
version = "{date}-{commit}".format(date=date,
commit=commit_id[0:7])
return version
def debian_exist(self):
"""
check whether debian or debianstatic directory under the repository
return: True if debian or debianstatic exist
False
"""
if os.path.isdir(self._repo_dir):
for filename in os.listdir(self._repo_dir):
if filename == "debian":
return True
return False
def generate_big_version(self):
"""
Generate the big version according to changelog
The big version is the latest version of debian/changelog
return: big version
"""
# If the repository has the debianstatic/repository name/,
# create a soft link to debian before compute version
debian_exist = self.debian_exist()
linked = False
if not debian_exist:
for filename in os.listdir(self._repo_dir):
if filename == "debianstatic":
debianstatic_dir = os.path.join(self._repo_dir,
"debianstatic")
for debianstatic_filename in os.listdir(debianstatic_dir):
if debianstatic_filename == self._repo_name:
debianstatic_repo_dir = "debianstatic/{0}".format(
self._repo_name)
common.link_dir(debianstatic_repo_dir, "debian",
self._repo_dir)
linked = True
if not debian_exist and not linked:
return None
cmd_args = ["dpkg-parsechangelog", "--show-field", "Version"]
version = common.run_command(cmd_args, directory=self._repo_dir)
if linked:
os.remove(os.path.join(self._repo_dir, "debian"))
return version
def generate_version_stage(self):
"""
Generate the version stage according to the stage of deveplopment
return: devel ,if the branch is master
rc, if the branch is not master
"""
current_branch = self.repo_operator.get_current_branch(self._repo_dir)
version_stage = ""
if "master" in current_branch:
version_stage = "devel"
else:
version_stage = "rc"
return version_stage
def generate_package_version(self, is_official_release):
"""
generate the version of package, just like:
1.1-1-devel-20160809150908-7396d91 or 1.1-1
:return: package version
"""
big_version = self.generate_big_version()
if big_version is None:
common.logging.warning(
"Failed to generate big version, maybe the {0} doesn't contain debian directory"
.format(self._repo_dir))
return None
if is_official_release:
version = big_version
else:
small_version = self.generate_small_version()
if small_version is None:
raise RuntimeError(
"Failed to generate version for {0}, due to the small version is None"
.format(self._repo_dir))
version = "{0}-{1}".format(big_version, small_version)
return version
class ChangelogUpdater(object):
def __init__(self, repo_dir, version):
"""
The module updates debian/changelog under the directory of repository
_repo_dir: the directory of the repository
_version: the new version which is going to be updated to changelog
"""
self._repo_dir = repo_dir
self._version = version
self.repo_operator = RepoOperator()
def debian_exist(self):
"""
check whether debian directory under the repository
return: True if debian exist
False
"""
if os.path.isdir(self._repo_dir):
for filename in os.listdir(self._repo_dir):
if filename == "debian":
return True
return False
def get_repo_name(self):
"""
get the name of the repository
:return: the name of the repository
"""
repo_url = self.repo_operator.get_repo_url(self._repo_dir)
repo_name = strip_suffix(os.path.basename(repo_url), ".git")
return repo_name
def update_changelog(self, message=None):
"""
add an entry to changelog
:param message: the message which is going to be added to changelog
return: Ture if changelog is updated
False, otherwise
"""
repo_name = self.get_repo_name()
debian_exist = self.debian_exist()
linked = False
if not debian_exist:
# Handle repository which contains debianstatic/repository_name folder,
# for example: debianstatic/on-http
for filename in os.listdir(self._repo_dir):
if filename == "debianstatic":
debianstatic_dir = os.path.join(self._repo_dir, "debianstatic")
for debianstatic_filename in os.listdir(debianstatic_dir):
if debianstatic_filename == repo_name:
debianstatic_repo_dir = "debianstatic/{0}".format(repo_name)
link_dir(debianstatic_repo_dir, "debian", self._repo_dir)
linked = True
if not debian_exist and not linked:
return False
print "start to update changelog of {0}".format(self._repo_dir)
# -v: Add a new changelog entry with version number specified
# -b: Force a version to be less than the current one
# -m: Don't change (maintain) the trailer line in the changelog entry; i.e.
# maintain the maintainer and date/time details
cmd_args = ["dch", "-v", self._version, "-b", "-m"]
if message is None:
message = "new release {0}".format(self._version)
cmd_args += ["-p", message]
proc = subprocess.Popen(cmd_args,
cwd=self._repo_dir,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
shell=False)
(out, err) = proc.communicate()
if linked:
os.remove(os.path.join(self._repo_dir, "debian"))
if proc.returncode != 0:
raise RuntimeError("Failed to add an entry for {0} in debian/changelog due to {1}".format(self._version, err))
return True
class UpdateManifest(object):
def __init__(self):
"""
__repo - the repository url being updated in the manifest
__branch - the branch name associated with __repo
__sliced_branch - the branch name sliced at any forward slashes
__commit - The commit id associated with the __repo and __branch that we want to update
__manifest_base_url - the base download url points to the collection of manifest files
__manifest_file - the desired manifest file to update which resides in __manifest_repository_url
__cleanup_directories - the path/name of directories created are appended here for cleanup in task_cleanup
__git_credentials - url, credentials pair for the access to github repos
quiet - used for testing to minimize text written to a terminal
repo_operator - Class instance of RepoOperator
:return: None
"""
self.__repo = None
self.__branch = None
self.__sliced_branch = None
self.__commit = None
self.__manifest_download_url = None
self.__manifest_file = None
self.__cleanup_directories = []
self.__git_credentials = None
self.__updated_manifest = None
self.__dryrun = False
self.quiet = False
self.repo_operator = RepoOperator()
#pylint: disable=no-self-use
def parse_args(self, args):
"""
Take in values from the user.
Repo, branch, and manifest_repo are required. This exits if they are not given.
:return: Parsed args for assignment
"""
parser = argparse.ArgumentParser()
parser.add_argument(
'-n',
'--dryrun',
help="Do not commit any changes, just print what would be done",
action="store_true")
parser.add_argument("--repo",
help="Git url to match for updating the commit-id",
action="store")
parser.add_argument("--branch",
help="The target branch for the named repo",
action="store")
parser.add_argument("--manifest_download_url",
help="The manifest derectory URL in bintray.",
action="store")
parser.add_argument(
"--commit",
help="OPTIONAL: The commit id to target an exact version",
action="store")
parser.add_argument(
"--manifest_file",
help=
"The target manifest file to be updated. Searches through all if left empty.",
action="store")
parser.add_argument("--git-credential",
help="Git URL and credentials comma separated",
action="append")
parser.add_argument(
'--updated-manifest',
help=
"Output file containing the name of the updated manifest, the download url, branch, commit of manifest repository. The key, value pairs will be passed to downstream jobs as parameters",
action="store")
parser = parser.parse_args(args)
return parser
def assign_args(self, args):
"""
Assign args to member variables and perform checks on branch and commit-id.
:param args: Parsed args from the user
:return:
"""
if args.repo:
self.__repo = args.repo
else:
print "\nMust specify repository url for cloning (--repo <git_url>)\n"
if args.branch:
self.__branch = args.branch
if "/" in self.__branch:
self.__sliced_branch = self.__branch.split("/")[-1]
else:
self.__sliced_branch = self.__branch
else:
print "\nMust specify a branch name (--branch <branch_name>)\n"
# if args.manifest_repo:
# self.__manifest_repository_url = args.manifest_repo
# else:
# print "\n Must specify a full repository url for retrieving <manifest>.json files\n"
if args.manifest_download_url:
self.__manifest_download_url = args.manifest_download_url
else:
print "\n Must specify a manifest base url for download <manifest>.json files\n"
if args.dryrun:
self.__dryrun = True
self.repo_operator.setup_git_dryrun(self.__dryrun)
if args.commit:
self.__commit = args.commit
if args.manifest_file:
self.__manifest_file = args.manifest_file
if args.git_credential:
self.__git_credentials = args.git_credential
self.repo_operator.setup_gitbit(credentials=self.__git_credentials)
if args.updated_manifest:
self.__updated_manifest = args.updated_manifest
def check_args(self):
"""
Check the values given for branch and commit-id
"""
if self.__branch:
try:
self.repo_operator.check_branch(self.__repo,
self.__sliced_branch)
except RuntimeError as error:
self.cleanup_and_exit(error, 1)
if self.__commit:
self.__check_commit()
def __check_commit(self):
"""
Check the format of the commit-id. It must be 40 hex characters.
Exits if it is not.
:return: None
"""
commit_match = re.match('^[0-9a-fA-F]{40}$', self.__commit)
if not commit_match:
self.cleanup_and_exit(
"Id, '{0}' is not valid. It must be a 40 character hex string."
.format(self.__commit), 1)
def download_manifest_file(self):
"""
Download the manifest json files. Return that directory name which stores manifest. The directory
is temporary and deleted in the cleanup_and_exit function
:return: A string containing the name of the folder where the manifest file was download.
"""
directory_name = tempfile.mkdtemp()
if os.path.isdir(directory_name):
pass
# For now script of 'upload to bintray' is seperated from this script.
# So before uploading this directory shouldn't be deleted'
# The blow code snippet will be unfolded when involved bintray upload functions into this script.
# self.__cleanup_directories.append(directory_name)
else:
self.cleanup_and_exit(
"Failed to make temporary directory for the repository: {0}".
format(url), 1)
try:
url = "/".join(
[self.__manifest_download_url, self.__manifest_file])
dest_dir = "/".join([directory_name, self.__manifest_file])
if os.environ['BINTRAY_USERNAME'] and os.environ['BINTRAY_API_KEY']:
print "Requests bintray with token"
auth = (os.environ['BINTRAY_USERNAME'].strip(),
os.environ['BINTRAY_API_KEY'].strip())
resp = requests.get(url, auth=auth)
else:
print "Requests without token"
resp = requests.get(url)
if resp.ok:
with open(dest_dir, "wb") as file_handle:
file_handle.write(resp.content)
elif resp.status_code == 404:
# If there's no manifest file in bintray server, init an empty one
print "can't find manifest in remote server, will use template manifest"
Manifest.instance_of_sample().dump_to_json_file(dest_dir)
else:
print "Unknown error, {0}".format(resp.status_code)
return directory_name
except RuntimeError as error:
self.cleanup_and_exit(error, 1)
def validate_manifest_files(self, *args):
"""
validate several manifest files
For example: validate_manifest_files(file1, file2) or
validate_manifest_files(file1, file2, file3)
"""
validate_result = True
for filename in args:
try:
manifest = Manifest(filename)
manifest.validate_manifest()
print "manifest file {0} is valid".format(filename)
except KeyError as error:
print "Failed to validate manifest file {0}".format(filename)
print "\nERROR: \n{0}".format(error.message)
validate_result = False
return validate_result
def cleanup_and_exit(self, message=None, code=0):
"""
Delete all files and folders made during this job which are named in self.cleanup_directories
:return: None
"""
if not self.quiet:
if message is not None:
print "\nERROR: {0}".format(message)
print "\nCleaning environment!\n"
for item in self.__cleanup_directories:
subprocess.check_output(["rm", "-rf", item])
sys.exit(code)
def get_updated_commit_message(self):
"""
get the updated repository commit message
"""
# get commit message based on the arguments repo, branch and commit
directory_name = tempfile.mkdtemp()
if os.path.isdir(directory_name):
self.__cleanup_directories.append(directory_name)
else:
self.cleanup_and_exit(
"Failed to make temporary directory for the repository: {0}".
format(url), 1)
try:
updated_repo_dir = self.repo_operator.clone_repo(
self.__repo, directory_name)
repo_commit_message = self.repo_operator.get_commit_message(
updated_repo_dir, self.__commit)
return repo_commit_message
except RuntimeError as error:
self.cleanup_and_exit(error, 1)
def upload_manifest_to_bintray(self, dir_name, bintray):
"""
Update manifest to bintray based on its contents and user arguments.
:param dir_name: The directory of the repository
:return: if repo is updated, return updated manifest file path and the manifest object
otherwise, return None, None
"""
def update_manifest_repo(self, dir_name, repo_commit_message):
"""
Update manifest repository based on its contents and user arguments.
:param dir_name: The directory of the repository
:return: if repo is updated, return updated manifest file path and the manifest object
otherwise, return None, None
"""
if self.__manifest_file is not None:
path_name = os.path.join(dir_name, self.__manifest_file)
if os.path.isfile(path_name):
try:
manifest = Manifest(path_name, self.__git_credentials)
manifest.update_manifest(self.__repo, self.__branch,
self.__commit)
if manifest.changed:
manifest.write_manifest_file(path_name, self.__dryrun)
return path_name, manifest
else:
print "No changes to {0}".format(manifest.name)
except KeyError as error:
self.cleanup_and_exit("Failed to create an Manifest instance for the manifest file {0}\nError:{1}"\
.format(self.__manifest_file, error.message),1)
except RuntimeError as error:
self.cleanup_and_exit(
"Failed to update manifest repo\nError:{0}".format(
error.message), 1)
else:
for item in os.listdir(dir_name):
path_name = os.path.join(dir_name, item)
if os.path.isfile(path_name):
try:
manifest = Manifest(path_name, self.__git_credentials)
manifest.update_manifest(self.__repo, self.__branch,
self.__commit)
if manifest.changed:
manifest.write_manifest_file(
path_name, self.__dryrun)
return path_name, manifest
else:
print "No changes to {0}".format(manifest.name)
except KeyError as error:
self.cleanup_and_exit("Failed to create an Manifest instance for the manifest file {0}\nError:{1}"\
.format(path_name, error.message),1)
except RuntimeError as error:
self.cleanup_and_exit(
"Failed to update manifest repo\nError:{0}".format(
error.message), 1)
return None, None
def write_downstream_parameters(self, filename, params):
"""
Add/append downstream parameter (java variable value pair) to the given
parameter file. If the file does not exist, then create the file.
:param filename: The parameter file that will be used for making environment
variable for downstream job.
:param params: the parameters dictionary
:return:
None on success
Raise any error if there is any
"""
if filename is None:
return
with open(filename, 'w') as fp:
try:
for key in params:
entry = "{key}={value}\n".format(key=key,
value=params[key])
fp.write(entry)
except IOError as error:
print "Unable to write parameter(s) for next step(s), exit"
self.cleanup_and_exit(error, 1)
def downstream_manifest_to_use(self, manifest_folder, file_with_path,
manifest, validate_result):
"""
Write file which contains the name of the manifest file most recently updated.
:param manifest_folder: the path of the manifest repository
:param file_with_path: The path to be split to claim the filename
:param manifest: the Manifest object of manifest file
"""
file_name = file_with_path.split('/')[-1]
downstream_parameters = {}
# If not validate_result, do not trigger downstream job
downstream_parameters['MANIFEST_FILE_VALID'] = validate_result
downstream_parameters[
'BUILD_REQUIREMENTS'] = manifest.build_requirements
downstream_parameters['MANIFEST_FILE_PATH'] = file_with_path
downstream_parameters['MANIFEST_FILE_URL'] = "".join(
[self.__manifest_download_url, file_name])
self.write_downstream_parameters(self.__updated_manifest,
downstream_parameters)
return
class ChangelogUpdater(object):
def __init__(self, repo_dir, version):
"""
The module updates debian/changelog under the directory of repository
_repo_dir: the directory of the repository
_version: the new version which is going to be updated to changelog
"""
self._repo_dir = repo_dir
self._version = version
self.repo_operator = RepoOperator()
def debian_exist(self):
"""
check whether debian directory under the repository
return: True if debian exist
False
"""
if os.path.isdir(self._repo_dir):
for filename in os.listdir(self._repo_dir):
if filename == "debian":
return True
return False
def get_repo_name(self):
"""
get the name of the repository
:return: the name of the repository
"""
repo_url = self.repo_operator.get_repo_url(self._repo_dir)
repo_name = common.strip_suffix(os.path.basename(repo_url), ".git")
return repo_name
def link_debianstatic(self):
"""
Handle repository which contains debianstatic/repository_name folder.
Create a soft link of debianstatic/repository_name to debian
for example: ln -s debianstatic/on-http debian
"""
repo_name = self.get_repo_name()
linked = False
for dir_name in os.listdir(self._repo_dir):
if dir_name == "debianstatic":
debianstatic_dir = os.path.join(self._repo_dir, "debianstatic")
for debianstatic_dir_name in os.listdir(debianstatic_dir):
if debianstatic_dir_name == repo_name:
debianstatic_repo_dir = "debianstatic/{0}".format(repo_name)
common.link_dir(debianstatic_repo_dir, "debian", self._repo_dir)
linked = True
return linked
def get_current_version(self):
"""
return the current version
"""
cmd_args = ["dpkg-parsechangelog", "--show-field", "Version"]
version = common.run_command(cmd_args, directory=self._repo_dir)
return version
def update_changelog(self, message=None):
"""
add an entry to changelog
:param message: the message which is going to be added to changelog
return: Ture if changelog is updated
False, otherwise
"""
repo_name = self.get_repo_name()
debian_exist = self.debian_exist()
linked = False
if not debian_exist:
linked = self.link_debianstatic()
if not debian_exist and not linked:
return
# -v: Add a new changelog entry with version number specified
# -b: Force a version to be less than the current one
# -m: Don't change (maintain) the trailer line in the changelog entry; i.e.
# maintain the maintainer and date/time details
try:
version = self.get_current_version()
# TBD: check whether the commit in manifest is the commit to jump version
if version == self._version:
print "[WARNING] The version of {0} is already {1}, skip the version bump action in debian/changelog for {0}"\
.format(repo_name, self._version)
return
print "start to update changelog of {0}".format(self._repo_dir)
cmd_args = ["dch", "-v", self._version, "-b", "-m"]
if message is None:
message = "new release {0}".format(self._version)
cmd_args += ["-p", message]
common.run_command(cmd_args, directory=self._repo_dir)
if linked:
os.remove(os.path.join(self._repo_dir, "debian"))
return
except Exception, err:
raise RuntimeError("Failed to add an entry for {0} in debian/changelog due to {1}".format(self._version, err))
