class ExistDirManifestGenerator(ManifestGenerator):
def __init__(self, dest, builddir, git_credential=None, force=False, jobs=1):
self._jenkins_author = config.gitbit_identity["username"]
self._dest_manifest_file = dest
self._builddir = builddir
self._force = force
self._jobs = jobs
self._manifest = Manifest.instance_of_sample()
self.repo_operator = RepoOperator(git_credential)
self.check_builddir()
def check_builddir(self):
"""
Checks the given builddir name and force flag.
Deletes exists directory if one already exists and --force is set
:return: None
"""
if not os.path.exists(self._builddir):
print "The {0} doesn't exist".format(self._builddir)
sys.exit(1)
def update_manifest(self):
"""
update the manifest with branch name
:return: None
"""
repositories = self._manifest.repositories
downstream_jobs = self._manifest.downstream_jobs
self.update_repositories_commit(repositories)
self.update_repositories_branch(repositories)
self.update_repositories_commit(downstream_jobs)
self.update_repositories_branch(downstream_jobs)
self._manifest.validate_manifest()
def update_repositories_branch(self, repositories):
"""
update the commit-id of repository with the latest commit id
:param repositories: a list of repository directory
:return: None
"""
for repo in repositories:
repo_dir = self.directory_for_repo(repo)
repo["branch"] = self.repo_operator.get_current_branch(repo_dir)
class VersionGenerator(object):
def __init__(self, repo_dir):
"""
This module compute the version of a repository
The version for candidate release: {big-version}~{version-stage}-{small-version}
The big version is parsed from debian/changelog
The version-stage is devel if branch is master; or rc if branch if not master
The samll version is consist of the commit hash and commit date of manifest repository
:return:None
"""
self._repo_dir = repo_dir
self.repo_operator = RepoOperator()
self._repo_name = self.get_repo_name()
def get_repo_name(self):
repo_url = self.repo_operator.get_repo_url(self._repo_dir)
repo_name = common.strip_suffix(os.path.basename(repo_url), ".git")
return repo_name
def generate_small_version(self):
"""
Generate the small version which consists of commit date and commit hash of manifest repository
According to small version, users can track the commit of all repositories in manifest file
return: small version
"""
if self._repo_name == "RackHD":
utc_now = datetime.utcnow()
utc_yesterday = utc_now + timedelta(days=-1)
version = utc_yesterday.strftime('%Y%m%dUTC')
return version
else:
commit_timestamp_str = self.repo_operator.get_lastest_commit_date(
self._repo_dir)
date = datetime.utcfromtimestamp(
int(commit_timestamp_str)).strftime('%Y%m%dUTC')
commit_id = self.repo_operator.get_lastest_commit_id(
self._repo_dir)
version = "{date}-{commit}".format(date=date,
commit=commit_id[0:7])
return version
def debian_exist(self):
"""
check whether debian or debianstatic directory under the repository
return: True if debian or debianstatic exist
False
"""
if os.path.isdir(self._repo_dir):
for filename in os.listdir(self._repo_dir):
if filename == "debian":
return True
return False
def generate_big_version(self):
"""
Generate the big version according to changelog
The big version is the latest version of debian/changelog
return: big version
"""
# If the repository has the debianstatic/repository name/,
# create a soft link to debian before compute version
debian_exist = self.debian_exist()
linked = False
if not debian_exist:
for filename in os.listdir(self._repo_dir):
if filename == "debianstatic":
debianstatic_dir = os.path.join(self._repo_dir,
"debianstatic")
for debianstatic_filename in os.listdir(debianstatic_dir):
if debianstatic_filename == self._repo_name:
debianstatic_repo_dir = "debianstatic/{0}".format(
self._repo_name)
common.link_dir(debianstatic_repo_dir, "debian",
self._repo_dir)
linked = True
if not debian_exist and not linked:
return None
cmd_args = ["dpkg-parsechangelog", "--show-field", "Version"]
version = common.run_command(cmd_args, directory=self._repo_dir)
if linked:
os.remove(os.path.join(self._repo_dir, "debian"))
return version
def generate_version_stage(self):
"""
Generate the version stage according to the stage of deveplopment
return: devel ,if the branch is master
rc, if the branch is not master
"""
current_branch = self.repo_operator.get_current_branch(self._repo_dir)
version_stage = ""
if "master" in current_branch:
version_stage = "devel"
else:
version_stage = "rc"
return version_stage
def generate_package_version(self, is_official_release):
"""
generate the version of package, just like:
1.1-1-devel-20160809150908-7396d91 or 1.1-1
:return: package version
"""
big_version = self.generate_big_version()
if big_version is None:
common.logging.warning(
"Failed to generate big version, maybe the {0} doesn't contain debian directory"
.format(self._repo_dir))
return None
if is_official_release:
version = big_version
else:
small_version = self.generate_small_version()
if small_version is None:
raise RuntimeError(
"Failed to generate version for {0}, due to the small version is None"
.format(self._repo_dir))
version = "{0}-{1}".format(big_version, small_version)
return version
class VersionGenerator(object):
def __init__(self, repo_dir):
"""
This module compute the version of a repository
The version for candidate release: {big-version}~{version-stage}-{small-version}
The big version is parsed from debian/changelog
The version-stage is devel if branch is master; or rc if branch if not master
The samll version is consist of the commit hash and commit date of manifest repository
:return:None
"""
self._repo_dir = repo_dir
self.repo_operator = RepoOperator()
self._repo_name = self.get_repo_name()
def get_repo_name(self):
repo_url = self.repo_operator.get_repo_url(self._repo_dir)
repo_name = common.strip_suffix(os.path.basename(repo_url), ".git")
return repo_name
def generate_small_version(self):
"""
Generate the small version which consists of commit date and commit hash of manifest repository
According to small version, users can track the commit of all repositories in manifest file
return: small version
"""
if self._repo_name == "RackHD":
utc_now = datetime.utcnow()
utc_yesterday = utc_now + timedelta(days=-1)
version = utc_yesterday.strftime('%Y%m%dUTC')
return version
else:
commit_timestamp_str = self.repo_operator.get_lastest_commit_date(self._repo_dir)
date = datetime.utcfromtimestamp(int(commit_timestamp_str)).strftime('%Y%m%dUTC')
commit_id = self.repo_operator.get_lastest_commit_id(self._repo_dir)
version = "{date}-{commit}".format(date=date, commit=commit_id[0:7])
return version
def debian_exist(self):
"""
check whether debian or debianstatic directory under the repository
return: True if debian or debianstatic exist
False
"""
if os.path.isdir(self._repo_dir):
for filename in os.listdir(self._repo_dir):
if filename == "debian":
return True
return False
def generate_big_version(self):
"""
Generate the big version according to changelog
The big version is the latest version of debian/changelog
return: big version
"""
# If the repository has the debianstatic/repository name/,
# create a soft link to debian before compute version
debian_exist = self.debian_exist()
linked = False
if not debian_exist:
for filename in os.listdir(self._repo_dir):
if filename == "debianstatic":
debianstatic_dir = os.path.join(self._repo_dir, "debianstatic")
for debianstatic_filename in os.listdir(debianstatic_dir):
if debianstatic_filename == self._repo_name:
debianstatic_repo_dir = "debianstatic/{0}".format(self._repo_name)
common.link_dir(debianstatic_repo_dir, "debian", self._repo_dir)
linked = True
if not debian_exist and not linked:
return None
cmd_args = ["dpkg-parsechangelog", "--show-field", "Version"]
version = common.run_command(cmd_args, directory=self._repo_dir)
if linked:
os.remove(os.path.join(self._repo_dir, "debian"))
return version
def generate_version_stage(self):
"""
Generate the version stage according to the stage of deveplopment
return: devel ,if the branch is master
rc, if the branch is not master
"""
current_branch = self.repo_operator.get_current_branch(self._repo_dir)
version_stage = ""
if "master" in current_branch:
version_stage = "devel"
else:
version_stage = "rc"
return version_stage
def generate_package_version(self, is_official_release):
"""
generate the version of package, just like:
1.1-1-devel-20160809150908-7396d91 or 1.1-1
:return: package version
"""
big_version = self.generate_big_version()
if big_version is None:
common.logging.warning("Failed to generate big version, maybe the {0} doesn't contain debian directory".format(self._repo_dir))
return None
if is_official_release:
version = big_version
else:
small_version = self.generate_small_version()
if small_version is None:
raise RuntimeError("Failed to generate version for {0}, due to the small version is None".format(self._repo_dir))
version = "{0}-{1}".format(big_version, small_version)
return version
class UpdateManifest(object):
def __init__(self):
"""
__repo - the repository url being updated in the manifest
__branch - the branch name associated with __repo
__sliced_branch - the branch name sliced at any forward slashes
__commit - The commit id associated with the __repo and __branch that we want to update
__manifest_repository_url - the repository url the points to the collection of manifest files
__manifest_file - the desired manifest file to update which resides in __manifest_repository_url
__cleanup_directories - the path/name of directories created are appended here for cleanup in task_cleanup
__git_credentials - url, credentials pair for the access to github repos
quiet - used for testing to minimize text written to a terminal
repo_operator - Class instance of RepoOperator
:return: None
"""
self.__repo = None
self.__branch = None
self.__sliced_branch = None
self.__commit = None
self.__manifest_repository_url = None
self.__manifest_file = None
self.__cleanup_directories = []
self.__git_credentials = None
self.__updated_manifest = None
self.__dryrun = False
self.quiet = False
self.repo_operator = RepoOperator()
#pylint: disable=no-self-use
def parse_args(self, args):
"""
Take in values from the user.
Repo, branch, and manifest_repo are required. This exits if they are not given.
:return: Parsed args for assignment
"""
parser = argparse.ArgumentParser()
parser.add_argument(
'-n',
'--dryrun',
help="Do not commit any changes, just print what would be done",
action="store_true")
parser.add_argument("--repo",
help="Git url to match for updating the commit-id",
action="store")
parser.add_argument("--branch",
help="The target branch for the named repo",
action="store")
parser.add_argument("--manifest_repo",
help="The manifest repository URL.",
action="store")
parser.add_argument(
"--commit",
help="OPTIONAL: The commit id to target an exact version",
action="store")
parser.add_argument(
"--manifest_file",
help=
"The target manifest file to update. Searches through all if left empty.",
action="store")
parser.add_argument("--git-credential",
help="Git URL and credentials comma separated",
action="append")
parser.add_argument(
'--updated_manifest',
help="file containing the name of the updated manifest",
action="store")
parser = parser.parse_args(args)
return parser
def assign_args(self, args):
"""
Assign args to member variables and perform checks on branch and commit-id.
:param args: Parsed args from the user
:return:
"""
if args.repo:
self.__repo = args.repo
else:
print "\nMust specify repository url for cloning (--repo <git_url>)\n"
if args.branch:
self.__branch = args.branch
if "/" in self.__branch:
self.__sliced_branch = self.__branch.split("/")[-1]
else:
self.__sliced_branch = self.__branch
else:
print "\nMust specify a branch name (--branch <branch_name>)\n"
if args.manifest_repo:
self.__manifest_repository_url = args.manifest_repo
else:
print "\n Must specify a full repository url for retrieving <manifest>.json files\n"
if args.dryrun:
self.__dryrun = True
self.repo_operator.setup_git_dryrun(self.__dryrun)
if args.commit:
self.__commit = args.commit
if args.manifest_file:
self.__manifest_file = args.manifest_file
if args.git_credential:
self.__git_credentials = args.git_credential
self.repo_operator.setup_gitbit(credentials=self.__git_credentials)
if args.updated_manifest:
self.__updated_manifest = args.updated_manifest
def check_args(self):
"""
Check the values given for branch and commit-id
"""
if self.__branch:
try:
self.repo_operator.check_branch(self.__repo,
self.__sliced_branch)
except RuntimeError as error:
self.cleanup_and_exit(error, 1)
if self.__commit:
self.__check_commit()
def __check_commit(self):
"""
Check the format of the commit-id. It must be 40 hex characters.
Exits if it is not.
:return: None
"""
commit_match = re.match('^[0-9a-fA-F]{40}$', self.__commit)
if not commit_match:
self.cleanup_and_exit(
"Id, '{0}' is not valid. It must be a 40 character hex string."
.format(self.__commit), 1)
def clone_manifest_repo(self):
"""
Clone the repository which holds the manifest json files. Return that directory name. The directory
is temporary and deleted in the cleanup_and_exit function
:return: A string containing the name of the folder where the manifest repo was cloned
"""
directory_name = tempfile.mkdtemp()
if os.path.isdir(directory_name):
self.__cleanup_directories.append(directory_name)
else:
self.cleanup_and_exit(
"Failed to make temporary directory for the repository: {0}".
format(url), 1)
try:
manifest_repo = self.repo_operator.clone_repo(
self.__manifest_repository_url, directory_name)
return manifest_repo
except RuntimeError as error:
self.cleanup_and_exit(error, 1)
def validate_manifest_files(self, *args):
"""
validate several manifest files
For example: validate_manifest_files(file1, file2) or
validate_manifest_files(file1, file2, file3)
"""
validate_result = True
for filename in args:
try:
manifest = Manifest(filename)
manifest.validate_manifest()
print "manifest file {0} is valid".format(filename)
except KeyError as error:
print "Failed to validate manifest file {0}".format(filename)
print "\nERROR: \n{0}".format(error.message)
validate_result = False
return validate_result
def cleanup_and_exit(self, message=None, code=0):
"""
Delete all files and folders made during this job which are named in self.cleanup_directories
:return: None
"""
if not self.quiet:
if message is not None:
print "\nERROR: {0}".format(message)
print "\nCleaning environment!\n"
for item in self.__cleanup_directories:
subprocess.check_output(["rm", "-rf", item])
sys.exit(code)
def get_updated_commit_message(self):
"""
get the updated repository commit message
"""
# get commit message based on the arguments repo, branch and commit
directory_name = tempfile.mkdtemp()
if os.path.isdir(directory_name):
self.__cleanup_directories.append(directory_name)
else:
self.cleanup_and_exit(
"Failed to make temporary directory for the repository: {0}".
format(url), 1)
try:
updated_repo_dir = self.repo_operator.clone_repo(
self.__repo, directory_name)
repo_commit_message = self.repo_operator.get_commit_message(
updated_repo_dir, self.__commit)
return repo_commit_message
except RuntimeError as error:
self.cleanup_and_exit(error, 1)
def update_manifest_repo(self, dir_name, repo_commit_message):
"""
Update manifest repository based on its contents and user arguments.
:param dir_name: The directory of the repository
:return: if repo is updated, return updated manifest file path and the manifest object
otherwise, return None, None
"""
url_short_name = self.__repo.split('/')[-1][:-4]
commit_message = "update to {0} */{1} commit #{2}\n{3}"\
.format(url_short_name, self.__branch, self.__commit, repo_commit_message)
if self.__manifest_file is not None:
path_name = os.path.join(dir_name, self.__manifest_file)
if os.path.isfile(path_name):
try:
manifest = Manifest(path_name, self.__git_credentials)
manifest.update_manifest(self.__repo, self.__branch,
self.__commit)
if manifest.changed:
manifest.write_manifest_file(dir_name, commit_message,
path_name, self.__dryrun)
return path_name, manifest
else:
print "No changes to {0}".format(manifest.name)
except KeyError as error:
self.cleanup_and_exit("Failed to create an Manifest instance for the manifest file {0}\nError:{1}"\
.format(self.__manifest_file, error.message),1)
except RuntimeError as error:
self.cleanup_and_exit(
"Failed to update manifest repo\nError:{0}".format(
error.message), 1)
else:
for item in os.listdir(dir_name):
path_name = os.path.join(dir_name, item)
if os.path.isfile(path_name):
try:
manifest = Manifest(path_name, self.__git_credentials)
manifest.update_manifest(self.__repo, self.__branch,
self.__commit)
if manifest.changed:
manifest.write_manifest_file(
dir_name, commit_message, path_name,
self.__dryrun)
return path_name, manifest
else:
print "No changes to {0}".format(manifest.name)
except KeyError as error:
self.cleanup_and_exit("Failed to create an Manifest instance for the manifest file {0}\nError:{1}"\
.format(path_name, error.message),1)
except RuntimeError as error:
self.cleanup_and_exit(
"Failed to update manifest repo\nError:{0}".format(
error.message), 1)
return None, None
def write_downstream_parameters(self, filename, params):
"""
Add/append downstream parameter (java variable value pair) to the given
parameter file. If the file does not exist, then create the file.
:param filename: The parameter file that will be used for making environment
variable for downstream job.
:param params: the parameters dictionary
:return:
None on success
Raise any error if there is any
"""
if filename is None:
return
with open(filename, 'w') as fp:
try:
for key in params:
entry = "{key}={value}\n".format(key=key,
value=params[key])
fp.write(entry)
except IOError as error:
print "Unable to write parameter(s) for next step(s), exit"
self.cleanup_and_exit(error, 1)
def downstream_manifest_to_use(self, manifest_folder, file_with_path,
manifest):
"""
Write file which contains the name of the manifest file most recently updated.
:param manifest_folder: the path of the manifest repository
:param file_with_path: The path to be split to claim the filename
:param manifest: the Manifest object of manifest file
"""
file_name = file_with_path.split('/')[-1]
downstream_parameters = {}
downstream_parameters['MANIFEST_FILE_NAME'] = file_name
downstream_parameters[
'BUILD_REQUIREMENTS'] = manifest.build_requirements
downstream_parameters[
'MANIFEST_FILE_REPO'] = self.__manifest_repository_url
try:
downstream_parameters[
'MANIFEST_FILE_BRANCH'] = self.repo_operator.get_current_branch(
manifest_folder)
downstream_parameters[
'MANIFEST_FILE_COMMIT'] = self.repo_operator.get_lastest_commit_id(
manifest_folder)
except RuntimeError as error:
self.cleanup_and_exit(error, 1)
self.write_downstream_parameters(self.__updated_manifest,
downstream_parameters)
return
def create_inject_properties_file(self):
"""
Create inject.properties file for env vars injection after excute shell.
If upstream_manifest_name exists then copy it to inject.properties, else
create an empty inject.properties file to avoid exceptions throw by plugin.
"""
if os.path.isfile(self.__updated_manifest):
try:
shutil.copyfile(self.__updated_manifest, "inject.properties")
except IOError as error:
print "ERROR: copy file {0} to inject.properties error.\n{1}".format(
self.__updated_manifest, error)
sys.exit(1)
else:
try:
open('inject.properties', 'a').close()
except IOError as error:
print "ERROR: create empty file inject.properties error.\n{0}".format(
error)
sys.exit(1)
return
