def main():
# parse arguments
args = parse_command_line(sys.argv[1:])
generator = ManifestGenerator(args.source_manifest,
args.dest_manifest,
args.branch,
args.builddir,
args.git_credentials,
jobs=args.jobs,
force=args.force)
try:
generator.update_manifest()
if args.force:
generator.set_force(args.force)
if args.publish:
if args.git_credentials and args.publish_branch:
repo_operator = RepoOperator(args.git_credentials)
commit_message = "add a manifest file for new branch {0}".format(
args.branch)
repo_dir = os.path.dirname(args.dest_manifest)
repo_operator.checkout_repo_branch(repo_dir,
args.publish_branch)
generator.generate_manifest()
repo_operator.push_repo_changes(repo_dir,
commit_message,
push_all=True)
else:
print "Please specify the git-credential and publish-branch if you want to push the new manifest"
sys.exit(1)
else:
generator.generate_manifest()
except Exception, e:
traceback.print_exc()
print "Failed to generate new manifest for {0} due to \n{1}\nExiting now".format(
args.branch, e)
sys.exit(1)
class ManifestActions(object):
"""
valid actions:
checkout: check out a set of repositories to match the manifest file
"""
valid_actions = ['checkout', 'branch', 'packagerefs', 'tag']
def __init__(self,
manifest_path,
builddir,
force=False,
git_credentials=None,
jobs=1,
actions=[],
branch_name=None,
tag_name=None):
"""
__force - Overwrite a directory if it exists
__git_credential - url, credentials pair for the access to github repos
__manifest - Repository manifest contents
__builddir - Destination for checked out repositories
__jobs - Number of parallel jobs to run
__actions -Supported actions
:return:
"""
self._force = force
self._git_credentials = git_credentials
self._builddir = builddir
self._manifest = None
self.handle_manifest(manifest_path)
self._jobs = jobs
self.actions = []
for action in actions:
self.add_action(action)
self._branch_name = branch_name
self._tag_name = tag_name
self.repo_operator = RepoOperator(self._git_credentials)
def set_force(self, force):
"""
Standard setter for force
:param force: if true, overwrite a directory if it exists
:return: None
"""
self._force = force
def get_force(self):
"""
Standard getter for force
:return: force
"""
return self._force
def set_branch_name(self, branch):
"""
Standard setter for branch_name
:param force: if true, overwrite a directory if it exists
:return: None
"""
self._branch_name = branch
def get_branch_name(self):
"""
Standard getter for branch_name
:return: force
"""
return self._branch_name
def set_git_credentials(self, git_credential):
"""
Standard setter for git_credentials
:param git_credential: url, credentials pair for the access to github repos
:return: None
"""
self._git_credentials = git_credential
self.repo_operator.setup_gitbit(credentials=self._git_credentials)
def get_manifest(self):
"""
Standard getter for manifest
:return: an instance of Manifest
"""
return self._manifest
def add_action(self, action):
"""
Add action to actions
:param action: a string, just like: checkout
:return: None
"""
if action not in self.valid_actions:
print "Unknown action '{0}' requested".format(action)
print "Valid actions are:"
for op in self.valid_actions:
print " {0}".format(op)
sys.exit(1)
else:
if action in ['branch', 'tag'] and self._git_credentials == None:
print "Must Specify git_credentials when try to write repository"
sys.exit(1)
self.actions.append(action)
def set_jobs(self, jobs):
"""
Standard setter for jobs
:param jobs: number of parallel jobs to run
:return: None
"""
self._jobs = jobs
if self._jobs < 1:
print "--jobs value must be an integer >=1"
sys.exit(1)
def handle_manifest(self, manifest_path):
"""
initial manifest and validate it
:param manifest_path: the path of manifest file
:return: None
"""
try:
self._manifest = Manifest(manifest_path)
self._manifest.validate_manifest()
except KeyError as error:
print "Failed to create a Manifest instance for the manifest file {0} \nERROR:\n{1}"\
.format(manifest_path, error.message)
sys.exit(1)
for repo in self._manifest.repositories:
repo['directory-name'] = self.directory_for_repo(repo)
def check_builddir(self):
"""
Checks the given builddir name and force flag.
Deletes exists directory if one already exists and --force is set
:return: None
"""
if os.path.exists(self._builddir):
if self._force:
shutil.rmtree(self._builddir)
print "Removing existing data at {0}".format(self._builddir)
else:
print "Unwilling to overwrite destination builddir of {0}".format(
self._builddir)
sys.exit(1)
os.makedirs(self._builddir)
def get_repositories(self):
"""
Issues checkout commands to dictionaries within a provided manifest
:return: None
"""
repo_list = self._manifest.repositories
try:
self.repo_operator.clone_repo_list(repo_list,
self._builddir,
jobs=self._jobs)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
def directory_for_repo(self, repo):
"""
Get the directory of a repository
:param repo: a dictionary
:return: the directary of repository
"""
if 'checked-out-directory-name' in repo:
repo_directory = repo['checked-out-directory-name']
else:
if 'repository' in repo:
repo_url = repo['repository']
repo_directory = strip_suffix(os.path.basename(repo_url),
".git")
else:
raise ValueError("no way to find basename")
repo_directory = os.path.join(self._builddir, repo_directory)
return repo_directory
def execute_actions(self):
"""
start to execute actions
:return: None
"""
# Start to check out a set of repositories within a manifest file
if 'checkout' in self.actions:
self.check_builddir()
self.get_repositories()
# Start to create branch and update package.json
if 'branch' in self.actions:
self.execute_branch_action()
if 'tag' in self.actions:
self.create_tag()
# Start to update the packge.json, for example:
# - git+https://github.com/RackHD/on-core.git
# +
if 'packagerefs' in self.actions:
self.update_package_references()
def execute_branch_action(self):
"""
execute the action "branch"
:return: None
"""
if self._branch_name is None:
raise ValueError("No setting for branch-name")
else:
print "create branch and update package.json for the repos..."
self.branch_existing_repositories()
self.checkout_branch_repositories(self._branch_name)
self.update_package_references(version=self._branch_name)
commit_message = "update the dependencies version to {0}".format(
self._branch_name)
self.push_changed_repositories(commit_message)
def branch_existing_repositories(self):
"""
Issues create branch commands to repos in a provided manifest
:return: None
"""
if self._branch_name is None:
print "Please provide the new branch name"
sys.exit(2)
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and create specified branch on each
for repo in repo_list:
self.create_repo_branch(repo, self._branch_name)
def create_repo_branch(self, repo, branch):
"""
create branch on the repos in the manifest file
:param repo: A dictionary
:return: None
"""
try:
repo_directory = self.directory_for_repo(repo)
repo_url = repo["repository"]
self.repo_operator.create_repo_branch(repo_url, repo_directory,
branch)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
def checkout_branch_repositories(self, branch):
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and checkout specified branch on each
for repo in repo_list:
self.checkout_repo_branch(repo, branch)
def checkout_repo_branch(self, repo, branch):
"""
checkout to a specify branch on repository
:param repo: A dictionary
:param branch: the specify branch name
:return: None
"""
try:
repo_directory = self.directory_for_repo(repo)
self.repo_operator.checkout_repo_branch(repo_directory, branch)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
def update_package_references(self, version=None):
print "Update internal package lists"
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and update package.json on each
for repo in repo_list:
self.update_repo_package_list(repo, pkg_version=version)
def update_repo_package_list(self, repo, pkg_version=None):
"""
Update the package.json of repository to point to new version
:param repo: a manifest repository entry
:param pkg_version: the version of package.json to point to
:return:
"""
repo_dir = repo['directory-name']
package_json_file = os.path.join(repo_dir, "package.json")
if not os.path.exists(package_json_file):
# if there's no package.json file, there is nothing more for us to do here
return
changes = False
log = ""
with open(package_json_file, "r") as fp:
package_data = json.load(fp)
if 'dependencies' in package_data:
for package, version in package_data['dependencies'].items():
new_version = self._update_dependency(
version, pkg_version=pkg_version)
if new_version != version:
log += " {0}:\n WAS {1}\n NOW {2}\n".format(
package, version, new_version)
package_data['dependencies'][package] = new_version
changes = True
if changes:
print "There are changes to dependencies for {0}\n{1}".format(
package_json_file, log)
os.remove(package_json_file)
new_file = package_json_file
with open(new_file, "w") as newfile:
json.dump(package_data, newfile, indent=4, sort_keys=True)
else:
print "There are NO changes to data for {0}".format(
package_json_file)
def _update_dependency(self, version, pkg_version=None):
"""
Check the specified package & version, and return a new package version if
the package is listed in the manifest.
:param version:
:return:
"""
if not version.startswith("git+"):
return version
url = strip_prefix(version, "git+")
url = url.split('#')[0]
new_url = url
if pkg_version is None:
for repo in self._manifest.repositories:
if new_url == repo['repository']:
if 'directory-name' in repo:
new_url = os.path.abspath(repo['directory-name'])
return new_url
else:
for repo in self._manifest.repositories:
if new_url == repo['repository']:
new_url = "git+{url}#{pkg_version}".format(
url=new_url, pkg_version=pkg_version)
return new_url
return version
def create_tag(self):
if self._tag_name is None:
raise ValueError("No setting for tag-name")
else:
print "create tag for the repos..."
self.create_tag_for_repositories()
def create_tag_for_repositories(self):
"""
Issues set_tagname commands to repos in a provided manifest
:return: None
"""
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and create specified tag on each
for repo in repo_list:
self.set_repo_tagname(repo)
def set_repo_tagname(self, repo):
"""
Add a tag to a repository
:param repo: A dictionary
:return: None
"""
try:
repo_url = repo["repository"]
repo_directory = self.directory_for_repo(repo)
self.repo_operator.set_repo_tagname(repo_url, repo_directory,
self._tag_name)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
def push_changed_repositories(self, commit_message):
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and publish changes on each
for repo in repo_list:
self.push_changed_repo(repo, commit_message)
def push_changed_repo(self, repo, commit_message):
"""
publish changes in the repository
:param repo: A dictionary
:param commit_message: the message to be added to the commit
:return: None
"""
repo_dir = repo['directory-name']
try:
self.repo_operator.push_repo_changes(repo_dir, commit_message)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
class ManifestActions(object):
"""
valid actions:
checkout: check out a set of repositories to match the manifest file
"""
valid_actions = ['checkout', 'branch', 'packagerefs', 'packagerefs-commit', 'tag']
def __init__(self, manifest_path, builddir, force=False, git_credentials=None, jobs=1, actions=[], branch_name=None, tag_name=None):
"""
__force - Overwrite a directory if it exists
__git_credential - url, credentials pair for the access to github repos
__manifest - Repository manifest contents
__builddir - Destination for checked out repositories
__jobs - Number of parallel jobs to run
__actions -Supported actions
:return:
"""
self._force = force
self._git_credentials = git_credentials
self._builddir = builddir
self._manifest = None
self.handle_manifest(manifest_path)
self._jobs = jobs
self.actions = []
for action in actions:
self.add_action(action)
self._branch_name = branch_name
self._tag_name = tag_name
self.repo_operator = RepoOperator(self._git_credentials)
def set_force(self, force):
"""
Standard setter for force
:param force: if true, overwrite a directory if it exists
:return: None
"""
self._force = force
def get_force(self):
"""
Standard getter for force
:return: force
"""
return self._force
def set_branch_name(self, branch):
"""
Standard setter for branch_name
:param force: if true, overwrite a directory if it exists
:return: None
"""
self._branch_name = branch
def get_branch_name(self):
"""
Standard getter for branch_name
:return: force
"""
return self._branch_name
def set_git_credentials(self, git_credential):
"""
Standard setter for git_credentials
:param git_credential: url, credentials pair for the access to github repos
:return: None
"""
self._git_credentials = git_credential
self.repo_operator.setup_gitbit(credentials=self._git_credentials)
def get_manifest(self):
"""
Standard getter for manifest
:return: an instance of Manifest
"""
return self._manifest
def add_action(self, action):
"""
Add action to actions
:param action: a string, just like: checkout
:return: None
"""
if action not in self.valid_actions:
print "Unknown action '{0}' requested".format(action)
print "Valid actions are:"
for op in self.valid_actions:
print " {0}".format(op)
sys.exit(1)
else:
if action in ['branch', 'tag'] and self._git_credentials == None:
print "Must Specify git_credentials when try to write repository"
sys.exit(1)
self.actions.append(action)
def set_jobs(self, jobs):
"""
Standard setter for jobs
:param jobs: number of parallel jobs to run
:return: None
"""
self._jobs = jobs
if self._jobs < 1:
print "--jobs value must be an integer >=1"
sys.exit(1)
def handle_manifest(self, manifest_path):
"""
initial manifest and validate it
:param manifest_path: the path of manifest file
:return: None
"""
try:
self._manifest = Manifest(manifest_path)
self._manifest.validate_manifest()
except KeyError as error:
print "Failed to create a Manifest instance for the manifest file {0} \nERROR:\n{1}"\
.format(manifest_path, error.message)
sys.exit(1)
for repo in self._manifest.repositories:
repo['directory-name'] = self.directory_for_repo(repo)
def check_builddir(self):
"""
Checks the given builddir name and force flag.
Deletes exists directory if one already exists and --force is set
:return: None
"""
if os.path.exists(self._builddir):
if self._force:
shutil.rmtree(self._builddir)
print "Removing existing data at {0}".format(self._builddir)
else:
print "Unwilling to overwrite destination builddir of {0}".format(self._builddir)
sys.exit(1)
os.makedirs(self._builddir)
def get_repositories(self):
"""
Issues checkout commands to dictionaries within a provided manifest
:return: None
"""
repo_list = self._manifest.repositories
try:
self.repo_operator.clone_repo_list(repo_list, self._builddir, jobs=self._jobs)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
def directory_for_repo(self, repo):
"""
Get the directory of a repository
:param repo: a dictionary
:return: the directary of repository
"""
if 'checked-out-directory-name' in repo:
repo_directory = repo['checked-out-directory-name']
else:
if 'repository' in repo:
repo_url = repo['repository']
repo_directory = strip_suffix(os.path.basename(repo_url), ".git")
else:
raise ValueError("no way to find basename")
repo_directory = os.path.join(self._builddir, repo_directory)
return repo_directory
def execute_actions(self):
"""
start to execute actions
:return: None
"""
# Start to check out a set of repositories within a manifest file
if 'checkout' in self.actions:
self.check_builddir()
self.get_repositories()
# Start to create branch and update package.json
if 'branch' in self.actions:
self.execute_branch_action()
if 'tag' in self.actions:
self.create_tag()
# Start to update the packge.json, for example:
# - git+https://github.com/RackHD/on-core.git
# +
if 'packagerefs' in self.actions:
self.update_package_references()
# Update package dependency with repo commit-id
if 'packagerefs-commit' in self.actions:
self.update_package_references(ref_type="commit")
def execute_branch_action(self):
"""
execute the action "branch"
:return: None
"""
if self._branch_name is None:
raise ValueError("No setting for branch-name")
else:
print "create branch and update package.json for the repos..."
self.branch_existing_repositories()
self.checkout_branch_repositories(self._branch_name)
self.update_package_references(version=self._branch_name)
commit_message = "update the dependencies version to {0}".format(self._branch_name)
self.push_changed_repositories(commit_message)
def branch_existing_repositories(self):
"""
Issues create branch commands to repos in a provided manifest
:return: None
"""
if self._branch_name is None:
print "Please provide the new branch name"
sys.exit(2)
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and create specified branch on each
for repo in repo_list:
self.create_repo_branch(repo, self._branch_name)
def create_repo_branch(self, repo, branch):
"""
create branch on the repos in the manifest file
:param repo: A dictionary
:return: None
"""
try:
repo_directory = self.directory_for_repo(repo)
repo_url = repo["repository"]
self.repo_operator.create_repo_branch(repo_url, repo_directory, branch)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
def checkout_branch_repositories(self, branch):
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and checkout specified branch on each
for repo in repo_list:
self.checkout_repo_branch(repo, branch)
def checkout_repo_branch(self, repo, branch):
"""
checkout to a specify branch on repository
:param repo: A dictionary
:param branch: the specify branch name
:return: None
"""
try:
repo_directory = self.directory_for_repo(repo)
self.repo_operator.checkout_repo_branch(repo_directory, branch)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
def update_package_references(self, version=None, ref_type="path"):
print "Update internal package lists"
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and update package.json on each
for repo in repo_list:
self.update_repo_package_list(repo, pkg_version=version, ref_type=ref_type)
def update_repo_package_list(self, repo, pkg_version=None, ref_type="path"):
"""
Update the package.json of repository to point to new version
:param repo: a manifest repository entry
:param pkg_version: the version of package.json to point to
:return:
"""
repo_dir = repo['directory-name']
package_json_file = os.path.join(repo_dir, "package.json")
if not os.path.exists(package_json_file):
# if there's no package.json file, there is nothing more for us to do here
return
changes = False
log = ""
with open(package_json_file, "r") as fp:
package_data = json.load(fp)
if 'dependencies' in package_data:
for package, version in package_data['dependencies'].items():
new_version = self._update_dependency(version, pkg_version=pkg_version, ref_type=ref_type)
if new_version != version:
log += " {0}:\n WAS {1}\n NOW {2}\n".format(package,
version,
new_version)
package_data['dependencies'][package] = new_version
changes = True
if changes:
print "There are changes to dependencies for {0}\n{1}".format(package_json_file, log)
os.remove(package_json_file)
new_file = package_json_file
with open(new_file, "w") as newfile:
json.dump(package_data, newfile, indent=4, sort_keys=True)
else:
print "There are NO changes to data for {0}".format(package_json_file)
def _update_dependency(self, version, pkg_version=None, ref_type="path"):
"""
Check the specified package & version, and return a new package version if
the package is listed in the manifest.
:param version:
:return:
"""
if not version.startswith("git+"):
return version
url = strip_prefix(version, "git+")
url = url.split('#')[0]
new_url = url
if pkg_version is None:
for repo in self._manifest.repositories:
if new_url == repo['repository']:
if ref_type == "path":
if 'directory-name' in repo:
new_url = os.path.abspath(repo['directory-name'])
return new_url
elif ref_type == "commit":
new_url = "git+{url}#{pkg_version}".format(url=new_url, pkg_version=repo['commit-id'])
return new_url
else:
for repo in self._manifest.repositories:
if new_url == repo['repository']:
new_url = "git+{url}#{pkg_version}".format(url=new_url, pkg_version=pkg_version)
return new_url
return version
def create_tag(self):
if self._tag_name is None:
raise ValueError("No setting for tag-name")
else:
print "create tag for the repos..."
self.create_tag_for_repositories()
def create_tag_for_repositories(self):
"""
Issues set_tagname commands to repos in a provided manifest
:return: None
"""
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and create specified tag on each
for repo in repo_list:
self.set_repo_tagname(repo)
def set_repo_tagname(self, repo):
"""
Add a tag to a repository
:param repo: A dictionary
:return: None
"""
try:
repo_url = repo["repository"]
repo_directory = self.directory_for_repo(repo)
self.repo_operator.set_repo_tagname(repo_url, repo_directory, self._tag_name)
except RuntimeError as error:
raise RuntimeError("Failed to create tag {0} for {1} \ndue to error: {2}".format(self._tag_name, repo_url, error))
def push_changed_repositories(self, commit_message):
repo_list = self._manifest.repositories
if repo_list is None:
print "No repository list found in manifest file"
sys.exit(2)
else:
# Loop through list of repos and publish changes on each
for repo in repo_list:
self.push_changed_repo(repo, commit_message)
def push_changed_repo(self, repo, commit_message):
"""
publish changes in the repository
:param repo: A dictionary
:param commit_message: the message to be added to the commit
:return: None
"""
repo_dir = repo['directory-name']
try:
self.repo_operator.push_repo_changes(repo_dir, commit_message)
except RuntimeError as error:
print "Exiting due to error: {0}".format(error)
sys.exit(1)
