def reset(self):
if (self.__config.cleanup_silently or prompt_yesno_question(
'Should the repository and file system to be reset automatically?\nThis will reset the entire repository inlcuding latest commits to comply to remote.\nThis will also delete untrackted files!')):
# arbitrary 20, but extensive enough to reset all hopefully
log_info("Executing reset (git reset --hard HEAD~20)")
self.__repo.git.reset('--hard', 'HEAD~20')
self.update_and_clean()
def create_issue(self, title: str, milestone: Milestone, body: str) -> int:
'''Function creates an issue in git hub with title,milestone,body,labels passed'''
if self.__config.dry_run:
log_info_dry('Skipping creation of issue with title ' + str(title))
return 0
if self.__config.debug and not prompt_yesno_question(
'[DEBUG] Would now create GitHub issue with title="' + title +
'", milestone=' + str(milestone) + '. Continue?'):
sys.exit()
log_info('Create GitHub issue with title "' + title + '"...')
try:
issue: Issue = self.__repo.create_issue(
title=title,
body=body,
milestone=milestone,
labels=[self.__config.issue_label_name, "CI/CD"],
assignee=self.__github.get_user().login)
self.__config.github_issue_no = issue.number
self.__cache.issues.update({issue.number: issue})
return self.__config.github_issue_no
except GithubException as e:
print(str(e))
return 0
def __set_path(config: Config, attr: str):
msg = {
'root_path': "path of the repository to work on",
}
while True:
if not hasattr(config, attr) or not getattr(config, attr):
path = prompt_enter_value(msg[attr])
else:
path = getattr(config, attr) # set by script param
if not __check_path(path):
setattr(config, attr, "")
continue
if (attr == "root_path") & (os.path.realpath(__file__).startswith(
os.path.abspath(path))):
log_error(
"Please copy and run the create release script in another place outside of the repository and execute again."
)
sys.exit()
try:
Git(path)
except InvalidGitRepositoryError:
log_error("Path is not a git repository.")
setattr(config, attr, path)
info = {
"root_path": "Executing release in path '",
}
log_info(info[attr] + str(getattr(config, attr)) + "'")
break
def __store_config(config: Config):
def config2dic(config, attrs):
data = dict()
for attr in attrs:
if hasattr(config, attr):
data[attr] = getattr(config, attr)
return data
def store_as_yaml(data, path, prefix=""):
with open(path, 'w') as outfile:
yaml.dump(data, outfile, default_flow_style=False)
log_info("{}Git configuration has been stored in {}!".format(
prefix, path))
config_dir = "config"
if not os.path.isdir(config_dir):
os.makedirs(config_dir)
log_info("Folder config has been created!")
attrs = ["oss", "gpg_loaded", "gpg_keyname"]
non_git_config_dic = config2dic(config, attrs)
non_git_config_path = os.path.join(config_dir, "non_git_config.yml")
store_as_yaml(non_git_config_dic, non_git_config_path, prefix="Non ")
attrs = [
"git_username", "two_factor_authentication", "branch_to_be_released",
"release_version", "next_version", "github_issue_no"
]
git_config_dic = config2dic(config, attrs)
git_config_path = os.path.join(config_dir, "git_config.yml")
store_as_yaml(git_config_dic, git_config_path)
def update_and_clean(self):
log_info("Executing update and cleanup (git pull origin && git submodule update && git clean -fd)")
self.pull()
self.__repo.git.submodule("update")
self.__repo.git.clean("-fd")
if not self.is_working_copy_clean():
log_error("Reset and cleanup did not work out. Other branches have local commits not yet pushed:")
log_info("\n" + self.__list_unpushed_commits())
sys.exit()
def assure_clean_working_copy(self) -> None:
if not self.is_working_copy_clean(True):
log_error("Working copy is not clean!")
if self.__config.cleanup_silently or prompt_yesno_question("Should I clean the repo for you? This will delete all untracked files and hardly reset the repository!"):
self.reset()
else:
log_info("Please cleanup your working copy first. Then run the script again.")
sys.exit()
else:
log_info("Working copy clean.")
def is_valid_branch(config: Config) -> bool:
'''This Method is responsible for checking branches in repository with branch entered by user'''
if git.cmd.Git(config.root_path).execute("git ls-remote --heads origin "+config.branch_to_be_released+" | wc -l") == "":
log_info("Branch is not known remotely.")
is_branch_valid = False
else:
log_info("Branch is valid.")
is_branch_valid = True
return is_branch_valid
def run_maven_process_and_handle_error(command: str, execpath: str=config.build_folder_abs):
log_info("Execute command '" + command + "'")
returncode = maven.run_maven_process(execpath, command)
if returncode == 1:
log_error("Maven execution failed, please see create_release.py.log for logs located at current directory.")
if prompt_yesno_question("Maven execution failed. Script is not able to recover from it by its own.\nCan you fix the problem right now? If so, would you like to retry the last maven execution and resume the script?"):
run_maven_process_and_handle_error(command, execpath)
else:
git_repo.reset()
sys.exit()
def update_and_clean(self):
log_info("Executing update and cleanup (git pull origin && git clean -fd)")
self.pull()
self.__repo.git.clean("-fd")
if not self.is_working_copy_clean():
log_error("Reset and cleanup did not work out. Other branches have local commits not yet pushed:")
log_info("\n" + self.__list_unpushed_commits())
if not prompt_yesno_question(
"Something went wrong during cleanup. Please check if you can perform the cleanup on your own. Resume the script?"):
self.reset()
sys.exit()
def update_documentation(self) -> None:
log_info("Changing the " + self.__config.wiki_version_overview_page + " file, updating the version number...")
version_decl = self.__config.cobigenwiki_title_name
new_version_decl = version_decl + " v" + self.__config.release_version
modified_file = os.path.join(self.__config.root_path, "documentation", self.__config.wiki_version_overview_page)
with FileInput(modified_file,
inplace=True) as file:
for line in file:
line = re.sub(r'' + version_decl + r'\s+v[0-9]\.[0-9]\.[0-9]', new_version_decl, line)
sys.stdout.write(line)
self.add_modified_files()
def run_maven_process(self, execpath: str, command: str) -> int:
maven_process = subprocess.Popen(command.split(), shell=True, stdout=PIPE, stderr=PIPE,
universal_newlines=True, bufsize=1, cwd=execpath, env=os.environ)
for line in maven_process.stdout:
log_info(line.strip())
maven_process.stdout.close()
for line in maven_process.stderr:
log_error(line.strip())
maven_process.stderr.close()
return_code = maven_process.wait()
return return_code
def pull(self, branch_name: str = None):
if not branch_name:
branch = self.__repo.active_branch.name
else:
branch = branch_name
try:
log_info('Pull changes from origin ...')
self.__repo.git.execute("git pull origin {}".format(branch).split(" "))
except GitCommandError:
log_error("Pull from origin/" + branch + " on " + self.__repo.working_tree_dir +
" is not possible as you might have uncommitted or untracked files. Fix the working tree, and then try again.")
if not prompt_yesno_question("Did you fix the issue manually? Resume script?"):
self.reset()
sys.exit()
def update_submodule(self, submodule_path: str) -> None:
sm_repo = GitRepo(self.__config, submodule_path)
sm_repo.checkout('master')
sm_repo.pull()
log_info("Changing the "+self.__config.wiki_version_overview_page + " file, updating the version number...")
version_decl = self.__config.cobigenwiki_title_name
new_version_decl = version_decl+" v"+self.__config.release_version
with FileInput(os.path.join(self.__config.wiki_submodule_path, self.__config.wiki_version_overview_page), inplace=True) as file:
for line in file:
line = re.sub(r''+version_decl+r'\s+v[0-9]\.[0-9]\.[0-9]', new_version_decl, line)
sys.stdout.write(line)
sm_repo.add([self.__config.wiki_version_overview_page], False)
sm_repo.commit("update wiki docs")
sm_repo.push()
def __set_path(config: Config, attr: str):
while True:
if not hasattr(config, attr) or not getattr(config, attr):
path = config.temp_root_path
if not hasattr(config, "two_factor_authentication"):
config.two_factor_authentication = prompt_yesno_question(
"Are you using two-factor authentication on GitHub?")
if config.two_factor_authentication:
repository_url = "[email protected]:" + config.github_repo + ".git"
else:
repository_url = "https://github.com/" + config.github_repo + ".git"
log_info("Cloning temporary repository from " +
repository_url + " to " + str(path) +
" for processing the release...")
Repo.clone_from(repository_url,
path,
multi_options=["--config core.longpaths=true"])
else:
path = getattr(config, attr) # set by script param
if not __check_path(path):
setattr(config, attr, "")
continue
if (attr == "root_path") & (os.path.realpath(__file__).startswith(
os.path.abspath(path))):
log_error(
"Please copy and run the create release script in another place outside of the repository and execute again."
)
sys.exit()
try:
Git(path)
except InvalidGitRepositoryError:
log_error("Path " + path + " is not a git repository.")
setattr(config, attr, path)
info = {
"root_path": "Executing release in path '",
}
log_info(info[attr] + str(getattr(config, attr)) + "'")
break
def set_version(self, version: str) -> List[str]:
log_info("Setting version to " + version + " with maven/tycho")
changed_files = list()
# For dev_eclipseplugin branch
if self.__config.branch_to_be_released == self.__config.branch_eclipseplugin:
self.__run_maven_and_handle_error(os.path.join(self.__config.build_folder_abs, "cobigen-eclipse"),
"mvn -Dtycho.mode=maven -U org.eclipse.tycho:tycho-versions-plugin:set-version -DnewVersion="+version)
changed_files.append(os.path.join(self.__config.build_folder_abs, "cobigen-eclipse", "pom.xml"))
changed_files.append(os.path.join(self.__config.build_folder_abs, "cobigen-eclipse", "META-INF", "MANIFEST.MF"))
self.__run_maven_and_handle_error(os.path.join(self.__config.build_folder_abs, "cobigen-eclipse-test"),
"mvn -Dtycho.mode=maven -U org.eclipse.tycho:tycho-versions-plugin:set-version -DnewVersion="+version)
changed_files.append(os.path.join(self.__config.build_folder_abs, "cobigen-eclipse-test", "pom.xml"))
changed_files.append(os.path.join(self.__config.build_folder_abs, "cobigen-eclipse-test", "META-INF", "MANIFEST.MF"))
self.__run_maven_and_handle_error(os.path.join(self.__config.build_folder_abs, "cobigen-eclipse-feature"),
"mvn -Dtycho.mode=maven -U org.eclipse.tycho:tycho-versions-plugin:set-version -DnewVersion="+version)
changed_files.append(os.path.join(self.__config.build_folder_abs, "cobigen-eclipse-feature", "pom.xml"))
changed_files.append(os.path.join(self.__config.build_folder_abs, "cobigen-eclipse-feature", "feature.xml"))
self.__run_maven_and_handle_error(os.path.join(self.__config.build_folder_abs, "cobigen-eclipse-updatesite"),
"mvn -Dtycho.mode=maven -U org.eclipse.tycho:tycho-versions-plugin:set-version -DnewVersion="+version)
changed_files.append(os.path.join(self.__config.build_folder_abs, "cobigen-eclipse-updatesite", "pom.xml"))
else:
toplevel_pom_path = os.path.join(self.__config.build_folder_abs, "pom.xml")
# For dev_mavenplugin branch
if self.__config.branch_to_be_released == self.__config.branch_mavenplugin:
pom = etree.parse(toplevel_pom_path)
for mapping in pom.findall("//mvn:properties", self.mavenNS):
log_info("Processing " + toplevel_pom_path + " ...")
version_node = mapping.find("mvn:cobigen.maven.version", self.mavenNS)
if self.__check_and_write_pom(pom, version_node, version, toplevel_pom_path):
changed_files.append(toplevel_pom_path)
# For dev_core branch
elif self.__config.branch_to_be_released == self.__config.branch_core:
pom = etree.parse(toplevel_pom_path)
for mapping in pom.findall("//mvn:properties", self.mavenNS):
log_info("Processing " + toplevel_pom_path + " ...")
version_node = mapping.find("mvn:cobigencore.version", self.mavenNS)
if self.__check_and_write_pom(pom, version_node, version, toplevel_pom_path):
changed_files.append(toplevel_pom_path)
# others
else:
for dirpath, dnames, fnames in os.walk(self.__config.build_folder_abs):
self.__ignore_folders_on_pom_search(dnames)
if "pom.xml" in fnames:
fpath = os.path.join(dirpath, "pom.xml")
log_info("Processing " + fpath + " ...")
with open(fpath) as file:
pom = etree.parse(file)
version_node = pom.find("mvn:version", self.mavenNS)
if self.__check_and_write_pom(pom, version_node, version, fpath):
changed_files.append(fpath)
return changed_files
def __print_cmd_help():
log_info("""
This script automates the deployment of CobiGen modules.
[WARNING]: The script will access and change the Github repository. Do not use it unless you want to deploy modules.
Options:
-c / --cleanup-silently [CAUTION] Will silently reset/clean your working copy automatically.
This will also delete non-tracked files from your local file system!
You will not be asked anymore!
-d / --debug: Script stops after each automatic step and asks the user to continue.
-g / --github-repo-id GitHub repository name to be released
-h / --help: Provides a short help about the intention and possible options.
-k / --gpg-key GPG key for code signing (for OSS release only)
-r / --local-repo Local repository clone to work on for the release
-t / --test: Script runs on a different repo for testing purpose. It also uses predefined
names and variables to shorten up the process.
-y / --dry-run: Will prevent from pushing to the remote repository, changing anything on GitHub
Issues/Milestones etc.
""")
def init_git_dependent_config(config: Config, github: GitHub, git_repo: GitRepo):
while(True):
config.branch_to_be_released = prompt_enter_value("the name of the branch to be released")
if(is_valid_branch(config)):
break
config.release_version = ""
version_pattern = re.compile(r'[0-9]\.[0-9]\.[0-9]')
while(not version_pattern.match(config.release_version)):
config.release_version = prompt_enter_value("release version number without 'v' in front")
config.next_version = ""
while(not version_pattern.match(config.next_version)):
config.next_version = prompt_enter_value("next version number (after releasing) without 'v' in front")
config.build_folder = __get_build_folder(config)
config.build_folder_abs = os.path.join(config.root_path, config.build_folder)
config.build_artifacts_root_search_path = __get_build_artifacts_root_search_path(config)
config.cobigenwiki_title_name = __get_cobigenwiki_title_name(config)
config.tag_name = __get_tag_name(config) + config.release_version
config.tag_core_name = __get_tag_name_specific_branch(config, config.branch_core)
if git_repo.exists_tag(config.tag_name):
log_error("Git tag " + config.tag_name + " already exists. Maybe you entered the wrong release version? Please fix the problem and try again.")
sys.exit()
config.issue_label_name = config.tag_name[:-7]
config.expected_milestone_name = config.tag_name[:-7] + "-v" + config.release_version
config.expected_core_milestone_name = config.tag_core_name[:-2] + "-v"
milestone = github.find_release_milestone()
if milestone:
log_info("Milestone '"+milestone.title+"' found!")
else:
log_error("Milestone not found! Searched for milestone with name '" + config.expected_milestone_name+"'. Aborting...")
sys.exit()
while(True):
github_issue_no: str = prompt_enter_value(
"release issue number without # prefix in case you already created one or type 'new' to create an issue automatically")
if github_issue_no == 'new':
issue_text = "This issue has been automatically created. It serves as a container for all release related commits"
config.github_issue_no = github.create_issue("Release " + config.expected_milestone_name, body=issue_text, milestone=milestone)
if not config.github_issue_no:
log_error("Could not create issue! Aborting...")
sys.exit()
else:
log_info('Successfully created issue #' + str(github_issue_no))
break
else:
try:
if github.find_issue(int(github_issue_no)):
config.github_issue_no = int(github_issue_no)
log_info("Issue #" + str(config.github_issue_no) + " found remotely!")
break
else:
log_error("Issue with number #" + str(config.github_issue_no) + " not found! Typo?")
except ValueError:
log_error("Please enter a number.")
def push(self, force: bool = False):
''' Boolean return type states, whether to continue process or abort'''
if (not force and not self.has_unpushed_commits()):
log_info("Nothing to be pushed.")
return
if (self.__config.test_run or self.__config.debug) and not prompt_yesno_question(
"[DEBUG] Changes will be pushed now. Continue?"):
self.reset()
sys.exit()
if self.__config.dry_run:
log_info_dry('Skipping pushing of changes.')
return
try:
log_info(
"Pushing to origin/" + self.__repo.active_branch.name + " in " + self.__repo.working_tree_dir + " ...")
self.__repo.git.execute("git push origin " + self.__repo.active_branch.name + " --tags")
except Exception as e:
if "no changes added to commit" in str(e):
log_info("No file is changed, nothing to commit.")
else:
if not prompt_yesno_question(
"Something went wrong during pushing. Please check if you can perform pushing on your own. Resume the script?"):
self.reset()
def __authenticate_git_user(self):
while True:
if not hasattr(self.__config, "two_factor_authentication"):
self.__config.two_factor_authentication = prompt_yesno_question(
"Are you using two-factor authentication on GitHub?")
if self.__config.two_factor_authentication:
self.__config.git_token = getpass.getpass(
"> Please enter your token: ")
while not self.__config.git_token:
self.__config.git_token = getpass.getpass(
"> Please enter your token: ")
else:
if not hasattr(self.__config, "git_username"):
self.__config.git_username = prompt_enter_value(
"your git user name")
else:
log_info("The stored Github username is {}".format(
self.__config.git_username))
self.__config.git_password = getpass.getpass(
"> Please enter your password: "******"> Please enter your password: "******"Authenticated.")
break
except BadCredentialsException:
log_info("Authentication error, please try again.")
continue
def merge(self, source: str, target: str) -> None:
if self.__config.dry_run:
log_info_dry("Would merge from " + source + " to " + target)
return
try:
self.checkout(target)
log_info("Executing git pull...")
self.pull()
log_info("Merging...")
self.__repo.git.execute("git merge " + self.__config.branch_to_be_released)
log_info("Adapting automatically generated merge commit message to include issue no.")
automatic_commit_message = self.__repo.git.execute("git log -1 --pretty=%B")
if "Merge" in automatic_commit_message and str(
self.__config.github_issue_no) not in automatic_commit_message:
self.__repo.git.execute('git commit --amend -m"#' + str(
self.__config.github_issue_no) + ' ' + automatic_commit_message + '"')
except Exception as ex:
log_error("Something went wrong, please check if merge conflicts exist and solve them.")
if self.__config.debug:
print(ex)
if not prompt_yesno_question(
"If there were conflicts you solved and committed, would you like to resume the script?"):
self.__repo.git.execute("git merge --abort")
self.reset()
sys.exit()
def commit(self, commit_message: str):
try:
if self.__list_uncommitted_files() != "":
log_info("Committing ...")
self.__repo.index.commit("#" + str(self.__config.github_issue_no) + " " + commit_message)
else:
log_info("Nothing to commit.")
except Exception as e:
if "no changes added to commit" in str(e):
log_info("No File is changed, Nothing to commit..")
def store_as_yaml(data, path, prefix=""):
with open(path, 'w') as outfile:
yaml.dump(data, outfile, default_flow_style=False)
log_info("{}Git configuration has been stored in {}!".format(
prefix, path))
