def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('--workdir', help='the temporary working directory', metavar='DIR', default="tags_work", required=False) args = parser.parse_args() workdir = os.path.realpath(args.workdir) utils.remove_if_exists(workdir) os.makedirs(workdir) # fetch all branches sapmachine_branches = utils.get_active_sapmachine_branches() # fetch all tags tags = utils.get_github_tags() # iterate all tags to find the latest jdk tag for a JDK major release jdk_tags = {} for tag in tags: # filter for jdk tags jdk_tag = JDKTag.from_string(tag['name']) if jdk_tag is None: continue # only remember the latest jdk tag (version comparison) # filter out jdk update tags with build number "0" like jdk-11.0.3+0 if ((jdk_tag.get_major() not in jdk_tags or jdk_tag.is_greater_than(jdk_tags[jdk_tag.get_major()])) and not (jdk_tag.get_update() > 0 and (jdk_tag.get_build_number() if jdk_tag.get_build_number() is not None else 0) == 0)): jdk_tags[jdk_tag.get_major()] = jdk_tag # clone the SapMachine repository git_target_dir = join(workdir, 'sapmachine') utils.git_clone('github.com/SAP/SapMachine.git', 'sapmachine', git_target_dir) # for each "sapmachine" branch, check whether it contains the latest jdk tag with matching major version for sapmachine_branch in sapmachine_branches: # get the latest jdk tag for this major version latest_tag = jdk_tags[sapmachine_branch[1]] print(str.format('latest tag for branch "{0}" is "{1}"', sapmachine_branch, latest_tag.as_string())) # check whether the jdk tag is already contained in the sapmachine branch _, out, _ = utils.run_cmd(str.format('git branch -a --contains tags/{0}', latest_tag.as_string()).split(' '), cwd=git_target_dir, std=True, throw=False) containing_branches_pattern = re.compile('{0}|pr-jdk-{1}.*'.format(sapmachine_branch[0], sapmachine_branch[1])) match = re.search(containing_branches_pattern, out) if match is None: # the jdk tag is not contained in a sapmachine branch # create a pull request branch and a pull request. print(str.format('creating pull request "pr-{0}" with base branch "{1}"', latest_tag.as_string(), sapmachine_branch)) utils.run_cmd(str.format('git checkout {0}', latest_tag.as_string()).split(' '), cwd=git_target_dir) utils.run_cmd(str.format('git checkout -b pr-{0}', latest_tag.as_string()).split(' '), cwd=git_target_dir) utils.run_cmd(str.format('git push origin pr-{0}', latest_tag.as_string()).split(' '), cwd=git_target_dir) pull_request = str.format('{{ "title": "Merge to tag {0}", "body": "please pull", "head": "pr-{1}", "base": "{2}" }}', latest_tag.as_string(), latest_tag.as_string(), sapmachine_branch[0]) utils.github_api_request('pulls', data=pull_request, method='POST') utils.remove_if_exists(workdir) return 0
def main(): parser = argparse.ArgumentParser() parser.add_argument('--msvs_version', required=True) parser.add_argument('--chrome_path') parser.add_argument('--skia_path') args = parser.parse_args() isolate_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'win_toolchain.isolate') with utils.print_timings(): with utils.tmp_dir() as tmp_dir: chrome_path = args.chrome_path if not chrome_path: print( 'Syncing Chrome from scratch. If you already have a checkout, ' 'specify --chrome_path to save time.') chrome_path = os.path.join(tmp_dir.name, 'src') if not os.path.isdir(chrome_path): utils.git_clone(REPO_CHROME, chrome_path) skia_path = args.skia_path if not skia_path: print( 'Syncing Skia from scratch. If you already have a checkout, ' 'specify --chrome_path to save time.') skia_path = os.path.join(tmp_dir.name, 'skia') if not os.path.isdir(skia_path): utils.git_clone(REPO_SKIA, skia_path) isolated_hash = gen_toolchain(chrome_path, args.msvs_version, isolate_file) update_toolchain_file(skia_path, args.msvs_version, isolated_hash)
def initial_deploy(): """ Performs a complete deploy of the project. """ # put ssh key set_deploy_key() # github host handshake run('ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts') # bitbucket host handshake run('ssh-keyscan -t rsa bitbucket.org >> ~/.ssh/known_hosts') # install necessary dependencies to handle the project install_project_handling_dependencies() # clone repository git_clone(env.server_git_url, env.server_root_dir) # checkout branch with cd(env.server_root_dir): git_checkout(env.branch) # dependencies installation (quickstart) with cd(env.server_root_dir): run('./quickstart.sh') # gunicorn installation and configuration gunicorn.install() gunicorn.add_gunicorn_service() gunicorn.start() # nginx installation and configuration nginx.install() nginx.add_django_site() nginx.start()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--msvs_version', required=True) parser.add_argument('--chrome_path') parser.add_argument('--skia_path') args = parser.parse_args() isolate_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'win_toolchain.isolate') with utils.print_timings(): with utils.tmp_dir() as tmp_dir: chrome_path = args.chrome_path if not chrome_path: print ('Syncing Chrome from scratch. If you already have a checkout, ' 'specify --chrome_path to save time.') chrome_path = os.path.join(tmp_dir.name, 'src') if not os.path.isdir(chrome_path): utils.git_clone(REPO_CHROME, chrome_path) skia_path = args.skia_path if not skia_path: print ('Syncing Skia from scratch. If you already have a checkout, ' 'specify --chrome_path to save time.') skia_path = os.path.join(tmp_dir.name, 'skia') if not os.path.isdir(skia_path): utils.git_clone(REPO_SKIA, skia_path) isolated_hash = gen_toolchain(chrome_path, args.msvs_version, isolate_file) update_toolchain_file(skia_path, args.msvs_version, isolated_hash)
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('--workdir', help='specify the working directory', metavar='DIR', default='docker_work', required=False) args = parser.parse_args() workdir = os.path.realpath(args.workdir) git_dir = join(workdir, 'sapmachine-infrastructure') utils.remove_if_exists(workdir) os.makedirs(workdir) releases = github_api_request('releases', per_page=100) infrastructure_tags = github_api_request( 'tags', repository='SapMachine-infrastructure', per_page=100) lts_release = None lts_release_major = 0 stable_release = None for release in releases: if release['prerelease']: continue version, version_part, major, build_number, sap_build_number, os_ext = utils.sapmachine_tag_components( release['name']) if utils.sapmachine_is_lts(major) and not lts_release: lts_release = release lts_release_major = major else: if not stable_release and major > lts_release_major: stable_release = release if lts_release and stable_release: break if lts_release or stable_release: utils.git_clone('github.com/SAP/SapMachine-infrastructure', 'master', git_dir) if lts_release and not stable_release: stable_release = lts_release if lts_release: print(str.format('found latest LTS release "{0}"', lts_release['name'])) process_release(lts_release, 'lts', infrastructure_tags, git_dir) if stable_release: print( str.format('found latest stable release "{0}"', stable_release['name'])) process_release(stable_release, 'stable', infrastructure_tags, git_dir) utils.remove_if_exists(workdir) return 0
def _on_install(self, _): self.unit.status = MaintenanceStatus("Installing apt packages") install_apt(packages=APT_REQUIREMENTS, update=True) self.unit.status = MaintenanceStatus("Preparing the environment") self._reset_environment() self.unit.status = MaintenanceStatus("Downloading srsLTE from Github") git_clone(GIT_REPO, output_folder=SRC_PATH, branch=GIT_REPO_TAG, depth=1) self.unit.status = MaintenanceStatus("Building srsLTE") shell(SRS_ENB_UE_BUILD_COMMAND) self.unit.status = MaintenanceStatus("Generating configuration files") copy_files(origin=CONFIG_ORIGIN_PATHS, destination=CONFIG_PATHS) self.unit.status = MaintenanceStatus("Generating systemd files") self._configure_srsenb_service() self._configure_srsue_service() service_enable(SRS_ENB_SERVICE) self._stored.installed = True
def main(argv=None): token = utils.get_github_api_accesstoken() asset_pattern = re.compile(utils.sapmachine_asset_pattern()) asset_map_jre = {} asset_map_jdk = {} releases = utils.get_github_releases() for release in releases: if release['prerelease'] is True: continue t = SapMachineTag.from_string(release['name']) if t is None: continue assets = release['assets'] for asset in assets: match = asset_pattern.match(asset['name']) if match is None: continue asset_image_type = match.group(1) asset_os = match.group(3) if asset_os == 'linux-x64': sapmachine_version = t.get_version() build_number = t.get_build_number() buildpack_version = str.format( '{0}.{1}.{2}_{3}.{4}.b{5}', sapmachine_version[0], sapmachine_version[1], sapmachine_version[2], sapmachine_version[3], sapmachine_version[4], build_number if build_number else '0') if asset_image_type == 'jre': asset_map_jre[buildpack_version] = asset[ 'browser_download_url'] else: asset_map_jdk[buildpack_version] = asset[ 'browser_download_url'] local_repo = join(os.getcwd(), 'gh-pages') utils.git_clone('github.com/SAP/SapMachine.git', 'gh-pages', local_repo) write_index_yaml( asset_map_jre, join(local_repo, 'assets', 'cf', 'jre', 'linux', 'x86_64')) write_index_yaml( asset_map_jdk, join(local_repo, 'assets', 'cf', 'jdk', 'linux', 'x86_64')) utils.git_commit(local_repo, 'Updated index.yml', ['assets']) utils.git_push(local_repo) utils.remove_if_exists(local_repo) return 0
def main(argv=None): token = utils.get_github_api_accesstoken() asset_pattern = re.compile(utils.sapmachine_asset_pattern()) asset_map = {} releases = utils.github_api_request('releases', per_page=100) for release in releases: if release['prerelease'] is True: continue version, version_part, major, build_number, sap_build_number, os_ext = utils.sapmachine_tag_components( release['name']) assets = release['assets'] if version is None or os_ext: continue for asset in assets: match = asset_pattern.match(asset['name']) if match is not None: asset_image_type = match.group(1) asset_os = match.group(3) if asset_os == 'linux-x64' and asset_image_type == 'jre': sapmachine_version = [ int(e) for e in version_part.split('.') ] sapmachine_version += [ 0 for sapmachine_version in range( 0, 5 - len(sapmachine_version)) ] if sap_build_number: sapmachine_version[4] = int(sap_build_number) buildpack_version = str.format( '{0}.{1}.{2}_{3}.{4}.b{5}', sapmachine_version[0], sapmachine_version[1], sapmachine_version[2], sapmachine_version[3], sapmachine_version[4], build_number if build_number else '0') asset_map[buildpack_version] = asset[ 'browser_download_url'] local_repo = join(os.getcwd(), 'gh-pages') utils.git_clone('github.com/SAP/SapMachine.git', 'gh-pages', local_repo) write_index_yaml( asset_map, join(local_repo, 'assets', 'cf', 'jre', 'linux', 'x86_64')) utils.git_commit(local_repo, 'Updated index.yml', ['assets']) utils.git_push(local_repo) utils.remove_if_exists(local_repo)
def push_to_git(files): local_repo = join(os.getcwd(), 'gh-pages') utils.git_clone('github.com/SAP/SapMachine.git', 'gh-pages', local_repo) for _file in files: location = join(local_repo, _file['location']) if not os.path.exists(os.path.dirname(location)): os.makedirs(os.path.dirname(location)) with open(location, 'w+') as out: out.write(_file['data']) utils.git_commit(local_repo, _file['commit_message'], [location]) utils.git_push(local_repo) utils.remove_if_exists(local_repo)
def main(argv=None): token = utils.get_github_api_accesstoken() github_api = 'https://api.github.com/repos/SAP/SapMachine/releases' asset_pattern = re.compile(utils.sapmachine_asset_pattern()) asset_map = {} request = Request(github_api) if token is not None: request.add_header('Authorization', str.format('token {0}', token)) response = json.loads(urlopen(request).read()) for release in response: if release['prerelease'] is True: continue version, version_part, major, build_number, sap_build_number, os_ext = utils.sapmachine_tag_components(release['name']) assets = release['assets'] if version is None or os_ext: continue for asset in assets: match = asset_pattern.match(asset['name']) if match is not None: asset_image_type = match.group(1) asset_os = match.group(3) if asset_os == 'linux-x64' and asset_image_type == 'jdk': sapmachine_version = [int(e) for e in version_part.split('.')] sapmachine_version += [0 for sapmachine_version in range(0, 5 - len(sapmachine_version))] if sap_build_number: sapmachine_version[4] = int(sap_build_number) buildpack_version = '.'.join([str(e) for e in sapmachine_version]) buildpack_version += str.format('_b{0}', build_number if build_number else '0') asset_map[buildpack_version] = asset['browser_download_url'] local_repo = join(os.getcwd(), 'gh-pages') utils.git_clone('github.com/SAP/SapMachine.git', 'gh-pages', local_repo) write_index_yaml(asset_map, join(local_repo, 'assets', 'cf', 'jre', 'linux', 'x86_64')) utils.git_commit(local_repo, 'Updated index.yml', ['assets']) utils.git_push(local_repo) utils.remove_if_exists(local_repo)
def get_flutter_skps(target_dir): """Creates SKPs using Flutter's skp_generator tool. Documentation is at https://github.com/flutter/tests/tree/master/skp_generator """ with utils.tmp_dir(): utils.git_clone('https://github.com/flutter/tests.git', '.') os.chdir('skp_generator') subprocess.check_call(['bash', 'build.sh']) # Fix invalid SKP file names. for f in os.listdir('skps'): original_file_name = os.path.splitext(f)[0] new_file_name = ''.join( [x if x.isalnum() else "_" for x in original_file_name]) if new_file_name != original_file_name: os.rename(os.path.join('skps', f), os.path.join('skps', new_file_name + '.skp')) copy_tree('skps', target_dir)
def initial_deploy(): """ Performs a complete deploy of the project. """ # put ssh key set_deploy_key() # github host handshake run('ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts') # install necessary dependencies to handle the project install_project_handling_dependencies() # clone repository git_clone(env.server_git_url, env.server_root_dir) # checkout branch with cd(env.server_root_dir): git_checkout(env.branch) # mysql installation install_mysql() # dependencies installation (quickstart) with cd(env.server_root_dir): run('./quickstart.sh') # bower installation bower.install() # gunicorn installation and configuration gunicorn.install() gunicorn.add_gunicorn_service() gunicorn.start() # nginx installation and configuration nginx.install() nginx.add_django_site() nginx.start()
def initial_deploy(): """ Performs a complete deploy of the project. """ # put ssh key ssh_key = '%s/fabfile/templates/ssh_key' ssh_key %= env.local_root_dir run('mkdir -p -m 0700 .ssh') put(ssh_key, '.ssh/id_rsa', mode=0600) # github host handshake run('ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts') # clone repository deb_handler.install('git') git_clone(env.server_git_url, env.server_root_dir) # checkout branch with cd(env.server_root_dir): git_checkout(env.branch) # mysql installation install_mysql() # dependencies installation (quickstart) with cd(env.server_root_dir): run('./quickstart.sh') # gunicorn installation and configuration gunicorn.install() gunicorn.add_gunicorn_service() gunicorn.start() # nginx installation and configuration nginx.install() nginx.add_django_site() nginx.start()
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('--workdir', help='the temporary working directory', metavar='DIR', default="tags_work", required=False) args = parser.parse_args() workdir = os.path.realpath(args.workdir) utils.remove_if_exists(workdir) os.makedirs(workdir) sapmachine_latest = 0 sapmachine_branches = [] # fetch all branches branches = utils.github_api_request('branches', per_page=100) # iterate all branches of the SapMachine repository for branch in branches: # filter for sapmachine branches match = branch_pattern.match(branch['name']) if match is not None: if match.group(1) is not None: # found sapmachine branch major = int(match.group(1)) print( str.format( 'found sapmachine branch "{0}" with major "{1}"', branch['name'], major)) sapmachine_branches.append([branch['name'], major]) sapmachine_latest = max(sapmachine_latest, major) else: print( str.format('found sapmachine branch "{0}"', branch['name'])) sapmachine_branches.append([branch['name'], 0]) sapmachine_latest += 1 # set the major version for "sapmachine" branch which always contains the latest changes from "jdk/jdk" for sapmachine_branch in sapmachine_branches: if sapmachine_branch[1] is 0: sapmachine_branch[1] = sapmachine_latest # fetch all tags jdk_tags = {} tags = utils.github_api_request('tags', per_page=300) # iterate all tags for tag in tags: # filter for jdk tags match = JDKTag.jdk_tag_pattern.match(tag['name']) if match is not None: # found a jdk tag jdk_tag = JDKTag(match) major = jdk_tag.get_major() # only store the latest jdk tag (version comparison) if major not in jdk_tags or jdk_tag.is_greater_than( jdk_tags[major]): # filter jdk update tags with build number "0" like jdk-11.0.3+0 if not (jdk_tag.is_update() and jdk_tag.get_build_number() == 0): jdk_tags[major] = jdk_tag # clone the SapMachine repository git_target_dir = join(workdir, 'sapmachine') utils.git_clone('github.com/SAP/SapMachine.git', 'sapmachine', git_target_dir) # for each "sapmachine" branch, check whether it contains the latest jdk tag with matching major version for sapmachine_branch in sapmachine_branches: # get the latest jdk tag for this major version latest_tag_for_branch = jdk_tags[sapmachine_branch[1]] print( str.format('latest tag for branch "{0}" is "{1}"', sapmachine_branch, latest_tag_for_branch.as_string())) # check whether the jdk tag is already contained in the sapmachine branch _, out, _ = utils.run_cmd(str.format( 'git branch -a --contains tags/{0}', latest_tag_for_branch.as_string()).split(' '), cwd=git_target_dir, std=True, throw=False) match = re.search(containing_branches_pattern, out) if match is None: # the jdk tag is not contained i a sapmachine branch # create a pull request branch and a pull request. print( str.format( 'creating pull request "pr-{0}" with base branch "{1}"', latest_tag_for_branch.as_string(), sapmachine_branch)) utils.run_cmd( str.format('git checkout {0}', latest_tag_for_branch.as_string()).split(' '), cwd=git_target_dir) utils.run_cmd( str.format('git checkout -b pr-{0}', latest_tag_for_branch.as_string()).split(' '), cwd=git_target_dir) utils.run_cmd( str.format('git push origin pr-{0}', latest_tag_for_branch.as_string()).split(' '), cwd=git_target_dir) pull_request = str.format( '{{ "title": "Merge to tag {0}", "body": "please pull", "head": "pr-{1}", "base": "{2}" }}', latest_tag_for_branch.as_string(), latest_tag_for_branch.as_string(), sapmachine_branch[0]) utils.github_api_request('pulls', data=pull_request, method='POST') utils.remove_if_exists(workdir) return 0
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('--workdir', help='the temporary working directory', metavar='DIR', default="tags_work", required=False) args = parser.parse_args() workdir = os.path.realpath(args.workdir) utils.remove_if_exists(workdir) os.makedirs(workdir) pull_requests = utils.github_api_request('pulls', per_page=100, url_parameter=['state=all']) # clone the SapMachine repository git_target_dir = join(workdir, 'sapmachine') utils.git_clone('github.com/SAP/SapMachine.git', 'sapmachine', git_target_dir) # fetch all branches branches = utils.github_api_request('branches', per_page=100) sapmachine_latest = 0 sapmachine_branches = [] # iterate all branches of the SapMachine repository for branch in branches: # filter for sapmachine branches match = branch_pattern.match(branch['name']) if match is not None: # found sapmachine branch if match.group(1) is not None: major = int(match.group(1)) sapmachine_branches.append([branch['name'], major]) sapmachine_latest = max(major, sapmachine_latest) else: sapmachine_branches.append([branch['name'], 0]) sapmachine_latest += 1 for branch in sapmachine_branches: if branch[1] == 0: branch[1] = sapmachine_latest major = branch[1] utils.run_cmd(str.format('git checkout {0}', branch[0]).split(' '), cwd=git_target_dir) # find the last merge commit and check wether it is a merge from the jdk branch _, commit_messages, _ = utils.run_cmd( 'git log --merges -n 50 --format=%s'.split(' '), cwd=git_target_dir, std=True, throw=False) _, commit_ids, _ = utils.run_cmd( 'git log --merges -n 50 --format=%H'.split(' '), cwd=git_target_dir, std=True, throw=False) if commit_messages and commit_ids: commit_messages = [ commit_message for commit_message in commit_messages.split(os.linesep) if commit_message ] commit_ids = [ commit_id for commit_id in commit_ids.split(os.linesep) if commit_id ] merge_commits = map(lambda x, y: [x, y], commit_messages, commit_ids) for merge_commit in merge_commits: commit_message = merge_commit[0] commit_id = merge_commit[1] match_merge_commit = re.search(merge_commit_pattern, commit_message) match_jdk_tag = re.search(JDKTag.jdk_tag_pattern, commit_message) if match_merge_commit is not None and match_jdk_tag is not None: jdk_tag = JDKTag(match_jdk_tag) print( str.format( 'found latest merge commit "{0}" for branch "{1}"', commit_message, branch)) _, tags, _ = utils.run_cmd(str.format('git tag --contains {0}', commit_id).split(' '), cwd=git_target_dir, std=True, throw=False) if not tags: # not tagged yet # create sapmachine tag create_sapmachine_tag(jdk_tag, commit_id, git_target_dir) run_jenkins_jobs(major, jdk_tag.as_sapmachine_tag()) elif not jdk_tag.is_ga(): tags = tags.splitlines() # check wether there is a JDK GA tag which has no corresponding sapmachine tag yet # get the commit to which the most recent (before GA) tag is pointing to _, jdk_tag_commit, _ = utils.run_cmd(str.format( 'git rev-list -n 1 {0}', jdk_tag.as_string()).split(' '), cwd=git_target_dir, std=True, throw=False) if jdk_tag_commit: jdk_tag_commit = jdk_tag_commit.rstrip() # get all tags associated with the commit _, tags_for_commit, _ = utils.run_cmd( str.format('git tag --contains {0}', jdk_tag_commit).split(' '), cwd=git_target_dir, std=True, throw=False) if tags_for_commit: tags_for_commit = tags_for_commit.splitlines() # search for a GA tag for tag in tags_for_commit: match = re.search(JDKTag.jdk_tag_pattern, tag) if match: as_jdk_tag = JDKTag(match) if as_jdk_tag.is_ga( ) and as_jdk_tag.as_sapmachine_tag( ) not in tags: # GA tag found # check whether there is already a pull request for this tag pull_request_title = str.format( 'Merge to tag {0}', as_jdk_tag.as_string()) pull_request_exits = False for pull_request in pull_requests: if pull_request[ 'title'] == pull_request_title: # there is already a pull request for this tag # don't create sapmachine tag pull_request_exits = True break if not pull_request_exits: # create sapmachine tag create_sapmachine_tag( as_jdk_tag, commit_id, git_target_dir) run_jenkins_jobs( major, as_jdk_tag.as_sapmachine_tag()) break else: print('already tagged ...') break utils.remove_if_exists(workdir) return 0
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('-t', '--tag', help='the SapMachine tag', metavar='TAG', required=True) parser.add_argument('--sha256sum', help='the sha256 sum', metavar='SHA256', required=True) parser.add_argument('-i', '--imagetype', help='The image type', metavar='IMAGETYPE', choices=['jdk', 'jre']) parser.add_argument('-p', '--prerelease', help='this is a pre-release', action='store_true', default=False) args = parser.parse_args() work_dir = join(os.getcwd(), 'cask_work') utils.remove_if_exists(work_dir) os.makedirs(work_dir) cask_version_pattern = re.compile('version \'((\d+\.?)+)(,(\d+))?\'') sapMachineTag = SapMachineTag.from_string(args.tag) if sapMachineTag is None: print(str.format("Tag {0} seems to be invalid. Aborting...", args.tag)) sys.exit() if args.prerelease: if sapMachineTag.get_build_number() is None: print('No build number given for pre-release. Aborting ...') sys.exit() cask_content = Template(pre_release_cask_template).substitute( MAJOR=sapMachineTag.get_major(), VERSION=sapMachineTag.get_version_string_without_build(), BUILD_NUMBER=sapMachineTag.get_build_number(), IMAGE_TYPE=args.imagetype, SHA256=args.sha256sum) cask_file_name = str.format('sapmachine{0}-ea-{1}.rb', sapMachineTag.get_major(), args.imagetype) else: cask_content = Template(release_cask_template).substitute( MAJOR=sapMachineTag.get_major(), VERSION=sapMachineTag.get_version_string_without_build(), IMAGE_TYPE=args.imagetype, SHA256=args.sha256sum) cask_file_name = str.format('sapmachine{0}-{1}.rb', sapMachineTag.get_major(), args.imagetype) homebrew_dir = join(work_dir, 'homebrew') cask_dir = join(homebrew_dir, 'Casks') cask_file_path = join(cask_dir, cask_file_name) utils.git_clone('github.com/SAP/homebrew-SapMachine', 'master', homebrew_dir) current_cask_version = None current_cask_build_number = None if os.path.exists(cask_file_path): with open(cask_file_path, 'r') as cask_file: cask_version_match = cask_version_pattern.search(cask_file.read()) if cask_version_match is not None: if len(cask_version_match.groups()) >= 1: current_cask_version = cask_version_match.group(1) if len(cask_version_match.groups()) >= 4: current_cask_build_number = cask_version_match.group(4) current_cask_version = versions.version_to_tuple( current_cask_version, current_cask_build_number) if current_cask_version is None or sapMachineTag.get_version_tuple( ) >= current_cask_version: print( str.format("Creating/updating cask for version {0}...", sapMachineTag.get_version_tuple())) with open(cask_file_path, 'w') as cask_file: cask_file.write(cask_content) utils.git_commit( homebrew_dir, str.format('Update {0} ({1}).', cask_file_name, sapMachineTag.get_version_string()), [join('Casks', cask_file_name)]) utils.git_push(homebrew_dir) else: print( str.format( "Current cask has version {0} which is higher than {1}, no update.", current_cask_version, sapMachineTag.get_version_tuple())) return 0
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('--workdir', help='specify the working directory', metavar='DIR', default='docker_work', required=False) args = parser.parse_args() workdir = os.path.realpath(args.workdir) git_dir = join(workdir, 'sapmachine-infrastructure') utils.remove_if_exists(workdir) os.makedirs(workdir) releases = github_api_request('releases', per_page=100) infrastructure_tags = utils.get_github_infrastructure_tags() docker_releases = {} stable_release = None for release in releases: if release['prerelease']: continue version, version_part, major, update, version_sap, build_number, os_ext = utils.sapmachine_tag_components( release['name']) if utils.sapmachine_is_lts(major): if major in docker_releases: _, _, _, lts_update, _, _, _ = utils.sapmachine_tag_components( docker_releases[major]['name']) if int(lts_update) < int(update): docker_releases[major] = release else: docker_releases[major] = release if stable_release == None: stable_release = release else: _, _, stable_major, stable_update, _, _, _ = utils.sapmachine_tag_components( stable_release['name']) if int(major) > int(stable_major) or ( int(major) == int(stable_major) and int(update) > int(stable_update)): stable_release = release print('Determined the following versions for processing:') for release in docker_releases: version, _, _, _, _, _, _ = utils.sapmachine_tag_components( docker_releases[release]['name']) print(str.format('LTS {1}: {0}', version, release)) stable_version, _, stable_major, _, _, _, _ = utils.sapmachine_tag_components( stable_release['name']) print(str.format('stable: {0}', stable_version)) if not (stable_version in docker_releases): docker_releases[stable_major] = stable_release utils.git_clone('github.com/SAP/SapMachine-infrastructure', 'master', git_dir) versions_dir = join(git_dir, 'dockerfiles', 'official') removed = [] for f in os.listdir(versions_dir): if not f in docker_releases: utils.remove_if_exists(join(versions_dir, f)) removed.append(join(versions_dir, f)) if removed != []: utils.git_commit(git_dir, 'remove discontinued versions', removed) for release in docker_releases: process_release(docker_releases[release], infrastructure_tags, git_dir) utils.remove_if_exists(workdir) return 0
CLONE_DIR = './repositories' OUTPUT_DIR = './output/' delete_dir(dir_path=CLONE_DIR) delete_dir(dir_path=OUTPUT_DIR) create_dir(dir_path=CLONE_DIR) create_dir(dir_path=OUTPUT_DIR) for project in PROJECTS: project_base_dirs = [] project_base_files = [] project_exclude_files = EXCLUDE_FILES + project["exclude_extension"] project_exclude_extension = EXCLUDE_EXTENSION + project["exclude_extension"] clone_repository_dir = os.path.join(CLONE_DIR, project["name"]) git_clone(CLONE_DIR, project["repository"]) for item in os.listdir(clone_repository_dir): item_path = os.path.join(clone_repository_dir, item) if os.path.isdir(item_path): if item not in EXCLUDE_DIRS: project_base_dirs.append(item_path) if os.path.isfile(item_path): if is_file_valid(item, project_exclude_files, project_exclude_extension): project_base_files.append(item_path) for base_dir in project_base_dirs: for root, dirs, files in os.walk(base_dir): for file in files: if is_file_valid(file, project_exclude_files,
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('-a', '--asset', help='the SapMachine asset file', metavar='ASSET', required=True) parser.add_argument('-j', '--jre', help='Build SapMachine JRE installer', action='store_true', default=False) parser.add_argument('-s', '--sapmachine-directory', help='specify the SapMachine GIT directory', metavar='DIR', required=True) args = parser.parse_args() cwd = os.getcwd() work_dir = join(cwd, 'msi_work') asset = os.path.realpath(args.asset) is_jre = args.jre sapmachine_git_dir = args.sapmachine_directory products = None product_id = None upgrade_code = None utils.remove_if_exists(work_dir) os.makedirs(work_dir) utils.extract_archive(asset, work_dir, remove_archive=False) sapmachine_folder = glob.glob(join(work_dir, 'sapmachine*')) os.rename(sapmachine_folder[0], join(work_dir, 'SourceDir')) _, _, version_output = utils.run_cmd( [join(work_dir, 'SourceDir', 'bin', 'java.exe'), '-version'], std=True) version, version_part, major, version_sap, build_number = utils.sapmachine_version_components( version_output, multiline=True) sapmachine_version = [e for e in version_part.split('.')] if len(sapmachine_version) < 3: sapmachine_version += [ '0' for sapmachine_version in range(0, 3 - len(sapmachine_version)) ] if len(sapmachine_version) == 4: sapmachine_version[3] = str((int(sapmachine_version[3]) << 8) & 0xFF00) if len(sapmachine_version) == 5: merged_version = str((int(sapmachine_version[3]) << 8) | (int(sapmachine_version[4]) & 0xFF)) del sapmachine_version[4] sapmachine_version[3] = merged_version sapmachine_version = '.'.join(sapmachine_version) shutil.copyfile( join(sapmachine_git_dir, 'src', 'java.base', 'windows', 'native', 'launcher', 'icons', 'awt.ico'), join(work_dir, 'sapmachine.ico')) write_as_rtf(join(sapmachine_git_dir, 'LICENSE'), join(work_dir, 'license.rtf')) infrastructure_dir = join(work_dir, 'sapmachine_infrastructure') templates_dir = join(infrastructure_dir, 'wix-templates') utils.git_clone('github.com/SAP/SapMachine-infrastructure', 'master', infrastructure_dir) with open(join(templates_dir, 'products.yml'), 'r') as products_yml: products = yaml.safe_load(products_yml.read()) image_type = 'jre' if is_jre else 'jdk' if products[image_type] is None or major not in products[image_type]: product_id = str(uuid.uuid4()) upgrade_code = str(uuid.uuid4()) if products[image_type] is None: products[image_type] = {} products[image_type][major] = { 'product_id': product_id, 'upgrade_code': upgrade_code } with open(join(templates_dir, 'products.yml'), 'w') as products_yml: products_yml.write(yaml.dump(products, default_flow_style=False)) utils.git_commit(infrastructure_dir, 'Updated product codes.', [join('wix-templates', 'products.yml')]) utils.git_push(infrastructure_dir) else: product_id = products[image_type][major]['product_id'] upgrade_code = products[image_type][major]['upgrade_code'] create_sapmachine_wxs( join( templates_dir, 'SapMachine.jre.wxs.template' if is_jre else 'SapMachine.jdk.wxs.template'), join(work_dir, 'SapMachine.wxs'), product_id, upgrade_code, sapmachine_version, major) shutil.copyfile(join(work_dir, 'SourceDir', 'release'), join(work_dir, 'release')) utils.remove_if_exists(join(work_dir, 'SourceDir', 'release')) utils.run_cmd( 'heat dir SourceDir -swall -srd -gg -platform x64 -template:module -cg SapMachineGroup -out SapMachineModule.wxs' .split(' '), cwd=work_dir) with open(join(work_dir, 'SapMachineModule.wxs'), 'r+') as sapmachine_module: sapmachine_module_content = sapmachine_module.read() sapmachine_module_content = sapmachine_module_content.replace( 'PUT-MODULE-NAME-HERE', 'SapMachineModule') sapmachine_module_content = sapmachine_module_content.replace( 'PUT-COMPANY-NAME-HERE', 'SapMachine Team') sapmachine_module.seek(0) sapmachine_module.truncate() sapmachine_module.write(sapmachine_module_content) utils.run_cmd('candle -arch x64 SapMachineModule.wxs'.split(' '), cwd=work_dir) utils.run_cmd('light SapMachineModule.wixobj'.split(' '), cwd=work_dir) utils.run_cmd('candle -arch x64 SapMachine.wxs'.split(' '), cwd=work_dir) utils.run_cmd('light -ext WixUIExtension SapMachine.wixobj'.split(' '), cwd=work_dir) msi_name = os.path.basename(asset) msi_name = os.path.splitext(msi_name)[0] os.rename(join(work_dir, 'SapMachine.msi'), join(cwd, str.format('{0}.msi', msi_name))) return 0
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('--workdir', help='the temporary working directory', metavar='DIR', default="tags_work", required=False) args = parser.parse_args() workdir = os.path.realpath(args.workdir) utils.remove_if_exists(workdir) os.makedirs(workdir) global git_target_dir git_target_dir = join(workdir, 'sapmachine') # fetch all branches sapmachine_branches = utils.get_active_sapmachine_branches() if sapmachine_branches is None or len(sapmachine_branches) == 0: print("No sapmachine branches found") return 0 # clone the SapMachine repository to the first branch in list utils.git_clone('github.com/SAP/SapMachine.git', sapmachine_branches[0][0], git_target_dir) # go through all sapmachine branches, find the latest OpenJDK merge commit and make sure it's correctly sapmachine-tagged for branch in sapmachine_branches: # checkout branch utils.run_cmd(str.format('git checkout {0}', branch[0]).split(' '), cwd=git_target_dir) # find latest merge commit merge_commits = get_merge_commits() jdk_tag = None for merge_commit in merge_commits: if re.search(merge_commit_pattern, merge_commit[0]) is None: print( str.format('merge commit "{0}" is no OpenJDK merge', merge_commit[0])) continue match_jdk_tag = re.search(JDKTag.tag_pattern, merge_commit[0]) if match_jdk_tag is None: print( str.format( 'merge commit "{0}" does not contain an OpenJDK tag pattern', merge_commit[0])) continue jdk_tag = JDKTag(match_jdk_tag) merge_commit_message = merge_commit[0] merge_commit_id = merge_commit[1] break if jdk_tag is None: print( str.format('No merge commits found for branch "{0}"', branch[0])) continue else: print( str.format( 'latest merge commit for branch "{0}" is {1}: "{2}"', branch[0], merge_commit_id, merge_commit_message)) # if the merge commit is already contained in a SapMachine tag, we're done if the merge commit's JDK tag is a ga tag # otherwise we run some logic to make sure a corresponding JDK ga tag is not missed out _, tags_of_merge_commit, _ = utils.run_cmd(str.format( 'git tag --contains {0}', merge_commit_id).split(' '), cwd=git_target_dir, std=True, throw=False) if tags_of_merge_commit: print( str.format('Merge commit for "{0}" was already tagged.', jdk_tag.as_string())) if not jdk_tag.is_ga(): check_for_untagged_ga(jdk_tag, tags_of_merge_commit.splitlines(), merge_commit_id) continue # this merge commit is not contained in a SapMachine tag but a corresponding sapmachine tag already exists pointing to somewhere else. # That's weird, but we don't change things now... if exists_tag(jdk_tag.as_sapmachine_tag_string()): print( str.format( '"{0}" is already tagged as "{1}" but does not include this merge commit. That could be a problem.', jdk_tag.as_string(), jdk_tag.as_sapmachine_tag_string())) continue # create the sapmachine tag create_sapmachine_tag(jdk_tag, merge_commit_id) if jdk_tag.is_ga(): # when the tag is a GA tag and the last non-ga tag was not yet tagged, we tag it to the same commit and build it make_sure_last_non_ga_is_tagged(jdk_tag, merge_commit_id) else: # build the new tag (EA release) run_jenkins_jobs(jdk_tag.get_major(), jdk_tag.as_sapmachine_tag_string()) utils.remove_if_exists(workdir) return 0
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('-t', '--tag', help='the SapMachine tag', metavar='TAG', required=True) parser.add_argument('--sha256sum', help='the sha256 sum', metavar='SHA256', required=True) parser.add_argument('-i', '--imagetype', help='The image type', metavar='IMAGETYPE', choices=['jdk', 'jre']) parser.add_argument('-p', '--prerelease', help='this is a pre-release', action='store_true', default=False) args = parser.parse_args() cwd = os.getcwd() work_dir = join(cwd, 'cask_work') tag = args.tag sha256sum = args.sha256sum image_type = args.imagetype is_prerelease = args.prerelease cask_version_pattern = re.compile('version \'((\d+\.?)+)(,(\d+))?\'') utils.remove_if_exists(work_dir) os.makedirs(work_dir) version, version_part, major, build_number, sap_build_number, os_ext = utils.sapmachine_tag_components( tag) if is_prerelease: if build_number is None: print('No build number given. Aborting ...') sys.exit() cask_content = Template(pre_release_cask_template).substitute( MAJOR=major, VERSION=version_part, BUILD_NUMBER=build_number, IMAGE_TYPE=image_type, SHA256=sha256sum) cask_file_name = str.format('sapmachine{0}-ea-{1}.rb', major, image_type) else: cask_content = Template(release_cask_template).substitute( MAJOR=major, VERSION=version_part, IMAGE_TYPE=image_type, SHA256=sha256sum) cask_file_name = str.format('sapmachine{0}-{1}.rb', major, image_type) homebrew_dir = join(work_dir, 'homebrew') cask_dir = join(homebrew_dir, 'Casks') utils.git_clone('github.com/SAP/homebrew-SapMachine', 'master', homebrew_dir) current_cask_version = None current_cask_build_number = None if os.path.exists(join(cask_dir, cask_file_name)): with open(join(cask_dir, cask_file_name), 'r') as cask_file: cask_version_match = cask_version_pattern.search(cask_file.read()) if cask_version_match is not None: if len(cask_version_match.groups()) >= 1: current_cask_version = cask_version_match.group(1) if len(cask_version_match.groups()) >= 4: current_cask_build_number = cask_version_match.group(4) current_cask_version = version_to_tuple(current_cask_version, current_cask_build_number) new_cask_version = version_to_tuple(version_part, build_number) if new_cask_version >= current_cask_version: with open(join(cask_dir, cask_file_name), 'w') as cask_file: cask_file.write(cask_content) utils.git_commit(homebrew_dir, str.format('Updated {0}.', cask_file_name), [join('Casks', cask_file_name)]) utils.git_push(homebrew_dir) return 0
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('-m', '--major', help='the SapMachine major version', metavar='MAJOR', required=True) args = parser.parse_args() token = utils.get_github_api_accesstoken() github_api = 'https://api.github.com/repos/SAP/SapMachine/releases' asset_pattern = re.compile(utils.sapmachine_asset_pattern()) asset_map = {} request = Request(github_api) if token is not None: request.add_header('Authorization', str.format('token {0}', token)) response = json.loads(urlopen(request).read()) for release in response: if release['prerelease'] is True: continue version, version_part, major, build_number, sap_build_number, os_ext = utils.sapmachine_tag_components( release['name']) assets = release['assets'] if version is None or os_ext: continue if args.major == major: for asset in assets: match = asset_pattern.match(asset['name']) if match is not None: asset_image_type = match.group(1) asset_os = match.group(3) if asset_os == 'linux-x64' and asset_image_type == 'jre': buildpack_version = '' parts = version_part.split('.') num_parts = len(parts) if num_parts == 3: buildpack_version = str.format( '{0}_', version_part) elif num_parts < 3: buildpack_version = str.format( '{0}{1}_', version_part, '.0' * (3 - num_parts)) elif num_parts > 3: buildpack_version = str.format( '{0}.{1}.{2}_{3}', parts[0], parts[1], parts[2], parts[3]) buildpack_version += str.format('b{0}', build_number) if sap_build_number: buildpack_version += str.format( 's{0}', sap_build_number) asset_map[buildpack_version] = asset[ 'browser_download_url'] local_repo = join(os.getcwd(), 'gh-pages') utils.git_clone('github.com/SAP/SapMachine.git', 'gh-pages', local_repo) write_index_yaml( asset_map, join(local_repo, 'assets', 'cf', 'jre', args.major, 'linux', 'x86_64')) utils.git_commit(local_repo, 'Updated index.yml', ['assets']) utils.git_push(local_repo) utils.remove_if_exists(local_repo)
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('-t', '--tag', help='the SapMachine tag', metavar='TAG', required=True) parser.add_argument( '-d', '--dual', help='this is going to be a dual architecture cask (x64 and aarch64)', action='store_true', default=False) args = parser.parse_args() work_dir = join(os.getcwd(), 'cask_work') utils.remove_if_exists(work_dir) os.makedirs(work_dir) raw_tag = args.tag sapMachineTag = SapMachineTag.from_string(raw_tag) if sapMachineTag is None: print(str.format("Tag {0} seems to be invalid. Aborting...", args.tag)) sys.exit(1) os_name = 'osx' if sapMachineTag.get_major() < 17 or ( sapMachineTag.get_major() == 17 and sapMachineTag.get_update() is None and sapMachineTag.get_build_number() < 21) else 'macos' prerelease = not sapMachineTag.is_ga() if prerelease: jdk_cask_file_name = str.format('sapmachine{0}-ea-jdk.rb', sapMachineTag.get_major()) jre_cask_file_name = str.format('sapmachine{0}-ea-jre.rb', sapMachineTag.get_major()) cask_tag = str.format('{0}-ea', sapMachineTag.get_major()) cask_version = str.format( '{0},{1}', sapMachineTag.get_version_string_without_build(), sapMachineTag.get_build_number()) ruby_version = 'version.before_comma' ea_ext = '-ea.' url_version1 = '#{version.before_comma}%2B#{version.after_comma}' url_version2 = '#{version.before_comma}-ea.#{version.after_comma}' else: jdk_cask_file_name = str.format('sapmachine{0}-jdk.rb', sapMachineTag.get_major()) jre_cask_file_name = str.format('sapmachine{0}-jre.rb', sapMachineTag.get_major()) cask_tag = str.format('{0}', sapMachineTag.get_major()) cask_version = str.format( '{0}', sapMachineTag.get_version_string_without_build()) ruby_version = 'version' ea_ext = '.' url_version1 = '#{version}' url_version2 = '#{version}' if args.dual: try: aarch_jdk_sha_url, aarch_jre_sha_url = utils.get_asset_url( raw_tag, os_name + '-aarch64', '.sha256.dmg.txt') intel_jdk_sha_url, intel_jre_sha_url = utils.get_asset_url( raw_tag, os_name + '-x64', '.sha256.dmg.txt') except Exception as e: print('Not both platforms ready yet') sys.exit(0) aarch_jdk_sha, code1 = utils.download_asset(aarch_jdk_sha_url) aarch_jre_sha, code2 = utils.download_asset(aarch_jre_sha_url) intel_jdk_sha, code3 = utils.download_asset(intel_jdk_sha_url) intel_jre_sha, code4 = utils.download_asset(intel_jre_sha_url) if code1 != 200 or code2 != 200 or code3 != 200 or code4 != 200: print('Download failed') sys.exit(1) aarch_jdk_sha = aarch_jdk_sha.split(' ')[0] aarch_jre_sha = aarch_jre_sha.split(' ')[0] intel_jdk_sha = intel_jdk_sha.split(' ')[0] intel_jre_sha = intel_jre_sha.split(' ')[0] jdk_cask_content = Template(duplex_cask_template).substitute( CASK_TAG=cask_tag, IMAGE_TYPE='jdk', CASK_VERSION=cask_version, URL_VERSION1=url_version1, URL_VERSION2=url_version2, OS_NAME=os_name, INTELSHA256=intel_jdk_sha, AARCHSHA256=aarch_jdk_sha, RUBY_VERSION=ruby_version, EA_EXT=ea_ext) jre_cask_content = Template(duplex_cask_template).substitute( CASK_TAG=cask_tag, IMAGE_TYPE='jre', CASK_VERSION=cask_version, URL_VERSION1=url_version1, URL_VERSION2=url_version2, OS_NAME=os_name, INTELSHA256=intel_jre_sha, AARCHSHA256=aarch_jre_sha, RUBY_VERSION=ruby_version, EA_EXT=ea_ext) else: try: intel_jdk_sha_url, intel_jre_sha_url = utils.get_asset_url( raw_tag, os_name + '-x64', '.sha256.dmg.txt') except Exception as e: print('Asset not found') sys.exit(1) intel_jdk_sha, code1 = utils.download_asset(intel_jdk_sha_url) intel_jre_sha, code2 = utils.download_asset(intel_jre_sha_url) if code1 != 200 or code2 != 200: print('Download failed') sys.exit(1) intel_jdk_sha = intel_jdk_sha.split(' ')[0] intel_jre_sha = intel_jre_sha.split(' ')[0] try: intel_jdk_url, intel_jre_url = utils.get_asset_url( raw_tag, os_name + '-x64', '.dmg') except Exception as e: print('Asset not found') sys.exit(1) jdk_cask_content = Template(cask_template).substitute( CASK_TAG=cask_tag, IMAGE_TYPE='jdk', CASK_VERSION=cask_version, SHA256=intel_jdk_sha, URL_VERSION1=url_version1, URL_VERSION2=url_version2, OS_NAME=os_name, RUBY_VERSION=ruby_version, EA_EXT=ea_ext) jre_cask_content = Template(cask_template).substitute( CASK_TAG=cask_tag, IMAGE_TYPE='jre', CASK_VERSION=cask_version, SHA256=intel_jre_sha, URL_VERSION1=url_version1, URL_VERSION2=url_version2, OS_NAME=os_name, RUBY_VERSION=ruby_version, EA_EXT=ea_ext) homebrew_dir = join(work_dir, 'homebrew') utils.git_clone('github.com/SAP/homebrew-SapMachine', 'master', homebrew_dir) jdk_replaced = replace_cask(jdk_cask_file_name, jdk_cask_content, sapMachineTag, homebrew_dir) jre_replaced = replace_cask(jre_cask_file_name, jre_cask_content, sapMachineTag, homebrew_dir) if jdk_replaced or jre_replaced: utils.git_push(homebrew_dir) utils.remove_if_exists(work_dir) return 0