def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('--workdir', help='the temporary working directory', metavar='DIR', default="tags_work", required=False) args = parser.parse_args() workdir = os.path.realpath(args.workdir) utils.remove_if_exists(workdir) os.makedirs(workdir) # fetch all branches sapmachine_branches = utils.get_active_sapmachine_branches() # fetch all tags tags = utils.get_github_tags() # iterate all tags to find the latest jdk tag for a JDK major release jdk_tags = {} for tag in tags: # filter for jdk tags jdk_tag = JDKTag.from_string(tag['name']) if jdk_tag is None: continue # only remember the latest jdk tag (version comparison) # filter out jdk update tags with build number "0" like jdk-11.0.3+0 if ((jdk_tag.get_major() not in jdk_tags or jdk_tag.is_greater_than(jdk_tags[jdk_tag.get_major()])) and not (jdk_tag.get_update() > 0 and (jdk_tag.get_build_number() if jdk_tag.get_build_number() is not None else 0) == 0)): jdk_tags[jdk_tag.get_major()] = jdk_tag # clone the SapMachine repository git_target_dir = join(workdir, 'sapmachine') utils.git_clone('github.com/SAP/SapMachine.git', 'sapmachine', git_target_dir) # for each "sapmachine" branch, check whether it contains the latest jdk tag with matching major version for sapmachine_branch in sapmachine_branches: # get the latest jdk tag for this major version latest_tag = jdk_tags[sapmachine_branch[1]] print(str.format('latest tag for branch "{0}" is "{1}"', sapmachine_branch, latest_tag.as_string())) # check whether the jdk tag is already contained in the sapmachine branch _, out, _ = utils.run_cmd(str.format('git branch -a --contains tags/{0}', latest_tag.as_string()).split(' '), cwd=git_target_dir, std=True, throw=False) containing_branches_pattern = re.compile('{0}|pr-jdk-{1}.*'.format(sapmachine_branch[0], sapmachine_branch[1])) match = re.search(containing_branches_pattern, out) if match is None: # the jdk tag is not contained in a sapmachine branch # create a pull request branch and a pull request. print(str.format('creating pull request "pr-{0}" with base branch "{1}"', latest_tag.as_string(), sapmachine_branch)) utils.run_cmd(str.format('git checkout {0}', latest_tag.as_string()).split(' '), cwd=git_target_dir) utils.run_cmd(str.format('git checkout -b pr-{0}', latest_tag.as_string()).split(' '), cwd=git_target_dir) utils.run_cmd(str.format('git push origin pr-{0}', latest_tag.as_string()).split(' '), cwd=git_target_dir) pull_request = str.format('{{ "title": "Merge to tag {0}", "body": "please pull", "head": "pr-{1}", "base": "{2}" }}', latest_tag.as_string(), latest_tag.as_string(), sapmachine_branch[0]) utils.github_api_request('pulls', data=pull_request, method='POST') utils.remove_if_exists(workdir) return 0
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('--workdir', help='specify the working directory', metavar='DIR', default='docker_work', required=False) args = parser.parse_args() workdir = os.path.realpath(args.workdir) git_dir = join(workdir, 'sapmachine-infrastructure') utils.remove_if_exists(workdir) os.makedirs(workdir) releases = github_api_request('releases', per_page=100) infrastructure_tags = github_api_request( 'tags', repository='SapMachine-infrastructure', per_page=100) lts_release = None lts_release_major = 0 stable_release = None for release in releases: if release['prerelease']: continue version, version_part, major, build_number, sap_build_number, os_ext = utils.sapmachine_tag_components( release['name']) if utils.sapmachine_is_lts(major) and not lts_release: lts_release = release lts_release_major = major else: if not stable_release and major > lts_release_major: stable_release = release if lts_release and stable_release: break if lts_release or stable_release: utils.git_clone('github.com/SAP/SapMachine-infrastructure', 'master', git_dir) if lts_release and not stable_release: stable_release = lts_release if lts_release: print(str.format('found latest LTS release "{0}"', lts_release['name'])) process_release(lts_release, 'lts', infrastructure_tags, git_dir) if stable_release: print( str.format('found latest stable release "{0}"', stable_release['name'])) process_release(stable_release, 'stable', infrastructure_tags, git_dir) utils.remove_if_exists(workdir) return 0
def main(argv=None): upstream_releases = utils.github_api_request(api='releases', url=None, owner='jvm-profiling-tools', repository='async-profiler', data=None, method='GET', per_page=100, content_type=None, url_parameter=[]) sap_releases = utils.github_api_request(api='releases', url=None, owner='SAP', repository='async-profiler', data=None, method='GET', per_page=100, content_type=None, url_parameter=[]) sap_tags = utils.github_api_request(api='tags', url=None, owner='SAP', repository='async-profiler', data=None, method='GET', per_page=100, content_type=None, url_parameter=[]) for release in reversed(upstream_releases): tag = release['tag_name'] if [x for x in sap_releases if x['tag_name'] == tag ] == [] and [x for x in sap_tags if x['name'] == tag] != []: data = json.dumps({ "tag_name": tag, "name": release['name'], "body": release['body'] }) response = utils.github_api_request( api='releases', url=None, owner='SAP', repository='async-profiler', data=data, method='POST', content_type='application/json') print(tag, end='') return 0 print('No new releases', end='') return 0
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('-m', '--major', help='the SapMachine major version', metavar='MAJOR', required=True) parser.add_argument('-s', '--separator', help='the separator char', metavar='SEPARATOR', required=False, default=' ') parser.add_argument('-p', '--include-prereleases', help='include pre-releases', action='store_true', default=False) args = parser.parse_args() requested_major = args.major separator = args.separator include_prereleases = args.include_prereleases tag_list = [] releases = github_api_request('releases') for release in releases: if release['prerelease'] is True and not include_prereleases: continue version, version_part, major, update, version_sap, build_number, os_ext = utils.sapmachine_tag_components(release['name']) if major is None or major != requested_major or os_ext: continue tag_list.append(release['name']) print(separator.join([tag for tag in tag_list]))
def validate_issue(issue_id): issue = github_api_request(str.format('issues/{0}', issue_id)) # check whether the issue exists if issue is None: raise Exception(str.format('No issue with id #{0} found.'), issue_id) return issue
def main(argv=None): token = utils.get_github_api_accesstoken() asset_pattern = re.compile(utils.sapmachine_asset_pattern()) asset_map = {} releases = utils.github_api_request('releases', per_page=100) for release in releases: if release['prerelease'] is True: continue version, version_part, major, build_number, sap_build_number, os_ext = utils.sapmachine_tag_components( release['name']) assets = release['assets'] if version is None or os_ext: continue for asset in assets: match = asset_pattern.match(asset['name']) if match is not None: asset_image_type = match.group(1) asset_os = match.group(3) if asset_os == 'linux-x64' and asset_image_type == 'jre': sapmachine_version = [ int(e) for e in version_part.split('.') ] sapmachine_version += [ 0 for sapmachine_version in range( 0, 5 - len(sapmachine_version)) ] if sap_build_number: sapmachine_version[4] = int(sap_build_number) buildpack_version = str.format( '{0}.{1}.{2}_{3}.{4}.b{5}', sapmachine_version[0], sapmachine_version[1], sapmachine_version[2], sapmachine_version[3], sapmachine_version[4], build_number if build_number else '0') asset_map[buildpack_version] = asset[ 'browser_download_url'] local_repo = join(os.getcwd(), 'gh-pages') utils.git_clone('github.com/SAP/SapMachine.git', 'gh-pages', local_repo) write_index_yaml( asset_map, join(local_repo, 'assets', 'cf', 'jre', 'linux', 'x86_64')) utils.git_commit(local_repo, 'Updated index.yml', ['assets']) utils.git_push(local_repo) utils.remove_if_exists(local_repo)
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('-t', '--tag', help='the SapMachine Git tag', metavar='TAG', required=True) args = parser.parse_args() releases = utils.github_api_request('releases', per_page=100) for release in releases: if release['tag_name'] == args.tag: print(release['published_at'].split('T')[0])
def is_pr_ok_to_test(pull_request): if pull_request['author_association'] == 'MEMBER' or pull_request['user'][ 'login'] == 'SapMachine': # is a sapmachine-team member or user 'SapMachine' # it is OK to test return True else: # not a sapmachine-team member # check for a 'retest this please' comment # by any sapmachine-team member comments = github_api_request(url=pull_request['comments_url']) for comment in comments: if comment['body'] == 'retest this please' and comment[ 'author_association'] == 'MEMBER': return True return False
def get_latest_non_ga_tag(jdk_tag): latest_non_ga_tag = None if jdk_tag.is_ga(): # fetch all tags tags = utils.github_api_request('tags', per_page=300) latest_non_ga_tag = None # iterate all tags for tag in tags: # filter for jdk tags match = JDKTag.jdk_tag_pattern.match(tag['name']) if match is not None: # found a jdk tag t = JDKTag(match) major = t.get_major() if major is jdk_tag.get_major() and not t.is_ga(): if latest_non_ga_tag is None or t.is_greater_than( latest_non_ga_tag): latest_non_ga_tag = t return latest_non_ga_tag
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('--workdir', help='specify the working directory', metavar='DIR', default='docker_work', required=False) args = parser.parse_args() workdir = os.path.realpath(args.workdir) git_dir = join(workdir, 'sapmachine-infrastructure') utils.remove_if_exists(workdir) os.makedirs(workdir) releases = github_api_request('releases', per_page=100) infrastructure_tags = utils.get_github_infrastructure_tags() docker_releases = {} stable_release = None for release in releases: if release['prerelease']: continue version, version_part, major, update, version_sap, build_number, os_ext = utils.sapmachine_tag_components( release['name']) if utils.sapmachine_is_lts(major): if major in docker_releases: _, _, _, lts_update, _, _, _ = utils.sapmachine_tag_components( docker_releases[major]['name']) if int(lts_update) < int(update): docker_releases[major] = release else: docker_releases[major] = release if stable_release == None: stable_release = release else: _, _, stable_major, stable_update, _, _, _ = utils.sapmachine_tag_components( stable_release['name']) if int(major) > int(stable_major) or ( int(major) == int(stable_major) and int(update) > int(stable_update)): stable_release = release print('Determined the following versions for processing:') for release in docker_releases: version, _, _, _, _, _, _ = utils.sapmachine_tag_components( docker_releases[release]['name']) print(str.format('LTS {1}: {0}', version, release)) stable_version, _, stable_major, _, _, _, _ = utils.sapmachine_tag_components( stable_release['name']) print(str.format('stable: {0}', stable_version)) if not (stable_version in docker_releases): docker_releases[stable_major] = stable_release utils.git_clone('github.com/SAP/SapMachine-infrastructure', 'master', git_dir) versions_dir = join(git_dir, 'dockerfiles', 'official') removed = [] for f in os.listdir(versions_dir): if not f in docker_releases: utils.remove_if_exists(join(versions_dir, f)) removed.append(join(versions_dir, f)) if removed != []: utils.git_commit(git_dir, 'remove discontinued versions', removed) for release in docker_releases: process_release(docker_releases[release], infrastructure_tags, git_dir) utils.remove_if_exists(workdir) return 0
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('-m', '--major', help='the SapMachine major version to build', metavar='MAJOR', required=True) parser.add_argument('-d', '--destination', help='the download destination', metavar='DIR', required=True) args = parser.parse_args() boot_jdk_major_max = int(args.major) boot_jdk_major_min = boot_jdk_major_max - 1 destination = os.path.realpath(args.destination) releases = utils.github_api_request('releases', per_page=100) platform = str.format('{0}-{1}_bin', utils.get_system(), utils.get_arch()) for release in releases: if release['prerelease']: continue version, version_part, major, build_number, sap_build_number, os_ext = utils.sapmachine_tag_components(release['name']) if major is None: continue major = int(major) if major <= boot_jdk_major_max and major >= boot_jdk_major_min: assets = release['assets'] for asset in assets: asset_name = asset['name'] asset_url = asset['browser_download_url'] if 'jdk' in asset_name and platform in asset_name and not asset_name.endswith('.txt'): archive_path = join(destination, asset_name) utils.remove_if_exists(archive_path) utils.download_artifact(asset_url, archive_path) boot_jdk_exploded = join(destination, 'boot_jdk') utils.remove_if_exists(boot_jdk_exploded) os.makedirs(boot_jdk_exploded) utils.extract_archive(archive_path, boot_jdk_exploded) sapmachine_folder = glob.glob(join(boot_jdk_exploded, 'sapmachine*')) if sapmachine_folder is not None: sapmachine_folder = sapmachine_folder[0] files = os.listdir(sapmachine_folder) for f in files: shutil.move(join(sapmachine_folder, f), boot_jdk_exploded) utils.remove_if_exists(sapmachine_folder) if utils.get_system() == 'osx': files = os.listdir(join(boot_jdk_exploded, 'Contents', 'Home')) for f in files: shutil.move(join(boot_jdk_exploded, 'Contents', 'Home', f), boot_jdk_exploded) utils.remove_if_exists(join(boot_jdk_exploded, 'Contents')) return 0 return 0
def main(argv=None): token = utils.get_github_api_accesstoken() org = 'SAP' repository = 'SapMachine' asset_pattern = re.compile(utils.sapmachine_asset_pattern()) major_dict = {} release_dict = {} image_dict = {} latest_link_dict = {} releases = utils.github_api_request('releases', per_page=100) for release in releases: version, version_part, major, build_number, sap_build_number, os_ext = utils.sapmachine_tag_components( release['name']) is_prerelease = release['prerelease'] if version is None or os_ext: continue if major in major_dict: if not is_prerelease and major_dict[major]: # this is not a pre-release but the release before this was a pre-release # remove all assets if major in release_dict: release_dict[major].clear_assets() # remove entry in the image dictionary if major in image_dict: del image_dict[major] else: if not major_dict[major]: # the release before this was a release # skip, we only keep the latest release version continue major_dict[major] = is_prerelease assets = release['assets'] if is_prerelease is not True and major not in latest_link_dict: latest_link_dict[major] = Template(latest_template).substitute( major=major, url=release['html_url']) has_dmg = False for asset in assets: match = asset_pattern.match(asset['name']) if match is not None: asset_image_type = match.group(1) if asset_image_type == 'jdk': asset_os = match.group(3) file_type = match.group(4) if asset_os == 'windows-x64' and file_type == '.msi': asset_os = 'windows-x64-installer' if asset_os == 'osx-x64': if file_type == '.dmg': has_dmg = True elif has_dmg: continue tag = release['name'] image_is_lts = utils.sapmachine_is_lts( major) and not release['prerelease'] if major not in image_dict: image_dict[major] = { 'label': str.format('SapMachine {0}', major), 'lts': image_is_lts, 'ea': release['prerelease'] } if major in release_dict: releases = release_dict[major] else: releases = Releases(major) release_dict[major] = releases release_dict[major].add_asset( asset['browser_download_url'], asset_os, tag) latest_lts_version = 0 latest_non_lts_version = 0 for major in image_dict: if image_dict[major]['lts'] and int(major) > latest_lts_version: latest_lts_version = int(major) if not image_dict[major]['lts'] and not image_dict[major][ 'ea'] and int(major) > latest_non_lts_version: latest_non_lts_version = int(major) json_root = {'imageTypes': [], 'os': [], 'assets': {}} for major in sorted(image_dict): add = False if image_dict[major]['lts']: if int(major) >= latest_lts_version: add = True else: if int(major) >= latest_non_lts_version and int( major) >= latest_lts_version: add = True if add: json_root['imageTypes'].append({ 'id': major, 'label': image_dict[major]['label'], 'lts': image_dict[major]['lts'], 'ea': image_dict[major]['ea'] }) else: del image_dict[major] def get_os_key(os): return os_description[os]['ordinal'] for os in sorted(os_description, key=get_os_key): json_root['os'].append({ 'key': os, 'value': os_description[os]['name'], 'ordinal': os_description[os]['ordinal'] }) for major in release_dict: if major in image_dict: json_root['assets'].update(release_dict[major].transform()) files = [{ 'location': join('assets', 'data', 'sapmachine_releases.json'), 'data': json.dumps(json_root, indent=4), 'commit_message': 'Updated release data.' }] for major in latest_link_dict: files.append({ 'location': join('latest', major, 'index.md'), 'data': latest_link_dict[major], 'commit_message': str.format('Updated latest link for SapMachine {0}', major) }) push_to_git(files) return 0
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('-t', '--tag', help='the SapMachine tag', metavar='MAJOR', required=True) parser.add_argument('-a', '--asset', help='the asset to upload', metavar='ASSET', required=True) args = parser.parse_args() tag = args.tag asset = args.asset rc = 0 asset_file = os.path.realpath(asset) asset_name = os.path.basename(asset_file) asset_mime_type = mimetypes.guess_type(asset_file) if asset_mime_type is None or asset_mime_type[0] is None: asset_mime_type = 'application/octet-stream' print( str.format('could not detect mime-type: falling back to "{0}"', asset_mime_type)) else: asset_mime_type = asset_mime_type[0] print(str.format('detected mime-type "{0}"', asset_mime_type)) releases = utils.github_api_request(api='releases', url=None, owner='SAP', repository='async-profiler', data=None, method='GET', per_page=100, content_type=None, url_parameter=[]) release_id = None upload_url = None for release in releases: if release['tag_name'] == tag: release_id = release['id'] upload_url = release['upload_url'] break if release_id is None: print("Could not find release") return 1 if asset is not None: ''' asset file is specified (-a) first check wether the asset already exists ''' gh_assets = utils.github_api_request(api=str.format( 'releases/{0}/assets', release_id), url=None, owner='SAP', repository='async-profiler', data=None, method='GET', per_page=50, content_type=None, url_parameter=[]) for gh_asset in gh_assets: if gh_asset['name'] == asset_name: ''' asset already exists -> skip ''' print( str.format('error: asset "{0}" already exists ...', asset_name)) return 1 upload_url = str( upload_url.split('{', 1)[0] + '?name=' + quote(asset_name)) ''' read the contents of the asset file ''' with open(asset_file, 'rb') as asset_file: asset_data = asset_file.read() asset_length = len(asset_data) retry = 2 rc = -1 while retry > 0: try: ''' upload the asset file ''' print( str.format( 'uploading asset "{0}" with a length of {1} bytes ...', asset_name, str(asset_length))) utils.github_api_request(url=upload_url, data=asset_data, method='POST', content_type=asset_mime_type) rc = 0 break except IOError as e: # _type, value, _traceback = sys.exc_info() # traceback.print_exception(_type, value, _traceback) print( str.format('Error uploading asset "{0}": {1}', asset_name, e)) retry -= 1 except URLError as e: print( str.format('Error uploading asset "{0}": {1}', asset_name, e)) retry -= 1 return rc
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('-p', '--pull-request', help='the Pull Request number', metavar='NUMBER', required=True) parser.add_argument('-c', '--comment', help='if set, comment in PR', action='store_true', default=False) args = parser.parse_args() # request the pull request information pull_request = github_api_request( str.format('pulls/{0}', args.pull_request)) comments_url = pull_request['comments_url'] pr_author = pull_request['user']['login'] user = github_api_request(url=pull_request['user']['url']) if not is_pr_ok_to_test(pull_request): message = create_request_for_admin_comment(pr_author) print(str.format('Pull Request validation failed: "{0}"', message)) if args.comment: github_api_request(url=comments_url, data=message, method='POST') return 0 # if the author is in the exception list, the validation is skipped if pr_author not in pr_author_exception_list: # validate the pull request body (description) try: validate_pull_request(pull_request['body']) except Exception as error: message = create_failure_comment( pr_author, str.format( 'this pull request doesn\'t meet the expectations. {0}', error)) print(str.format('Pull Request validation failed: "{0}"', message)) if args.comment: github_api_request(url=comments_url, data=message, method='POST') return 0 # all formal requirements are met if args.comment: github_api_request(url=comments_url, data=create_success_comment(pr_author), method='POST') # check wether the complete validation has to run pull_request_files = github_api_request( str.format('pulls/{0}/files', args.pull_request)) roots_requiring_verification = [ 'make', 'src', 'test', 'Makefile', 'configure', '.github' ] requires_verification = False for pr_file in pull_request_files: print( str.format('Pull request changes file: "{0}"', pr_file['filename'])) root = next(part for part in pr_file['filename'].split(os.path.sep) if part) if root in roots_requiring_verification: requires_verification = True break if requires_verification: print("Pull Request requires further verification.") return 1 else: print("Pull Request requires no further verification.") return 2
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('--workdir', help='the temporary working directory', metavar='DIR', default="tags_work", required=False) args = parser.parse_args() workdir = os.path.realpath(args.workdir) utils.remove_if_exists(workdir) os.makedirs(workdir) sapmachine_latest = 0 sapmachine_branches = [] # fetch all branches branches = utils.github_api_request('branches', per_page=100) # iterate all branches of the SapMachine repository for branch in branches: # filter for sapmachine branches match = branch_pattern.match(branch['name']) if match is not None: if match.group(1) is not None: # found sapmachine branch major = int(match.group(1)) print( str.format( 'found sapmachine branch "{0}" with major "{1}"', branch['name'], major)) sapmachine_branches.append([branch['name'], major]) sapmachine_latest = max(sapmachine_latest, major) else: print( str.format('found sapmachine branch "{0}"', branch['name'])) sapmachine_branches.append([branch['name'], 0]) sapmachine_latest += 1 # set the major version for "sapmachine" branch which always contains the latest changes from "jdk/jdk" for sapmachine_branch in sapmachine_branches: if sapmachine_branch[1] is 0: sapmachine_branch[1] = sapmachine_latest # fetch all tags jdk_tags = {} tags = utils.github_api_request('tags', per_page=300) # iterate all tags for tag in tags: # filter for jdk tags match = JDKTag.jdk_tag_pattern.match(tag['name']) if match is not None: # found a jdk tag jdk_tag = JDKTag(match) major = jdk_tag.get_major() # only store the latest jdk tag (version comparison) if major not in jdk_tags or jdk_tag.is_greater_than( jdk_tags[major]): # filter jdk update tags with build number "0" like jdk-11.0.3+0 if not (jdk_tag.is_update() and jdk_tag.get_build_number() == 0): jdk_tags[major] = jdk_tag # clone the SapMachine repository git_target_dir = join(workdir, 'sapmachine') utils.git_clone('github.com/SAP/SapMachine.git', 'sapmachine', git_target_dir) # for each "sapmachine" branch, check whether it contains the latest jdk tag with matching major version for sapmachine_branch in sapmachine_branches: # get the latest jdk tag for this major version latest_tag_for_branch = jdk_tags[sapmachine_branch[1]] print( str.format('latest tag for branch "{0}" is "{1}"', sapmachine_branch, latest_tag_for_branch.as_string())) # check whether the jdk tag is already contained in the sapmachine branch _, out, _ = utils.run_cmd(str.format( 'git branch -a --contains tags/{0}', latest_tag_for_branch.as_string()).split(' '), cwd=git_target_dir, std=True, throw=False) match = re.search(containing_branches_pattern, out) if match is None: # the jdk tag is not contained i a sapmachine branch # create a pull request branch and a pull request. print( str.format( 'creating pull request "pr-{0}" with base branch "{1}"', latest_tag_for_branch.as_string(), sapmachine_branch)) utils.run_cmd( str.format('git checkout {0}', latest_tag_for_branch.as_string()).split(' '), cwd=git_target_dir) utils.run_cmd( str.format('git checkout -b pr-{0}', latest_tag_for_branch.as_string()).split(' '), cwd=git_target_dir) utils.run_cmd( str.format('git push origin pr-{0}', latest_tag_for_branch.as_string()).split(' '), cwd=git_target_dir) pull_request = str.format( '{{ "title": "Merge to tag {0}", "body": "please pull", "head": "pr-{1}", "base": "{2}" }}', latest_tag_for_branch.as_string(), latest_tag_for_branch.as_string(), sapmachine_branch[0]) utils.github_api_request('pulls', data=pull_request, method='POST') utils.remove_if_exists(workdir) return 0
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('-t', '--tag', help='the SapMachine tag', metavar='MAJOR', required=True) parser.add_argument('-d', '--description', help='the description of the release', required=False) parser.add_argument('-p', '--prerelease', help='this is a pre-release', action='store_true', default=False) parser.add_argument('-a', '--asset', help='the asset to upload', metavar='ASSET', required=False) args = parser.parse_args() tag = args.tag prerelease = args.prerelease asset = args.asset description = '' if args.description is None else args.description rc = 0 if asset is not None: asset_file = os.path.realpath(asset) asset_name = os.path.basename(asset_file) asset_mime_type = mimetypes.guess_type(asset_file) if asset_mime_type is None or asset_mime_type[0] is None: asset_mime_type = 'application/octet-stream' print(str.format('could not detect mime-type: falling back to "{0}"', asset_mime_type)) else: asset_mime_type = asset_mime_type[0] print(str.format('detected mime-type "{0}"', asset_mime_type)) releases = utils.get_github_releases() release_id = None upload_url = None for release in releases: if release['tag_name'] == tag: release_id = release['id'] upload_url = release['upload_url'] break if release_id is None: ''' release does not exist yet -> create it ''' data = json.dumps({ "tag_name": tag, "name": tag, "body": description, "draft": False, "prerelease": prerelease }) response = utils.github_api_request('releases', data=data, method='POST', content_type='application/json') release_id = response['id'] upload_url = response['upload_url'] print(str.format('created release "{0}"', tag)) if asset is not None: ''' asset file is specified (-a) first check wether the asset already exists ''' gh_assets = utils.github_api_request(str.format('releases/{0}/assets', release_id), per_page=50) for gh_asset in gh_assets: if gh_asset['name'] == asset_name: ''' asset already exists -> skip ''' print(str.format('error: asset "{0}" already exists ...', asset_name)) return 1 upload_url = str(upload_url.split('{', 1)[0] + '?name=' + quote(asset_name)) ''' read the contents of the asset file ''' with open(asset_file, 'rb') as asset_file: asset_data = asset_file.read() asset_length = len(asset_data) retry = 2 rc = -1 while retry > 0: try: ''' upload the asset file ''' print(str.format('uploading asset "{0}" with a length of {1} bytes ...', asset_name, str(asset_length))) utils.github_api_request(url=upload_url, data=asset_data, method='POST', content_type=asset_mime_type) rc = 0 break except IOError as e: # _type, value, _traceback = sys.exc_info() # traceback.print_exception(_type, value, _traceback) print(str.format('Error uploading asset "{0}": {1}', asset_name, e)) retry -= 1 except URLError as e: print(str.format('Error uploading asset "{0}": {1}', asset_name, e)) retry -= 1 return rc
def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument('--workdir', help='the temporary working directory', metavar='DIR', default="tags_work", required=False) args = parser.parse_args() workdir = os.path.realpath(args.workdir) utils.remove_if_exists(workdir) os.makedirs(workdir) pull_requests = utils.github_api_request('pulls', per_page=100, url_parameter=['state=all']) # clone the SapMachine repository git_target_dir = join(workdir, 'sapmachine') utils.git_clone('github.com/SAP/SapMachine.git', 'sapmachine', git_target_dir) # fetch all branches branches = utils.github_api_request('branches', per_page=100) sapmachine_latest = 0 sapmachine_branches = [] # iterate all branches of the SapMachine repository for branch in branches: # filter for sapmachine branches match = branch_pattern.match(branch['name']) if match is not None: # found sapmachine branch if match.group(1) is not None: major = int(match.group(1)) sapmachine_branches.append([branch['name'], major]) sapmachine_latest = max(major, sapmachine_latest) else: sapmachine_branches.append([branch['name'], 0]) sapmachine_latest += 1 for branch in sapmachine_branches: if branch[1] == 0: branch[1] = sapmachine_latest major = branch[1] utils.run_cmd(str.format('git checkout {0}', branch[0]).split(' '), cwd=git_target_dir) # find the last merge commit and check wether it is a merge from the jdk branch _, commit_messages, _ = utils.run_cmd( 'git log --merges -n 50 --format=%s'.split(' '), cwd=git_target_dir, std=True, throw=False) _, commit_ids, _ = utils.run_cmd( 'git log --merges -n 50 --format=%H'.split(' '), cwd=git_target_dir, std=True, throw=False) if commit_messages and commit_ids: commit_messages = [ commit_message for commit_message in commit_messages.split(os.linesep) if commit_message ] commit_ids = [ commit_id for commit_id in commit_ids.split(os.linesep) if commit_id ] merge_commits = map(lambda x, y: [x, y], commit_messages, commit_ids) for merge_commit in merge_commits: commit_message = merge_commit[0] commit_id = merge_commit[1] match_merge_commit = re.search(merge_commit_pattern, commit_message) match_jdk_tag = re.search(JDKTag.jdk_tag_pattern, commit_message) if match_merge_commit is not None and match_jdk_tag is not None: jdk_tag = JDKTag(match_jdk_tag) print( str.format( 'found latest merge commit "{0}" for branch "{1}"', commit_message, branch)) _, tags, _ = utils.run_cmd(str.format('git tag --contains {0}', commit_id).split(' '), cwd=git_target_dir, std=True, throw=False) if not tags: # not tagged yet # create sapmachine tag create_sapmachine_tag(jdk_tag, commit_id, git_target_dir) run_jenkins_jobs(major, jdk_tag.as_sapmachine_tag()) elif not jdk_tag.is_ga(): tags = tags.splitlines() # check wether there is a JDK GA tag which has no corresponding sapmachine tag yet # get the commit to which the most recent (before GA) tag is pointing to _, jdk_tag_commit, _ = utils.run_cmd(str.format( 'git rev-list -n 1 {0}', jdk_tag.as_string()).split(' '), cwd=git_target_dir, std=True, throw=False) if jdk_tag_commit: jdk_tag_commit = jdk_tag_commit.rstrip() # get all tags associated with the commit _, tags_for_commit, _ = utils.run_cmd( str.format('git tag --contains {0}', jdk_tag_commit).split(' '), cwd=git_target_dir, std=True, throw=False) if tags_for_commit: tags_for_commit = tags_for_commit.splitlines() # search for a GA tag for tag in tags_for_commit: match = re.search(JDKTag.jdk_tag_pattern, tag) if match: as_jdk_tag = JDKTag(match) if as_jdk_tag.is_ga( ) and as_jdk_tag.as_sapmachine_tag( ) not in tags: # GA tag found # check whether there is already a pull request for this tag pull_request_title = str.format( 'Merge to tag {0}', as_jdk_tag.as_string()) pull_request_exits = False for pull_request in pull_requests: if pull_request[ 'title'] == pull_request_title: # there is already a pull request for this tag # don't create sapmachine tag pull_request_exits = True break if not pull_request_exits: # create sapmachine tag create_sapmachine_tag( as_jdk_tag, commit_id, git_target_dir) run_jenkins_jobs( major, as_jdk_tag.as_sapmachine_tag()) break else: print('already tagged ...') break utils.remove_if_exists(workdir) return 0
def check_for_untagged_ga(jdk_tag, tags_of_merge_commit, merge_commit_id): # get the commit to which jdk_tag is pointing to global git_target_dir _, jdk_tag_commit, _ = utils.run_cmd(str.format( 'git rev-list -n 1 {0}', jdk_tag.as_string()).split(' '), cwd=git_target_dir, std=True, throw=False) if jdk_tag_commit is None: print( str.format( 'no commit found for JDK tag {0}, that should not happen', jdk_tag.as_string())) return # get all tags associated with the commit _, tags_of_jdk_tag_commit, _ = utils.run_cmd(str.format( 'git tag --contains {0}', jdk_tag_commit.rstrip()).split(' '), cwd=git_target_dir, std=True, throw=False) if tags_of_jdk_tag_commit is None: print( str.format( 'no tags found for commit of JDK tag {0}, that should not happen', jdk_tag.as_string())) return # search for a GA tag that matches jdk_tag's update version ga_tag = None for tag in tags_of_jdk_tag_commit.splitlines(): ga_tag = JDKTag.from_string(tag) if ga_tag is not None: if ga_tag.is_ga() and jdk_tag.is_same_update_version(ga_tag): break else: ga_tag = None if not ga_tag is None: print( str.format('found GA tag for JDK tag "{0}": {1}', jdk_tag.as_string(), ga_tag.as_string())) else: print( str.format('no GA tag found for JDK tag "{0}"', jdk_tag.as_string())) return # check whether this GA tag is already one of the tags of the merge commit sapmachine_ga_tag_string = ga_tag.as_sapmachine_tag_string() if sapmachine_ga_tag_string in tags_of_merge_commit: print( str.format('GA tag {0} already tagged as {1}', ga_tag.as_string(), sapmachine_ga_tag_string)) return # this merge commit is not contained in an existing corresponding sapmachine GA tag. # That's weird, but we don't change things now... if exists_tag(sapmachine_ga_tag_string): print( str.format( 'GA tag {0} is already tagged as {1} but does not include this merge commit. That could be a problem.', ga_tag.as_string(), sapmachine_ga_tag_string)) return # don't create a sapmachine tag if there is already a pull request for this tag pull_request_title = str.format('Merge to tag {0}', ga_tag.as_string()) pull_request_exits = False global pull_requests if pull_requests is None: pull_requests = utils.github_api_request('pulls', per_page=100, url_parameter=['state=all']) for pull_request in pull_requests: if pull_request['title'] == pull_request_title: pull_request_exits = True break if pull_request_exits: print( str.format('a pull request for GA tag {0} exists', ga_tag.as_string())) return # create sapmachine tag create_sapmachine_tag(ga_tag, merge_commit_id) # make sure the last tag before the GA tag has been built make_sure_last_non_ga_is_tagged(ga_tag, merge_commit_id)