def main(): ballerina_bot_token = os.environ[constants.ENV_BALLERINA_BOT_TOKEN] github = Github(ballerina_bot_token) repo = github.get_repo(constants.BALLERINA_ORG_NAME + '/' + sys.argv[1]) code_owner_content = repo.get_contents('.github/CODEOWNERS') owners = code_owner_content.decoded_content.decode().split("*")[1].split( "@") encryption_key = os.environ['ENV_USER_ENCRYPTION_KEY'] fernet = Fernet(encryption_key) with open('dependabot/resources/github_users_encrypted.csv', 'rb') as enc_file: encrypted_csv = enc_file.read() decrypted = fernet.decrypt(encrypted_csv) with open('dependabot/resources/github_users_decrypted.csv', 'wb') as dec_file: dec_file.write(decrypted) message = "*" + str(sys.argv[1]) + "* daily build failure" + "\n" +\ "Please visit <https://github.com/ballerina-platform/" + str(sys.argv[1]) + "/actions?query=workflow%3A%22Daily+build%22|the daily build page> for more information" +"\n" for owner in owners: with open('dependabot/resources/github_users_decrypted.csv', 'r') as read_obj: user_file = csv.DictReader(read_obj) owner = owner.strip() for row in user_file: if row['gh-username'] == owner: message += "<users/" + row['user-id'] + ">" + "\n" notify_chat.send_message(message)
def main(): global send_reminder_chat global lag_reminder_modules update_lang_version() updated_readme = get_updated_readme() # Write to local README file f = open(README_FILE, 'w') f.write(updated_readme) f.close() try: update_readme, commit = utils.commit_file( 'ballerina-release', README_FILE, updated_readme, constants.DASHBOARD_UPDATE_BRANCH, '[Automated] Update extension dependency dashboard') except GithubException as e: print('Error occurred while committing README.md', e) sys.exit(1) try: image = Image.open(constants.PIE_CHART_IMAGE, mode='r') img_byte_arr = io.BytesIO() image.save(img_byte_arr, format='JPEG') if update_readme: utils.commit_image_file('ballerina-release', constants.PIE_CHART_IMAGE, img_byte_arr.getvalue(), constants.DASHBOARD_UPDATE_BRANCH, '[Automated] Update status pie chart') except GithubException as e: print('Error occurred while committing status pie chart', e) sys.exit(1) if update_readme: utils.open_pr_and_merge( 'ballerina-release', '[Automated] Update Extension Dependency Dashboard', 'Update extension dependency dashboard', constants.DASHBOARD_UPDATE_BRANCH) if send_reminder_chat == 'true' and len(lag_reminder_modules) > 0: chat_message = distribution_lag_statement.replace( '<code>ballerina-distribution</code>', '*ballerina-distribution*') + ".\n\n" chat_message += "*Reminder* on the following modules\' dependency update..." + "\n" for module in lag_reminder_modules: lag_status_link = module[MODULE_PULL_REQUEST] if lag_status_link == "": lag_status_link = constants.BALLERINA_ORG_URL + module[ MODULE_NAME] chat_message += utils.get_module_message( module, lag_status_link) print("\n" + utils.get_sanitised_chat_message(chat_message)) notify_chat.send_message(chat_message) else: print('No changes to ' + README_FILE + ' file')
def main(): ballerina_bot_token = os.environ[constants.ENV_BALLERINA_BOT_TOKEN] github = Github(ballerina_bot_token) repo = github.get_repo(constants.BALLERINA_ORG_NAME + '/' + sys.argv[1]) code_owner_content = repo.get_contents('.github/CODEOWNERS') owners = code_owner_content.decoded_content.decode().split("*")[1].split("@") encryption_key = os.environ['ENV_USER_ENCRYPTION_KEY'] fernet = Fernet(encryption_key) with open('dependabot/resources/github_users_encrypted.csv', 'rb') as enc_file: encrypted_csv = enc_file.read() decrypted = fernet.decrypt(encrypted_csv) with open('dependabot/resources/github_users_decrypted.csv', 'wb') as dec_file: dec_file.write(decrypted) workflow_name_and_description = '%22Daily+build%22|the daily build page' message_body = "daily build failure" _, repo_name, workflow_name, github_action_type = sys.argv if workflow_name != "" : workflow_name_and_description = '%22' + workflow_name.replace(" ", "+") + '%22|' + 'the ' + workflow_name + ' page' if github_action_type == "notify-ballerinax-connector-build-failure" : message_body = "build using ballerina docker image failed" message = "*" + str(repo_name) + "* " + str(message_body) + "\n" +\ "Please visit <https://github.com/ballerina-platform/" + str(repo_name) +\ "/actions?query=workflow%3A" + str(workflow_name_and_description) + "> for more information" +"\n" for owner in owners : with open('dependabot/resources/github_users_decrypted.csv', 'r') as read_obj: user_file = csv.DictReader(read_obj) owner = owner.strip() for row in user_file: if row['gh-username'] == owner: message += "<users/" + row['user-id'] + ">" + "\n" notify_chat.send_message(message)
def update_lang_version(branch_name, lang_version): dist_repo = github.get_repo(constants.BALLERINA_ORG_NAME + '/ballerina-distribution', ref=branch_name) properties_content = dist_repo.get_contents(constants.GRADLE_PROPERTIES_FILE) properties_content = properties_content.decoded_content.decode(constants.ENCODING) synced = False updated_properties_file = '' module = {'name': 'ballerina-distribution', 'auto_merge': True} for line in properties_content.splitlines(): if line.startswith('ballerinaLangVersion'): if line.split('=')[1] == lang_version: print("[Info] lang version is already synced in " + branch_name + " branch") Synced = True break else: updated_properties_file += 'ballerinaLangVersion' + '=' + lang_version + '\n' else: updated_properties_file += line + '\n' if not synced: committed = commit_file(dist_repo, constants.GRADLE_PROPERTIES_FILE, updated_properties_file, branch_name, COMMIT_MESSAGE_PREFIX + lang_version) print("[Info] Update lang version in " + branch_name + " branch") if committed: pr = create_pull_request(dist_repo, branch_name) utils.approve_pr(module, 'TRUE', pr.number) ref = dist_repo.get_git_ref('heads/' + branch_name + "_temp") pending = True wait_cycles = 0 while pending: if wait_cycles < MAX_WAIT_CYCLES: time.sleep(SLEEP_INTERVAL) pending, passing, failing_pr_checks = check_pending_pr_checks(dist_repo, pr) if not pending: if len(failing_pr_checks) > 0: passing = all(check.startswith('codecov') for check in failing_pr_checks) if passing: if(pr.mergeable_state != 'dirty'): try: pr.merge() ref.delete() log_message = "[Info] Automated lang version update PR merged. PR: " + pr.html_url print(log_message) except Exception as e: print("[Error] Error occurred while merging version update PR " , e) else: notify_chat.send_message("[Info] Automated ballerina-distribution version update PR is unmerged due to conflicts." + "\n" +\ "Please visit <" + pr.html_url + "|the build page> for more information") else: notify_chat.send_message("[Info] Automated ballerina-distribution version update PR has failed checks." + "\n" +\ "Please visit <" + pr.html_url + "|the build page> for more information") pr.edit(state = 'closed') ref.delete() else: wait_cycles += 1 else: notify_chat.send_message("[Info] Automated ballerina-distribution version update PR is unmerged due to pr checks timeout." + "\n" +\ "Please visit <" + pr.html_url + "|the build page> for more information") break
def wait_for_current_level_build(level): global MAX_WAIT_CYCLES print("[Info] Waiting for level '" + str(level) + "' module build.") total_modules = len(current_level_modules) if level == 5: # In level 5 http takes around 30 min for PR build and build each # Changes timeout to 80 minutes MAX_WAIT_CYCLES = 140 if level == 6: # In level 6 c2c takes around 40 min for PR build and build each # Changes timeout to 100 minutes MAX_WAIT_CYCLES = 200 wait_cycles = 0 global status_completed_modules status_completed_modules = 0 while status_completed_modules != total_modules: for idx, module in enumerate(current_level_modules): if module[MODULE_STATUS] == MODULE_STATUS_IN_PROGRESS: if module[MODULE_CONCLUSION] == MODULE_CONCLUSION_PR_PENDING: check_pending_pr_checks(idx) else: # Build checks test check_pending_build_checks(idx) if wait_cycles < MAX_WAIT_CYCLES: time.sleep(SLEEP_INTERVAL) wait_cycles = wait_cycles + 1 else: # Force stop script with all in progress modules printed print( 'Dependency bump script timed out. Following modules are in pending state' ) for module in current_level_modules: if module[MODULE_STATUS] == MODULE_STATUS_IN_PROGRESS: print(module['name']) sys.exit(1) module_release_failure = False chat_message = "Dependency update to lang version \'" + lang_version + "\'.\n" pr_checks_failed_modules = list( filter( lambda s: s[MODULE_CONCLUSION] == MODULE_CONCLUSION_PR_CHECK_FAILURE, current_level_modules)) if len(pr_checks_failed_modules) != 0: module_release_failure = True chat_message += 'Following modules\' Automated Dependency Update PRs have failed checks...' + "\n" for module in pr_checks_failed_modules: chat_message += "<" + module[ MODULE_CREATED_PR].html_url + "|" + module['name'] + ">" + "\n" pr_merged_failed_modules = list( filter( lambda s: s[MODULE_CONCLUSION] == MODULE_CONCLUSION_PR_MERGE_FAILURE, current_level_modules)) if len(pr_merged_failed_modules) != 0: module_release_failure = True chat_message += 'Following modules\' Automated Dependency Update PRs could not be merged...' + "\n" for module in pr_merged_failed_modules: chat_message += "<" + module[ MODULE_CREATED_PR].html_url + "|" + module['name'] + ">" + "\n" build_checks_failed_modules = list( filter( lambda s: s[MODULE_CONCLUSION] == MODULE_CONCLUSION_BUILD_FAILURE, current_level_modules)) if len(build_checks_failed_modules) != 0: module_release_failure = True chat_message += 'Following modules\' Timestamped Build checks have failed...' + "\n" for module in build_checks_failed_modules: build_actions_page = constants.BALLERINA_ORG_URL + module['name'] + "/actions/workflows/" + \ module[MODULE_BUILD_ACTION_FILE] + ".yml" chat_message += "<" + build_actions_page + "|" + module[ 'name'] + ">" + "\n" build_version_failed_modules = list( filter( lambda s: s[MODULE_CONCLUSION ] == MODULE_CONCLUSION_VERSION_CANNOT_BE_IDENTIFIED, current_level_modules)) if len(build_version_failed_modules) != 0: module_release_failure = True chat_message += 'Following modules\' latest Timestamped Build Version cannot be identified...' + "\n" for module in build_version_failed_modules: build_actions_page = constants.BALLERINA_ORG_URL + module['name'] + "/actions/workflows/" + \ module[MODULE_BUILD_ACTION_FILE] + ".yml" chat_message += "<" + build_actions_page + "|" + module[ 'name'] + ">" + "\n" if module_release_failure: print(chat_message) chat_message += "After following up on the above, retrigger the <" + \ "https://github.com/ballerina-platform/ballerina-release/actions/workflows/update_dependency_version.yml" + \ "|Dependency Update Workflow>" notify_chat.send_message(chat_message) sys.exit(1)
def main(): global lang_version global extensions_file global all_modules global current_level_modules try: extensions_file = utils.read_json_file(constants.EXTENSIONS_FILE) except Exception as e: print('[Error] Error while loading modules list ', e) sys.exit(1) print("Workflow invoked of type '" + event_type + "'") if event_type == 'schedule' and not extensions_file['auto_bump']: print( "Schedule workflow invoked, exiting script as 'auto_bump' flag in modules_list.json is false." ) return if override_ballerina_version != '': lang_version = override_ballerina_version else: lang_version = utils.get_latest_lang_version() bal_version = {'version': lang_version} try: utils.write_json_file(constants.LANG_VERSION_FILE, bal_version) except Exception as e: print('Failed to write to file latest_ballerina_lang_version.json', e) sys.exit() try: updated_file_content = open(constants.LANG_VERSION_FILE, 'r').read() update = utils.commit_file( 'ballerina-release', constants.LANG_VERSION_FILE, updated_file_content, constants.EXTENSIONS_UPDATE_BRANCH, '[Automated] Update Workflow Lang Version')[0] if update: utils.open_pr_and_merge( 'ballerina-release', '[Automated] Update Dependency Bump Workflow Triggered Version', 'Update bumped ballerina lang version', constants.EXTENSIONS_UPDATE_BRANCH) else: print('No changes to ' + constants.LANG_VERSION_FILE + ' file') except GithubException as e: print( 'Error occurred while committing latest_ballerinalang_version.md', e) sys.exit(1) print('Workflow started with Ballerina Lang version : ' + lang_version) all_modules = extensions_file['modules'] last_level = all_modules[-1]['level'] print( 'Start dependency bump to extensions packed in ballerina-distribution') for i in range(last_level): current_level = i + 1 current_level_modules = list( filter(lambda s: s['level'] == current_level, all_modules)) for idx, module in enumerate(current_level_modules): print("[Info] Check lang dependency in module '" + module['name'] + "'") update_module(idx, current_level) if auto_merge_pull_requests.lower() == 'true': module_release_failure, chat_message = wait_for_current_level_build( current_level) if module_release_failure: print(chat_message) chat_message += "After following up on the above, retrigger the <" + \ "https://github.com/ballerina-platform/ballerina-release/actions/workflows/update_dependency_version.yml" + \ "|Dependency Update Workflow>" notify_chat.send_message(chat_message) sys.exit(1) print( 'Successfully bumped dependencies in extensions packed in ballerina-distribution' ) central_module_level = extensions_file['central_modules'][-1]['level'] print('Start dependency bump to extensions available only in central') for j in range(last_level, central_module_level): current_level = j + 1 current_level_modules = list( filter(lambda s: s['level'] == current_level, extensions_file['central_modules'])) for idx, module in enumerate(current_level_modules): print("[Info] Check lang dependency in module '" + module['name'] + "'") update_module(idx, current_level) if auto_merge_pull_requests.lower() == 'true': _, _ = wait_for_current_level_build(current_level) print( 'Successfully bumped dependencies in extensions available in central')
def wait_for_current_level_build(level, is_stdlib_module): global MAX_WAIT_CYCLES global send_notification global status_completed_modules print("[Info] Waiting for level '" + str(level) + "' module build.") total_modules = len(current_level_modules) if level == 5: # In level 5 http takes around 30 min for PR build and build each # Changes timeout to 80 minutes MAX_WAIT_CYCLES = 140 if level == 6: # In level 6 c2c takes around 52 min for PR build and build each # Changes timeout to 140 minutes MAX_WAIT_CYCLES = 280 wait_cycles = 0 status_completed_modules = 0 while status_completed_modules != total_modules: for idx, module in enumerate(current_level_modules): if module[MODULE_STATUS] == MODULE_STATUS_IN_PROGRESS: if module[MODULE_CONCLUSION] == MODULE_CONCLUSION_PR_PENDING: check_pending_pr_checks(idx) else: # Build checks test check_pending_build_checks(idx) if wait_cycles < MAX_WAIT_CYCLES: time.sleep(SLEEP_INTERVAL) wait_cycles = wait_cycles + 1 else: # Force stop script with all in progress modules printed print( 'Dependency update script timed out. Following modules are in pending state' ) for module in current_level_modules: if module[MODULE_STATUS] == MODULE_STATUS_IN_PROGRESS: print(module['name']) sys.exit(1) module_release_failure = False chat_message_send = False chat_message = "Dependency update to lang version *" + lang_version + "*\n\n" print() pr_checks_failed_modules = list( filter( lambda s: s[MODULE_CONCLUSION] == MODULE_CONCLUSION_PR_CHECK_FAILURE, current_level_modules)) if len(pr_checks_failed_modules) != 0: module_release_failure = True pr_failed_message = 'Following modules\' Automated Dependency Update PRs have failed checks...' send_chat, partial_chat_message = get_chat_message( pr_checks_failed_modules, pr_failed_message, True) chat_message_send = chat_message_send or send_chat chat_message += partial_chat_message pr_merged_failed_modules = list( filter( lambda s: s[MODULE_CONCLUSION] == MODULE_CONCLUSION_PR_MERGE_FAILURE, current_level_modules)) if len(pr_merged_failed_modules) != 0: module_release_failure = True pr_merged_failed_message = 'Following modules\' Automated Dependency Update PRs could not be merged...' send_chat, partial_chat_message = get_chat_message( pr_merged_failed_modules, pr_merged_failed_message, True) chat_message_send = chat_message_send or send_chat chat_message += partial_chat_message build_checks_failed_modules = list( filter( lambda s: s[MODULE_CONCLUSION] == MODULE_CONCLUSION_BUILD_FAILURE, current_level_modules)) if len(build_checks_failed_modules) != 0: module_release_failure = True build_checks_failed_message = 'Following modules\' Timestamped Build checks have failed...' send_chat, partial_chat_message = get_chat_message( build_checks_failed_modules, build_checks_failed_message, False) chat_message_send = chat_message_send or send_chat chat_message += partial_chat_message build_version_failed_modules = list( filter( lambda s: s[MODULE_CONCLUSION ] == MODULE_CONCLUSION_VERSION_CANNOT_BE_IDENTIFIED, current_level_modules)) if len(build_version_failed_modules) != 0: module_release_failure = True build_version_failed_message = 'Following modules\' latest Timestamped Build Version cannot be identified...' send_chat, partial_chat_message = get_chat_message( build_version_failed_modules, build_version_failed_message, False) chat_message_send = chat_message_send or send_chat chat_message += partial_chat_message if is_stdlib_module: chat_message += "After following up on the above, trigger the <" + \ "https://github.com/ballerina-platform/ballerina-release/actions/workflows/update_dependency_version.yml" + \ "|Dependency Update Workflow>" if send_notification == 'true' and chat_message_send: print('Failing modules that is being notified:') print(utils.get_sanitised_chat_message(chat_message)) notify_chat.send_message(chat_message) elif chat_message_send: print('Failing modules that is NOT being notified:') print(utils.get_sanitised_chat_message(chat_message)) if module_release_failure: sys.exit(1)
def wait_for_current_level_build(level): global MAX_WAIT_CYCLES print("[Info] Waiting for level '" + str(level) + "' module build.") total_modules = len(current_level_modules) if level == 5: # In level 5 http takes around 30 min for PR build and build each # Changes timeout to 80 minutes MAX_WAIT_CYCLES = 140 if level == 6: # In level 6 c2c takes around 40 min for PR build and build each # Changes timeout to 100 minutes MAX_WAIT_CYCLES = 200 wait_cycles = 0 global status_completed_modules status_completed_modules = 0 while status_completed_modules != total_modules: for idx, module in enumerate(current_level_modules): if module[MODULE_STATUS] == MODULE_STATUS_IN_PROGRESS: if module[MODULE_CONCLUSION] == MODULE_CONCLUSION_PR_PENDING: check_pending_pr_checks(idx) else: # Build checks test check_pending_build_checks(idx) if wait_cycles < MAX_WAIT_CYCLES: time.sleep(SLEEP_INTERVAL) wait_cycles = wait_cycles + 1 else: # Force stop script with all in progress modules printed print('Dependency bump script timed out. Following modules are in pending state') for module in current_level_modules: if module[MODULE_STATUS] == MODULE_STATUS_IN_PROGRESS: print(module['name']) sys.exit(1) module_release_failure = False chat_message = "" pr_checks_failed_modules = list( filter(lambda s: s[MODULE_CONCLUSION] == MODULE_CONCLUSION_PR_CHECK_FAILURE, current_level_modules)) if len(pr_checks_failed_modules) != 0: module_release_failure = True print('Following modules dependency PRs have failed checks...') chat_message += 'Following modules dependency PRs have failed checks...' + "\n" for module in pr_checks_failed_modules: print(module['name']) chat_message += module['name'] + "\n" pr_merged_failed_modules = list( filter(lambda s: s[MODULE_CONCLUSION] == MODULE_CONCLUSION_PR_MERGE_FAILURE, current_level_modules)) if len(pr_merged_failed_modules) != 0: module_release_failure = True chat_message += 'Following modules dependency PRs could not be merged...' + "\n" for module in pr_merged_failed_modules: chat_message += module['name'] + "\n" build_checks_failed_modules = list( filter(lambda s: s[MODULE_CONCLUSION] == MODULE_CONCLUSION_BUILD_FAILURE, current_level_modules)) if len(build_checks_failed_modules) != 0: module_release_failure = True chat_message += 'Following modules timestamped build checks failed...' + "\n" for module in build_checks_failed_modules: chat_message += module['name'] + "\n" build_version_failed_modules = list( filter(lambda s: s[MODULE_CONCLUSION] == MODULE_CONCLUSION_VERSION_CANNOT_BE_IDENTIFIED, current_level_modules)) if len(build_version_failed_modules) != 0: module_release_failure = True chat_message += 'Following modules timestamped build version cannot be identified...' + "\n" for module in build_version_failed_modules: chat_message += module['name'] + "\n" if module_release_failure: print(chat_message) notify_chat.send_message(chat_message) sys.exit(1)
def main(): repo = github.get_repo(constants.BALLERINA_ORG_NAME + '/' + 'ballerina-lang') branches = repo.get_branches() temp_branch = 'sync-2201.1.x-' + date.today().strftime("%d-%m-%Y") for branch in branches: if (branch.name == '2201.1.x'): patch_branch = branch break # Check whether master branch already has an unmerged PR from temporary branch, delete if exists pulls = repo.get_pulls(state='open') for pull in pulls: if (pull.head.ref == temp_branch): print("Master branch already has an open pull request from " + temp_branch) pull.edit(state='closed') break # if temporary branch exists, delete it for branch in branches: if branch.name == temp_branch: ref = repo.get_git_ref('heads/' + temp_branch) ref.delete() break # create the temporary branch from patch branch repo.create_git_ref(ref='refs/heads/' + temp_branch, sha=patch_branch.commit.sha) ref = repo.get_git_ref('heads/' + temp_branch) pr = create_pull_request(repo, temp_branch) time.sleep(60) pending = True wait_cycles = 0 if (pr.mergeable_state != 'dirty'): while pending: if wait_cycles < MAX_WAIT_CYCLES: time.sleep(SLEEP_INTERVAL) pending, passing, failing_pr_checks = check_pending_pr_checks( repo, pr) if not pending: if len(failing_pr_checks) > 0: passing = all( check.startswith('codecov') for check in failing_pr_checks) if passing: try: pr.merge() ref.delete() log_message = "[Info] Automated master update PR merged. PR: " + pr.html_url print(log_message) except Exception as e: print( "[Error] Error occurred while merging master update PR ", e) else: notify_chat.send_message("[Info] Automated ballerina-lang master update PR has failed checks." + "\n" +\ "Please visit <" + pr.html_url + "|the build page> for more information") pr.edit(state='closed') ref.delete() else: wait_cycles += 1 else: notify_chat.send_message("[Info] Automated ballerina-lang master update PR is unmerged due to pr checks timeout." + "\n" +\ "Please visit <" + pr.html_url + "|the build page> for more information") break else: notify_chat.send_message("[Info] Automated ballerina-lang master update PR is unmerged due to conflicts with the master." + "\n" +\ "Please visit <" + pr.html_url + "|the build page> for more information")