def main():
ballerina_bot_token = os.environ[constants.ENV_BALLERINA_BOT_TOKEN]
github = Github(ballerina_bot_token)
repo = github.get_repo(constants.BALLERINA_ORG_NAME + '/' + sys.argv[1])
code_owner_content = repo.get_contents('.github/CODEOWNERS')
owners = code_owner_content.decoded_content.decode().split("*")[1].split(
"@")
encryption_key = os.environ['ENV_USER_ENCRYPTION_KEY']
fernet = Fernet(encryption_key)
with open('dependabot/resources/github_users_encrypted.csv',
'rb') as enc_file:
encrypted_csv = enc_file.read()
decrypted = fernet.decrypt(encrypted_csv)
with open('dependabot/resources/github_users_decrypted.csv',
'wb') as dec_file:
dec_file.write(decrypted)
message = "*" + str(sys.argv[1]) + "* daily build failure" + "\n" +\
"Please visit <https://github.com/ballerina-platform/" + str(sys.argv[1]) + "/actions?query=workflow%3A%22Daily+build%22|the daily build page> for more information" +"\n"
for owner in owners:
with open('dependabot/resources/github_users_decrypted.csv',
'r') as read_obj:
user_file = csv.DictReader(read_obj)
owner = owner.strip()
for row in user_file:
if row['gh-username'] == owner:
message += "<users/" + row['user-id'] + ">" + "\n"
notify_chat.send_message(message)
def main():
global send_reminder_chat
global lag_reminder_modules
update_lang_version()
updated_readme = get_updated_readme()
# Write to local README file
f = open(README_FILE, 'w')
f.write(updated_readme)
f.close()
try:
update_readme, commit = utils.commit_file(
'ballerina-release', README_FILE, updated_readme,
constants.DASHBOARD_UPDATE_BRANCH,
'[Automated] Update extension dependency dashboard')
except GithubException as e:
print('Error occurred while committing README.md', e)
sys.exit(1)
try:
image = Image.open(constants.PIE_CHART_IMAGE, mode='r')
img_byte_arr = io.BytesIO()
image.save(img_byte_arr, format='JPEG')
if update_readme:
utils.commit_image_file('ballerina-release',
constants.PIE_CHART_IMAGE,
img_byte_arr.getvalue(),
constants.DASHBOARD_UPDATE_BRANCH,
'[Automated] Update status pie chart')
except GithubException as e:
print('Error occurred while committing status pie chart', e)
sys.exit(1)
if update_readme:
utils.open_pr_and_merge(
'ballerina-release',
'[Automated] Update Extension Dependency Dashboard',
'Update extension dependency dashboard',
constants.DASHBOARD_UPDATE_BRANCH)
if send_reminder_chat == 'true' and len(lag_reminder_modules) > 0:
chat_message = distribution_lag_statement.replace(
'<code>ballerina-distribution</code>',
'*ballerina-distribution*') + ".\n\n"
chat_message += "*Reminder* on the following modules\' dependency update..." + "\n"
for module in lag_reminder_modules:
lag_status_link = module[MODULE_PULL_REQUEST]
if lag_status_link == "":
lag_status_link = constants.BALLERINA_ORG_URL + module[
MODULE_NAME]
chat_message += utils.get_module_message(
module, lag_status_link)
print("\n" + utils.get_sanitised_chat_message(chat_message))
notify_chat.send_message(chat_message)
else:
print('No changes to ' + README_FILE + ' file')
def main():
ballerina_bot_token = os.environ[constants.ENV_BALLERINA_BOT_TOKEN]
github = Github(ballerina_bot_token)
repo = github.get_repo(constants.BALLERINA_ORG_NAME + '/' + sys.argv[1])
code_owner_content = repo.get_contents('.github/CODEOWNERS')
owners = code_owner_content.decoded_content.decode().split("*")[1].split("@")
encryption_key = os.environ['ENV_USER_ENCRYPTION_KEY']
fernet = Fernet(encryption_key)
with open('dependabot/resources/github_users_encrypted.csv', 'rb') as enc_file:
encrypted_csv = enc_file.read()
decrypted = fernet.decrypt(encrypted_csv)
with open('dependabot/resources/github_users_decrypted.csv', 'wb') as dec_file:
dec_file.write(decrypted)
workflow_name_and_description = '%22Daily+build%22|the daily build page'
message_body = "daily build failure"
_, repo_name, workflow_name, github_action_type = sys.argv
if workflow_name != "" :
workflow_name_and_description = '%22' + workflow_name.replace(" ", "+") + '%22|' + 'the ' + workflow_name + ' page'
if github_action_type == "notify-ballerinax-connector-build-failure" :
message_body = "build using ballerina docker image failed"
message = "*" + str(repo_name) + "* " + str(message_body) + "\n" +\
"Please visit <https://github.com/ballerina-platform/" + str(repo_name) +\
"/actions?query=workflow%3A" + str(workflow_name_and_description) + "> for more information" +"\n"
for owner in owners :
with open('dependabot/resources/github_users_decrypted.csv', 'r') as read_obj:
user_file = csv.DictReader(read_obj)
owner = owner.strip()
for row in user_file:
if row['gh-username'] == owner:
message += "<users/" + row['user-id'] + ">" + "\n"
notify_chat.send_message(message)
def update_lang_version(branch_name, lang_version):
dist_repo = github.get_repo(constants.BALLERINA_ORG_NAME + '/ballerina-distribution', ref=branch_name)
properties_content = dist_repo.get_contents(constants.GRADLE_PROPERTIES_FILE)
properties_content = properties_content.decoded_content.decode(constants.ENCODING)
synced = False
updated_properties_file = ''
module = {'name': 'ballerina-distribution', 'auto_merge': True}
for line in properties_content.splitlines():
if line.startswith('ballerinaLangVersion'):
if line.split('=')[1] == lang_version:
print("[Info] lang version is already synced in " + branch_name + " branch")
Synced = True
break
else:
updated_properties_file += 'ballerinaLangVersion' + '=' + lang_version + '\n'
else:
updated_properties_file += line + '\n'
if not synced:
committed = commit_file(dist_repo, constants.GRADLE_PROPERTIES_FILE, updated_properties_file,
branch_name, COMMIT_MESSAGE_PREFIX + lang_version)
print("[Info] Update lang version in " + branch_name + " branch")
if committed:
pr = create_pull_request(dist_repo, branch_name)
utils.approve_pr(module, 'TRUE', pr.number)
ref = dist_repo.get_git_ref('heads/' + branch_name + "_temp")
pending = True
wait_cycles = 0
while pending:
if wait_cycles < MAX_WAIT_CYCLES:
time.sleep(SLEEP_INTERVAL)
pending, passing, failing_pr_checks = check_pending_pr_checks(dist_repo, pr)
if not pending:
if len(failing_pr_checks) > 0:
passing = all(check.startswith('codecov') for check in failing_pr_checks)
if passing:
if(pr.mergeable_state != 'dirty'):
try:
pr.merge()
ref.delete()
log_message = "[Info] Automated lang version update PR merged. PR: " + pr.html_url
print(log_message)
except Exception as e:
print("[Error] Error occurred while merging version update PR " , e)
else:
notify_chat.send_message("[Info] Automated ballerina-distribution version update PR is unmerged due to conflicts." + "\n" +\
"Please visit <" + pr.html_url + "|the build page> for more information")
else:
notify_chat.send_message("[Info] Automated ballerina-distribution version update PR has failed checks." + "\n" +\
"Please visit <" + pr.html_url + "|the build page> for more information")
pr.edit(state = 'closed')
ref.delete()
else:
wait_cycles += 1
else:
notify_chat.send_message("[Info] Automated ballerina-distribution version update PR is unmerged due to pr checks timeout." + "\n" +\
"Please visit <" + pr.html_url + "|the build page> for more information")
break
def wait_for_current_level_build(level):
global MAX_WAIT_CYCLES
print("[Info] Waiting for level '" + str(level) + "' module build.")
total_modules = len(current_level_modules)
if level == 5:
# In level 5 http takes around 30 min for PR build and build each
# Changes timeout to 80 minutes
MAX_WAIT_CYCLES = 140
if level == 6:
# In level 6 c2c takes around 40 min for PR build and build each
# Changes timeout to 100 minutes
MAX_WAIT_CYCLES = 200
wait_cycles = 0
global status_completed_modules
status_completed_modules = 0
while status_completed_modules != total_modules:
for idx, module in enumerate(current_level_modules):
if module[MODULE_STATUS] == MODULE_STATUS_IN_PROGRESS:
if module[MODULE_CONCLUSION] == MODULE_CONCLUSION_PR_PENDING:
check_pending_pr_checks(idx)
else:
# Build checks test
check_pending_build_checks(idx)
if wait_cycles < MAX_WAIT_CYCLES:
time.sleep(SLEEP_INTERVAL)
wait_cycles = wait_cycles + 1
else:
# Force stop script with all in progress modules printed
print(
'Dependency bump script timed out. Following modules are in pending state'
)
for module in current_level_modules:
if module[MODULE_STATUS] == MODULE_STATUS_IN_PROGRESS:
print(module['name'])
sys.exit(1)
module_release_failure = False
chat_message = "Dependency update to lang version \'" + lang_version + "\'.\n"
pr_checks_failed_modules = list(
filter(
lambda s: s[MODULE_CONCLUSION] ==
MODULE_CONCLUSION_PR_CHECK_FAILURE, current_level_modules))
if len(pr_checks_failed_modules) != 0:
module_release_failure = True
chat_message += 'Following modules\' Automated Dependency Update PRs have failed checks...' + "\n"
for module in pr_checks_failed_modules:
chat_message += "<" + module[
MODULE_CREATED_PR].html_url + "|" + module['name'] + ">" + "\n"
pr_merged_failed_modules = list(
filter(
lambda s: s[MODULE_CONCLUSION] ==
MODULE_CONCLUSION_PR_MERGE_FAILURE, current_level_modules))
if len(pr_merged_failed_modules) != 0:
module_release_failure = True
chat_message += 'Following modules\' Automated Dependency Update PRs could not be merged...' + "\n"
for module in pr_merged_failed_modules:
chat_message += "<" + module[
MODULE_CREATED_PR].html_url + "|" + module['name'] + ">" + "\n"
build_checks_failed_modules = list(
filter(
lambda s: s[MODULE_CONCLUSION] == MODULE_CONCLUSION_BUILD_FAILURE,
current_level_modules))
if len(build_checks_failed_modules) != 0:
module_release_failure = True
chat_message += 'Following modules\' Timestamped Build checks have failed...' + "\n"
for module in build_checks_failed_modules:
build_actions_page = constants.BALLERINA_ORG_URL + module['name'] + "/actions/workflows/" + \
module[MODULE_BUILD_ACTION_FILE] + ".yml"
chat_message += "<" + build_actions_page + "|" + module[
'name'] + ">" + "\n"
build_version_failed_modules = list(
filter(
lambda s: s[MODULE_CONCLUSION
] == MODULE_CONCLUSION_VERSION_CANNOT_BE_IDENTIFIED,
current_level_modules))
if len(build_version_failed_modules) != 0:
module_release_failure = True
chat_message += 'Following modules\' latest Timestamped Build Version cannot be identified...' + "\n"
for module in build_version_failed_modules:
build_actions_page = constants.BALLERINA_ORG_URL + module['name'] + "/actions/workflows/" + \
module[MODULE_BUILD_ACTION_FILE] + ".yml"
chat_message += "<" + build_actions_page + "|" + module[
'name'] + ">" + "\n"
if module_release_failure:
print(chat_message)
chat_message += "After following up on the above, retrigger the <" + \
"https://github.com/ballerina-platform/ballerina-release/actions/workflows/update_dependency_version.yml" + \
"|Dependency Update Workflow>"
notify_chat.send_message(chat_message)
sys.exit(1)
def main():
global lang_version
global extensions_file
global all_modules
global current_level_modules
try:
extensions_file = utils.read_json_file(constants.EXTENSIONS_FILE)
except Exception as e:
print('[Error] Error while loading modules list ', e)
sys.exit(1)
print("Workflow invoked of type '" + event_type + "'")
if event_type == 'schedule' and not extensions_file['auto_bump']:
print(
"Schedule workflow invoked, exiting script as 'auto_bump' flag in modules_list.json is false."
)
return
if override_ballerina_version != '':
lang_version = override_ballerina_version
else:
lang_version = utils.get_latest_lang_version()
bal_version = {'version': lang_version}
try:
utils.write_json_file(constants.LANG_VERSION_FILE, bal_version)
except Exception as e:
print('Failed to write to file latest_ballerina_lang_version.json', e)
sys.exit()
try:
updated_file_content = open(constants.LANG_VERSION_FILE, 'r').read()
update = utils.commit_file(
'ballerina-release', constants.LANG_VERSION_FILE,
updated_file_content, constants.EXTENSIONS_UPDATE_BRANCH,
'[Automated] Update Workflow Lang Version')[0]
if update:
utils.open_pr_and_merge(
'ballerina-release',
'[Automated] Update Dependency Bump Workflow Triggered Version',
'Update bumped ballerina lang version',
constants.EXTENSIONS_UPDATE_BRANCH)
else:
print('No changes to ' + constants.LANG_VERSION_FILE + ' file')
except GithubException as e:
print(
'Error occurred while committing latest_ballerinalang_version.md',
e)
sys.exit(1)
print('Workflow started with Ballerina Lang version : ' + lang_version)
all_modules = extensions_file['modules']
last_level = all_modules[-1]['level']
print(
'Start dependency bump to extensions packed in ballerina-distribution')
for i in range(last_level):
current_level = i + 1
current_level_modules = list(
filter(lambda s: s['level'] == current_level, all_modules))
for idx, module in enumerate(current_level_modules):
print("[Info] Check lang dependency in module '" + module['name'] +
"'")
update_module(idx, current_level)
if auto_merge_pull_requests.lower() == 'true':
module_release_failure, chat_message = wait_for_current_level_build(
current_level)
if module_release_failure:
print(chat_message)
chat_message += "After following up on the above, retrigger the <" + \
"https://github.com/ballerina-platform/ballerina-release/actions/workflows/update_dependency_version.yml" + \
"|Dependency Update Workflow>"
notify_chat.send_message(chat_message)
sys.exit(1)
print(
'Successfully bumped dependencies in extensions packed in ballerina-distribution'
)
central_module_level = extensions_file['central_modules'][-1]['level']
print('Start dependency bump to extensions available only in central')
for j in range(last_level, central_module_level):
current_level = j + 1
current_level_modules = list(
filter(lambda s: s['level'] == current_level,
extensions_file['central_modules']))
for idx, module in enumerate(current_level_modules):
print("[Info] Check lang dependency in module '" + module['name'] +
"'")
update_module(idx, current_level)
if auto_merge_pull_requests.lower() == 'true':
_, _ = wait_for_current_level_build(current_level)
print(
'Successfully bumped dependencies in extensions available in central')
def wait_for_current_level_build(level, is_stdlib_module):
global MAX_WAIT_CYCLES
global send_notification
global status_completed_modules
print("[Info] Waiting for level '" + str(level) + "' module build.")
total_modules = len(current_level_modules)
if level == 5:
# In level 5 http takes around 30 min for PR build and build each
# Changes timeout to 80 minutes
MAX_WAIT_CYCLES = 140
if level == 6:
# In level 6 c2c takes around 52 min for PR build and build each
# Changes timeout to 140 minutes
MAX_WAIT_CYCLES = 280
wait_cycles = 0
status_completed_modules = 0
while status_completed_modules != total_modules:
for idx, module in enumerate(current_level_modules):
if module[MODULE_STATUS] == MODULE_STATUS_IN_PROGRESS:
if module[MODULE_CONCLUSION] == MODULE_CONCLUSION_PR_PENDING:
check_pending_pr_checks(idx)
else:
# Build checks test
check_pending_build_checks(idx)
if wait_cycles < MAX_WAIT_CYCLES:
time.sleep(SLEEP_INTERVAL)
wait_cycles = wait_cycles + 1
else:
# Force stop script with all in progress modules printed
print(
'Dependency update script timed out. Following modules are in pending state'
)
for module in current_level_modules:
if module[MODULE_STATUS] == MODULE_STATUS_IN_PROGRESS:
print(module['name'])
sys.exit(1)
module_release_failure = False
chat_message_send = False
chat_message = "Dependency update to lang version *" + lang_version + "*\n\n"
print()
pr_checks_failed_modules = list(
filter(
lambda s: s[MODULE_CONCLUSION] ==
MODULE_CONCLUSION_PR_CHECK_FAILURE, current_level_modules))
if len(pr_checks_failed_modules) != 0:
module_release_failure = True
pr_failed_message = 'Following modules\' Automated Dependency Update PRs have failed checks...'
send_chat, partial_chat_message = get_chat_message(
pr_checks_failed_modules, pr_failed_message, True)
chat_message_send = chat_message_send or send_chat
chat_message += partial_chat_message
pr_merged_failed_modules = list(
filter(
lambda s: s[MODULE_CONCLUSION] ==
MODULE_CONCLUSION_PR_MERGE_FAILURE, current_level_modules))
if len(pr_merged_failed_modules) != 0:
module_release_failure = True
pr_merged_failed_message = 'Following modules\' Automated Dependency Update PRs could not be merged...'
send_chat, partial_chat_message = get_chat_message(
pr_merged_failed_modules, pr_merged_failed_message, True)
chat_message_send = chat_message_send or send_chat
chat_message += partial_chat_message
build_checks_failed_modules = list(
filter(
lambda s: s[MODULE_CONCLUSION] == MODULE_CONCLUSION_BUILD_FAILURE,
current_level_modules))
if len(build_checks_failed_modules) != 0:
module_release_failure = True
build_checks_failed_message = 'Following modules\' Timestamped Build checks have failed...'
send_chat, partial_chat_message = get_chat_message(
build_checks_failed_modules, build_checks_failed_message, False)
chat_message_send = chat_message_send or send_chat
chat_message += partial_chat_message
build_version_failed_modules = list(
filter(
lambda s: s[MODULE_CONCLUSION
] == MODULE_CONCLUSION_VERSION_CANNOT_BE_IDENTIFIED,
current_level_modules))
if len(build_version_failed_modules) != 0:
module_release_failure = True
build_version_failed_message = 'Following modules\' latest Timestamped Build Version cannot be identified...'
send_chat, partial_chat_message = get_chat_message(
build_version_failed_modules, build_version_failed_message, False)
chat_message_send = chat_message_send or send_chat
chat_message += partial_chat_message
if is_stdlib_module:
chat_message += "After following up on the above, trigger the <" + \
"https://github.com/ballerina-platform/ballerina-release/actions/workflows/update_dependency_version.yml" + \
"|Dependency Update Workflow>"
if send_notification == 'true' and chat_message_send:
print('Failing modules that is being notified:')
print(utils.get_sanitised_chat_message(chat_message))
notify_chat.send_message(chat_message)
elif chat_message_send:
print('Failing modules that is NOT being notified:')
print(utils.get_sanitised_chat_message(chat_message))
if module_release_failure:
sys.exit(1)
def wait_for_current_level_build(level):
global MAX_WAIT_CYCLES
print("[Info] Waiting for level '" + str(level) + "' module build.")
total_modules = len(current_level_modules)
if level == 5:
# In level 5 http takes around 30 min for PR build and build each
# Changes timeout to 80 minutes
MAX_WAIT_CYCLES = 140
if level == 6:
# In level 6 c2c takes around 40 min for PR build and build each
# Changes timeout to 100 minutes
MAX_WAIT_CYCLES = 200
wait_cycles = 0
global status_completed_modules
status_completed_modules = 0
while status_completed_modules != total_modules:
for idx, module in enumerate(current_level_modules):
if module[MODULE_STATUS] == MODULE_STATUS_IN_PROGRESS:
if module[MODULE_CONCLUSION] == MODULE_CONCLUSION_PR_PENDING:
check_pending_pr_checks(idx)
else:
# Build checks test
check_pending_build_checks(idx)
if wait_cycles < MAX_WAIT_CYCLES:
time.sleep(SLEEP_INTERVAL)
wait_cycles = wait_cycles + 1
else:
# Force stop script with all in progress modules printed
print('Dependency bump script timed out. Following modules are in pending state')
for module in current_level_modules:
if module[MODULE_STATUS] == MODULE_STATUS_IN_PROGRESS:
print(module['name'])
sys.exit(1)
module_release_failure = False
chat_message = ""
pr_checks_failed_modules = list(
filter(lambda s: s[MODULE_CONCLUSION] == MODULE_CONCLUSION_PR_CHECK_FAILURE, current_level_modules))
if len(pr_checks_failed_modules) != 0:
module_release_failure = True
print('Following modules dependency PRs have failed checks...')
chat_message += 'Following modules dependency PRs have failed checks...' + "\n"
for module in pr_checks_failed_modules:
print(module['name'])
chat_message += module['name'] + "\n"
pr_merged_failed_modules = list(
filter(lambda s: s[MODULE_CONCLUSION] == MODULE_CONCLUSION_PR_MERGE_FAILURE, current_level_modules))
if len(pr_merged_failed_modules) != 0:
module_release_failure = True
chat_message += 'Following modules dependency PRs could not be merged...' + "\n"
for module in pr_merged_failed_modules:
chat_message += module['name'] + "\n"
build_checks_failed_modules = list(
filter(lambda s: s[MODULE_CONCLUSION] == MODULE_CONCLUSION_BUILD_FAILURE, current_level_modules))
if len(build_checks_failed_modules) != 0:
module_release_failure = True
chat_message += 'Following modules timestamped build checks failed...' + "\n"
for module in build_checks_failed_modules:
chat_message += module['name'] + "\n"
build_version_failed_modules = list(
filter(lambda s: s[MODULE_CONCLUSION] == MODULE_CONCLUSION_VERSION_CANNOT_BE_IDENTIFIED, current_level_modules))
if len(build_version_failed_modules) != 0:
module_release_failure = True
chat_message += 'Following modules timestamped build version cannot be identified...' + "\n"
for module in build_version_failed_modules:
chat_message += module['name'] + "\n"
if module_release_failure:
print(chat_message)
notify_chat.send_message(chat_message)
sys.exit(1)
def main():
repo = github.get_repo(constants.BALLERINA_ORG_NAME + '/' +
'ballerina-lang')
branches = repo.get_branches()
temp_branch = 'sync-2201.1.x-' + date.today().strftime("%d-%m-%Y")
for branch in branches:
if (branch.name == '2201.1.x'):
patch_branch = branch
break
# Check whether master branch already has an unmerged PR from temporary branch, delete if exists
pulls = repo.get_pulls(state='open')
for pull in pulls:
if (pull.head.ref == temp_branch):
print("Master branch already has an open pull request from " +
temp_branch)
pull.edit(state='closed')
break
# if temporary branch exists, delete it
for branch in branches:
if branch.name == temp_branch:
ref = repo.get_git_ref('heads/' + temp_branch)
ref.delete()
break
# create the temporary branch from patch branch
repo.create_git_ref(ref='refs/heads/' + temp_branch,
sha=patch_branch.commit.sha)
ref = repo.get_git_ref('heads/' + temp_branch)
pr = create_pull_request(repo, temp_branch)
time.sleep(60)
pending = True
wait_cycles = 0
if (pr.mergeable_state != 'dirty'):
while pending:
if wait_cycles < MAX_WAIT_CYCLES:
time.sleep(SLEEP_INTERVAL)
pending, passing, failing_pr_checks = check_pending_pr_checks(
repo, pr)
if not pending:
if len(failing_pr_checks) > 0:
passing = all(
check.startswith('codecov')
for check in failing_pr_checks)
if passing:
try:
pr.merge()
ref.delete()
log_message = "[Info] Automated master update PR merged. PR: " + pr.html_url
print(log_message)
except Exception as e:
print(
"[Error] Error occurred while merging master update PR ",
e)
else:
notify_chat.send_message("[Info] Automated ballerina-lang master update PR has failed checks." + "\n" +\
"Please visit <" + pr.html_url + "|the build page> for more information")
pr.edit(state='closed')
ref.delete()
else:
wait_cycles += 1
else:
notify_chat.send_message("[Info] Automated ballerina-lang master update PR is unmerged due to pr checks timeout." + "\n" +\
"Please visit <" + pr.html_url + "|the build page> for more information")
break
else:
notify_chat.send_message("[Info] Automated ballerina-lang master update PR is unmerged due to conflicts with the master." + "\n" +\
"Please visit <" + pr.html_url + "|the build page> for more information")
