def setup_mongo_repo_env(ctx):
mongo_dir = config.REPO_ROOT / 'mongo'
python3_venv_dir = 'python3-venv'
def run_cmds(cmds):
for cmd in cmds:
res = ctx.run(cmd)
get_logger().info('Ran cmd: %s', res.command)
with ctx.cd(str(mongo_dir)):
if not (mongo_dir / python3_venv_dir).exists():
install_venv_cmds = [
f'python3 -m venv {python3_venv_dir}',
]
run_cmds(install_venv_cmds)
else:
get_logger().warning(
'Found existing Python3 virtualenv at %s, skipping creating a new one',
str(mongo_dir / python3_venv_dir))
with ctx.prefix('source python3-venv/bin/activate'):
install_cmds = [
'pip install -r etc/pip/dev-requirements.txt',
'pip install regex',
]
run_cmds(install_cmds)
def jira(self):
"""
lazily get a jira client.
"""
if not self._jira:
while True:
try:
_jira = jira.JIRA(
options={'server': JIRA_URL},
basic_auth=(self.username, self.jira_pwd),
validate=True,
logging=False,
max_retries=3,
timeout=5, # I think the unit is seconds.
)
if _jira:
self._jira = _jira
break
except jira.exceptions.JIRAError as e:
get_logger().warning(
'Failed to login to Jira. Please re-enter your username and password. '
'If the failure persists, please login to Jira manually in a browser. '
'If that still doesn\'t work, seek help in #new-server-eng-help'
)
get_logger().debug(e)
self.reset_jira_credentials()
return self._jira
def add_comment(ticket, comment, **kwargs):
try:
jirac = config.Config().jira
jirac.add_comment(ticket, comment, **kwargs)
except jira.exceptions.JIRAError as e:
get_logger().error('Failed to add comment "%s" to ticket %s due to Jira error: %s', comment, ticket, e.text)
except requests.exceptions.ReadTimeout as e:
get_logger().error('Failed to connect to Jira: %s', repr(e))
def checkout_branch(ctx, branch, silent=False):
original_branch = cur_branch_name(ctx)
ctx.run(f'git checkout {branch}')
with ctx.cd(ent_repo_rel_path):
ctx.run(f'git checkout {branch}')
if not silent:
get_logger().info(f'Checked out existing branch {branch}')
return original_branch
def get_ticket_conf(ctx):
branch = cur_branch_name(ctx)
ticket_conf = config.Config().in_progress_tickets.get(branch)
if not ticket_conf:
get_logger().critical(
'Unknown branch "%s". Please ensure the branch is created with the "start" command',
branch)
raise InvalidConfigError()
return ticket_conf
def _do_download(ctx, download_config):
with ctx.cd(str(config.HOME)):
local_path = config.HOME / download_config.relative_local
if local_path.exists():
get_logger().warning('File %s exists. If you\'d like to re-download this '
'file, please delete the local copy first.', local_path)
else:
cmd = f'curl -o {download_config.relative_local} {download_config.remote}'
get_logger().info(cmd)
ctx.run(cmd)
def install_ninja(ctx):
try:
# Use ninja as a sentinel to check if this step as run.
ctx.run('ninja --version')
except UnexpectedExit:
ctx.run('brew install ninja icecream ccache')
else:
get_logger().warning('ninja appears to be already installed, skipping install')
(config.HOME / 'Library' / 'LaunchAgents').mkdir(parents=True, exist_ok=True)
def create_dir(ctx, conf, dir_absolute):
d = pathlib.Path(dir_absolute)
if d.exists() and d.owner() == config.USER:
get_logger().warning(f'Directory {d} exists and is owned by the current user, skipping creation')
return True
# Need Invoke for sudo. Can't use native Python mkdir().
ctx.sudo(f'mkdir -p {d}', warn=False, password=conf.get_sudo_pwd(ctx))
ctx.sudo(f'chown {config.USER} {d}', warn=False, password=conf.get_sudo_pwd(ctx))
get_logger().info(f'Created directory {d}')
def refresh_repos(ctx, branch):
original_branch = checkout_branch(ctx, branch, silent=True)
try:
ctx.run(f'git pull --rebase origin {branch}')
with ctx.cd(ent_repo_rel_path):
ctx.run(f'git pull --rebase origin {branch}')
get_logger().info(f'Pulled latest changes from {branch} branch')
finally:
checkout_branch(ctx, original_branch)
def install_shell_profile(ctx):
config.CONFIG_DIR.mkdir(parents=True, exist_ok=True)
profile = config.CONFIG_DIR / 'profile'
with open(profile, 'w') as fh:
fh.write(shell_profile_template)
todo = actionable('TODO:')
get_logger().info(f'{todo} Please add the following line to your shell config file, if you haven\'t already ')
get_logger().info(' done so. The default shell config file is ~/.profile. If you\'re using a')
get_logger().info(' different shell, e.g. zsh, you may have a different config file, e.g. ~/.zshrc')
get_logger().info('')
get_logger().info(actionable(' source %s'), str(profile))
def load():
if CONFIG_FILE.exists():
with open(str(CONFIG_FILE), 'rb') as fh:
try:
return pickle.load(fh, fix_imports=False)
except (EOFError, KeyError, TypeError, AttributeError) as e:
get_logger().error('%s: %s', type(e), str(e))
get_logger().warning(
'Could not read config file at %s, using empty config '
'as fallback', str(CONFIG_FILE))
return _ConfigImpl()
def _download_executable_tarball(ctx, download_config, default_name, pretty_name):
bin_dir = config.HOME / 'bin'
if (bin_dir / pretty_name).exists():
get_logger().warning('File/Directory %s already exists. Skipping install', str(bin_dir / pretty_name))
return
download_config.relative_local = f'bin/{pretty_name}.tar.xz'
_do_download(ctx, download_config)
with ctx.cd(str(bin_dir)):
ctx.run(f'tar -xvzf {pretty_name}.tar.xz')
ctx.run(f'rm -f {pretty_name}.tar.xz')
ctx.run(f'mv {default_name} {pretty_name}')
def upload(existing_cr, repo_name):
has_changes = ctx.run(f'git diff {ticket_conf.base_branch}').stdout.strip()
if has_changes:
get_logger().info(f'Submitting code review for the {repo_name} repo')
else:
get_logger().info(f'There are no changes in the {repo_name} repository, skipping code review')
return
cmd = f'python {str(config.UPLOAD_PY)} --rev {ticket_conf.base_branch}...' # Yes three dots.
cmd += ' --nojira -y --git_similarity 90 --check-clang-format --check-eslint'
if existing_cr is not None:
# Continue an existing CR.
cmd += f' -i {existing_cr}'
else:
# Start a new CR.
commit_msg = ctx.run('git log -1 --pretty=%B').stdout.strip()
cr_title = input(f'Please enter the title for this code review (without the ticket number). '
f'Default: {commit_msg}')
if not cr_title:
cr_title = commit_msg
else:
# Add the ticket number to the description.
cr_title = f'{git.cur_branch_name(ctx).upper()} {commit_msg}'
cmd += f' -t "{cr_title}"'
get_logger().info('Opening browser to authenticate with OAuth2... ')
# Simulate some newline inputs to get the browser to open.
sim_stdin = io.StringIO(initial_value='\n\n')
res = ctx.run(cmd, in_stream=sim_stdin)
if existing_cr is None:
cr_url = re.search('Issue created. URL: (.*)', res.stdout.strip()).group(1)
cr_issue_number = cr_url.split('/')[-1]
get_logger().info(f'Code review created: {cr_url}')
ticket_number = git.cur_branch_name(ctx).upper()
jira.transition_ticket(ticket_number, 'In Progress', 'Start Code Review')
jira.add_comment(
ticket_number,
f'Code Review: {cr_url}',
visibility={'type': 'role', 'value': 'Developers'}
)
return cr_issue_number
else:
get_logger().info(f'Code review updated')
return existing_cr
def do_commit(repo):
has_changes = ctx.run('git diff HEAD').stdout.strip()
if has_changes:
branch = git.cur_branch_name(ctx)
ctx.run('git add -u', echo=True, hide=False)
ctx.run(f'git commit -m "{branch.upper()} {raw_commit_msg}"', echo=True, hide=False)
commit_hash = ctx.run('git rev-parse --verify HEAD').stdout.strip()
if repo == 'community':
commit_info.community = commit_hash
elif repo == 'enterprise':
commit_info.enterprise = commit_hash
else:
raise ValueError('Unknown repo type %s', repo)
else:
get_logger().info('No changes found in repo: %s', repo)
def reset_jira_credentials(self):
"""
Reset Jira credentials; for when the user accidentally entered the wrong information.
:return:
"""
if self.username is not None:
self._username = None
self._jira_pwd = None
try:
keyring.delete_password(JIRA_URL, self.username)
except keyring.errors.PasswordDeleteError as e:
get_logger().error(str(e))
else:
get_logger().warning(
'Attempting to delete Jira password without a username')
def anew(ctx, ticket_number, project='server', base_branch='master'):
"""
Step 0: Start a ticket or continue working on an existing ticket
A git branch is created for every Jira ticket and there is a one-to-one correspondence. If you need to
work on two pieces of the same Jira ticket concurrently and have them reviewed separately, consider
filing another ticket and linking it with the original ticket. This way, more information can be captured in
Jira for future reference.
:param ticket_number: Digits of the Jira ticket.
:param project: Jira project. Default: server.
:param base_branch: The base branch of this ticket. Default: master
"""
helpers.check_mongo_repo_root()
try:
ticket_number = int(ticket_number)
except ValueError:
get_logger().critical('Ticket number must be an integer, got "%"', ticket_number)
raise InvalidConfigError()
branch_name = f'{project}-{ticket_number}'
# This is an in-progress ticket.
if branch_name in config.Config().in_progress_tickets:
git.checkout_branch(ctx, branch_name)
# Starting a new ticket.
else:
git.refresh_repos(ctx, base_branch)
git.checkout_branch(ctx, base_branch, silent=True)
git.new_branch(ctx, branch_name)
issue = jira.transition_ticket(
f'{branch_name.upper()}',
from_status='Open',
transition_name='Start Progress'
)
conf = config.TicketConfig()
conf.base_branch = base_branch
if issue:
conf.ticket_summary = issue.fields.summary
else:
conf.ticket_summary = branch_name.upper()
config.Config().in_progress_tickets[branch_name] = conf
def get_sudo_pwd(self, ctx):
if not self._sudo_pwd:
while True:
sudo_pwd = getpass.getpass(
prompt=actionable('Please enter your sudo password: '******'ls', warn=False, hide='both', password=sudo_pwd)
except invoke.exceptions.AuthFailure as e:
get_logger().error(str(e))
continue
self._sudo_pwd = sudo_pwd
break
return self._sudo_pwd
def download_evergreen(ctx):
bin_dir = config.HOME / 'bin'
if (bin_dir / 'evergreen').exists():
get_logger().warning(
'File %s already exists. Skipping downloading evergreen CLI',
str(bin_dir / 'evergreen'))
else:
dc = DownloadConfig(
'https://evergreen.mongodb.com/clients/darwin_amd64/evergreen',
relative_local='bin/evergreen')
_do_download(ctx, dc)
with ctx.cd(str(bin_dir)):
# chmod is cheap enough that we'll just always do it instead of checking if it's already done.
ctx.run('chmod +x evergreen')
def download_clang_format(ctx):
bin_dir = config.HOME / 'bin'
if (bin_dir / 'clang-format').exists():
get_logger().warning('File %s already exists. Skipping install',
str(bin_dir / 'clang-format'))
return
dc = DownloadConfig(config.CLANG_FORMAT_URL)
default_name = 'clang+llvm-3.8.0-x86_64-apple-darwin'
pretty_name = 'llvm-3.8.0'
_download_executable_tarball(ctx, dc, default_name=default_name, pretty_name=pretty_name)
with ctx.cd(str(bin_dir)):
# softlink clang-format to PATH.
ctx.run(f'ln -s {str(bin_dir / pretty_name / "bin" / "clang-format")} clang-format')
def macos(ctx):
"""
Set up macOS for MongoDB server development.
If you're running the workflow tool for the first time, please use the bootstrap script from README.md.
"""
conf = config.Config()
funcs = [
# Do tasks that require user interaction first.
(lambda: ssh_keys(ctx), 'Configure SSH Keys'),
(lambda: create_dir(ctx, conf, '/data'),
'Create MongoDB Data Directory'),
(lambda: create_dir(ctx, conf, '/opt/mongodbtoolchain/revisions'),
'Create MongoDB Toolchain Directory'),
(lambda: create_dir(ctx, conf, str(config.HOME / 'bin')),
'Create User bin Directory'),
# Then do the automated tasks that don't require user interaction.
(lambda: evergreen_yaml(conf), 'Configure Evergeen'),
(lambda: clone_repos(ctx), 'Clone MongoDB Repositories'),
(lambda: download_clang_format(ctx), 'Download clang-format'),
(lambda: download_eslint(ctx), 'Download eslint'),
(lambda: download_evergreen(ctx), 'Download evergreen CLI'),
(lambda: install_ninja(ctx), 'Install ninja, icecream, ccache'),
# Next do mongo repo setup. These tasks require the system setup steps above to have run.
(lambda: setup_mongo_repo_env(ctx), 'Setup the mongo Repository'),
# Do tasks that require followup work last.
(lambda: install_githooks(ctx), 'Install Git Hooks'),
(lambda: install_shell_profile(ctx), 'Install Shell Profile'),
(lambda: post_task_instructions(), 'Post Setup Instructions')
]
for func in funcs:
log_func(func[0], func[1])
get_logger().info('Finished setting up macOS for MongoDB development!')
get_logger().info(
f'Please go over any action items above highlighted in {actionable("green")}.'
)
def zzz(ctx, force=False):
"""
Step 6: Cleanup. Remove local branches and close Jira ticket
:param force: Force delete the local branch even if it's not fully merged
"""
helpers.check_mongo_repo_root()
ticket_conf = helpers.get_ticket_conf(ctx)
feature_branch = git.cur_branch_name(ctx)
base_branch = ticket_conf.base_branch
get_logger().info(f'🍦 Congrats on completing {feature_branch.upper()}! 🍦')
input(actionable(f'Press any key to remove local branches and close the Jira ticket'))
config.Config().in_progress_tickets.pop(feature_branch)
ctx.run(f'git checkout {base_branch}')
try:
if force:
ctx.run(f'git branch -D {feature_branch}')
else:
ctx.run(f'git branch --delete {feature_branch}')
except UnexpectedExit as e:
get_logger().error(e)
get_logger().error(f'Failed to delete branch, please manually delete your local branch {feature_branch}')
jira.transition_ticket(
feature_branch.upper(),
'In Code Review',
'Close Issue'
)
def evergreen_yaml(conf):
# initialize Jira to get the jira user name for Evergreen.
if config.EVG_CONFIG_FILE.exists():
get_logger().info(
'Found existing ~/.evergreen.yml, skipping adding Evergreen configuration'
)
get_logger().info(
'Please ensure your ~/.evergreen.yml was generated by this tool. If not, '
'make sure you know what\'s in there')
else:
settings_url = 'https://evergreen.mongodb.com/login/key'
while True:
res = requests.post(settings_url,
json={
'username': conf.username,
'password': conf.jira_pwd
})
if res.status_code != 200:
get_logger().error(
'Failed to fetch API key from evergreen. Error: %s',
str(res))
req_input('Press any key to retry...')
conf.reset_jira_credentials()
continue
res_json = res.json()
evg_config = evergreen_yaml_template.format(
res_json['user'], res_json['api_key'])
with open(config.EVG_CONFIG_FILE, 'w') as fh:
fh.write(evg_config)
break
return True
def clone_repos(ctx):
config.REPO_ROOT.mkdir(exist_ok=True)
get_logger().info('Placing MongoDB Git repositories in %s', config.REPO_ROOT)
with ctx.cd(str(config.REPO_ROOT)):
for repo_config in config.REQUIRED_REPOS:
repo_dir = config.REPO_ROOT / repo_config.relative_local
if repo_dir.exists():
get_logger().warning('Local directory %s exists.', str(repo_dir))
get_logger().warning('If you\'d like to re-clone, please delete this directory first')
else:
cmd = f'git clone {repo_config.remote} {repo_dir}'
get_logger().info(cmd)
ctx.run(cmd, hide=False)
def install_githooks(ctx):
hooks_dir = config.HOME / '.githooks'
# Dupe of path defined in REQUIRED_REPO.
kernel_tools_dir = config.REPO_ROOT / 'kernel-tools'
mongo_dir = config.REPO_ROOT / 'mongo'
hooks_dir.mkdir(exist_ok=True, parents=True)
if not (hooks_dir / 'mongo').exists():
ctx.run(
f'ln -s {str(kernel_tools_dir / "githooks")} {str(hooks_dir / "mongo")}'
)
with ctx.cd(str(mongo_dir)):
ctx.run(f'source buildscripts/install-hooks -f')
todo = actionable('TODO:')
get_logger().info(
f'{todo} Please consult with your mentor on which githooks are needed. Some hooks may be'
)
get_logger().info(
' unnecessarily cumbersome for your project. You can delete any unneeded hooks in '
)
get_logger().info(f' `%s` and rerun `workflow macos.setup`',
str(kernel_tools_dir / "githooks" / "pre-push"))
def ship(ctx):
"""
Step 5: Provide instructions on pushing your changes to master
Also add the URL of the latest patch build to the Jira ticket.
"""
helpers.check_mongo_repo_root()
ticket_conf = helpers.get_ticket_conf(ctx)
cur_branch = git.cur_branch_name(ctx)
if ticket_conf.patch_ids:
jira.add_comment(
cur_branch.upper(),
f'Patch Build: {urllib.parse.urljoin(config.EVG_PATCH_URL_BASE, ticket_conf.patch_ids[-1])}',
visibility={'type': 'role', 'value': 'Developers'}
)
else:
get_logger().warning('No patch builds were created for this ticket, not adding patch build URL to Jira')
# This will implicitly check for uncommitted changes on the feature branch
git.refresh_repos(ctx, ticket_conf.base_branch)
lines = [
actionable('Please run the following commands to push your changes to the upstream MongoDB repository:'),
'',
f' git rebase --interactive {ticket_conf.base_branch}',
f' git checkout {ticket_conf.base_branch}',
f' git merge --ff-only {cur_branch}',
f' git push origin {ticket_conf.base_branch} --dry-run',
f' git push origin {ticket_conf.base_branch}',
'',
'As part of `git rebase --interactive`, you should squash your local commits into one commit. Please refer to',
'this guide for an intro to interactive rebase: https://git-scm.com/docs/git-rebase#_interactive_mode',
'',
'If you encounter errors during any of the above steps, please ask your mentor for advice.',
'',
'Finally, when you\'ve pushed your changes, run `workflow zzz` to delete your local branches'
]
log.log_multiline(get_logger().info, lines)
def post_task_instructions():
lines = [
actionable('Note on Using "compiledb":'),
' A Clang JSON Compilation Database (compiledb) can be generated for the mongo repository by running',
'',
' `ninja compiledb` (from the mongo repo at ~/mongodb/mongo)',
'',
' It enables features like jump to definition and semantic code completion in code editors. Please refer',
' to the following web page on how to integrate compiledb with your favorite editor',
'',
' https://sarcasm.github.io/notes/dev/compilation-database.html#text-editors-and-ides',
'',
' When you switch branches or add/remove files, compiledb needs to be updated by running `ninja compiledb`',
'',
' If you\'d like to use an editor that "just works", The CLion IDE is a good option. You just need',
' to install it and open the "mongo" directory. Code completion and jumping to definitions will',
' automatically work',
'',
' To install CLion, run: `brew cask install clion`'
]
log_multiline(get_logger().info, lines)
get_logger().warning('Please run mongod with `--dbpath /opt/data` as macOS 10.15+ does not allow modifying /')
def run():
invoke_config = {
'run': {
'hide': True # Don't print stdout or stderr.
},
'NINJA_STATUS': '[%f/%t (%p) %es] ' # make the ninja output even nicer
}
ns = Collection.from_module(tasks, config=invoke_config)
ns.add_collection(
Collection.from_module(setupenv, name='setup', config=invoke_config))
ns.add_collection(
Collection.from_module(helpers, name='helpers', config=invoke_config))
proj_info = pkg_resources.require("server_workflow_tool")[0]
p = Program(binary='workflow',
name=proj_info.project_name,
namespace=ns,
version=proj_info.version)
p.parse_core(sys.argv[1:])
if p.args.debug.value:
get_logger(level=logging.DEBUG)
else:
get_logger(level=logging.INFO)
c = Config()
try:
p.run()
except (InvalidConfigError, RequireUserInputError):
# These errors are not actionable right now.
sys.exit(1)
finally:
c.dump()
def transition_ticket(ticket, from_status, transition_name):
from_id = _server_status_map[from_status]
transition_id = _server_transition_map[transition_name]
to_status = _server_transition_to_map[transition_name]
try:
jirac = config.Config().jira
issue = jirac.issue(ticket)
if issue.fields.status.id == from_id:
get_logger().info(f'Transitioning {ticket} in Jira from "{from_status}"" to {to_status}"')
jirac.transition_issue(issue, transition_id)
else:
get_logger().info(f'{ticket} is not "{from_status}", skipping transition to "{to_status}"')
return issue
except jira.exceptions.JIRAError as e:
get_logger().error('Failed to do transition "%s" for ticket %s due to Jira error: %s',
transition_name, ticket, e.text)
except requests.exceptions.ReadTimeout as e:
get_logger().error('Failed to connect to Jira: %s', repr(e))
def patch(ctx, finalize='yes', alias='required'):
"""
Step 3: Run a patch build in Evergreen CI
:param alias: the Evergreen alias for determining the set of tasks to run, defaults to "required".
:param finalize: pass in any falsy value to avoid kicking off the patch build immediately.
"""
helpers.check_mongo_repo_root()
ticket_conf = helpers.get_ticket_conf(ctx)
if not ticket_conf.commits:
get_logger().warning('Did not find any commits on this branch. Please make sure you run `commit` '
'before `patch`')
# Use the commit message from the latest commit as the patch build description.
commit_msg = ctx.run('git log -1 --pretty=%B').stdout.strip()
cmd = f'evergreen patch --alias {alias} --description "{commit_msg}" --yes'
if finalize == 'yes':
# Any other value specified through the commandline is considered falsy.
cmd += ' --finalize'
res = ctx.run(cmd, hide=False, echo=True)
# Sketchy regex parsing of the output of "evergreen patch"
evg_url = re.search("Build : (.*)", res.stdout, re.MULTILINE).group(1)
patch_id = evg_url.split('/')[-1]
with ctx.cd(git.ent_repo_rel_path):
ctx.run(f'evergreen patch-set-module --id {patch_id} --module enterprise --yes', hide=False, echo=True)
ticket_conf = helpers.get_ticket_conf(ctx)
ticket_conf.patch_ids.append(patch_id)
if finalize != 'yes':
# Point the user to the patch build URL if finalize is false.
webbrowser.open(evg_url)
else:
get_logger().info(f'Patch build starting at URL: {evg_url}')
get_logger().info('You can configure Slack and Email notifications for your patch builds at '
'https://evergreen.mongodb.com/notifications')
def setUpClass(cls) -> None:
get_logger(logging.INFO)