def test_upload_to_pypi(testing, python2, python3, mocker, library_test_repo):
"""upload_to_pypi should create a dist based on a version and upload to pypi or pypitest"""
twine_env = {
'PYPI_USERNAME': '******',
'PYPI_PASSWORD': '******',
'PYPITEST_USERNAME': '******',
'PYPITEST_PASSWORD': '******',
}
library_test_repo = RepoInfo(**{
**library_test_repo._asdict(),
'python2': python2,
'python3': python3,
})
def check_call_func(*args, **kwargs):
"""Patch check_call to skip twine"""
command_args = args[0]
if not command_args[0].endswith("twine"):
call(*args, **kwargs)
if command_args[0].endswith("twine"):
# Need to assert twine here since temp directory is random which makes it assert to assert
assert kwargs['env']['TWINE_USERNAME'] == (
twine_env['PYPITEST_USERNAME'] if testing else twine_env['PYPI_USERNAME']
)
assert kwargs['env']['TWINE_PASSWORD'] == (
twine_env['PYPITEST_PASSWORD'] if testing else twine_env['PYPI_PASSWORD']
)
assert args[0][1] == 'upload'
if testing:
assert args[0][2:4] == ["--repository-url", "https://test.pypi.org/legacy/"]
dist_files = args[0][-2:]
assert len([name for name in dist_files if name.endswith(".whl")]) == 1
assert len([name for name in dist_files if name.endswith(".tar.gz")]) == 1
if command_args[0].endswith("virtualenv"):
assert command_args[-1] == ("python3" if python3 else "python2")
if 'bdist_wheel' in command_args:
if python2 and python3:
assert "--universal" in command_args
mocker.patch('lib.check_call', side_effect=check_call_func)
call_mock = mocker.patch('lib.call', side_effect=check_call_func)
check_output_mock = mocker.patch('lib.check_output', return_value=b'x=y=z')
mocker.patch.dict('os.environ', twine_env, clear=False)
upload_to_pypi(repo_info=library_test_repo, testing=testing)
assert check_output_mock.call_count == 1
assert call_mock.call_args[1] == {'env': {'x': 'y=z'}}
def load_repos_info(channel_lookup):
"""
Load repo information from JSON and looks up channel ids for each repo
Args:
channel_lookup (dict): Map of channel names to channel ids
Returns:
list of RepoInfo: Information about the repositories
"""
with open(os.path.join(SCRIPT_DIR, "repos_info.json"), "r", encoding="utf-8") as f:
repos_info = json.load(f)
infos = [
RepoInfo(
name=repo_info["name"],
repo_url=repo_info["repo_url"],
ci_hash_url=(
repo_info["ci_hash_url"]
if repo_info.get("project_type") == WEB_APPLICATION_TYPE
else None
),
rc_hash_url=(
repo_info["rc_hash_url"]
if repo_info.get("project_type") == WEB_APPLICATION_TYPE
else None
),
prod_hash_url=(
repo_info["prod_hash_url"]
if repo_info.get("project_type") == WEB_APPLICATION_TYPE
else None
),
channel_id=channel_lookup[repo_info["channel_name"]],
project_type=repo_info.get("project_type"),
web_application_type=repo_info.get("web_application_type"),
packaging_tool=repo_info.get("packaging_tool"),
versioning_strategy=repo_info.get("versioning_strategy", FILE_VERSION),
)
for repo_info in repos_info["repos"]
if repo_info.get("repo_url")
]
# some basic validation for sanity checking
for info in infos:
if info.project_type == WEB_APPLICATION_TYPE:
if info.web_application_type not in VALID_WEB_APPLICATION_TYPES:
raise Exception(
f"Unexpected web application type {info.web_application_type} for {info.name}"
)
elif info.project_type == LIBRARY_TYPE:
if info.packaging_tool not in VALID_PACKAGING_TOOL_TYPES:
raise Exception(
f"Unexpected packaging tool {info.packaging_tool} for {info.name}"
)
if info.versioning_strategy not in VALID_VERSIONING_STRATEGIES:
raise Exception(
f"Unexpected versioning strategy {info.versioning_strategy} for {info.name}"
)
return infos
def test_load_repos_info(mocker):
"""
load_repos_info should match channels with repositories
"""
json_load = mocker.patch('lib.json.load', autospec=True, return_value={
'repos': [
{
"name": "bootcamp-ecommerce",
"repo_url": "https://github.com/mitodl/bootcamp-ecommerce.git",
"rc_hash_url": "https://bootcamp-ecommerce-rc.herokuapp.com/static/hash.txt",
"prod_hash_url": "https://bootcamp-ecommerce.herokuapp.com/static/hash.txt",
"channel_name": "bootcamp-eng",
"project_type": "web_application",
"python2": False,
"python3": True,
}
]
})
assert load_repos_info({
'bootcamp-eng': 'bootcamp_channel_id'
}) == [
RepoInfo(
name='bootcamp-ecommerce',
repo_url='https://github.com/mitodl/bootcamp-ecommerce.git',
rc_hash_url="https://bootcamp-ecommerce-rc.herokuapp.com/static/hash.txt",
prod_hash_url="https://bootcamp-ecommerce.herokuapp.com/static/hash.txt",
channel_id='bootcamp_channel_id',
project_type='web_application',
python2=False,
python3=True,
),
]
assert json_load.call_count == 1
def load_repos_info(channel_lookup):
"""
Load repo information from JSON and looks up channel ids for each repo
Args:
channel_lookup (dict): Map of channel names to channel ids
Returns:
list of RepoInfo: Information about the repositories
"""
with open(os.path.join(SCRIPT_DIR, "repos_info.json")) as f:
repos_info = json.load(f)
return [
RepoInfo(
name=repo_info['name'],
repo_url=repo_info.get('repo_url'),
rc_hash_url=(repo_info['rc_hash_url']
if repo_info.get('project_type')
== WEB_APPLICATION_TYPE else None),
prod_hash_url=(repo_info['prod_hash_url']
if repo_info.get('project_type')
== WEB_APPLICATION_TYPE else None),
channel_id=channel_lookup[repo_info['channel_name']],
project_type=repo_info.get('project_type'),
python2=repo_info.get('python2'),
python3=repo_info.get('python3'),
announcements=repo_info.get('announcements'),
) for repo_info in repos_info['repos']
]
async def test_update_version(
mocker,
test_repo,
versioning_strategy,
expected_python,
expected_js,
expected_version_file,
readonly,
):
"""Call update_version on project"""
repo_info = RepoInfo(
**{
**test_repo._asdict(),
"versioning_strategy": versioning_strategy,
})
new_version = "12.34.56"
working_dir = "/tmp/a/directory"
update_py_mock = mocker.patch("version.update_python_version")
update_js_mock = mocker.async_patch("version.update_npm_version")
update_version_file_mock = mocker.async_patch(
"version.update_version_file")
await update_version(
repo_info=repo_info,
new_version=new_version,
working_dir=working_dir,
readonly=readonly,
)
if expected_python:
update_py_mock.assert_called_once_with(
new_version=new_version,
working_dir=working_dir,
readonly=readonly,
)
else:
assert update_py_mock.called is False
if expected_js:
update_js_mock.assert_called_once_with(
new_version=new_version,
working_dir=working_dir,
readonly=readonly,
)
else:
assert update_js_mock.called is False
if expected_version_file:
update_version_file_mock.assert_called_once_with(
new_version=new_version,
working_dir=working_dir,
readonly=readonly,
)
else:
assert update_version_file_mock.called is False
async def test_publish(mocker, test_repo_directory, library_test_repo,
packaging_tool):
"""publish should call upload_to_pypi or upload_to_npm depending on the packaging_tool"""
upload_to_pypi_mocked = mocker.async_patch("publish.upload_to_pypi")
upload_to_npm_mocked = mocker.async_patch("publish.upload_to_npm")
library_test_repo = RepoInfo(
**{
**library_test_repo._asdict(),
"packaging_tool": packaging_tool,
})
github_access_token = "git hub"
npm_token = "npm token"
version = "9.8.7"
working_dir = "/a/working/dir"
init_mock = mocker.patch(
"publish.init_working_dir",
side_effect=async_context_manager_yielder(working_dir),
)
await publish(
repo_info=library_test_repo,
github_access_token=github_access_token,
npm_token=npm_token,
version=version,
)
init_mock.assert_called_once_with(github_access_token,
library_test_repo.repo_url,
branch=f"v{version}")
if packaging_tool == SETUPTOOLS:
assert upload_to_npm_mocked.called is False
upload_to_pypi_mocked.assert_called_once_with(project_dir=working_dir)
elif packaging_tool == NPM:
assert upload_to_pypi_mocked.called is False
upload_to_npm_mocked.assert_called_once_with(
project_dir=working_dir,
npm_token=npm_token,
)
def load_repos_info(channel_lookup):
"""
Load repo information from JSON and looks up channel ids for each repo
Args:
channel_lookup (dict): Map of channel names to channel ids
Returns:
list of RepoInfo: Information about the repositories
"""
with open(os.path.join(SCRIPT_DIR, "repos_info.json")) as f:
repos_info = json.load(f)
return [
RepoInfo(
name=repo_info['name'],
repo_url=repo_info['repo_url'],
rc_hash_url=repo_info['rc_hash_url'],
prod_hash_url=repo_info['prod_hash_url'],
channel_id=channel_lookup[repo_info['channel_name']],
) for repo_info in repos_info['repos']
]
'Intended Audience :: Developers',
'Intended Audience :: Education',
'Programming Language :: Python',
],
include_package_data=True,
zip_safe=False,
)
"""
WEB_TEST_REPO_INFO = RepoInfo(
name='doof_repo',
repo_url='http://github.com/mitodl/doof.git',
prod_hash_url='http://doof.example.com/hash.txt',
rc_hash_url='http://doof-rc.example.com/hash.txt',
channel_id='doof',
project_type=WEB_APPLICATION_TYPE,
python2=False,
python3=True,
announcements=False,
)
# pylint: disable=redefined-outer-name, unused-argument
@pytest.fixture
def test_repo_directory():
"""Helper function to make a repo for testing"""
with make_test_repo() as working_dir:
yield working_dir
async def test_upload_to_pypi(testing, python2, python3, mocker, library_test_repo, test_repo_directory):
"""upload_to_pypi should create a dist based on a version and upload to pypi or pypitest"""
twine_env = {
'PYPI_USERNAME': '******',
'PYPI_PASSWORD': '******',
'PYPITEST_USERNAME': '******',
'PYPITEST_PASSWORD': '******',
}
library_test_repo = RepoInfo(**{
**library_test_repo._asdict(),
'python2': python2,
'python3': python3,
})
def check_call_func(*args, env, **kwargs):
"""Patch check_call to skip twine"""
command_args = args[0]
env = {
**os.environ,
**env
} if env is not None else os.environ
if not command_args[0].endswith("twine"):
call(*args, **kwargs, env=env)
if command_args[0].endswith("twine"):
# Need to assert twine here since temp directory is random which makes it assert to assert
assert env['TWINE_USERNAME'] == (
twine_env['PYPITEST_USERNAME'] if testing else twine_env['PYPI_USERNAME']
)
assert env['TWINE_PASSWORD'] == (
twine_env['PYPITEST_PASSWORD'] if testing else twine_env['PYPI_PASSWORD']
)
assert args[0][1] == 'upload'
if testing:
assert args[0][2:4] == ["--repository-url", "https://test.pypi.org/legacy/"]
dist_files = args[0][-2:]
assert len([name for name in dist_files if name.endswith(".whl")]) == 1
assert len([name for name in dist_files if name.endswith(".tar.gz")]) == 1
if 'bdist_wheel' in command_args:
if python2 and python3:
assert "--universal" in command_args
mocker.async_patch('lib.check_call', side_effect=check_call_func)
call_mock = mocker.async_patch('lib.call', side_effect=check_call_func)
mocker.async_patch('lib.check_output', return_value=b'x=y=z')
mocker.patch.dict('os.environ', twine_env, clear=False)
init_working_dir_mock = mocker.patch(
'lib.init_working_dir', side_effect=async_context_manager_yielder(test_repo_directory)
)
version = "4.5.6"
token = "token"
await upload_to_pypi(
repo_info=library_test_repo, testing=testing, github_access_token=token, version=version
)
assert call_mock.call_args[1] == {'env': {'x': 'y=z'}, 'cwd': test_repo_directory}
init_working_dir_mock.assert_called_once_with(token, library_test_repo.repo_url, branch=f"v{version}")
)
from repo_info import RepoInfo
pytestmark = pytest.mark.asyncio
GITHUB_ACCESS = 'github'
SLACK_ACCESS = 'slack'
TEST_REPOS_INFO = [
RepoInfo(
name='doof_repo',
repo_url='http://github.com/mitodl/doof.git',
prod_hash_url='http://doof.example.com/hash.txt',
rc_hash_url='http://doof-rc.example.com/hash.txt',
channel_id='doof',
)
]
# pylint: disable=redefined-outer-name
class DoofSpoof(Bot):
"""Testing bot"""
def __init__(self):
"""Since the testing bot isn't contacting slack or github we don't need these tokens here"""
super().__init__(
websocket=Mock(),
slack_access_token=SLACK_ACCESS,
github_access_token=GITHUB_ACCESS,
'Development Status :: 2 - Pre-Alpha',
'Intended Audience :: Developers',
'Intended Audience :: Education',
'Programming Language :: Python',
],
include_package_data=True,
zip_safe=False,
)
"""
WEB_TEST_REPO_INFO = RepoInfo(
name="doof_repo",
repo_url="http://github.com/mitodl/doof.git",
prod_hash_url="http://doof.example.com/hash.txt",
rc_hash_url="http://doof-rc.example.com/hash.txt",
ci_hash_url="http://doof-ci.example.com/hash.txt",
channel_id="doof",
project_type=WEB_APPLICATION_TYPE,
web_application_type=DJANGO,
packaging_tool=None,
versioning_strategy=PYTHON_VERSION,
)
# pylint: disable=redefined-outer-name, unused-argument
@pytest.fixture
def test_repo_directory():
"""Helper function to make a repo for testing"""
with make_test_repo() as working_dir:
yield working_dir
async def test_load_repos_info(mocker):
"""
load_repos_info should match channels with repositories
"""
json_load = mocker.patch(
"lib.json.load",
autospec=True,
return_value={
"repos": [
{
"name": "bootcamp-ecommerce",
"repo_url":
"https://github.com/mitodl/bootcamp-ecommerce.git",
"ci_hash_url":
"https://bootcamp-ecommerce-ci.herokuapp.com/static/hash.txt",
"rc_hash_url":
"https://bootcamp-ecommerce-rc.herokuapp.com/static/hash.txt",
"prod_hash_url":
"https://bootcamp-ecommerce.herokuapp.com/static/hash.txt",
"channel_name": "bootcamp-eng",
"project_type": WEB_APPLICATION_TYPE,
"web_application_type": DJANGO,
"versioning_strategy": PYTHON_VERSION,
},
{
"name": "bootcamp-ecommerce-library",
"repo_url":
"https://github.com/mitodl/bootcamp-ecommerce-library.git",
"channel_name": "bootcamp-library",
"project_type": LIBRARY_TYPE,
"packaging_tool": NPM,
"versioning_strategy": NPM_VERSION,
},
{
"name": "ocw-hugo-projects",
"repo_url":
"https://github.com/mitodl/ocw-hugo-projects.git",
"channel_name": "ocw-hugo-projects",
"project_type": "library",
"packaging_tool": "none",
"versioning_strategy": FILE_VERSION,
},
]
},
)
expected_web_application = RepoInfo(
name="bootcamp-ecommerce",
repo_url="https://github.com/mitodl/bootcamp-ecommerce.git",
ci_hash_url=
"https://bootcamp-ecommerce-ci.herokuapp.com/static/hash.txt",
rc_hash_url=
"https://bootcamp-ecommerce-rc.herokuapp.com/static/hash.txt",
prod_hash_url=
"https://bootcamp-ecommerce.herokuapp.com/static/hash.txt",
channel_id="bootcamp_channel_id",
project_type=WEB_APPLICATION_TYPE,
web_application_type=DJANGO,
packaging_tool=None,
versioning_strategy=PYTHON_VERSION,
)
expected_npm_library = RepoInfo(
name="bootcamp-ecommerce-library",
repo_url="https://github.com/mitodl/bootcamp-ecommerce-library.git",
ci_hash_url=None,
rc_hash_url=None,
prod_hash_url=None,
channel_id="bootcamp_library_channel_id",
project_type=LIBRARY_TYPE,
web_application_type=None,
packaging_tool=NPM,
versioning_strategy=NPM_VERSION,
)
expected_file_library = RepoInfo(
name="ocw-hugo-projects",
repo_url="https://github.com/mitodl/ocw-hugo-projects.git",
ci_hash_url=None,
rc_hash_url=None,
prod_hash_url=None,
channel_id="ocw_hugo_channel_id",
project_type=LIBRARY_TYPE,
web_application_type=None,
packaging_tool=NONE,
versioning_strategy=FILE_VERSION,
)
assert load_repos_info({
"bootcamp-eng": "bootcamp_channel_id",
"bootcamp-library": "bootcamp_library_channel_id",
"ocw-hugo-projects": "ocw_hugo_channel_id",
}) == [
expected_web_application, expected_npm_library, expected_file_library
]
assert json_load.call_count == 1