def main(args=None):
parser = argparse.ArgumentParser(
description='Create wheels for all given project versions and upload them to the given index.')
parser.add_argument('requirements',
help='requirements.txt style file specifying which project versions to package.')
parser.add_argument('index', help='The index to upload the packaged software to.')
parser.add_argument('--batch',
default=False,
action='store_true',
help='Batch mode. Do not prompt for credentials')
parser.add_argument('--user',
default=os.environ.get('DEVPI_USER'),
help='The user to log in as.')
parser.add_argument('--password',
default=os.environ.get('DEVPI_PASSWORD'),
help='Password of the user.')
parser.add_argument('--blacklist', help='Packages matched by this requirements.txt style file will never be build.')
parser.add_argument('--pure-index',
help='The index to use for pure packages. Any non-pure package will be uploaded '
'to the index given as positional argument. Packages already found in the '
'pure index will not be built, either.'
)
parser.add_argument('--junit-xml',
help='Write information about the build success / failure to a JUnit-compatible XML file.')
parser.add_argument('--run-id',
help='Add the given string to all entries in the XML output, allowing to distinguish output from multiple runs in a merged XML.')
parser.add_argument('--dry-run', help='Build missing wheels, but do not modify the state of the devpi server.',
action='store_true')
parser.add_argument('--client-cert', help='Client key to use to authenticate with the devpi server.', default=None)
args = parser.parse_args(args=args)
if not args.batch:
if args.user is None:
args.user = input('Username: '******'Password: ')
packages = requirements.read_exact_versions(args.requirements)
with wheeler.Builder() as builder, DevpiClient(args.index, args.user, args.password,
client_cert=args.client_cert) as devpi_client:
if args.pure_index:
with DevpiClient(args.pure_index, args.user, args.password,
client_cert=args.client_cert) as pure_index_client:
processor = Processor(builder, devpi_client, args.blacklist, pure_index_client,
junit_xml=args.junit_xml, dry_run=args.dry_run, run_id=args.run_id)
processor.build_packages(packages)
else:
processor = Processor(builder, devpi_client, args.blacklist, junit_xml=args.junit_xml, dry_run=args.dry_run,
run_id=args.run_id)
processor.build_packages(packages)
def TestServer(users={}, indices={}, config={}, fail_on_output=['Traceback']):
"""
Starts a devpi server to be used within tests.
"""
with temporary_dir() as server_dir:
server_options = {
'port': 2414,
'serverdir': server_dir}
server_options.update(config)
if 'serverdir' not in config:
prefill_serverdir(server_options)
with DevpiServer(server_options) as url:
with DevpiClient(url, 'root', '') as client:
for user, kwargs in iteritems(users):
client.create_user(user, **kwargs)
for index, kwargs in iteritems(indices):
client.create_index(index, **kwargs)
yield client
_assert_no_logged_errors(fail_on_output, server_options['serverdir'] + '/.xproc/devpi-server/xprocess.log')
def TestServer(users={}, indices={}, config={}, fail_on_output=['Traceback']):
"""
Starts a devpi server to be used within tests.
"""
with temporary_dir() as server_dir:
server_options = {
'host': os.getenv('DEVPI_PLUMBER_SERVER_HOST', 'localhost'),
'port': os.getenv('DEVPI_PLUMBER_SERVER_PORT', 2414),
'serverdir': server_dir,
}
server_options.update(config)
initialize_serverdir(server_options)
with DevpiServer(server_options) as url:
with DevpiClient(url, 'root', '') as client:
for user, kwargs in iteritems(users):
client.create_user(user, **kwargs)
for index, kwargs in iteritems(indices):
client.create_index(index, **kwargs)
yield client
_assert_no_logged_errors(fail_on_output,
server_options['serverdir'] + '/server.log')
def test_dry_run(self):
"""
Test that a dry run produces the same ``junit.xml`` as a run without dry-run but does not modify the server
state.
"""
with TestServer(USERS, INDICES) as devpi:
with DevpiClient(devpi.server_url + '/' + INDEX, USER,
PASSWORD) as client:
client.upload(
'tests/fixture/pure_package/dist/test_package-0.1.dev1-py2.py3-none-any.whl'
)
tempdir = tempfile.mkdtemp()
try:
junit_filename = os.path.join(tempdir, 'junit.xml')
main([
'tests/fixture/sample_junit.txt',
devpi.url + '/' + INDEX,
USER,
PASSWORD,
'--junit-xml',
junit_filename,
'--dry-run',
])
self._assert_junit_xml_content(junit_filename)
finally:
shutil.rmtree(tempdir)
self.assertFalse(
package_version_exists(devpi, INDEX, 'progressbar', '2.2'))
def test_reports_junit_xml(devpi, tmpdir):
with DevpiClient(devpi.server_url + '/' + INDEX,
USER,
PASSWORD) as client:
client.upload('tests/fixture/pure_package/dist/test_package-0.1.dev1-py2.py3-none-any.whl')
junit_filename = str(tmpdir.join('junit.xml'))
main(['tests/fixture/sample_junit.txt', devpi.url + '/' + INDEX,
'--user={}'.format(USER),
'--password={}'.format(PASSWORD), '--junit-xml', junit_filename])
_assert_junit_xml_content(junit_filename)
def test_devpi_client(self):
with TestServer() as test_server:
with DevpiClient(test_server.server_url) as devpi:
devpi.create_user("user",
password="******",
email="*****@*****.**")
self.assertEqual(
200,
requests.get(devpi.server_url + "/user").status_code)
self.assertIn("credentials valid",
devpi.login("user", "password"))
self.assertEquals("user", devpi.user)
def main(args=None):
parser = argparse.ArgumentParser(
description='A utility to batch-remove packages from a Devpi server.',
epilog='The arguments --dev-only and --version-filter can be combined. In this case only packages passing both filters will be removed.',
)
parser.add_argument('server', help='The devpi server to operate on.')
parser.add_argument('index_spec', metavar='user[/index]', help='The index from which to remove the packages. If only the user part is specified, all indices of that user will be cleaned.')
parser.add_argument('package_specification', help='The specification of the package version(s) to remove.')
parser.add_argument('--batch', help='Assume yes on confirmation questions.', action='store_true')
parser.add_argument('--dev-only', help='Remove only development versions as specified by PEP 440.', action='store_true')
parser.add_argument('--version-filter', metavar='REGEX', help='Remove only versions in which the given regular expression can be found.')
parser.add_argument('--versions-to-keep', type=int, help='Number of versions to keep.')
parser.add_argument('--force', help='Temporarily make indices volatile to enable package removal.', action='store_true')
parser.add_argument('--password', help='The password with which to authenticate.')
parser.add_argument('--login', help='The user name to user for authentication. Defaults to the user of the indices to operate on.')
args = parser.parse_args(args=args)
login_user = args.login if args.login else args.index_spec.split('/')[0]
password = args.password
if password is None:
password = getpass.getpass()
try:
with DevpiClient(args.server, login_user, password) as client:
packages_by_index = list_packages_by_index(
client, args.index_spec, args.package_specification, args.dev_only, args.version_filter
)
for index, packages in packages_by_index.items():
print('Packages to be deleted from {}: '.format(index))
sorted_packages = sorted(packages)
for package in sorted_packages:
print(' * {}'.format(package))
if not args.batch:
confirmation = input('Enter "yes" to confirm: ')
if confirmation != 'yes':
print('Aborting...')
return
for index, packages in packages_by_index.items():
print('Cleaning {}…'.format(index))
# if len(packages) > 1:
# # Make iteration over the packages display a progress bar
# packages = ProgressBar()(packages)
remove_packages(client, index, packages, args.force, args.versions_to_keep)
except DevpiClientError as client_error:
print_(client_error, file=sys.stderr)
sys.exit(1)
def test_run_id(devpi, tmpdir):
RUN_ID = 'my_run_id'
with DevpiClient(devpi.server_url + '/' + INDEX, USER, PASSWORD) as client:
client.upload(
'tests/fixture/pure_package/dist/test_package-0.1.dev1-py2.py3-none-any.whl'
)
junit_filename = str(tmpdir.join('junit.xml'))
main([
'tests/fixture/sample_junit.txt', devpi.url + '/' + INDEX, USER,
PASSWORD, '--junit-xml', junit_filename, '--run-id', RUN_ID
])
_assert_junit_xml_content(junit_filename, RUN_ID)
def test_not_built_if_on_pure(devpi):
"""
Verify that packages are not built and re-uploaded if they are already on the pure index.
"""
with DevpiClient(devpi.server_url + '/' + PURE_INDEX,
USER,
PASSWORD) as pure_client:
pure_client.upload('tests/fixture/pure_package/dist/test_package-0.1.dev1-py2.py3-none-any.whl')
with patch.object(
wheeler.Builder, 'build', autospec=True, side_effect=Exception('Should not build!')
) as mock_build:
main(['tests/fixture/sample_test_package.txt', devpi.url + '/' + INDEX,
'--user={}'.format(USER),
'--password={}'.format(PASSWORD), '--pure-index={}'.format(pure_client.url)])
assert not mock_build.called
def test_handles_non_ascii_build_output(devpi, tmpdir, monkeypatch):
monkeypatch.setenv('PIP_INDEX_URL', devpi.url + '/' + PURE_INDEX)
with DevpiClient(devpi.server_url + '/' + PURE_INDEX,
USER,
PASSWORD) as client:
client.upload('tests/fixture/non-ascii_package/dist/non-ascii-package-0.1.dev1.tar.gz')
junit_filename = str(tmpdir.join('junit.xml'))
# our main assertion is that the main does not fail
main(['tests/fixture/sample_non-ascii.txt', devpi.url + '/' + INDEX,
'--user={}'.format(USER),
'--password={}'.format(PASSWORD), '--junit-xml', junit_filename])
# check that the build failure shows up in the build output
assert 'No non-ascii 4 you!' in open(junit_filename).read()
def test_reports_junit_xml(self):
with TestServer(USERS, INDICES) as devpi:
with DevpiClient(devpi.server_url + '/' + INDEX, USER,
PASSWORD) as client:
client.upload(
'tests/fixture/pure_package/dist/test_package-0.1.dev1-py2.py3-none-any.whl'
)
tempdir = tempfile.mkdtemp()
try:
junit_filename = os.path.join(tempdir, 'junit.xml')
main([
'tests/fixture/sample_junit.txt', devpi.url + '/' + INDEX,
USER, PASSWORD, '--junit-xml', junit_filename
])
self._assert_junit_xml_content(junit_filename)
finally:
shutil.rmtree(tempdir)
def test_dry_run(devpi, tmpdir):
"""
Test that a dry run produces the same ``junit.xml`` as a run without dry-run but does not modify the server
state.
"""
with DevpiClient(devpi.server_url + '/' + INDEX,
USER, PASSWORD) as client:
client.upload('tests/fixture/pure_package/dist/test_package-0.1.dev1-py2.py3-none-any.whl')
junit_filename = str(tmpdir.join('junit.xml'))
main([
'tests/fixture/sample_junit.txt',
devpi.url + '/' + INDEX,
'--user={}'.format(USER),
'--password={}'.format(PASSWORD),
'--junit-xml', junit_filename,
'--dry-run',
])
_assert_junit_xml_content(junit_filename)
assert not _package_version_exists(devpi, INDEX, 'progressbar', '2.2')
def main(args=None):
parser = argparse.ArgumentParser(
description=
'Create wheels for all given project versions and upload them to the given index.'
)
parser.add_argument(
'requirements',
help=
'requirements.txt style file specifying which project versions to package.'
)
parser.add_argument('index',
help='The index to upload the packaged software to.')
parser.add_argument('user', help='The user to log in as.')
parser.add_argument('password', help='Password of the user.')
parser.add_argument(
'--blacklist',
help=
'Packages matched by this requirements.txt style file will never be build.'
)
parser.add_argument(
'--pure-index',
help=
'The index to use for pure packages. Any non-pure package will be uploaded '
'to the index given as positional argument. Packages already found in the '
'pure index will not be built, either.')
parser.add_argument(
'--junit-xml',
help=
'Write information about the build success / failure to a JUnit-compatible XML file.'
)
parser.add_argument(
'--dry-run',
help=
'Build missing wheels, but do not modify the state of the devpi server.',
action='store_true')
parser.add_argument(
'--client-cert',
help='Client key to use to authenticate with the devpi server.',
default=None)
args = parser.parse_args(args=args)
packages = requirements.read(args.requirements)
with wheeler.Builder() as builder, DevpiClient(
args.index, args.user, args.password,
client_cert=args.client_cert) as devpi_client:
if args.pure_index:
with DevpiClient(
args.pure_index,
args.user,
args.password,
client_cert=args.client_cert) as pure_index_client:
processor = Processor(builder,
devpi_client,
args.blacklist,
pure_index_client,
junit_xml=args.junit_xml,
dry_run=args.dry_run)
processor.build_packages(packages)
else:
processor = Processor(builder,
devpi_client,
args.blacklist,
junit_xml=args.junit_xml,
dry_run=args.dry_run)
processor.build_packages(packages)