def test_update_all_packages_with_ignorelist(monkeypatch):
"""Test calling update_all_packages()"""
public_pkg_1 = PkgFile("Flask", "1.0")
public_pkg_2 = PkgFile("requests", "1.0")
private_pkg_1 = PkgFile("my_private_pkg", "1.0")
private_pkg_2 = PkgFile("my_other_private_pkg", "1.0")
roots_mock = {
"/opt/pypi": [
public_pkg_1,
private_pkg_1,
],
"/data/pypi": [public_pkg_2, private_pkg_2],
}
def core_listdir_mock(directory):
return roots_mock.get(directory, [])
monkeypatch.setattr(manage.core, "listdir", core_listdir_mock)
monkeypatch.setattr(manage, "update", Mock(return_value=None))
destdir = None
dry_run = False
stable_only = True
update_all_packages(
roots=list(roots_mock.keys()),
destdir=destdir,
dry_run=dry_run,
stable_only=stable_only,
ignorelist=["my_private_pkg", "my_other_private_pkg"],
)
manage.update.assert_called_once_with( # pylint: disable=no-member
frozenset([public_pkg_1, public_pkg_2]), destdir, dry_run, stable_only)
def main(argv=None):
"""Application entrypoint for pypiserver.
This function drives the application (as opposed to the library)
implementation of pypiserver. Usage from the commandline will result in
this function being called.
"""
import pypiserver
if argv is None:
# The first item in sys.argv is the name of the python file being
# executed, which we don't need
argv = sys.argv[1:]
config = Config.from_args(argv)
init_logging(
level=config.log_level,
filename=config.log_file,
frmt=config.log_frmt,
stream=config.log_stream,
)
# Check to see if we were asked to run an update command instead of running
# the server
if isinstance(config, UpdateConfig):
from pypiserver.manage import update_all_packages
update_all_packages(
config.roots,
config.download_directory,
dry_run=not config.execute,
stable_only=config.allow_unstable,
ignorelist=config.ignorelist,
)
return
# Fixes #49:
# The gevent server adapter needs to patch some
# modules BEFORE importing bottle!
if config.server_method.startswith("gevent"):
import gevent.monkey # @UnresolvedImport
gevent.monkey.patch_all()
from pypiserver import bottle
bottle.debug(config.verbosity > 1)
bottle._stderr = ft.partial(_logwrite, logging.getLogger(bottle.__name__),
logging.INFO)
# Here `app` is a Bottle instance, which we pass to bottle.run() to run
# the server
app = pypiserver.app_from_config(config)
bottle.run(
app=app,
host=config.host,
port=config.port,
server=config.server_method,
)
def test_update_all_packages_with_blacklist(monkeypatch):
"""Test calling update_all_packages()"""
public_pkg_1 = PkgFile('Flask', '1.0')
public_pkg_2 = PkgFile('requests', '1.0')
private_pkg_1 = PkgFile('my_private_pkg', '1.0')
private_pkg_2 = PkgFile('my_other_private_pkg', '1.0')
roots_mock = {
'/opt/pypi': [
public_pkg_1,
private_pkg_1,
],
'/data/pypi': [public_pkg_2, private_pkg_2],
}
def core_listdir_mock(directory):
return roots_mock.get(directory, [])
monkeypatch.setattr(manage.core, 'listdir', core_listdir_mock)
monkeypatch.setattr(
manage.core, 'read_lines',
Mock(return_value=['my_private_pkg', 'my_other_private_pkg']))
monkeypatch.setattr(manage, 'update', Mock(return_value=None))
destdir = None
dry_run = False
stable_only = True
blacklist_file = '/root/pkg_blacklist'
update_all_packages(
roots=list(roots_mock.keys()),
destdir=destdir,
dry_run=dry_run,
stable_only=stable_only,
blacklist_file=blacklist_file,
)
manage.update.assert_called_once_with( # pylint: disable=no-member
frozenset([public_pkg_1, public_pkg_2]), destdir, dry_run, stable_only)
manage.core.read_lines.assert_called_once_with(blacklist_file) # pylint: disable=no-member
def main(argv=None):
import pypiserver
if argv is None:
argv = sys.argv
command = "serve"
c = pypiserver.Configuration(**pypiserver.default_config())
update_dry_run = True
update_directory = None
update_stable_only = True
try:
opts, roots = getopt.getopt(argv[1:], "i:p:a:r:d:P:Uuvxoh", [
"interface=", "passwords=", "authenticate=", "port=", "root=",
"server=", "fallback-url=", "disable-fallback", "overwrite",
"hash-algo=", "log-file=", "log-frmt=", "log-req-frmt=",
"log-res-frmt=", "log-err-frmt=", "welcome=", "cache-control=",
"version", "help"
])
except getopt.GetoptError:
err = sys.exc_info()[1]
sys.exit("usage error: %s" % (err, ))
for k, v in opts:
if k in ("-p", "--port"):
try:
c.port = int(v)
except Exception:
err = sys.exc_info()[1]
sys.exit("Invalid port(%r) due to: %s" % (v, err))
elif k in ("-a", "--authenticate"):
c.authenticated = [
a.lower() for a in re.split("[, ]+", v.strip(" ,")) if a
]
if c.authenticated == ['.']:
c.authenticated = []
else:
actions = ("list", "download", "update")
for a in c.authenticated:
if a not in actions:
errmsg = "Action '%s' for option `%s` not one of %s!"
sys.exit(errmsg % (a, k, actions))
elif k in ("-i", "--interface"):
c.host = v
elif k in ("-r", "--root"):
roots.append(v)
elif k == "--disable-fallback":
c.redirect_to_fallback = False
elif k == "--fallback-url":
c.fallback_url = v
elif k == "--server":
c.server = v
elif k == "--welcome":
c.welcome_file = v
elif k == "--version":
print("pypiserver %s\n" % pypiserver.__version__)
return
elif k == "-U":
command = "update"
elif k == "-x":
update_dry_run = False
elif k == "-u":
update_stable_only = False
elif k == "-d":
update_directory = v
elif k in ("-P", "--passwords"):
c.password_file = v
elif k in ("-o", "--overwrite"):
c.overwrite = True
elif k in ("--hash-algo"):
c.hash_algo = None if not pypiserver.str2bool(v,
c.hash_algo) else v
elif k == "--log-file":
c.log_file = v
elif k == "--log-frmt":
c.log_frmt = v
elif k == "--log-req-frmt":
c.log_req_frmt = v
elif k == "--log-res-frmt":
c.log_res_frmt = v
elif k == "--log-err-frmt":
c.log_err_frmt = v
elif k == "--cache-control":
c.cache_control = v
elif k == "-v":
c.verbosity += 1
elif k in ("-h", "--help"):
print(usage())
sys.exit(0)
if (not c.authenticated and c.password_file != '.'
or c.authenticated and c.password_file == '.'):
auth_err = "When auth-ops-list is empty (-a=.), password-file (-P=%r) must also be empty ('.')!"
sys.exit(auth_err % c.password_file)
if len(roots) == 0:
roots.append(os.path.expanduser("~/packages"))
roots = [os.path.abspath(x) for x in roots]
c.root = roots
verbose_levels = [
logging.WARNING, logging.INFO, logging.DEBUG, logging.NOTSET
]
log_level = list(zip(verbose_levels, range(c.verbosity)))[-1][0]
init_logging(level=log_level, filename=c.log_file, frmt=c.log_frmt)
if command == "update":
from pypiserver.manage import update_all_packages
update_all_packages(roots,
update_directory,
dry_run=update_dry_run,
stable_only=update_stable_only)
return
# Fixes #49:
# The gevent server adapter needs to patch some
# modules BEFORE importing bottle!
if c.server and c.server.startswith('gevent'):
import gevent.monkey # @UnresolvedImport
gevent.monkey.patch_all()
from pypiserver import bottle
if c.server not in bottle.server_names:
sys.exit("unknown server %r. choose one of %s" %
(c.server, ", ".join(bottle.server_names.keys())))
bottle.debug(c.verbosity > 1)
bottle._stderr = ft.partial(pypiserver._logwrite,
logging.getLogger(bottle.__name__),
logging.INFO)
app = pypiserver.app(**vars(c))
bottle.run(app=app, host=c.host, port=c.port, server=c.server)
def main(argv=None):
if argv is None:
argv = sys.argv
global packages
command = "serve"
host = "0.0.0.0"
port = 8080
server = DEFAULT_SERVER
redirect_to_fallback = True
fallback_url = "http://pypi.python.org/simple"
authed_ops_list = ['update']
password_file = None
overwrite = False
verbosity = 1
log_file = None
log_frmt = "g%(asctime)s|%(levelname)s|%(thread)d|%(message)s"
log_req_frmt = "%(bottle.request)s"
log_res_frmt = "%(status)s"
log_err_frmt = "%(body)s: %(exception)s \n%(traceback)s"
add_template = ""
cache_control = None
update_dry_run = True
update_directory = None
update_stable_only = True
try:
opts, roots = getopt.getopt(argv[1:], "i:p:a:r:d:P:Uuvxoh", [
"interface=",
"passwords=",
"authenticate=",
"port=",
"root=",
"server=",
"fallback-url=",
"disable-fallback",
"overwrite",
"log-file=",
"log-frmt=",
"log-req-frmt=",
"log-res-frmt=",
"log-err-frmt=",
"cache-control=",
"version",
"help",
"add-template="
])
except getopt.GetoptError:
err = sys.exc_info()[1]
sys.exit("usage error: %s" % (err,))
for k, v in opts:
if k in ("-p", "--port"):
try:
port = int(v)
except Exception as ex:
sys.exit("Invalid port(%r)!" % v)
elif k in ("-a", "--authenticate"):
authed_ops_list = [a.lower()
for a in re.split("[, ]+", v.strip(" ,"))
if a]
if authed_ops_list == ['.']:
authed_ops_list = []
else:
actions = ("list", "download", "update")
for a in authed_ops_list:
if a not in actions:
errmsg = "Action '%s' for option `%s` not one of %s!"
sys.exit(errmsg % (a, k, actions))
elif k in ("-i", "--interface"):
host = v
elif k in ("-r", "--root"):
roots.append(v)
elif k == "--disable-fallback":
redirect_to_fallback = False
elif k == "--fallback-url":
fallback_url = v
elif k == "--server":
server = v
elif k == "--version":
from pypiserver import __version__
print("pypiserver %s\n" % __version__)
return
elif k == "-U":
command = "update"
elif k == "-x":
update_dry_run = False
elif k == "-u":
update_stable_only = False
elif k == "-d":
update_directory = v
elif k in ("-P", "--passwords"):
password_file = v
elif k in ("-o", "--overwrite"):
overwrite = True
elif k == "--log-file":
log_file = v
elif k == "--log-frmt":
log_frmt = v
elif k == "--log-req-frmt":
log_req_frmt = v
elif k == "--log-res-frmt":
log_res_frmt = v
elif k == "--log-err-frmt":
log_err_frmt = v
elif k == "--cache-control":
cache_control = v
elif k == "-v":
verbosity += 1
elif k == "--add-template":
add_template = v
elif k in ("-h", "--help"):
print(usage())
sys.exit(0)
if (not authed_ops_list and password_file != '.' or
authed_ops_list and password_file == '.'):
auth_err = "When auth-ops-list is empty (-a=.), password-file (-P=%r) must also be empty ('.')!"
sys.exit(auth_err % password_file)
if len(roots) == 0:
roots.append(os.path.expanduser("~/packages"))
roots=[os.path.abspath(x) for x in roots]
verbose_levels=[
logging.WARNING, logging.INFO, logging.DEBUG, logging.NOTSET]
log_level=list(zip(verbose_levels, range(verbosity)))[-1][0]
init_logging(level=log_level, filename=log_file, frmt=log_frmt)
if command == "update":
from pypiserver.manage import update_all_packages
update_all_packages(
roots, update_directory, update_dry_run, stable_only=update_stable_only)
return
# Fixes #49:
# The gevent server adapter needs to patch some
# modules BEFORE importing bottle!
if server and server.startswith('gevent'):
import gevent.monkey # @UnresolvedImport
gevent.monkey.patch_all()
from pypiserver.bottle import server_names, run
if server not in server_names:
sys.exit("unknown server %r. choose one of %s" % (
server, ", ".join(server_names.keys())))
from pypiserver import __version__, app
a=app(
root=roots,
redirect_to_fallback=redirect_to_fallback,
authenticated=authed_ops_list,
password_file=password_file,
fallback_url=fallback_url,
overwrite=overwrite,
log_req_frmt=log_req_frmt, log_res_frmt=log_res_frmt, log_err_frmt=log_err_frmt,
cache_control=cache_control,
add_template=add_template,
)
log.info("This is pypiserver %s serving %r on http://%s:%s\n\n",
__version__, ", ".join(roots), host, port)
run(app=a, host=host, port=port, server=server)
def main(argv=None):
import pypiserver
if argv is None:
argv = sys.argv
command = "serve"
c = pypiserver.Configuration(**pypiserver.default_config())
update_dry_run = True
update_directory = None
update_stable_only = True
try:
opts, roots = getopt.getopt(argv[1:], "i:p:a:r:d:P:Uuvxoh", [
"interface=",
"passwords=",
"authenticate=",
"port=",
"root=",
"server=",
"fallback-url=",
"disable-fallback",
"overwrite",
"hash-algo=",
"log-file=",
"log-frmt=",
"log-req-frmt=",
"log-res-frmt=",
"log-err-frmt=",
"welcome=",
"cache-control=",
"version",
"help"
])
except getopt.GetoptError:
err = sys.exc_info()[1]
sys.exit("usage error: %s" % (err,))
for k, v in opts:
if k in ("-p", "--port"):
try:
c.port = int(v)
except Exception:
err = sys.exc_info()[1]
sys.exit("Invalid port(%r) due to: %s" % (v, err))
elif k in ("-a", "--authenticate"):
if '{' in v:
try:
v = ast.literal_eval(v)
except SyntaxError:
message = 'Could not parse auth string %s! Please ensure string is correctly formatted.' % v
print(message)
sys.exit(message)
if (not isinstance(v, dict) or not all([isinstance(i, list) for i in v.values()])):
message = 'Matrix auth string must be a dict of lists. Please see the README for details.'
print(message)
sys.exit(message)
if isinstance(v, dict):
c.authenticated = {}
for user in v:
c.authenticated[user] = [a.lower() for a in v[user] if a]
if c.authenticated[user] == ['.']:
c.authenticated[user] = []
else:
actions = ("list", "download", "update")
for a in c.authenticated[user]:
if a not in actions:
errmsg = "Action '%s' for option `%s` not one of %s!"
sys.exit(errmsg % (a, k, actions))
else:
c.authenticated = [a.lower()
for a in re.split("[, ]+", v.strip(" ,"))
if a]
if c.authenticated == ['.']:
c.authenticated = []
else:
actions = ("list", "download", "update")
for a in c.authenticated:
if a not in actions:
errmsg = "Action '%s' for option `%s` not one of %s!"
sys.exit(errmsg % (a, k, actions))
elif k in ("-i", "--interface"):
c.host = v
elif k in ("-r", "--root"):
roots.append(v)
elif k == "--disable-fallback":
c.redirect_to_fallback = False
elif k == "--fallback-url":
c.fallback_url = v
elif k == "--server":
c.server = v
elif k == "--welcome":
c.welcome_file = v
elif k == "--version":
print("pypiserver %s\n" % pypiserver.__version__)
return
elif k == "-U":
command = "update"
elif k == "-x":
update_dry_run = False
elif k == "-u":
update_stable_only = False
elif k == "-d":
update_directory = v
elif k in ("-P", "--passwords"):
c.password_file = v
elif k in ("-o", "--overwrite"):
c.overwrite = True
elif k in ("--hash-algo"):
c.hash_algo = None if not pypiserver.str2bool(v, c.hash_algo) else v
elif k == "--log-file":
c.log_file = v
elif k == "--log-frmt":
c.log_frmt = v
elif k == "--log-req-frmt":
c.log_req_frmt = v
elif k == "--log-res-frmt":
c.log_res_frmt = v
elif k == "--log-err-frmt":
c.log_err_frmt = v
elif k == "--cache-control":
c.cache_control = v
elif k == "-v":
c.verbosity += 1
elif k in ("-h", "--help"):
print(usage())
sys.exit(0)
if (not c.authenticated and c.password_file != '.' or
c.authenticated and c.password_file == '.'):
auth_err = "When auth-ops-list is empty (-a=.), password-file (-P=%r) must also be empty ('.')!"
sys.exit(auth_err % c.password_file)
if len(roots) == 0:
roots.append(os.path.expanduser("~/packages"))
roots=[os.path.abspath(x) for x in roots]
c.root = roots
verbose_levels=[
logging.WARNING, logging.INFO, logging.DEBUG, logging.NOTSET]
log_level=list(zip(verbose_levels, range(c.verbosity)))[-1][0]
init_logging(level=log_level, filename=c.log_file, frmt=c.log_frmt)
if command == "update":
from pypiserver.manage import update_all_packages
update_all_packages(roots, update_directory,
dry_run=update_dry_run, stable_only=update_stable_only)
return
# Fixes #49:
# The gevent server adapter needs to patch some
# modules BEFORE importing bottle!
if c.server and c.server.startswith('gevent'):
import gevent.monkey # @UnresolvedImport
gevent.monkey.patch_all()
from pypiserver import bottle
if c.server not in bottle.server_names:
sys.exit("unknown server %r. choose one of %s" % (
c.server, ", ".join(bottle.server_names.keys())))
bottle.debug(c.verbosity > 1)
bottle._stderr = ft.partial(pypiserver._logwrite,
logging.getLogger(bottle.__name__), logging.INFO)
app = pypiserver.app(**vars(c))
bottle.run(app=app, host=c.host, port=c.port, server=c.server)
def main(argv: t.Sequence[str] = None) -> None:
"""Application entrypoint for pypiserver.
This function drives the application (as opposed to the library)
implementation of pypiserver. Usage from the commandline will result in
this function being called.
"""
# pylint: disable=import-outside-toplevel
import pypiserver # pylint: disable=redefined-outer-name
if argv is None:
# The first item in sys.argv is the name of the python file being
# executed, which we don't need
argv = sys.argv[1:]
config = Config.from_args(argv)
init_logging(
level=config.log_level,
filename=config.log_file,
frmt=config.log_frmt,
stream=config.log_stream,
)
# Check to see if we were asked to run an update command instead of running
# the server
if isinstance(config, UpdateConfig):
from pypiserver.manage import update_all_packages
update_all_packages(
config.roots,
config.download_directory,
dry_run=not config.execute,
stable_only=config.allow_unstable,
ignorelist=config.ignorelist,
)
return
# Fixes #49:
# The gevent server adapter needs to patch some
# modules BEFORE importing bottle!
if config.server_method.startswith("gevent"):
import gevent.monkey # @UnresolvedImport
gevent.monkey.patch_all()
from pypiserver import bottle
bottle.debug(config.verbosity > 1)
bottle._stderr = ft.partial( # pylint: disable=protected-access
_logwrite, logging.getLogger(bottle.__name__), logging.INFO)
# Here `app` is a Bottle instance, which we pass to bottle.run() to run
# the server
app = pypiserver.app_from_config(config)
if config.server_method == "gunicorn":
# When bottle runs gunicorn, gunicorn tries to pull its arguments from
# sys.argv. Because pypiserver's arguments don't match gunicorn's,
# this leads to errors.
# Gunicorn can be configured by using a `gunicorn.conf.py` config file
# or by specifying the `GUNICORN_CMD_ARGS` env var. See gunicorn
# docs for more info.
sys.argv = ["gunicorn"]
wsgi_kwargs = {"handler_class": WsgiHandler}
if config.server_method == "auto":
expected_server = guess_auto_server()
extra_kwargs = (wsgi_kwargs
if expected_server is AutoServer.WsgiRef else {})
log.debug(
"Server 'auto' selected. Expecting bottle to run '%s'. "
"Passing extra keyword args: %s",
expected_server.name,
extra_kwargs,
)
else:
extra_kwargs = wsgi_kwargs if config.server_method == "wsgiref" else {}
log.debug("Running bottle with selected server '%s'",
config.server_method)
bottle.run(
app=app,
host=config.host,
port=config.port,
server=config.server_method,
**extra_kwargs,
)
