def package_updates(session, packages_dict, include_prerelease=False):
"""
I *think* this returns a list of packages that need updating...
Seems like the route for this fuction expects args from a url that look
like:
/updates?packageids='pkg1'|'pkg2'&versions='vers1'|'vers2'
where "|" might be encoded as %7C
Parameters
----------
session : :class:`sqlalchemy.orm.session.Session`
packages_dict : dict
Dict of {package.name, version}.
include_prerelease : bool
"""
logger.debug("db.package_updates(...)")
package_versions = [
"{}~~{}".format(pkg, vers) for pkg, vers in packages_dict.items()
]
query = (session.query(Version).join(Package).filter(
Version.version == Package.latest_version).filter(
Version.package_id.in_(packages_dict.keys())).filter(
~Version.thing.in_(package_versions)))
return query.order_by(Version.package_id).all()
def download(pkg_id=None, version=None):
"""
Indirectly used by `nuget install`.
"""
logger.debug("Route: /download")
if pkg_id is None:
pkg_id = request.args.get('Id')
if version is None:
version = request.args.get('Version')
pkg_name = db.find_pkg_by_id(session, pkg_id).name
path = core.get_package_path(pkg_name, version)
# Make sure we're trying to download from our package direcory
path = Path(current_app.config['SERVER_PATH']
) / current_app.config['PACKAGE_DIR'] / path
abs_path = Path.cwd() / path
logger.debug("Path to package: %s" % abs_path)
db.increment_download_count(session, pkg_name, version)
filename = "{}.{}.nupkg".format(pkg_name, version)
logger.debug("File name: %s" % filename)
logger.debug("sending file")
result = send_file(str(abs_path),
mimetype="application/zip",
as_attachment=True,
attachment_filename=filename)
header_str = str(result.headers).replace("\r\n", "\r\n ").strip()
logger.debug("Header: \n {}".format(header_str))
return result, 200
def determine_dependencies(metadata_element, namespace):
""""""
# TODO: python-ify
logger.debug("Parsing dependencies.")
dependencies = []
dep = metadata_element.find('nuspec:dependencies', namespace)
if et.iselement(dep):
logger.debug("Found dependencies.")
dep_no_fw = dep.findall('nuspec:dependency', namespace)
if dep_no_fw:
logger.debug("Found dependencies not specific to any framework.")
for dependency in dep_no_fw:
d = {
'framework': None,
'id': str(dependency.attrib['id']),
'version': str(dependency.attrib['version']),
}
dependencies.append(d)
dep_fw = dep.findall('nuspec:group', namespace)
if dep_fw:
logger.debug("Found dependencies specific to a framework")
for group in dep_fw:
group_elem = group.findall('nuspec:dependency', namespace)
for dependency in group_elem:
d = {
'framework': str(group.attrib['targetFramework']),
'id': str(dependency.attrib['id']),
'version': str(dependency.attrib['version']),
}
dependencies.append(d)
else:
logger.debug("No dependencies found.")
return dependencies
def hash_file(file, algorithm=hashlib.md5):
"""
Parameters
----------
file : :class:`pathlib.Path` object
algorithm : Hash algorithm constructor
One of the hash algorithms present in the `hashlib` module.
Returns:
--------
hash_ : bytes
Note that the returned `hash_` value is in binary. To make it a human
readable string, use `binascii.hexlify(hash_).decode('utf-8')`.
"""
file = str(file)
logger.debug("Hashing file %s" % file)
m = algorithm()
with open(file, 'rb') as openf:
m.update(openf.read())
hash_ = m.hexdigest()
logger.debug("%s hash: %s, %s" % (algorithm.__name__, hash_, file))
return m.digest()
def insert_or_update_package(session, package_name, title, latest_version):
"""
Parameters
----------
session : :class:`sqlalchemy.orm.session.Session`
package_name : str
The NuGet name of the package - the "id" tag in the NuSpec file.
title : str
latest_version : str
"""
logger.debug("db.insert_or_update_package(...)")
sql = session.query(Package).filter(Package.name == package_name)
obj = sql.one_or_none()
if obj is None:
pkg = Package(name=package_name,
title=title,
latest_version=latest_version)
session.add(pkg)
else:
sql.update({
Package.name: package_name,
Package.title: title,
Package.latest_version: latest_version
})
session.commit()
def save_file(file, pkg_name, version):
"""
Parameters
----------
file : :class:`werkzeug.datastructures.FileStorage` object
The file as retrieved by Flask.
pkg_name : str
version : str
Returns:
--------
local_path : :class:`pathlib.Path`
The path to the saved file.
"""
# Save the package file to the local package dir. Thus far it's
# just been floating around in magic Flask land.
server_path = Path(current_app.config['SERVER_PATH'])
package_dir = Path(current_app.config['PACKAGE_DIR'])
local_path = server_path / package_dir
local_path = local_path / pkg_name / (version + ".nupkg")
# Check if the package's directory already exists. Create if needed.
create_parent_dirs(local_path)
logger.debug("Saving uploaded file to filesystem.")
try:
file.save(str(local_path))
except Exception as err: # TODO: specify exceptions
logger.error("Unknown exception: %s" % err)
raise err
else:
logger.info("Succesfully saved package to '%s'" % str(local_path))
return local_path
def index():
"""
Used for web interface.
"""
logger.debug("Route: /index")
version = "<TODO>"
return render_template("web/index.html", server_version=version)
def delete_version(session, package_name, version):
"""
Parameters
----------
session : :class:`sqlalchemy.orm.session.Session`
package_name : str
The NuGet name of the package - the "id" tag in the NuSpec file.
version : str
"""
msg = "db.delete_version({}, {})"
logger.debug(msg.format(package_name, version))
sql = (session.query(Version).join(Package).filter(
Package.name == package_name).filter(Version.version == version))
version = sql.one()
pkg = version.package
session.delete(version)
# update the Package.latest_version value, or delete the Package
versions = (session.query(Version).join(Package).filter(
Package.name == package_name)).all()
if len(versions) > 0:
pkg.latest_version = max(v.version for v in versions)
else:
logger.info("No more versions exist. Deleting package %s" % pkg)
session.delete(pkg)
session.commit()
def delete(package=None, version=None):
"""
Used by `nuget delete`.
"""
logger.debug("Route: /delete")
if not core.require_auth(request.headers):
return "api_error: Missing or Invalid API key", 401 # TODO
if package is not None:
pkg_name = package
else:
pkg_name = request.args.get('id')
if version is None:
version = request.args.get('version')
path = core.get_package_path(pkg_name, version)
path = Path(current_app.config['SERVER_PATH']
) / current_app.config['PACKAGE_DIR'] / path
if path.exists():
path.unlink()
try:
db.delete_version(session, pkg_name, version)
except NoResultFound:
msg = "Version '{}' of Package '{}' was not found."
return msg.format(pkg_name, version), 404
logger.info("Sucessfully deleted package %s version %s." %
(pkg_name, version))
return '', 204
def count():
"""
Not sure which nuget command uses this...
"""
logger.debug("Route: /count")
resp = make_response(str(db.count_packages(session)))
resp.headers['Content-Type'] = 'text/plain; charset=utf-8'
return resp
def find_pkg_by_id(session, package_id):
query = (session.query(Package).filter(Package.package_id == package_id))
# TODO: Error handling
result = query.one()
logger.debug(result)
return result
def extract_nuspec(file):
"""
Parameters
----------
file : :class:`pathlib.Path` object or str
The file as retrieved by Flask.
"""
pkg = ZipFile(str(file), 'r')
logger.debug("Parsing uploaded file.")
nuspec_file = None
pattern = re.compile(r'^.*\.nuspec$', re.IGNORECASE)
nuspec_file = list(filter(pattern.search, pkg.namelist()))
if len(nuspec_file) > 1:
logger.error("Multiple NuSpec files found within the package.")
raise ApiException("api_error: multiple nuspec files found")
elif len(nuspec_file) == 0:
logger.error("No NuSpec file found in the package.")
raise ApiException("api_error: nuspec file not found") # TODO
nuspec_file = nuspec_file[0]
with pkg.open(nuspec_file, 'r') as openf:
nuspec_string = openf.read()
logger.debug("NuSpec string:")
logger.debug(nuspec_string)
logger.debug("Parsing NuSpec file XML")
nuspec = et.fromstring(nuspec_string)
if not et.iselement(nuspec):
msg = "`nuspec` expected to be type `xml...Element`. Got {}"
raise TypeError(msg.format(type(nuspec)))
return nuspec
def count_packages(session):
"""
Count the number of packages on the server.
Parameters
----------
session : :class:`sqlalchemy.orm.session.Session`
Returns
-------
int
"""
logger.debug("db.count_packages()")
return session.query(func.count(Package.package_id)).scalar()
def insert_version(session, **kwargs):
"""Insert a new version of an existing package."""
logger.debug("db.insert_version(...)")
kwargs['created'] = dt.datetime.utcnow()
if 'dependencies' in kwargs:
kwargs['dependencies'] = json.dumps(kwargs['dependencies'])
if 'is_prerelease' not in kwargs:
kwargs['is_prerelease'] = 0
if 'require_license_acceptance' not in kwargs:
kwargs['require_license_acceptance'] = 0
version = Version(**kwargs)
session.add(version)
session.commit()
logger.debug(version)
def extract_namespace(nuspec):
"""
Extract the namespce from the NuSpec file.
Parameters
----------
nuspec : :class:`lxml.etree.Element` object
The parsed nuspec data.
Returns
-------
namespace : str
"""
namespace = nuspec.xpath('namespace-uri(.)')
logger.debug("Found namespace: %s" % namespace)
return namespace
def validate_id_and_version(session, package_name, version):
"""
Not exactly sure what this is supposed to do, but I *think* it simply
makes sure that the given pacakge_id and version exist... So that's
what I've decided to make it do.
Parameters
----------
session : :class:`sqlalchemy.orm.session.Session`
package_name : str
The NuGet name of the package - the "id" tag in the NuSpec file.
version : str
"""
logger.debug("db.validate_id_and_version(...)")
query = (session.query(Version).join(Package).filter(
Package.name == package_name).filter(Version.version == version))
return session.query(query.exists()).scalar()
def updates():
"""
I thought this was `nuget restore` but that looks to be
`GET http://localhost:5000/Packages(Id='xunit',Version='2.3.1')`
"""
logger.debug("Route: /updates")
ids = request.args.get('packageids').strip("'").split('|')
versions = request.args.get('versions').strip("'").split('|')
include_prerelease = request.args.get('includeprerelease', default=False)
pkg_to_vers = {k: v for k, v in zip(ids, versions)}
results = db.package_updates(session, pkg_to_vers, include_prerelease)
feed = FeedWriter('GetUpdates', request.url_root)
resp = make_response(feed.write_to_output(results))
resp.headers['Content-Type'] = FEED_CONTENT_TYPE_HEADER
return resp
def search_packages(session,
include_prerelease=False,
order_by=desc(Version.version_download_count),
filter_=None,
search_query=None):
"""
Parameters
----------
session : :class:`sqlalchemy.orm.session.Session`
include_prerelease : bool
order_by : :class:`sqlalchemy.sql.operators.ColumnOperators`
filder_ : str
One of ('is_absolute_latest_version', 'is_latest_version').
search_query : str
"""
logger.debug("db.search_packages(...)")
query = session.query(Version).join(Package)
if search_query is not None:
search_query = "%" + search_query + "%"
query = query.filter(
or_(Package.name.like(search_query),
Package.title.like(search_query)))
if not include_prerelease:
query = query.filter(Version.is_prerelease.isnot(True))
known_filters = ('is_absolute_latest_version', 'IsLatestVersion')
if filter_ is None:
pass
elif filter_ in known_filters:
query = query.filter(Version.version == Package.latest_version)
else:
raise ValueError("Unknown filter '{}'".format(filter_))
if order_by is not None:
query = query.order_by(order_by)
results = query.all()
logger.debug("Found %d results." % len(results))
return results
def increment_download_count(session, package_name, version):
"""
Increment the download count for a given package version.
Parameters
----------
session : :class:`sqlalchemy.orm.session.Session`
package_name : str
The NuGet name of the package - the "id" tag in the NuSpec file.
version : str
"""
msg = "db.increment_download_count(%s, %s)"
logger.debug(msg % (package_name, version))
obj = (session.query(Version).join(Package).filter(
Package.name == package_name).filter(
Version.version == version)).one()
obj.version_download_count += 1
obj.package.download_count += 1
session.commit()
logger.debug("Finished increment_download_count")
def meta():
"""
The `nuget list` command calls this route.
"""
logger.debug("route: /$metadata")
resp = make_response("""<?xml version="1.0" encoding="utf-8"?>
<service xml:base="{}" xmlns="http://www.w3.org/2007/app" xmlns:atom="http://www.w3.org/2005/Atom">
<workspace>
<atom:title type="text">Default</atom:title>
<collection href="Packages">
<atom:title type="text">Packages</atom:title>
</collection>
</workspace>
</service>""".format(request.url_root))
resp.headers['Content-Type'] = 'application/atomsvc+xml; charset=utf-8'
resp.headers['Cache-Control'] = 'no-cache'
resp.headers['Pragma'] = 'no-cache'
resp.headers['Expires'] = '-1'
logger.debug("Returning resp: %s" % str(resp))
return resp
def encode_file(file, hash_):
"""
Parameters
----------
file : :class:`pathlib.Path` object
hash_ : bytes
The value returned by `hash_file()`.
Returns:
--------
hash_ : bytes
filesize : int
"""
logger.debug("Encoding in Base64.")
hash_ = base64.b64encode(hash_)
# Get the filesize of the uploaded file. Used later.
filesize = os.path.getsize(file)
logger.debug("File size: %d bytes" % filesize)
return hash_.decode('utf-8'), filesize
def find_by_pkg_name(session, package_name, version=None):
"""
Find a package by name. If version is `None`, returns all versions.
Parameters
----------
session : :class:`sqlalchemy.orm.session.Session`
package_name : str
The NuGet name of the package - the "id" tag in the NuSpec file.
version : str
The version of the package to download. If `None`, then return all
versions.
Returns
-------
results : list of :class:`Version`
"""
logger.debug("db.find_by_pkg_name('%s', version='%s')" %
(package_name, version))
query = (session.query(Version).join(Package).filter(
Package.name == package_name))
stmt = query.statement.compile(dialect=sqlite.dialect(),
compile_kwargs={"literal_binds": True})
logger.debug(stmt)
if version:
query = query.filter(Version.version == version)
query.order_by(desc(Version.version))
results = query.all()
logger.info("Found %d results." % len(results))
logger.debug(results)
return results
def create_app():
"""
Application Factory.
"""
instance_path = Path("/var/www/pynuget")
config_file = instance_path / "config.py"
testing = os.getenv(ENV_VAR, None) == 'TESTING'
app = Flask(__name__)
app.config.from_object('pynuget.default_config')
# Override some default vars if we're running using the flask dev server.
if os.getenv(ENV_VAR, None) == "LOCAL_DEV":
app.config['LOG_PATH'] = os.getenv('PYNUGET_LOG_PATH')
app.config['SERVER_PATH'] = os.getenv('PYNUGET_SERVER_PATH')
if config_file.exists():
logger.debug("Found config file: %s. Loading..." % str(config_file))
app.config.from_pyfile(str(config_file))
# Update logging to also log to a file.
# This *should* modify the logger that was created in __init__.py...
if not testing:
setup_logging(to_console=False, to_file=True,
log_path=app.config['LOG_PATH'])
# Register blueprints
app.register_blueprint(pages)
@app.teardown_appcontext
def teardown_db_session(exception):
session = getattr(g, 'session', None)
if session is not None:
session.close()
return app
def search():
"""
Used by `nuget list`.
"""
logger.debug("Route: /search")
logger.debug(request.args)
# TODO: Cleanup this and db.search_pacakges call sig.
include_prerelease = request.args.get('includePrerelease', default=False)
order_by = request.args.get('$orderBy', default='Id')
filter_ = request.args.get('$filter', default=None)
top = request.args.get('$top', default=30)
skip = request.args.get('$skip', default=0)
sem_ver_level = request.args.get('semVerLevel', default='2.0.0')
search_query = request.args.get('searchTerm', default=None)
target_framework = request.args.get('targetFramework', default='')
# Some of the terms have quotes surrounding them
search_query = search_query.strip("'")
target_framework = target_framework.strip("'")
results = db.search_packages(
session,
include_prerelease=include_prerelease,
#order_by=order_by,
filter_=filter_,
search_query=search_query,
)
feed = FeedWriter('Search', request.url_root)
resp = make_response(feed.write_to_output(results))
logger.debug("Finished FeedWriter.write_to_output")
resp.headers['Content-Type'] = FEED_CONTENT_TYPE_HEADER
# Keep this line for future debugging. Will fill up the logs *very*
# quickly if there are a lot of packages on the server.
# logger.debug(resp.data.decode('utf-8').replace('><',' >\n<'))
return resp
def find_by_id(func_args=None):
"""
Used by `nuget install`.
It looks like the NuGet client expects this to send back a list of all
versions for the package, and then the client handles extraction of
an individual version.
Note that this is different from the newer API
/Packages(Id='pkg_name',Version='0.1.3')
which appears to move the version selection to server-side.
"""
logger.debug("Route: /find_by_id")
logger.debug(" args: {}".format(request.args))
logger.debug(" header: {}".format(request.headers))
if func_args is not None:
# Using the newer API
logger.debug(func_args)
# Parse the args. From what I can tell, the only things sent here are:
# 'Id': the name of the package
# 'Version': the version string.
# Looks like:
# "Id='NuGetTest',Version='0.0.2'"
# Naive implementation
regex = re.compile(r"^Id='(?P<name>.+)',Version='(?P<version>.+)'$")
match = regex.search(func_args)
if match is None:
msg = "Unable to parse the arg string `{}`!"
logger.error(msg.format(func_args))
return msg.format(func_args), 500
pkg_name = match.group('name')
version = match.group('version')
logger.debug("{}, {}".format(pkg_name, version))
else:
# old API
pkg_name = request.args.get('id')
sem_ver_level = request.args.get('semVerLevel', default=None)
version = None
# Some terms are quoted
pkg_name = pkg_name.strip("'")
results = db.find_by_pkg_name(session, pkg_name, version)
logger.debug(results)
feed = FeedWriter('FindPackagesById', request.url_root)
resp = make_response(feed.write_to_output(results))
resp.headers['Content-Type']
# This will spam logs!
#logger.debug(resp.data.decode('utf-8').replace('><',' >\n<'))
return resp
def push():
"""
Used by `nuget push`.
"""
logger.debug("push()")
logger.debug(" args: {}".format(request.args))
logger.debug(" header: {}".format(request.headers))
if not core.require_auth(request.headers):
return "api_error: Missing or Invalid API key", 401
logger.debug("Checking for uploaded file.")
if 'package' not in request.files:
logger.error("Package file was not uploaded.")
return "error: File not uploaded", 409
file = request.files['package']
# Save the file to a temporary location
file = core.save_file(file, "_temp", str(uuid4()))
# Open the zip file that was sent and extract out the .nuspec file."
try:
nuspec = core.extract_nuspec(file)
except Exception as err:
logger.error("Exception: %s" % err)
return "api_error: Zero or multiple nuspec files found", 400
# The NuSpec XML file uses namespaces.
ns = {'nuspec': core.extract_namespace(nuspec)}
# Make sure both the ID and the version are provided in the .nuspec file.
try:
metadata, pkg_name, version = core.parse_nuspec(nuspec, ns)
except ApiException as err:
return str(err), 400
except Exception as err:
logger.error(err)
return str(err), 400
valid_id = re.compile('^[A-Z0-9\.\~\+\_\-]+$', re.IGNORECASE)
# Make sure that the ID and version are sane
if not re.match(valid_id, pkg_name) or not re.match(valid_id, version):
logger.error("Invalid ID or version.")
return "api_error: Invlaid ID or Version", 400
# and that we don't already have that ID+version in our database
if db.validate_id_and_version(session, pkg_name, version):
logger.error("Package %s version %s already exists" %
(pkg_name, version))
return "api_error: Package version already exists", 409
# Hash the uploaded file and encode the hash in Base64. For some reason.
try:
# rename our file.
# Check if the package's directory already exists. Create if needed.
new = file.parent.parent / pkg_name / (version + ".nupkg")
core.create_parent_dirs(new)
logger.debug("Renaming %s to %s" % (str(file), new))
file.rename(new)
hash_, filesize = core.hash_and_encode_file(str(new))
except Exception as err:
logger.error("Exception: %s" % err)
return "api_error: Unable to save file", 500
try:
dependencies = core.determine_dependencies(metadata, ns)
except Exception as err:
logger.error("Exception: %s" % err)
return "api_error: Unable to parse dependencies.", 400
logger.debug(dependencies)
# and finaly, update our database.
logger.debug("Updating database entries.")
db.insert_or_update_package(session,
package_name=pkg_name,
title=et_to_str(
metadata.find('nuspec:title', ns)),
latest_version=version)
pkg_id = (session.query(
db.Package).filter(db.Package.name == pkg_name).one()).package_id
logger.debug("package_id = %d" % pkg_id)
db.insert_version(
session,
authors=et_to_str(metadata.find('nuspec:authors', ns)),
copyright_=et_to_str(metadata.find('nuspec:copyright', ns)),
dependencies=dependencies,
description=et_to_str(metadata.find('nuspec:description', ns)),
package_hash=hash_,
package_hash_algorithm='SHA512',
package_size=filesize,
icon_url=et_to_str(metadata.find('nuspec:iconUrl', ns)),
is_prerelease='-' in version,
license_url=et_to_str(metadata.find('nuspec:licenseUrl', ns)),
owners=et_to_str(metadata.find('nuspec:owners', ns)),
package_id=pkg_id,
project_url=et_to_str(metadata.find('nuspec:projectUrl', ns)),
release_notes=et_to_str(metadata.find('nuspec:releaseNotes', ns)),
require_license_acceptance=et_to_str(
metadata.find('nuspec:requireLicenseAcceptance', ns)) == 'true',
tags=et_to_str(metadata.find('nuspec:tags', ns)),
title=et_to_str(metadata.find('nuspec:id', ns)),
version=version,
)
logger.info(
"Sucessfully updated database entries for package %s version %s." %
(pkg_name, version))
resp = make_response('', 201)
return resp
