def package_updates(session, packages_dict, include_prerelease=False): """ I *think* this returns a list of packages that need updating... Seems like the route for this fuction expects args from a url that look like: /updates?packageids='pkg1'|'pkg2'&versions='vers1'|'vers2' where "|" might be encoded as %7C Parameters ---------- session : :class:`sqlalchemy.orm.session.Session` packages_dict : dict Dict of {package.name, version}. include_prerelease : bool """ logger.debug("db.package_updates(...)") package_versions = [ "{}~~{}".format(pkg, vers) for pkg, vers in packages_dict.items() ] query = (session.query(Version).join(Package).filter( Version.version == Package.latest_version).filter( Version.package_id.in_(packages_dict.keys())).filter( ~Version.thing.in_(package_versions))) return query.order_by(Version.package_id).all()
def download(pkg_id=None, version=None): """ Indirectly used by `nuget install`. """ logger.debug("Route: /download") if pkg_id is None: pkg_id = request.args.get('Id') if version is None: version = request.args.get('Version') pkg_name = db.find_pkg_by_id(session, pkg_id).name path = core.get_package_path(pkg_name, version) # Make sure we're trying to download from our package direcory path = Path(current_app.config['SERVER_PATH'] ) / current_app.config['PACKAGE_DIR'] / path abs_path = Path.cwd() / path logger.debug("Path to package: %s" % abs_path) db.increment_download_count(session, pkg_name, version) filename = "{}.{}.nupkg".format(pkg_name, version) logger.debug("File name: %s" % filename) logger.debug("sending file") result = send_file(str(abs_path), mimetype="application/zip", as_attachment=True, attachment_filename=filename) header_str = str(result.headers).replace("\r\n", "\r\n ").strip() logger.debug("Header: \n {}".format(header_str)) return result, 200
def determine_dependencies(metadata_element, namespace): """""" # TODO: python-ify logger.debug("Parsing dependencies.") dependencies = [] dep = metadata_element.find('nuspec:dependencies', namespace) if et.iselement(dep): logger.debug("Found dependencies.") dep_no_fw = dep.findall('nuspec:dependency', namespace) if dep_no_fw: logger.debug("Found dependencies not specific to any framework.") for dependency in dep_no_fw: d = { 'framework': None, 'id': str(dependency.attrib['id']), 'version': str(dependency.attrib['version']), } dependencies.append(d) dep_fw = dep.findall('nuspec:group', namespace) if dep_fw: logger.debug("Found dependencies specific to a framework") for group in dep_fw: group_elem = group.findall('nuspec:dependency', namespace) for dependency in group_elem: d = { 'framework': str(group.attrib['targetFramework']), 'id': str(dependency.attrib['id']), 'version': str(dependency.attrib['version']), } dependencies.append(d) else: logger.debug("No dependencies found.") return dependencies
def hash_file(file, algorithm=hashlib.md5): """ Parameters ---------- file : :class:`pathlib.Path` object algorithm : Hash algorithm constructor One of the hash algorithms present in the `hashlib` module. Returns: -------- hash_ : bytes Note that the returned `hash_` value is in binary. To make it a human readable string, use `binascii.hexlify(hash_).decode('utf-8')`. """ file = str(file) logger.debug("Hashing file %s" % file) m = algorithm() with open(file, 'rb') as openf: m.update(openf.read()) hash_ = m.hexdigest() logger.debug("%s hash: %s, %s" % (algorithm.__name__, hash_, file)) return m.digest()
def insert_or_update_package(session, package_name, title, latest_version): """ Parameters ---------- session : :class:`sqlalchemy.orm.session.Session` package_name : str The NuGet name of the package - the "id" tag in the NuSpec file. title : str latest_version : str """ logger.debug("db.insert_or_update_package(...)") sql = session.query(Package).filter(Package.name == package_name) obj = sql.one_or_none() if obj is None: pkg = Package(name=package_name, title=title, latest_version=latest_version) session.add(pkg) else: sql.update({ Package.name: package_name, Package.title: title, Package.latest_version: latest_version }) session.commit()
def save_file(file, pkg_name, version): """ Parameters ---------- file : :class:`werkzeug.datastructures.FileStorage` object The file as retrieved by Flask. pkg_name : str version : str Returns: -------- local_path : :class:`pathlib.Path` The path to the saved file. """ # Save the package file to the local package dir. Thus far it's # just been floating around in magic Flask land. server_path = Path(current_app.config['SERVER_PATH']) package_dir = Path(current_app.config['PACKAGE_DIR']) local_path = server_path / package_dir local_path = local_path / pkg_name / (version + ".nupkg") # Check if the package's directory already exists. Create if needed. create_parent_dirs(local_path) logger.debug("Saving uploaded file to filesystem.") try: file.save(str(local_path)) except Exception as err: # TODO: specify exceptions logger.error("Unknown exception: %s" % err) raise err else: logger.info("Succesfully saved package to '%s'" % str(local_path)) return local_path
def index(): """ Used for web interface. """ logger.debug("Route: /index") version = "<TODO>" return render_template("web/index.html", server_version=version)
def delete_version(session, package_name, version): """ Parameters ---------- session : :class:`sqlalchemy.orm.session.Session` package_name : str The NuGet name of the package - the "id" tag in the NuSpec file. version : str """ msg = "db.delete_version({}, {})" logger.debug(msg.format(package_name, version)) sql = (session.query(Version).join(Package).filter( Package.name == package_name).filter(Version.version == version)) version = sql.one() pkg = version.package session.delete(version) # update the Package.latest_version value, or delete the Package versions = (session.query(Version).join(Package).filter( Package.name == package_name)).all() if len(versions) > 0: pkg.latest_version = max(v.version for v in versions) else: logger.info("No more versions exist. Deleting package %s" % pkg) session.delete(pkg) session.commit()
def delete(package=None, version=None): """ Used by `nuget delete`. """ logger.debug("Route: /delete") if not core.require_auth(request.headers): return "api_error: Missing or Invalid API key", 401 # TODO if package is not None: pkg_name = package else: pkg_name = request.args.get('id') if version is None: version = request.args.get('version') path = core.get_package_path(pkg_name, version) path = Path(current_app.config['SERVER_PATH'] ) / current_app.config['PACKAGE_DIR'] / path if path.exists(): path.unlink() try: db.delete_version(session, pkg_name, version) except NoResultFound: msg = "Version '{}' of Package '{}' was not found." return msg.format(pkg_name, version), 404 logger.info("Sucessfully deleted package %s version %s." % (pkg_name, version)) return '', 204
def count(): """ Not sure which nuget command uses this... """ logger.debug("Route: /count") resp = make_response(str(db.count_packages(session))) resp.headers['Content-Type'] = 'text/plain; charset=utf-8' return resp
def find_pkg_by_id(session, package_id): query = (session.query(Package).filter(Package.package_id == package_id)) # TODO: Error handling result = query.one() logger.debug(result) return result
def extract_nuspec(file): """ Parameters ---------- file : :class:`pathlib.Path` object or str The file as retrieved by Flask. """ pkg = ZipFile(str(file), 'r') logger.debug("Parsing uploaded file.") nuspec_file = None pattern = re.compile(r'^.*\.nuspec$', re.IGNORECASE) nuspec_file = list(filter(pattern.search, pkg.namelist())) if len(nuspec_file) > 1: logger.error("Multiple NuSpec files found within the package.") raise ApiException("api_error: multiple nuspec files found") elif len(nuspec_file) == 0: logger.error("No NuSpec file found in the package.") raise ApiException("api_error: nuspec file not found") # TODO nuspec_file = nuspec_file[0] with pkg.open(nuspec_file, 'r') as openf: nuspec_string = openf.read() logger.debug("NuSpec string:") logger.debug(nuspec_string) logger.debug("Parsing NuSpec file XML") nuspec = et.fromstring(nuspec_string) if not et.iselement(nuspec): msg = "`nuspec` expected to be type `xml...Element`. Got {}" raise TypeError(msg.format(type(nuspec))) return nuspec
def count_packages(session): """ Count the number of packages on the server. Parameters ---------- session : :class:`sqlalchemy.orm.session.Session` Returns ------- int """ logger.debug("db.count_packages()") return session.query(func.count(Package.package_id)).scalar()
def insert_version(session, **kwargs): """Insert a new version of an existing package.""" logger.debug("db.insert_version(...)") kwargs['created'] = dt.datetime.utcnow() if 'dependencies' in kwargs: kwargs['dependencies'] = json.dumps(kwargs['dependencies']) if 'is_prerelease' not in kwargs: kwargs['is_prerelease'] = 0 if 'require_license_acceptance' not in kwargs: kwargs['require_license_acceptance'] = 0 version = Version(**kwargs) session.add(version) session.commit() logger.debug(version)
def extract_namespace(nuspec): """ Extract the namespce from the NuSpec file. Parameters ---------- nuspec : :class:`lxml.etree.Element` object The parsed nuspec data. Returns ------- namespace : str """ namespace = nuspec.xpath('namespace-uri(.)') logger.debug("Found namespace: %s" % namespace) return namespace
def validate_id_and_version(session, package_name, version): """ Not exactly sure what this is supposed to do, but I *think* it simply makes sure that the given pacakge_id and version exist... So that's what I've decided to make it do. Parameters ---------- session : :class:`sqlalchemy.orm.session.Session` package_name : str The NuGet name of the package - the "id" tag in the NuSpec file. version : str """ logger.debug("db.validate_id_and_version(...)") query = (session.query(Version).join(Package).filter( Package.name == package_name).filter(Version.version == version)) return session.query(query.exists()).scalar()
def updates(): """ I thought this was `nuget restore` but that looks to be `GET http://localhost:5000/Packages(Id='xunit',Version='2.3.1')` """ logger.debug("Route: /updates") ids = request.args.get('packageids').strip("'").split('|') versions = request.args.get('versions').strip("'").split('|') include_prerelease = request.args.get('includeprerelease', default=False) pkg_to_vers = {k: v for k, v in zip(ids, versions)} results = db.package_updates(session, pkg_to_vers, include_prerelease) feed = FeedWriter('GetUpdates', request.url_root) resp = make_response(feed.write_to_output(results)) resp.headers['Content-Type'] = FEED_CONTENT_TYPE_HEADER return resp
def search_packages(session, include_prerelease=False, order_by=desc(Version.version_download_count), filter_=None, search_query=None): """ Parameters ---------- session : :class:`sqlalchemy.orm.session.Session` include_prerelease : bool order_by : :class:`sqlalchemy.sql.operators.ColumnOperators` filder_ : str One of ('is_absolute_latest_version', 'is_latest_version'). search_query : str """ logger.debug("db.search_packages(...)") query = session.query(Version).join(Package) if search_query is not None: search_query = "%" + search_query + "%" query = query.filter( or_(Package.name.like(search_query), Package.title.like(search_query))) if not include_prerelease: query = query.filter(Version.is_prerelease.isnot(True)) known_filters = ('is_absolute_latest_version', 'IsLatestVersion') if filter_ is None: pass elif filter_ in known_filters: query = query.filter(Version.version == Package.latest_version) else: raise ValueError("Unknown filter '{}'".format(filter_)) if order_by is not None: query = query.order_by(order_by) results = query.all() logger.debug("Found %d results." % len(results)) return results
def increment_download_count(session, package_name, version): """ Increment the download count for a given package version. Parameters ---------- session : :class:`sqlalchemy.orm.session.Session` package_name : str The NuGet name of the package - the "id" tag in the NuSpec file. version : str """ msg = "db.increment_download_count(%s, %s)" logger.debug(msg % (package_name, version)) obj = (session.query(Version).join(Package).filter( Package.name == package_name).filter( Version.version == version)).one() obj.version_download_count += 1 obj.package.download_count += 1 session.commit() logger.debug("Finished increment_download_count")
def meta(): """ The `nuget list` command calls this route. """ logger.debug("route: /$metadata") resp = make_response("""<?xml version="1.0" encoding="utf-8"?> <service xml:base="{}" xmlns="http://www.w3.org/2007/app" xmlns:atom="http://www.w3.org/2005/Atom"> <workspace> <atom:title type="text">Default</atom:title> <collection href="Packages"> <atom:title type="text">Packages</atom:title> </collection> </workspace> </service>""".format(request.url_root)) resp.headers['Content-Type'] = 'application/atomsvc+xml; charset=utf-8' resp.headers['Cache-Control'] = 'no-cache' resp.headers['Pragma'] = 'no-cache' resp.headers['Expires'] = '-1' logger.debug("Returning resp: %s" % str(resp)) return resp
def encode_file(file, hash_): """ Parameters ---------- file : :class:`pathlib.Path` object hash_ : bytes The value returned by `hash_file()`. Returns: -------- hash_ : bytes filesize : int """ logger.debug("Encoding in Base64.") hash_ = base64.b64encode(hash_) # Get the filesize of the uploaded file. Used later. filesize = os.path.getsize(file) logger.debug("File size: %d bytes" % filesize) return hash_.decode('utf-8'), filesize
def find_by_pkg_name(session, package_name, version=None): """ Find a package by name. If version is `None`, returns all versions. Parameters ---------- session : :class:`sqlalchemy.orm.session.Session` package_name : str The NuGet name of the package - the "id" tag in the NuSpec file. version : str The version of the package to download. If `None`, then return all versions. Returns ------- results : list of :class:`Version` """ logger.debug("db.find_by_pkg_name('%s', version='%s')" % (package_name, version)) query = (session.query(Version).join(Package).filter( Package.name == package_name)) stmt = query.statement.compile(dialect=sqlite.dialect(), compile_kwargs={"literal_binds": True}) logger.debug(stmt) if version: query = query.filter(Version.version == version) query.order_by(desc(Version.version)) results = query.all() logger.info("Found %d results." % len(results)) logger.debug(results) return results
def create_app(): """ Application Factory. """ instance_path = Path("/var/www/pynuget") config_file = instance_path / "config.py" testing = os.getenv(ENV_VAR, None) == 'TESTING' app = Flask(__name__) app.config.from_object('pynuget.default_config') # Override some default vars if we're running using the flask dev server. if os.getenv(ENV_VAR, None) == "LOCAL_DEV": app.config['LOG_PATH'] = os.getenv('PYNUGET_LOG_PATH') app.config['SERVER_PATH'] = os.getenv('PYNUGET_SERVER_PATH') if config_file.exists(): logger.debug("Found config file: %s. Loading..." % str(config_file)) app.config.from_pyfile(str(config_file)) # Update logging to also log to a file. # This *should* modify the logger that was created in __init__.py... if not testing: setup_logging(to_console=False, to_file=True, log_path=app.config['LOG_PATH']) # Register blueprints app.register_blueprint(pages) @app.teardown_appcontext def teardown_db_session(exception): session = getattr(g, 'session', None) if session is not None: session.close() return app
def search(): """ Used by `nuget list`. """ logger.debug("Route: /search") logger.debug(request.args) # TODO: Cleanup this and db.search_pacakges call sig. include_prerelease = request.args.get('includePrerelease', default=False) order_by = request.args.get('$orderBy', default='Id') filter_ = request.args.get('$filter', default=None) top = request.args.get('$top', default=30) skip = request.args.get('$skip', default=0) sem_ver_level = request.args.get('semVerLevel', default='2.0.0') search_query = request.args.get('searchTerm', default=None) target_framework = request.args.get('targetFramework', default='') # Some of the terms have quotes surrounding them search_query = search_query.strip("'") target_framework = target_framework.strip("'") results = db.search_packages( session, include_prerelease=include_prerelease, #order_by=order_by, filter_=filter_, search_query=search_query, ) feed = FeedWriter('Search', request.url_root) resp = make_response(feed.write_to_output(results)) logger.debug("Finished FeedWriter.write_to_output") resp.headers['Content-Type'] = FEED_CONTENT_TYPE_HEADER # Keep this line for future debugging. Will fill up the logs *very* # quickly if there are a lot of packages on the server. # logger.debug(resp.data.decode('utf-8').replace('><',' >\n<')) return resp
def find_by_id(func_args=None): """ Used by `nuget install`. It looks like the NuGet client expects this to send back a list of all versions for the package, and then the client handles extraction of an individual version. Note that this is different from the newer API /Packages(Id='pkg_name',Version='0.1.3') which appears to move the version selection to server-side. """ logger.debug("Route: /find_by_id") logger.debug(" args: {}".format(request.args)) logger.debug(" header: {}".format(request.headers)) if func_args is not None: # Using the newer API logger.debug(func_args) # Parse the args. From what I can tell, the only things sent here are: # 'Id': the name of the package # 'Version': the version string. # Looks like: # "Id='NuGetTest',Version='0.0.2'" # Naive implementation regex = re.compile(r"^Id='(?P<name>.+)',Version='(?P<version>.+)'$") match = regex.search(func_args) if match is None: msg = "Unable to parse the arg string `{}`!" logger.error(msg.format(func_args)) return msg.format(func_args), 500 pkg_name = match.group('name') version = match.group('version') logger.debug("{}, {}".format(pkg_name, version)) else: # old API pkg_name = request.args.get('id') sem_ver_level = request.args.get('semVerLevel', default=None) version = None # Some terms are quoted pkg_name = pkg_name.strip("'") results = db.find_by_pkg_name(session, pkg_name, version) logger.debug(results) feed = FeedWriter('FindPackagesById', request.url_root) resp = make_response(feed.write_to_output(results)) resp.headers['Content-Type'] # This will spam logs! #logger.debug(resp.data.decode('utf-8').replace('><',' >\n<')) return resp
def push(): """ Used by `nuget push`. """ logger.debug("push()") logger.debug(" args: {}".format(request.args)) logger.debug(" header: {}".format(request.headers)) if not core.require_auth(request.headers): return "api_error: Missing or Invalid API key", 401 logger.debug("Checking for uploaded file.") if 'package' not in request.files: logger.error("Package file was not uploaded.") return "error: File not uploaded", 409 file = request.files['package'] # Save the file to a temporary location file = core.save_file(file, "_temp", str(uuid4())) # Open the zip file that was sent and extract out the .nuspec file." try: nuspec = core.extract_nuspec(file) except Exception as err: logger.error("Exception: %s" % err) return "api_error: Zero or multiple nuspec files found", 400 # The NuSpec XML file uses namespaces. ns = {'nuspec': core.extract_namespace(nuspec)} # Make sure both the ID and the version are provided in the .nuspec file. try: metadata, pkg_name, version = core.parse_nuspec(nuspec, ns) except ApiException as err: return str(err), 400 except Exception as err: logger.error(err) return str(err), 400 valid_id = re.compile('^[A-Z0-9\.\~\+\_\-]+$', re.IGNORECASE) # Make sure that the ID and version are sane if not re.match(valid_id, pkg_name) or not re.match(valid_id, version): logger.error("Invalid ID or version.") return "api_error: Invlaid ID or Version", 400 # and that we don't already have that ID+version in our database if db.validate_id_and_version(session, pkg_name, version): logger.error("Package %s version %s already exists" % (pkg_name, version)) return "api_error: Package version already exists", 409 # Hash the uploaded file and encode the hash in Base64. For some reason. try: # rename our file. # Check if the package's directory already exists. Create if needed. new = file.parent.parent / pkg_name / (version + ".nupkg") core.create_parent_dirs(new) logger.debug("Renaming %s to %s" % (str(file), new)) file.rename(new) hash_, filesize = core.hash_and_encode_file(str(new)) except Exception as err: logger.error("Exception: %s" % err) return "api_error: Unable to save file", 500 try: dependencies = core.determine_dependencies(metadata, ns) except Exception as err: logger.error("Exception: %s" % err) return "api_error: Unable to parse dependencies.", 400 logger.debug(dependencies) # and finaly, update our database. logger.debug("Updating database entries.") db.insert_or_update_package(session, package_name=pkg_name, title=et_to_str( metadata.find('nuspec:title', ns)), latest_version=version) pkg_id = (session.query( db.Package).filter(db.Package.name == pkg_name).one()).package_id logger.debug("package_id = %d" % pkg_id) db.insert_version( session, authors=et_to_str(metadata.find('nuspec:authors', ns)), copyright_=et_to_str(metadata.find('nuspec:copyright', ns)), dependencies=dependencies, description=et_to_str(metadata.find('nuspec:description', ns)), package_hash=hash_, package_hash_algorithm='SHA512', package_size=filesize, icon_url=et_to_str(metadata.find('nuspec:iconUrl', ns)), is_prerelease='-' in version, license_url=et_to_str(metadata.find('nuspec:licenseUrl', ns)), owners=et_to_str(metadata.find('nuspec:owners', ns)), package_id=pkg_id, project_url=et_to_str(metadata.find('nuspec:projectUrl', ns)), release_notes=et_to_str(metadata.find('nuspec:releaseNotes', ns)), require_license_acceptance=et_to_str( metadata.find('nuspec:requireLicenseAcceptance', ns)) == 'true', tags=et_to_str(metadata.find('nuspec:tags', ns)), title=et_to_str(metadata.find('nuspec:id', ns)), version=version, ) logger.info( "Sucessfully updated database entries for package %s version %s." % (pkg_name, version)) resp = make_response('', 201) return resp