def update_latest(self, version, save=True):
assert isinstance(version, PackageVersion)
assert version.package == self
if not self.latest or semver.Version(version.version) > semver.Version(self.latest.version):
self.latest = version
if save:
self.save()
def check_platforms(cls, platforms):
if not platforms:
return
cur_platform = pf.get_platform()
cur_version = semver.Version(cur_platform.version)
cur_kernel = semver.Version(cur_platform.kernel)
for platform_name, cpus in platforms.iteritems():
if isinstance(cpus, (str, int)):
cpus = [int(cpus)] if cpus.strip() else None
cpus = cpus or [32, 64]
# Check cpu
if cur_platform.cpu not in cpus:
continue
# Check system
system, version = pf.split_platform(platform_name)
if system in (cur_platform.system, cur_platform.dist):
if not version:
return
elif (system == 'Windows' and semver.Spec(version).match(
cur_kernel)) or \
semver.Spec(version).match(cur_version):
return
raise NotMatchError(
"Current {!r} doesn't match platform require: {!r}".format(
cur_platform, platforms))
def check_version(self, upgrade_version):
self.http_handler.log_ok('Checking version')
current_version = get_agent_version()
if semver.Version(upgrade_version) == semver.Version(current_version):
raise MessageError('Upgrade version: {} == agent current version: '
'{}, will not upgrade'.format(
upgrade_version, current_version))
self.http_handler.log_ok('Version valid')
def get(self, package, version, scope=None):
package = str(refstring.Package(scope, package))
try:
version = semver.Selector(version)
except ValueError as exc:
flask.abort(404)
# Check our database of available packages.
package_obj = Package.objects(name=package).first()
if not package_obj:
return self.not_found(package, version)
# Find a matching version.
versions = PackageVersion.objects(package=package_obj)
key = lambda x: semver.Version(x.version)
best = version.best_of(versions, key=key)
if not best:
return self.not_found(package, version)
try:
return json.loads(best.manifest)
except json.JSONDecodeError:
app.logger.error("invalid manifest found: {}@{}".format(best.package.name, best.version))
flask.abort(505)
def version():
"""
Creates a version
:return: a version type
:rtype: Version
"""
return semver.Version(
major=1, minor=2, patch=3, prerelease="alpha.1.2", build="build.11.e0f985a"
)
def install_dependencies(self, deps, current_dir):
"""
Install all dependencies specified in the dictionary *deps*.
"""
install_deps = []
for name, req in deps.items():
if not isinstance(req, manifest.Requirement):
req = manifest.Requirement.from_line(req, name=name)
try:
have_package = self.find_package(name, req.internal)
if isinstance(have_package, InvalidPackage):
raise PackageNotFound
if have_package.get('__is_link'):
# If the package is a link -- try re installing always as the
# link may have been invalidated.
raise PackageNotFound
except PackageNotFound as exc:
install_deps.append((name, req))
else:
if req.type == 'registry':
if not req.selector(semver.Version(have_package['version'])):
print(' Warning: Dependency "{}@{}" unsatisfied, have "{}" installed'
.format(name, req.selector, have_package.identifier))
else:
print(' Skipping satisfied dependency "{}@{}", have "{}" installed'
.format(name, req.selector, have_package.identifier))
else:
# Must be a Git URL or a relative path.
print(' Skipping "{}" dependency, have "{}" installed'
.format(req.type, name, have_package.identifier))
if self.recursive:
self.install_dependencies_for(have_package, have_package.directory, None)
if not install_deps:
return True
for name, req in install_deps:
print(' Installing "{}" ({})'.format(name, req))
if req.type == 'registry':
if not self.install_from_registry(name, req.selector, internal=req.internal, regs=req.registry)[0]:
return False
elif req.type == 'git':
if not self.install_from_git(req.git_url, req.recursive, req.internal)[0]:
return False
elif req.type == 'path':
path = req.path
if not os.path.isabs(path):
path = os.path.join(current_dir, path)
if not self.install_from_directory(path, req.link, internal=req.internal)[0]:
return False
else:
raise RuntimeError('unexpected dependency data: "{}" -> {!r}'.format(name, req))
return True
def drop(all, package, user, reown, yes, keep_files):
"""
Drop the registry data, a specific package or user.
"""
if all:
if not yes and not prompt('Are you sure you want to drop all data?'):
sys.exit(0)
print('Dropping collection: user')
models.User.drop_collection()
print('Dropping collection: package')
models.Package.drop_collection()
print('Dropping collection: package_version')
models.PackageVersion.drop_collection()
print('Dropping collection: migration_revision')
models.MigrationRevision.drop_collection()
if not keep_files:
print('Deleting registry data directory ...')
if os.path.isdir(config.prefix):
shutil.rmtree(config.prefix)
sys.exit(0)
if package:
ref = refstring.parse(package)
package = models.Package.objects(name=str(ref.package)).first()
if not package:
print('Package "{}" does not exist.'.format(ref.package))
sys.exit(1)
versions = []
for pkgv in models.PackageVersion.objects(package=package):
if not ref.version or ref.version(semver.Version(pkgv.version)):
versions.append(pkgv)
if ref.version and not versions:
print('No versions matching "{}@{}"'.format(ref.package, ref.version))
sys.exit(1)
if ref.version:
if yes:
print('Dropping the following versions of "{}"'.format(ref.package))
else:
print('Are you sure you want to drop the following versions of "{}"?'
.format(ref.package))
for pkgv in versions:
print(' -', pkgv.version)
if not yes and not prompt('Confirm? '):
sys.exit(0)
print('Dropping "{}@{}" ...'.format(ref.package, ref.version))
else:
if not yes and not prompt('Do you really want to drop "{}" and all its versions?'
.format(ref.package)):
sys.exit(0)
print('Dropping "{}" ...'.format(ref.package))
for pkgv in versions:
directory = pkgv.get_directory()
pkgv.delete()
if not keep_files:
try:
shutil.rmtree(directory)
except OSError as exc:
print(' Warning: error deleting files for "{}@{}": {}'
.format(ref.package, pkgv.version, exc))
continue
if not ref.version:
directory = package.get_directory()
package.delete()
if not keep_files:
try:
shutil.rmtree(directory)
except OSError as exc:
print(' Warning: error deleting files for "{}": {}'
.format(ref.package, exc))
sys.exit(0)
if user:
user_obj = models.User.objects(name=user).first()
if not user_obj:
print('User "{}" does not exist.'.format(user))
sys.exit(1)
if not yes and not prompt('Do you really want to drop user "{}"?'.format(user)):
sys.exit(0)
packages = models.Package.objects(owner=user_obj).all()
if not reown and packages:
reown = input('Which user should the {} packages be transfered to? '.format(len(packages)))
if reown:
reown_obj = models.User.objects(name=reown).first()
if not reown_obj:
print('User "{}" does not exist'.format(reown))
sys.exit(1)
if reown_obj == user_obj:
print('Target user can not match the user that will be dropped.')
sys.exit(1)
if packages and not reown:
print('Please specify a user the will own the users packages with --reown')
sys.exit(1)
if packages:
print('Transferring packages ...')
for package in packages:
package.owner = reown_obj
package.save()
print('Dropping user "{}" ...'.format(user))
user_obj.delete()
sys.exit(0)
'holiday': {
'holiday_eve_index': 9999,
'holiday_eve_text': [],
'holiday_index': 9999,
'holiday_text': []
}
}
params_file_path = os.path.join(os.path.dirname(__file__), 'params', 'params.json')
logger.info('Installing parameters file')
expected_params_version = '1.0.0'
if os.path.exists(params_file_path):
with open(params_file_path, 'r') as f:
try:
params_version = semver.Version(json.loads(f.read())['params_version'])
logger.info('Detected parameters file version {}'.format(params_version))
if not params_version == semver.Version(expected_params_version):
logger.info('This version requires parameters file version {}. Creating a default one'.format(
expected_params_version))
os.rename(params_file_path, os.path.join(os.path.dirname(params_file_path), 'params_archive.json'))
write_params_file(params_file_path, params_default)
except json.decoder.JSONDecodeError:
logger.info('parameters file too old - no version. Creating default')
f.close()
os.rename(params_file_path, os.path.join(os.path.dirname(params_file_path), 'params_archive.json'))
write_params_file(params_file_path, params_default)
else:
logger.info('Parameters file not found, creating a default one')
write_params_file(params_file_path, params_default)
def _validate_version(field):
try:
semver.Version(field.value)
except ValueError as exc:
field.errors.append(str(exc))
def post(self, finally_, package, version, scope=None):
try:
# Just make sure the scope and package are valid.
package = refstring.Package(scope, package)
version = semver.Version(version)
except ValueError:
flask.abort(404)
replies = []
# Find the authenticated user. We should always get one because we
# use the same mechanism in httpauth.
user = User.objects(name=httpauth.username()).first()
if not user.validated:
return email_not_verified(user)
if config.enforce_package_namespaces and \
package.scope != user.name and not user.superuser:
return bad_request('You can only upload packages into your own namespace. '
' Rename your package to "@{}/{}"'.format(user.name, package.name))
# Find the package information in our database.
pkg = Package.objects(name=str(package)).first()
if pkg and pkg.owner != user:
return unauthorized_access(user, pkg)
pkgversion = PackageVersion.objects(package=pkg, version=str(version)).first()
# We only expect 1 file to be uploaded per request.
if len(request.files) != 1:
return bad_request('zero or more than 1 file(s) uploaded')
filename, storage = next(request.files.items())
if filename == 'package.json':
return bad_request('"package.json" can not be uploaded directly')
# Get the directory and filename to upload the file to.
directory = os.path.join(config.prefix, str(package), str(version))
absfile = os.path.join(directory, filename)
# Check if the upload should be forced even if the file already exists.
force = request.args.get('force', 'false').lower().strip() == 'true'
if os.path.isfile(absfile) and not force:
return bad_request('file "{} already exists'.format(filename))
# Handle package source distributions special: Unpack the package.json
# and README.md file and store the information in the database.
if filename == registry_client.get_package_archive_name(package, version):
# Save the uploaded file to a temporary path. Make sure it gets
# deleted when we're done with the request.
tmp = tempfile.NamedTemporaryFile(suffix='_' + filename, delete=False)
tmp.close()
finally_.append(lambda: fs.silentremove(tmp.name))
storage.save(tmp.name)
# Extract the package.json and README.md files.
files = {}
with tarfile.open(tmp.name, mode='r') as tar:
for fn in ['package.json', 'README.md']:
try:
files[fn] = tar.extractfile(fn).read().decode('utf8')
except KeyError:
pass
if 'package.json' not in files:
return bad_request('The uploaded package distribution archive does '
'not container a package.json file')
# Parse the manifest.
try:
pkgmf_json = json.loads(files['package.json'])
pkgmf = manifest.parse_dict(pkgmf_json)
except (json.JSONDecodeError, manifest.InvalidPackageManifest) as exc:
return bad_request('Invalid package manifest: {}'.format(exc))
if not pkgmf.license:
return bad_request('Packages uploaded to the registry must specify '
'the `license` field.')
if pkgmf.name != str(package) or pkgmf.version != version:
return bad_request('The uploaded package distribution achive does '
'not match with the target version. You are trying to uploaded '
'the archive to "{}@{}" but the manifest says it is actually '
'"{}".'.format(package, version, pkgmf.identifer))
# Now that we validated the archive and its manifest, we can copy
# it into our target directory.
if not os.path.isdir(directory):
os.makedirs(directory)
shutil.move(tmp.name, absfile)
# If the package did not exist yet, make sure it exists in the
# database.
if not pkg:
replies.append('Added package "{}" to user "{}"'.format(package, user.name))
pkg = Package(name=str(package), owner=user)
pkg.save()
# Same for the version.
if not pkgversion:
replies.append('Added new package version "{}"'.format(pkgmf.identifier))
pkgversion = PackageVersion(package=pkg, version=str(version))
else:
replies.append('Updated package version "{}"'.format(pkgmf.identifier))
pkgversion.readme = files.get('README.md', '')
pkgversion.manifest = files['package.json']
pkgversion.add_file(filename)
pkgversion.save()
# Update the 'latest' member in the Package.
if pkg.update_latest(pkgversion):
replies.append('{} is now the newest version of package "{}"'.format(
pkgversion.version, pkg.name))
# This does not appear to be a package distribution archive.
# We only allow additional files after a distribution was uploaded
# at least once.
else:
if not pkgversion:
return bad_request('Additional file uploads are only allowed after '
'a package distribution was uploaded at least once.')
if os.path.isfile(absfile):
replies.append('File "{}" updated.'.format(filename))
else:
replies.append('File "{}" saved.'.format(filename))
# Simply save the file to the directory.
storage.save(absfile)
pkgmf = None
pkgversion.add_file(filename)
pkgversion.save()
return {'message': '\n'.join(replies)}