def test_build_check(self):
spec = semantic_version.Spec('<=0.1.1-rc1')
version = semantic_version.Version('0.1.1-rc1+4.2')
self.assertTrue(version in spec,
"%r should be in %r" % (version, spec))
def test_invalid(self):
for invalid in self.invalid_specs:
with self.subTest(spec=invalid):
with self.assertRaises(ValueError, msg="Spec(%r) should be invalid" % invalid):
semantic_version.Spec(invalid)
def test_prerelease_check(self):
strict_spec = semantic_version.Spec('>=0.1.1-')
lax_spec = semantic_version.Spec('>=0.1.1')
version = semantic_version.Version('0.1.1-rc1+4.2')
self.assertFalse(version in lax_spec, "%r should not be in %r" % (version, lax_spec))
self.assertFalse(version in strict_spec, "%r should not be in %r" % (version, strict_spec))
def set(self, values):
if 'name' not in values:
logging.error('Missing the name for one of "%s"\'s packages!', self._mod.title)
self._valid = False
return
for name in values:
if name not in self._fields:
logging.warning('Ignoring unknown option "%s" for package "%s"!', name, values['name'])
self.name = values['name']
self.notes = values.get('notes', '')
self.status = values.get('status', 'recommended')
self.dependencies = values.get('dependencies', [])
self.environment = values.get('environment', [])
self.files = {}
self.filelist = values.get('filelist', [])
self.executables = values.get('executables', [])
# Validate this package's options
if self.status not in ('required', 'recommended', 'optional'):
logging.error('Unknown status "%s" for package "%s" (of Mod "%s")!', self.status, self.name, self._mod.title)
self._valid = False
for dep in self.dependencies:
if 'id' not in dep:
logging.error('Missing the ID for one of package "%s"\'s dependencies!', self.name)
self._valid = False
if 'version' not in dep:
logging.error('Missing the version of one of package "%s"\'s dependencies!', self.name)
self._valid = False
else:
if re.match('\d.*', dep['version']):
# Make a spec out of this version
dep['version'] = '==' + dep['version']
try:
semantic_version.Spec(dep['version'])
except ValueError as exc:
logging.error('Failed to parse version for one of package "%s"\'s dependencies! (%s)', self.name, str(exc))
self._valid = False
return
for name in dep:
if name not in ('id', 'version', 'packages'):
logging.warning('Ignored unknown option for "%s" dependency of package "%s"!', dep.get('id', '???'), self.name)
for env in self.environment:
if 'type' not in env:
logging.error('Missing the type for one of "%s"\'s environment conditions!', self.name)
self._valid = False
elif 'value' not in env:
logging.error('Missing the value for one of "%s"\'s environment conditions!', self.name)
self._valid = False
elif 'not' not in env:
env['not'] = False
elif env['type'] == 'cpu_feature':
# TODO
pass
elif env['type'] == 'os':
if env['value'] not in ('windows', 'linux', 'macos'):
logging.error('Unknown operating system "%s" for package "%s"!', env['value'], self.name)
self._valid = False
elif env['type'] == 'bool':
env['value'] = bool_parser.parse(env['value'])
if not env['value']:
logging.error('Invalid boolean expression "%s" for package "%s"!', env['value'], self.name)
else:
logging.error('Unknown environment condition "%s" for package "%s"!', env['type'], self.name)
self._valid = False
_files = values.get('files', [])
self.set_files(_files)
mid = self._mod.mid
if mid == '':
raise Exception('Package "%s" initialized with Mod %s which has no ID!' % (self.name, self._mod.title))
def version_specification(self) -> semver.Spec:
"""Retrieve version specification based on specified version."""
if not self._version_spec:
self._version_spec = semver.Spec(self.version)
return self._version_spec
def _install_from_tmp_dir( # pylint: disable=too-many-branches
self, tmp_dir, requirements=None):
tmp_manifest = self.load_manifest(tmp_dir)
assert set(["name", "version"]) <= set(tmp_manifest.keys())
pkg_dirname = self.get_install_dirname(tmp_manifest)
pkg_dir = join(self.package_dir, pkg_dirname)
cur_manifest = self.load_manifest(pkg_dir)
tmp_semver = None
cur_semver = None
try:
tmp_semver = semantic_version.Version(
tmp_manifest['version'], partial=True)
if cur_manifest:
cur_semver = semantic_version.Version(
cur_manifest['version'], partial=True)
except ValueError:
pass
# package should satisfy requirements
if requirements:
mismatch_error = (
"Package version %s doesn't satisfy requirements %s" %
(tmp_manifest['version'], requirements))
try:
assert tmp_semver and tmp_semver in semantic_version.Spec(
requirements), mismatch_error
except (AssertionError, ValueError):
assert tmp_manifest['version'] == requirements, mismatch_error
# check if package already exists
if cur_manifest:
# 0-overwrite, 1-rename, 2-fix to a version
action = 0
if "__src_url" in cur_manifest:
if cur_manifest['__src_url'] != tmp_manifest.get("__src_url"):
action = 1
elif "__src_url" in tmp_manifest:
action = 2
else:
if tmp_semver and (not cur_semver or tmp_semver > cur_semver):
action = 1
elif tmp_semver and cur_semver and tmp_semver != cur_semver:
action = 2
# rename
if action == 1:
target_dirname = "%s@%s" % (pkg_dirname,
cur_manifest['version'])
if "__src_url" in cur_manifest:
target_dirname = "%s@src-%s" % (
pkg_dirname,
hashlib.md5(cur_manifest['__src_url']).hexdigest())
os.rename(pkg_dir, join(self.package_dir, target_dirname))
# fix to a version
elif action == 2:
target_dirname = "%s@%s" % (pkg_dirname,
tmp_manifest['version'])
if "__src_url" in tmp_manifest:
target_dirname = "%s@src-%s" % (
pkg_dirname,
hashlib.md5(tmp_manifest['__src_url']).hexdigest())
pkg_dir = join(self.package_dir, target_dirname)
# remove previous/not-satisfied package
if isdir(pkg_dir):
util.rmtree_(pkg_dir)
os.rename(tmp_dir, pkg_dir)
assert isdir(pkg_dir)
self.cache_reset()
return pkg_dir
def test_get_package_api_loader(self):
spec = semantic_version.Spec('*')
self.loader.load_class_package(self.api_pkg_name, spec)
self.loader.api_loader.load_class_package.assert_called_with(
self.api_pkg_name, spec)
parser.add_argument("-l",
"--logging",
type=str,
choices=["info", "debug"],
help="Logging level for output")
parser.add_argument("--logfile", type=str, help="Log file")
parser.add_argument('package', help='package to analyze')
parser.add_argument('version', help='version to analyze')
args = parser.parse_args()
if args.logging:
set_logging(args.logging, args.logfile)
logging.debug("Executed as: " + str(sys.argv))
constraint = semantic_version.Spec(
'==' + str(semantic_version.Version.coerce(args.version)))
(release, count, lag_released,
dependencies) = lag_package_transitive(args.package, constraint)
# print("Release found: {} {}".format(args.package, release))
# print("Lag (number of releases): {}".format(count))
# print("Lag (release dates): {}".format(lag_released))
# print("Dependencies: {}".format(','.join(dependencies)))
# for dependency in dependencies:
# (package, spec) = split_dependency(dependency)
# print(" {}: {}".format(package, str(spec)))
# (release, count, lag_released, dependencies) = lag_package(package, spec)
# print(" Release found: {} {}".format(package, release))
# print(" Lag (number of releases): {}".format(count))
# print(" Lag (release dates): {}".format(lag_released))
# print(" Dependencies: {}".format(','.join(dependencies)))
from __future__ import print_function, absolute_import
import os
import sys
import subprocess
from distutils.cmd import Command
from distutils.errors import (CompileError, DistutilsFileError,
DistutilsExecError)
import semantic_version
from .extension import RustExtension
from .utils import cpython_feature, get_rust_version
MIN_VERSION = semantic_version.Spec('>=1.15')
class test_rust(Command):
""" Run cargo test"""
description = "test rust extensions"
user_options = []
def initialize_options(self):
self.extensions = ()
def finalize_options(self):
self.extensions = [
ext for ext in self.distribution.rust_extensions
if isinstance(ext, RustExtension)
]
import logging
import os
import dateutil.parser as dp
import docker
import semantic_version
import itertools
import re
from utils import misc
from crawler_exceptions import (DockerutilsNoJsonLog, DockerutilsException)
from timeout_utils import (Timeout, TimeoutError)
from dockerevent import DockerContainerEvent
# version at which docker image layer organization changed
VERSION_SPEC = semantic_version.Spec('>=1.10.0')
logger = logging.getLogger('crawlutils')
SUPPORTED_DRIVERS = ['btrfs', 'devicemapper', 'aufs', 'vfs']
def exec_dockerps():
"""
Returns a list of docker inspect jsons, one for each container.
This call executes the `docker inspect` command every time it is invoked.
"""
try:
client = docker.Client(base_url='unix://var/run/docker.sock',
version='auto')
from oslo_log import log as logging
from oslo_service import service
import semantic_version
from muranoagent import bunch
from muranoagent.common import config
from muranoagent.common import messaging
from muranoagent import exceptions as exc
from muranoagent import execution_plan_queue
from muranoagent import execution_plan_runner
from muranoagent import execution_result as ex_result
CONF = config.CONF
LOG = logging.getLogger(__name__)
max_format_version = semantic_version.Spec('<=2.2.0')
class MuranoAgent(service.Service):
def __init__(self):
self._queue = execution_plan_queue.ExecutionPlanQueue()
super(MuranoAgent, self).__init__()
@staticmethod
def _load_package(name):
try:
LOG.debug('Loading plugin %s', name)
__import__(name)
except Exception:
LOG.warn('Cannot load package %s', name, exc_info=True)
pass
def test_simple(self):
for valid in self.valid_specs:
version = semantic_version.Spec(valid)
self.assertEqual(valid, str(version))
def test_build_check(self):
strict_spec = semantic_version.Spec('<=0.1.1-rc1+')
lax_spec = semantic_version.Spec('<=0.1.1-rc1')
version = semantic_version.Version('0.1.1-rc1+4.2')
self.assertTrue(version in lax_spec, "%r should be in %r" % (version, lax_spec))
self.assertFalse(version in strict_spec, "%r should not be in %r" % (version, strict_spec))
def __init__(self, strict=False, **kwargs):
self.data = {
'name': '',
'description': '',
'short_description': '',
'owners': [],
'authors': [],
'license': '',
'tags': [],
'versions': {},
'screenshots': {},
'stats': {
'views': 0,
'date': {
'created': (datetime.datetime.utcnow().strftime(
Constants.date_format)),
'last-updated': (datetime.datetime.utcnow().strftime(
Constants.date_format))
}
}
}
if strict:
for k in [
'name', 'description', 'short_description', 'authors',
'license', 'tags', 'versions', 'screenshots', 'owners'
]:
if k not in kwargs:
raise KeyError(k)
for k, v in kwargs.items():
if k in ['name', 'description', 'license']:
self.data[k] = str(v)
elif k in ['authors', 'tags']:
self.data[k] = [str(x) for x in v]
elif k == 'owners':
owners = [str(x) for x in v]
for owner in owners:
if not Constants.user_pattern.match(owner):
raise HTTPBadRequest(detail=Messages.user_bad_name)
if len(owners) == 0:
raise HTTPBadRequest(detail=Messages.empty_owner_list)
self.data[k] = owners
elif k == 'versions':
data = {}
for ver, value in v.items():
files = {}
for file_url_unchecked, f in value['files'].items():
file_url = parse_url(file_url_unchecked)
file_dir = file_name = file_path = None
if ('path' not in f and 'dir' not in f
and 'name' not in f):
f['path'] # raise an error
if 'path' in f:
check_path(str(f['path']))
file_path = os.path.join('/', str(f['path']))
file_dir, file_name = split_path(file_path)
else:
check_path(str(f['name']))
check_filename(str(f['name']))
file_dir, file_name = str(f['dir']), str(f['name'])
file_path = os.path.join('/', f['dir'], f['name'])
file_dir = os.path.normpath(file_dir)
file_name = os.path.normpath(file_name)
file_path = os.path.normpath(file_path)
files[str(file_url)] = {
'dir': file_dir,
'name': file_name,
'path': file_path
}
dependencies = {}
for dep_name, dep_info in value['depends'].items():
if dep_info['type'] not in [
'recommended', 'optional', 'required'
]:
raise ValueError(Messages.wrong_dep_type)
semver.Spec(dep_info['version'])
dependencies[str(dep_name)] = {
'version': str(dep_info['version']),
'type': str(dep_info['type'])
}
data[str(semver.Version.coerce(str(ver)))] = {
'files': files,
'depends': dependencies,
'changes': value['changes']
}
self.data[k] = data
elif k == 'screenshots':
self.data[k] = {
parse_url(str(url)): str(desc)
for url, desc in v.items()
}
elif k == 'short_description':
self.data[k] = str(v)[:Constants.sdesc_len]