def _get_version(self, spec):
spec = self._parse_spec(spec)
version = spec['version']
version_sign = spec['version_sign']
resource_name = spec['resource_name']
if version_sign == '==':
return os.path.join(self.fpath, spec['resource_name'], version)
found = self.iter_contents(resource_name)
if version is None:
sc = semver.compare
sorted_vers = sorted(found,
cmp=lambda a, b: sc(a['version'],
b['version']),
reverse=True)
if not sorted_vers:
raise ResourceNotFound(spec)
version = sorted_vers[0]['version']
else:
version = '{}{}'.format(version_sign, version)
matched = filter(lambda x: semver.match(x['version'], version),
found)
sorted_vers = sorted(matched,
cmp=lambda a, b: semver.compare(a['version'],
b['version']),
reverse=True)
version = next((x['version'] for x in sorted_vers
if semver.match(x['version'], version)),
None)
if version is None:
raise ResourceNotFound(spec)
return version
def decide(tree, args):
bare_tree, configuration, tree_version = Interpreter._parse_tree(tree)
if configuration != {}:
time = None if len(args) == 1 else args[1]
context_result = Interpreter._rebuild_context(configuration, args[0], time)
context = context_result["context"]
else:
context = Interpreter.join_decide_args(args)
# Convert timezones as integers into standard +/hh:mm format
# This should only happen when no time generated value is required
context = Interpreter._convert_timezones_to_standard_format(configuration, context)
if semver.match(tree_version, ">=1.0.0") and semver.match(tree_version, "<2.0.0"):
decision = InterpreterV1.decide(configuration, bare_tree, context)
elif semver.match(tree_version, ">=2.0.0") and semver.match(tree_version, "<3.0.0"):
decision = InterpreterV2.decide(configuration, bare_tree, context)
else:
raise CraftAiDecisionError(
"""Invalid decision tree format, "{}" is currently not a valid version.""".
format(tree_version)
)
decision["context"] = context
return decision
def complex_semver_match(version, version_specifier):
"""
Custom semver match function which also supports complex semver specifiers
such as >=1.6, <2.0, etc.
NOTE: This function also supports special "all" version specifier. When "all"
is specified, any version provided will be considered valid.
:rtype: ``bool``
"""
if version_specifier == 'all':
return True
split_version_specifier = version_specifier.split(',')
if len(split_version_specifier) == 1:
# No comma, we can do a simple comparision
return semver.match(version, version_specifier)
else:
# Compare part by part
for version_specifier_part in split_version_specifier:
version_specifier_part = version_specifier_part.strip()
if not version_specifier_part:
continue
if not semver.match(version, version_specifier_part):
return False
return True
def get_catalog_branch(version):
if semver.match(version, "<=1.6.0"):
return "master"
elif semver.match(version, ">1.6.0") and semver.match(version, "<2.0.0"):
return "v1.6-release"
elif semver.match(version, ">=2.0.0"):
return "v2.0-release"
else:
print "Unknown version"
sys.exit(1)
def exclude_update_from_latest(feature):
"""
Exclude 'update from latest' scenario if branch version is lower than latest
"""
if feature.name in PKG_UPDATE_SUITES:
branch = CONF.feature.branch
if branch == 'latest': # Excludes when trying to update from latest to latest
match = True
else:
for br in [branch, 'latest']:
last_version = get_scalaraizr_latest_version(br)
if len(last_version.split('.')) == 3:
minor = int(last_version.split('.')[2][0])
else:
minor = '0'
if last_version.strip().endswith('-1'):
last_version = last_version.strip()[:-2]
if br == branch:
to_version = last_version.split('.')[0] + '.' + last_version.split('.')[1] + '.' + str(minor)
LOG.debug("Testing branch version: %s" % to_version)
else:
latest_version = last_version.split('.')[0] + '.' + last_version.split('.')[1] + '.' + str(minor)
LOG.debug("Latest version: %s" % latest_version)
match = semver.match(latest_version, '>' + to_version)
if match:
scenario = [s for s in feature.scenarios if s.name == 'Update from latest to branch from ScalrUI'][0]
feature.scenarios.remove(scenario)
LOG.info("Removed scenario: %s" % scenario)
def connect(self):
"""Enumerate and connect to the first USB HID interface."""
def passphrase_handler(_):
log.debug('using %s passphrase for %s',
'non-empty' if self.passphrase else 'empty', self)
return self._defs.PassphraseAck(passphrase=self.passphrase)
for d in self._defs.HidTransport.enumerate():
log.debug('endpoint: %s', d)
transport = self._defs.HidTransport(d)
connection = self._defs.Client(transport)
connection.callback_PassphraseRequest = passphrase_handler
f = connection.features
log.debug('connected to %s %s', self, f.device_id)
log.debug('label : %s', f.label)
log.debug('vendor : %s', f.vendor)
current_version = '{}.{}.{}'.format(f.major_version,
f.minor_version,
f.patch_version)
log.debug('version : %s', current_version)
log.debug('revision : %s', binascii.hexlify(f.revision))
if not semver.match(current_version, self.required_version):
fmt = ('Please upgrade your {} firmware to {} version'
' (current: {})')
raise ValueError(fmt.format(self, self.required_version,
current_version))
connection.ping(msg='', pin_protection=True) # unlock PIN
return connection
raise interface.NotFoundError('{} not connected'.format(self))
def check_deps(dependencies, parent=""):
if isinstance(dependencies, dict):
for pkg_name, exp_ver in dependencies.items():
if parent:
full_name = "{} ({})".format(pkg_name, parent)
else:
full_name = pkg_name
meta = getPackageMeta(pkg_name)
ver = meta.__version__
if MAJOR_MINOR_REGEX.fullmatch(ver):
ver += ".0" # Add a fictive patch number to fit semver format
if not semver.match(ver, exp_ver):
raise RuntimeError("Incompatible '{}' package version. "
"Expected: {} "
"Found: {}".
format(pkg_name, exp_ver, ver))
if hasattr(meta, "__dependencies__"):
deps = meta.__dependencies__
check_deps(deps, full_name)
else:
pkg = dependencies if isinstance(dependencies, str) else \
dependencies.__name__
meta = getPackageMeta(pkg)
deps = meta.__dependencies__
check_deps(deps)
def _load_client(name, client_type, hid_transport,
passphrase_ack, identity_type,
required_version, call_exception):
def empty_passphrase_handler(_):
return passphrase_ack(passphrase='')
for d in hid_transport.enumerate():
connection = client_type(hid_transport(d))
connection.callback_PassphraseRequest = empty_passphrase_handler
f = connection.features
log.debug('connected to %s %s', name, f.device_id)
log.debug('label : %s', f.label)
log.debug('vendor : %s', f.vendor)
current_version = '{}.{}.{}'.format(f.major_version,
f.minor_version,
f.patch_version)
log.debug('version : %s', current_version)
log.debug('revision : %s', binascii.hexlify(f.revision))
if not semver.match(current_version, required_version):
fmt = 'Please upgrade your {} firmware to {} version (current: {})'
raise ValueError(fmt.format(name,
required_version,
current_version))
yield ClientWrapper(connection=connection,
identity_type=identity_type,
device_name=name,
call_exception=call_exception)
def getActualNodeVersion(nodeVersionPackage, nodeVersionsHtml):
nodeVersionsFile = open(nodeVersionsHtml)
nodeVersionsContent = nodeVersionsFile.read()
nodeVersionsFile.close()
pattern = r"<a.*?>v(\d*)\.(\d*)\.(\d*)/</a>"
regex = re.compile(pattern)
allVersions = []
stableVersions = []
for match in regex.finditer(nodeVersionsContent):
version = "%s.%s.%s" % (match.group(1), match.group(2), match.group(3))
allVersions.append(version)
minorVersion = int(match.group(2), 10)
if minorVersion % 2 == 0:
stableVersions.append(version)
stableVersions.sort(semver.compare)
stableVersions.reverse()
allVersions.sort(semver.compare)
allVersions.reverse()
if nodeVersionPackage == "*": return stableVersions[0]
for version in allVersions:
if semver.match(version, nodeVersionPackage): return version
return None
def validate_extensions(extensions):
"""Process the extension dependencies."""
ext_map = dict(
(ext.name, ext) for ext in extensions
)
for ext in extensions:
for dependency, versions in ext.requirements.items():
ext_dependency = ext_map.get(dependency, None)
if not ext_dependency:
raise MissingDependency(
'{0} is required by {1} but is not loaded.'.format(
ext.name,
dependency,
)
)
for version in versions:
if not semver.match(ext.version, version):
raise InvalidDependency(
'{0}-{1} required by {2} but found {0}-{3}.'.format(
dependency,
version,
ext.name,
ext.version,
)
)
return extensions
def semverMatch(have,want):
try:
if have.startswith('v'):
have = have[1:]
if want.startswith('~'):
haveList = have.split('.')
wantList = want[1:].split('.')
return haveList[0] == wantList[0] and haveList[1] == wantList[1] and int(haveList[2]) >= int(wantList[2])
elif want.startswith('^'):
wantList = want[1:].split('.')
greaterOrEqual = '>=%s.%s.%s' % (wantList[0], wantList[1], wantList[2])
lessThan = '<%s.%s.%s' % (int(wantList[0]) + 1, 0, 0)
return semver.match(have,greaterOrEqual) and semver.match(have, lessThan)
else:
return semver.match(have,want)
except:
return False
def __init__(self, server="localhost:8983",
detect_live_nodes=False,
user=None,
password=None,
timeout=10,
webappdir='solr',
version='5.3.0',
request_retries=1,
use_https=False):
self.user = user
self.password = password
self.timeout = timeout
self.webappdir = webappdir
self.version = version
self.request_retries = request_retries
if not semver.match(version, MIN_SUPPORTED_VERSION) and semver.match(version, MAX_SUPPORTED_VERSION):
raise Exception("Unsupported version %s" % version)
if semver.match(self.version, '<5.4.0'):
self.zk_path = '/{webappdir}/zookeeper'.format(webappdir=self.webappdir)
else:
self.zk_path = '/{webappdir}/admin/zookeeper'.format(webappdir=self.webappdir)
protocol = "https" if use_https else "http"
self.url_template = '{protocol}://{{server}}/{webappdir}/'.format(protocol=protocol, webappdir=self.webappdir)
if type(server) == str:
self.url = self.url_template.format(server=server)
servers = [self.url, self.url]
if detect_live_nodes:
url = servers[0]
self.servers = self.detect_nodes(url)
else:
self.servers = servers
if type(server) == list:
servers = [self.url_template.format(server=a) for a in server]
if detect_live_nodes:
url = servers[0]
self.servers = self.detect_nodes(url)
else:
self.servers = servers
self.client = _Request(self)
def __init__(self, package, versionspec):
# if something deprecated '>=4.1.0' we want it to raise during the 4.1.0-dev, and 4.1.0-rc,
# not just when we release 4.1.0, so remove any extra-tags.
versionspec = versionspec.split('-')[0]
current_version = traitlets.import_item(package+'.__version__')
self.match = semver.match(current_version, versionspec)
self.package = package
self.spec = versionspec
def match(version, spec):
if version[0] == 'v':
version = version[1:]
if spec[0] == '^':
base = spec[1:]
parsed_base = semver.parse(base)
if parsed_base['major'] > 0:
top = semver.bump_major(base)
elif parsed_base['minor'] > 0:
top = semver.bump_minor(base)
else:
top = semver.bump_patch(base)
return semver.match(version, ">="+base) and semver.match(version, "<="+top)
else:
try:
return semver.match(version, spec)
except ValueError:
# this happens when the spec isn't an expression, in which case we need an exact match
return semver.parse(version) == semver.parse(spec)
def find_semantic_version(releases, versions):
matching_releases = []
for release in releases:
if release.count('.') == 2:
try:
if semver.match(release, versions[0]) \
and semver.match(release, versions[1]):
matching_releases.append(release)
except:
pass
if len(matching_releases) == 1:
out = matching_releases[0]
elif len(matching_releases) == 2:
out = semver.max_ver(matching_releases[0], matching_releases[1])
elif len(matching_releases) > 2:
out = sorted(matching_releases)[-1]
else:
out = False
return out
def constraint_match(constraint_string, version):
if not constraint_string:
return True
constraints = [c.strip() for c in constraint_string.split(',') if c.strip()]
for c in constraints:
if not re.match(r'[><=][=]?\d+(\.\d+)*', c):
raise InvalidConstraintException('invalid constraint: %s' % c)
return all(semver.match(version, c) for c in constraints)
def is_compatible(self):
if self.s.name and self.compatibility:
compatible_version = self.compatibility.get(self.s.name)
if not compatible_version:
return False
if self.s.version:
return semver.match(self.s.version, compatible_version)
return False
return True
def extract_bower_zipfile(self, zip_file, dest, expected_version=None):
bower_json = None
root = None
deps_installed = []
for info in zip_file.infolist():
if PurePath(info.filename).name == "bower.json":
with zip_file.open(info) as f:
bower_json = json.load(f)
root = str(PurePath(info.filename).parent)
break
version = bower_json["version"]
if expected_version is not None:
expected_version = Bower.clean_semver(expected_version)
if not semver.match(version, expected_version):
click.secho("error: versions do not match ({} =/= {})".format(
version, expected_version))
raise InvalidPackageError
if "dependencies" in bower_json:
for package, version in bower_json["dependencies"].items():
url = Bower.get_package_url(package)
deps_installed.extend(self.get_bower_package(
url, dest=dest, version=version))
ignore_patterns = [GitIgnorePattern(ig) for ig in bower_json["ignore"]]
path_spec = PathSpec(ignore_patterns)
namelist = [path for path in zip_file.namelist()
if PurePath(path).parts[0] == root]
ignored = list(path_spec.match_files(namelist))
for path in namelist:
dest_path = PurePath(
bower_json["name"],
*PurePath(path).parts[1:])
if path in ignored:
continue
for path in ignored:
for parent in PurePath(path):
if parent in ignored:
continue
if path.endswith("/"):
if list(path_spec.match_files([str(dest_path)])):
ignored.append(PurePath(path))
elif not (dest / dest_path).is_dir():
(dest / dest_path).mkdir(parents=True)
else:
target_path = dest / dest_path.parent / dest_path.name
source = zip_file.open(path)
target = target_path.open("wb")
with source, target:
shutil.copyfileobj(source, target)
deps_installed.append((bower_json["name"], bower_json["version"]))
return deps_installed
def test_should_raise_value_error_for_unexpected_match_expression():
with pytest.raises(ValueError):
match("2.3.7", "=2.3.7")
with pytest.raises(ValueError):
match("2.3.7", "~2.3.7")
with pytest.raises(ValueError):
match("2.3.7", "^2.3.7")
def test_should_raise_value_error_for_invalid_match_expression():
with pytest.raises(ValueError):
match('1.0.0', '')
with pytest.raises(ValueError):
match('1.0.0', '!')
with pytest.raises(ValueError):
match('1.0.0', '1.0.0')
def _check_version(self):
"""Check if the Brayns' version is sufficient enough."""
status = utils.http_request(HTTP_METHOD_GET, self.http_url, 'version')
if status.code != HTTP_STATUS_OK:
raise Exception('Cannot obtain version from Brayns')
version = '.'.join(str(x) for x in [status.contents['major'], status.contents['minor'],
status.contents['patch']])
import semver
if semver.match(version, '<{0}'.format(MINIMAL_VERSION)):
raise Exception('Brayns does not satisfy minimal required version; '
'needed {0}, got {1}'.format(MINIMAL_VERSION, version))
def get_arch(): "Returns architecture name for PyV8 binary" is_64bit = sys.maxsize > 2**32 system_name = platform.system() if system_name == 'Darwin': if semver.match(platform.mac_ver()[0], '<10.7.0'): return 'mac106' return 'osx' if system_name == 'Windows': return 'win64' if is_64bit else 'win32' if system_name == 'Linux': return 'linux64' if is_64bit else 'linux32'
def test_should_not_raise_value_error_for_expected_match_expression():
assert match("2.3.7", "<2.4.0") is True
assert match("2.3.7", ">2.3.5") is True
assert match("2.3.7", "<=2.3.9") is True
assert match("2.3.7", ">=2.3.5") is True
assert match("2.3.7", "==2.3.7") is True
assert match("2.3.7", "!=2.3.7") is False
def get_bower_package(self, url, dest=None, version=None,
process_deps=True):
dest = dest or Path(".") / "assets"
parsed_url = urlparse(url)
if parsed_url.scheme == "git" or parsed_url.path.endswith(".git"):
if parsed_url.netloc == "github.com":
user, repo = parsed_url.path[1:-4].split("/")
response = get(github_api_uri +
"/repos/{}/{}/tags".format(user, repo))
tags = response.json()
target = None
if not len(tags):
click.secho("fatal: no tags exist for {}/{}".format(
user, repo), fg="red")
raise InvalidPackageError
if version is None:
target = tags[0]
else:
for tag in tags:
if semver.match(tag["name"],
Bower.clean_semver(version)):
target = tag
break
if not target:
click.secho(
"fatal: failed to find matching tag for "
"{user}/{repo} {version}".format(user, repo, version),
fg="red")
raise VersionNotFoundError
click.secho("installing {}/{}#{}".format(
user, repo, tags[0]["name"]), fg="green")
return self.get_bower_package(
url=target["zipball_url"],
dest=dest,
version=version)
raise NotImplementedError
click.echo("git clone {url}".format(url=url))
cmd = envoy.run('git clone {url} "{dest}"'.format(
url=url, dest=dest))
elif parsed_url.scheme in ("http", "https"):
zip_dest = download_file(url, dest=self.temp_dir,
label="{dest_basename}",
expected_extension="zip")
with zipfile.ZipFile(zip_dest, "r") as pkg:
return self.extract_bower_zipfile(pkg, dest,
expected_version=version)
# pkg.extractall(str(dest))
else:
click.secho("protocol currently unsupported :(")
sys.exit(1)
def _parse_tree(tree_object):
# Checking definition of tree_object
if not isinstance(tree_object, dict):
raise CraftAiDecisionError("Invalid decision tree format, the given json is not an object.")
# Checking version existence
tree_version = tree_object.get("_version")
if not tree_version:
raise CraftAiDecisionError(
"""Invalid decision tree format, unable to find the version"""
""" informations."""
)
# Checking version and tree validity according to version
if re.compile(r"\d+.\d+.\d+").match(tree_version) is None:
raise CraftAiDecisionError(
"""Invalid decision tree format, "{}" is not a valid version.""".
format(tree_version)
)
elif semver.match(tree_version, ">=1.0.0") and semver.match(tree_version, "<3.0.0"):
if tree_object.get("configuration") is None:
raise CraftAiDecisionError(
"""Invalid decision tree format, no configuration found"""
)
if tree_object.get("trees") is None:
raise CraftAiDecisionError(
"""Invalid decision tree format, no tree found."""
)
bare_tree = tree_object.get("trees")
configuration = tree_object.get("configuration")
else:
raise CraftAiDecisionError(
"""Invalid decision tree format, {} is not a supported"""
""" version.""".
format(tree_version)
)
return bare_tree, configuration, tree_version
def get_arch():
"Returns architecture name for PyV8 binary"
suffix = is_python3 and '-p3' or ''
p = lambda a: '%s%s' % (a, suffix)
is_64bit = sys.maxsize > 2**32
system_name = platform.system()
if system_name == 'Darwin':
if semver.match(platform.mac_ver()[0], '<10.7.0'):
return p('mac106')
return p('osx')
if system_name == 'Windows':
return p('win64') if is_64bit else p('win32')
if system_name == 'Linux':
return p('linux64') if is_64bit else p('linux32')
def complex_semver_match(version, version_specifier):
"""
Custom semver match function which also supports complex semver specifiers
such as >=1.6, <2.0, etc.
:rtype: ``bool``
"""
split_version_specifier = version_specifier.split(',')
if len(split_version_specifier) == 1:
# No comma, we can do a simple comparision
return semver.match(version, version_specifier)
else:
# Compare part by part
for version_specifier_part in split_version_specifier:
version_specifier_part = version_specifier_part.strip()
if not version_specifier_part:
continue
if not semver.match(version, version_specifier_part):
return False
return True
def cluster_health(self):
"""
Determine the state of all nodes and collections in the cluster. Problematic nodes or
collections are returned, along with their state, otherwise an `OK` message is returned
:return: a dict representing the status of the cluster
:rtype: dict
"""
res = []
if semver.match(self.version, '<5.4.0'):
params = {'detail': 'true', 'path': '/clusterstate.json'}
response = self.client.get(
('/{webappdir}/zookeeper'.format(webappdir=self.webappdir)), params).result
data = json.loads(response['znode']['data'])
collections = self.list()
for coll in collections:
shards = data[coll]['shards']
for shard, shard_info in iteritems(shards):
replicas = shard_info['replicas']
for replica, info in iteritems(replicas):
state = info['state']
if state != 'active':
item = {"collection": coll,
"replica": replica,
"shard": shard,
"info": info,
}
res.append(item)
else:
params = {'action': 'CLUSTERSTATUS', 'wt': 'json'}
response = self.client.get(
('/{webappdir}/admin/collections'.format(webappdir=self.webappdir)), params).result
for collection_name, collection in list(response.dict['cluster']['collections'].items()):
for shard_name, shard in list(collection['shards'].items()):
for replica_name, replica in list(shard['replicas'].items()):
if replica['state'] != 'active':
item = {"collection": collection_name,
"replica": replica_name,
"shard": shard_name,
"info": replica}
res.append(item)
if not res:
return {"status": "OK"}
return {"status": "NOT OK", "details": res}
def __new__(cls, *args, **kwds):
from . import notifiers
system = platform.system()
if system == 'Linux':
cls = notifiers.NotifySendNotifier
elif system == 'Darwin':
cls = notifiers.NotificationCenterNotifier
elif system == 'Windows':
if semver.match(platform.release(), '<6.2.9200'):
cls = notifiers.BalloonNotifier
else:
cls = notifiers.ToasterNotifier
else:
cls = notifiers.GrowlNotifier
return cls(*args, **kwds)
def get_arch():
"Returns architecture name for PyV8 binary"
suffix = is_python3 and "-p3" or ""
p = lambda a: "%s%s" % (a, suffix)
is_64bit = sys.maxsize > 2 ** 32
system_name = platform.system()
if system_name == "Darwin":
try:
if semver.match(platform.mac_ver()[0], "<10.7.0"):
return p("mac106")
except:
pass
return p("osx")
if system_name == "Windows":
return p("win64") if is_64bit else p("win32")
if system_name == "Linux":
return p("linux64") if is_64bit else p("linux32")
