def _parse_tree(tree_object):
# Checking definition of tree_object
if not isinstance(tree_object, dict):
raise CraftAiDecisionError(
"Invalid decision tree format, the given json is not an object."
)
# Checking version existence
tree_version = tree_object.get("_version")
if not tree_version:
raise CraftAiDecisionError(
"""Invalid decision tree format, unable to find the version"""
""" informations.""")
# Checking version and tree validity according to version
if re.compile(r"\d+.\d+.\d+").match(tree_version) is None:
raise CraftAiDecisionError(
"""Invalid decision tree format, "{}" is not a valid version."""
.format(tree_version))
elif semver.match(tree_version, ">=1.0.0") and semver.match(
tree_version, "<3.0.0"):
if tree_object.get("configuration") is None:
raise CraftAiDecisionError(
"""Invalid decision tree format, no configuration found""")
if tree_object.get("trees") is None:
raise CraftAiDecisionError(
"""Invalid decision tree format, no tree found.""")
bare_tree = tree_object.get("trees")
configuration = tree_object.get("configuration")
else:
raise CraftAiDecisionError(
"""Invalid decision tree format, {} is not a supported"""
""" version.""".format(tree_version))
return bare_tree, configuration, tree_version
def _get_version(self, spec):
spec = self._parse_spec(spec)
version = spec['version']
version_sign = spec['version_sign']
resource_name = spec['resource_name']
if version_sign == '==':
return os.path.join(self.fpath, spec['resource_name'], version)
found = self.iter_contents(resource_name)
if version is None:
sc = semver.compare
sorted_vers = sorted(found,
cmp=lambda a, b: sc(a['version'],
b['version']),
reverse=True)
if not sorted_vers:
raise ResourceNotFound(spec)
version = sorted_vers[0]['version']
else:
version = '{}{}'.format(version_sign, version)
matched = filter(lambda x: semver.match(x['version'], version),
found)
sorted_vers = sorted(matched,
cmp=lambda a, b: semver.compare(a['version'],
b['version']),
reverse=True)
version = next((x['version'] for x in sorted_vers
if semver.match(x['version'], version)),
None)
if version is None:
raise ResourceNotFound(spec)
return version
def complex_semver_match(version, version_specifier):
"""
Custom semver match function which also supports complex semver specifiers
such as >=1.6, <2.0, etc.
NOTE: This function also supports special "all" version specifier. When "all"
is specified, any version provided will be considered valid.
:rtype: ``bool``
"""
if version_specifier == 'all':
return True
split_version_specifier = version_specifier.split(',')
if len(split_version_specifier) == 1:
# No comma, we can do a simple comparision
return semver.match(version, version_specifier)
else:
# Compare part by part
for version_specifier_part in split_version_specifier:
version_specifier_part = version_specifier_part.strip()
if not version_specifier_part:
continue
if not semver.match(version, version_specifier_part):
return False
return True
def get_k8s_supported_versions():
k8s_releases_json = None
attempt = 0
while attempt < 3:
try:
with urllib.request.urlopen(
'https://storage.googleapis.com/minikube/k8s_releases.json'
) as f:
k8s_releases_json = json.loads(f.read().decode('utf-8'))
break
except:
time.sleep(5)
k8s_releases_json = None
attempt += 1
if not k8s_releases_json:
print(
"Failed to get K8S releases from https://storage.googleapis.com/minikube/k8s_releases.json !"
)
sys.exit(1)
versions = []
for r in k8s_releases_json:
version = r['version'].strip().strip('v')
if semver.match(version, '>=' + K8S_MIN_VERSION) and semver.match(
version, '<=' + K8S_MAX_VERSION):
versions.append(version)
return sorted(versions,
key=lambda v: list(map(int, v.split('.'))),
reverse=True)
def get_k8s_supported_versions():
k8s_releases_json = None
attempt = 0
while attempt < 3:
try:
with urllib.request.urlopen(
"https://storage.googleapis.com/minikube/k8s_releases.json"
) as resp:
k8s_releases_json = json.loads(resp.read().decode("utf-8"))
break
except: # noqa pylint: disable=bare-except
time.sleep(5)
k8s_releases_json = None
attempt += 1
if not k8s_releases_json:
print(
"Failed to get K8S releases from https://storage.googleapis.com/minikube/k8s_releases.json !"
)
sys.exit(1)
versions = []
for release in k8s_releases_json:
version = release["version"].strip().strip("v")
if semver.match(version, ">=" + K8S_MIN_VERSION) and semver.match(
version, "<=" + K8S_MAX_VERSION):
versions.append(version)
versions += KUBEADM_VERSIONS
return sorted(versions,
key=lambda v: list(map(int, v.split("."))),
reverse=True)
def decide(tree, args):
bare_tree, configuration, tree_version = Interpreter._parse_tree(tree)
if configuration != {}:
time = None if len(args) == 1 else args[1]
context_result = Interpreter._rebuild_context(configuration, args[0], time)
context = context_result["context"]
else:
context = Interpreter.join_decide_args(args)
# Convert timezones as integers into standard +/hh:mm format
# This should only happen when no time generated value is required
context = Interpreter._convert_timezones_to_standard_format(configuration, context)
if semver.match(tree_version, ">=1.0.0") and semver.match(tree_version, "<2.0.0"):
decision = InterpreterV1.decide(configuration, bare_tree, context)
elif semver.match(tree_version, ">=2.0.0") and semver.match(tree_version, "<3.0.0"):
decision = InterpreterV2.decide(configuration, bare_tree, context)
else:
raise CraftAiDecisionError(
"""Invalid decision tree format, "{}" is currently not a valid version.""".
format(tree_version)
)
decision["context"] = context
return decision
def _check_features(self):
res = super(WideSkyHasFeaturesOperation, self)._check_features()
# Ensure this is WideSky
if self._about_data["productName"] not in (
"Widesky Semantic Database Toolkit",
"WideSky",
):
# Not recognised, stop here.
return res
# Get the WideSky version, preferring moduleVersion over productVersion
ver = self._about_data.get("moduleVersion",
self._about_data["productVersion"])
for feature in self._features:
if feature in (
HaystackSession.FEATURE_HISREAD_MULTI,
HaystackSession.FEATURE_HISWRITE_MULTI,
):
try:
res[feature] = semver.match(ver, ">=0.5.0")
except ValueError:
# Unrecognised version string
return res
elif feature == HaystackSession.FEATURE_ID_UUID:
try:
res[feature] = semver.match(ver, ">=0.8.0")
except ValueError:
return res
return res
def decide(tree, args):
bare_tree, configuration, tree_version = Interpreter._parse_tree(tree)
if configuration != {}:
time = None if len(args) == 1 else args[1]
context_result = Interpreter._rebuild_context(
configuration, args[0], time)
context = context_result["context"]
else:
context = Interpreter.join_decide_args(args)
# Convert timezones as integers into standard +/hh:mm format
# This should only happen when no time generated value is required
context = Interpreter._convert_timezones_to_standard_format(
configuration, context)
if semver.match(tree_version, ">=1.0.0") and semver.match(
tree_version, "<2.0.0"):
decision = InterpreterV1.decide(configuration, bare_tree, context)
elif semver.match(tree_version, ">=2.0.0") and semver.match(
tree_version, "<3.0.0"):
decision = InterpreterV2.decide(configuration, bare_tree, context)
else:
raise CraftAiDecisionError(
"""Invalid decision tree format, "{}" is currently not a valid version."""
.format(tree_version))
decision["context"] = context
return decision
def complex_semver_match(version, version_specifier):
"""
Custom semver match function which also supports complex semver specifiers
such as >=1.6, <2.0, etc.
NOTE: This function also supports special "all" version specifier. When "all"
is specified, any version provided will be considered valid.
:rtype: ``bool``
"""
if version_specifier == 'all':
return True
split_version_specifier = version_specifier.split(',')
if len(split_version_specifier) == 1:
# No comma, we can do a simple comparision
return semver.match(version, version_specifier)
else:
# Compare part by part
for version_specifier_part in split_version_specifier:
version_specifier_part = version_specifier_part.strip()
if not version_specifier_part:
continue
if not semver.match(version, version_specifier_part):
return False
return True
def deploy(self, version, timeout, options=None):
if options is None:
options = {}
self.registry_port = get_free_port()
if container_is_running(self.host_client, "minikube"):
self.host_client.containers.get("minikube").remove(force=True,
v=True)
self.k8s_version = version
if self.k8s_version[0] != "v":
self.k8s_version = "v" + self.k8s_version
if not options:
options = {
"name": "minikube",
"privileged": True,
"environment": {
"K8S_VERSION": self.k8s_version,
"TIMEOUT": str(timeout)
},
"ports": {
"8080/tcp": None,
"8443/tcp": None,
"2375/tcp": None,
"%d/tcp" % self.registry_port: self.registry_port,
},
"volumes": {
"/tmp/scratch": {
"bind": "/tmp/scratch",
"mode": "rw"
}
},
}
if MINIKUBE_VERSION:
self.version = MINIKUBE_VERSION
elif semver.match(self.k8s_version.lstrip("v"), ">=1.11.0"):
self.version = MINIKUBE_KUBEADM_VERSION
else:
self.version = MINIKUBE_LOCALKUBE_VERSION
if self.version == "latest" or semver.match(
self.version.lstrip("v"),
">" + MINIKUBE_LOCALKUBE_VERSION.lstrip("v")):
options["command"] = "/lib/systemd/systemd"
else:
options["command"] = "sleep inf"
print("\nDeploying minikube %s cluster ..." % self.k8s_version)
image, _ = self.host_client.images.build(
path=os.path.join(TEST_SERVICES_DIR, "minikube"),
buildargs={"MINIKUBE_VERSION": self.version},
tag="minikube:%s" % self.version,
rm=True,
forcerm=True,
)
self.container = self.host_client.containers.run(image.id,
detach=True,
**options)
self.name = self.container.name
self.container.exec_run("start-minikube.sh", detach=True)
self.load_kubeconfig(timeout=timeout)
self.client = self.get_client()
self.get_bootstrapper()
def _get_vm_abi_versions(self):
"""
Check the version of the node and retrieve the correct values for abi and vm version
"""
if semver.match(self.client.api_version, "<=1.4.0"):
return CONTRACT_ROMA_VM, CONTRACT_ROMA_ABI
if semver.match(self.client.api_version, "<3.0.0"):
return CONTRACT_MINERVA_VM, CONTRACT_MINERVA_ABI
raise UnsupportedNodeVersion(f"Version {self.client.api_version} is not supported")
def get_catalog_branch(version):
if semver.match(version, "<=1.6.0"):
return "master"
elif semver.match(version, ">1.6.0") and semver.match(version, "<2.0.0"):
return "v1.6-release"
elif semver.match(version, ">=2.0.0"):
return "v2.0-release"
else:
print "Unknown version"
sys.exit(1)
def get_catalog_branch(version):
if semver.match(version, "<=1.6.0"):
return "master"
elif semver.match(version, ">1.6.0") and semver.match(version, "<2.0.0"):
return "v1.6-release"
elif semver.match(version, ">=2.0.0"):
return "v2.0-release"
else:
print "Unknown version"
sys.exit(1)
def check_k8s_version(k8s_version):
assert k8s_version, "K8S version not defined"
k8s_latest_version = retry(get_latest_k8s_version, urllib.error.URLError)
if k8s_version.lower() == "latest":
k8s_version = k8s_latest_version
k8s_version = k8s_version.lstrip("v")
assert re.match(r"^\d+\.\d+\.\d+$", k8s_version), "Invalid K8S version '%s'" % k8s_version
assert semver.match(k8s_version, ">=" + K8S_MIN_VERSION), "K8S version %s not supported" % k8s_version
assert semver.match(k8s_version, "<=" + k8s_latest_version), "K8S version %s not supported" % k8s_version
return "v" + k8s_version
def desugar_range_and_match(version, semver_range):
if 'x' in semver_range:
if len(semver_range.split('.')) < 3:
range_floor = semver_range.replace('x', '0.0')
range_ceiling = semver.bump_major(range_floor)
else:
range_floor = semver_range.replace('x','0')
range_ceiling = semver.bump_minor(range_floor)
if not semver.match(version, ">="+range_floor):
return False
if not semver.match(version, "<"+range_ceiling):
return False
return True
def __init__(self,
server="localhost:8983",
detect_live_nodes=False,
user=None,
password=None,
timeout=10,
webappdir='solr',
version='5.3.0',
request_retries=1,
use_https=False):
self.user = user
self.password = password
self.timeout = timeout
self.webappdir = webappdir
self.version = version
self.request_retries = request_retries
if not semver.match(version, MIN_SUPPORTED_VERSION) and semver.match(
version, MAX_SUPPORTED_VERSION):
raise Exception("Unsupported version %s" % version)
if semver.match(self.version, '<5.4.0'):
self.zk_path = '/{webappdir}/zookeeper'.format(
webappdir=self.webappdir)
else:
self.zk_path = '/{webappdir}/admin/zookeeper'.format(
webappdir=self.webappdir)
protocol = "https" if use_https else "http"
self.url_template = '{protocol}://{{server}}/{webappdir}/'.format(
protocol=protocol, webappdir=self.webappdir)
if type(server) == str:
self.url = self.url_template.format(server=server)
servers = [self.url, self.url]
if detect_live_nodes:
url = servers[0]
self.servers = self.detect_nodes(url)
else:
self.servers = servers
if type(server) == list:
servers = [self.url_template.format(server=a) for a in server]
if detect_live_nodes:
url = servers[0]
self.servers = self.detect_nodes(url)
else:
self.servers = servers
self.client = _Request(self)
def is_server_valid(server_list, network):
# Check requested server is on list
available_servers = [s['name'] for s in server_list]
if network not in available_servers:
raise NetworkUnavailableError(
'Requested network not present on network: {}'.format(network))
# Check network version
server_details = next(s for s in server_list if s['name'] == network)
if server_details['versions'] != '*':
version_constraints = server_details['versions'].split(',')
# Build required version (e.g.: 0.9.1-alpha2 -> 0.9.0)
network_version_required = semver.parse(__version__)
network_version_required['prerelease'] = None
network_version_required['build'] = None
network_version_required['patch'] = 0
network_version_required = semver.format_version(
**network_version_required)
if not all(
semver.match(network_version_required, c)
for c in version_constraints):
raise IncompatibleLedgerVersion(
"Requested network does not support required version\n" +
"Required version: {}\nNetwork supports: {}".format(
network_version_required, ', '.join(version_constraints)))
# Return true if server valid
return True
def getActualNodeVersion(nodeVersionPackage, nodeVersionsHtml):
nodeVersionsFile = open(nodeVersionsHtml)
nodeVersionsContent = nodeVersionsFile.read()
nodeVersionsFile.close()
pattern = r"<a.*?>v(\d*)\.(\d*)\.(\d*)/</a>"
regex = re.compile(pattern)
allVersions = []
stableVersions = []
for match in regex.finditer(nodeVersionsContent):
version = "%s.%s.%s" % (match.group(1), match.group(2), match.group(3))
allVersions.append(version)
minorVersion = int(match.group(2), 10)
if minorVersion % 2 == 0:
stableVersions.append(version)
stableVersions.sort(semver.compare)
stableVersions.reverse()
allVersions.sort(semver.compare)
allVersions.reverse()
if nodeVersionPackage == "*": return stableVersions[0]
for version in allVersions:
if semver.match(version, nodeVersionPackage): return version
return None
def huber_loss(y_true, y_pred, max_grad=1.):
"""Calculate the huber loss.
See https://en.wikipedia.org/wiki/Huber_loss
Parameters
----------
y_true: np.array, tf.Tensor
Target value.
y_pred: np.array, tf.Tensor
Predicted value.
max_grad: float, optional
Positive floating point value. Represents the maximum possible
gradient magnitude.
Returns
-------
tf.Tensor
The huber loss.
"""
diff = y_pred - y_true
if semver.match(tf.__version__, '<1.0.0'):
huber_val = tf.select(
tf.abs(diff) < max_grad, 0.5 * tf.square(diff),
max_grad * (tf.abs(diff) - 0.5 * max_grad))
else:
huber_val = tf.where(
tf.abs(diff) < max_grad, 0.5 * tf.square(diff),
max_grad * (tf.abs(diff) - 0.5 * max_grad))
return huber_val
def _apply_compatibility(self, results, thingtalk_version):
if semver.match(thingtalk_version, "<1.3.0"):
# convert stream-join "=>" to "join"
for result in results:
code = result['code']
if len(code) == 0 or code[0] == 'policy':
continue
start = 0
if code[0] == 'executor':
while code[start] != ':':
start += 1
start += 1
has_now = code[start] == 'now'
if has_now:
start += 2 # "now" & "=>"
last_arrow = None
for i in range(len(code) - 1, start, -1):
if code[i] == '=>':
last_arrow = i
break
if last_arrow is None:
continue
for i in range(start, last_arrow):
if code[i] == '=>':
code[i] = 'join'
def version_match(actual, requirement):
# if '.' not in actual and '*' not in actual:
# actual += '.0.0'
return semver.satisfies(actual, requirement)
if not re.match('[<>=!].*', requirement):
requirement = '== ' + requirement
return semver.match(actual, requirement)
## SEMVER!!
return (requirement == actual
or (requirement[0] == '^'
and semver_compare(actual, requirement[1:]) >= 0
and # Above requirement
semver_compare(actual, semver_cap(
requirement[1:])) < 0 # but below rollover
) or (requirement.startswith('> ')
and semver_compare(actual, requirement[2:]) > 0)
or (requirement.startswith('>= ')
and semver_compare(actual, requirement[3:]) >= 0)
or (requirement.startswith('= ')
and semver_compare(actual, requirement[2:]) == 0)
or (requirement.startswith('<= ')
and semver_compare(actual, requirement[3:]) <= 0)
or (requirement.startswith('< ')
and semver_compare(actual, requirement[2:]) < 0)
or semver_compare(actual, requirement) == 0)
def exclude_update_from_latest(feature):
"""
Exclude 'update from latest' scenario if branch version is lower than latest
"""
if feature.name in PKG_UPDATE_SUITES:
branch = CONF.feature.branch
if branch == 'latest': # Excludes when trying to update from latest to latest
match = True
else:
for br in [branch, 'latest']:
last_version = get_scalaraizr_latest_version(br)
if len(last_version.split('.')) == 3:
minor = int(last_version.split('.')[2][0])
else:
minor = '0'
if last_version.strip().endswith('-1'):
last_version = last_version.strip()[:-2]
if br == branch:
to_version = last_version.split('.')[0] + '.' + last_version.split('.')[1] + '.' + str(minor)
LOG.debug("Testing branch version: %s" % to_version)
else:
latest_version = last_version.split('.')[0] + '.' + last_version.split('.')[1] + '.' + str(minor)
LOG.debug("Latest version: %s" % latest_version)
match = semver.match(latest_version, '>' + to_version)
if match:
scenario = [s for s in feature.scenarios if s.name == 'Update from latest to branch from ScalrUI'][0]
feature.scenarios.remove(scenario)
LOG.info("Removed scenario: %s" % scenario)
def connect(self):
"""Enumerate and connect to the first USB HID interface."""
def passphrase_handler(_):
log.debug('using %s passphrase for %s',
'non-empty' if self.passphrase else 'empty', self)
return self._defs.PassphraseAck(passphrase=self.passphrase)
for d in self._defs.HidTransport.enumerate():
log.debug('endpoint: %s', d)
transport = self._defs.HidTransport(d)
connection = self._defs.Client(transport)
connection.callback_PassphraseRequest = passphrase_handler
f = connection.features
log.debug('connected to %s %s', self, f.device_id)
log.debug('label : %s', f.label)
log.debug('vendor : %s', f.vendor)
current_version = '{}.{}.{}'.format(f.major_version,
f.minor_version,
f.patch_version)
log.debug('version : %s', current_version)
log.debug('revision : %s', binascii.hexlify(f.revision))
if not semver.match(current_version, self.required_version):
fmt = ('Please upgrade your {} firmware to {} version'
' (current: {})')
raise ValueError(fmt.format(self, self.required_version,
current_version))
connection.ping(msg='', pin_protection=True) # unlock PIN
return connection
raise interface.NotFoundError('{} not connected'.format(self))
def get_module_info(slug):
print('Checking ' + slug)
sys.stdout.flush()
resp = requests.get(
forgeUrl + '/v3/modules/' + slug +
'?exclude_fields=readme%20changelog%20license%20reference')
modinfo = resp.json()
name = modinfo['name']
thismod = modules[name]
if 'releases' in modinfo:
thismod.pop('_mkenv_matchversion', None)
vSort(modinfo['releases'])
for r in modinfo['releases']:
print(' version {}'.format(r['version']))
if functools.reduce(
lambda x, y: x and semver.match(r['version'], y),
functools.reduce(
lambda x, y: x + fix_semver(y),
flatten(thismod['_mkenv_versions'].values()), []),
True):
print(' matches requirements: {}'.format(pp_deps(name)))
thismod['_mkenv_matchversion'] = r['version']
break
else:
print('thismodersions?')
#pprint.pprint(modinfo)
return {name: {**modinfo, **thismod}}
def check_brightside_version():
v = version()
log.debug("%s version: %s" % (BRIGHTSIDE_NAME, v))
required_version = ">=1.0.1"
if not semver.match(v, required_version):
print("pybright requires %s %s but it is %s" %
(BRIGHTSIDE_NAME, required_version, v))
def _override_artifact_ids(self, conf, version, artifact_ids):
"""
Override the `artifact_ids` if `version` matches any pattern in the `artifactIdVersions` config.
Consider each `(pattern, override_artifact_ids)` in
`artifactIdVersions`. Assume that `pattern` is a `semver.match`
pattern. If `version` matches any `pattern`, then return the
corresponding `override_artifact_ids`. If no pattern matches, return
`artifact_ids` unchanged.
The set of `pattern`s must be disjoint - `version` must match zero or one `pattern`.
"""
result = artifact_ids
version_pattern_matches = 0
for version_pattern, version_artifact_ids in conf.get(
'artifactIdVersions', {}).items():
if semver.match(version, version_pattern):
version_pattern_matches += 1
if version_pattern_matches > 1:
raise AssertionError(
"version {version} of {aid} matches more than one artifactIdVersions pattern"
.format(version=version, aid=self.artifact))
result = version_artifact_ids
return result
def get_uninitialized_variables(variables=None):
"""Return a list of uninitialized tf variables.
Parameters
----------
variables: tf.Variable, list(tf.Variable), optional
Filter variable list to only those that are uninitialized. If no
variables are specified the list of all variables in the graph
will be used.
Returns
-------
list(tf.Variable)
List of uninitialized tf variables.
"""
sess = tf.get_default_session()
if variables is None:
variables = tf.global_variables()
else:
variables = list(variables)
if len(variables) == 0:
return []
if semver.match(tf.__version__, '<1.0.0'):
init_flag = sess.run(
tf.pack([tf.is_variable_initialized(v) for v in variables]))
else:
init_flag = sess.run(
tf.stack([tf.is_variable_initialized(v) for v in variables]))
return [v for v, f in zip(variables, init_flag) if not f]
def mean_huber_loss(y_true, y_pred, max_grad=1.):
"""Return mean huber loss.
Same as huber_loss, but takes the mean over all values in the
output tensor.
Parameters
----------
y_true: np.array, tf.Tensor
Target value.
y_pred: np.array, tf.Tensor
Predicted value.
max_grad: float, optional
Positive floating point value. Represents the maximum possible
gradient magnitude.
Returns
-------
tf.Tensor
The mean huber loss.
"""
diff = y_pred - y_true
if semver.match(tf.__version__, '<1.0.0'):
huber_val = tf.select(
tf.abs(diff) < max_grad, 0.5 * tf.square(diff),
max_grad * (tf.abs(diff) - 0.5 * max_grad))
else:
huber_val = tf.where(
tf.abs(diff) < max_grad, 0.5 * tf.square(diff),
max_grad * (tf.abs(diff) - 0.5 * max_grad))
return tf.reduce_mean(huber_val)
def matches_version(version, version_range):
if version_range is None or not version:
return True
# For the purposes of upgrade comparison, treat dev versions as alphas
version = normalize_version_to_semver(version).replace("dev", "a")
version_range = "".join(get_version_and_operator_from_range(version_range))
return match(version, version_range)
def connect(self):
"""Enumerate and connect to the first USB HID interface."""
def passphrase_handler(_):
log.debug('using %s passphrase for %s',
'non-empty' if self.passphrase else 'empty', self)
return self._defs.PassphraseAck(passphrase=self.passphrase)
def create_pin_handler(conn):
if not sys.stdin.closed and os.isatty(sys.stdin.fileno()):
return conn.callback_PinMatrixRequest # CLI-based PIN handler
def qt_handler(_):
# pylint: disable=import-error
from PyQt5.QtWidgets import QApplication, QInputDialog, QLineEdit
label = (
'Use the numeric keypad to describe number positions.\n'
'The layout is:\n'
' 7 8 9\n'
' 4 5 6\n'
' 1 2 3\n'
'Please enter PIN:')
app = QApplication([])
qd = QInputDialog()
qd.setTextEchoMode(QLineEdit.Password)
qd.setLabelText(label)
qd.show()
app.exec_()
return self._defs.PinMatrixAck(pin=qd.textValue())
return qt_handler
for d in self._defs.Transport.enumerate():
log.debug('endpoint: %s', d)
transport = self._defs.Transport(d)
connection = self._defs.Client(transport)
connection.callback_PassphraseRequest = passphrase_handler
connection.callback_PinMatrixRequest = create_pin_handler(
connection)
f = connection.features
log.debug('connected to %s %s', self, f.device_id)
log.debug('label : %s', f.label)
log.debug('vendor : %s', f.vendor)
current_version = '{}.{}.{}'.format(f.major_version,
f.minor_version,
f.patch_version)
log.debug('version : %s', current_version)
log.debug('revision : %s', binascii.hexlify(f.revision))
if not semver.match(current_version, self.required_version):
fmt = ('Please upgrade your {} firmware to {} version'
' (current: {})')
raise ValueError(
fmt.format(self, self.required_version, current_version))
try:
connection.ping(msg='', pin_protection=True) # unlock PIN
except Exception as e:
log.exception('ping failed: %s', e)
connection.close() # so the next HID open() will succeed
raise
return connection
raise interface.NotFoundError('{} not connected'.format(self))
def connect(self):
"""Enumerate and connect to the first USB HID interface."""
def passphrase_handler(_):
log.debug('using %s passphrase for %s',
'non-empty' if self.passphrase else 'empty', self)
return self._defs.PassphraseAck(passphrase=self.passphrase)
for d in self._defs.Transport.enumerate():
log.debug('endpoint: %s', d)
transport = self._defs.Transport(d)
connection = self._defs.Client(transport)
connection.callback_PassphraseRequest = passphrase_handler
f = connection.features
log.debug('connected to %s %s', self, f.device_id)
log.debug('label : %s', f.label)
log.debug('vendor : %s', f.vendor)
current_version = '{}.{}.{}'.format(f.major_version,
f.minor_version,
f.patch_version)
log.debug('version : %s', current_version)
log.debug('revision : %s', binascii.hexlify(f.revision))
if not semver.match(current_version, self.required_version):
fmt = ('Please upgrade your {} firmware to {} version'
' (current: {})')
raise ValueError(
fmt.format(self, self.required_version, current_version))
connection.ping(msg='', pin_protection=True) # unlock PIN
return connection
raise interface.NotFoundError('{} not connected'.format(self))
def _load_client(name, client_type, hid_transport,
passphrase_ack, identity_type,
required_version, call_exception):
def empty_passphrase_handler(_):
return passphrase_ack(passphrase='')
for d in hid_transport.enumerate():
connection = client_type(hid_transport(d))
connection.callback_PassphraseRequest = empty_passphrase_handler
f = connection.features
log.debug('connected to %s %s', name, f.device_id)
log.debug('label : %s', f.label)
log.debug('vendor : %s', f.vendor)
current_version = '{}.{}.{}'.format(f.major_version,
f.minor_version,
f.patch_version)
log.debug('version : %s', current_version)
log.debug('revision : %s', binascii.hexlify(f.revision))
if not semver.match(current_version, required_version):
fmt = 'Please upgrade your {} firmware to {} version (current: {})'
raise ValueError(fmt.format(name,
required_version,
current_version))
yield ClientWrapper(connection=connection,
identity_type=identity_type,
device_name=name,
call_exception=call_exception)
def verify_gpg_version():
"""Make sure that the installed GnuPG is not too old."""
existing_gpg = keyring.gpg_version().decode('ascii')
required_gpg = '>=2.1.11'
msg = 'Existing GnuPG has version "{}" ({} required)'.format(existing_gpg,
required_gpg)
assert semver.match(existing_gpg, required_gpg), msg
def validate_extensions(extensions):
"""Process the extension dependencies."""
ext_map = dict(
(ext.name, ext) for ext in extensions
)
for ext in extensions:
for dependency, versions in ext.requirements.items():
ext_dependency = ext_map.get(dependency, None)
if not ext_dependency:
raise MissingDependency(
'{0} is required by {1} but is not loaded.'.format(
ext.name,
dependency,
)
)
for version in versions:
if not semver.match(ext.version, version):
raise InvalidDependency(
'{0}-{1} required by {2} but found {0}-{3}.'.format(
dependency,
version,
ext.name,
ext.version,
)
)
return extensions
def _load_client(name, client_type, hid_transport,
passphrase_ack, identity_type,
required_version, call_exception):
def empty_passphrase_handler(_):
return passphrase_ack(passphrase='')
for d in hid_transport.enumerate():
connection = client_type(hid_transport(d))
connection.callback_PassphraseRequest = empty_passphrase_handler
f = connection.features
log.debug('connected to %s %s', name, f.device_id)
log.debug('label : %s', f.label)
log.debug('vendor : %s', f.vendor)
current_version = '{}.{}.{}'.format(f.major_version,
f.minor_version,
f.patch_version)
log.debug('version : %s', current_version)
log.debug('revision : %s', binascii.hexlify(f.revision))
if not semver.match(current_version, required_version):
fmt = 'Please upgrade your {} firmware to {} version (current: {})'
raise ValueError(fmt.format(name,
required_version,
current_version))
yield ClientWrapper(connection=connection,
identity_type=identity_type,
device_name=name,
call_exception=call_exception)
def validate_extensions(extensions):
"""Process the extension dependencies."""
ext_map = dict((ext.name, ext) for ext in extensions)
for ext in extensions:
for dependency, versions in ext.requirements.items():
ext_dependency = ext_map.get(dependency, None)
if not ext_dependency:
raise MissingDependency(
'{0} is required by {1} but is not loaded.'.format(
ext.name,
dependency,
))
for version in versions:
if not semver.match(ext.version, version):
raise InvalidDependency(
'{0}-{1} required by {2} but found {0}-{3}.'.format(
dependency,
version,
ext.name,
ext.version,
))
return extensions
def _check_version(self, tc_instance):
if tc_instance.config.get("compatible") and tc_instance.config[
'compatible']['framework']['name']:
framework = tc_instance.config['compatible']['framework']
# Check if version requirement is available
# and that the testcase is meant for this framework
if framework['version'] and framework['name'] == "Icetea":
ver_str = framework['version']
fw_version = get_fw_version()
try:
if not self._check_major_version(fw_version, ver_str):
result = self._wrong_version(
tc_instance, ver_str,
"Testcase not suitable for version >1.0.0. "
"Please install Icetea {}".format(ver_str))
return result
except ValueError:
# Unable to convert fw_version to integer, let's just proceed.
return None
if ver_str[0].isdigit():
return self._wrong_version(
tc_instance,
ver_str) if fw_version != ver_str else None
# Handle case where the version is a version number without comparison operators
if not semver.match(fw_version, ver_str):
result = self._wrong_version(tc_instance, ver_str)
return result
return None
else:
return None
def check_deps(dependencies, parent=""):
if isinstance(dependencies, dict):
for pkg_name, exp_ver in dependencies.items():
if parent:
full_name = "{} ({})".format(pkg_name, parent)
else:
full_name = pkg_name
meta = getPackageMeta(pkg_name)
ver = meta.__version__
if MAJOR_MINOR_REGEX.fullmatch(ver):
ver += ".0" # Add a fictive patch number to fit semver format
if not semver.match(ver, exp_ver):
raise RuntimeError("Incompatible '{}' package version. "
"Expected: {} "
"Found: {}".
format(pkg_name, exp_ver, ver))
if hasattr(meta, "__dependencies__"):
deps = meta.__dependencies__
check_deps(deps, full_name)
else:
pkg = dependencies if isinstance(dependencies, str) else \
dependencies.__name__
meta = getPackageMeta(pkg)
deps = meta.__dependencies__
check_deps(deps)
def get_report_template(report_version: str,
default_template: str = "index.md"):
"""
Retrieve and return the most appropriate template based on the
chaostoolkit-lib version used when running the experiment.
"""
env = Environment(loader=PackageLoader('chaosreport', 'template'))
env.filters["pretty_date"] = lambda d: str(
maya.MayaDT.from_datetime(dateparser.parse(d)))
env.globals["pretty_duration"] = lambda d0, d1: date.delta(
dateparser.parse(d0), dateparser.parse(d1), words=False)[0]
if not report_version:
return env.get_template(default_template)
templates = []
for name in env.list_templates(["md"]):
if name in ["index.md", "header.md"]:
continue
_, _, v = name.split('_')
v, _ = v.rsplit('.md', 1)
templates.append((semver.parse_version_info(v), name))
templates = sorted(templates, key=lambda vinfo: vinfo[0])
report_version = report_version.replace('rc1', '-rc1')
for (vinfo, name) in templates:
if semver.match(
report_version,
"<={v}".format(v=semver.format_version(**vinfo._asdict()))):
return env.get_template(name)
# none of the old versions matched, we can use the latest template
return env.get_template(default_template)
def assert_min_version(path, min_version):
version = discover_version(path)
if not semver.match(to_semver(version), f">={to_semver(min_version)}"):
fail(
f"{path} is version {version} which is less than the required version {min_version}"
)
return version
def semverMatch(have,want):
try:
if have.startswith('v'):
have = have[1:]
if want.startswith('~'):
haveList = have.split('.')
wantList = want[1:].split('.')
return haveList[0] == wantList[0] and haveList[1] == wantList[1] and int(haveList[2]) >= int(wantList[2])
elif want.startswith('^'):
wantList = want[1:].split('.')
greaterOrEqual = '>=%s.%s.%s' % (wantList[0], wantList[1], wantList[2])
lessThan = '<%s.%s.%s' % (int(wantList[0]) + 1, 0, 0)
return semver.match(have,greaterOrEqual) and semver.match(have, lessThan)
else:
return semver.match(have,want)
except:
return False
def __init__(self, server="localhost:8983",
detect_live_nodes=False,
user=None,
password=None,
timeout=10,
webappdir='solr',
version='5.3.0',
request_retries=1,
use_https=False):
self.user = user
self.password = password
self.timeout = timeout
self.webappdir = webappdir
self.version = version
self.request_retries = request_retries
if not semver.match(version, MIN_SUPPORTED_VERSION) and semver.match(version, MAX_SUPPORTED_VERSION):
raise Exception("Unsupported version %s" % version)
if semver.match(self.version, '<5.4.0'):
self.zk_path = '/{webappdir}/zookeeper'.format(webappdir=self.webappdir)
else:
self.zk_path = '/{webappdir}/admin/zookeeper'.format(webappdir=self.webappdir)
protocol = "https" if use_https else "http"
self.url_template = '{protocol}://{{server}}/{webappdir}/'.format(protocol=protocol, webappdir=self.webappdir)
if type(server) == str:
self.url = self.url_template.format(server=server)
servers = [self.url, self.url]
if detect_live_nodes:
url = servers[0]
self.servers = self.detect_nodes(url)
else:
self.servers = servers
if type(server) == list:
servers = [self.url_template.format(server=a) for a in server]
if detect_live_nodes:
url = servers[0]
self.servers = self.detect_nodes(url)
else:
self.servers = servers
self.client = _Request(self)
def find_semantic_version(releases, versions):
matching_releases = []
for release in releases:
if release.count('.') == 2:
try:
if semver.match(release, versions[0]) \
and semver.match(release, versions[1]):
matching_releases.append(release)
except:
pass
if len(matching_releases) == 1:
out = matching_releases[0]
elif len(matching_releases) == 2:
out = semver.max_ver(matching_releases[0], matching_releases[1])
elif len(matching_releases) > 2:
out = sorted(matching_releases)[-1]
else:
out = False
return out
def __init__(self, package, versionspec):
# if something deprecated '>=4.1.0' we want it to raise during the 4.1.0-dev, and 4.1.0-rc,
# not just when we release 4.1.0, so remove any extra-tags.
versionspec = versionspec.split('-')[0]
current_version = traitlets.import_item(package+'.__version__')
self.match = semver.match(current_version, versionspec)
self.package = package
self.spec = versionspec
def match(version, spec):
if version[0] == 'v':
version = version[1:]
if spec[0] == '^':
base = spec[1:]
parsed_base = semver.parse(base)
if parsed_base['major'] > 0:
top = semver.bump_major(base)
elif parsed_base['minor'] > 0:
top = semver.bump_minor(base)
else:
top = semver.bump_patch(base)
return semver.match(version, ">="+base) and semver.match(version, "<="+top)
else:
try:
return semver.match(version, spec)
except ValueError:
# this happens when the spec isn't an expression, in which case we need an exact match
return semver.parse(version) == semver.parse(spec)
def constraint_match(constraint_string, version):
if not constraint_string:
return True
constraints = [c.strip() for c in constraint_string.split(',') if c.strip()]
for c in constraints:
if not re.match(r'[><=][=]?\d+(\.\d+)*', c):
raise InvalidConstraintException('invalid constraint: %s' % c)
return all(semver.match(version, c) for c in constraints)
def is_compatible(self):
if self.s.name and self.compatibility:
compatible_version = self.compatibility.get(self.s.name)
if not compatible_version:
return False
if self.s.version:
return semver.match(self.s.version, compatible_version)
return False
return True
def extract_bower_zipfile(self, zip_file, dest, expected_version=None):
bower_json = None
root = None
deps_installed = []
for info in zip_file.infolist():
if PurePath(info.filename).name == "bower.json":
with zip_file.open(info) as f:
bower_json = json.load(f)
root = str(PurePath(info.filename).parent)
break
version = bower_json["version"]
if expected_version is not None:
expected_version = Bower.clean_semver(expected_version)
if not semver.match(version, expected_version):
click.secho("error: versions do not match ({} =/= {})".format(
version, expected_version))
raise InvalidPackageError
if "dependencies" in bower_json:
for package, version in bower_json["dependencies"].items():
url = Bower.get_package_url(package)
deps_installed.extend(self.get_bower_package(
url, dest=dest, version=version))
ignore_patterns = [GitIgnorePattern(ig) for ig in bower_json["ignore"]]
path_spec = PathSpec(ignore_patterns)
namelist = [path for path in zip_file.namelist()
if PurePath(path).parts[0] == root]
ignored = list(path_spec.match_files(namelist))
for path in namelist:
dest_path = PurePath(
bower_json["name"],
*PurePath(path).parts[1:])
if path in ignored:
continue
for path in ignored:
for parent in PurePath(path):
if parent in ignored:
continue
if path.endswith("/"):
if list(path_spec.match_files([str(dest_path)])):
ignored.append(PurePath(path))
elif not (dest / dest_path).is_dir():
(dest / dest_path).mkdir(parents=True)
else:
target_path = dest / dest_path.parent / dest_path.name
source = zip_file.open(path)
target = target_path.open("wb")
with source, target:
shutil.copyfileobj(source, target)
deps_installed.append((bower_json["name"], bower_json["version"]))
return deps_installed
def test_should_raise_value_error_for_unexpected_match_expression():
with pytest.raises(ValueError):
match("2.3.7", "=2.3.7")
with pytest.raises(ValueError):
match("2.3.7", "~2.3.7")
with pytest.raises(ValueError):
match("2.3.7", "^2.3.7")
def test_should_raise_value_error_for_invalid_match_expression():
with pytest.raises(ValueError):
match('1.0.0', '')
with pytest.raises(ValueError):
match('1.0.0', '!')
with pytest.raises(ValueError):
match('1.0.0', '1.0.0')
def get_arch(): "Returns architecture name for PyV8 binary" is_64bit = sys.maxsize > 2**32 system_name = platform.system() if system_name == 'Darwin': if semver.match(platform.mac_ver()[0], '<10.7.0'): return 'mac106' return 'osx' if system_name == 'Windows': return 'win64' if is_64bit else 'win32' if system_name == 'Linux': return 'linux64' if is_64bit else 'linux32'
def _check_version(self):
"""Check if the Brayns' version is sufficient enough."""
status = utils.http_request(HTTP_METHOD_GET, self.http_url, 'version')
if status.code != HTTP_STATUS_OK:
raise Exception('Cannot obtain version from Brayns')
version = '.'.join(str(x) for x in [status.contents['major'], status.contents['minor'],
status.contents['patch']])
import semver
if semver.match(version, '<{0}'.format(MINIMAL_VERSION)):
raise Exception('Brayns does not satisfy minimal required version; '
'needed {0}, got {1}'.format(MINIMAL_VERSION, version))
def test_should_not_raise_value_error_for_expected_match_expression():
assert match("2.3.7", "<2.4.0") is True
assert match("2.3.7", ">2.3.5") is True
assert match("2.3.7", "<=2.3.9") is True
assert match("2.3.7", ">=2.3.5") is True
assert match("2.3.7", "==2.3.7") is True
assert match("2.3.7", "!=2.3.7") is False
def get_bower_package(self, url, dest=None, version=None,
process_deps=True):
dest = dest or Path(".") / "assets"
parsed_url = urlparse(url)
if parsed_url.scheme == "git" or parsed_url.path.endswith(".git"):
if parsed_url.netloc == "github.com":
user, repo = parsed_url.path[1:-4].split("/")
response = get(github_api_uri +
"/repos/{}/{}/tags".format(user, repo))
tags = response.json()
target = None
if not len(tags):
click.secho("fatal: no tags exist for {}/{}".format(
user, repo), fg="red")
raise InvalidPackageError
if version is None:
target = tags[0]
else:
for tag in tags:
if semver.match(tag["name"],
Bower.clean_semver(version)):
target = tag
break
if not target:
click.secho(
"fatal: failed to find matching tag for "
"{user}/{repo} {version}".format(user, repo, version),
fg="red")
raise VersionNotFoundError
click.secho("installing {}/{}#{}".format(
user, repo, tags[0]["name"]), fg="green")
return self.get_bower_package(
url=target["zipball_url"],
dest=dest,
version=version)
raise NotImplementedError
click.echo("git clone {url}".format(url=url))
cmd = envoy.run('git clone {url} "{dest}"'.format(
url=url, dest=dest))
elif parsed_url.scheme in ("http", "https"):
zip_dest = download_file(url, dest=self.temp_dir,
label="{dest_basename}",
expected_extension="zip")
with zipfile.ZipFile(zip_dest, "r") as pkg:
return self.extract_bower_zipfile(pkg, dest,
expected_version=version)
# pkg.extractall(str(dest))
else:
click.secho("protocol currently unsupported :(")
sys.exit(1)
def _parse_tree(tree_object):
# Checking definition of tree_object
if not isinstance(tree_object, dict):
raise CraftAiDecisionError("Invalid decision tree format, the given json is not an object.")
# Checking version existence
tree_version = tree_object.get("_version")
if not tree_version:
raise CraftAiDecisionError(
"""Invalid decision tree format, unable to find the version"""
""" informations."""
)
# Checking version and tree validity according to version
if re.compile(r"\d+.\d+.\d+").match(tree_version) is None:
raise CraftAiDecisionError(
"""Invalid decision tree format, "{}" is not a valid version.""".
format(tree_version)
)
elif semver.match(tree_version, ">=1.0.0") and semver.match(tree_version, "<3.0.0"):
if tree_object.get("configuration") is None:
raise CraftAiDecisionError(
"""Invalid decision tree format, no configuration found"""
)
if tree_object.get("trees") is None:
raise CraftAiDecisionError(
"""Invalid decision tree format, no tree found."""
)
bare_tree = tree_object.get("trees")
configuration = tree_object.get("configuration")
else:
raise CraftAiDecisionError(
"""Invalid decision tree format, {} is not a supported"""
""" version.""".
format(tree_version)
)
return bare_tree, configuration, tree_version
def get_arch():
"Returns architecture name for PyV8 binary"
suffix = is_python3 and '-p3' or ''
p = lambda a: '%s%s' % (a, suffix)
is_64bit = sys.maxsize > 2**32
system_name = platform.system()
if system_name == 'Darwin':
if semver.match(platform.mac_ver()[0], '<10.7.0'):
return p('mac106')
return p('osx')
if system_name == 'Windows':
return p('win64') if is_64bit else p('win32')
if system_name == 'Linux':
return p('linux64') if is_64bit else p('linux32')
def complex_semver_match(version, version_specifier):
"""
Custom semver match function which also supports complex semver specifiers
such as >=1.6, <2.0, etc.
:rtype: ``bool``
"""
split_version_specifier = version_specifier.split(',')
if len(split_version_specifier) == 1:
# No comma, we can do a simple comparision
return semver.match(version, version_specifier)
else:
# Compare part by part
for version_specifier_part in split_version_specifier:
version_specifier_part = version_specifier_part.strip()
if not version_specifier_part:
continue
if not semver.match(version, version_specifier_part):
return False
return True
def cluster_health(self):
"""
Determine the state of all nodes and collections in the cluster. Problematic nodes or
collections are returned, along with their state, otherwise an `OK` message is returned
:return: a dict representing the status of the cluster
:rtype: dict
"""
res = []
if semver.match(self.version, '<5.4.0'):
params = {'detail': 'true', 'path': '/clusterstate.json'}
response = self.client.get(
('/{webappdir}/zookeeper'.format(webappdir=self.webappdir)), params).result
data = json.loads(response['znode']['data'])
collections = self.list()
for coll in collections:
shards = data[coll]['shards']
for shard, shard_info in iteritems(shards):
replicas = shard_info['replicas']
for replica, info in iteritems(replicas):
state = info['state']
if state != 'active':
item = {"collection": coll,
"replica": replica,
"shard": shard,
"info": info,
}
res.append(item)
else:
params = {'action': 'CLUSTERSTATUS', 'wt': 'json'}
response = self.client.get(
('/{webappdir}/admin/collections'.format(webappdir=self.webappdir)), params).result
for collection_name, collection in list(response.dict['cluster']['collections'].items()):
for shard_name, shard in list(collection['shards'].items()):
for replica_name, replica in list(shard['replicas'].items()):
if replica['state'] != 'active':
item = {"collection": collection_name,
"replica": replica_name,
"shard": shard_name,
"info": replica}
res.append(item)
if not res:
return {"status": "OK"}
return {"status": "NOT OK", "details": res}
def get_arch():
"Returns architecture name for PyV8 binary"
suffix = is_python3 and "-p3" or ""
p = lambda a: "%s%s" % (a, suffix)
is_64bit = sys.maxsize > 2 ** 32
system_name = platform.system()
if system_name == "Darwin":
try:
if semver.match(platform.mac_ver()[0], "<10.7.0"):
return p("mac106")
except:
pass
return p("osx")
if system_name == "Windows":
return p("win64") if is_64bit else p("win32")
if system_name == "Linux":
return p("linux64") if is_64bit else p("linux32")
def __new__(cls, *args, **kwds):
from . import notifiers
system = platform.system()
if system == 'Linux':
cls = notifiers.NotifySendNotifier
elif system == 'Darwin':
cls = notifiers.NotificationCenterNotifier
elif system == 'Windows':
if semver.match(platform.release(), '<6.2.9200'):
cls = notifiers.BalloonNotifier
else:
cls = notifiers.ToasterNotifier
else:
cls = notifiers.GrowlNotifier
return cls(*args, **kwds)