def parse(data: str, event: str):
if version.Version(lark.__version__) >= version.Version("1.0.0"):
out = parser_fast.parse(event + data)
else:
out = transformer.transform(parser.parse(event + data))
# TODO: Provide sanity checks for object construction e.g a linear target should always have LinearAttributes
return out
def forward(self, input, mask, dim):
"""
"""
# print("="*50)
# print("4. Xsoftmax:")
# print("input: ", input.shape)
# print("mask: ", mask.shape)
# print("dim: ", dim)
self.dim = dim
if version.Version(torch.__version__) >= version.Version('1.2.0a'):
rmask = ~(mask.bool())
else:
rmask = (
1 - mask).byte() # This line is not supported by Onnx tracing.
output = input.masked_fill(rmask, float('-inf'))
WINDOW_SIZE = opts.window_size
topk_scores, topk_ids = output.topk(
k=WINDOW_SIZE, dim=-1, largest=True) # [bs, head, q_len, k]
threshod = topk_scores[:, :, :, -1].unsqueeze(
-1).detach() # [bs, head, q_len, 1]
output = output.masked_fill(output < threshod, -1e9)
# print("output: ", output.shape, output[0, 0, 0, :])
output = torch.softmax(output, self.dim)
output.masked_fill_(rmask, 0)
self.save_for_backward(output)
return output
def test_model_knn_regressor_radius(self):
model, X = self._fit_model(RadiusNeighborsRegressor())
model_onnx = convert_sklearn(model,
"KNN regressor",
[("input", FloatTensorType([None, 4]))],
target_opset=TARGET_OPSET,
options={id(model): {
'optim': 'cdist'
}})
sess = InferenceSession(model_onnx.SerializeToString())
X = X[:5]
got = sess.run(None, {'input': X.astype(numpy.float32)})[0]
exp = model.predict(X.astype(numpy.float32))
if any(numpy.isnan(got.ravel())):
# The model is unexpectedly producing nan values
# not on all platforms.
rows = [
'--EXP--',
str(exp), '--GOT--',
str(got), '--EVERY-OUTPUT--'
]
for out in enumerate_model_node_outputs(model_onnx,
add_node=False):
onx = select_model_inputs_outputs(model_onnx, out)
sess = InferenceSession(onx.SerializeToString())
res = sess.run(None, {'input': X.astype(numpy.float32)})
rows.append('--{}--'.format(out))
rows.append(str(res))
if (pv.Version(ort_version) < pv.Version("1.4.0")):
return
raise AssertionError('\n'.join(rows))
assert_almost_equal(exp.ravel(), got.ravel(), decimal=3)
def check_versions():
"""
This endpoint validates software version using packaging.version lib,
it supports with letters even. For more information please check
https://www.python.org/dev/peps/pep-0440/
"""
if request.json == None:
abort(400, 'no parameters')
if 'version_1' in request.json and 'version_2' in request.json:
version_1 = request.json.get('version_1')
version_2 = request.json.get('version_2')
else:
abort(400, 'missing parameters')
# check versions format
try:
version.Version(version_1)
version.Version(version_2)
version_1 = version.parse(version_1)
version_2 = version.parse(version_2)
except InvalidVersion:
abort(400, 'format error')
comparison = compare(version_1, version_2)
return jsonify(
{'comparison': STRING_RESPONSE % (version_1, comparison, version_2)})
def check_version(parsed_args: argparse.Namespace) -> None:
info = get_version(parsed_args.master)
master_version = info["master"]["version"]
client_version = info["client"]["version"]
if not master_version:
print(
termcolor.colored(
"Master not found at {}. "
"Hint: Remember to set the DET_MASTER environment variable "
"to the correct Determined master IP or use the '-m' flag.".format(
parsed_args.master
),
"yellow",
)
)
elif version.Version(client_version) < version.Version(master_version):
print(
termcolor.colored(
"CLI version {} is less than master version {}. "
"Consider upgrading the CLI.".format(client_version, master_version),
"yellow",
)
)
elif version.Version(client_version) > version.Version(master_version):
print(
termcolor.colored(
"Master version {} is less than CLI version {}. "
"Consider upgrading the master.".format(master_version, client_version),
"yellow",
)
)
def main():
"""Main entry point."""
if len(sys.argv) < 2:
print("Usage {} <image_name>".format(sys.argv[0]))
return
url = "https://registry.hub.docker.com/v1/repositories/{}/tags".format(
sys.argv[1])
response = requests.get(url)
response.raise_for_status()
tags = json.loads(response.text)
versions = []
for item in tags:
try:
versions.append(version.parse(item["name"]))
except version.InvalidVersion:
pass
versions.sort(reverse=True)
if len(versions):
latest_version = versions.pop(0)
else:
latest_version = version.Version("0.0.0")
next_version = version.Version("{}.{}.{}".format(
latest_version.release[0],
latest_version.release[1],
latest_version.release[2] + 1,
))
print(next_version)
def _ScalaVersion() -> version.Version:
if _SparkVersion().major >= 3:
# https://spark.apache.org/docs/3.0.0/#downloading
return version.Version('2.12')
else:
# https://spark.apache.org/docs/2.4.0/#downloading
return version.Version('2.11')
def test_module_install_version(
tmpdir: py.path.local,
snippetcompiler_clean,
modules_v2_dir: str,
dev: bool,
) -> None:
"""
Make sure that the module install results in a module instance with the appropriate version information.
:param dev: whether to add a dev tag to the version
"""
module_name: str = "minimalv2module"
module_path: str = os.path.join(str(tmpdir), module_name)
module_version: version.Version = version.Version(
"1.2.3") if not dev else version.Version("1.2.3.dev0")
# set up simple project and activate the snippetcompiler venv
project: module.Project = snippetcompiler_clean.setup_for_snippet("")
# install module
module_from_template(
os.path.join(modules_v2_dir, module_name),
module_path,
new_version=module_version,
)
os.chdir(project.path)
ModuleTool().install(editable=True, path=module_path)
# check version
mod: module.Module = ModuleTool().get_module(module_name)
assert mod.version == module_version
class TestCustomTransformerTSNE(unittest.TestCase):
@unittest.skipIf(pv.Version(ort_version) <= pv.Version("0.3.0"),
reason="TopK is failing.")
def test_custom_pipeline_scaler(self):
digits = datasets.load_digits(n_class=6)
Xd = digits.data[:20]
yd = digits.target[:20]
n_samples, n_features = Xd.shape
ptsne_knn = PredictableTSNE()
ptsne_knn.fit(Xd, yd)
update_registered_converter(
PredictableTSNE,
"CustomPredictableTSNE",
predictable_tsne_shape_calculator,
predictable_tsne_converter,
)
model_onnx = convert_sklearn(
ptsne_knn,
"predictable_tsne",
[("input", FloatTensorType([None, Xd.shape[1]]))],
target_opset=TARGET_OPSET)
dump_data_and_model(Xd.astype(numpy.float32)[:7],
ptsne_knn,
model_onnx,
basename="CustomTransformerTSNEkNN-OneOffArray")
trace_line = []
def my_parser(scope, model, inputs, custom_parsers=None):
trace_line.append(model)
return _parse_sklearn_simple_model(scope, model, inputs,
custom_parsers)
model_onnx = convert_sklearn(
ptsne_knn,
"predictable_tsne",
[("input", FloatTensorType([None, Xd.shape[1]]))],
custom_parsers={PredictableTSNE: my_parser},
target_opset=TARGET_OPSET)
assert len(trace_line) == 1
dump_data_and_model(
Xd.astype(numpy.float32)[:7],
ptsne_knn,
model_onnx,
basename="CustomTransformerTSNEkNNCustomParser-OneOffArray")
update_registered_parser(PredictableTSNE, my_parser)
model_onnx = convert_sklearn(
ptsne_knn,
"predictable_tsne",
[("input", FloatTensorType([None, Xd.shape[1]]))],
target_opset=TARGET_OPSET)
assert len(trace_line) == 2
class TestSklearnTfidfVectorizerSparse(unittest.TestCase):
@unittest.skipIf(
TARGET_OPSET < 9,
# issue with encoding
reason="https://github.com/onnx/onnx/pull/1734")
@unittest.skipIf(pv.Version(ort.__version__) <= pv.Version("0.2.1"),
reason="sparse not supported")
def test_model_tfidf_transform_bug(self):
categories = [
"alt.atheism",
"soc.religion.christian",
"comp.graphics",
"sci.med",
]
twenty_train = fetch_20newsgroups(subset="train",
categories=categories,
shuffle=True,
random_state=0)
text_clf = Pipeline([("vect", CountVectorizer()),
("tfidf", TfidfTransformer())])
twenty_train.data[0] = "bruît " + twenty_train.data[0]
text_clf.fit(twenty_train.data, twenty_train.target)
model_onnx = convert_sklearn(text_clf,
name="DocClassifierCV-Tfidf",
initial_types=[("input",
StringTensorType([5]))],
target_opset=TARGET_OPSET)
dump_data_and_model(twenty_train.data[5:10],
text_clf,
model_onnx,
basename="SklearnPipelineTfidfTransformer")
def test_active_env_check_constraints(caplog, tmpvenv_active: str) -> None:
"""
Verify that the env.ActiveEnv.check() method's constraints parameter is taken into account as expected.
"""
caplog.set_level(logging.WARNING)
in_scope: Pattern[str] = re.compile("test-package-.*")
constraints: List[Requirement] = [
Requirement.parse("test-package-one~=1.0")
]
def check_log(version: Optional[version.Version]) -> None:
assert f"Incompatibility between constraint test-package-one~=1.0 and installed version {version}" in {
rec.message
for rec in caplog.records
}
assert env.ActiveEnv.check(in_scope)
caplog.clear()
assert not env.ActiveEnv.check(in_scope, constraints)
check_log(None)
caplog.clear()
create_install_package("test-package-one", version.Version("1.0.0"), [])
assert env.ActiveEnv.check(in_scope, constraints)
caplog.clear()
v: version.Version = version.Version("2.0.0")
create_install_package("test-package-one", v, [])
assert not env.ActiveEnv.check(in_scope, constraints)
check_log(v)
def is_version_compatible(
provider_v_in: Union[str, v.Version],
consumer_v_in: Union[str, v.Version],
raise_exception: bool = True,
) -> bool:
"""
Check provider supports consumer and throws exception if not
Set the spec so that the consumer has to be within a micro/patch release of the provider
NOTE - this isn't semver - breaks when have > v1 release as then treats minor as breaking,
e.g. 2.2.5 is not compat with 2.1.5
"""
consumer_v = v.Version(consumer_v_in) if isinstance(consumer_v_in, str) else consumer_v_in
provider_v = v.Version(provider_v_in) if isinstance(provider_v_in, str) else provider_v_in
provider_spec = SpecifierSet(f"~={provider_v.major}.{provider_v.minor}.0")
log.debug(f"Provider spec {provider_spec}, Consumer version {consumer_v}")
if consumer_v not in provider_spec:
if raise_exception:
raise VersionMismatch(
f"Consumer ({consumer_v}) and Provider ({provider_spec}) API versions not compatible"
)
return False
return True
def test_active_env_check_basic(
caplog,
tmpdir: str,
tmpvenv_active: str,
) -> None:
"""
Verify that the env.ActiveEnv.check() method detects all possible forms of incompatibilities within the environment.
"""
caplog.set_level(logging.WARNING)
in_scope_test: Pattern[str] = re.compile("test-package-.*")
in_scope_nonext: Pattern[str] = re.compile("nonexistant-package")
def assert_all_checks(expect_test: Tuple[bool, str] = (True, ""),
expect_nonext: Tuple[bool,
str] = (True, "")) -> None:
for in_scope, expect in [(in_scope_test, expect_test),
(in_scope_nonext, expect_nonext)]:
caplog.clear()
assert env.ActiveEnv.check(in_scope) == expect[0]
if not expect[0]:
assert expect[1] in {rec.message for rec in caplog.records}
assert_all_checks()
create_install_package("test-package-one", version.Version("1.0.0"), [])
assert_all_checks()
create_install_package("test-package-two", version.Version("1.0.0"),
[Requirement.parse("test-package-one~=1.0")])
assert_all_checks()
create_install_package("test-package-one", version.Version("2.0.0"), [])
assert_all_checks(expect_test=(
False,
"Incompatibility between constraint test-package-one~=1.0 and installed version 2.0.0"
))
def check_ovs_and_skip(test):
ovs = ovs_lib.BaseOVS()
current_ovs_version = version.Version(ovs.config['ovs_version'])
if current_ovs_version < version.Version(ovs_version):
test.skipTest("This test requires OVS version %s or higher." %
ovs_version)
return f(test)
def test_ovr_classification_decision_function_binary(self):
model, X = fit_classification_model(
OneVsRestClassifier(LogisticRegression()), 2)
options = {id(model): {'raw_scores': True}}
model_onnx = convert_sklearn(
model,
"ovr classification",
[("input", FloatTensorType([None, X.shape[1]]))],
options=options,
target_opset=TARGET_OPSET)
self.assertIsNotNone(model_onnx)
dump_data_and_model(
X,
model,
model_onnx,
basename="SklearnOVRClassificationDecisionFunctionBinary",
methods=['predict', 'decision_function_binary'])
if pv.Version(ort_version) < pv.Version("1.0.0"):
return
options = {id(model): {'raw_scores': True, 'zipmap': False}}
model_onnx = convert_sklearn(
model,
"ovr classification",
[("input", FloatTensorType([None, X.shape[1]]))],
options=options,
target_opset=TARGET_OPSET)
sess = InferenceSession(model_onnx.SerializeToString())
got = sess.run(None, {'input': X})[1]
dec = model.decision_function(X)
assert_almost_equal(got[:, 1], dec, decimal=4)
assert_almost_equal(-got[:, 0], dec, decimal=4)
def test_auto_updates_pip(self):
"""Installing a plugin should automatically update pip if it's
out-of-date.
"""
# arrange
old_pip_version = "20.0.1"
assert (subprocess.run([
disthelpers.get_pip_path(),
"install",
"-U",
f"pip=={old_pip_version}",
]).returncode == 0)
assert version.Version(
get_pkg_version("pip")) == version.Version(old_pip_version)
plugin_name = "junit4"
plugin_version = "v1.0.0"
cmd = [
*pluginmanager.plugin_category.install.as_name_tuple(),
"--plugin-spec",
f"{plugin_name}{pluginmanager.PLUGIN_SPEC_SEP}{plugin_version}",
]
# act
repobee.run(cmd)
# assert
assert version.Version(
get_pkg_version("pip")) > version.Version(old_pip_version)
def __init__(self, cache_dir: str, split: str):
with tf.io.gfile.GFile(utils.get_cached_info_path(cache_dir, split)) as f:
split_info = json.load(f)
features = split_info["features"]
with tf.io.gfile.GFile(utils.get_cached_stats_path(cache_dir, split)) as f:
stats = json.load(f)
version_when_cached = version.Version(
split_info.get("seqio_version", "0.pre"))
version_with_true_dtypes = version.Version("0.0.0")
if version_when_cached < version_with_true_dtypes:
# Assume that all int64 features are really int32.
for name, feat in features.items():
if feat["dtype"] == "int64":
logging.info("Casting cached '%s' to int32.", name)
feat["dtype"] = "int32"
# Use `FixedLenSequenceFeature` for sequences with variable length.
def _feature_config(shape, dtype):
if dtype in ("int32", "bool"):
# int32 and bool are stored as int64 in the tf.train.Example protobuf.
# TODO(adarob): Support other conversions.
dtype = "int64"
if shape and shape[0] is None:
return tf.io.FixedLenSequenceFeature(
shape[1:], dtype, allow_missing=True)
return tf.io.FixedLenFeature(shape, dtype)
feature_description = {
feat: _feature_config(**desc) for feat, desc in features.items()
}
def read_file_fn(filepattern):
ds = tf.data.TFRecordDataset(filepattern)
ds = ds.map(
lambda pb: tf.io.parse_single_example(pb, feature_description),
num_parallel_calls=tf.data.experimental.AUTOTUNE)
# Cast features back to the types from the info JSON since some features
# must be cast for storage (e.g., in32 is stored as int64).
ds = ds.map(
lambda x: {k: tf.cast(v, features[k]["dtype"]) for k, v in x.items()},
num_parallel_calls=tf.data.experimental.AUTOTUNE)
# Legacy cached datasets may use old "_plaintext" suffix. Rename to
# "_pretokenized".
ds = _rename_plaintext_to_pretokenized(ds)
return ds
split_to_filepattern = {
split: "%s-*-of-*%d" % (
utils.get_cached_tfrecord_prefix(cache_dir, split),
split_info["num_shards"])
}
super().__init__(
read_file_fn=read_file_fn,
split_to_filepattern=split_to_filepattern,
num_input_examples={split: stats["examples"]}
)
def test_documentation_examples(self):
this = os.path.abspath(os.path.dirname(__file__))
fold = os.path.normpath(os.path.join(this, '..', 'examples'))
found = os.listdir(fold)
tested = 0
for name in found:
if name.startswith("plot_") and name.endswith(".py"):
if (name == "plot_pipeline_lightgbm.py" and pv.Version(
onnxruntime.__version__) < pv.Version('1.0.0')):
continue
print("run %r" % name)
try:
mod = import_source(fold, os.path.splitext(name)[0])
assert mod is not None
except FileNotFoundError:
# try another way
cmds = [sys.executable, "-u", os.path.join(fold, name)]
p = subprocess.Popen(cmds,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
res = p.communicate()
out, err = res
st = err.decode('ascii', errors='ignore')
if len(st) > 0 and 'Traceback' in st:
if "No such file or directory: 'dot'" in st:
# dot not installed, this part
# is tested in onnx framework
pass
elif '"dot" not found in path.' in st:
# dot not installed, this part
# is tested in onnx framework
pass
elif ('Please fix either the inputs or '
'the model.') in st:
# onnxruntime datasets changed in master branch,
# still the same in released version on pypi
pass
elif ('Current official support for domain ai.onnx '
'is till opset 12.') in st:
# one example is using opset 13 but onnxruntime
# only support up to opset 12.
pass
elif "'str' object has no attribute 'decode'" in st:
# unstable bug in scikit-learn<0.24
pass
else:
raise RuntimeError(
"Example '{}' (cmd: {} - exec_prefix='{}') "
"failed due to\n{}"
"".format(name, cmds, sys.exec_prefix, st))
tested += 1
if tested == 0:
raise RuntimeError("No example was tested.")
def skl2onnx_convert_lightgbm(scope, operator, container):
options = scope.get_options(operator.raw_operator)
if 'split' in options:
if pv.Version(oml_version) < pv.Version('1.9.2'):
warnings.warn(
"Option split was released in version 1.9.2 but %s is "
"installed. It will be ignored." % oml_version)
operator.split = options['split']
else:
operator.split = None
convert_lightgbm(scope, operator, container)
def test_transformer_creation_without_optional_args(
name_from_base,
create_model,
sagemaker_session,
tensorflow_inference_version,
tensorflow_inference_py_version,
):
if version.Version(tensorflow_inference_version) < version.Version("1.11"):
pytest.skip(
"Legacy TF version requires explicit image URI, and "
"this logic is tested in test_create_model_with_custom_image.")
model_name = "generated-model-name"
name_from_base.return_value = model_name
model = Mock()
create_model.return_value = model
base_job_name = "tensorflow"
tf = TensorFlow(
entry_point=SCRIPT_PATH,
framework_version=tensorflow_inference_version,
py_version=tensorflow_inference_py_version,
role=ROLE,
sagemaker_session=sagemaker_session,
instance_count=INSTANCE_COUNT,
instance_type=INSTANCE_TYPE,
base_job_name=base_job_name,
)
tf.latest_training_job = _TrainingJob(sagemaker_session, "some-job-name")
tf.transformer(INSTANCE_COUNT, INSTANCE_TYPE)
name_from_base.assert_called_with(base_job_name)
create_model.assert_called_with(
role=ROLE,
vpc_config_override="VPC_CONFIG_DEFAULT",
entry_point=None,
enable_network_isolation=False,
name=model_name,
)
model.transformer.assert_called_with(
INSTANCE_COUNT,
INSTANCE_TYPE,
accept=None,
assemble_with=None,
env=None,
max_concurrent_transforms=None,
max_payload=None,
output_kms_key=None,
output_path=None,
strategy=None,
tags=None,
volume_kms_key=None,
)
def forward(self, hidden_states, residual_states, input_mask):
out = self.conv(hidden_states.permute(0,2,1).contiguous()).permute(0,2,1).contiguous()
if version.Version(torch.__version__) >= version.Version('1.2.0a'):
rmask = (1-input_mask).bool()
else:
rmask = (1-input_mask).byte()
out.masked_fill_(rmask.unsqueeze(-1).expand(out.size()), 0)
out = ACT2FN[self.conv_act](self.dropout(out))
output_states = MaskedLayerNorm(self.LayerNorm, residual_states + out, input_mask)
return output_states
def test_enable_sm_metrics_if_fw_ver_is_at_least_1_15(
sagemaker_session, tensorflow_training_version,
tensorflow_training_py_version):
if version.Version(tensorflow_training_version) < version.Version("1.15"):
pytest.skip("This test is for TF 1.15 and higher.")
tf = _build_tf(
sagemaker_session,
framework_version=tensorflow_training_version,
py_version=tensorflow_training_py_version,
)
assert tf.enable_sagemaker_metrics
def forward(self, input, mask, dim):
self.dim = dim
if version.Version(torch.__version__) >= version.Version("1.2.0a"):
rmask = ~(mask.bool())
else:
rmask = (1 - mask).byte() # This line is not supported by Onnx tracing.
output = input.masked_fill(rmask, float("-inf"))
output = torch.softmax(output, self.dim)
output.masked_fill_(rmask, 0)
self.save_for_backward(output)
return output
def forward(self, input, mask, dim):
self.dim = dim
if version.Version(torch.__version__) >= version.Version('1.2.0a'):
rmask = (1 - mask).bool()
else:
rmask = (1 - mask).byte()
output = input.masked_fill(rmask, float('-inf'))
output = torch.softmax(output, self.dim)
output.masked_fill_(rmask, 0)
self.save_for_backward(output)
return output
class TestSklearnArrayFeatureExtractor(unittest.TestCase):
@unittest.skipIf(ColumnTransformer is None
or pv.Version(ort_version) <= pv.Version("0.4.0"),
reason="onnxruntime too old")
def test_array_feature_extractor(self):
data_to_cluster = pd.DataFrame([[1, 2, 3.5, 4.5], [1, 2, 1.7, 4.0],
[2, 4, 2.4, 4.3], [2, 4, 2.5, 4.0]],
columns=[1, 2, 3, 4])
cat_attributes_clustering = [1, 2]
num_attributes_clustering = [3, 4] # this is of length 12 in reality
gmm = GaussianMixture(n_components=2, random_state=1)
ohe_cat = [
OneHotEncoder(categories='auto', sparse=False, drop=None)
for i in cat_attributes_clustering
]
ct_cat = ColumnTransformer(
[("oneHotEncoder" + str(i), ohe_cat[i], [i])
for i, item in enumerate(cat_attributes_clustering)],
remainder='passthrough')
onehotencoding_pipeline = Pipeline([
("columnTransformer", ct_cat),
])
clustering_pipeline = Pipeline([('onehotencoder_and_scaler',
onehotencoding_pipeline),
('clustering', gmm)])
clustering_pipeline.fit(X=data_to_cluster)
initial_type = [
('float_input',
FloatTensorType([
None,
len([*cat_attributes_clustering, *num_attributes_clustering])
]))
]
data = data_to_cluster.values.astype(np.float32)
# checks the first step
model_onnx = to_onnx(clustering_pipeline.steps[0][1],
initial_types=initial_type,
target_opset=TARGET_OPSET)
dump_data_and_model(data,
clustering_pipeline.steps[0][1],
model_onnx,
basename="SklearnArrayFeatureExtractorStep0")
# checks the whole pipeline
model_onnx = to_onnx(clustering_pipeline,
initial_types=initial_type,
target_opset=TARGET_OPSET)
dump_data_and_model(data,
clustering_pipeline,
model_onnx,
basename="SklearnArrayFeatureExtractor")
def test_disable_sm_metrics_if_fw_ver_is_less_than_1_15(
sagemaker_session, tensorflow_training_version,
tensorflow_training_py_version):
if version.Version(tensorflow_training_version) > version.Version("1.14"):
pytest.skip("This test is for TF 1.14 and lower.")
tf = _build_tf(
sagemaker_session,
framework_version=tensorflow_training_version,
py_version=tensorflow_training_py_version,
image_uri="old-image",
)
assert tf.enable_sagemaker_metrics is None
def __lt__(self, other: Any) -> bool:
if not isinstance(other, Version):
raise TypeError("Cannot compare Version with type {}".format(
other.__class__.__qualname__))
import packaging.version as pv
return (pv.Version(self.base) < pv.Version(other.base)
and _blank(self.stage, "") < _blank(other.stage, "")
and _blank(self.revision, 0) < _blank(other.revision, 0)
and _blank(self.distance, 0) < _blank(other.distance, 0)
and _blank(self.commit, "") < _blank(other.commit, "")
and bool(self.dirty) < bool(other.dirty))
def check_lib_version(lib_name: str, checked_version: str,
operator) -> (Optional[bool], str):
"""
Checks if a library is installed, and if it is, checks the operator(lib.__version__, checked_version) as a result.
This bool result along with a string analysis of result is returned.
If the library is not installed at all, then returns None instead, along with a string explaining
that the library is not installed
Args:
lib_name: lower case str name of the library that must be imported.
checked_version: semver string that is compared against lib.__version__.
operator: binary callable function func(a, b) -> bool; that compares lib.__version__ against version in
some manner. Must return a boolean.
Returns:
A tuple of results:
- Bool or None. Bool if the library could be imported, and the result of
operator(lib.__version__, checked_version) or False if __version__ is not implemented in lib.
None is passed if the library is not installed at all.
- A string analysis of the check.
"""
try:
if '.' in lib_name:
mod = import_class_by_path(lib_name)
else:
mod = __import__(lib_name)
if hasattr(mod, '__version__'):
lib_ver = version.Version(mod.__version__)
match_ver = version.Version(checked_version)
if operator(lib_ver, match_ver):
msg = f"Lib {lib_name} version is satisfied !"
return True, msg
else:
msg = (
f"Lib {lib_name} version ({lib_ver}) is not {operator.__name__} than required version {checked_version}.\n"
f"Please upgrade the lib using either pip or conda to the latest version."
)
return False, msg
else:
msg = (
f"Lib {lib_name} does not implement __version__ in its init file. "
f"Could not check version compatibility.")
return False, msg
except (ImportError, ModuleNotFoundError):
pass
msg = f"Lib {lib_name} has not been installed. Please use pip or conda to install this package."
return None, msg
class TestOnnxOperatorsSparse(unittest.TestCase):
@unittest.skipIf(TARGET_OPSET < 11,
reason="only available for opset >= 11")
@unittest.skipIf(pv.Version(ort_version) < pv.Version(THRESHOLD),
reason="fails with onnxruntime < %s" % THRESHOLD)
def test_onnx_init_dense(self):
X = np.array([1, 2, 3, 4, 5, 6]).astype(np.float32).reshape((3, 2))
node = OnnxAdd('X', X, output_names=['Y'], op_version=TARGET_OPSET)
model_def = node.to_onnx({'X': X},
outputs=[('Y', FloatTensorType())])
sess = InferenceSession(model_def.SerializeToString())
res = sess.run(None, {'X': X})[0]
assert_almost_equal(X + X, res)
@unittest.skipIf(TARGET_OPSET < 11,
reason="only available for opset >= 11")
@unittest.skipIf(pv.Version(ort_version) < pv.Version(THRESHOLD),
reason="fails with onnxruntime < %s" % THRESHOLD)
def test_onnx_init_sparse_coo(self):
row = np.array([0, 0, 1, 3, 1], dtype=np.float32)
col = np.array([0, 2, 1, 3, 1], dtype=np.float32)
data = np.array([1, 1, 1, 1, 1], dtype=np.float32)
X = coo_matrix((data, (row, col)), shape=(4, 4))
node = OnnxAdd(
'X', X, output_names=['Y'],
op_version=TARGET_OPSET)
model_def = node.to_onnx(
{'X': X}, outputs=[('Y', FloatTensorType())])
try:
sess = InferenceSession(model_def.SerializeToString())
except (RuntimeError, OrtInvalidArgument):
# Sparse tensor is not supported for constant.
return
try:
res = sess.run(None, {'X': X})[0]
except RuntimeError as e:
# Sparse tensor is not supported for constant.
warnings.warn(
"Unable to run with %r\n---\n%s\n%s" % (
{'X': X}, model_def, e))
return
assert_almost_equal(X + X, res)
def test_onnx_example_pdist(self):
x = np.array([1, 2, 4, 5, 5, 4]).astype(np.float32).reshape((3, 2))
opv = _TARGET_OPSET_
diff = OnnxSub('next_in', 'next', output_names=['diff'],
op_version=opv)
id_next = OnnxIdentity(
'next_in', output_names=['next_out'],
op_version=opv)
norm = OnnxReduceSumSquare(
diff, output_names=['norm'], axes=[1],
op_version=opv)
flat = OnnxSqueezeApi11(
norm, output_names=['scan_out'], axes=[1],
op_version=opv)
scan_body = id_next.to_onnx(
OrderedDict([('next_in', x), ('next', FloatTensorType())]),
outputs=[('next_out', FloatTensorType([3, 2])),
('scan_out', FloatTensorType([3]))],
other_outputs=[flat],
target_opset=opv)
sess = InferenceSession(scan_body.SerializeToString())
res = sess.run(None, {'next_in': x, 'next': x[:1]})
assert_almost_equal(x, res[0])
exp = np.array([0., 18., 20.], dtype=np.float32)
assert_almost_equal(exp, res[1])
node = OnnxScan(
'x', 'x', output_names=['y', 'z'],
num_scan_inputs=1, body=scan_body.graph,
op_version=opv)
model_def = node.to_onnx({'x': x},
outputs=[('y', FloatTensorType([3, 2])),
('z', FloatTensorType([3, 3]))])
try:
onnx.checker.check_model(model_def)
except ValidationError as e:
if pv.Version(onnx__version__) <= pv.Version("1.5.0"):
warnings.warn(e)
else:
raise e
sess = InferenceSession(model_def.SerializeToString())
res = sess.run(None, {'x': x})
exp = squareform(pdist(x, metric="sqeuclidean"))
assert_almost_equal(x, res[0])
assert_almost_equal(exp, res[1])
