Exemplo n.º 1
0
def parse(data: str, event: str):
    if version.Version(lark.__version__) >= version.Version("1.0.0"):
        out = parser_fast.parse(event + data)
    else:
        out = transformer.transform(parser.parse(event + data))
    # TODO: Provide sanity checks for object construction e.g a linear target should always have LinearAttributes
    return out
Exemplo n.º 2
0
    def forward(self, input, mask, dim):
        """
        """
        # print("="*50)
        # print("4. Xsoftmax:")
        # print("input: ", input.shape)
        # print("mask: ", mask.shape)
        # print("dim: ", dim)
        self.dim = dim
        if version.Version(torch.__version__) >= version.Version('1.2.0a'):
            rmask = ~(mask.bool())
        else:
            rmask = (
                1 - mask).byte()  # This line is not supported by Onnx tracing.

        output = input.masked_fill(rmask, float('-inf'))

        WINDOW_SIZE = opts.window_size
        topk_scores, topk_ids = output.topk(
            k=WINDOW_SIZE, dim=-1, largest=True)  # [bs, head, q_len, k]
        threshod = topk_scores[:, :, :, -1].unsqueeze(
            -1).detach()  # [bs, head, q_len, 1]
        output = output.masked_fill(output < threshod, -1e9)
        # print("output: ", output.shape, output[0, 0, 0, :])

        output = torch.softmax(output, self.dim)
        output.masked_fill_(rmask, 0)
        self.save_for_backward(output)
        return output
 def test_model_knn_regressor_radius(self):
     model, X = self._fit_model(RadiusNeighborsRegressor())
     model_onnx = convert_sklearn(model,
                                  "KNN regressor",
                                  [("input", FloatTensorType([None, 4]))],
                                  target_opset=TARGET_OPSET,
                                  options={id(model): {
                                               'optim': 'cdist'
                                           }})
     sess = InferenceSession(model_onnx.SerializeToString())
     X = X[:5]
     got = sess.run(None, {'input': X.astype(numpy.float32)})[0]
     exp = model.predict(X.astype(numpy.float32))
     if any(numpy.isnan(got.ravel())):
         # The model is unexpectedly producing nan values
         # not on all platforms.
         rows = [
             '--EXP--',
             str(exp), '--GOT--',
             str(got), '--EVERY-OUTPUT--'
         ]
         for out in enumerate_model_node_outputs(model_onnx,
                                                 add_node=False):
             onx = select_model_inputs_outputs(model_onnx, out)
             sess = InferenceSession(onx.SerializeToString())
             res = sess.run(None, {'input': X.astype(numpy.float32)})
             rows.append('--{}--'.format(out))
             rows.append(str(res))
         if (pv.Version(ort_version) < pv.Version("1.4.0")):
             return
         raise AssertionError('\n'.join(rows))
     assert_almost_equal(exp.ravel(), got.ravel(), decimal=3)
Exemplo n.º 4
0
def check_versions():
    """
    This endpoint validates software version using packaging.version lib,
    it supports with letters even. For more information please check
    https://www.python.org/dev/peps/pep-0440/
    """
    if request.json == None:
        abort(400, 'no parameters')

    if 'version_1' in request.json and 'version_2' in request.json:
        version_1 = request.json.get('version_1')
        version_2 = request.json.get('version_2')
    else:
        abort(400, 'missing parameters')

    # check versions format
    try:
        version.Version(version_1)
        version.Version(version_2)
        version_1 = version.parse(version_1)
        version_2 = version.parse(version_2)

    except InvalidVersion:
        abort(400, 'format error')

    comparison = compare(version_1, version_2)

    return jsonify(
        {'comparison': STRING_RESPONSE % (version_1, comparison, version_2)})
Exemplo n.º 5
0
def check_version(parsed_args: argparse.Namespace) -> None:
    info = get_version(parsed_args.master)

    master_version = info["master"]["version"]
    client_version = info["client"]["version"]
    if not master_version:
        print(
            termcolor.colored(
                "Master not found at {}. "
                "Hint: Remember to set the DET_MASTER environment variable "
                "to the correct Determined master IP or use the '-m' flag.".format(
                    parsed_args.master
                ),
                "yellow",
            )
        )
    elif version.Version(client_version) < version.Version(master_version):
        print(
            termcolor.colored(
                "CLI version {} is less than master version {}. "
                "Consider upgrading the CLI.".format(client_version, master_version),
                "yellow",
            )
        )
    elif version.Version(client_version) > version.Version(master_version):
        print(
            termcolor.colored(
                "Master version {} is less than CLI version {}. "
                "Consider upgrading the master.".format(master_version, client_version),
                "yellow",
            )
        )
Exemplo n.º 6
0
def main():
    """Main entry point."""
    if len(sys.argv) < 2:
        print("Usage {} <image_name>".format(sys.argv[0]))
        return
    url = "https://registry.hub.docker.com/v1/repositories/{}/tags".format(
        sys.argv[1])
    response = requests.get(url)
    response.raise_for_status()

    tags = json.loads(response.text)

    versions = []
    for item in tags:
        try:
            versions.append(version.parse(item["name"]))
        except version.InvalidVersion:
            pass

    versions.sort(reverse=True)
    if len(versions):
        latest_version = versions.pop(0)
    else:
        latest_version = version.Version("0.0.0")
    next_version = version.Version("{}.{}.{}".format(
        latest_version.release[0],
        latest_version.release[1],
        latest_version.release[2] + 1,
    ))
    print(next_version)
Exemplo n.º 7
0
def _ScalaVersion() -> version.Version:
    if _SparkVersion().major >= 3:
        # https://spark.apache.org/docs/3.0.0/#downloading
        return version.Version('2.12')
    else:
        # https://spark.apache.org/docs/2.4.0/#downloading
        return version.Version('2.11')
Exemplo n.º 8
0
def test_module_install_version(
    tmpdir: py.path.local,
    snippetcompiler_clean,
    modules_v2_dir: str,
    dev: bool,
) -> None:
    """
    Make sure that the module install results in a module instance with the appropriate version information.
    :param dev: whether to add a dev tag to the version
    """
    module_name: str = "minimalv2module"
    module_path: str = os.path.join(str(tmpdir), module_name)
    module_version: version.Version = version.Version(
        "1.2.3") if not dev else version.Version("1.2.3.dev0")

    # set up simple project and activate the snippetcompiler venv
    project: module.Project = snippetcompiler_clean.setup_for_snippet("")

    # install module
    module_from_template(
        os.path.join(modules_v2_dir, module_name),
        module_path,
        new_version=module_version,
    )
    os.chdir(project.path)
    ModuleTool().install(editable=True, path=module_path)

    # check version
    mod: module.Module = ModuleTool().get_module(module_name)
    assert mod.version == module_version
Exemplo n.º 9
0
class TestCustomTransformerTSNE(unittest.TestCase):
    @unittest.skipIf(pv.Version(ort_version) <= pv.Version("0.3.0"),
                     reason="TopK is failing.")
    def test_custom_pipeline_scaler(self):

        digits = datasets.load_digits(n_class=6)
        Xd = digits.data[:20]
        yd = digits.target[:20]
        n_samples, n_features = Xd.shape

        ptsne_knn = PredictableTSNE()
        ptsne_knn.fit(Xd, yd)

        update_registered_converter(
            PredictableTSNE,
            "CustomPredictableTSNE",
            predictable_tsne_shape_calculator,
            predictable_tsne_converter,
        )

        model_onnx = convert_sklearn(
            ptsne_knn,
            "predictable_tsne",
            [("input", FloatTensorType([None, Xd.shape[1]]))],
            target_opset=TARGET_OPSET)

        dump_data_and_model(Xd.astype(numpy.float32)[:7],
                            ptsne_knn,
                            model_onnx,
                            basename="CustomTransformerTSNEkNN-OneOffArray")

        trace_line = []

        def my_parser(scope, model, inputs, custom_parsers=None):
            trace_line.append(model)
            return _parse_sklearn_simple_model(scope, model, inputs,
                                               custom_parsers)

        model_onnx = convert_sklearn(
            ptsne_knn,
            "predictable_tsne",
            [("input", FloatTensorType([None, Xd.shape[1]]))],
            custom_parsers={PredictableTSNE: my_parser},
            target_opset=TARGET_OPSET)
        assert len(trace_line) == 1

        dump_data_and_model(
            Xd.astype(numpy.float32)[:7],
            ptsne_knn,
            model_onnx,
            basename="CustomTransformerTSNEkNNCustomParser-OneOffArray")

        update_registered_parser(PredictableTSNE, my_parser)
        model_onnx = convert_sklearn(
            ptsne_knn,
            "predictable_tsne",
            [("input", FloatTensorType([None, Xd.shape[1]]))],
            target_opset=TARGET_OPSET)

        assert len(trace_line) == 2
class TestSklearnTfidfVectorizerSparse(unittest.TestCase):
    @unittest.skipIf(
        TARGET_OPSET < 9,
        # issue with encoding
        reason="https://github.com/onnx/onnx/pull/1734")
    @unittest.skipIf(pv.Version(ort.__version__) <= pv.Version("0.2.1"),
                     reason="sparse not supported")
    def test_model_tfidf_transform_bug(self):
        categories = [
            "alt.atheism",
            "soc.religion.christian",
            "comp.graphics",
            "sci.med",
        ]
        twenty_train = fetch_20newsgroups(subset="train",
                                          categories=categories,
                                          shuffle=True,
                                          random_state=0)
        text_clf = Pipeline([("vect", CountVectorizer()),
                             ("tfidf", TfidfTransformer())])
        twenty_train.data[0] = "bruît " + twenty_train.data[0]
        text_clf.fit(twenty_train.data, twenty_train.target)
        model_onnx = convert_sklearn(text_clf,
                                     name="DocClassifierCV-Tfidf",
                                     initial_types=[("input",
                                                     StringTensorType([5]))],
                                     target_opset=TARGET_OPSET)
        dump_data_and_model(twenty_train.data[5:10],
                            text_clf,
                            model_onnx,
                            basename="SklearnPipelineTfidfTransformer")
Exemplo n.º 11
0
def test_active_env_check_constraints(caplog, tmpvenv_active: str) -> None:
    """
    Verify that the env.ActiveEnv.check() method's constraints parameter is taken into account as expected.
    """
    caplog.set_level(logging.WARNING)
    in_scope: Pattern[str] = re.compile("test-package-.*")
    constraints: List[Requirement] = [
        Requirement.parse("test-package-one~=1.0")
    ]

    def check_log(version: Optional[version.Version]) -> None:
        assert f"Incompatibility between constraint test-package-one~=1.0 and installed version {version}" in {
            rec.message
            for rec in caplog.records
        }

    assert env.ActiveEnv.check(in_scope)

    caplog.clear()
    assert not env.ActiveEnv.check(in_scope, constraints)
    check_log(None)

    caplog.clear()
    create_install_package("test-package-one", version.Version("1.0.0"), [])
    assert env.ActiveEnv.check(in_scope, constraints)

    caplog.clear()
    v: version.Version = version.Version("2.0.0")
    create_install_package("test-package-one", v, [])
    assert not env.ActiveEnv.check(in_scope, constraints)
    check_log(v)
Exemplo n.º 12
0
def is_version_compatible(
    provider_v_in: Union[str, v.Version],
    consumer_v_in: Union[str, v.Version],
    raise_exception: bool = True,
) -> bool:
    """
    Check provider supports consumer and throws exception if not

    Set the spec so that the consumer has to be within a micro/patch release of the provider
    NOTE - this isn't semver - breaks when have > v1 release as then treats minor as breaking,
    e.g. 2.2.5 is not compat with 2.1.5
    """
    consumer_v = v.Version(consumer_v_in) if isinstance(consumer_v_in, str) else consumer_v_in
    provider_v = v.Version(provider_v_in) if isinstance(provider_v_in, str) else provider_v_in

    provider_spec = SpecifierSet(f"~={provider_v.major}.{provider_v.minor}.0")

    log.debug(f"Provider spec {provider_spec}, Consumer version {consumer_v}")
    if consumer_v not in provider_spec:
        if raise_exception:
            raise VersionMismatch(
                f"Consumer ({consumer_v}) and Provider ({provider_spec}) API versions not compatible"
            )
        return False
    return True
Exemplo n.º 13
0
def test_active_env_check_basic(
    caplog,
    tmpdir: str,
    tmpvenv_active: str,
) -> None:
    """
    Verify that the env.ActiveEnv.check() method detects all possible forms of incompatibilities within the environment.
    """
    caplog.set_level(logging.WARNING)

    in_scope_test: Pattern[str] = re.compile("test-package-.*")
    in_scope_nonext: Pattern[str] = re.compile("nonexistant-package")

    def assert_all_checks(expect_test: Tuple[bool, str] = (True, ""),
                          expect_nonext: Tuple[bool,
                                               str] = (True, "")) -> None:
        for in_scope, expect in [(in_scope_test, expect_test),
                                 (in_scope_nonext, expect_nonext)]:
            caplog.clear()
            assert env.ActiveEnv.check(in_scope) == expect[0]
            if not expect[0]:
                assert expect[1] in {rec.message for rec in caplog.records}

    assert_all_checks()
    create_install_package("test-package-one", version.Version("1.0.0"), [])
    assert_all_checks()
    create_install_package("test-package-two", version.Version("1.0.0"),
                           [Requirement.parse("test-package-one~=1.0")])
    assert_all_checks()
    create_install_package("test-package-one", version.Version("2.0.0"), [])
    assert_all_checks(expect_test=(
        False,
        "Incompatibility between constraint test-package-one~=1.0 and installed version 2.0.0"
    ))
Exemplo n.º 14
0
 def check_ovs_and_skip(test):
     ovs = ovs_lib.BaseOVS()
     current_ovs_version = version.Version(ovs.config['ovs_version'])
     if current_ovs_version < version.Version(ovs_version):
         test.skipTest("This test requires OVS version %s or higher." %
                       ovs_version)
     return f(test)
Exemplo n.º 15
0
 def test_ovr_classification_decision_function_binary(self):
     model, X = fit_classification_model(
         OneVsRestClassifier(LogisticRegression()), 2)
     options = {id(model): {'raw_scores': True}}
     model_onnx = convert_sklearn(
         model,
         "ovr classification",
         [("input", FloatTensorType([None, X.shape[1]]))],
         options=options,
         target_opset=TARGET_OPSET)
     self.assertIsNotNone(model_onnx)
     dump_data_and_model(
         X,
         model,
         model_onnx,
         basename="SklearnOVRClassificationDecisionFunctionBinary",
         methods=['predict', 'decision_function_binary'])
     if pv.Version(ort_version) < pv.Version("1.0.0"):
         return
     options = {id(model): {'raw_scores': True, 'zipmap': False}}
     model_onnx = convert_sklearn(
         model,
         "ovr classification",
         [("input", FloatTensorType([None, X.shape[1]]))],
         options=options,
         target_opset=TARGET_OPSET)
     sess = InferenceSession(model_onnx.SerializeToString())
     got = sess.run(None, {'input': X})[1]
     dec = model.decision_function(X)
     assert_almost_equal(got[:, 1], dec, decimal=4)
     assert_almost_equal(-got[:, 0], dec, decimal=4)
Exemplo n.º 16
0
    def test_auto_updates_pip(self):
        """Installing a plugin should automatically update pip if it's
        out-of-date.
        """
        # arrange
        old_pip_version = "20.0.1"
        assert (subprocess.run([
            disthelpers.get_pip_path(),
            "install",
            "-U",
            f"pip=={old_pip_version}",
        ]).returncode == 0)
        assert version.Version(
            get_pkg_version("pip")) == version.Version(old_pip_version)

        plugin_name = "junit4"
        plugin_version = "v1.0.0"
        cmd = [
            *pluginmanager.plugin_category.install.as_name_tuple(),
            "--plugin-spec",
            f"{plugin_name}{pluginmanager.PLUGIN_SPEC_SEP}{plugin_version}",
        ]

        # act
        repobee.run(cmd)

        # assert
        assert version.Version(
            get_pkg_version("pip")) > version.Version(old_pip_version)
Exemplo n.º 17
0
  def __init__(self, cache_dir: str, split: str):

    with tf.io.gfile.GFile(utils.get_cached_info_path(cache_dir, split)) as f:
      split_info = json.load(f)
      features = split_info["features"]

    with tf.io.gfile.GFile(utils.get_cached_stats_path(cache_dir, split)) as f:
      stats = json.load(f)

    version_when_cached = version.Version(
        split_info.get("seqio_version", "0.pre"))
    version_with_true_dtypes = version.Version("0.0.0")
    if version_when_cached < version_with_true_dtypes:
      # Assume that all int64 features are really int32.
      for name, feat in features.items():
        if feat["dtype"] == "int64":
          logging.info("Casting cached '%s' to int32.", name)
          feat["dtype"] = "int32"

    # Use `FixedLenSequenceFeature` for sequences with variable length.
    def _feature_config(shape, dtype):
      if dtype in ("int32", "bool"):
        # int32 and bool are stored as int64 in the tf.train.Example protobuf.
        # TODO(adarob): Support other conversions.
        dtype = "int64"
      if shape and shape[0] is None:
        return tf.io.FixedLenSequenceFeature(
            shape[1:], dtype, allow_missing=True)
      return tf.io.FixedLenFeature(shape, dtype)

    feature_description = {
        feat: _feature_config(**desc) for feat, desc in features.items()
    }

    def read_file_fn(filepattern):
      ds = tf.data.TFRecordDataset(filepattern)
      ds = ds.map(
          lambda pb: tf.io.parse_single_example(pb, feature_description),
          num_parallel_calls=tf.data.experimental.AUTOTUNE)
      # Cast features back to the types from the info JSON since some features
      # must be cast for storage (e.g., in32 is stored as int64).
      ds = ds.map(
          lambda x: {k: tf.cast(v, features[k]["dtype"]) for k, v in x.items()},
          num_parallel_calls=tf.data.experimental.AUTOTUNE)
      # Legacy cached datasets may use old "_plaintext" suffix. Rename to
      # "_pretokenized".
      ds = _rename_plaintext_to_pretokenized(ds)
      return ds

    split_to_filepattern = {
        split: "%s-*-of-*%d" % (
            utils.get_cached_tfrecord_prefix(cache_dir, split),
            split_info["num_shards"])
    }

    super().__init__(
        read_file_fn=read_file_fn,
        split_to_filepattern=split_to_filepattern,
        num_input_examples={split: stats["examples"]}
    )
    def test_documentation_examples(self):

        this = os.path.abspath(os.path.dirname(__file__))
        fold = os.path.normpath(os.path.join(this, '..', 'examples'))
        found = os.listdir(fold)
        tested = 0
        for name in found:
            if name.startswith("plot_") and name.endswith(".py"):
                if (name == "plot_pipeline_lightgbm.py" and pv.Version(
                        onnxruntime.__version__) < pv.Version('1.0.0')):
                    continue
                print("run %r" % name)
                try:
                    mod = import_source(fold, os.path.splitext(name)[0])
                    assert mod is not None
                except FileNotFoundError:
                    # try another way
                    cmds = [sys.executable, "-u", os.path.join(fold, name)]
                    p = subprocess.Popen(cmds,
                                         stdout=subprocess.PIPE,
                                         stderr=subprocess.PIPE)
                    res = p.communicate()
                    out, err = res
                    st = err.decode('ascii', errors='ignore')
                    if len(st) > 0 and 'Traceback' in st:
                        if "No such file or directory: 'dot'" in st:
                            # dot not installed, this part
                            # is tested in onnx framework
                            pass
                        elif '"dot" not found in path.' in st:
                            # dot not installed, this part
                            # is tested in onnx framework
                            pass
                        elif ('Please fix either the inputs or '
                              'the model.') in st:
                            # onnxruntime datasets changed in master branch,
                            # still the same in released version on pypi
                            pass
                        elif ('Current official support for domain ai.onnx '
                              'is till opset 12.') in st:
                            # one example is using opset 13 but onnxruntime
                            # only support up to opset 12.
                            pass
                        elif "'str' object has no attribute 'decode'" in st:
                            # unstable bug in scikit-learn<0.24
                            pass
                        else:
                            raise RuntimeError(
                                "Example '{}' (cmd: {} - exec_prefix='{}') "
                                "failed due to\n{}"
                                "".format(name, cmds, sys.exec_prefix, st))
                tested += 1
        if tested == 0:
            raise RuntimeError("No example was tested.")
Exemplo n.º 19
0
def skl2onnx_convert_lightgbm(scope, operator, container):
    options = scope.get_options(operator.raw_operator)
    if 'split' in options:
        if pv.Version(oml_version) < pv.Version('1.9.2'):
            warnings.warn(
                "Option split was released in version 1.9.2 but %s is "
                "installed. It will be ignored." % oml_version)
        operator.split = options['split']
    else:
        operator.split = None
    convert_lightgbm(scope, operator, container)
def test_transformer_creation_without_optional_args(
    name_from_base,
    create_model,
    sagemaker_session,
    tensorflow_inference_version,
    tensorflow_inference_py_version,
):
    if version.Version(tensorflow_inference_version) < version.Version("1.11"):
        pytest.skip(
            "Legacy TF version requires explicit image URI, and "
            "this logic is tested in test_create_model_with_custom_image.")

    model_name = "generated-model-name"
    name_from_base.return_value = model_name

    model = Mock()
    create_model.return_value = model

    base_job_name = "tensorflow"
    tf = TensorFlow(
        entry_point=SCRIPT_PATH,
        framework_version=tensorflow_inference_version,
        py_version=tensorflow_inference_py_version,
        role=ROLE,
        sagemaker_session=sagemaker_session,
        instance_count=INSTANCE_COUNT,
        instance_type=INSTANCE_TYPE,
        base_job_name=base_job_name,
    )
    tf.latest_training_job = _TrainingJob(sagemaker_session, "some-job-name")
    tf.transformer(INSTANCE_COUNT, INSTANCE_TYPE)

    name_from_base.assert_called_with(base_job_name)
    create_model.assert_called_with(
        role=ROLE,
        vpc_config_override="VPC_CONFIG_DEFAULT",
        entry_point=None,
        enable_network_isolation=False,
        name=model_name,
    )
    model.transformer.assert_called_with(
        INSTANCE_COUNT,
        INSTANCE_TYPE,
        accept=None,
        assemble_with=None,
        env=None,
        max_concurrent_transforms=None,
        max_payload=None,
        output_kms_key=None,
        output_path=None,
        strategy=None,
        tags=None,
        volume_kms_key=None,
    )
Exemplo n.º 21
0
    def forward(self, hidden_states, residual_states, input_mask):
        out = self.conv(hidden_states.permute(0,2,1).contiguous()).permute(0,2,1).contiguous()
        if version.Version(torch.__version__) >= version.Version('1.2.0a'):
            rmask = (1-input_mask).bool()
        else:
            rmask = (1-input_mask).byte()
        out.masked_fill_(rmask.unsqueeze(-1).expand(out.size()), 0)
        out = ACT2FN[self.conv_act](self.dropout(out))
        output_states = MaskedLayerNorm(self.LayerNorm, residual_states + out, input_mask)

        return output_states
def test_enable_sm_metrics_if_fw_ver_is_at_least_1_15(
        sagemaker_session, tensorflow_training_version,
        tensorflow_training_py_version):
    if version.Version(tensorflow_training_version) < version.Version("1.15"):
        pytest.skip("This test is for TF 1.15 and higher.")

    tf = _build_tf(
        sagemaker_session,
        framework_version=tensorflow_training_version,
        py_version=tensorflow_training_py_version,
    )
    assert tf.enable_sagemaker_metrics
Exemplo n.º 23
0
    def forward(self, input, mask, dim):
        self.dim = dim
        if version.Version(torch.__version__) >= version.Version("1.2.0a"):
            rmask = ~(mask.bool())
        else:
            rmask = (1 - mask).byte()  # This line is not supported by Onnx tracing.

        output = input.masked_fill(rmask, float("-inf"))
        output = torch.softmax(output, self.dim)
        output.masked_fill_(rmask, 0)
        self.save_for_backward(output)
        return output
Exemplo n.º 24
0
    def forward(self, input, mask, dim):
        self.dim = dim
        if version.Version(torch.__version__) >= version.Version('1.2.0a'):
            rmask = (1 - mask).bool()
        else:
            rmask = (1 - mask).byte()

        output = input.masked_fill(rmask, float('-inf'))
        output = torch.softmax(output, self.dim)
        output.masked_fill_(rmask, 0)
        self.save_for_backward(output)
        return output
Exemplo n.º 25
0
class TestSklearnArrayFeatureExtractor(unittest.TestCase):
    @unittest.skipIf(ColumnTransformer is None
                     or pv.Version(ort_version) <= pv.Version("0.4.0"),
                     reason="onnxruntime too old")
    def test_array_feature_extractor(self):
        data_to_cluster = pd.DataFrame([[1, 2, 3.5, 4.5], [1, 2, 1.7, 4.0],
                                        [2, 4, 2.4, 4.3], [2, 4, 2.5, 4.0]],
                                       columns=[1, 2, 3, 4])
        cat_attributes_clustering = [1, 2]
        num_attributes_clustering = [3, 4]  # this is of length 12 in reality
        gmm = GaussianMixture(n_components=2, random_state=1)
        ohe_cat = [
            OneHotEncoder(categories='auto', sparse=False, drop=None)
            for i in cat_attributes_clustering
        ]
        ct_cat = ColumnTransformer(
            [("oneHotEncoder" + str(i), ohe_cat[i], [i])
             for i, item in enumerate(cat_attributes_clustering)],
            remainder='passthrough')
        onehotencoding_pipeline = Pipeline([
            ("columnTransformer", ct_cat),
        ])
        clustering_pipeline = Pipeline([('onehotencoder_and_scaler',
                                         onehotencoding_pipeline),
                                        ('clustering', gmm)])
        clustering_pipeline.fit(X=data_to_cluster)
        initial_type = [
            ('float_input',
             FloatTensorType([
                 None,
                 len([*cat_attributes_clustering, *num_attributes_clustering])
             ]))
        ]
        data = data_to_cluster.values.astype(np.float32)

        # checks the first step
        model_onnx = to_onnx(clustering_pipeline.steps[0][1],
                             initial_types=initial_type,
                             target_opset=TARGET_OPSET)
        dump_data_and_model(data,
                            clustering_pipeline.steps[0][1],
                            model_onnx,
                            basename="SklearnArrayFeatureExtractorStep0")

        # checks the whole pipeline
        model_onnx = to_onnx(clustering_pipeline,
                             initial_types=initial_type,
                             target_opset=TARGET_OPSET)
        dump_data_and_model(data,
                            clustering_pipeline,
                            model_onnx,
                            basename="SklearnArrayFeatureExtractor")
def test_disable_sm_metrics_if_fw_ver_is_less_than_1_15(
        sagemaker_session, tensorflow_training_version,
        tensorflow_training_py_version):
    if version.Version(tensorflow_training_version) > version.Version("1.14"):
        pytest.skip("This test is for TF 1.14 and lower.")

    tf = _build_tf(
        sagemaker_session,
        framework_version=tensorflow_training_version,
        py_version=tensorflow_training_py_version,
        image_uri="old-image",
    )
    assert tf.enable_sagemaker_metrics is None
Exemplo n.º 27
0
    def __lt__(self, other: Any) -> bool:
        if not isinstance(other, Version):
            raise TypeError("Cannot compare Version with type {}".format(
                other.__class__.__qualname__))

        import packaging.version as pv

        return (pv.Version(self.base) < pv.Version(other.base)
                and _blank(self.stage, "") < _blank(other.stage, "")
                and _blank(self.revision, 0) < _blank(other.revision, 0)
                and _blank(self.distance, 0) < _blank(other.distance, 0)
                and _blank(self.commit, "") < _blank(other.commit, "")
                and bool(self.dirty) < bool(other.dirty))
Exemplo n.º 28
0
def check_lib_version(lib_name: str, checked_version: str,
                      operator) -> (Optional[bool], str):
    """
    Checks if a library is installed, and if it is, checks the operator(lib.__version__, checked_version) as a result.
    This bool result along with a string analysis of result is returned.

    If the library is not installed at all, then returns None instead, along with a string explaining
    that the library is not installed

    Args:
        lib_name: lower case str name of the library that must be imported.
        checked_version: semver string that is compared against lib.__version__.
        operator: binary callable function func(a, b) -> bool; that compares lib.__version__ against version in
            some manner. Must return a boolean.

    Returns:
        A tuple of results:
        -   Bool or None. Bool if the library could be imported, and the result of
            operator(lib.__version__, checked_version) or False if __version__ is not implemented in lib.
            None is passed if the library is not installed at all.
        -   A string analysis of the check.
    """
    try:
        if '.' in lib_name:
            mod = import_class_by_path(lib_name)
        else:
            mod = __import__(lib_name)

        if hasattr(mod, '__version__'):
            lib_ver = version.Version(mod.__version__)
            match_ver = version.Version(checked_version)

            if operator(lib_ver, match_ver):
                msg = f"Lib {lib_name} version is satisfied !"
                return True, msg
            else:
                msg = (
                    f"Lib {lib_name} version ({lib_ver}) is not {operator.__name__} than required version {checked_version}.\n"
                    f"Please upgrade the lib using either pip or conda to the latest version."
                )
                return False, msg
        else:
            msg = (
                f"Lib {lib_name} does not implement __version__ in its init file. "
                f"Could not check version compatibility.")
            return False, msg
    except (ImportError, ModuleNotFoundError):
        pass

    msg = f"Lib {lib_name} has not been installed. Please use pip or conda to install this package."
    return None, msg
Exemplo n.º 29
0
class TestOnnxOperatorsSparse(unittest.TestCase):

    @unittest.skipIf(TARGET_OPSET < 11,
                     reason="only available for opset >= 11")
    @unittest.skipIf(pv.Version(ort_version) < pv.Version(THRESHOLD),
                     reason="fails with onnxruntime < %s" % THRESHOLD)
    def test_onnx_init_dense(self):
        X = np.array([1, 2, 3, 4, 5, 6]).astype(np.float32).reshape((3, 2))

        node = OnnxAdd('X', X, output_names=['Y'], op_version=TARGET_OPSET)

        model_def = node.to_onnx({'X': X},
                                 outputs=[('Y', FloatTensorType())])

        sess = InferenceSession(model_def.SerializeToString())
        res = sess.run(None, {'X': X})[0]

        assert_almost_equal(X + X, res)

    @unittest.skipIf(TARGET_OPSET < 11,
                     reason="only available for opset >= 11")
    @unittest.skipIf(pv.Version(ort_version) < pv.Version(THRESHOLD),
                     reason="fails with onnxruntime < %s" % THRESHOLD)
    def test_onnx_init_sparse_coo(self):
        row = np.array([0, 0, 1, 3, 1], dtype=np.float32)
        col = np.array([0, 2, 1, 3, 1], dtype=np.float32)
        data = np.array([1, 1, 1, 1, 1], dtype=np.float32)
        X = coo_matrix((data, (row, col)), shape=(4, 4))

        node = OnnxAdd(
            'X', X, output_names=['Y'],
            op_version=TARGET_OPSET)

        model_def = node.to_onnx(
            {'X': X}, outputs=[('Y', FloatTensorType())])

        try:
            sess = InferenceSession(model_def.SerializeToString())
        except (RuntimeError, OrtInvalidArgument):
            # Sparse tensor is not supported for constant.
            return
        try:
            res = sess.run(None, {'X': X})[0]
        except RuntimeError as e:
            # Sparse tensor is not supported for constant.
            warnings.warn(
                "Unable to run with %r\n---\n%s\n%s" % (
                    {'X': X}, model_def, e))
            return
        assert_almost_equal(X + X, res)
    def test_onnx_example_pdist(self):
        x = np.array([1, 2, 4, 5, 5, 4]).astype(np.float32).reshape((3, 2))

        opv = _TARGET_OPSET_
        diff = OnnxSub('next_in', 'next', output_names=['diff'],
                       op_version=opv)
        id_next = OnnxIdentity(
            'next_in', output_names=['next_out'],
            op_version=opv)
        norm = OnnxReduceSumSquare(
            diff, output_names=['norm'], axes=[1],
            op_version=opv)
        flat = OnnxSqueezeApi11(
            norm, output_names=['scan_out'], axes=[1],
            op_version=opv)
        scan_body = id_next.to_onnx(
            OrderedDict([('next_in', x), ('next', FloatTensorType())]),
            outputs=[('next_out', FloatTensorType([3, 2])),
                     ('scan_out', FloatTensorType([3]))],
            other_outputs=[flat],
            target_opset=opv)

        sess = InferenceSession(scan_body.SerializeToString())
        res = sess.run(None, {'next_in': x, 'next': x[:1]})
        assert_almost_equal(x, res[0])
        exp = np.array([0., 18., 20.], dtype=np.float32)
        assert_almost_equal(exp, res[1])

        node = OnnxScan(
            'x', 'x', output_names=['y', 'z'],
            num_scan_inputs=1, body=scan_body.graph,
            op_version=opv)
        model_def = node.to_onnx({'x': x},
                                 outputs=[('y', FloatTensorType([3, 2])),
                                          ('z', FloatTensorType([3, 3]))])
        try:
            onnx.checker.check_model(model_def)
        except ValidationError as e:
            if pv.Version(onnx__version__) <= pv.Version("1.5.0"):
                warnings.warn(e)
            else:
                raise e

        sess = InferenceSession(model_def.SerializeToString())
        res = sess.run(None, {'x': x})

        exp = squareform(pdist(x, metric="sqeuclidean"))
        assert_almost_equal(x, res[0])
        assert_almost_equal(exp, res[1])