Пример #1
0
def _post_model(files, push_api, auth_api, tries, max_tries, extra_headers,
                options) -> Optional[str]:
    '''Attempts to post the model to Acumos, returns the docker image URI'''
    headers = {
        'Authorization': get_jwt(auth_api),
        'isCreateMicroservice':
        'true' if options.create_microservice else 'false',
        'deploy': 'true' if options.deploy else 'false'
    }

    if extra_headers is not None:
        headers.update(extra_headers)

    resp = requests.post(push_api, files=files, headers=headers)
    if resp.ok:
        logger.info("Model pushed successfully to {}".format(push_api))
        if options.create_microservice:
            try:
                docker_image_uri = resp.json()["dockerImageUri"]
                logger.info(
                    f"Acumos model docker image successfully created: {docker_image_uri}"
                )
                return docker_image_uri
            except KeyError:
                logger.warning(
                    "Docker image URI could not be found in server response, "
                    "on-boarding server is probably running a version prior to Demeter."
                )
    else:
        clear_jwt()
        if resp.status_code == 401 and tries != max_tries:
            logger.warning(
                'Model push failed due to an authorization failure. Clearing credentials and trying again'
            )
            _post_model(files, push_api, auth_api, tries + 1, max_tries,
                        extra_headers, options)
        elif resp.status_code == 500 and tries != max_tries:
            with ExitStack() as stack:
                print(
                    "\x1b[31m Warning : Status code  500 received, Trying with the 0.4.0 Clio schema\x1b[39m \n"
                )
                dump_dir = str(files['model'][1]).split('name=')[1].split(
                    'model.zip')[0].split('\'')[1]
                meta_clio = stack.enter_context(
                    open(path_join(dump_dir, 'metadata_clio.json')))
                model = stack.enter_context(
                    open(path_join(dump_dir, 'model.zip'), 'rb'))
                proto = stack.enter_context(
                    open(path_join(dump_dir, 'model.proto')))
                files_Clio = {
                    'model': ('model.zip', model, 'application/zip'),
                    'metadata':
                    ('metadata.json', meta_clio, 'application/json'),
                    'schema': ('model.proto', proto, 'text/plain')
                }
                _post_model(files_Clio, push_api, auth_api, tries + 1,
                            max_tries, extra_headers, options)
        else:
            raise AcumosError("Model push failed: {}".format(
                _ServerResponse(resp.status_code, resp.reason, resp.text)))
def test_session_push_sklearn():
    '''Tests basic model pushing functionality with sklearn'''
    clear_jwt()

    with _patch_auth():
        with MockServer() as server:
            iris = load_iris()
            X = iris.data
            y = iris.target

            clf = RandomForestClassifier(random_state=0)
            clf.fit(X, y)

            columns = [
                'sepallength', 'sepalwidth', 'petallength', 'petalwidth'
            ]
            X_df = pd.DataFrame(X, columns=columns)

            DataFrame = create_dataframe('DataFrame', X_df)
            Predictions = create_namedtuple('Predictions',
                                            [('predictions', List[int])])

            def predict(df: DataFrame) -> Predictions:
                '''Predicts the class of iris'''
                X = np.column_stack(df)
                yhat = clf.predict(X)
                preds = Predictions(predictions=yhat)
                return preds

            model = Model(predict=predict)

            model_url, auth_url, _, _ = server.config
            s = AcumosSession(model_url, auth_url)
            s.push(model, name='sklearn_iris_push')
Пример #3
0
def _patch_auth(clear_token=True):
    '''Convenience CM to patch interactive prompts used for authentication'''
    if clear_token:
        clear_jwt()

    with mock.patch('acumos.auth.gettoken', lambda x: _FAKE_TOKEN):
        yield
def test_auth_envvar():
    '''Tests that environment variables can be used in place of interactive input'''
    clear_jwt()
    with _patch_environ(**{
            _USERNAME_VAR: _FAKE_USERNAME,
            _PASSWORD_VAR: _FAKE_PASSWORD
    }):
        _push_dummy_model()

    clear_jwt()
    with _patch_environ(**{_TOKEN_VAR: _FAKE_TOKEN}):
        _push_dummy_model()
def test_extra_header():
    '''Tests that extra headers are correctly sent to the onboarding server'''
    clear_jwt()
    # in the mock onboarding server, this extra test header acts as another auth header
    with _patch_environ(**{_TOKEN_VAR: 'wrongtoken'}):
        extra_headers = {'X-Test-Header': _FAKE_TOKEN}
        _push_dummy_model(extra_headers)

    clear_jwt()
    with pytest.raises(AcumosError):
        with _patch_environ(**{_TOKEN_VAR: 'wrongtoken'}):
            extra_headers = {'X-Test-Header': 'wrongtoken'}
            _push_dummy_model(extra_headers)
Пример #6
0
def test_session_push():
    '''Tests various session push scenarios'''
    # allow users to push using username and password env vars
    clear_jwt()
    with _patch_environ(**{
            _USERNAME_VAR: _FAKE_USERNAME,
            _PASSWORD_VAR: _FAKE_PASSWORD
    }):
        _push_dummy_model(use_auth_url=True)

    # verify auth url is required for username and password auth
    clear_jwt()
    with _patch_environ(**{
            _USERNAME_VAR: _FAKE_USERNAME,
            _PASSWORD_VAR: _FAKE_PASSWORD
    }):
        with pytest.raises(AcumosError):
            _push_dummy_model(use_auth_url=False)

    # allow users to push using a token env var
    clear_jwt()
    with _patch_environ(**{_TOKEN_VAR: _FAKE_TOKEN}):
        _push_dummy_model()

    # allow users to push by providing a token via interactive prompt
    clear_jwt()
    with _patch_auth():
        _push_dummy_model()

    # verify we can recover from a stale token
    _set_jwt('invalidtoken')
    with _patch_auth():
        _push_dummy_model()
Пример #7
0
def _post_model(files, push_api, auth_api, tries, max_tries, extra_headers):
    '''Attempts to post the model to Acumos'''
    headers = {'Authorization': get_jwt(auth_api)}
    if extra_headers is not None:
        headers.update(extra_headers)

    r = requests.post(push_api, files=files, headers=headers)

    if r.status_code == 201:
        logger.info("Model pushed successfully to {}".format(push_api))
    else:
        clear_jwt()
        if r.status_code == 401 and tries != max_tries:
            logger.warning('Model push failed due to an authorization failure. Clearing credentials and trying again')
            _post_model(files, push_api, auth_api, tries + 1, max_tries, extra_headers)
        else:
            raise AcumosError("Model push failed: {}".format(_ServerResponse(r.status_code, r.reason, r.text)))
def test_session_push_keras():
    '''Tests basic model pushing functionality with keras'''
    clear_jwt()

    with _patch_auth():
        with MockServer() as server:
            iris = load_iris()
            X = iris.data
            y = pd.get_dummies(iris.target).values

            clf = Sequential()
            clf.add(Dense(3, input_dim=4, activation='relu'))
            clf.add(Dense(3, activation='softmax'))
            clf.compile(loss='categorical_crossentropy',
                        optimizer='adam',
                        metrics=['accuracy'])
            clf.fit(X, y)

            columns = [
                'sepallength', 'sepalwidth', 'petallength', 'petalwidth'
            ]
            X_df = pd.DataFrame(X, columns=columns)

            DataFrame = create_dataframe('DataFrame', X_df)
            Predictions = create_namedtuple('Predictions',
                                            [('predictions', List[int])])

            def predict(df: DataFrame) -> Predictions:
                '''Predicts the class of iris'''
                X = np.column_stack(df)
                yhat = clf.predict(X)
                preds = Predictions(predictions=yhat)
                return preds

            model = Model(predict=predict)

            model_url, auth_url, _, _ = server.config
            s = AcumosSession(model_url, auth_url)
            s.push(model, name='keras_iris_push')