def __init__(self, model):
self.model = model
self.session = whylogs.get_or_create_session("/opt/ml/model/" +
WHYLOGS_YML)
self.ylog = self.create_logger()
self.last_upload_time = datetime.datetime.utcnow()
atexit.register(self.ylog.close)
def log_session(dataset_name, session_data):
session = get_or_create_session()
df = pd.DataFrame(session_data)
df["timestamp"] = pd.to_datetime(df["timestamp"], unit="ms")
df_minutes = df.groupby(pd.Grouper(key="timestamp", freq="min"))
for minute_batch, batch in df_minutes:
with session.logger(dataset_name=dataset_name,
dataset_timestamp=minute_batch) as logger:
logger.log_dataframe(batch)
def __generate_reference_profile():
session = get_or_create_session()
with session.logger("mytestytest",
dataset_timestamp=datetime.datetime(2021, 6,
2)) as logger:
for _ in range(5):
logger.log({"uniform_integers": np.random.randint(0, 50)})
logger.log({"nulls": None})
return logger.profile
def __generate_categorical_reference_profile():
session = get_or_create_session()
credit_cards = pd.DataFrame([
{
"credit_card": "6011 1111 1111 1117"
},
{
"credit_card": "6011-0009-9013-9424"
},
{
"credit_card": "3530 1113 3330 0000"
},
{
"credit_card": "3566-0020-2036-0505"
},
])
return session.log_dataframe(credit_cards, "test.data")
def __generate_categorical_target_profile():
session = get_or_create_session()
credit_cards = pd.DataFrame([
{
"credit_card": "3714-496353-98431"
},
{
"credit_card": "3787 344936 71000"
},
{
"credit_card": "3056 930902 5904"
},
{
"credit_card": "3065 133242 2899"
},
])
return session.log_dataframe(credit_cards, "test.data")
from flask import Flask, jsonify
from flask_cors import CORS
from joblib import load
from utils import MessageException, message_exception_handler
from whylogs import get_or_create_session
# Load environment variables
load_dotenv()
# Initialize Dataset
df = pd.read_csv(os.environ["DATASET_URL"])
# Load model with joblib
model = load(os.environ["MODEL_PATH"])
whylabs_session = get_or_create_session(os.environ["WHYLABS_CONFIG"])
whylabs_logger = None
def create_app(config_object="settings"):
"""Create application factory, as explained here: http://flask.pocoo.org/docs/patterns/appfactories/.
:param config_object: The configuration object to use.
"""
app = Flask(__name__.split(".")[0])
# Adding CORS
CORS(app)
# Adding Logging
import pandas as pd
import time
from whylogs.core.datasetprofile import dataframe_profile
from whylogs import get_or_create_session
if __name__ == "__main__":
df = pd.read_csv("data/lending-club-accepted-10.csv")
print(df.head())
session = get_or_create_session()
with session.logger("segment",
segments=[[{
"key": "home_ownership",
"value": "RENT"
}], [{
"key": "home_ownership",
"value": "MORTGAGE"
}]],
cache=1) as logger:
print(session.get_config())
logger.log_dataframe(df)
profile_seg = logger.segemented_profiles
with session.logger("my_rotated_seg",
segments=["home_ownership"],
with_rotation_time="s",
cache=1) as logger:
print(session.get_config())
logger.log_dataframe(df)
time.sleep(2)
logger.log_dataframe(df)
def __init__(self, session=None):
logger.debug("Creating a real session for WhyLogsRun")
self._session = session if session else get_or_create_session()
import boto3
import botocore
import flask
import my_model as my_model_code
import numpy as np
import torch
import torchvision.transforms as transforms
from PIL import Image
from whylogs import get_or_create_session
prefix = "/opt/ml/"
# location of you checkpoint in sagemaker container
model_path = os.path.join(prefix, "model")
whylogs_session = get_or_create_session()
logger = whylogs_session.logger(dataset_name="my_deployed_model",
dataset_timestamp=datetime.datetime.now(
datetime.timezone.utc),
with_rotation_time="5m")
# simple custom metric for embedding
embedding_centr = np.linalg.norm(np.ones(1024))
# loads the model into memory from disk and returns it
def model_fn(model_dir):
model_config = {
"backbone": "resnet",
"outlayer": "C5",
