def __init__(self, *args, **kwargs):
self.project_name = os.environ.get('PROJECT_NAME')
self.deployment_name = os.environ.get('DEPLOYMENT_NAME')
# Initialize bitcoin wallet
xpub_wif = os.environ.get('HDKEY_XPUB_WIF', None)
if xpub_wif is None:
raise NoXPUBFoud
try:
xpub = HDKey(xpub_wif)
self.wallet = Wallet.create("Wallet", keys=xpub, network='bitcoin', witness_type='segwit')
except Exception as e:
raise e
# Initialize Ubiops Client
ubiops_api_token = os.environ.get('UBIOPS_API_TOKEN', None)
internal_ubiops_api_host = os.environ.get('INT_API_URL')
ubiops_conf = ubiops.Configuration(
host=internal_ubiops_api_host,
api_key={'Authorization': 'Token {}'.format(ubiops_api_token)},
)
client = ubiops.ApiClient(ubiops_conf)
self.ubiops_api = ubiops.api.CoreApi(client)
environment_variables = self.ubiops_api.deployment_environment_variables_list(self.project_name, self.deployment_name)
self.last_used_path_index = os.environ.get('LAST_USED_PATH_INDEX', '0/0')
self.last_used_path_index_env = None
for env in environment_variables:
if env.name == 'LAST_USED_PATH_INDEX':
self.last_used_path_index_env = env
print("Wallet initialized successfuly.")
def init_client():
config_access_token = Config().get('auth.tmp_access_token')
config_service_token = Config().get('auth.service_token')
config_api = Config().get('auth.api')
try:
configuration = api.Configuration()
configuration.host = config_api
if config_access_token and len(config_access_token) > 0:
configuration.api_key_prefix['Authorization'] = 'Bearer'
configuration.api_key['Authorization'] = config_access_token
elif config_service_token and len(config_service_token) > 0:
configuration.api_key_prefix['Authorization'] = ''
configuration.api_key['Authorization'] = config_service_token
else:
raise Exception("No access or service token found.")
client = api.ApiClient(configuration)
client.user_agent = "UbiOps/cli/%s" % VERSION
core_api = api.CoreApi(client)
assert core_api.service_status().status == 'ok'
return core_api
except Exception:
raise UnAuthorizedException("Unauthorized. Please, use 'ubiops signin' first.")
def __init__(self, base_directory, context):
"""
Initialisation method for the deployment. It can for example be used for loading modules that have to be kept in
memory or setting up connections. Load your external model files (such as pickles or .h5 files) here.
:param str base_directory: absolute path to the directory where the deployment.py file is located
:param dict context: a dictionary containing details of the deployment that might be useful in your code.
It contains the following keys:
- deployment (str): name of the deployment
- version (str): name of the version
- input_type (str): deployment input type, either 'structured' or 'plain'
- output_type (str): deployment output type, either 'structured' or 'plain'
- language (str): programming language the deployment is running
- environment_variables (str): the custom environment variables configured for the deployment.
You can also access those as normal environment variables via os.environ
"""
# Setup a api config
configuration = ubiops.Configuration()
# Configure API key authorization using environment variables
# https://ubiops.com/docs/deployments/environment-variables/
configuration.api_key['Authorization'] = os.environ['YOUR_API_KEY']
configuration.api_key_prefix['Authorization'] = ''
# Defining host is optional and default to https://api.ubiops.com/v2.1
configuration.host = "https://api.ubiops.com/v2.1"
# Enter a context with an instance of the API client
api_client = ubiops.ApiClient(configuration)
# Create an instance of the API class
self.api_instance = ubiops.CoreApi(api_client)
print("Initialising blob storage Deployment")
def __init__(self, project_name: str, deployment_name: str,
deployment_version: str, input_spec: dict, output_spec: dict,
api_key: str, api_host: str):
super().__init__(project_name=project_name,
deployment_name=deployment_name,
deployment_version=deployment_version,
input_spec=input_spec,
output_spec=output_spec,
api_key=api_key,
api_host=api_host)
self.url_fmt = '{host}/projects/{proj}/'
self.req_fmt = 'deployments/{dname}/versions/{dver}/request'
self.req_url = os.path.join(
self.url_fmt.format(host=self.api_host, proj=self.project_name),
self.req_fmt.format(dname=self.deployment_name,
dver=self.deployment_version))
self.client = ubiops.ApiClient(
ubiops.Configuration(api_key={'Authorization': self.api_key},
host=self.api_host))
self.api = ubiops.CoreApi(self.client)
self.input_factory = UbiOpsInputRequestData(
spec=self.input_spec, api=self.api, project_name=self.project_name)
self.output_factory = UbiOpsOutputRequestData(
spec=self.output_spec,
api=self.api,
project_name=self.project_name)
def raise_for_status(api_endpoint, token):
configuration = api.Configuration()
configuration.host = api_endpoint
configuration.api_key_prefix['Authorization'] = ''
configuration.api_key['Authorization'] = token
client = api.CoreApi(api.ApiClient(configuration))
client.user_agent = "UbiOps/cli/%s" % VERSION
assert client.service_status().status == 'ok'
client.api_client.close()
# Get email from token
service_user = user(api_endpoint=api_endpoint, token=token)
user_config = Config()
user_config.set('auth.api', api_endpoint)
user_config.set('auth.email', service_user['email'])
user_config.set('auth.service_token', token)
user_config.delete_option('auth.tmp_access_token')
user_config.write()
def main(req: func.HttpRequest):
"""
Deployment request that is HTTP Triggered.
:param req: HttpRequest object
"""
# Get the POST request body
req_body = req.get_json()
configuration = ubiops.Configuration()
configuration.api_key['Authorization'] = f"Token {TOKEN}"
client = ubiops.ApiClient(configuration)
api = ubiops.api.CoreApi(client)
r = api.pipeline_requests_create(project_name=PROJECT_NAME,
pipeline_name=PIPELINE_NAME,
data=req_body)
return f"Response of pipeline request is {r}"
def ubiops_request(event, context):
"""
Pipeline batch request triggered from a message on a Cloud Pub/Sub topic.
:param dict event: Event payload.
:param google.cloud.functions.Context context: Metadata for the event.
"""
pubsub_message = base64.b64decode(event['data']).decode('utf-8')
# The API Token for UbiOps is hardcoded for simplicity in this example.
# This should *absolutely never* be done in a production like environment.
# Instead make use of the solutions provided, in this case by Google, to handle secrets and passwords.
configuration = ubiops.Configuration()
configuration.api_key['Authorization'] = 'Token abcdefghijklmnopqrstuvwxyz'
client = ubiops.ApiClient(configuration)
api = ubiops.api.CoreApi(client)
api.batch_pipeline_requests_create(project_name='test-project',
pipeline_name='test-pipeline',
data=[pubsub_message])
def main(req: func.HttpRequest):
"""
Deployment request that is HTTP Triggered.
:param req: HttpRequest object
"""
# Get the POST request body
req_body = req.get_json()
configuration = ubiops.Configuration()
configuration.api_key['Authorization'] = 'Token <YOUR TOKEN HERE>'
client = ubiops.ApiClient(configuration)
api = ubiops.api.CoreApi(client)
r = api.pipeline_requests_create(
project_name=PROJECT_NAME,
pipeline_name=PIPELINE_NAME,
data={'data': json.dumps(req_body['value']), 'training': False}
)
return json.dumps({'output': f"Response of pipeline request is {r}"})
def _connect_api(api_token):
client = ubiops.ApiClient(
ubiops.Configuration(api_key={'Authorization': api_token},
host='https://api.ubiops.com/v2.1'))
return ubiops.CoreApi(client)
# Connect with your UbiOps environment
API_TOKEN = '<INSERT API_TOKEN WITH PROJECT EDITOR RIGHTS>' # Make sure this is in the format "Token token-code"
PROJECT_NAME = '<INSERT PROJECT NAME IN YOUR ACCOUNT>'
DEPLOYMENT_NAME = 'mnist-streamlit'
# API setup
if PROJECT_NAME and API_TOKEN and DEPLOYMENT_NAME:
# Only reconnect if API object is not in session state
if 'ubiops_api' not in st.session_state:
with st.spinner("Connecting to UbiOps API"):
configuration = ubiops.Configuration(host="https://api.ubiops.com/v2.1")
configuration.api_key['Authorization'] = API_TOKEN
client = ubiops.ApiClient(configuration)
st.session_state.ubiops_api = ubiops.CoreApi(client)
deployment_info = st.session_state.ubiops_api.deployments_get(PROJECT_NAME,DEPLOYMENT_NAME)
print(deployment_info)
sleep(2) # sleep for 2s to showcase progress spinners
# Use the streamlit session to store API object
if(st.session_state.ubiops_api.service_status().status == 'ok' ):
st.success("Connected to UbiOps API!")
else:
st.error("Not connected!")
def main(cfg):
if cfg.wandb.project:
import wandb
from wandb.keras import WandbCallback
wandb.init(project=cfg.wandb.project)
callbacks = [WandbCallback()]
else:
callbacks = []
data_path = Path("/pfs/faces/data/imdb_crop")
#data_path = Path("/home/raoulfasel/Documents/pachyderm/age_gender_estimation/data/imdb_crop")
csv_path = Path(to_absolute_path("./")).joinpath("meta", f"{cfg.data.db}.csv")
#csv_path = Path(to_absolute_path("/pfs/faces")).joinpath("meta", f"{cfg.data.db}.csv")
print(csv_path)
df = pd.read_csv(str(csv_path))
train, val = train_test_split(df, random_state=42, test_size=0.1)
train_gen = ImageSequence(cfg, train, "train", data_path)
val_gen = ImageSequence(cfg, val, "val", data_path)
strategy = tf.distribute.MirroredStrategy()
with strategy.scope():
model = get_model(cfg)
opt = get_optimizer(cfg)
scheduler = get_scheduler(cfg)
model.compile(optimizer=opt,
loss=["sparse_categorical_crossentropy", "sparse_categorical_crossentropy"],
metrics=['accuracy'])
#checkpoint_dir = Path(to_absolute_path("age_gender_estimation")).joinpath("checkpoint")
checkpoint_dir = Path(to_absolute_path("/pfs/build")).joinpath("checkpoint")
print(checkpoint_dir)
checkpoint_dir.mkdir(exist_ok=True)
filename = "_".join([cfg.model.model_name,
str(cfg.model.img_size),
"weights.{epoch:02d}-{val_loss:.2f}.hdf5"])
callbacks.extend([
LearningRateScheduler(schedule=scheduler),
ModelCheckpoint(str(checkpoint_dir) + "/" + filename,
monitor="val_loss",
verbose=1,
save_best_only=True,
mode="auto")
])
model.fit(train_gen, epochs=cfg.train.epochs, callbacks=callbacks, validation_data=val_gen,
workers=multiprocessing.cpu_count())
model.save("tensorflow_deployment_package/tensorflow_model.h5")
with open('/opt/ubiops/token', 'r') as reader:
API_TOKEN = reader.read()
client = ubiops.ApiClient(ubiops.Configuration(api_key={'Authorization': API_TOKEN},
host='https://api.ubiops.com/v2.1'))
api = ubiops.CoreApi(client)
# Create the deployment
deployment_template = ubiops.DeploymentCreate(
name=DEPLOYMENT_NAME,
description='Tensorflow deployment',
input_type='structured',
output_type='structured',
input_fields=[
ubiops.DeploymentInputFieldCreate(
name='input_image',
data_type='blob',
),
],
output_fields=[
ubiops.DeploymentOutputFieldCreate(
name='output_image',
data_type='blob'
),
],
labels={"demo": "tensorflow"}
)
api.deployments_create(
project_name=PROJECT_NAME,
data=deployment_template
)
# Create the version
version_template = ubiops.DeploymentVersionCreate(
version=DEPLOYMENT_VERSION,
language='python3.8',
instance_type="2048mb",
minimum_instances=0,
maximum_instances=1,
maximum_idle_time=1800 # = 30 minutes
)
api.deployment_versions_create(
project_name=PROJECT_NAME,
deployment_name=DEPLOYMENT_NAME,
data=version_template
)
# Zip the deployment package
shutil.make_archive('tensorflow_deployment_package', 'zip', '.', 'tensorflow_deployment_package')
# Upload the zipped deployment package
file_upload_result =api.revisions_file_upload(
project_name=PROJECT_NAME,
deployment_name=DEPLOYMENT_NAME,
version=DEPLOYMENT_VERSION,
file='tensorflow_deployment_package.zip'
)
