def read_image_from_url_cv2(url):
import_cv2()
logger.info("Read cv2 image from: {0}".format(url))
return cv2.imdecode(
np.asarray(bytearray(
urlopen(Request(url=url, headers=headers), timeout=100).read()),
dtype="uint8"), cv2.IMREAD_UNCHANGED)
def upload_bytes(self,
bytes_data,
key,
bucket_name=None,
overwrite=False,
encrypt=False,
acl=None):
"""
Uploads bytes to S3
This is provided as a convenience to drop a string in S3. It uses the
boto infrastructure to ship a file to s3.
Args:
bytes_data: `bytes`. bytes to set as content for the key.
key: `str`. S3 key that will point to the file.
bucket_name: `str`. Name of the bucket in which to store the file.
overwrite: `bool`. A flag to decide whether or not to overwrite the key
if it already exists.
encrypt: `bool`. If True, the file will be encrypted on the server-side
by S3 and will be stored in an encrypted form while at rest in S3.
acl: `str`. ACL to use for uploading, e.g. "public-read".
"""
if not bucket_name:
bucket_name = self.bucket
if bucket_name is None:
raise MlChainError("bucket can't be None",
code="S000",
status_code=500)
filelike_buffer = BytesIO(bytes_data)
try:
self.client.upload_fileobj(filelike_buffer, bucket_name, key)
except Exception as ex:
logger.info(str(ex))
raise MlChainError(str(ex), code="S006", status_code=500)
def load(self):
original_cuda_variable = os.environ.get('CUDA_VISIBLE_DEVICES')
if original_cuda_variable is None:
os.environ['CUDA_VISIBLE_DEVICES'] = str(next(gpus))
else:
logger.info(
f"Skipping automatic GPU selection for gunicorn worker since CUDA_VISIBLE_DEVICES environment variable is already set to {original_cuda_variable}"
)
serve_model = get_model(entry_file, serve_model=True)
if serve_model is None:
raise Exception(
f"Can not init model class from {entry_file}. Please check mlconfig.yaml or {entry_file} or mlchain run -m {{mode}}!"
)
if isinstance(serve_model, ServeModel):
if (not self.autofrontend) and model_id is not None:
from mlchain.server.autofrontend import register_autofrontend
register_autofrontend(model_id,
serve_model=serve_model,
version=version,
endpoint=os.getenv('NGROK_URL'))
self.autofrontend = True
if self.server == 'flask':
from mlchain.server.flask_server import FlaskServer
app = FlaskServer(serve_model,
name=name,
api_format=api_format,
version=version,
authentication=authentication,
static_url_path=static_url_path,
static_folder=static_folder,
template_folder=template_folder)
app.register_swagger()
if cors:
from flask_cors import CORS
CORS(app.app)
return app.app
if self.server == 'quart':
from mlchain.server.quart_server import QuartServer
app = QuartServer(serve_model,
name=name,
api_format=api_format,
version=version,
authentication=authentication,
static_url_path=static_url_path,
static_folder=static_folder,
template_folder=template_folder)
app.register_swagger()
if cors:
from quart_cors import cors as CORS
CORS(app.app)
return app.app
return None
def delete(self, path, bucket=None):
bucket = bucket or self.bucket
if bucket is None:
raise MlChainError("bucket can't be None",
code="S000",
status_code=500)
try:
self.client.remove_object(bucket, path)
except Exception as ex:
logger.info(str(ex))
raise MlChainError(str(ex), code="S011", status_code=500)
def listdir(self, path, bucket=None):
bucket = bucket or self.bucket
if bucket is None:
raise MlChainError("bucket can't be None",
code="S000",
status_code=500)
try:
results = self.list(bucket_name=bucket, prefix=path)
return {'files': results['keys'], 'dirs': results['prefixes']}
except Exception as ex:
logger.info(str(ex))
raise MlChainError(str(ex), code="S010", status_code=500)
def register_autofrontend(model_id,
serve_model,
version='latest',
endpoint=None):
mlchain_management = os.getenv('MLCHAIN_URL', None)
if endpoint is None:
endpoint = ''
autofrontend_template = AutofrontendConfig()
if serve_model.config is not None:
out_configs = serve_model.config
else:
out_configs = {}
for name, func in serve_model.get_all_func().items():
if name in out_configs:
out_config = out_configs[name]
if 'config' in out_config:
config = out_config['config']
else:
config = None
else:
config = None
autofrontend_template.add_endpoint(func,
f'{endpoint}/call/{name}',
output_config=config)
if os.path.exists("Readme.md"):
description = open("Readme.md", encoding='utf-8').read()
else:
description = ""
if os.path.exists("changelog.md"):
changelog = open("changelog.md", encoding='utf-8').read()
else:
changelog = ""
if mlchain_management and model_id is not None:
config_version = {
"model_id": model_id,
"version": version,
"input_config": autofrontend_template.input_config,
"output_config": autofrontend_template.output_config,
'endpoint': endpoint,
'readme': description,
'changelog': changelog
}
try:
import requests
res = requests.post(f'{mlchain_management}/version/create',
json=config_version)
logger.info(str(res.json()))
except Exception as ex:
logger.error(ex)
def upload_cv2(self,
img,
key,
bucket_name=None,
overwrite=False,
quality=95,
img_fmt='.jpg',
acl=None):
"""
Uploads bytes to S3
This is provided as a convenience to drop a string in S3. It uses the
boto infrastructure to ship a file to s3.
Args:
bytes_data: `bytes`. bytes to set as content for the key.
key: `str`. S3 key that will point to the file.
bucket_name: `str`. Name of the bucket in which to store the file.
overwrite: `bool`. A flag to decide whether or not to overwrite the key
if it already exists.
encrypt: `bool`. If True, the file will be encrypted on the server-side
by S3 and will be stored in an encrypted form while at rest in S3.
acl: `str`. ACL to use for uploading, e.g. "public-read".
"""
if not bucket_name:
bucket_name = self.bucket
if bucket_name is None:
raise MlChainError("bucket can't be None",
code="S000",
status_code=500)
try:
jpg_formats = ['.JPG', '.JPEG']
png_formats = ['.PNG']
encode_params = None
img_fmt = '.{0}'.format(img_fmt.strip('.')).upper()
cv2 = import_cv2()
if img_fmt in jpg_formats:
encode_params = [cv2.IMWRITE_JPEG_QUALITY, quality]
elif img_fmt in png_formats:
encode_params = [cv2.IMWRITE_PNG_COMPRESSION, quality]
ret, buf = cv2.imencode(img_fmt, img, encode_params)
filelike_buffer = BytesIO(buf.tostring())
self.client.upload_fileobj(filelike_buffer, bucket_name, key)
except Exception as ex:
logger.info(str(ex))
raise MlChainError(str(ex), code="S007", status_code=500)
def upload_dir(self, dir_path, prefix, bucket_name=None):
bucket_name = bucket_name or self.bucket
if bucket_name is None:
raise MlChainError("bucket can't be None",
code="S000",
status_code=500)
try:
for file in glob.glob(os.path.join(dir_path, '*'), recursive=True):
file_name = file[len(dir_path):].replace('\\', '/')
if file_name.startswith('/'):
file_name = file_name[1:]
key = os.path.join(prefix, file_name).replace('\\', '/')
if os.path.isfile(file):
self.upload_file(file, key, bucket_name)
else:
self.upload_dir(file, key, bucket_name)
except Exception as ex:
logger.info(str(ex))
raise MlChainError(str(ex), code="S008", status_code=500)
def read_image_from_url_pil(url):
logger.info("Read pil image from: {0}".format(url))
file = BytesIO(
urlopen(Request(url=url, headers=headers), timeout=100).read())
img = Image.open(file)
return img
def run_command(entry_file, host, port, bind, wrapper, server, workers, config,
name, mode, api_format, ngrok, kws):
kws = list(kws)
if isinstance(entry_file, str) and not os.path.exists(entry_file):
kws = [f'--entry_file={entry_file}'] + kws
entry_file = None
from mlchain import config as mlconfig
default_config = False
if config is None:
default_config = True
config = 'mlconfig.yaml'
config_path = copy.deepcopy(config)
if os.path.isfile(config_path) and os.path.exists(config_path):
config = mlconfig.load_file(config_path)
if config is None:
raise SystemExit(
"Config file {0} are not supported".format(config_path))
else:
if not default_config:
raise SystemExit("Can't find config file {0}".format(config_path))
else:
raise SystemExit(
"Can't find mlchain config file. Please double check your current working directory. Or use `mlchain init` to initialize a new ones here."
)
if 'mode' in config and 'env' in config['mode']:
if mode in config['mode']['env']:
config['mode']['default'] = mode
elif mode is not None:
available_mode = list(config['mode']['env'].keys())
available_mode = [
each for each in available_mode if each != 'default'
]
raise SystemExit(
f"No {mode} mode are available. Found these mode in config file: {available_mode}"
)
mlconfig.load_config(config)
for kw in kws:
if kw.startswith('--'):
tokens = kw[2:].split('=', 1)
if len(tokens) == 2:
key, value = tokens
mlconfig.mlconfig.update({key: value})
else:
raise AssertionError("Unexpected param {0}".format(kw))
else:
raise AssertionError("Unexpected param {0}".format(kw))
model_id = mlconfig.get_value(None, config, 'model_id', None)
entry_file = mlconfig.get_value(entry_file, config, 'entry_file',
'server.py')
if entry_file.strip() == '':
raise SystemExit(f"Entry file cannot be empty")
if not os.path.exists(entry_file):
raise SystemExit(
f"Entry file {entry_file} not found in current working directory.")
host = mlconfig.get_value(host, config, 'host', 'localhost')
port = mlconfig.get_value(port, config, 'port', 5000)
server = mlconfig.get_value(server, config, 'server', 'flask')
if len(bind) == 0:
bind = None
bind = mlconfig.get_value(bind, config, 'bind', [])
wrapper = mlconfig.get_value(wrapper, config, 'wrapper', None)
if wrapper == 'gunicorn' and os.name == 'nt':
logger.warning(
'Gunicorn warper are not supported on Windows. Switching to None instead.'
)
wrapper = None
workers = None
if 'gunicorn' in config:
workers = mlconfig.get_value(workers, config['gunicorn'], 'workers',
None)
if workers is None and 'hypercorn' in config.keys():
workers = mlconfig.get_value(workers, config['hypercorn'], 'workers',
None)
workers = int(workers) if workers is not None else 1
name = mlconfig.get_value(name, config, 'name', None)
cors = mlconfig.get_value(None, config, 'cors', False)
static_folder = mlconfig.get_value(None, config, 'static_folder', None)
static_url_path = mlconfig.get_value(None, config, 'static_url_path', None)
template_folder = mlconfig.get_value(None, config, 'template_folder', None)
version = mlconfig.get_value(None, config, 'version', '0.0')
version = str(version)
api_format = mlconfig.get_value(api_format, config, 'api_format', None)
api_keys = os.getenv('API_KEYS', None)
if api_keys is not None:
api_keys = api_keys.split(';')
api_keys = mlconfig.get_value(api_keys, config, 'api_keys', None)
if api_keys is None:
authentication = None
else:
authentication = Authentication(api_keys)
import logging
logging.root = logging.getLogger(name)
logger.debug(
dict(entry_file=entry_file,
host=host,
port=port,
bind=bind,
wrapper=wrapper,
server=server,
workers=workers,
name=name,
mode=mode,
api_format=api_format,
kws=kws))
bind = list(bind)
if ngrok:
from pyngrok import ngrok as pyngrok
endpoint = pyngrok.connect(port=port)
logger.info("Ngrok url: {0}".format(endpoint))
os.environ['NGROK_URL'] = endpoint
if server == 'grpc':
from mlchain.server.grpc_server import GrpcServer
app = get_model(entry_file, serve_model=True)
if app is None:
raise Exception(
"Can not init model class from {0}. Please check mlconfig.yaml or {0} or mlchain run -m {{mode}}!"
.format(entry_file))
app = GrpcServer(app, name=name)
app.run(host, port)
elif wrapper == 'gunicorn':
from gunicorn.app.base import BaseApplication
gpus = select_gpu()
class GunicornWrapper(BaseApplication):
def __init__(self, server_, **kwargs):
assert server_.lower() in ['quart', 'flask']
self.server = server_.lower()
self.options = kwargs
self.autofrontend = False
super(GunicornWrapper, self).__init__()
def load_config(self):
config = {
key: value
for key, value in self.options.items()
if key in self.cfg.settings and value is not None
}
for key, value in config.items():
self.cfg.set(key.lower(), value)
from mlchain.base.gunicorn_config import post_worker_init
self.cfg.set("post_worker_init", post_worker_init)
def load(self):
original_cuda_variable = os.environ.get('CUDA_VISIBLE_DEVICES')
if original_cuda_variable is None:
os.environ['CUDA_VISIBLE_DEVICES'] = str(next(gpus))
else:
logger.info(
f"Skipping automatic GPU selection for gunicorn worker since CUDA_VISIBLE_DEVICES environment variable is already set to {original_cuda_variable}"
)
serve_model = get_model(entry_file, serve_model=True)
if serve_model is None:
raise Exception(
f"Can not init model class from {entry_file}. Please check mlconfig.yaml or {entry_file} or mlchain run -m {{mode}}!"
)
if isinstance(serve_model, ServeModel):
if (not self.autofrontend) and model_id is not None:
from mlchain.server.autofrontend import register_autofrontend
register_autofrontend(model_id,
serve_model=serve_model,
version=version,
endpoint=os.getenv('NGROK_URL'))
self.autofrontend = True
if self.server == 'flask':
from mlchain.server.flask_server import FlaskServer
app = FlaskServer(serve_model,
name=name,
api_format=api_format,
version=version,
authentication=authentication,
static_url_path=static_url_path,
static_folder=static_folder,
template_folder=template_folder)
app.register_swagger()
if cors:
from flask_cors import CORS
CORS(app.app)
return app.app
if self.server == 'quart':
from mlchain.server.quart_server import QuartServer
app = QuartServer(serve_model,
name=name,
api_format=api_format,
version=version,
authentication=authentication,
static_url_path=static_url_path,
static_folder=static_folder,
template_folder=template_folder)
app.register_swagger()
if cors:
from quart_cors import cors as CORS
CORS(app.app)
return app.app
return None
if host is not None and port is not None:
bind.append('{0}:{1}'.format(host, port))
bind = list(set(bind))
gunicorn_config = config.get('gunicorn', {})
gunicorn_env = ['worker_class', 'threads', 'workers']
if workers is not None:
gunicorn_config['workers'] = workers
for k in gunicorn_env:
if get_env(k) in os.environ:
gunicorn_config[k] = os.environ[get_env(k)]
if server == 'flask' and 'worker_class' in gunicorn_config:
if 'uvicorn' in gunicorn_config['worker_class']:
logger.warning(
"Can't use flask with uvicorn. change to gthread")
gunicorn_config['worker_class'] = 'gthread'
GunicornWrapper(server, bind=bind, **gunicorn_config).run()
elif wrapper == 'hypercorn' and server == 'quart':
from mlchain.server.quart_server import QuartServer
app = get_model(entry_file, serve_model=True)
if app is None:
raise Exception(
"Can not init model class from {0}. Please check mlconfig.yaml or {0} or mlchain run -m {{mode}}!"
.format(entry_file))
app = QuartServer(app,
name=name,
version=version,
api_format=api_format,
authentication=authentication,
static_url_path=static_url_path,
static_folder=static_folder,
template_folder=template_folder)
app.run(host,
port,
bind=bind,
cors=cors,
gunicorn=False,
hypercorn=True,
**config.get('hypercorn', {}),
model_id=model_id)
app = get_model(entry_file)
if app is None:
raise Exception(
"Can not init model class from {0}. Please check mlconfig.yaml or {0} or mlchain run -m {{mode}}!"
.format(entry_file))
if isinstance(app, MLServer):
if app.__class__.__name__ == 'FlaskServer':
app.run(host, port, cors=cors, gunicorn=False)
elif app.__class__.__name__ == 'QuartServer':
app.run(host, port, cors=cors, gunicorn=False, hypercorn=False)
elif app.__class__.__name__ == 'GrpcServer':
app.run(host, port)
elif isinstance(app, ServeModel):
if server not in ['quart', 'grpc']:
server = 'flask'
if server == 'flask':
from mlchain.server.flask_server import FlaskServer
app = FlaskServer(app,
name=name,
api_format=api_format,
version=version,
authentication=authentication,
static_url_path=static_url_path,
static_folder=static_folder,
template_folder=template_folder)
app.run(host,
port,
cors=cors,
gunicorn=False,
model_id=model_id,
threads=workers > 1)
elif server == 'quart':
from mlchain.server.quart_server import QuartServer
app = QuartServer(app,
name=name,
api_format=api_format,
version=version,
authentication=authentication,
static_url_path=static_url_path,
static_folder=static_folder,
template_folder=template_folder)
app.run(host,
port,
cors=cors,
gunicorn=False,
hypercorn=False,
model_id=model_id,
workers=workers)
elif server == 'grpc':
from mlchain.server.grpc_server import GrpcServer
app = GrpcServer(app, name=name)
app.run(host, port)
def list(self,
bucket_name=None,
prefix='',
delimiter='/',
page_size=None,
max_items=None,
keys=True,
prefixes=True):
"""
Lists prefixes and contents in a bucket under prefix.
Args:
bucket_name: `str`. the name of the bucket
prefix: `str`. a key prefix
delimiter: `str`. the delimiter marks key hierarchy.
page_size: `str`. pagination size
max_items: `int`. maximum items to return
keys: `bool`. if it should include keys
prefixes: `boll`. if it should include prefixes
"""
if not bucket_name:
bucket_name = self.bucket
if bucket_name is None:
raise MlChainError("bucket can't be None",
code="S000",
status_code=500)
config = {
'PageSize': page_size,
'MaxItems': max_items,
}
try:
paginator = self.client.get_paginator('list_objects_v2')
response = paginator.paginate(Bucket=bucket_name,
Prefix=prefix,
Delimiter=delimiter,
PaginationConfig=config)
def get_keys(contents):
list_keys = []
for cont in contents:
list_keys.append(cont['Key'][len(prefix):])
return list_keys
def get_prefixes(page_prefixes):
list_prefixes = []
for pref in page_prefixes:
list_prefixes.append(pref['Prefix'][len(prefix):-1])
return list_prefixes
results = {'keys': [], 'prefixes': []}
for page in response:
if prefixes:
results['prefixes'] += get_prefixes(
page.get('CommonPrefixes', []))
if keys:
results['keys'] += get_keys(page.get('Contents', []))
return results
except Exception as ex:
logger.info(str(ex))
raise MlChainError(str(ex), code="S009", status_code=500)