def test_session_push_sklearn():
'''Tests basic model pushing functionality with sklearn'''
clear_jwt()
with _patch_auth():
with MockServer() as server:
iris = load_iris()
X = iris.data
y = iris.target
clf = RandomForestClassifier(random_state=0)
clf.fit(X, y)
columns = [
'sepallength', 'sepalwidth', 'petallength', 'petalwidth'
]
X_df = pd.DataFrame(X, columns=columns)
DataFrame = create_dataframe('DataFrame', X_df)
Predictions = create_namedtuple('Predictions',
[('predictions', List[int])])
def predict(df: DataFrame) -> Predictions:
'''Predicts the class of iris'''
X = np.column_stack(df)
yhat = clf.predict(X)
preds = Predictions(predictions=yhat)
return preds
model = Model(predict=predict)
model_url, auth_url, _, _ = server.config
s = AcumosSession(model_url, auth_url)
s.push(model, name='sklearn_iris_push')
def _dumped_model(model: Model, model_name: str = 'test-model') -> str:
"""Dumps a model an returns its path"""
with TemporaryDirectory() as dump_dir:
session = AcumosSession()
session.dump(model, model_name, dump_dir)
model_dir = os.path.join(dump_dir, model_name)
yield model_dir
def push_model(self, CSV_filenames, push_api, auth_api):
from acumos.session import AcumosSession
session = AcumosSession(push_api=push_api, auth_api=auth_api)
model, req = self.generate_model(CSV_filenames)
try:
session.push(model, 'VmPredictor', req) # creates ./my-iris.zip
return True
except Exception as e:
print(">> Error: Model push error {:}".format(e))
return False
def _push_dummy_model(extra_headers=None):
'''Generic dummy model push routine'''
def my_transform(x: int, y: int) -> int:
return x + y
model = Model(transform=my_transform)
with MockServer() as server:
model_url, auth_url, _, _ = server.config
s = AcumosSession(model_url, auth_url)
s.push(model, name='my-model', extra_headers=extra_headers)
def dump_model(self, CSV_filenames, model_dir):
from acumos.session import AcumosSession
model, req = self.generate_model(CSV_filenames)
session = AcumosSession()
try:
if not exists(model_dir):
makedirs(model_dir)
session.dump(model, 'VmPredictor', model_dir,
req) # creates ./my-iris.zip
return True
except Exception as e:
print(">> Error: Model dump error {:}".format(e))
return False
def test_session_dump_zip(replace: bool):
'''Tests session dump zip'''
def my_transform(x: int, y: int) -> int:
return x + y
model = Model(transform=my_transform)
model_name = 'my-model'
session = AcumosSession()
with tempfile.TemporaryDirectory() as tdir:
model_zip_path = Path(tdir) / f"{model_name}.zip"
session.dump_zip(model, model_name, model_zip_path)
import zipfile
with zipfile.ZipFile(model_zip_path, "r") as model_zip:
assert set(model_zip.namelist()) == set(_REQ_FILES)
if replace is False:
with pytest.raises(AcumosError):
session.dump_zip(model, model_name,
model_zip_path) # file already exists
else:
session.dump_zip(
model, model_name, model_zip_path,
replace=replace) # file already exists but it will be replaced
def _mock_model(yield_model=True):
'''Context manager that yields an acumos.wrapped.WrappedModel model for testing purposes'''
def add(x: int, y: int) -> int:
return x + y
def multiply(x: int, y: int) -> int:
return x * y
model = Model(add=add, multiply=multiply)
session = AcumosSession()
with TemporaryDirectory() as tdir:
session.dump(model, 'test-model', tdir)
model_dir = os.path.join(tdir, 'test-model')
wrapped_model = load_model(model_dir)
yield wrapped_model if yield_model else model_dir
def _push_dummy_model(extra_headers=None,
use_model_url=True,
use_auth_url=False,
options=None):
'''Generic dummy model push routine'''
def my_transform(x: int, y: int) -> int:
return x + y
model = Model(transform=my_transform)
with MockServer() as server:
_model_url, _auth_url, _, _ = server.config
model_url = _model_url if use_model_url else None
auth_url = _auth_url if use_auth_url else None
session = AcumosSession(model_url, auth_url)
session.push(model,
name='my-model',
extra_headers=extra_headers,
options=options)
def test_session_push_keras():
'''Tests basic model pushing functionality with keras'''
clear_jwt()
with _patch_auth():
with MockServer() as server:
iris = load_iris()
X = iris.data
y = pd.get_dummies(iris.target).values
clf = Sequential()
clf.add(Dense(3, input_dim=4, activation='relu'))
clf.add(Dense(3, activation='softmax'))
clf.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
clf.fit(X, y)
columns = [
'sepallength', 'sepalwidth', 'petallength', 'petalwidth'
]
X_df = pd.DataFrame(X, columns=columns)
DataFrame = create_dataframe('DataFrame', X_df)
Predictions = create_namedtuple('Predictions',
[('predictions', List[int])])
def predict(df: DataFrame) -> Predictions:
'''Predicts the class of iris'''
X = np.column_stack(df)
yhat = clf.predict(X)
preds = Predictions(predictions=yhat)
return preds
model = Model(predict=predict)
model_url, auth_url, _, _ = server.config
s = AcumosSession(model_url, auth_url)
s.push(model, name='keras_iris_push')
def test_session_dump(replace: bool):
'''Tests session dump'''
def my_transform(x: int, y: int) -> int:
return x + y
model = Model(transform=my_transform)
model_name = 'my-model'
session = AcumosSession()
with tempfile.TemporaryDirectory() as tdir:
session.dump(model, model_name, tdir)
model_dir = path_join(tdir, model_name)
assert set(listdir(model_dir)) == set(_REQ_FILES)
if replace is False:
with pytest.raises(AcumosError):
session.dump(model, model_name, tdir) # file already exists
else:
session.dump(
model, model_name, tdir,
replace=replace) # file already exists but it will be replaced
def test_session_dump():
'''Tests session dump'''
def my_transform(x: int, y: int) -> int:
return x + y
model = Model(transform=my_transform)
model_name = 'my-model'
s = AcumosSession()
with tempfile.TemporaryDirectory() as tdir:
s.dump(model, model_name, tdir)
model_dir = path_join(tdir, model_name)
assert set(listdir(model_dir)) == set(_REQ_FILES)
with pytest.raises(AcumosError):
s.dump(model, model_name, tdir) # file already exists
def keras_evaluate(config):
taskComplete = False
useSklearn = True
model = None
if useSklearn:
# formulate the pipelien to be used
from image_classifier.keras_model.prediction_formatter import Formatter
model, reqs = model_create_pipeline(config['model_path'], config['label_path'],
config['num_top_predictions'])
if 'dump_model' in config and config['dump_model']:
from acumos.session import AcumosSession
from os import makedirs
if not os.path.exists(config['dump_model']):
makedirs(config['dump_model'])
print("Dumping new model to '{:}'...".format(config['dump_model']))
session = AcumosSession()
session.dump(model, MODEL_NAME, config['dump_model'], reqs) # creates ./my-iris.zip
taskComplete = True
if 'push_address' in config and config['push_address']: # and 'auth_address' in config and config['push_address']:
from acumos.session import AcumosSession,Options
if config['create_microservice']=='False' :
if config['license_path'] is not 'None' :
opts=Options(create_microservice=False,license=config['license_path'])
else :
opts=Options(create_microservice=False)
if config['create_microservice']=='True' :
if config['license_path'] is not 'None' :
opts=Options(create_microservice=True,license=config['license_path'])
else :
opts=Options(create_microservice=True)
session = AcumosSession(push_api=config['push_address']) #, auth_api=config['auth_address'])
print("Pushing new model to '{:}'".format(config['push_address'])) #,auth '{:}'...".format(config['push_address'], config['auth_address']))
session.push(model, MODEL_NAME, reqs, options=opts)
taskComplete = True
"""
Disable non-sklearn path for now
else:
from image_classifier.keras import inception_v4
from image_classifier.keras.image_decoder import ImageDecoder
# Load test image!
img = ImageDecoder.get_processed_image_keras_file(config['image']) # load image through keras
# img = evaluate_image.get_processed_image_cv(config['image'])
# Run prediction on test image
model, model_path = inception_v4.create_model(weights='imagenet', include_top=True, model_path=model_path)
preds = model.predict(img)
"""
preds = None
if model and not taskComplete: # means we need to run a prediction/classify
if not os.path.exists(config['image']) and (config['image_list'] and not os.path.exists(config['image_list'])):
print("The target image/list '{}'/'{}' was not found, please check input arguments.".format(config['image'], config['image_list']))
sys.exit(-1)
listImages = []
if config['image_list']: # provided a list to run with?
dfImages = pd.read_csv(config['image_list'], header=None, names=['file'], delimiter=",")
listImages = dfImages['file'].tolist()
if len(listImages) == 0: # make a list if just one item
listImages = [config['image']]
wrapped_model = model # 5/31/18, simplify to reuse already created model
type_in = wrapped_model.classify.input_type
for idx in range(len(listImages)):
curImage = listImages[idx]
print("Attempting classification of image [{:}]: {:}...".format(idx, curImage))
X = create_sample(curImage)
classify_in = type_in(*X)
pred_raw = wrapped_model.classify.wrapped(classify_in)
if len(pred_raw._fields) == 1: # type is nested?, go down one level
pred_raw = getattr(pred_raw, pred_raw._fields[0])
# already a wrapped response
predNew = pd.DataFrame(pred_raw)
predNew[Formatter.COL_NAME_IDX] = idx
if preds is None:
preds = predNew
else:
preds = preds.append(predNew, ignore_index=True)
preds.reset_index(drop=True, inplace=True)
return preds
y: target_onehot
})
print("Epoch {} | Loss {}".format(epoch, loss))
prediction = tf.argmax(logits, 1)
yhat = sess.run([prediction], {x: data})[0]
# note: this predicts on the training set for illustration purposes only
print(classification_report(target, yhat))
# =============================================================================
# create a acumos model from the tensorflow model
# =============================================================================
X_df = pd.DataFrame(
data,
columns=['sepal_length', 'sepal_width', 'petal_length', 'petal_width'])
IrisDataFrame = create_dataframe('IrisDataFrame', X_df)
def classify_iris(df: IrisDataFrame) -> List[int]:
'''Returns an array of iris classifications'''
X = np.column_stack(df)
return prediction.eval({x: X}, sess)
model = Model(classify=classify_iris)
session = AcumosSession()
session.dump(model, 'model', '.') # creates ./model
from acumos.session import AcumosSession
if __name__ == '__main__':
'''Main'''
iris = load_iris()
X = iris.data
y = iris.target
clf = RandomForestClassifier(random_state=0)
clf.fit(X, y)
columns = ['sepallength', 'sepalwidth', 'petallength', 'petalwidth']
X_df = pd.DataFrame(X, columns=columns)
DataFrame = create_dataframe('DataFrame', X_df)
Predictions = create_namedtuple('Predictions',
[('predictions', List[int])])
def predict(df: DataFrame) -> Predictions:
'''Predicts the class of iris'''
X = np.column_stack(df)
yhat = clf.predict(X)
preds = Predictions(predictions=yhat)
return preds
model = Model(transform=predict)
s = AcumosSession(None)
s.dump(model, 'model', '.')
# This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ===============LICENSE_END=========================================================
'''
Dumps an example model for illustrating acumos_model_runner usage
'''
from collections import Counter
from acumos.session import AcumosSession
from acumos.modeling import Model, List, Dict
def add(x: int, y: int) -> int:
'''Adds two numbers'''
return x + y
def count(strings: List[str]) -> Dict[str, int]:
'''Counts the occurrences of words in `strings`'''
return Counter(strings)
if __name__ == '__main__':
'''Main'''
model = Model(add=add, count=count)
session = AcumosSession()
session.dump(model, 'example-model', '.')
PORT = 3001
pw = os.environ['ACUMOS_PASSWORD']
user = os.environ['ACUMOS_USERNAME']
MODEL_PATH = "Acumos Property Assistant"
pd.set_option('display.float_format', lambda x: '%.2f' % x)
from acumos.modeling import Model, List, Dict, create_namedtuple, create_dataframe
from acumos.session import AcumosSession
# Session Endpoints for Acumos Challenge model upload and auth.
# session = AcumosSession(push_api="https://acumos-challenge.org/onboarding-app/v2/models",
# auth_api="https://acumos-challenge.org/onboarding-app/v2/auth")
session = AcumosSession()
print('done')
REDFIN_TRAIN_CSV = os.path.join("assets", "redfin_2018_8_boston.csv")
def rename_col(x):
return x.lower().replace(' ', '_').replace('$',
'cost').replace('/', '_per_')
# ### Main Redfin Acumos Model
#
# Using recently sold properties to predict current property values, includes qualitative (encoded) and quantitative metrics.
class RedfinAcumosModel:
from acumos.modeling import Model, List, create_dataframe
iris = load_iris()
X = iris.data
y = pd.get_dummies(iris.target).values
clf = Sequential()
clf.add(Dense(3, input_dim=4, activation='relu'))
clf.add(Dense(3, activation='softmax'))
clf.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
clf.fit(X, y)
X_df = pd.DataFrame(
X, columns=['sepal_length', 'sepal_width', 'petal_length', 'petal_width'])
IrisDataFrame = create_dataframe('IrisDataFrame', X_df)
def classify_iris(df: IrisDataFrame) -> List[int]:
'''Returns an array of iris classifications'''
X = np.column_stack(df)
return clf.predict_classes(X)
model = Model(classify=classify_iris)
session = AcumosSession()
session.dump_zip(model, 'keras', './keras.zip',
replace=True) # creates ./keras.zip
from sklearn.pipeline import Pipeline
pw = os.environ['ACUMOS_PASSWORD']
user = os.environ['ACUMOS_USERNAME']
MODEL_PATH = "Acumos Property Assistant"
pd.set_option('display.float_format', lambda x: '%.2f' % x)
mp.use('TkAgg')
from acumos.modeling import Model, List, Dict, create_namedtuple, create_dataframe
from acumos.session import AcumosSession
# Session Endpoints for Acumos Challenge model upload and auth.
# session = AcumosSession(push_api="https://acumos-challenge.org/onboarding-app/v2/models",
# auth_api="https://acumos-challenge.org/onboarding-app/v2/auth")
session = AcumosSession()
print('starting demo')
# In[19]:
REDFIN_TRAIN_CSV = os.path.join("assets", "redfin_2018_8_boston.csv")
REDFIN_TEST_CSV = os.path.join("assets", "redfin_2018_active_boston.csv")
# In[20]:
def rename_col(x):
return x.lower().replace(' ', '_').replace('$',
'cost').replace('/', '_per_')
print('Acumos version is {}'.format(acumos.__version__))
model = Model(loopback=echo)
# Test the model function
val = 'hi'
out = model.loopback.inner(val)
if out is None or out != val:
print('Failed to get expected output, giving up.')
sys.exit()
else:
print('model function says {}'.format(out))
# Serialize the model to support web on-boarding
session = AcumosSession()
subdir = 'bundle-loopback'
print('dumping model to subdir {}'.format(subdir))
session.dump(model, 'loopback', subdir)
# Un-comment the following lines to:
# 1. configure client library with authentication
# 2. on-board this model to an acumos instance
#clear_jwt();
#token = 'API_TOKEN_FROM_YOUR_ACUMOS_ACCOUNT'
#_configuration(jwt=token)
#host='acumos-host-name.org'
#print('creating session using host {}'.format(host))
#session=AcumosSession(
# push_api='http://{}/upload'.format(host),
# auth_api='http://{}/auth'.format(host))
def keras_evaluate(config):
taskComplete = False
useSklearn = True
listImages = []
if 'image_list' in config and config['image_list']:
dfImages = pd.read_csv(config['image_list'], header=None, names=['file'], delimiter=",")
listImages = dfImages['file'].tolist()
config['image'] = listImages[0]
X = create_sample(config['image'])
if useSklearn:
# formulate the pipelien to be used
from image_classifier.keras_model.prediction_formatter import Formatter
model, reqs = model_create_pipeline(config['model_path'], config['label_path'],
config['num_top_predictions'])
if 'push_address' in config and 'auth_address' in config and config['push_address']:
from acumos.session import AcumosSession
session = AcumosSession(push_api=config['push_address'], auth_api=config['auth_address'])
print("Pushing new model to upload '{:}', auth '{:}'...".format(config['push_address'], config['auth_address']))
session.push(model, MODEL_NAME, reqs) # creates ./my-iris.zip
taskComplete = True
if 'dump_model' in config and config['dump_model']:
from acumos.session import AcumosSession
from os import makedirs
if not os.path.exists(config['dump_model']):
makedirs(config['dump_model'])
print("Dumping new model to '{:}'...".format(config['dump_model']))
session = AcumosSession()
session.dump(model, MODEL_NAME, config['dump_model'], reqs) # creates ./my-iris.zip
taskComplete = True
preds = None
if not taskComplete: # means we need to run a prediction/classify
import tempfile
from acumos.session import _dump_model, _copy_dir
from os.path import join as path_join
from acumos.wrapped import load_model
if not listImages:
listImages = [config['image']]
preds = None
# temporarily wrap model to a temp directory (to get 'wrapped' functionality)
with tempfile.TemporaryDirectory() as tdir: # create temp dir
with _dump_model(model, MODEL_NAME, reqs) as dump_dir: # dump model to temp dir
_copy_dir(dump_dir, tdir, MODEL_NAME) # relocate for load_model below
model_dir = path_join(tdir, MODEL_NAME)
wrapped_model = load_model(model_dir) # load to wrapped model
type_in = wrapped_model.classify._input_type
for idx in range(len(listImages)):
curImage = listImages[idx]
print("Attempting classification of image [{:}]: {:}...".format(idx, curImage))
X = create_sample(curImage)
classify_in = type_in(*tuple(col for col in X.values.T))
pred_raw = wrapped_model.classify.from_wrapped(classify_in).as_wrapped()
# already a wrapped response
predNew = pd.DataFrame(np.column_stack(pred_raw), columns=pred_raw._fields)
predNew[Formatter.COL_NAME_IDX] = idx
if preds is None:
preds = predNew
else:
preds = preds.append(predNew, ignore_index=True)
preds.reset_index(drop=True, inplace=True)
"""
Disable non-sklearn path for now
else:
from image_classifier.keras import inception_v4
from image_classifier.keras.image_decoder import ImageDecoder
# Load test image!
img = ImageDecoder.get_processed_image_keras_file(config['image']) # load image through keras
# img = evaluate_image.get_processed_image_cv(config['image'])
# Run prediction on test image
model, model_path = inception_v4.create_model(weights='imagenet', include_top=True, model_path=model_path)
preds = model.predict(img)
"""
return preds
def main(config={}):
from face_privacy_filter.transform_detect import FaceDetectTransform
from face_privacy_filter.transform_region import RegionTransform
from face_privacy_filter._version import MODEL_NAME
import argparse
parser = argparse.ArgumentParser()
submain = parser.add_argument_group(
'main execution and evaluation functionality')
submain.add_argument(
'-p',
'--predict_path',
type=str,
default='',
help=
"save detections from model (model must be provided via 'dump_model')")
submain.add_argument(
'-i',
'--input',
type=str,
default='',
help=
'absolute path to input data (image or csv, only during prediction / dump)'
)
submain.add_argument('-c',
'--csv_input',
dest='csv_input',
action='store_true',
default=False,
help='input as CSV format not an image')
submain.add_argument('-f',
'--function',
type=str,
default='detect',
help='which type of model to generate',
choices=['detect', 'pixelate'])
submain.add_argument(
'-s',
'--suppress_image',
dest='suppress_image',
action='store_true',
default=False,
help='do not create an extra row for a returned image')
subopts = parser.add_argument_group(
'model creation and configuration options')
subopts.add_argument(
'-a',
'--push_address',
help=
'server address to push the model (e.g. http://localhost:8887/v2/models)',
default=os.getenv('ACUMOS_PUSH', ""))
subopts.add_argument(
'-A',
'--auth_address',
help=
'server address for login and push of the model (e.g. http://localhost:8887/v2/auth)',
default=os.getenv('ACUMOS_AUTH', ""))
subopts.add_argument(
'-d',
'--dump_model',
help='dump model to a pickle directory for local running',
default='')
config.update(vars(
parser.parse_args())) # pargs, unparsed = parser.parse_known_args()
if not config['predict_path']:
print("Attempting to create new model for dump or push...")
elif not os.path.exists(config['input']):
print(
"Prediction requested but target input '{:}' was not found, please check input arguments."
.format(config['input']))
sys.exit(-1)
# refactor the raw samples from upstream image classifier
if config['function'] == "detect":
transform = FaceDetectTransform(
include_image=not config['suppress_image'])
pipeline, reqs = model_create_pipeline(transform, config['function'],
False, True)
elif config['function'] == "pixelate":
transform = RegionTransform()
pipeline, reqs = model_create_pipeline(transform, config['function'],
True, False)
else:
print("Error: Functional mode '{:}' unknown, aborting create".format(
config['function']))
print(pipeline)
print(getattr(pipeline, config['function']))
# formulate the pipeline to be used
model_name = MODEL_NAME + "_" + config['function']
if config['push_address'] and config['auth_address']:
from acumos.session import AcumosSession
print("Pushing new model to '{:}'...".format(config['push_address']))
session = AcumosSession(push_api=config['push_address'],
auth_api=config['auth_address'])
session.push(pipeline, model_name,
reqs) # pushes model directly to servers
if config['dump_model']:
from acumos.session import AcumosSession
from os import makedirs
if not os.path.exists(config['dump_model']):
makedirs(config['dump_model'])
print("Dumping new model to '{:}'...".format(config['dump_model']))
session = AcumosSession()
session.dump(pipeline, model_name, config['dump_model'],
reqs) # creates model subdirectory
if config['predict_path']:
print("Using newly created model for local prediction...")
if not config['csv_input']:
inputDf = FaceDetectTransform.generate_in_df(config['input'])
else:
inputDf = pd.read_csv(config['input'],
converters={
FaceDetectTransform.COL_IMAGE_DATA:
FaceDetectTransform.read_byte_arrays
})
func_action = getattr(
pipeline,
config['function']) # simplify to just use loaded model 6/1
pred_raw = func_action.wrapped(inputDf)
transform_out = func_action.from_wrapped(pred_raw).as_wrapped()
dfPred = pd.DataFrame(list(zip(*transform_out)),
columns=transform_out._fields)
if not config['csv_input']:
dfPred = FaceDetectTransform.suppress_image(dfPred)
if config['predict_path']:
print("Writing prediction to file '{:}'...".format(
config['predict_path']))
if not config['csv_input']:
dfPred.to_csv(config['predict_path'], sep=",", index=False)
else:
FaceDetectTransform.generate_out_image(dfPred,
config['predict_path'])
if dfPred is not None:
print("Predictions:\n{:}".format(dfPred))
def main(config={}):
import argparse
from image_mood_classifier.prediction_formatter import Formatter
from image_mood_classifier._version import MODEL_NAME
parser = argparse.ArgumentParser()
submain = parser.add_argument_group('main execution and evaluation functionality')
submain.add_argument('-p', '--predict_path', type=str, default='', help="Save predictions from model (model must be provided via 'dump_model')")
submain.add_argument('-i', '--input', type=str, default='', help='Absolute path to input training data file. (for now must be a header-less CSV)')
submain.add_argument('-C', '--cuda_env', type=str, default='', help='Anything special to inject into CUDA_VISIBLE_DEVICES environment string')
subopts = parser.add_argument_group('model creation and configuration options')
subopts.add_argument('-l', '--labels', type=str, default='', help="Path to label one-column file with one row for each input")
subopts.add_argument('-m', '--model_type', type=str, default='rf', help='specify the underlying classifier type (rf (randomforest), svc (SVM))', choices=['svm', 'rf'])
subopts.add_argument('-f', '--feature_nomask', dest='feature_nomask', default=False, action='store_true', help='create masked samples on input')
subopts.add_argument('-n', '--add_softnoise', dest='softnoise', default=False, action='store_true', help='do not add soft noise to classification inputs')
subopts.add_argument('-a', '--push_address', help='server address to push the model (e.g. http://localhost:8887/upload)', default=os.getenv('ACUMOS_PUSH', ""))
subopts.add_argument('-A', '--auth_address', help='server address for login and push of the model (e.g. http://localhost:8887/auth)', default=os.getenv('ACUMOS_AUTH', ""))
subopts.add_argument('-d', '--dump_model', help='dump model to a pickle directory for local running', default='')
subopts.add_argument('-s', '--summary', type=int, dest='summary', default=0, help='summarize top N image classes are strong for which label class (only in training)')
config.update(vars(parser.parse_args())) # pargs, unparsed = parser.parse_known_args()
if not os.path.exists(config['input']):
print("The target input '{:}' was not found, please check input arguments.".format(config['input']))
sys.exit(-1)
print("Loading raw samples...")
rawDf = pd.read_csv(config['input'], delimiter=",")
# If you want to use a GPU set its index here
if config['cuda_env']:
os.environ['CUDA_VISIBLE_DEVICES'] = config['cuda_env']
if not config['predict_path'] and config['labels']:
if not os.path.exists(config['labels']):
print("The target labels '{:}' was not found, please check input arguments.".format(config['labels']))
sys.exit(-1)
# refactor the raw samples from upstream image classifier
print("=================input_softnoise=config['softnoise']:%s",config['softnoise'])
formatter = Formatter(input_softnoise=config['softnoise'])
print("=======================formater:%s", formatter)
print("Loading labels to train a new model...")
rawLabel = pd.read_csv(config['labels'], header=None, delimiter=",")
if len(rawLabel.columns) != 1:
print("Error, not currently programmed to best-of class selection to a singleton.")
sys.exit(-1)
rawLabel = rawLabel[0].tolist()
formatter.learn_input_mapping(rawDf, "tag", "image", "score")
print("=======================formater:%s", formatter)
print("Converting block of {:} responses into training data, utilizing {:} images...".format(len(rawDf), len(rawLabel)))
objRefactor = formatter.transform_raw_sample(rawDf, rawLabel, None if config['feature_nomask'] else Formatter.SAMPLE_GENERATE_MASKING)
print("Generated {:} total samples (skip-masking: {:})".format(len(objRefactor['values']), config['feature_nomask']))
clf = model_archive() # debug helper
# run summary?
if config['summary']:
df_combined = pd.DataFrame(objRefactor['values'], columns=objRefactor['columns'])
df_combined['_labels'] = objRefactor['labels']
groupSet = df_combined.groupby('_labels')
for nameG, rowsG in groupSet:
df_sum = rowsG.sum(axis=0, numeric_only=True)
series_top = df_sum.sort_values(ascending=False)
print("Label: '{:}', top {:} classes...".format(nameG, config['summary']))
print(series_top[0:config['summary']])
# create pipeline to dump via client library
if config['push_address'] or config['dump_model']:
if clf is None:
clf = classifier_train(objRefactor['values'], objRefactor['labels'], config['model_type'])
model, reqs = model_create_pipeline(formatter, clf)
model_archive(clf) # debug helper
# formulate the pipeline to be used
if config['push_address']:
from acumos.session import AcumosSession
session = AcumosSession(push_api=config['push_address'], auth_api=config['auth_address'])
session.push(model, MODEL_NAME, reqs) # creates ./my-iris.zip
print("Pushing new model to '{:}'...".format(config['push_address']))
if config['dump_model']:
from acumos.session import AcumosSession
from os import makedirs
if not os.path.exists(config['dump_model']):
makedirs(config['dump_model'])
print("Dumping new model to '{:}'...".format(config['dump_model']))
session = AcumosSession()
session.dump(model, MODEL_NAME, config['dump_model'], reqs) # creates ./my-iris.zip
else:
if not config['dump_model'] or not os.path.exists(config['dump_model']):
print("Attempting to predict from a dumped model, but model not found.".format(config['dump_model']))
sys.exit(-1)
print("Attempting predict/transform on input sample...")
from acumos.wrapped import load_model
model = load_model(config['dump_model'])
type_in = model.classify._input_type
classify_in = type_in(*tuple(col for col in rawDf.values.T))
out_wrapped = model.classify.from_wrapped(classify_in).as_wrapped()
# for now, peel out top sample from classify set
dfPred = pd.DataFrame(out_wrapped[0])
if config['predict_path']:
print("Writing prediction to file '{:}'...".format(config['predict_path']))
dfPred.to_csv(config['predict_path'], sep=",", index=False)
if dfPred is not None:
print("Predictions:\n{:}".format(dfPred))
with mlflow.start_run():
lr = ElasticNet(alpha=alpha, l1_ratio=l1_ratio, random_state=42)
lr.fit(train_x, train_y)
predicted_qualities = lr.predict(test_x)
(rmse, mae, r2) = eval_metrics(test_y, predicted_qualities)
print("Elasticnet model (alpha=%f, l1_ratio=%f):" % (alpha, l1_ratio))
print(" RMSE: %s" % rmse)
print(" MAE: %s" % mae)
print(" R2: %s" % r2)
mlflow.log_param("alpha", alpha)
mlflow.log_param("l1_ratio", l1_ratio)
mlflow.log_metric("rmse", rmse)
mlflow.log_metric("r2", r2)
mlflow.log_metric("mae", mae)
mlflow.sklearn.log_model(lr, "model")
# Acumos part
from acumos.modeling import Model, List, Dict, create_namedtuple, create_dataframe
from acumos.session import AcumosSession, Requirements
model = Model(eval=eval_metrics)
#Exporting Models
session = AcumosSession()
session.dump(model, 'SK model with Gaia', '.')
})
print("Epoch {} | Loss {}".format(epoch, loss))
prediction = tf.argmax(logits, 1)
yhat = sess.run([prediction], {x: data})[0]
# note: this predicts on the training set for illustration purposes only
print(classification_report(target, yhat))
# =============================================================================
# create a acumos model from the tensorflow model
# =============================================================================
X_df = pd.DataFrame(
data,
columns=['sepal_length', 'sepal_width', 'petal_length', 'petal_width'])
IrisDataFrame = create_dataframe('IrisDataFrame', X_df)
def classify_iris(df: IrisDataFrame) -> List[int]:
'''Returns an array of iris classifications'''
X = np.column_stack(df)
return prediction.eval({x: X}, sess)
model = Model(classify=classify_iris)
session = AcumosSession()
session.dump_zip(model, 'tensorflow', './tensorflow.zip',
replace=True) # creates ./tensorflow.zip
if __name__ == '__main__':
'''Test area'''
def tokenize(value: str) -> List[str]:
'''Segments text into tokens'''
return value.split()
def count(value: List[str]) -> Dict[str, int]:
'''Returns a count of tokens'''
return Counter(value)
# define models
tokenizer = Model(tokenize=tokenize)
counter = Model(count=count)
# save models
session = AcumosSession()
session.dump(tokenizer, 'tokenizer', '.')
session.dump(counter, 'counter', '.')
# instantiate runners
runner1 = Runner('tokenizer', 3330)
runner2 = Runner('counter', 3331)
# call individual methods
runner1.call('tokenize',
{'value': 'hello world'}) # {'value': ['hello', 'world']}
runner2.call(
'count',
{'value': ['hello', 'world']}) # {'value': {'hello': 1, 'world': 1}}
# create and call chain
"""
import io
import PIL
from acumos.modeling import Model, create_namedtuple
from acumos.session import AcumosSession
ImageShape = create_namedtuple('ImageShape', [('width', int), ('height', int)])
def get_format(data: bytes) -> str:
'''Returns the format of an image'''
buffer = io.BytesIO(data)
img = PIL.Image.open(buffer)
return img.format
def get_shape(data: bytes) -> ImageShape:
'''Returns the width and height of an image'''
buffer = io.BytesIO(data)
img = PIL.Image.open(buffer)
shape = ImageShape(width=img.width, height=img.height)
return shape
model = Model(get_format=get_format, get_shape=get_shape)
session = AcumosSession()
session.dump(model, 'image-model', '.') # creates ./image-model
y_train = train[:, n_pixels // 2:]
X_test = test[:, :(n_pixels + 1) // 2]
y_test = test[:, n_pixels // 2:]
knn = KNeighborsRegressor()
knn.fit(X_train, y_train)
# =============================================================================
# Acumos specific code
# =============================================================================
# represents a single "flattened" [1 x n] image array
FlatImage = create_namedtuple('FlatImage', [('image', List[float])])
# represents a collection of flattened image arrays
FlatImages = create_namedtuple('FlatImages', [('images', List[FlatImage])])
def complete_faces(images: FlatImages) -> FlatImages:
'''Predicts the bottom half of each input image'''
X = np.vstack(images).squeeze(
) # creates an [m x n] matrixs with m images and n pixels
yhat = knn.predict(X)
return FlatImages([FlatImage(row) for row in yhat])
model = Model(complete_faces=complete_faces)
session = AcumosSession()
session.dump(model, 'face-model', '.') # creates ./face-model
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ===============LICENSE_END=========================================================
'''
Dumps an example model for illustrating acumos_model_runner usage
'''
from acumos.session import AcumosSession
from acumos.modeling import Model, new_type
# allow users to specify a "raw" bytes type. no protobuf message is generated here
Image = new_type(bytes, 'Image', {
'dcae_input_name': 'a',
'dcae_output_name': 'a'
}, 'example description')
def image_func(image: Image) -> Image:
'''Return an image'''
return Image(image)
if __name__ == '__main__':
'''Main'''
model = Model(imgae_func=image_func)
session = AcumosSession()
session.dump_zip(model, 'raw', './raw.zip',
replace=True) # creates ./raw.zip
assert (res.f < 1.0)
print('Test square')
ci = SquareMessage(2.0)
res = square(ci)
assert (res.d >= ci.d * ci.d)
print('Test subtract')
ci = ComputeInput(1.0, 2.0, "string")
res = subtract(ci)
assert (res.f < ci.f1 and res.f < ci.f2)
# Dump and on-board the methods as models
# This relies on the user entering a password on the command line
push = 'http://cognita-dev1-vm01-core.eastus.cloudapp.azure.com:8090/onboarding-app/v2/models'
auth = 'http://cognita-dev1-vm01-core.eastus.cloudapp.azure.com:8090/onboarding-app/v2/auth'
session = AcumosSession(push_api=push, auth_api=auth)
for f in add, average, concatenate, classify, ingest, manipulate, multiply, output, padd, paverage, pmultiply, poutput, predict, psubtract, square, subtract:
d = {f.__name__: f}
model = Model(**d)
subdir = 'dump_' + f.__name__
# if the dump dir exists, assume it was pushed also
if os.path.isdir(subdir):
print('Found dump of model {}, skipping'.format(f.__name__))
else:
print('Dumping model {}'.format(f.__name__))
session.dump(model, subdir, '.')
print('Pushing model {}'.format(f.__name__))
session.push(model, f.__name__)
from acumos.modeling import Model, List, Dict, create_namedtuple, create_dataframe
from acumos.session import AcumosSession
# replace these fake APIs with ones appropriate for your instance!
session = AcumosSession(push_api="https://my.acumos.instance.com/upload",
auth_api="https://my.acumos.instance.com/auth")
