def _dumped_model(model: Model, model_name: str = 'test-model') -> str: """Dumps a model an returns its path""" with TemporaryDirectory() as dump_dir: session = AcumosSession() session.dump(model, model_name, dump_dir) model_dir = os.path.join(dump_dir, model_name) yield model_dir
def dump_model(self, CSV_filenames, model_dir): from acumos.session import AcumosSession model, req = self.generate_model(CSV_filenames) session = AcumosSession() try: if not exists(model_dir): makedirs(model_dir) session.dump(model, 'VmPredictor', model_dir, req) # creates ./my-iris.zip return True except Exception as e: print(">> Error: Model dump error {:}".format(e)) return False
def _mock_model(yield_model=True): '''Context manager that yields an acumos.wrapped.WrappedModel model for testing purposes''' def add(x: int, y: int) -> int: return x + y def multiply(x: int, y: int) -> int: return x * y model = Model(add=add, multiply=multiply) session = AcumosSession() with TemporaryDirectory() as tdir: session.dump(model, 'test-model', tdir) model_dir = os.path.join(tdir, 'test-model') wrapped_model = load_model(model_dir) yield wrapped_model if yield_model else model_dir
def test_session_dump(): '''Tests session dump''' def my_transform(x: int, y: int) -> int: return x + y model = Model(transform=my_transform) model_name = 'my-model' s = AcumosSession() with tempfile.TemporaryDirectory() as tdir: s.dump(model, model_name, tdir) model_dir = path_join(tdir, model_name) assert set(listdir(model_dir)) == set(_REQ_FILES) with pytest.raises(AcumosError): s.dump(model, model_name, tdir) # file already exists
def test_session_dump(replace: bool): '''Tests session dump''' def my_transform(x: int, y: int) -> int: return x + y model = Model(transform=my_transform) model_name = 'my-model' session = AcumosSession() with tempfile.TemporaryDirectory() as tdir: session.dump(model, model_name, tdir) model_dir = path_join(tdir, model_name) assert set(listdir(model_dir)) == set(_REQ_FILES) if replace is False: with pytest.raises(AcumosError): session.dump(model, model_name, tdir) # file already exists else: session.dump( model, model_name, tdir, replace=replace) # file already exists but it will be replaced
model = Model(loopback=echo) # Test the model function val = 'hi' out = model.loopback.inner(val) if out is None or out != val: print('Failed to get expected output, giving up.') sys.exit() else: print('model function says {}'.format(out)) # Serialize the model to support web on-boarding session = AcumosSession() subdir = 'bundle-loopback' print('dumping model to subdir {}'.format(subdir)) session.dump(model, 'loopback', subdir) # Un-comment the following lines to: # 1. configure client library with authentication # 2. on-board this model to an acumos instance #clear_jwt(); #token = 'API_TOKEN_FROM_YOUR_ACUMOS_ACCOUNT' #_configuration(jwt=token) #host='acumos-host-name.org' #print('creating session using host {}'.format(host)) #session=AcumosSession( # push_api='http://{}/upload'.format(host), # auth_api='http://{}/auth'.format(host)) #print('on-boarding model to host {}'.format(host)) #session.push(model, 'hello-world')
clf = RandomForestClassifier(random_state=0) clf.fit(X, y) # here, an appropriate NamedTuple type is inferred from a pandas DataFrame X_df = pd.DataFrame( X, columns=['sepal_length', 'sepal_width', 'petal_length', 'petal_width']) IrisDataFrame = create_dataframe('IrisDataFrame', X_df) # ================================================================================== # # or equivalently: # # IrisDataFrame = create_namedtuple('IrisDataFrame', [('sepal_length', List[float]), # ('sepal_width', List[float]), # ('petal_length', List[float]), # ('petal_width', List[float])]) # ================================================================================== def classify_iris(df: IrisDataFrame) -> List[int]: '''Returns an array of iris classifications''' X = np.column_stack(df) return clf.predict(X) model = Model(classify=classify_iris) session = AcumosSession() session.dump(model, 'iris_sklearn', '/temp')
with mlflow.start_run(): lr = ElasticNet(alpha=alpha, l1_ratio=l1_ratio, random_state=42) lr.fit(train_x, train_y) predicted_qualities = lr.predict(test_x) (rmse, mae, r2) = eval_metrics(test_y, predicted_qualities) print("Elasticnet model (alpha=%f, l1_ratio=%f):" % (alpha, l1_ratio)) print(" RMSE: %s" % rmse) print(" MAE: %s" % mae) print(" R2: %s" % r2) mlflow.log_param("alpha", alpha) mlflow.log_param("l1_ratio", l1_ratio) mlflow.log_metric("rmse", rmse) mlflow.log_metric("r2", r2) mlflow.log_metric("mae", mae) mlflow.sklearn.log_model(lr, "model") # Acumos part from acumos.modeling import Model, List, Dict, create_namedtuple, create_dataframe from acumos.session import AcumosSession, Requirements model = Model(eval=eval_metrics) #Exporting Models session = AcumosSession() session.dump(model, 'SK model with Gaia', '.')
y: target_onehot }) print("Epoch {} | Loss {}".format(epoch, loss)) prediction = tf.argmax(logits, 1) yhat = sess.run([prediction], {x: data})[0] # note: this predicts on the training set for illustration purposes only print(classification_report(target, yhat)) # ============================================================================= # create a acumos model from the tensorflow model # ============================================================================= X_df = pd.DataFrame( data, columns=['sepal_length', 'sepal_width', 'petal_length', 'petal_width']) IrisDataFrame = create_dataframe('IrisDataFrame', X_df) def classify_iris(df: IrisDataFrame) -> List[int]: '''Returns an array of iris classifications''' X = np.column_stack(df) return prediction.eval({x: X}, sess) model = Model(classify=classify_iris) session = AcumosSession() session.dump(model, 'model', '.') # creates ./model
# This file is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ===============LICENSE_END========================================================= ''' Dumps an example model for illustrating acumos_model_runner usage ''' from collections import Counter from acumos.session import AcumosSession from acumos.modeling import Model, List, Dict def add(x: int, y: int) -> int: '''Adds two numbers''' return x + y def count(strings: List[str]) -> Dict[str, int]: '''Counts the occurrences of words in `strings`''' return Counter(strings) if __name__ == '__main__': '''Main''' model = Model(add=add, count=count) session = AcumosSession() session.dump(model, 'example-model', '.')
# In[39]: cols = list( filter( lambda x: 'property' not in x and 'location' not in x and 'city' not in x, redfin.X.columns.values)) # print(test_df[:10]) print(cols) # In[40]: acumos_model = Model(appraise=appraise, appraise_multiple=appraise_multiple) if os.path.isdir(MODEL_PATH): shutil.rmtree(MODEL_PATH) session.dump(acumos_model, MODEL_PATH, '.') # session.push(model, MODEL_PATH) # usable with active credentials print('Acumos %s saved' % MODEL_PATH) # ### Sanity Check # # As a sanity check, we test Run the model on the existing training data (should be similar estimates for price). # In[41]: redfin.train_model() results = redfin.model.predict(redfin.X) n = 50 zipped_data = list(zip(list(redfin.y[:n]), results[:n])) for pair in zipped_data: print("Actual: $%.2f, Predicted: $%.2f" % (pair[0], pair[1]))
'''Test area''' def tokenize(value: str) -> List[str]: '''Segments text into tokens''' return value.split() def count(value: List[str]) -> Dict[str, int]: '''Returns a count of tokens''' return Counter(value) # define models tokenizer = Model(tokenize=tokenize) counter = Model(count=count) # save models session = AcumosSession() session.dump(tokenizer, 'tokenizer', '.') session.dump(counter, 'counter', '.') # instantiate runners runner1 = Runner('tokenizer', 3330) runner2 = Runner('counter', 3331) # call individual methods runner1.call('tokenize', {'value': 'hello world'}) # {'value': ['hello', 'world']} runner2.call( 'count', {'value': ['hello', 'world']}) # {'value': {'hello': 1, 'world': 1}} # create and call chain runner1.create_chain('count_tokens', 'tokenize', runner2, 'count')
from acumos.modeling import List from acumos.session import AcumosSession from acumos.modeling import Model if __name__ == "__main": mydata = pd.read_csv("./dataset/creditcard.csv") X = np.array(mydata.iloc[:, 2:31]) y = np.array(mydata.iloc[:, -1]) x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42) clf = tree.DecisionTreeClassifier() lr = LogisticRegression() svm = SVC() model = VotingClassifier(estimators=[('lr', lr), ('clf', clf), ('svm', svm)]) model.fit(x_train, y_train) def fraud_predict(X: List[float]) -> List[int]: ''' :param X: a list of float :return: a list of int ''' yhat = model.predict(np.array(X).reshape(1, -1)) return yhat acumos_model = Model(classify=fraud_predict) session = AcumosSession() session.dump(model, 'fraud_detection', '.')
# # class IrisDataFrame(NamedTuple): # '''DataFrame corresponding to the Iris dataset''' # sepal_length: List[float] # sepal_width: List[float] # petal_length: List[float] # petal_width: List[float] # ============================================================================= # ============================================================================= # # A pandas DataFrame can also be used to infer appropriate NamedTuple types: # # import pandas as pd # from acumos.modeling import create_dataframe # # X_df = pd.DataFrame(X, columns=['sepal_length', 'sepal_width', 'petal_length', 'petal_width']) # IrisDataFrame = create_dataframe('IrisDataFrame', X_df) # ============================================================================= def classify_iris(df: IrisDataFrame) -> List[int]: '''Returns an array of iris classifications''' X = np.column_stack(df) return clf.predict(X) model = Model(classify=classify_iris) session = AcumosSession() session.dump(model, 'my-iris', '.') # creates ./my-iris
import mlflow session = AcumosSession() def add_numbers(x: int, y: int) -> int: '''Returns the sum of x and y''' return x + y if __name__ == "__main__": warnings.filterwarnings("ignore") np.random.seed(40) a = float(sys.argv[1]) if len(sys.argv) > 1 else 1 b = float(sys.argv[2]) if len(sys.argv) > 2 else 2 with mlflow.start_run(): sum = add_numbers(a, b) print("Addition code (a=%f, b=%f):" % (a, b)) print(" Sum: %f" % sum) mlflow.log_param("a", a) mlflow.log_param("b", b) mlflow.log_metric("sum", sum) model = Model(add=add_numbers) session.dump(model, 'my-model', '.') # creates ~/my-model
""" import io import PIL from acumos.modeling import Model, create_namedtuple from acumos.session import AcumosSession ImageShape = create_namedtuple('ImageShape', [('width', int), ('height', int)]) def get_format(data: bytes) -> str: '''Returns the format of an image''' buffer = io.BytesIO(data) img = PIL.Image.open(buffer) return img.format def get_shape(data: bytes) -> ImageShape: '''Returns the width and height of an image''' buffer = io.BytesIO(data) img = PIL.Image.open(buffer) shape = ImageShape(width=img.width, height=img.height) return shape model = Model(get_format=get_format, get_shape=get_shape) session = AcumosSession() session.dump(model, 'image-model', '.') # creates ./image-model
from acumos.session import AcumosSession if __name__ == '__main__': '''Main''' iris = load_iris() X = iris.data y = iris.target clf = RandomForestClassifier(random_state=0) clf.fit(X, y) columns = ['sepallength', 'sepalwidth', 'petallength', 'petalwidth'] X_df = pd.DataFrame(X, columns=columns) DataFrame = create_dataframe('DataFrame', X_df) Predictions = create_namedtuple('Predictions', [('predictions', List[int])]) def predict(df: DataFrame) -> Predictions: '''Predicts the class of iris''' X = np.column_stack(df) yhat = clf.predict(X) preds = Predictions(predictions=yhat) return preds model = Model(transform=predict) s = AcumosSession(None) s.dump(model, 'model', '.')
y_train = train[:, n_pixels // 2:] X_test = test[:, :(n_pixels + 1) // 2] y_test = test[:, n_pixels // 2:] knn = KNeighborsRegressor() knn.fit(X_train, y_train) # ============================================================================= # Acumos specific code # ============================================================================= # represents a single "flattened" [1 x n] image array FlatImage = create_namedtuple('FlatImage', [('image', List[float])]) # represents a collection of flattened image arrays FlatImages = create_namedtuple('FlatImages', [('images', List[FlatImage])]) def complete_faces(images: FlatImages) -> FlatImages: '''Predicts the bottom half of each input image''' X = np.vstack(images).squeeze( ) # creates an [m x n] matrixs with m images and n pixels yhat = knn.predict(X) return FlatImages([FlatImage(row) for row in yhat]) model = Model(complete_faces=complete_faces) session = AcumosSession() session.dump(model, 'face-model', '.') # creates ./face-model
def keras_evaluate(config): taskComplete = False useSklearn = True model = None if useSklearn: # formulate the pipelien to be used from image_classifier.keras_model.prediction_formatter import Formatter model, reqs = model_create_pipeline(config['model_path'], config['label_path'], config['num_top_predictions']) if 'dump_model' in config and config['dump_model']: from acumos.session import AcumosSession from os import makedirs if not os.path.exists(config['dump_model']): makedirs(config['dump_model']) print("Dumping new model to '{:}'...".format(config['dump_model'])) session = AcumosSession() session.dump(model, MODEL_NAME, config['dump_model'], reqs) # creates ./my-iris.zip taskComplete = True if 'push_address' in config and config['push_address']: # and 'auth_address' in config and config['push_address']: from acumos.session import AcumosSession,Options if config['create_microservice']=='False' : if config['license_path'] is not 'None' : opts=Options(create_microservice=False,license=config['license_path']) else : opts=Options(create_microservice=False) if config['create_microservice']=='True' : if config['license_path'] is not 'None' : opts=Options(create_microservice=True,license=config['license_path']) else : opts=Options(create_microservice=True) session = AcumosSession(push_api=config['push_address']) #, auth_api=config['auth_address']) print("Pushing new model to '{:}'".format(config['push_address'])) #,auth '{:}'...".format(config['push_address'], config['auth_address'])) session.push(model, MODEL_NAME, reqs, options=opts) taskComplete = True """ Disable non-sklearn path for now else: from image_classifier.keras import inception_v4 from image_classifier.keras.image_decoder import ImageDecoder # Load test image! img = ImageDecoder.get_processed_image_keras_file(config['image']) # load image through keras # img = evaluate_image.get_processed_image_cv(config['image']) # Run prediction on test image model, model_path = inception_v4.create_model(weights='imagenet', include_top=True, model_path=model_path) preds = model.predict(img) """ preds = None if model and not taskComplete: # means we need to run a prediction/classify if not os.path.exists(config['image']) and (config['image_list'] and not os.path.exists(config['image_list'])): print("The target image/list '{}'/'{}' was not found, please check input arguments.".format(config['image'], config['image_list'])) sys.exit(-1) listImages = [] if config['image_list']: # provided a list to run with? dfImages = pd.read_csv(config['image_list'], header=None, names=['file'], delimiter=",") listImages = dfImages['file'].tolist() if len(listImages) == 0: # make a list if just one item listImages = [config['image']] wrapped_model = model # 5/31/18, simplify to reuse already created model type_in = wrapped_model.classify.input_type for idx in range(len(listImages)): curImage = listImages[idx] print("Attempting classification of image [{:}]: {:}...".format(idx, curImage)) X = create_sample(curImage) classify_in = type_in(*X) pred_raw = wrapped_model.classify.wrapped(classify_in) if len(pred_raw._fields) == 1: # type is nested?, go down one level pred_raw = getattr(pred_raw, pred_raw._fields[0]) # already a wrapped response predNew = pd.DataFrame(pred_raw) predNew[Formatter.COL_NAME_IDX] = idx if preds is None: preds = predNew else: preds = preds.append(predNew, ignore_index=True) preds.reset_index(drop=True, inplace=True) return preds
def main(config={}): from face_privacy_filter.transform_detect import FaceDetectTransform from face_privacy_filter.transform_region import RegionTransform from face_privacy_filter._version import MODEL_NAME import argparse parser = argparse.ArgumentParser() submain = parser.add_argument_group( 'main execution and evaluation functionality') submain.add_argument( '-p', '--predict_path', type=str, default='', help= "save detections from model (model must be provided via 'dump_model')") submain.add_argument( '-i', '--input', type=str, default='', help= 'absolute path to input data (image or csv, only during prediction / dump)' ) submain.add_argument('-c', '--csv_input', dest='csv_input', action='store_true', default=False, help='input as CSV format not an image') submain.add_argument('-f', '--function', type=str, default='detect', help='which type of model to generate', choices=['detect', 'pixelate']) submain.add_argument( '-s', '--suppress_image', dest='suppress_image', action='store_true', default=False, help='do not create an extra row for a returned image') subopts = parser.add_argument_group( 'model creation and configuration options') subopts.add_argument( '-a', '--push_address', help= 'server address to push the model (e.g. http://localhost:8887/v2/models)', default=os.getenv('ACUMOS_PUSH', "")) subopts.add_argument( '-A', '--auth_address', help= 'server address for login and push of the model (e.g. http://localhost:8887/v2/auth)', default=os.getenv('ACUMOS_AUTH', "")) subopts.add_argument( '-d', '--dump_model', help='dump model to a pickle directory for local running', default='') config.update(vars( parser.parse_args())) # pargs, unparsed = parser.parse_known_args() if not config['predict_path']: print("Attempting to create new model for dump or push...") elif not os.path.exists(config['input']): print( "Prediction requested but target input '{:}' was not found, please check input arguments." .format(config['input'])) sys.exit(-1) # refactor the raw samples from upstream image classifier if config['function'] == "detect": transform = FaceDetectTransform( include_image=not config['suppress_image']) pipeline, reqs = model_create_pipeline(transform, config['function'], False, True) elif config['function'] == "pixelate": transform = RegionTransform() pipeline, reqs = model_create_pipeline(transform, config['function'], True, False) else: print("Error: Functional mode '{:}' unknown, aborting create".format( config['function'])) print(pipeline) print(getattr(pipeline, config['function'])) # formulate the pipeline to be used model_name = MODEL_NAME + "_" + config['function'] if config['push_address'] and config['auth_address']: from acumos.session import AcumosSession print("Pushing new model to '{:}'...".format(config['push_address'])) session = AcumosSession(push_api=config['push_address'], auth_api=config['auth_address']) session.push(pipeline, model_name, reqs) # pushes model directly to servers if config['dump_model']: from acumos.session import AcumosSession from os import makedirs if not os.path.exists(config['dump_model']): makedirs(config['dump_model']) print("Dumping new model to '{:}'...".format(config['dump_model'])) session = AcumosSession() session.dump(pipeline, model_name, config['dump_model'], reqs) # creates model subdirectory if config['predict_path']: print("Using newly created model for local prediction...") if not config['csv_input']: inputDf = FaceDetectTransform.generate_in_df(config['input']) else: inputDf = pd.read_csv(config['input'], converters={ FaceDetectTransform.COL_IMAGE_DATA: FaceDetectTransform.read_byte_arrays }) func_action = getattr( pipeline, config['function']) # simplify to just use loaded model 6/1 pred_raw = func_action.wrapped(inputDf) transform_out = func_action.from_wrapped(pred_raw).as_wrapped() dfPred = pd.DataFrame(list(zip(*transform_out)), columns=transform_out._fields) if not config['csv_input']: dfPred = FaceDetectTransform.suppress_image(dfPred) if config['predict_path']: print("Writing prediction to file '{:}'...".format( config['predict_path'])) if not config['csv_input']: dfPred.to_csv(config['predict_path'], sep=",", index=False) else: FaceDetectTransform.generate_out_image(dfPred, config['predict_path']) if dfPred is not None: print("Predictions:\n{:}".format(dfPred))
def keras_evaluate(config): taskComplete = False useSklearn = True listImages = [] if 'image_list' in config and config['image_list']: dfImages = pd.read_csv(config['image_list'], header=None, names=['file'], delimiter=",") listImages = dfImages['file'].tolist() config['image'] = listImages[0] X = create_sample(config['image']) if useSklearn: # formulate the pipelien to be used from image_classifier.keras_model.prediction_formatter import Formatter model, reqs = model_create_pipeline(config['model_path'], config['label_path'], config['num_top_predictions']) if 'push_address' in config and 'auth_address' in config and config['push_address']: from acumos.session import AcumosSession session = AcumosSession(push_api=config['push_address'], auth_api=config['auth_address']) print("Pushing new model to upload '{:}', auth '{:}'...".format(config['push_address'], config['auth_address'])) session.push(model, MODEL_NAME, reqs) # creates ./my-iris.zip taskComplete = True if 'dump_model' in config and config['dump_model']: from acumos.session import AcumosSession from os import makedirs if not os.path.exists(config['dump_model']): makedirs(config['dump_model']) print("Dumping new model to '{:}'...".format(config['dump_model'])) session = AcumosSession() session.dump(model, MODEL_NAME, config['dump_model'], reqs) # creates ./my-iris.zip taskComplete = True preds = None if not taskComplete: # means we need to run a prediction/classify import tempfile from acumos.session import _dump_model, _copy_dir from os.path import join as path_join from acumos.wrapped import load_model if not listImages: listImages = [config['image']] preds = None # temporarily wrap model to a temp directory (to get 'wrapped' functionality) with tempfile.TemporaryDirectory() as tdir: # create temp dir with _dump_model(model, MODEL_NAME, reqs) as dump_dir: # dump model to temp dir _copy_dir(dump_dir, tdir, MODEL_NAME) # relocate for load_model below model_dir = path_join(tdir, MODEL_NAME) wrapped_model = load_model(model_dir) # load to wrapped model type_in = wrapped_model.classify._input_type for idx in range(len(listImages)): curImage = listImages[idx] print("Attempting classification of image [{:}]: {:}...".format(idx, curImage)) X = create_sample(curImage) classify_in = type_in(*tuple(col for col in X.values.T)) pred_raw = wrapped_model.classify.from_wrapped(classify_in).as_wrapped() # already a wrapped response predNew = pd.DataFrame(np.column_stack(pred_raw), columns=pred_raw._fields) predNew[Formatter.COL_NAME_IDX] = idx if preds is None: preds = predNew else: preds = preds.append(predNew, ignore_index=True) preds.reset_index(drop=True, inplace=True) """ Disable non-sklearn path for now else: from image_classifier.keras import inception_v4 from image_classifier.keras.image_decoder import ImageDecoder # Load test image! img = ImageDecoder.get_processed_image_keras_file(config['image']) # load image through keras # img = evaluate_image.get_processed_image_cv(config['image']) # Run prediction on test image model, model_path = inception_v4.create_model(weights='imagenet', include_top=True, model_path=model_path) preds = model.predict(img) """ return preds
def main(config={}): import argparse from image_mood_classifier.prediction_formatter import Formatter from image_mood_classifier._version import MODEL_NAME parser = argparse.ArgumentParser() submain = parser.add_argument_group('main execution and evaluation functionality') submain.add_argument('-p', '--predict_path', type=str, default='', help="Save predictions from model (model must be provided via 'dump_model')") submain.add_argument('-i', '--input', type=str, default='', help='Absolute path to input training data file. (for now must be a header-less CSV)') submain.add_argument('-C', '--cuda_env', type=str, default='', help='Anything special to inject into CUDA_VISIBLE_DEVICES environment string') subopts = parser.add_argument_group('model creation and configuration options') subopts.add_argument('-l', '--labels', type=str, default='', help="Path to label one-column file with one row for each input") subopts.add_argument('-m', '--model_type', type=str, default='rf', help='specify the underlying classifier type (rf (randomforest), svc (SVM))', choices=['svm', 'rf']) subopts.add_argument('-f', '--feature_nomask', dest='feature_nomask', default=False, action='store_true', help='create masked samples on input') subopts.add_argument('-n', '--add_softnoise', dest='softnoise', default=False, action='store_true', help='do not add soft noise to classification inputs') subopts.add_argument('-a', '--push_address', help='server address to push the model (e.g. http://localhost:8887/upload)', default=os.getenv('ACUMOS_PUSH', "")) subopts.add_argument('-A', '--auth_address', help='server address for login and push of the model (e.g. http://localhost:8887/auth)', default=os.getenv('ACUMOS_AUTH', "")) subopts.add_argument('-d', '--dump_model', help='dump model to a pickle directory for local running', default='') subopts.add_argument('-s', '--summary', type=int, dest='summary', default=0, help='summarize top N image classes are strong for which label class (only in training)') config.update(vars(parser.parse_args())) # pargs, unparsed = parser.parse_known_args() if not os.path.exists(config['input']): print("The target input '{:}' was not found, please check input arguments.".format(config['input'])) sys.exit(-1) print("Loading raw samples...") rawDf = pd.read_csv(config['input'], delimiter=",") # If you want to use a GPU set its index here if config['cuda_env']: os.environ['CUDA_VISIBLE_DEVICES'] = config['cuda_env'] if not config['predict_path'] and config['labels']: if not os.path.exists(config['labels']): print("The target labels '{:}' was not found, please check input arguments.".format(config['labels'])) sys.exit(-1) # refactor the raw samples from upstream image classifier print("=================input_softnoise=config['softnoise']:%s",config['softnoise']) formatter = Formatter(input_softnoise=config['softnoise']) print("=======================formater:%s", formatter) print("Loading labels to train a new model...") rawLabel = pd.read_csv(config['labels'], header=None, delimiter=",") if len(rawLabel.columns) != 1: print("Error, not currently programmed to best-of class selection to a singleton.") sys.exit(-1) rawLabel = rawLabel[0].tolist() formatter.learn_input_mapping(rawDf, "tag", "image", "score") print("=======================formater:%s", formatter) print("Converting block of {:} responses into training data, utilizing {:} images...".format(len(rawDf), len(rawLabel))) objRefactor = formatter.transform_raw_sample(rawDf, rawLabel, None if config['feature_nomask'] else Formatter.SAMPLE_GENERATE_MASKING) print("Generated {:} total samples (skip-masking: {:})".format(len(objRefactor['values']), config['feature_nomask'])) clf = model_archive() # debug helper # run summary? if config['summary']: df_combined = pd.DataFrame(objRefactor['values'], columns=objRefactor['columns']) df_combined['_labels'] = objRefactor['labels'] groupSet = df_combined.groupby('_labels') for nameG, rowsG in groupSet: df_sum = rowsG.sum(axis=0, numeric_only=True) series_top = df_sum.sort_values(ascending=False) print("Label: '{:}', top {:} classes...".format(nameG, config['summary'])) print(series_top[0:config['summary']]) # create pipeline to dump via client library if config['push_address'] or config['dump_model']: if clf is None: clf = classifier_train(objRefactor['values'], objRefactor['labels'], config['model_type']) model, reqs = model_create_pipeline(formatter, clf) model_archive(clf) # debug helper # formulate the pipeline to be used if config['push_address']: from acumos.session import AcumosSession session = AcumosSession(push_api=config['push_address'], auth_api=config['auth_address']) session.push(model, MODEL_NAME, reqs) # creates ./my-iris.zip print("Pushing new model to '{:}'...".format(config['push_address'])) if config['dump_model']: from acumos.session import AcumosSession from os import makedirs if not os.path.exists(config['dump_model']): makedirs(config['dump_model']) print("Dumping new model to '{:}'...".format(config['dump_model'])) session = AcumosSession() session.dump(model, MODEL_NAME, config['dump_model'], reqs) # creates ./my-iris.zip else: if not config['dump_model'] or not os.path.exists(config['dump_model']): print("Attempting to predict from a dumped model, but model not found.".format(config['dump_model'])) sys.exit(-1) print("Attempting predict/transform on input sample...") from acumos.wrapped import load_model model = load_model(config['dump_model']) type_in = model.classify._input_type classify_in = type_in(*tuple(col for col in rawDf.values.T)) out_wrapped = model.classify.from_wrapped(classify_in).as_wrapped() # for now, peel out top sample from classify set dfPred = pd.DataFrame(out_wrapped[0]) if config['predict_path']: print("Writing prediction to file '{:}'...".format(config['predict_path'])) dfPred.to_csv(config['predict_path'], sep=",", index=False) if dfPred is not None: print("Predictions:\n{:}".format(dfPred))
assert (res.f < 1.0) print('Test square') ci = SquareMessage(2.0) res = square(ci) assert (res.d >= ci.d * ci.d) print('Test subtract') ci = ComputeInput(1.0, 2.0, "string") res = subtract(ci) assert (res.f < ci.f1 and res.f < ci.f2) # Dump and on-board the methods as models # This relies on the user entering a password on the command line push = 'http://cognita-dev1-vm01-core.eastus.cloudapp.azure.com:8090/onboarding-app/v2/models' auth = 'http://cognita-dev1-vm01-core.eastus.cloudapp.azure.com:8090/onboarding-app/v2/auth' session = AcumosSession(push_api=push, auth_api=auth) for f in add, average, concatenate, classify, ingest, manipulate, multiply, output, padd, paverage, pmultiply, poutput, predict, psubtract, square, subtract: d = {f.__name__: f} model = Model(**d) subdir = 'dump_' + f.__name__ # if the dump dir exists, assume it was pushed also if os.path.isdir(subdir): print('Found dump of model {}, skipping'.format(f.__name__)) else: print('Dumping model {}'.format(f.__name__)) session.dump(model, subdir, '.') print('Pushing model {}'.format(f.__name__)) session.push(model, f.__name__)
iris = load_iris() X = iris.data y = iris.target clf = RandomForestClassifier(random_state=0) clf.fit(X, y) # here, an appropriate NamedTuple type is inferred from a pandas DataFrame X_df = pd.DataFrame(X, columns=['sepal_length', 'sepal_width', 'petal_length', 'petal_width']) IrisDataFrame = create_dataframe('IrisDataFrame', X_df) # ================================================================================== # # or equivalently: # # IrisDataFrame = create_namedtuple('IrisDataFrame', [('sepal_length', List[float]), # ('sepal_width', List[float]), # ('petal_length', List[float]), # ('petal_width', List[float])]) # ================================================================================== def classify_iris(df: IrisDataFrame) -> List[int]: '''Returns an array of iris classifications''' X = np.column_stack(df) return clf.predict(X) model = Model(classify=classify_iris) session = AcumosSession() session.dump(model,'iris_sklearn','/Users/guy/Desktop')