def test_file(): with TemporaryDirectory() as tmpdir: print(tmpdir) data = mlrun.run.get_dataitem(tmpdir + "/test1.txt") data.put("abc") assert data.get() == b"abc", "failed put/get test" assert data.stat().size == 3, "got wrong file size" print(data.stat()) context = mlrun.get_or_create_ctx("test-file") context.artifact_path = tmpdir k1 = context.log_artifact("k1", body="abc", local_path="x.txt") k2 = context.log_dataset("k2", df=df, format="csv", db_key="k2key") print("k2 url:", k2.uri) # test that we can get the artifact as dataitem assert k1.to_dataitem().get( encoding="utf-8") == "abc", "wrong .dataitem result" alist = listdir(tmpdir) print(alist) assert mlrun.run.get_dataitem( tmpdir).listdir() == alist, "failed listdir" expected = ["test1.txt", "x.txt", "k2.csv"] for a in expected: assert a in alist, f"artifact {a} was not generated" new_fd = mlrun.run.get_dataitem(tmpdir + "/k2.csv").as_df() assert len(new_fd) == 5, "failed dataframe test" assert (mlrun.run.get_dataitem(tmpdir + "/x.txt").get() == b"abc" ), "failed to log in file artifact" name = k2.uri artifact, _ = mlrun.artifacts.get_artifact_meta(name) print(artifact.to_yaml()) mlrun.artifacts.update_dataset_meta(artifact, extra_data={"k1": k1}, column_metadata={"age": "great"}) artifact, _ = mlrun.artifacts.get_artifact_meta(name) print(artifact.to_yaml()) assert artifact.column_metadata == { "age": "great" }, "failed artifact update test"
def apply_mlrun(model, context: mlrun.MLClientCtx = None, X_test=None, y_test=None, model_name=None, generate_test_set=True, **kwargs): """ Wrap the given model with MLRun model, saving the model's attributes and methods while giving it mlrun's additional features. examples:: model = LogisticRegression() X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) model = apply_mlrun(model, context, X_test=X_test, y_test=y_test) model.fit(X_train, y_train) :param model: The model to wrap. :param context: MLRun context to work with. If no context is given it will be retrieved via 'mlrun.get_or_create_ctx(None)' :param X_test: X test data (for accuracy and plots generation) :param y_test: y test data (for accuracy and plots generation) :param model_name: model artifact name :param generate_test_set: will generate a test_set dataset artifact :return: The model with MLRun's interface. """ if context is None: context = mlrun.get_or_create_ctx("mlrun_sklearn") kwargs["X_test"] = X_test kwargs["y_test"] = y_test kwargs["generate_test_set"] = generate_test_set mh = SKLearnModelHandler(model_name=model_name or "model", model=model, context=context) # Add MLRun's interface to the model: MLMLRunInterface.add_interface(mh, context, model_name, kwargs) return mh
def handler(context, event): ctx = get_or_create_ctx('myfunc', event=event) p1 = ctx.get_param('p1', 1) p2 = ctx.get_param('p2', 'a-string') context.logger.info( f'Run: {ctx.name} uid={ctx.uid}:{ctx.iteration} Params: p1={p1}, p2={p2}') time.sleep(1) # log scalar values (KFP metrics) ctx.log_result('accuracy', p1 * 2) ctx.log_result('latency', p1 * 3) # log various types of artifacts (and set UI viewers) ctx.log_artifact('test.txt', body=b'abc is 123') ctx.log_artifact('test.html', body=b'<b> Some HTML <b>', viewer='web-app') context.logger.info('run complete!') return ctx.to_json()
def my_job(): # load MLRUN runtime context (will be set by the runtime framework e.g. KubeFlow) context = get_or_create_ctx('train') # get parameters from the runtime context (or use defaults) p1 = context.get_param('p1', 1) p2 = context.get_param('p2', 'a-string') # access input metadata, values, files, and secrets (passwords) print(f'Run: {context.name} (uid={context.uid})') print(f'Params: p1={p1}, p2={p2}') print('accesskey = {}'.format(context.get_secret('ACCESS_KEY'))) print('file\n{}\n'.format(context.get_input('infile.txt').get())) # RUN some useful code e.g. ML training, data prep, etc. # log scalar result values (job result metrics) context.log_result('accuracy', p1 * 2) context.log_result('loss', p1 * 3) context.set_label('framework', 'sklearn') # log various types of artifacts (file, web page, table), will be versioned and visible in the UI context.log_artifact('model.txt', body=b'abc is 123', labels={'framework': 'xgboost'}) context.log_artifact('results.html', body=b'<b> Some HTML <b>', viewer='web-app') context.log_artifact( TableArtifact('dataset.csv', '1,2,3\n4,5,6\n', viewer='table', header=['A', 'B', 'C'])) # create a chart output (will show in the pipelines UI) chart = ChartArtifact('chart.html') chart.labels = {'type': 'roc'} chart.header = ['Epoch', 'Accuracy', 'Loss'] for i in range(1, 8): chart.add_row([i, i / 20 + 0.75, 0.30 - i / 20]) context.log_artifact(chart)
def apply_mlrun( model: keras.Model, context: mlrun.MLClientCtx = None, auto_log: bool = True, use_horovod: bool = None, ) -> keras.Model: """ Wrap the given model with MLRun model, saving the model's attributes and methods while giving it mlrun's additional features. :param model: The model to wrap. :param context: MLRun context to work with. If no context is given it will be retrieved via 'mlrun.get_or_create_ctx(None)' :param auto_log: Whether or not to apply MLRun's auto logging on the model. Defaulted to True. :param use_horovod: Whether or not to use horovod - a distributed training framework. Defaulted to None, meaning it will be read from context if available and if not - False. :return: The model with MLRun's interface. """ # Get parameters defaults: # # Context: if context is None: context = mlrun.get_or_create_ctx(KerasMLRunInterface.DEFAULT_CONTEXT_NAME) # # Use horovod: if use_horovod is None: use_horovod = ( context.labels.get("kind", "") == "mpijob" if context is not None else False ) # Add MLRun's interface to the model: KerasMLRunInterface.add_interface(model=model) # Initialize horovod if needed: if use_horovod is True: model.use_horovod() # Add auto-logging if needed: if auto_log: model.auto_log(context=context) return model
def apply_mlrun(model, context: mlrun.MLClientCtx = None, X_test: Union[np.ndarray, pd.DataFrame] = None, y_test: Union[np.ndarray, pd.DataFrame] = None, model_name: str = None, generate_test_set: bool = True, **kwargs): """ Wrap the given model with MLRun model, saving the model's attributes and methods while giving it mlrun's additional features. examples:: model = LGBMClassifier() X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) model = apply_mlrun(model, context, X_test=X_test, y_test=y_test) model.fit(X_train, y_train) :param model: The model which will have the fit() function wrapped :param context: MLRun context to work with. If no context is given it will be retrieved via 'mlrun.get_or_create_ctx(None)' :param X_test: X_test dataset :param y_test: y_test dataset :param model_name: The model artifact name (Optional) :param generate_test_set: Generates a test_set dataset artifact :return: The model with MLRun's interface. """ if context is None: context = mlrun.get_or_create_ctx("mlrun_lgbm") kwargs["X_test"] = X_test kwargs["y_test"] = y_test kwargs["generate_test_set"] = generate_test_set mh = LGBMModelHandler(model_name=model_name or "model", model=model, context=context) # Add MLRun's interface to the model: MLMLRunInterface.add_interface(mh, context, model_name, kwargs) return mh
def myfunction(context, event): ctx = get_or_create_ctx("nuclio-test", event=event) p1 = ctx.get_param("p1", 1) p2 = ctx.get_param("p2", "a-string") context.logger.info( f"Run: {ctx.name} uid={ctx.uid}:{ctx.iteration} Params: p1={p1}, p2={p2}" ) time.sleep(1) # log scalar values (KFP metrics) ctx.log_result("accuracy", p1 * 2) ctx.log_result("latency", p1 * 3) # log various types of artifacts (and set UI viewers) ctx.log_artifact("test.txt", body=b"abc is 123") ctx.log_artifact("test.html", body=b"<b> Some HTML <b>", viewer="web-app") context.logger.info("run complete!") return ctx.to_json()
def test_log_dask_to_azure(auth_method): verify_auth_parameters_and_configure_env(auth_method) artifact_path = "az://" + config["env"].get("AZURE_CONTAINER") + "/" A = np.random.randint(0, 100, size=(10000, 4)) df = pd.DataFrame(data=A, columns=list("ABCD")) ddf = dd.from_pandas(df, npartitions=4) context = mlrun.get_or_create_ctx("test") context.log_dataset( key="test_data", df=ddf, artifact_path=artifact_path, format="parquet", stats=False, ) print(f"testing context.artifact_path: {artifact_path}") dataitem = context.get_dataitem(f"{artifact_path}test_data.parquet") ddf2 = dataitem.as_df(df_module=dd) df2 = ddf2.compute() pd.testing.assert_frame_equal(df, df2)
def my_job(): # load MLRUN runtime context (will be set by the runtime framework e.g. KubeFlow) context = get_or_create_ctx('best_fit') criteria = context.get_param('criteria', 'accuracy') max_column = f'output.{criteria}' iter_table = context.get_object('iterations.csv').get() df = pd.read_csv(BytesIO(iter_table), encoding='utf-8') print(df.head()) #json.loads(context.get_object('iterations').get()) row = df[max_column].idxmax() context.log_result('best_row', int(df.loc[row, 'iter'])) context.log_result('best_result', df.loc[row].to_json()) context.log_artifact('model.txt', body=b'abc is 123') context.log_artifact('best_fit.html', body=bytes(('<b> best fit is: {} </b>'.format( df.loc[row].to_json())).encode()), viewer='web-app')
def __init__(self, model: Module, context: mlrun.MLClientCtx = None): """ Initialize an interface for running training and evaluation on the given parameters. :param model: The model to train / evaluate. :param context: MLRun context to use. If None, the context will be taken from 'mlrun.get_or_create_ctx()'. """ # Set the context: if context is None: context = mlrun.get_or_create_ctx(self.DEFAULT_CONTEXT_NAME) # Store the model: self._model = model self._context = context # Prepare methods parameters: self._training_set = None # type: DataLoader self._loss_function = None # type: Module self._optimizer = None # type: Optimizer self._validation_set = None # type: DataLoader self._metric_functions = None # type: List[MetricFunctionType] self._scheduler = None self._scheduler_step_frequency = None # type: int self._epochs = None # type: int self._training_iterations = None # type: int self._validation_iterations = None # type: int self._callbacks = [] # type: List[Callback] self._use_cuda = None # type: bool self._use_horovod = None # type: bool # Prepare inner attributes: self._model_in_cuda = False self._hvd = None self._training_sampler = None # type: DistributedSampler self._validation_sampler = None # type: DistributedSampler self._callbacks_handler = None # type: CallbacksHandler
def test_local_coxph_train(): ctx = get_or_create_ctx(name="tasks survive trainer") data_url = "https://raw.githubusercontent.com/mlrun/demos/0.6.x/customer-churn-prediction/WA_Fn-UseC_-Telco-Customer-Churn.csv" src = mlrun.get_dataitem(data_url) data_clean(context=ctx, src=src, cleaned_key="artifacts/inputs/cleaned-data", encoded_key="artifacts/inputs/encoded-data") fn = import_function("function.yaml") fn.run(params={ "strata_cols": ['InternetService', 'StreamingMovies', 'StreamingTV', 'PhoneService'], "encode_cols": { "Contract": "Contract", "PaymentMethod": "Payment" }, "models_dest": 'models/cox' }, inputs={"dataset": "artifacts/inputs/encoded-data.csv"}, local=True) model = load(open("models/cox/km/model.pkl", "rb")) ans = model.predict([1, 10, 30, 100, 200]) assert (list(np.around(ans, 3)) == [0.969, 0.869, 0.781, 0.668, 0.668])
# Pyspark example called by mlrun_spark_k8s.ipynb from mlrun import get_or_create_ctx from pyspark.sql import SparkSession # Acquire MLRun context mlctx = get_or_create_ctx('spark-function') # Get MLRun parameters mlctx.logger.info('!@!@!@!@!@ Getting env variables') READ_OPTIONS = mlctx.get_param('data_sources') QUERY = mlctx.get_param('query') WRITE_OPTIONS = mlctx.get_param('write_options') # Create spark session spark = SparkSession.builder.appName('Spark function').getOrCreate() # Loading data from a JDBC source for data_source in READ_OPTIONS: spark.read.load(**READ_OPTIONS[data_source]).createOrReplaceTempView(data_source) # Transform the data using SQL query spark.sql(QUERY).write.save(**WRITE_OPTIONS) # write the result datadrame to destination mlctx.logger.info('!@!@!@!@!@ Saved') spark.stop()
local_path="model.txt", labels={"framework": "xgboost"}, ) context.log_artifact( "html_result", body=b"<b> Some HTML <b>", local_path="result.html" ) # create a chart output (will show in the pipelines UI) chart = ChartArtifact("chart") chart.labels = {"type": "roc"} chart.header = ["Epoch", "Accuracy", "Loss"] for i in range(1, 8): chart.add_row([i, i / 20 + 0.75, 0.30 - i / 20]) context.log_artifact(chart) raw_data = { "first_name": ["Jason", "Molly", "Tina", "Jake", "Amy"], "last_name": ["Miller", "Jacobson", "Ali", "Milner", "Cooze"], "age": [42, 52, 36, 24, 73], "testScore": [25, 94, 57, 62, 70], } df = pd.DataFrame(raw_data, columns=["first_name", "last_name", "age", "testScore"]) context.log_dataset("mydf", df=df, stats=True) if __name__ == "__main__": context = get_or_create_ctx("train") p1 = context.get_param("p1", 1) p2 = context.get_param("p2", "a-string") my_job(context, p1, p2)
from tensorflow.keras.applications import ResNet50 from tensorflow.keras.layers import Flatten, Dense from tensorflow.keras.models import Model from tensorflow.keras.preprocessing.image import ImageDataGenerator from tensorflow.keras.optimizers import Adadelta, SGD from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint from sklearn.model_selection import train_test_split from mlrun import get_or_create_ctx from mlrun.artifacts import ChartArtifact # Acquire MLRun context and parameters: mlctx = get_or_create_ctx('trainer') DATA_PATH = mlctx.get_param('data_path') MODEL_DIR = f"{mlctx.artifact_path}/{mlctx.get_param('model_dir')}" CHECKPOINTS_DIR = f"{mlctx.artifact_path}/{mlctx.get_param('checkpoints_dir')}" IMAGE_WIDTH = mlctx.get_param('image_width', 128) IMAGE_HEIGHT = mlctx.get_param('image_height', 128) IMAGE_CHANNELS = mlctx.get_param('image_channels', 3) # RGB color IMAGE_SIZE = (IMAGE_WIDTH, IMAGE_HEIGHT) IMAGE_SHAPE = (IMAGE_WIDTH, IMAGE_HEIGHT, IMAGE_CHANNELS) EPOCHS = mlctx.get_param('epochs', 1) BATCH_SIZE = mlctx.get_param('batch_size', 16) # RANDOM_STATE must be a parameter for reproducibility: RANDOM_STATE = mlctx.get_param('random_state', 1) TEST_SIZE = mlctx.get_param('test_size', 0.2) # kubeflow outputs/inputs
# context.log_result('loss', p2 * 3) # context.set_label('framework', 'sklearn') # # log various types of artifacts (file, web page, table), will be versioned and visible in the UI # context.log_artifact('model', body=b'Omlette is a silly cat', local_path='model.txt', labels={'framework': 'xgboost'}) # context.log_artifact('html_result', body=b'<b> Some HTML <b>', local_path='result.html') # context.log_artifact(TableArtifact('dataset', '1,2,3\n4,5,6\n', visible=True, # header=['A', 'B', 'C']), local_path='dataset.csv') # # create a chart output (will show in the pipelines UI) # chart = ChartArtifact('chart') # chart.labels = {'type': 'roc'} # chart.header = ['Epoch', 'Accuracy', 'Loss'] # for i in range(1, 8): # chart.add_row([i, i/20+0.75, 0.30-i/20]) # context.log_artifact(chart) # raw_data = {'first_name': ['Jason', 'Molly', 'Tina', 'Jake', 'Amy'], # 'last_name': ['Miller', 'Jacobson', 'Ali', 'Milner', 'Cooze'], # 'age': [42, 52, 36, 24, 73], # 'testScore': [25, 94, 57, 62, 70]} # df = pd.DataFrame(raw_data, columns=[ # 'first_name', 'last_name', 'age', 'testScore']) # context.log_dataset('mydf', df=df, stats=True) if __name__ == "__main__": context = get_or_create_ctx('alexp_train') p1 = context.get_param('test_data') p2 = context.get_param('train_set_count') my_job(context, p1, p2)
# Pyspark example called by mlrun_spark_k8s.ipynb from mlrun import get_or_create_ctx from pyspark.sql import SparkSession # Acquire MLRun context mlctx = get_or_create_ctx("spark-function") # Get MLRun parameters mlctx.logger.info("!@!@!@!@!@ Getting env variables") READ_OPTIONS = mlctx.get_param("data_sources") QUERY = mlctx.get_param("query") WRITE_OPTIONS = mlctx.get_param("write_options") # Create spark session spark = SparkSession.builder.appName("Spark function").getOrCreate() # Loading data from a JDBC source for data_source in READ_OPTIONS: spark.read.load( **READ_OPTIONS[data_source]).createOrReplaceTempView(data_source) # Transform the data using SQL query spark.sql(QUERY).write.save(**WRITE_OPTIONS) # write the result datadrame to destination mlctx.logger.info("!@!@!@!@!@ Saved") spark.stop()
import random as rand # horovod import horovod.tensorflow.keras as hvd # tensorflow import tensorflow as tf from tensorflow.keras.callbacks import ReduceLROnPlateau # mlrun # from mlrun import get_or_create_ctx # from mlrun.artifacts import ChartArtifact import mlrun import mlrun.artifacts # MLRun context mlctx = mlrun.get_or_create_ctx('horovod-trainer') images = mlctx.get_param('images') annotations = mlctx.get_param('annotations') model_artifacts = mlctx.get_param('model_artifacts') # image batch and epocs IMAGE_WIDTH = mlctx.get_param('image_width', 224) IMAGE_HEIGHT = mlctx.get_param('image_height', 224) IMAGE_CHANNELS = mlctx.get_param('image_channels', 3) # RGB color IMAGE_SIZE = (IMAGE_WIDTH, IMAGE_HEIGHT) IMAGE_SHAPE = (IMAGE_WIDTH, IMAGE_HEIGHT, IMAGE_CHANNELS) EPOCHS = mlctx.get_param('epochs', 1) BS = mlctx.get_param('batch_size', 16) # parameters for reproducibility: RANDOM_STATE = mlctx.get_param('random_state', 1)
def test_log_large_dask_dataframe_to_azure(auth_method): # Create the environmental variables verify_auth_parameters_and_configure_env(auth_method) A = np.random.random_sample(size=(25000000, 6)) df = pd.DataFrame(data=A, columns=list("ABCDEF")) ddf = dd.from_pandas(df, npartitions=10).persist() size = ddf.memory_usage().sum().compute() print(f"demo data has size: {size // 1e6} MB") # Verify that the size of the dataframe is > 1GB, and so # will write a collection of files, instead of a single # file assert (size // 1e6) > 1100 # Create environmental vars context = mlrun.get_or_create_ctx("test") # Define the artifact location target_path = "az://" + config["env"].get("AZURE_CONTAINER") + "/" context.log_dataset( key="demo_data", df=ddf, format="parquet", artifact_path=target_path, stats=True, ) data_item2 = mlrun.get_dataitem(f"{target_path}demo_data.parquet") ddf2 = data_item2.as_df(df_module=dd) # Check that a collection of files is written to Azure, # rather than a single parquet file from adlfs import AzureBlobFileSystem fs = AzureBlobFileSystem( account_name=os.getenv("AZURE_STORAGE_ACCOUNT_NAME"), account_key=os.getenv("AZURE_STORAGE_ACCOUNT_KEY"), connection_string=os.getenv("AZURE_STORAGE_CONNECTION_STRING"), tenant_id=os.getenv("AZURE_STORAGE_TENANT_ID"), client_id=os.getenv("AZURE_STORAGE_CLIENT_ID"), client_secret=os.getenv("AZURE_STORAGE_CLIENT_SECRET"), sas_token=os.getenv("AZURE_STORAGE_SAS_TOKEN"), ) # Verify that a directory was created, rather than a file path = target_path.partition("//")[2] path = os.path.join(path, "demo_data.parquet") assert fs.isdir(path) is True # Verify that a collection of files was written files = fs.ls(path) assert len(files) > 4 df2 = ddf2.compute() df2 = df2.reset_index(drop=True) df = ddf.compute() df = df.reset_index(drop=True) # Verify that the returned dataframe matches the original pd.testing.assert_frame_equal(df, df2, check_index_type=False, check_less_precise=True)
print(f'Run: {ctx.name} (uid={ctx.uid})') print(f'Params: p1={p1}, p2={p2}') print('accesskey = {}'.format(ctx.get_secret('ACCESS_KEY'))) print('file\n{}\n'.format(ctx.get_object('infile.txt').get())) # log scalar values (KFP metrics) ctx.log_result('accuracy', p1 * 2) ctx.log_result('latency', p1 * 3) # log various types of artifacts (and set UI viewers) ctx.log_artifact('test.txt', body=b'abc is 123') ctx.log_artifact('test.html', body=b'<b> Some HTML <b>', viewer='web-app') table = TableArtifact('tbl.csv', '1,2,3\n4,5,6\n', viewer='table', header=['A', 'B', 'C']) ctx.log_artifact(table) chart = ChartArtifact('chart.html') chart.header = ['Hour', 'One', 'Two'] for i in range(1, 4): chart.add_row([i, 1 + 2, 2 * i]) ctx.log_artifact(chart) if __name__ == "__main__": ex = get_or_create_ctx('mytask') my_func(ex) ex.commit('aa')
#!/usr/bin/env python # coding: utf-8 # In[2]: import yaml import json from mlrun import get_or_create_ctx, run_start context = get_or_create_ctx('sparkk8s') sparkk8s_spec = context.get_param('sparkk8sspec', {}) # IGUAZIO environment AUTH_SECRET = sparkk8s_spec.get('metadata.AUTH_SECRET', "shell-um81v5npue-qh8oc-v3io-auth") FUSE_SECRET = sparkk8s_spec.get('metadata.FUSE_SECRET', "shell-um81v5npue-qh8oc-v3io-fuse") rundb = sparkk8s_spec.get('rundb', '') run_start(runtime=sparkk8s_spec, rundb=rundb, command='sparkk8s://')
from mlrun import get_or_create_ctx mlrun_ctx = get_or_create_ctx('ctx') mlrun_ctx.logger.info("I am running") mlrun_ctx.log_result('result1',12)
def apply_mlrun( model: keras.Model, model_name: str = None, model_path: str = None, model_format: str = TFKerasModelHandler.ModelFormats.SAVED_MODEL, save_traces: bool = False, modules_map: Union[Dict[str, Union[None, str, List[str]]], str] = None, custom_objects_map: Union[Dict[str, Union[str, List[str]]], str] = None, custom_objects_directory: str = None, context: mlrun.MLClientCtx = None, auto_log: bool = True, tensorboard_directory: str = None, mlrun_callback_kwargs: Dict[str, Any] = None, tensorboard_callback_kwargs: Dict[str, Any] = None, use_horovod: bool = None, ) -> TFKerasModelHandler: """ Wrap the given model with MLRun model, saving the model's attributes and methods while giving it mlrun's additional features. :param model: The model to wrap. :param model_name: The model name to use for storing the model artifact. If not given, the tf.keras.Model.name will be used. :param model_path: The model's store object path. Mandatory for evaluation (to know which model to update). :param model_format: The format to use for saving and loading the model. Should be passed as a member of the class 'ModelFormats'. Defaulted to 'ModelFormats.SAVED_MODEL'. :param save_traces: Whether or not to use functions saving (only available for the 'SavedModel' format) for loading the model later without the custom objects dictionary. Only from tensorflow version >= 2.4.0. Using this setting will increase the model saving size. :param modules_map: A dictionary of all the modules required for loading the model. Each key is a path to a module and its value is the object name to import from it. All the modules will be imported globally. If multiple objects needed to be imported from the same module a list can be given. The map can be passed as a path to a json file as well. For example: { "module1": None, # => import module1 "module2": ["func1", "func2"], # => from module2 import func1, func2 "module3.sub_module": "func3", # => from module3.sub_module import func3 } If the model path given is of a store object, the modules map will be read from the logged modules map artifact of the model. :param custom_objects_map: A dictionary of all the custom objects required for loading the model. Each key is a path to a python file and its value is the custom object name to import from it. If multiple objects needed to be imported from the same py file a list can be given. The map can be passed as a path to a json file as well. For example: { "/.../custom_optimizer.py": "optimizer", "/.../custom_layers.py": ["layer1", "layer2"] } All the paths will be accessed from the given 'custom_objects_directory', meaning each py file will be read from 'custom_objects_directory/<MAP VALUE>'. If the model path given is of a store object, the custom objects map will be read from the logged custom object map artifact of the model. Notice: The custom objects will be imported in the order they came in this dictionary (or json). If a custom object is depended on another, make sure to put it below the one it relies on. :param custom_objects_directory: Path to the directory with all the python files required for the custom objects. Can be passed as a zip file as well (will be extracted during the run before loading the model). If the model path given is of a store object, the custom objects files will be read from the logged custom object artifact of the model. :param context: MLRun context to work with. If no context is given it will be retrieved via 'mlrun.get_or_create_ctx(None)' :param auto_log: Whether or not to apply MLRun's auto logging on the model. Defaulted to True. :param tensorboard_directory: If context is not given, or if wished to set the directory even with context, this will be the output for the event logs of tensorboard. If not given, the 'tensorboard_dir' parameter will be tried to be taken from the provided context. If not found in the context, the default tensorboard output directory will be: /User/.tensorboard/<PROJECT_NAME> or if working on local, the set artifacts path. :param mlrun_callback_kwargs: Key word arguments for the MLRun callback. For further information see the documentation of the class 'MLRunLoggingCallback'. Note that both 'context' and 'auto_log' parameters are already given here. :param tensorboard_callback_kwargs: Key word arguments for the tensorboard callback. For further information see the documentation of the class 'TensorboardLoggingCallback'. Note that both 'context' and 'auto_log' parameters are already given here. :param use_horovod: Whether or not to use horovod - a distributed training framework. Defaulted to None, meaning it will be read from context if available and if not - False. :return: The model with MLRun's interface. """ # Get parameters defaults: # # Context: if context is None: context = mlrun.get_or_create_ctx( TFKerasMLRunInterface.DEFAULT_CONTEXT_NAME) # # Use horovod: if use_horovod is None: use_horovod = (context.labels.get("kind", "") == "mpijob" if context is not None else False) # Create a model handler: handler = TFKerasModelHandler( model_name=model_name, model_path=model_path, model=model, model_format=model_format, save_traces=save_traces, context=context, modules_map=modules_map, custom_objects_map=custom_objects_map, custom_objects_directory=custom_objects_directory, ) # Add MLRun's interface to the model: TFKerasMLRunInterface.add_interface(model=model) # Initialize horovod if needed: if use_horovod is True: model.use_horovod() # Add auto-logging if needed: if auto_log: # Set the kwargs dictionaries defaults: mlrun_callback_kwargs = ({} if mlrun_callback_kwargs is None else mlrun_callback_kwargs) tensorboard_callback_kwargs = ({} if tensorboard_callback_kwargs is None else tensorboard_callback_kwargs) # Add the additional parameters to Tensorboard's callback kwargs dictionary: tensorboard_callback_kwargs[ "tensorboard_directory"] = tensorboard_directory # Add the additional parameters to MLRun's callback kwargs dictionary: mlrun_callback_kwargs["model_handler"] = handler # Add the logging callbacks with the provided parameters: model.auto_log( context=context, mlrun_callback_kwargs=mlrun_callback_kwargs, tensorboard_callback_kwargs=tensorboard_callback_kwargs, ) return handler
def train( model: Module, training_set: DataLoader, loss_function: Module, optimizer: Optimizer, validation_set: DataLoader = None, metric_functions: List[MetricFunctionType] = None, scheduler=None, scheduler_step_frequency: Union[int, float, str] = "epoch", epochs: int = 1, training_iterations: int = None, validation_iterations: int = None, callbacks_list: List[Callback] = None, use_cuda: bool = True, use_horovod: bool = None, auto_log: bool = True, model_name: str = None, modules_map: Union[Dict[str, Union[None, str, List[str]]], str] = None, custom_objects_map: Union[Dict[str, Union[str, List[str]]], str] = None, custom_objects_directory: str = None, tensorboard_directory: str = None, mlrun_callback_kwargs: Dict[str, Any] = None, tensorboard_callback_kwargs: Dict[str, Any] = None, context: mlrun.MLClientCtx = None, ) -> PyTorchModelHandler: """ Use MLRun's PyTorch interface to train the model with the given parameters. For more information and further options regarding the auto logging, see 'PyTorchMLRunInterface' documentation. Notice for auto-logging: In order to log the model to MLRun, its class (torch.Module) must be in the custom objects map or the modules map. :param model: The model to train. :param training_set: A data loader for the training process. :param loss_function: The loss function to use during training. :param optimizer: The optimizer to use during the training. :param validation_set: A data loader for the validation process. :param metric_functions: The metrics to use on training and validation. :param scheduler: Scheduler to use on the optimizer at the end of each epoch. The scheduler must have a 'step' method with no input. :param scheduler_step_frequency: The frequency in which to step the given scheduler. Can be equal to one of the strings 'epoch' (for at the end of every epoch) and 'batch' (for at the end of every batch), or an integer that specify per how many iterations to step or a float percentage (0.0 < x < 1.0) for per x / iterations to step. Defaulted to 'epoch'. :param epochs: Amount of epochs to perform. Defaulted to a single epoch. :param training_iterations: Amount of iterations (batches) to perform on each epoch's training. If 'None' the entire training set will be used. :param validation_iterations: Amount of iterations (batches) to perform on each epoch's validation. If 'None' the entire validation set will be used. :param callbacks_list: The callbacks to use on this run. :param use_cuda: Whether or not to use cuda. Only relevant if cuda is available. Defaulted to True. :param use_horovod: Whether or not to use horovod - a distributed training framework. Defaulted to False. :param auto_log: Whether or not to apply auto-logging (to both MLRun and Tensorboard). Defaulted to True. IF True, the custom objects are not optional. :param model_name: The model name to use for storing the model artifact. If not given, the model's class name will be used. :param modules_map: A dictionary of all the modules required for loading the model. Each key is a path to a module and its value is the object name to import from it. All the modules will be imported globally. If multiple objects needed to be imported from the same module a list can be given. The map can be passed as a path to a json file as well. For example: { "module1": None, # => import module1 "module2": ["func1", "func2"], # => from module2 import func1, func2 "module3.sub_module": "func3", # => from module3.sub_module import func3 } If the model path given is of a store object, the modules map will be read from the logged modules map artifact of the model. :param custom_objects_map: A dictionary of all the custom objects required for loading the model. Each key is a path to a python file and its value is the custom object name to import from it. If multiple objects needed to be imported from the same py file a list can be given. The map can be passed as a path to a json file as well. For example: { "/.../custom_optimizer.py": "optimizer", "/.../custom_layers.py": ["layer1", "layer2"] } All the paths will be accessed from the given 'custom_objects_directory', meaning each py file will be read from 'custom_objects_directory/<MAP VALUE>'. If the model path given is of a store object, the custom objects map will be read from the logged custom object map artifact of the model. Notice: The custom objects will be imported in the order they came in this dictionary (or json). If a custom object is depended on another, make sure to put it below the one it relies on. :param custom_objects_directory: Path to the directory with all the python files required for the custom objects. Can be passed as a zip file as well (will be extracted during the run before loading the model). If the model path given is of a store object, the custom objects files will be read from the logged custom object artifact of the model. :param tensorboard_directory: If context is not given, or if wished to set the directory even with context, this will be the output for the event logs of tensorboard. If not given, the 'tensorboard_dir' parameter will be tried to be taken from the provided context. If not found in the context, the default tensorboard output directory will be: /User/.tensorboard/<PROJECT_NAME> or if working on local, the set artifacts path. :param mlrun_callback_kwargs: Key word arguments for the MLRun callback. For further information see the documentation of the class 'MLRunLoggingCallback'. Note that both 'context', 'custom_objects' and 'auto_log' parameters are already given here. :param tensorboard_callback_kwargs: Key word arguments for the tensorboard callback. For further information see the documentation of the class 'TensorboardLoggingCallback'. Note that both 'context' and 'auto_log' parameters are already given here. :param context: The context to use for the logs. :return: A model handler with the provided model and parameters. :raise ValueError: If 'auto_log' is set to True and one all of the custom objects or modules parameters given is None. """ # Get the context if not given: context = ( context if context is not None else mlrun.get_or_create_ctx(PyTorchMLRunInterface.DEFAULT_CONTEXT_NAME) ) # Create the model handler: handler = PyTorchModelHandler( model_name=model_name, model=model, modules_map=modules_map, custom_objects_map=custom_objects_map, custom_objects_directory=custom_objects_directory, context=context, ) # Initialize the interface: interface = PyTorchMLRunInterface(model=model, context=context) # Add auto logging: if auto_log: # Parse the custom objects and the kwargs: mlrun_callback_kwargs, tensorboard_callback_kwargs = _parse_callbacks_kwargs( handler=handler, tensorboard_directory=tensorboard_directory, mlrun_callback_kwargs=mlrun_callback_kwargs, tensorboard_callback_kwargs=tensorboard_callback_kwargs, ) # Add the logging callbacks with the provided parameters: interface.add_auto_logging_callbacks( mlrun_callback_kwargs=mlrun_callback_kwargs, tensorboard_callback_kwargs=tensorboard_callback_kwargs, ) # Train: interface.train( training_set=training_set, loss_function=loss_function, optimizer=optimizer, validation_set=validation_set, metric_functions=metric_functions, scheduler=scheduler, scheduler_step_frequency=scheduler_step_frequency, epochs=epochs, training_iterations=training_iterations, validation_iterations=validation_iterations, callbacks=callbacks_list, use_cuda=use_cuda, use_horovod=use_horovod, ) return handler
# move image to processed images directory shutil.move(imagePath, new_path) # saves computed encodings to avoid repeating computations df_x = pd.DataFrame(knownEncodings, columns=['c' + str(i).zfill(3) for i in range(128)]).reset_index(drop=True) df_y = pd.DataFrame(knownLabels, columns=['label']).reset_index(drop=True) df_details = pd.DataFrame([['initial training'] * 3] * len(df_x), columns=['imgUrl', 'camera', 'time']) df_details['time'] = [datetime.datetime.utcnow()] * len(df_x) df_details['imgUrl'] = urls data_df = pd.concat([df_x, df_y, df_details], axis=1) data_df['fileName'] = fileNames client.write(backend='kv', table=params.encodings_path, dfs=data_df, index_cols=['fileName']) encoding_path = "encoding" # with open('encodings_path.txt', 'w+') as f: # f.write('encodings') context.log_artifact('encodings_path', body=encoding_path) # os.remove('encodings_path.txt') if __name__ == '__main__': context = get_or_create_ctx('encoding') encode_images(context)
labels={'framework': 'xgboost'}, ) context.log_artifact('html_result', body=b'<b> Some HTML <b>', local_path='result.html') # create a chart output (will show in the pipelines UI) chart = ChartArtifact('chart') chart.labels = {'type': 'roc'} chart.header = ['Epoch', 'Accuracy', 'Loss'] for i in range(1, 8): chart.add_row([i, i / 20 + 0.75, 0.30 - i / 20]) context.log_artifact(chart) raw_data = { 'first_name': ['Jason', 'Molly', 'Tina', 'Jake', 'Amy'], 'last_name': ['Miller', 'Jacobson', 'Ali', 'Milner', 'Cooze'], 'age': [42, 52, 36, 24, 73], 'testScore': [25, 94, 57, 62, 70], } df = pd.DataFrame(raw_data, columns=['first_name', 'last_name', 'age', 'testScore']) context.log_dataset('mydf', df=df, stats=True) if __name__ == "__main__": context = get_or_create_ctx('train') p1 = context.get_param('p1', 1) p2 = context.get_param('p2', 'a-string') my_job(context, p1, p2)
# coding: utf-8 # In[75]: import os # In[76]: from pyspark.conf import SparkConf from pyspark.sql import SparkSession from pyspark.sql.functions import concat, col from mlrun import get_or_create_ctx mlrun_ctx = get_or_create_ctx('sparkjob') # Initiate a new Spark Session spark = SparkSession.builder.appName( "Spark Session with Default Configurations").getOrCreate() # Retrieve and view all the default Spark configurations: # conf = spark.sparkContext._conf.getAll() # print(conf) conf = spark.sparkContext._conf # In[77]: # Relative path to the NoSQL table within the parent platform data container table = os.path.join(os.getenv("V3IO_HOME_URL"), "examples", "bank") # In[72]: df2 = spark.read.format("io.iguaz.v3io.spark.sql.kv").load(table)
# to deprecate, use mlrun.mlutils.models and make this model a parameter instead: from tensorflow.keras.applications import EfficientNetB7 from tensorflow.keras.layers import Flatten, Dense from tensorflow.keras.models import Model from tensorflow.keras.preprocessing.image import ImageDataGenerator from tensorflow.keras.optimizers import Adadelta, SGD from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint from sklearn.model_selection import train_test_split from mlrun import get_or_create_ctx from mlrun.artifacts import ChartArtifact # Acquire MLRun context and parameters: mlctx = get_or_create_ctx('horovod-trainer') DATA_PATH = mlctx.get_param('data_path') MODEL_DIR = mlctx.get_param('model_dir', 'models') CHECKPOINTS_DIR = mlctx.get_param('checkpoints_dir') IMAGE_WIDTH = mlctx.get_param('image_width', 128) IMAGE_HEIGHT = mlctx.get_param('image_height', 128) IMAGE_CHANNELS = mlctx.get_param('image_channels', 3) # RGB color IMAGE_SIZE = (IMAGE_WIDTH, IMAGE_HEIGHT) IMAGE_SHAPE = (IMAGE_WIDTH, IMAGE_HEIGHT, IMAGE_CHANNELS) EPOCHS = mlctx.get_param('epochs', 1) BATCH_SIZE = mlctx.get_param('batch_size', 16) # RANDOM_STATE must be a parameter for reproducibility: RANDOM_STATE = mlctx.get_param('random_state', 1) TEST_SIZE = mlctx.get_param('test_size', 0.2) # kubeflow outputs/inputs
def evaluate( model_path: str, dataset: DataLoader, model: Module = None, loss_function: Module = None, metric_functions: List[MetricFunctionType] = None, iterations: int = None, callbacks_list: List[Callback] = None, use_cuda: bool = True, use_horovod: bool = False, auto_log: bool = True, model_name: str = None, modules_map: Union[Dict[str, Union[None, str, List[str]]], str] = None, custom_objects_map: Union[Dict[str, Union[str, List[str]]], str] = None, custom_objects_directory: str = None, mlrun_callback_kwargs: Dict[str, Any] = None, context: mlrun.MLClientCtx = None, ) -> Tuple[PyTorchModelHandler, List[MetricValueType]]: """ Use MLRun's PyTorch interface to evaluate the model with the given parameters. For more information and further options regarding the auto logging, see 'PyTorchMLRunInterface' documentation. Notice for auto-logging: In order to log the model to MLRun, its class (torch.Module) must be in the custom objects map or the modules map. :param model_path: The model's store object path. Mandatory for evaluation (to know which model to update). :param dataset: A data loader for the validation process. :param model: The model to evaluate. IF None, the model will be loaded from the given store model path. :param loss_function: The loss function to use during training. :param metric_functions: The metrics to use on training and validation. :param iterations: Amount of iterations (batches) to perform on the dataset. If 'None' the entire dataset will be used. :param callbacks_list: The callbacks to use on this run. :param use_cuda: Whether or not to use cuda. Only relevant if cuda is available. Defaulted to True. :param use_horovod: Whether or not to use horovod - a distributed training framework. Defaulted to False. :param auto_log: Whether or not to apply auto-logging to MLRun. Defaulted to True. :param model_name: The model name to use for storing the model artifact. If not given, the model's class name will be used. :param modules_map: A dictionary of all the modules required for loading the model. Each key is a path to a module and its value is the object name to import from it. All the modules will be imported globally. If multiple objects needed to be imported from the same module a list can be given. The map can be passed as a path to a json file as well. For example: { "module1": None, # => import module1 "module2": ["func1", "func2"], # => from module2 import func1, func2 "module3.sub_module": "func3", # => from module3.sub_module import func3 } If the model path given is of a store object, the modules map will be read from the logged modules map artifact of the model. :param custom_objects_map: A dictionary of all the custom objects required for loading the model. Each key is a path to a python file and its value is the custom object name to import from it. If multiple objects needed to be imported from the same py file a list can be given. The map can be passed as a path to a json file as well. For example: { "/.../custom_optimizer.py": "optimizer", "/.../custom_layers.py": ["layer1", "layer2"] } All the paths will be accessed from the given 'custom_objects_directory', meaning each py file will be read from 'custom_objects_directory/<MAP VALUE>'. If the model path given is of a store object, the custom objects map will be read from the logged custom object map artifact of the model. Notice: The custom objects will be imported in the order they came in this dictionary (or json). If a custom object is depended on another, make sure to put it below the one it relies on. :param custom_objects_directory: Path to the directory with all the python files required for the custom objects. Can be passed as a zip file as well (will be extracted during the run before loading the model). If the model path given is of a store object, the custom objects files will be read from the logged custom object artifact of the model. :param mlrun_callback_kwargs: Key word arguments for the MLRun callback. For further information see the documentation of the class 'MLRunLoggingCallback'. Note that both 'context', 'custom_objects' and 'auto_log' parameters are already given here. :param context: The context to use for the logs. :return: A tuple of: [0] = Initialized model handler with the evaluated model. [1] = The evaluation metrics results list. """ # Get the context if not given: context = ( context if context is not None else mlrun.get_or_create_ctx(PyTorchMLRunInterface.DEFAULT_CONTEXT_NAME) ) # Create the model handler: handler = PyTorchModelHandler( model_path=model_path, model_name=model_name, model=model, modules_map=modules_map, custom_objects_map=custom_objects_map, custom_objects_directory=custom_objects_directory, context=context, ) # Check if the model is needed to be loaded: if model is None: handler.load() # Initialize the interface: interface = PyTorchMLRunInterface(model=handler.model, context=context) # Add auto logging: if auto_log: # Parse the custom objects and the kwargs: mlrun_callback_kwargs, _ = _parse_callbacks_kwargs( handler=handler, tensorboard_directory=None, mlrun_callback_kwargs=mlrun_callback_kwargs, tensorboard_callback_kwargs=None, ) # Add the logging callbacks with the provided parameters: interface.add_auto_logging_callbacks( mlrun_callback_kwargs=mlrun_callback_kwargs, add_tensorboard_logger=False ) # Evaluate: return ( handler, interface.evaluate( dataset=dataset, loss_function=loss_function, metric_functions=metric_functions, iterations=iterations, callbacks=callbacks_list, use_cuda=use_cuda, use_horovod=use_horovod, ), )
# to deprecate, use mlrun.mlutils.models and make this model a parameter instead: from tensorflow.keras.applications.vgg16 import VGG16 from tensorflow.keras.layers import Flatten, Dense from tensorflow.keras.models import Model from tensorflow.keras.preprocessing.image import ImageDataGenerator from tensorflow.keras.optimizers import Adadelta from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint from sklearn.model_selection import train_test_split from mlrun import get_or_create_ctx from mlrun.artifacts import ChartArtifact # Acquire MLRun context and parameters: mlctx = get_or_create_ctx("horovod-trainer") DATA_PATH = mlctx.get_param("data_path") MODEL_DIR = mlctx.get_param("model_dir", "models") CHECKPOINTS_DIR = mlctx.get_param("checkpoints_dir") IMAGE_WIDTH = mlctx.get_param("image_width", 128) IMAGE_HEIGHT = mlctx.get_param("image_height", 128) IMAGE_CHANNELS = mlctx.get_param("image_channels", 3) # RGB color IMAGE_SIZE = (IMAGE_WIDTH, IMAGE_HEIGHT) IMAGE_SHAPE = (IMAGE_WIDTH, IMAGE_HEIGHT, IMAGE_CHANNELS) EPOCHS = mlctx.get_param("epochs", 1) BATCH_SIZE = mlctx.get_param("batch_size", 16) # RANDOM_STATE must be a parameter for reproducibility: RANDOM_STATE = mlctx.get_param("random_state", 1) TEST_SIZE = mlctx.get_param("test_size", 0.2) # kubeflow outputs/inputs
def apply_mlrun( model: keras.Model, custom_objects_map: Union[Dict[str, Union[str, List[str]]], str] = None, custom_objects_directory: str = None, context: mlrun.MLClientCtx = None, auto_log: bool = True, mlrun_callback_kwargs: Dict[str, Any] = None, tensorboard_callback_kwargs: Dict[str, Any] = None, use_horovod: bool = None, ) -> keras.Model: """ Wrap the given model with MLRun model, saving the model's attributes and methods while giving it mlrun's additional features. :param model: The model to wrap. :param custom_objects_map: A dictionary of all the custom objects required for loading the model. Each key is a path to a python file and its value is the custom object name to import from it. If multiple objects needed to be imported from the same py file a list can be given. The map can be passed as a path to a json file as well. For example: { "/.../custom_optimizer.py": "optimizer", "/.../custom_layers.py": ["layer1", "layer2"] } All the paths will be accessed from the given 'custom_objects_directory', meaning each py file will be read from 'custom_objects_directory/<MAP VALUE>'. If the model path given is of a store object, the custom objects map will be read from the logged custom object map artifact of the model. Notice: The custom objects will be imported in the order they came in this dictionary (or json). If a custom object is depended on another, make sure to put it below the one it relies on. :param custom_objects_directory: Path to the directory with all the python files required for the custom objects. Can be passed as a zip file as well (will be extracted during the run before loading the model). If the model path given is of a store object, the custom objects files will be read from the logged custom object artifact of the model. :param context: MLRun context to work with. If no context is given it will be retrieved via 'mlrun.get_or_create_ctx(None)' :param auto_log: Whether or not to apply MLRun's auto logging on the model. Defaulted to True. :param mlrun_callback_kwargs: Key word arguments for the MLRun callback. For further information see the documentation of the class 'MLRunLoggingCallback'. Note that both 'context' and 'auto_log' parameters are already given here. :param tensorboard_callback_kwargs: Key word arguments for the tensorboard callback. For further information see the documentation of the class 'TensorboardLoggingCallback'. Note that both 'context' and 'auto_log' parameters are already given here. :param use_horovod: Whether or not to use horovod - a distributed training framework. Defaulted to None, meaning it will be read from context if available and if not - False. :return: The model with MLRun's interface. """ # Get parameters defaults: # # Context: if context is None: context = mlrun.get_or_create_ctx( KerasMLRunInterface.DEFAULT_CONTEXT_NAME) # # Use horovod: if use_horovod is None: use_horovod = (context.labels.get("kind", "") == "mpijob" if context is not None else False) # Add MLRun's interface to the model: KerasMLRunInterface.add_interface(model=model) # Initialize horovod if needed: if use_horovod is True: model.use_horovod() # Add auto-logging if needed: if auto_log: # Set the kwargs dictionaries defaults: mlrun_callback_kwargs = ({} if mlrun_callback_kwargs is None else mlrun_callback_kwargs) tensorboard_callback_kwargs = ({} if tensorboard_callback_kwargs is None else tensorboard_callback_kwargs) # Add the custom objects to MLRun's callback kwargs dictionary: mlrun_callback_kwargs["custom_objects_map"] = custom_objects_map mlrun_callback_kwargs[ "custom_objects_directory"] = custom_objects_directory # Add the logging callbacks with the provided parameters: model.auto_log( context=context, mlrun_callback_kwargs=mlrun_callback_kwargs, tensorboard_callback_kwargs=tensorboard_callback_kwargs, ) return model