def experiment(self): print(""" _____ _ _ | ____|_ ___ __ ___ _ __(_)_ __ ___ ___ _ __ | |_ | _| \ \/ / '_ \ / _ \ '__| | '_ ` _ \ / _ \ '_ \| __| | |___ > <| |_) | __/ | | | | | | | | __/ | | | |_ |_____/_/\_\ .__/ \___|_| |_|_| |_| |_|\___|_| |_|\__| |_| """) data_paths = self.dict_args['data_paths'] yaml_path = self.dict_args['yaml_path'] train_data_path, eval_data_path, pred_data_path = data_paths.strip( ).split(' ') # print(f"{train_data_path} | {eval_data_path} | {test_data_path}") train_args = { "cmd": "fit", "yaml_path": yaml_path, "data_path": train_data_path } eval_args = {"cmd": "evaluate", "data_path": eval_data_path} pred_args = {"cmd": "predict", "data_path": pred_data_path} Igel(**train_args) Igel(**eval_args) Igel(**pred_args)
def test_export(mock_args): """ test the export model functionality """ assert mock_args is not None Igel(**mock_args.fit) Igel(**mock_args.export) assert Constants.onnx_model_file.exists() == True
def experiment(self): """ run a whole experiment: this is a pipeline that includes fit, evaluate and predict. """ print(""" _____ _ _ | ____|_ ___ __ ___ _ __(_)_ __ ___ ___ _ __ | |_ | _| \ \/ / '_ \ / _ \ '__| | '_ ` _ \ / _ \ '_ \| __| | |___ > <| |_) | __/ | | | | | | | | __/ | | | |_ |_____/_/\_\ .__/ \___|_| |_|_| |_| |_|\___|_| |_|\__| |_| """) d = dict(self.dict_args) d.pop('cmd') if not d: default_train_data_path = './train_data.csv' default_eval_data_path = './eval_data.csv' default_test_data_path = './test_data.csv' default_yaml_path = './igel.yaml' print(f"" f"{'*' * 10} You entered interactive mode! {'*' * 10} \n" f"This is happening because you didn't enter all mandatory arguments in order to use the cli\n" f"Therefore, you will need to provide few information before proceeding.\n") train_data_path = input( f"enter path to your data: [{default_train_data_path}] ") or default_train_data_path eval_data_path = input( f"enter path to your data: [{default_eval_data_path}] ") or default_eval_data_path test_data_path = input( f"enter path to your data: [{default_test_data_path}] ") or default_test_data_path yaml_path = input( f"enter path to your yaml file: [{default_yaml_path}] ") or default_yaml_path # prepare the dict arguments: train_args = {"cmd": "fit", "yaml_path": yaml_path, "data_path": train_data_path} eval_args = {"cmd": "evaluate", "data_path": eval_data_path} pred_args = {"cmd": "predict", "data_path": test_data_path} else: data_paths = self.dict_args['data_paths'] yaml_path = self.dict_args['yaml_path'] train_data_path, eval_data_path, pred_data_path = data_paths.strip().split(' ') # print(f"{train_data_path} | {eval_data_path} | {test_data_path}") train_args = {"cmd": "fit", "yaml_path": yaml_path, "data_path": train_data_path} eval_args = {"cmd": "evaluate", "data_path": eval_data_path} pred_args = {"cmd": "predict", "data_path": pred_data_path} Igel(**train_args) Igel(**eval_args) Igel(**pred_args)
def experiment(data_paths: str, yaml_path: str) -> None: """ train, evaluate and use pre-trained model for predictions in one command """ train_data_path, eval_data_path, pred_data_path = data_paths.strip().split( " ") Igel(cmd="fit", data_path=train_data_path, yaml_path=yaml_path) Igel(cmd="evaluate", data_path=eval_data_path) Igel(cmd="predict", data_path=pred_data_path)
def test_fit(mock_args): """ test the fit model functionality """ assert mock_args is not None Igel(**mock_args.fit) assert Constants.model_results_dir.exists() == True assert Constants.description_file.exists() == True assert Constants.evaluation_file.exists() == False
def fit(self, *args, **kwargs): print(""" _____ _ _ |_ _| __ __ _(_)_ __ (_)_ __ __ _ | || '__/ _` | | '_ \| | '_ \ / _` | | || | | (_| | | | | | | | | | (_| | |_||_| \__,_|_|_| |_|_|_| |_|\__, | |___/ """) Igel(**self.dict_args)
def evaluate(self, *args, **kwargs): print(""" _____ _ _ _ | ____|_ ____ _| |_ _ __ _| |_(_) ___ _ __ | _| \ \ / / _` | | | | |/ _` | __| |/ _ \| '_ \ | |___ \ V / (_| | | |_| | (_| | |_| | (_) | | | | |_____| \_/ \__,_|_|\__,_|\__,_|\__|_|\___/|_| |_| """) Igel(**self.dict_args)
def predict(self, *args, **kwargs): print(""" ____ _ _ _ _ | _ \ _ __ ___ __| (_) ___| |_(_) ___ _ __ | |_) | '__/ _ \/ _` | |/ __| __| |/ _ \| '_ \ | __/| | | __/ (_| | | (__| |_| | (_) | | | | |_| |_| \___|\__,_|_|\___|\__|_|\___/|_| |_| """) Igel(**self.dict_args)
async def predict(data: dict = Body(...)): """ parse json data received from client, use pre-trained model to generate predictions and send them back to client """ try: logger.info( f"received request successfully, data will be parsed and used as inputs to generate predictions" ) # convert values to list in order to convert it later to pandas dataframe data = { k: [v] if not isinstance(v, list) else v for k, v in data.items() } # convert received data to dataframe df = pd.DataFrame(data, index=None) df.to_csv(temp_post_req_data_path, index=False) # use igel to generate predictions model_resutls_path = os.environ.get(Constants.model_results_path) logger.info(f"model_results path: {model_resutls_path}") if not model_resutls_path: logger.warning( f"Please provide path to the model_results directory generated by igel using the cli!" ) else: model_path = Path(model_resutls_path) / Constants.model_file description_file = (Path(model_resutls_path) / Constants.description_file) prediction_file = (Path(model_resutls_path) / Constants.prediction_file) res = Igel( cmd="predict", data_path=str(temp_post_req_data_path), model_path=model_path, description_file=description_file, prediction_file=prediction_file, ) # remove temp file: remove_temp_data_file(temp_post_req_data_path) logger.info("sending predictions back to client...") return {"prediction": res.predictions.to_numpy().tolist()} except FileNotFoundError as ex: remove_temp_data_file(temp_post_req_data_path) logger.exception(ex)
def predict(self, *args, **kwargs): print(""" ____ _ _ _ _ | _ \\ _ __ ___ __| (_) ___| |_(_) ___ _ __ | |_) | '__/ _ \\/ _` | |/ __| __| |/ _ \\| '_ \ | __/| | | __/ (_| | | (__| |_| | (_) | | | | |_| |_| \\___|\\__,_|_|\\___|\\__|_|\\___/|_| |_| """) d = dict(self.dict_args) d.pop("cmd") if not d: self._accept_user_input() Igel(**self.dict_args)
def evaluate(self, *args, **kwargs): print(""" _____ _ _ _ | ____|_ ____ _| |_ _ __ _| |_(_) ___ _ __ | _| \\ \\ / / _` | | | | |/ _` | __| |/ _ \\| '_ \ | |___ \\ V / (_| | | |_| | (_| | |_| | (_) | | | | |_____| \\_/ \\__,_|_|\\__,_|\\__,_|\\__|_|\\___/|_| |_| """) d = dict(self.dict_args) d.pop("cmd") if not d: self._accept_user_input() Igel(**self.dict_args)
def fit(self, *args, **kwargs): print(r""" _____ _ _ |_ _| __ __ _(_)_ __ (_)_ __ __ _ | || '__/ _` | | '_ \| | '_ \ / _` | | || | | (_| | | | | | | | | | (_| | |_||_| \__,_|_|_| |_|_|_| |_|\__, | |___/ """) d = dict(self.dict_args) d.pop("cmd") if not d: self._accept_user_input(yaml_needed=True) Igel(**self.dict_args)
from igel import Igel """ The goal of igel is to use ML without writing code. Therefore, the right and simplest way to use igel is from terminal. You can run ` igel fit -dp path_to_dataset -yml path_to_yaml_file`. Alternatively, you can write code if you want. This example below demonstrates how to use igel if you want to write code. However, I suggest you try and use the igel CLI. Type igel -h in your terminal to know more. =============================================================================================================== This example fits a machine learning model on the indian-diabetes dataset - default model here is the neural network and the configuration are provided in neural-network.yaml file - You can switch to random forest by providing the random-forest.yaml as the config file in the parameters """ mock_fit_params = { 'data_path': '../data/indian-diabetes/train-indians-diabetes.csv', #'data_path': './data.json', 'yaml_path': './neural-network.yaml', 'cmd': 'fit'} Igel(**mock_fit_params)
from igel import Igel """ The goal of igel is to use ML without writing code. Therefore, the right and simplest way to use igel is from terminal. You can run ` igel predict -dp path_to_dataset`. Alternatively, you can write code if you want. This example below demonstrates how to use igel if you want to write code. However, I suggest you try and use the igel CLI. Type igel -h in your terminal to know more. =============================================================================================================== This example uses the pre-fitted machine learning model to generate predictions """ mock_pred_params = { 'data_path': '../data/clustering-data/train.csv', 'cmd': 'predict' } Igel(**mock_pred_params)
def predict(data_path: str) -> None: """ Use an existing machine learning model to generate predictions """ Igel(cmd="predict", data_path=data_path)
def evaluate(data_path: str) -> None: """ Evaluate the performance of an existing machine learning model """ Igel(cmd="evaluate", data_path=data_path)
def fit(data_path: str, yaml_path: str) -> None: """ fit/train a machine learning model """ Igel(cmd="fit", data_path=data_path, yaml_path=yaml_path)
def export(model_path: str) -> None: """ Export an existing machine learning model to ONNX """ Igel(cmd="export", model_path=model_path)
import sys import json from igel import Igel arg = sys.argv[1] payload = json.loads(arg) print(f"Executing {payload['cmd']} command") print(f"path to the data \n {payload['data_path']}") print(f"path to the yaml file \n {payload['yaml_path']}") Igel(**payload) sys.stdout.flush()
from igel import Igel """ The goal of igel is to use ML without writing code. Therefore, the right and simplest way to use igel is from terminal. You can run ` igel evaluate -dp path_to_dataset`. Alternatively, you can write code if you want. This example below demonstrates how to use igel if you want to write code. However, I suggest you try and use the igel CLI. Type igel -h in your terminal to know more. """ mock_eval_params = { 'data_path': '../data/iris/eval-Iris.csv', 'cmd': 'evaluate' } Igel(**mock_eval_params)