import comet_ml # Comet.ml can log training metrics, parameters, do version control and parameter optimization import os # os handles directory/workspace changes import pandas as pd # Pandas to load the data initially # import modin.pandas as pd # Optimized distributed version of Pandas import numpy as np # Mathematical operations package, allowing also for missing values representation import torch # PyTorch for tensor and deep learning operations import plotly.graph_objs as go # Plotly for interactive and pretty plots import data_utils as du # Data science and machine learning relevant methods from model_interpreter.model_interpreter import ModelInterpreter # Model interpretability class import shap # Model-agnostic interpretability package inspired on Shapley values from datetime import datetime # datetime to use proper date and time formats import pickle # Save and load Python objects du.random_seed du.set_random_seed(42) du.random_seed du.set_pandas_library(lib='pandas') import pixiedust # Debugging in Jupyter Notebook cells # Change to scripts directory os.chdir('../../scripts') import Models # Script with all the machine learning model classes # Change to parent directory (presumably "eICU-mortality-prediction") os.chdir('..')
import sys # Parse the terminal arguments parser = argparse.ArgumentParser() parser.add_argument('--comet_ml_api_key', type=str, help='Comet.ml API key') parser.add_argument('--comet_ml_workspace', type=str, help='Comet.ml workspace where data the experiment will be uploaded') parser.add_argument('--comet_ml_project_name', type=str, help='Comet.ml project name where data the experiment will be uploaded') parser.add_argument('--comet_ml_save_model', type=bool, help='Boolean to decide whether the trained models are uploaded to Comet.ml') parser.add_argument('--config_file_name', type=str, help='Name of the configuration file.') args = parser.parse_args() # Load the configuration dictionary stream_config = open(args.config_file_name, 'r') config = yaml.load(stream_config, Loader=yaml.FullLoader) print(f'DEBUG: config={config}') # Set the random seed for reproducibility du.set_random_seed(config.get('random_seed', 42)) # Add the Comet ML credentials to the configuration dictionary config['comet_ml_api_key'] = args.comet_ml_api_key config['comet_ml_workspace'] = args.comet_ml_workspace config['comet_ml_project_name'] = args.comet_ml_project_name config['comet_ml_save_model'] = args.comet_ml_save_model # Make sure that all None configuration are correctly formated as None, not a string for key, val in config.items(): if str(val).lower() == 'none': config[key] = None # Start ray # ray.init(address='auto', resources=dict(CPU=120, GPU=120)) ray.init(address='auto') print('DEBUG: Started Ray.') # NOTE: These could actually just be the current VM's resources. If it's the head node, # we might need some extra resources just to add new nodes.