import comet_ml # Comet.ml can log training metrics, parameters, do version control and parameter optimization
import os # os handles directory/workspace changes
import pandas as pd # Pandas to load the data initially
# import modin.pandas as pd # Optimized distributed version of Pandas
import numpy as np # Mathematical operations package, allowing also for missing values representation
import torch # PyTorch for tensor and deep learning operations
import plotly.graph_objs as go # Plotly for interactive and pretty plots
import data_utils as du # Data science and machine learning relevant methods
from model_interpreter.model_interpreter import ModelInterpreter # Model interpretability class
import shap # Model-agnostic interpretability package inspired on Shapley values
from datetime import datetime # datetime to use proper date and time formats
import pickle # Save and load Python objects
du.random_seed
du.set_random_seed(42)
du.random_seed
du.set_pandas_library(lib='pandas')
import pixiedust # Debugging in Jupyter Notebook cells
# Change to scripts directory
os.chdir('../../scripts')
import Models # Script with all the machine learning model classes
# Change to parent directory (presumably "eICU-mortality-prediction")
os.chdir('..')
import sys
# Parse the terminal arguments
parser = argparse.ArgumentParser()
parser.add_argument('--comet_ml_api_key', type=str, help='Comet.ml API key')
parser.add_argument('--comet_ml_workspace', type=str, help='Comet.ml workspace where data the experiment will be uploaded')
parser.add_argument('--comet_ml_project_name', type=str, help='Comet.ml project name where data the experiment will be uploaded')
parser.add_argument('--comet_ml_save_model', type=bool, help='Boolean to decide whether the trained models are uploaded to Comet.ml')
parser.add_argument('--config_file_name', type=str, help='Name of the configuration file.')
args = parser.parse_args()
# Load the configuration dictionary
stream_config = open(args.config_file_name, 'r')
config = yaml.load(stream_config, Loader=yaml.FullLoader)
print(f'DEBUG: config={config}')
# Set the random seed for reproducibility
du.set_random_seed(config.get('random_seed', 42))
# Add the Comet ML credentials to the configuration dictionary
config['comet_ml_api_key'] = args.comet_ml_api_key
config['comet_ml_workspace'] = args.comet_ml_workspace
config['comet_ml_project_name'] = args.comet_ml_project_name
config['comet_ml_save_model'] = args.comet_ml_save_model
# Make sure that all None configuration are correctly formated as None, not a string
for key, val in config.items():
if str(val).lower() == 'none':
config[key] = None
# Start ray
# ray.init(address='auto', resources=dict(CPU=120, GPU=120))
ray.init(address='auto')
print('DEBUG: Started Ray.')
# NOTE: These could actually just be the current VM's resources. If it's the head node,
# we might need some extra resources just to add new nodes.
