def main(): sagemaker_session = LocalSession() sagemaker_session.config = {'local': {'local_code': True}} dummy_model_file = Path("dummy.model") dummy_model_file.touch() with tarfile.open("model.tar.gz", "w:gz") as tar: tar.add(dummy_model_file.as_posix()) # For local training a dummy role will be sufficient role = DUMMY_IAM_ROLE model = SKLearnModel(role=role, model_data='file://./model.tar.gz', framework_version='0.23-1', py_version='py3', source_dir='code', entry_point='inference.py') print('Deploying endpoint in local mode') print( 'Note: if launching for the first time in local mode, container image download might take a few minutes to complete.' ) predictor = model.deploy( initial_instance_count=1, instance_type='local', ) do_inference_on_local_endpoint(predictor) print('About to delete the endpoint to stop paying (if in cloud mode).') predictor.delete_endpoint(predictor.endpoint_name)
def main(parser=argparse.ArgumentParser()): import logging logging.basicConfig(level=logging.WARN) parser.add_argument('--profile', type=str) parser.add_argument('--local-mode', action='store_true') parser.add_argument('--instance-type', type=str, default=DEFAULT_INSTANCE_TYPE) parser.add_argument('--region', type=str, default=DEFAULT_REGION) parser.add_argument('--arn', type=str, default=None) parser.add_argument('--max-runtime', type=int, default=DEFAULT_RUNTIME, help='seconds') args = parser.parse_args() boto_session = Session(profile_name=args.profile, region_name=args.region) sagemaker_session = sagemaker.Session(boto_session=boto_session) role = args.arn if args.arn is not None else sagemaker.get_execution_role( sagemaker_session) hyperparameters = { 'gpu': 0 if args.instance_type.startswith('ml.p') else -1, 'mjcf': 'env/ant_simple.xml', 'action-dim': 8, 'obs-dim': 28, 'skip-step': 10, 'algorithm': 'TRPO', 'foot-list': 'right_back_foot left_back_foot front_right_foot front_left_foot' } chainer_estimator = Chainer( entry_point='train.py', source_dir='../', role=role, image_name=IMAGE, framework_version='5.0.0', sagemaker_session=LocalSession(boto_session) if args.local_mode else sagemaker_session, train_instance_count=1, train_instance_type='local' if args.local_mode else args.instance_type, hyperparameters=hyperparameters, base_job_name='roboschool-TRPO-skipstep10-continue-1', train_max_run=args.max_runtime) chainer_estimator.fit(wait=args.local_mode)
def main(parser=argparse.ArgumentParser()): import logging logging.basicConfig(level=logging.WARN) parser.add_argument('--profile', type=str, default='default') parser.add_argument('--local-mode', action='store_true') parser.add_argument('--instance-type', type=str, default=DEFAULT_INSTANCE_TYPE) parser.add_argument('--region', type=str, default=DEFAULT_REGION) parser.add_argument('--arn', type=str, default=None) parser.add_argument('--max-runtime', type=int, default=DEFAULT_RUNTIME, help='seconds') args = parser.parse_args() boto_session = Session(profile_name=args.profile, region_name=args.region) sagemaker_session = LocalSession( boto_session) if args.local_mode else sagemaker.Session(boto_session) role = args.arn if args.arn is not None else sagemaker.get_execution_role( sagemaker_session) gpu = 0 if args.instance_type.startswith( 'ml.p') and not args.local_mode else -1 hyperparameters = { 'gpu': gpu, } chainer_estimator = Chainer( entry_point='train.py', source_dir='./', role=role, image_name=IMAGE, framework_version='5.0.0', sagemaker_session=sagemaker_session, train_instance_count=1, train_instance_type='local' if args.local_mode else args.instance_type, hyperparameters=hyperparameters, base_job_name='chainer-sagemaker-sample', train_max_run=args.max_runtime) chainer_estimator.fit(wait=args.local_mode)
def sagemaker_local_session(region): return LocalSession(boto_session=boto3.Session(region_name=region))
def sagemaker_local_session(): return LocalSession(boto_session=boto3.Session(region_name=REGION_NAME))
def setUpClass(cls) -> None: cls.role = "arn:aws:iam::111111111111:role/service-role/AmazonSageMaker-ExecutionRole-20200101T000001" cls.sagemaker_session = LocalSession() cls.sagemaker_session.config = {"local": {"local_code": True}}
# 2. Docker Desktop has to be installed on your computer, and running. # 3. Open terminal and run the following commands: # docker build -t sagemaker-sklearn-rf-regressor-local container/. ######################################################################################################################## import tarfile import boto3 import pandas as pd from sagemaker import Model, LocalSession from sagemaker.deserializers import CSVDeserializer from sklearn.datasets import fetch_california_housing from sklearn.model_selection import train_test_split from sagemaker.predictor import Predictor from sagemaker.serializers import CSVSerializer sagemaker_session = LocalSession() sagemaker_session.config = {'local': {'local_code': True}} DUMMY_IAM_ROLE = 'arn:aws:iam::111111111111:role/service-role/AmazonSageMaker-ExecutionRole-20200101T000001' s3 = boto3.client('s3') def main(): image_name = "sagemaker-sklearn-rf-regressor-local" # Prepare data for model inference - we use the Boston housing dataset print('Preparing data for model inference') data = fetch_california_housing() X_train, X_test, y_train, y_test = train_test_split(data.data, data.target,
def page(state): # Streamlit session state print(state.mlflow_res.shape) # Delete any existing mlflow experiments naned '0' try: mlflow.delete_experiment('0') except: pass # Create a new experiment with unique ID exp_uniq_name = create_mlflow_exp() # Title and Description st.title('Training') st.markdown(""" This page guides you through the selection of algorithms and hyperparameters to tune the algorithms **Start by loading the data**. """) data_file = st.file_uploader("Upload data...", type="csv", key='train_data') try: text_io = io.TextIOWrapper(data_file) except: pass if data_file is not None: W = pd.read_csv(data_file) y = W.iloc[:, 0] X = W.iloc[:, 1:] st.write(W) # Options to select Model st.subheader('Select Training Method') model_types = ['XGBoost'] sel_model = st.radio('Which model to use for training?', model_types) params = {} if sel_model == 'XGBoost': # Get parameters for XGBoost params['num-rounds'] = st.sidebar.number_input( label="Number of boosting rounds or trees", min_value=5, max_value=100, value=10, step=1) params['max-depth'] = st.sidebar.number_input( label="Maximum depth of trees", min_value=2, max_value=10, value=2, step=1) params['eta'] = st.sidebar.number_input( label="Step size shrinkage for each boosting step", min_value=0.05, max_value=0.5, value=0.1, step=0.05) params['subsample'] = st.sidebar.number_input( label= "Subsample ratio of the training instances. Setting it to 0.5 means that XGBoost would randomly sample half of the training data prior to growing trees. and this will prevent overfitting", min_value=0.0, max_value=1.0, value=0.75, step=0.05) params['colsample-bytree'] = st.sidebar.number_input( label= "Subsample ratio of columns when constructing each tree. Subsampling occurs once for every tree constructed", min_value=0.0, max_value=1.0, value=0.75, step=0.05) params['num-folds'] = st.sidebar.number_input( label="Number of folds to use in cross-validation", min_value=2, max_value=20, value=10, step=1) # Sagemaker Training options instance_types = [ 'local', 'ml.m5.large', 'ml.m5.xlarge', 'ml.m5.4xlarge', 'ml.m5.24xlarge', 'ml.g4dn.xlarge', 'ml.g4dn.4xlarge', 'ml.g4dn.16xlarge' ] sagemaker_instance = st.sidebar.selectbox( 'Instance type for Sagemaker training', instance_types) if len(params) > 0: cv_submit = st.sidebar.button('Run CV') train_submit = st.sidebar.button('Run Training') best_train_submit = st.sidebar.button( 'Run Training with Best CV Parameters') print(state.best_train_submit_button) print(cv_submit) if best_train_submit and state.mlflow_res.shape[0] == 0: st.warning( 'Please run at least one CV run before training on the best CV parameters' ) if cv_submit or train_submit or best_train_submit: if sel_model == 'XGBoost': model_title = 'XGBoost' image_name = IMAGE_NAME model_name_suffix = 'penguin_xgb_model.json' if sagemaker_instance == 'local': sagemaker_session = LocalSession(boto_session=boto3.Session( region_name=REGION_NAME)) instance_type = 'local' params['categorical-columns'] = "island,sex" params['train-file'] = "penguins.csv" if cv_submit: params['do-cv'] = 1 else: params['do-cv'] = 0 if best_train_submit: state.best_train_submit_button = True param_names = [ col.split('params.')[1] for col in state.mlflow_res.columns if col.startswith('params.') ] for key in param_names: params[key] = state.mlflow_res.loc[ state.mlflow_res.index[state.sel_best_run] - 1, 'params.' + key] print(1) train_res = do_train(W, project_dir, sagemaker_session, instance_type, role=ROLE, image_name=IMAGE_NAME, params=params, model_name_suffix=model_name_suffix) # Select mlflow runs mlflow_res = mlflow.search_runs() logger.info(mlflow_res.columns) sel_cols = ['run_id', 'experiment_id'] sel_cols.extend([ col for col in mlflow_res.columns if col.startswith(('metrics', 'params')) ]) logger.info(sel_cols) mlflow_res = mlflow_res.loc[:, sel_cols] mlflow_res = mlflow_res.set_index('run_id') mlflow_res.index = np.arange(1, len(mlflow_res) + 1) #Assign the mlflow_res variable to the state. This will persisit for the rest of the session unless modified again here state.mlflow_res = mlflow_res print(state.mlflow_res) if state.mlflow_res.shape[0] != 0: st.markdown("**Metrics and Parameters Table**") st.write(state.mlflow_res) sel_best = st.selectbox( 'Which parameters to choose for the model? Pick a row number from the table above', state.mlflow_res.index) state.sel_best_run = sel_best # Plot CV plots sel_x = st.selectbox('X-axis for the CV plot', [ col for col in state.mlflow_res.columns if col.startswith('params') ]) sel_y = st.selectbox('X-axis for the CV plot', [ col for col in state.mlflow_res.columns if col.startswith('metrics') ]) print(sel_x, sel_y) plot_scatter(state.mlflow_res, sel_x, sel_y) if state.best_train_submit_button: # Plot Confusion Matrix plot_confusion_matrix(train_res[1].get_label().astype(int), train_res[5], classes=np.asarray(list(train_res[2].keys())), title='Confusion matrix, without normalization') plot_confusion_matrix(train_res[1].get_label().astype(int), train_res[5], classes=np.asarray(list(train_res[2].keys())), normalize=True, title='Normalized confusion matrix')