def main(argv=None): parser = argparse.ArgumentParser(description='SageMaker Training Job') parser.add_argument('--s3TestingData', type=str, help='Training Dataset.') parser.add_argument('--s3OutputData', type=str, help='Location to place training results.') parser.add_argument('--s3ModelData', type=str, help='Location for Model Artifacts') parser.add_argument('--ModelName', type=str, help='Location for Model Artifacts') args = parser.parse_args() training_data_file = utils.s3_get_file(args.S3TrainingData) print('Starting the training.') try: # read in training data raw_data = pd.read_csv(training_data_file) # pre process the training data x_train, y_train, vocab_size, max_length = data_process(raw_data) # create keras classifier and fit the model optimized_classifier = generate_model(x_train, y_train, vocab_size, max_length) # make predictions with training data predictions = optimized_classifier.predict(y_train) # save the model to the hard drive optimized_classifier.model.save( os.path.join(model_path, args.ModelName)) # upload model to s3 utils.s3_upload_file(args.S3ModelData, os.path.join(model_path, args.ModelName)) print('Training is complete.') except Exception as e: # Write out an error file. This will be returned as the failure # Reason in the DescribeTrainingJob result. trc = traceback.format_exc() with open(os.path.join(output_path, 'failure'), 'w') as s: s.write('Exception during training: ' + str(e) + '\n' + trc) # Printing this causes the exception to be in the training job logs print('Exception during training: ' + str(e) + '\n' + trc, file=sys.stderr) # A non-zero exit code causes the training job to be marked as Failed. sys.exit(255)
def main(argv=None): parser = argparse.ArgumentParser(description='SageMaker Training Job') parser.add_argument('--s3RawData', type=str, help='Training Dataset.') parser.add_argument('--s3TrainingData', type=str, help='Testing Dataset.') parser.add_argument('--s3TestingData', type=str, help='Location to place training results.') args = parser.parse_args() data_file = utils.s3_get_file(args.s3RawData) print('Starting the preprocess.') try: # read in training data train_data = pd.read_csv(data_file) # pre process the training data X_data, y_data, vocab_size, max_length = data_process(raw_data) X_train, X_test, y_train, y_test = train_test_split(X_data, y_data, test_size=0.33, random_state=12345) train = pd.concat([X_train, y_train], axis=1, join='inner') test = pd.concat([X_test, y_test], axis=1, join='inner') # give columns numeric names cols = [str(i) for i in list(range(max_length + 1))] cols[0] = 'sentiment' train.columns = cols test.columns = cols # write data to local disk train.to_csv(os.path.join(model_path, 'train.csv'), sep=',') test.to_csv(os.path.join(model_path, 'test.csv'), sep=',') # upload training and test data to s3 utils.s3_upload_file(args.s3TrainingData, os.path.join(model_path, 'train.csv')) utils.s3_upload_file(args.s3TestingData, os.path.join(model_path, 'test.csv')) print('Preprocess is complete.') except Exception as e: # Write out an error file. This will be returned as the failure # Reason in the DescribeTrainingJob result. trc = traceback.format_exc() with open(os.path.join(output_path, 'failure'), 'w') as s: s.write('Exception during training: ' + str(e) + '\n' + trc) # Printing this causes the exception to be in the training job logs print('Exception during training: ' + str(e) + '\n' + trc, file=sys.stderr) # A non-zero exit code causes the training job to be marked as Failed. sys.exit(255)
def main(argv=None): parser = argparse.ArgumentParser(description='Confusion Matrix Job') parser.add_argument('--s3_targets', type=str, help='Training Dataset.') parser.add_argument('--s3_predictions', type=str, help='Location to place training results.') args = parser.parse_args() try: targets_data_file = utils.s3_get_file(args.s3_testing_data, input_path) predictions_data_file = utils.s3_get_file(args.s3_testing_data, input_path) target_data = pd.read_csv(targets_data_file, index=False) prediction_data = pd.read_csv(predictions_data_file, index=False) target_sentiment = target_data['sentiment'] encoder = LabelEncoder() encoder.fit(target_sentiment) encoded_target_sentiment = encoder.transform(target_sentiment) dummy_encoded_target_sentiment = np_utils.to_categorical( encoded_target_sentiment) predicted_sentiment = encoder.inverse_transform(prediction_data) #classes = np.array(['NEGATIVE', 'NEUTRAL', 'POSITIVE'] classes = list(target_sentiment.unique()) cm = confusion_matrix(target_sentiment, predicted_sentiment, labels=classes) data = [] for target_index, target_row in enumerate(cm): for predicted_index, count in enumerate(target_row): data.append( (classes[target_index], classes[predicted_index], count)) df_cm = pd.DataFrame(data, columns=['target', 'predicted', 'count']) cm_file = os.path.join(output_path, 'confusion_matrix.csv') if not os.path.exists(output_path): os.makedirs(output_path) with file_io.FileIO(cm_file, 'w') as f: df_cm.to_csv(f, columns=['target', 'predicted', 'count'], header=False, index=False) metadata = { 'outputs': [{ 'type': 'confusion_matrix', 'format': 'csv', 'schema': [ { 'name': 'target', 'type': 'CATEGORY' }, { 'name': 'predicted', 'type': 'CATEGORY' }, { 'name': 'count', 'type': 'NUMBER' }, ], 'source': cm_file, # Convert vocab to string because for boolean values we want "True|False" to match csv data. 'labels': list(map(str, vocab)), }] } with file_io.FileIO('/mlpipeline-ui-metadata.json', 'w') as f: json.dump(metadata, f) accuracy = accuracy_score(df['target'], df['predicted']) metrics = { 'metrics': [{ 'name': 'accuracy-score', 'numberValue': accuracy, 'format': "PERCENTAGE", }] } with file_io.FileIO('/mlpipeline-metrics.json', 'w') as f: json.dump(metrics, f) except Exception as e: # Write out an error file. This will be returned as the failure # Reason in the DescribeTrainingJob result. trc = traceback.format_exc() with open(os.path.join(output_path, 'failure'), 'w') as s: s.write('Exception during training: ' + str(e) + '\n' + trc) # Printing this causes the exception to be in the training job logs print('Exception during training: ' + str(e) + '\n' + trc, file=sys.stderr) # A non-zero exit code causes the training job to be marked as Failed. sys.exit(255)
def main(argv=None): parser = argparse.ArgumentParser(description='SageMaker Training Job') parser.add_argument('--s3_training_data', type=str, help='Training Dataset.') parser.add_argument('--s3_training_predictions', type=str, help='Location to place training results.') parser.add_argument('--s3_model_artifacts', type=str, help='Location for Model Artifacts') parser.add_argument('--model_name', type=str, help='Location for Model Artifacts') parser.add_argument('--max_length', type=int, help='Location for Model Artifacts') parser.add_argument('--vocab_size', type=int, help='Location for Model Artifacts') args = parser.parse_args() training_data_file = utils.s3_get_file(args.s3_training_data, input_path) print('Starting the training.') try: # read in training data to pandas dataframe. raw_data = pd.read_csv(training_data_file) y_train = raw_data['sentiment'] # one hot encode the sentiment encoder = LabelEncoder() encoder.fit(y_train) encoded_train_y = encoder.transform(y_train) dummy_train_y = np_utils.to_categorical(encoded_train_y) print(dummy_train_y) collist = raw_data.columns.tolist() collist.remove('sentiment') x_train = raw_data[collist] print(x_train) # create keras classifier and fit the model optimized_classifier = generate_model(x_train, dummy_train_y, args.vocab_size, args.max_length) # make predictions with training data predictions = pd.DataFrame(optimized_classifier.predict(x_train)) predictions = pd.concat([y_train, predictions], axis=1, join='inner') if not os.path.exists(output_path): os.makedirs(output_path) #write predictions to file system predictions.to_csv(os.path.join(output_path, 'predictions.csv'), sep=',') if not os.path.exists(model_path): os.makedirs(model_path) # save the model to the hard drive optimized_classifier.model.save(os.path.join(model_path, '{}.h5'.format(args.model_name))) # upload model to s3 utils.s3_upload_file('{}/{}.h5'.format(args.s3_model_artifacts, args.model_name), os.path.join(model_path, '{}.h5'.format(args.model_name))) # upload training predictions utils.s3_upload_file('{}/predictions.csv'.format(args.s3_training_predictions), os.path.join(output_path, 'predictions.csv')) with open('/tmp/s3_training_predictions.txt', 'w') as f: f.write('{}/predictions.csv'.format(args.s3_training_predictions)) with open('/tmp/s3_model_artifacts.txt', 'w') as f: f.write('{}/{}.h5'.format(args.s3_model_artifacts, args.model_name)) print('Training is complete.') except Exception as e: # Write out an error file. This will be returned as the failure # Reason in the DescribeTrainingJob result. trc = traceback.format_exc() with open(os.path.join(output_path, 'failure'), 'w') as s: s.write('Exception during training: ' + str(e) + '\n' + trc) # Printing this causes the exception to be in the training job logs print( 'Exception during training: ' + str(e) + '\n' + trc, file=sys.stderr) # A non-zero exit code causes the training job to be marked as Failed. sys.exit(255)
def main(argv=None): parser = argparse.ArgumentParser(description='SageMaker Training Job') parser.add_argument('--s3_raw_data', type=str, help='Training Dataset.') parser.add_argument('--model_name', type=str, help='Model Name.') args = parser.parse_args() rightnow = datetime.datetime.now() s3url = S3Url(args.s3_raw_data) # marking a unique model run s3_model_run_id = "{}_{}{}{}{}{}{}".format(args.model_name, rightnow.year, rightnow.month, rightnow.day, rightnow.hour, rightnow.minute, rightnow.second) s3_training_predictions = "s3://{}/models/{}/data/train/pred".format( s3url.bucket, s3_model_run_id) s3_testing_predictions = "s3://{}/models/{}/data/test/pred".format( s3url.bucket, s3_model_run_id) s3_training_data = "s3://{}/models/{}/data/train/input".format( s3url.bucket, s3_model_run_id) s3_testing_data = "s3://{}/models/{}/data/test/input".format( s3url.bucket, s3_model_run_id) s3_model_artifacts = "s3://{}/models/{}/artifacts".format( s3url.bucket, s3_model_run_id) data_file = utils.s3_get_file(args.s3_raw_data, input_path) print('Starting the preprocess.') try: # read in training data raw_data = pd.read_csv(data_file) # pre process the training data X_data, y_data, vocab_size, max_length = data_process(raw_data) X_train, X_test, y_train, y_test = train_test_split(X_data, y_data, test_size=0.33, random_state=12345) X_train = pd.DataFrame(X_train) X_test = pd.DataFrame(X_test) train = pd.concat([X_train, y_train], axis=1, join='inner') test = pd.concat([X_test, y_test], axis=1, join='inner') # give columns numeric names cols = [str(i) for i in list(range(max_length + 1))] cols[max_length] = 'sentiment' train.columns = cols test.columns = cols if not os.path.exists(output_path): os.makedirs(output_path) # write data to local disk train.to_csv(os.path.join(output_path, 'train.csv'), sep=',', index=False) test.to_csv(os.path.join(output_path, 'test.csv'), sep=',', index=False) # upload training and test data to s3 utils.s3_upload_file('{}/train.csv'.format(s3_training_data), os.path.join(output_path, 'train.csv')) utils.s3_upload_file('{}/test.csv'.format(s3_testing_data), os.path.join(output_path, 'test.csv')) with open('/tmp/s3_training_predictions.txt', 'w') as f: f.write(s3_training_predictions) with open('/tmp/s3_testing_predictions.txt', 'w') as f: f.write(s3_testing_predictions) with open('/tmp/s3_training_data.txt', 'w') as f: f.write('{}/train.csv'.format(s3_training_data)) with open('/tmp/s3_testing_data.txt', 'w') as f: f.write('{}/test.csv'.format(s3_testing_data)) with open('/tmp/s3_model_artifacts.txt', 'w') as f: f.write(s3_model_artifacts) with open('/tmp/max_length.txt', 'w') as f: f.write(str(max_length)) with open('/tmp/vocab_size.txt', 'w') as f: f.write(str(vocab_size)) print('Preprocess is complete.') except Exception as e: # Write out an error file. This will be returned as the failure # Reason in the DescribeTrainingJob result. trc = traceback.format_exc() with open(os.path.join(output_path, 'failure'), 'w') as s: s.write('Exception during training: ' + str(e) + '\n' + trc) # Printing this causes the exception to be in the training job logs print('Exception during training: ' + str(e) + '\n' + trc, file=sys.stderr) # A non-zero exit code causes the training job to be marked as Failed. sys.exit(255)