def fetchFileFromS3viaLocalApi(s3FullPath, s3Direct=False): """ Takes S3fullPath and downloads the file and return localFileLocation :param s3FullPath: S3 file location to download :return: localFileLocation of the downloaded file """ try: s3util = S3Util.S3LocalUtility() bucketName, s3filePath = parsefilePath(s3FullPath) localFileLocation = s3util.s3Config.s3_local_folder_path + s3filePath logger.info(localFileLocation) if os.path.isfile(localFileLocation) and s3Direct is False: return localFileLocation if bucketName is None: raise FileNotFoundError( "Bucket not well defined according to format and no file was found in local file system" + str(localFileLocation)) makeDirectories(localFileLocation) with FileLock(str(localFileLocation) + ".lock"): with Lock(): if not os.path.isfile(localFileLocation) or s3Direct is True: logger.info("Downloading file from S3: Bucket: " + str(bucketName) + " S3 file path: " + str(s3filePath) + " via Local API") s3util.downloadFileFromS3(s3filePath, localFileLocation, bucketName) return localFileLocation except Exception as e: msg = "Exception While downloading file from S3 Utility: " + str( type(e).__name__) + " " + str(e) logger.error(msg) raise Exception(msg)
def checkIfS3FileExist(s3FullPath): """ Check if S3 file exist or not :param s3FullPath: S3 file path :return: True/False """ s3util = S3Util.S3LocalUtility() bucketName, s3filePath = parsefilePath(s3FullPath) return s3util.ifS3FileExist(s3filePath, bucketName)
def checkIfS3FileExist(s3FullPath): try: s3util = S3Util.S3LocalUtility() bucketName, s3filePath = parsefilePath(s3FullPath) return s3util.ifS3FileExist(s3filePath, bucketName) except Exception as e: msg = "Exception in checkIfS3FileExist: " + str( type(e).__name__) + " " + str(e) logger.error(msg) raise Exception(msg)
def saveDftoS3(df, s3FullPath, **kwargs): """ Saves csv file in S3 :param df: Dataframe to save to S3 :param s3FullPath: S3 path to save the csv :param kwargs: arguments to save csv :return: """ try: s3util = S3Util.S3LocalUtility() bucketName, s3filePath = parsefilePath(s3FullPath) localFileLocation = s3util.s3Config.s3_local_folder_path + s3filePath makeDirectories(localFileLocation) df.to_csv(localFileLocation, **kwargs) s3util.uploadFileToS3(localFileLocation, s3filePath, bucketName) except Exception as e: msg = "Exception While saving dataframe to csv file and uploading to S3: " + str( type(e).__name__) + " " + str(e) logger.error(msg) raise Exception(msg)
import keras import tensorflow as tf import common_utils.ConfigurationSetups from prediction_service.constants import Constants from common_utils.MLLogger import MLLogging from threading import Lock import prediction_service.constants from prediction_service.deserializer_predictor.context.PipelinePredictionContext import PipelinePredictionContext, PredictionContextError from common_utils import S3Util, ServiceHelper import configurations import numpy as np s3Config = S3Util.S3Configuration() logger = MLLogging.getLog() class KerasDeserializerPredictor(PipelinePredictionContext): """ Keras Algorithms Deserializer and Predictor """ def __init__(self): self.dataFrame = None def performPredictions(self, dataFrame, pipelineFilename, requestArguments): """ Perform predictions for all Keras based algorithms. This deserializer predictor requires thread and process lock to predict due to tensorflow sessions. :param dataFrame: Dataframe to perform predictions on