def fetchFileFromS3viaLocalApi(s3FullPath, s3Direct=False):
"""
Takes S3fullPath and downloads the file and return localFileLocation
:param s3FullPath: S3 file location to download
:return: localFileLocation of the downloaded file
"""
try:
s3util = S3Util.S3LocalUtility()
bucketName, s3filePath = parsefilePath(s3FullPath)
localFileLocation = s3util.s3Config.s3_local_folder_path + s3filePath
logger.info(localFileLocation)
if os.path.isfile(localFileLocation) and s3Direct is False:
return localFileLocation
if bucketName is None:
raise FileNotFoundError(
"Bucket not well defined according to format and no file was found in local file system"
+ str(localFileLocation))
makeDirectories(localFileLocation)
with FileLock(str(localFileLocation) + ".lock"):
with Lock():
if not os.path.isfile(localFileLocation) or s3Direct is True:
logger.info("Downloading file from S3: Bucket: " +
str(bucketName) + " S3 file path: " +
str(s3filePath) + " via Local API")
s3util.downloadFileFromS3(s3filePath, localFileLocation,
bucketName)
return localFileLocation
except Exception as e:
msg = "Exception While downloading file from S3 Utility: " + str(
type(e).__name__) + " " + str(e)
logger.error(msg)
raise Exception(msg)
def checkIfS3FileExist(s3FullPath):
"""
Check if S3 file exist or not
:param s3FullPath: S3 file path
:return: True/False
"""
s3util = S3Util.S3LocalUtility()
bucketName, s3filePath = parsefilePath(s3FullPath)
return s3util.ifS3FileExist(s3filePath, bucketName)
def checkIfS3FileExist(s3FullPath):
try:
s3util = S3Util.S3LocalUtility()
bucketName, s3filePath = parsefilePath(s3FullPath)
return s3util.ifS3FileExist(s3filePath, bucketName)
except Exception as e:
msg = "Exception in checkIfS3FileExist: " + str(
type(e).__name__) + " " + str(e)
logger.error(msg)
raise Exception(msg)
def saveDftoS3(df, s3FullPath, **kwargs):
"""
Saves csv file in S3
:param df: Dataframe to save to S3
:param s3FullPath: S3 path to save the csv
:param kwargs: arguments to save csv
:return:
"""
try:
s3util = S3Util.S3LocalUtility()
bucketName, s3filePath = parsefilePath(s3FullPath)
localFileLocation = s3util.s3Config.s3_local_folder_path + s3filePath
makeDirectories(localFileLocation)
df.to_csv(localFileLocation, **kwargs)
s3util.uploadFileToS3(localFileLocation, s3filePath, bucketName)
except Exception as e:
msg = "Exception While saving dataframe to csv file and uploading to S3: " + str(
type(e).__name__) + " " + str(e)
logger.error(msg)
raise Exception(msg)
import keras
import tensorflow as tf
import common_utils.ConfigurationSetups
from prediction_service.constants import Constants
from common_utils.MLLogger import MLLogging
from threading import Lock
import prediction_service.constants
from prediction_service.deserializer_predictor.context.PipelinePredictionContext import PipelinePredictionContext, PredictionContextError
from common_utils import S3Util, ServiceHelper
import configurations
import numpy as np
s3Config = S3Util.S3Configuration()
logger = MLLogging.getLog()
class KerasDeserializerPredictor(PipelinePredictionContext):
"""
Keras Algorithms Deserializer and Predictor
"""
def __init__(self):
self.dataFrame = None
def performPredictions(self, dataFrame, pipelineFilename,
requestArguments):
"""
Perform predictions for all Keras based algorithms. This deserializer predictor requires thread and process lock to predict due to tensorflow sessions.
:param dataFrame: Dataframe to perform predictions on