def fetchFileFromS3viaLocalApi(s3FullPath, s3Direct=False):
"""
Takes S3fullPath and downloads the file and return localFileLocation
:param s3FullPath: S3 file location to download
:return: localFileLocation of the downloaded file
"""
try:
s3util = S3Util.S3LocalUtility()
bucketName, s3filePath = parsefilePath(s3FullPath)
localFileLocation = s3util.s3Config.s3_local_folder_path + s3filePath
logger.info(localFileLocation)
if os.path.isfile(localFileLocation) and s3Direct is False:
return localFileLocation
if bucketName is None:
raise FileNotFoundError(
"Bucket not well defined according to format and no file was found in local file system"
+ str(localFileLocation))
makeDirectories(localFileLocation)
with FileLock(str(localFileLocation) + ".lock"):
with Lock():
if not os.path.isfile(localFileLocation) or s3Direct is True:
logger.info("Downloading file from S3: Bucket: " +
str(bucketName) + " S3 file path: " +
str(s3filePath) + " via Local API")
s3util.downloadFileFromS3(s3filePath, localFileLocation,
bucketName)
return localFileLocation
except Exception as e:
msg = "Exception While downloading file from S3 Utility: " + str(
type(e).__name__) + " " + str(e)
logger.error(msg)
raise Exception(msg)
def checkIfS3FileExist(s3FullPath):
"""
Check if S3 file exist or not
:param s3FullPath: S3 file path
:return: True/False
"""
s3util = S3Util.S3LocalUtility()
bucketName, s3filePath = parsefilePath(s3FullPath)
return s3util.ifS3FileExist(s3filePath, bucketName)
def checkIfS3FileExist(s3FullPath):
try:
s3util = S3Util.S3LocalUtility()
bucketName, s3filePath = parsefilePath(s3FullPath)
return s3util.ifS3FileExist(s3filePath, bucketName)
except Exception as e:
msg = "Exception in checkIfS3FileExist: " + str(
type(e).__name__) + " " + str(e)
logger.error(msg)
raise Exception(msg)
def saveDftoS3(df, s3FullPath, **kwargs):
"""
Saves csv file in S3
:param df: Dataframe to save to S3
:param s3FullPath: S3 path to save the csv
:param kwargs: arguments to save csv
:return:
"""
try:
s3util = S3Util.S3LocalUtility()
bucketName, s3filePath = parsefilePath(s3FullPath)
localFileLocation = s3util.s3Config.s3_local_folder_path + s3filePath
makeDirectories(localFileLocation)
df.to_csv(localFileLocation, **kwargs)
s3util.uploadFileToS3(localFileLocation, s3filePath, bucketName)
except Exception as e:
msg = "Exception While saving dataframe to csv file and uploading to S3: " + str(
type(e).__name__) + " " + str(e)
logger.error(msg)
raise Exception(msg)