def fetchFileFromS3viaLocalApi(s3FullPath, s3Direct=False): """ Takes S3fullPath and downloads the file and return localFileLocation :param s3FullPath: S3 file location to download :return: localFileLocation of the downloaded file """ try: s3util = S3Util.S3LocalUtility() bucketName, s3filePath = parsefilePath(s3FullPath) localFileLocation = s3util.s3Config.s3_local_folder_path + s3filePath logger.info(localFileLocation) if os.path.isfile(localFileLocation) and s3Direct is False: return localFileLocation if bucketName is None: raise FileNotFoundError( "Bucket not well defined according to format and no file was found in local file system" + str(localFileLocation)) makeDirectories(localFileLocation) with FileLock(str(localFileLocation) + ".lock"): with Lock(): if not os.path.isfile(localFileLocation) or s3Direct is True: logger.info("Downloading file from S3: Bucket: " + str(bucketName) + " S3 file path: " + str(s3filePath) + " via Local API") s3util.downloadFileFromS3(s3filePath, localFileLocation, bucketName) return localFileLocation except Exception as e: msg = "Exception While downloading file from S3 Utility: " + str( type(e).__name__) + " " + str(e) logger.error(msg) raise Exception(msg)
def checkIfS3FileExist(s3FullPath): """ Check if S3 file exist or not :param s3FullPath: S3 file path :return: True/False """ s3util = S3Util.S3LocalUtility() bucketName, s3filePath = parsefilePath(s3FullPath) return s3util.ifS3FileExist(s3filePath, bucketName)
def checkIfS3FileExist(s3FullPath): try: s3util = S3Util.S3LocalUtility() bucketName, s3filePath = parsefilePath(s3FullPath) return s3util.ifS3FileExist(s3filePath, bucketName) except Exception as e: msg = "Exception in checkIfS3FileExist: " + str( type(e).__name__) + " " + str(e) logger.error(msg) raise Exception(msg)
def saveDftoS3(df, s3FullPath, **kwargs): """ Saves csv file in S3 :param df: Dataframe to save to S3 :param s3FullPath: S3 path to save the csv :param kwargs: arguments to save csv :return: """ try: s3util = S3Util.S3LocalUtility() bucketName, s3filePath = parsefilePath(s3FullPath) localFileLocation = s3util.s3Config.s3_local_folder_path + s3filePath makeDirectories(localFileLocation) df.to_csv(localFileLocation, **kwargs) s3util.uploadFileToS3(localFileLocation, s3filePath, bucketName) except Exception as e: msg = "Exception While saving dataframe to csv file and uploading to S3: " + str( type(e).__name__) + " " + str(e) logger.error(msg) raise Exception(msg)