def _upload_file_to_s3(self, bucket_name, file_path, s3_path): logging.debug("Start uploading %s to bucket=%s path=%s", file_path, bucket_name, s3_path) metadata = {} if os.path.getsize(file_path) < 64 * 1024 * 1024: if s3_path.endswith("txt") or s3_path.endswith("log") or s3_path.endswith("err") or s3_path.endswith("out"): metadata['ContentType'] = "text/plain; charset=utf-8" logging.info("Content type %s for file path %s", "text/plain; charset=utf-8", file_path) elif s3_path.endswith("html"): metadata['ContentType'] = "text/html; charset=utf-8" logging.info("Content type %s for file path %s", "text/html; charset=utf-8", file_path) else: logging.info("No content type provied for %s", file_path) else: if s3_path.endswith("txt") or s3_path.endswith("log") or s3_path.endswith("err") or s3_path.endswith("out"): logging.info("Going to compress file log file %s to %s", file_path, file_path + ".gz") compress_file_fast(file_path, file_path + ".gz") file_path += ".gz" s3_path += ".gz" else: logging.info("Processing file without compression") logging.info("File is too large, do not provide content type") self.client.upload_file(file_path, bucket_name, s3_path, ExtraArgs=metadata) logging.info("Upload %s to %s. Meta: %s", file_path, s3_path, metadata) # last two replacements are specifics of AWS urls: https://jamesd3142.wordpress.com/2018/02/28/amazon-s3-and-the-plus-symbol/ return "https://s3.amazonaws.com/{bucket}/{path}".format(bucket=bucket_name, path=s3_path).replace('+', '%2B').replace(' ', '%20')
def _upload_file_to_s3(self, bucket_name, file_path, s3_path): logging.debug( "Start uploading %s to bucket=%s path=%s", file_path, bucket_name, s3_path ) metadata = {} if os.path.getsize(file_path) < 64 * 1024 * 1024: if ( s3_path.endswith("txt") or s3_path.endswith("log") or s3_path.endswith("err") or s3_path.endswith("out") ): metadata["ContentType"] = "text/plain; charset=utf-8" logging.info( "Content type %s for file path %s", "text/plain; charset=utf-8", file_path, ) elif s3_path.endswith("html"): metadata["ContentType"] = "text/html; charset=utf-8" logging.info( "Content type %s for file path %s", "text/html; charset=utf-8", file_path, ) elif s3_path.endswith("css"): metadata["ContentType"] = "text/css; charset=utf-8" logging.info( "Content type %s for file path %s", "text/css; charset=utf-8", file_path, ) elif s3_path.endswith("js"): metadata["ContentType"] = "text/javascript; charset=utf-8" logging.info( "Content type %s for file path %s", "text/css; charset=utf-8", file_path, ) else: logging.info("No content type provied for %s", file_path) else: if re.search(r"\.(txt|log|err|out)$", s3_path) or re.search( r"\.log\..*(?<!\.gz)$", s3_path ): logging.info( "Going to compress file log file %s to %s", file_path, file_path + ".gz", ) compress_file_fast(file_path, file_path + ".gz") file_path += ".gz" s3_path += ".gz" else: logging.info("Processing file without compression") logging.info("File is too large, do not provide content type") self.client.upload_file(file_path, bucket_name, s3_path, ExtraArgs=metadata) logging.info("Upload %s to %s. Meta: %s", file_path, s3_path, metadata) # last two replacements are specifics of AWS urls: # https://jamesd3142.wordpress.com/2018/02/28/amazon-s3-and-the-plus-symbol/ url = f"{self.download_host}/{bucket_name}/{s3_path}" return url.replace("+", "%2B").replace(" ", "%20")