def upload_image_s3(self):
self.logger.info("Starting S3 upload")
# add credentials to URL
url = urllib.parse.urlparse(self.task["upload_uri"])
qs = urllib.parse.parse_qs(url.query)
qs["keyId"] = Setting.s3_access_key
qs["secretAccessKey"] = Setting.s3_secret_key
# setup upload logging
uploader_log = io.StringIO()
uploader_logger = logging.getLogger("uploader_log")
uploader_logger.propagate = True
uploader_logger.setLevel(logging.DEBUG)
uploader_logger.addHandler(logging.StreamHandler(stream=uploader_log))
# init and test storage
uploader_logger.info("initializing S3")
s3_storage = KiwixStorage(
urllib.parse.SplitResult(
"https",
url.netloc,
url.path,
urllib.parse.urlencode(qs, doseq=True),
url.fragment,
).geturl())
uploader_logger.debug(
f"S3 initialized for {s3_storage.url.netloc}/{s3_storage.bucket_name}"
)
# torrent
dl_url = urllib.parse.urlparse(self.task["download_uri"])
upload_torrent = "torrent" in dl_url.scheme
if upload_torrent:
parts = list(urllib.parse.urlsplit(dl_url.geturl()))
parts[0] = parts[0].replace("+torrent", "")
dl_url = urllib.parse.urlparse(urllib.parse.urlunsplit(parts))
uploader_logger.info(
f"Creating torrent file for {self.img_path.name}")
torrent_path = self.img_path.with_suffix(
f"{self.img_path.suffix}.torrent")
download_url = f"{dl_url.geturl()}/{self.img_path.name}"
torrent = torf.Torrent(
path=self.img_path,
trackers=[
"https://opentracker.xyz:443/announce",
"http://torrent.nwps.ws:80/announce",
"udp://tracker.open-internet.nl:6969/announce",
"udp://tracker.coppersurfer.tk:6969/announce",
"udp://tracker.openbittorrent.com:80/announce",
],
webseeds=[download_url],
)
torrent.generate()
torrent.write(torrent_path)
uploader_logger.info(f".. created {torrent_path.name}")
uploader_logger.info(f"Uploading {torrent_path.name}")
s3_storage.upload_file(fpath=str(torrent_path),
key=torrent_path.name)
uploader_logger.info(".. uploaded")
torrent_path.unlink()
# upload
uploader_logger.info(f"Uploading to {self.img_path.name}")
try:
hook = ImageTransferHook(output=uploader_log, fpath=self.img_path)
s3_storage.upload_file(fpath=str(self.img_path),
key=self.img_path.name,
Callback=hook)
uploaded = True
except Exception as exc:
uploaded = False
uploader_logger.error(f"uploader failed: {exc}")
uploader_logger.exception(exc)
else:
uploader_logger.info("uploader ran successfuly.")
# setting autodelete
try:
# make sure autodelete is above bucket's min retention (should be 1d)
expire_on = datetime.datetime.now() + datetime.timedelta(
days=max([2, self.task["media_duration"]]))
uploader_logger.info(f"Setting autodelete to {expire_on}")
s3_storage.set_object_autodelete_on(key=self.img_path.name,
on=expire_on)
except Exception as exc:
uploader_logger.error("Failed to set autodelete")
uploader_logger.exception(exc)
if upload_torrent:
try:
uploader_logger.info(
f"Setting torrent autodelete to {expire_on}")
s3_storage.set_object_autodelete_on(key=torrent_path.name,
on=expire_on)
except Exception as exc:
uploader_logger.error("Failed to set autodelete on torrent")
uploader_logger.exception(exc)
self.logger.info("collecting uploader log")
try:
self.logs["uploader_log"] = uploader_log.getvalue()
uploader_log.close()
except Exception as exc:
self.logger.error(f"Failed to collect logs: {exc}")
# remove image
try:
self.logger.info("removing image file: {}".format(
self.img_path.name))
self.img_path.unlink()
except Exception as exp:
self.logger.error("Unable to remove image file: {}".format(exp))
self.logger.exception(exp)
if not uploaded:
raise subprocess.SubprocessError("S3 upload failed")
def s3_upload_file(
src_path,
upload_uri,
filesize,
private_key, # not used
resume=False, # not supported
move=False, # not relevant
delete=False,
compress=False, # not relevant
bandwidth=None, # not supported
cipher=None, # not relevant
delete_after=None, # nb of days to expire upload file after
):
started_on = now()
s3_storage = KiwixStorage(rebuild_uri(upload_uri, scheme="https").geturl())
logger.debug(f"S3 initialized for {s3_storage.url.netloc}/{s3_storage.bucket_name}")
key = upload_uri.path[1:]
if upload_uri.path.endswith("/"):
key += src_path.name
try:
logger.info(f"Uploading to {key}")
hook = FileTransferHook(filename=src_path)
s3_storage.upload_file(fpath=str(src_path), key=key, Callback=hook)
print("", flush=True)
except Exception as exc:
# as there is no resume, uploading to existing URL will result in DELETE+UPLOAD
# if credentials doesn't allow DELETE or if there is an unsatisfied
# retention, will raise PermissionError
logger.error(f"uploader failed: {exc}")
logger.exception(exc)
return 1
ended_on = now()
logger.info("uploader ran successfuly.")
# setting autodelete
if delete_after is not None:
try:
# set expiration after bucket's min retention.
# bucket retention is 1d minumum.
# can be configured to loger value.
# if expiration before bucket min retention, raises 400 Bad Request
# on compliance
expire_on = (
datetime.datetime.now()
+ datetime.timedelta(days=delete_after or 1)
# adding 1mn to prevent clash with bucket's equivalent min retention
+ datetime.timedelta(seconds=60)
)
logger.info(f"Setting autodelete to {expire_on}")
s3_storage.set_object_autodelete_on(key=key, on=expire_on)
except Exception as exc:
logger.error(f"Failed to set autodelete: {exc}")
logger.exception(exc)
if delete:
remove_source_file(src_path)
display_stats(filesize, started_on, ended_on)
return 0
def download_image_s3(self):
self.logger.info("Starting s3 image download")
# add credentials to URL
url = urllib.parse.urlparse(self.task["download_uri"])
qs = urllib.parse.parse_qs(url.query)
qs["keyId"] = Setting.s3_access_key
qs["secretAccessKey"] = Setting.s3_secret_key
# setup download logging
downloader_log = io.StringIO()
downloader_logger = logging.getLogger("downloader_log")
downloader_logger.propagate = True
downloader_logger.setLevel(logging.DEBUG)
downloader_logger.addHandler(logging.StreamHandler(stream=downloader_log))
# init and test storage
downloader_logger.info("initializing S3")
s3_storage = KiwixStorage(
urllib.parse.SplitResult(
"https",
url.netloc,
url.path,
urllib.parse.urlencode(qs, doseq=True),
url.fragment,
).geturl()
)
downloader_logger.debug(
f"S3 initialized for {s3_storage.url.netloc}/{s3_storage.bucket_name}"
)
# download
downloader_logger.info(f"Downloading from {self.img_path.name}")
try:
hook = ImageTransferHook(
output=downloader_log,
size=s3_storage.get_object_stat(key=self.img_path.name).size,
name=self.img_path.name,
)
s3_storage.download_file(
key=self.img_path.name, fpath=str(self.img_path), Callback=hook
)
downloaded = True
except Exception as exc:
downloaded = False
downloader_logger.error(f"downloader failed: {exc}")
downloader_logger.exception(exc)
else:
downloader_logger.info("downloader ran successfuly.")
if downloaded:
# image downloaded, mark for autodeletion
try:
autodelete_on = datetime.datetime.now() + datetime.timedelta(days=1)
downloader_logger.info(f"Setting autodelete to now ({autodelete_on})")
s3_storage.set_object_autodelete_on(
key=self.img_path.name, on=autodelete_on
)
except Exception as exc:
downloader_logger.error(
"Failed to set autodelete (normal if before bucket retention)"
)
downloader_logger.exception(exc)
self.logger.info("collecting downloader log")
try:
self.logs["downloader_log"] = downloader_log.getvalue()
downloader_log.close()
except Exception as exc:
self.logger.error(f"Failed to collect logs: {exc}")
if not downloaded:
raise subprocess.SubprocessError("S3 download failed")
