def finish_scraper_log_upload(self):
# stop and remove previous (watch) container. might be already stopped
stop_container(self.docker, self.log_uploader.name, timeout=60 * 10)
remove_container(self.docker, self.log_uploader.name, force=True)
self.log_uploader = None
# restart without watch to make sure it's complete
self.upload_log(watch=False)
try:
self.log_uploader.reload()
exit_code = wait_container(self.docker,
self.log_uploader.name,
timeout=20 * 60)["StatusCode"]
# connexion exception can be thrown by socket, urllib, requests
except Exception as exc:
logger.error(f"log upload could not complete. {exc}")
stop_container(self.docker, self.log_uploader.name)
exit_code = -1
finally:
logger.info(f"Scraper log upload complete: {exit_code}")
if exit_code != 0:
logger.error(
f"Log Uploader:: {get_container_logs(self.docker, self.log_uploader.name)}"
)
remove_container(self.docker, self.log_uploader.name, force=True)
def finish_scraper_log_upload(self):
# stop and remove previous (watch) container. might be already stopped
try:
stop_container(self.docker,
self.log_uploader.name,
timeout=60 * 10)
remove_container(self.docker, self.log_uploader.name, force=True)
except docker.errors.NotFound as exc:
# failure here is unexpected (container should exist) but happens randomly
# catching this to prevent task from crashing while there might be
# ZIM files to continue uploading after
logger.warning(f"Log uploader container missing: {exc}")
logger.warning("Expect full-log upload next (long)")
self.log_uploader = None
try:
# restart without watch to make sure it's complete
self.upload_log(watch=False)
self.log_uploader.reload()
# should log uploader above have been gone, we might expect this to fail
# on super large mwoffliner with verbose mode on (20mn not enough for 20GB)
exit_code = wait_container(self.docker,
self.log_uploader.name,
timeout=20 * 60)["StatusCode"]
# connexion exception can be thrown by socket, urllib, requests
except Exception as exc:
logger.error(f"log upload could not complete. {exc}")
stop_container(self.docker, self.log_uploader.name)
exit_code = -1
finally:
logger.info(f"Scraper log upload complete: {exit_code}")
if exit_code != 0:
logger.error(
f"Log Uploader:: {get_container_logs(self.docker, self.log_uploader.name)}"
)
remove_container(self.docker, self.log_uploader.name, force=True)
def sync_tasks_and_containers(self):
# list of completed containers (successfuly ran)
completed_containers = list_containers(self.docker,
all=True,
filters={
"label": ["zimtask=yes"],
"exited": 0
})
# list of running containers
running_containers = list_containers(
self.docker, filters={"label": ["zimtask=yes"]})
# list of task_ids for running containers
running_task_ids = [
get_label_value(self.docker, container.name, "task_id")
for container in running_containers
]
# remove completed containers
for container in completed_containers:
logger.info(
f"container {container.name} exited successfuly, removing.")
remove_container(self.docker, container.name)
# make sure we are tracking task for all running containers
for task_id in running_task_ids:
if task_id not in self.tasks.keys():
logger.info(f"found running container for {task_id}.")
self.update_task_data(task_id)
# filter our tasks register of gone containers
for task_id in list(self.tasks.keys()):
if task_id not in running_task_ids:
logger.info(
f"task {task_id} is not running anymore, unwatching.")
self.tasks.pop(task_id, None)