def start_uploader(self, upload_dir, filename):
logger.info(f"Starting uploader for /{upload_dir}/{filename}")
self.uploader = start_uploader(
self.docker,
self.task,
self.username,
self.host_task_workdir,
upload_dir,
filename,
move=True,
delete=True,
compress=False,
resume=False,
watch=False,
)
def start_uploader(self, upload_dir, filename):
logger.info(f"Starting uploader for {upload_dir}/{filename}")
self.uploader = start_uploader(
self.docker,
self.task,
"zim",
self.username,
self.host_task_workdir,
upload_dir,
filename,
move=True,
delete=False, # zim delete on task exit to allow parallel zimcheck
compress=False,
resume=False,
)
def upload_log(self, watch=False):
if not self.scraper:
logger.error("can't start a log uploader without a scraper…")
return # scraper gone, we can't access log
log_path = pathlib.Path(self.scraper.attrs["LogPath"])
self.log_uploader = start_uploader(
self.docker,
self.task,
self.username,
log_path.parent,
"logs",
log_path.name,
move=False,
delete=False,
compress=True,
resume=True,
watch="6h" if watch else None,
)
def upload_scraper_log(self):
if not self.scraper:
logger.error("No scraper to upload it's logs…")
return # scraper gone, we can't access log
logger.debug("Dumping docker logs to file…")
filename = f"{self.task['_id']}_{self.task['config']['task_name']}.log"
try:
fpath = self.task_workdir / filename
with open(fpath, "wb") as fh:
for line in container_logs(
self.docker,
self.scraper.name,
stdout=True,
stderr=True,
stream=True,
):
fh.write(line)
except Exception as exc:
logger.error(f"Unable to dump logs to {fpath}")
logger.exception(exc)
return False
logger.debug("Starting log uploader container…")
self.log_uploader = start_uploader(
self.docker,
self.task,
"logs",
self.username,
host_workdir=self.host_task_workdir,
upload_dir="",
filename=filename,
move=False,
delete=True,
compress=True,
resume=True,
)
