def start_uploader(self, upload_dir, filename): logger.info(f"Starting uploader for /{upload_dir}/{filename}") self.uploader = start_uploader( self.docker, self.task, self.username, self.host_task_workdir, upload_dir, filename, move=True, delete=True, compress=False, resume=False, watch=False, )
def start_uploader(self, upload_dir, filename): logger.info(f"Starting uploader for {upload_dir}/{filename}") self.uploader = start_uploader( self.docker, self.task, "zim", self.username, self.host_task_workdir, upload_dir, filename, move=True, delete=False, # zim delete on task exit to allow parallel zimcheck compress=False, resume=False, )
def upload_log(self, watch=False): if not self.scraper: logger.error("can't start a log uploader without a scraper…") return # scraper gone, we can't access log log_path = pathlib.Path(self.scraper.attrs["LogPath"]) self.log_uploader = start_uploader( self.docker, self.task, self.username, log_path.parent, "logs", log_path.name, move=False, delete=False, compress=True, resume=True, watch="6h" if watch else None, )
def upload_scraper_log(self): if not self.scraper: logger.error("No scraper to upload it's logs…") return # scraper gone, we can't access log logger.debug("Dumping docker logs to file…") filename = f"{self.task['_id']}_{self.task['config']['task_name']}.log" try: fpath = self.task_workdir / filename with open(fpath, "wb") as fh: for line in container_logs( self.docker, self.scraper.name, stdout=True, stderr=True, stream=True, ): fh.write(line) except Exception as exc: logger.error(f"Unable to dump logs to {fpath}") logger.exception(exc) return False logger.debug("Starting log uploader container…") self.log_uploader = start_uploader( self.docker, self.task, "logs", self.username, host_workdir=self.host_task_workdir, upload_dir="", filename=filename, move=False, delete=True, compress=True, resume=True, )