def do_one_wiki(self): """ run dump of specified type for one wiki, for given date unless it is among the wikis we skip, has already been run for the date, or some other process has the lock and is therefore presumably already dumping it """ if not skip_wiki(self.wiki.db_name, self.wiki.config): dumpdir = MiscDumpDir(self.args["config"], self.args["date"]) if not exists(dumpdir.get_dumpdir(self.wiki.db_name)): os.makedirs(dumpdir.get_dumpdir(self.wiki.db_name)) status_info = StatusInfo(self.args["config"], self.wiki.date, self.wiki.db_name) status = status_info.get_status() if status == "done:all" and not self.flags["forcerun"]: log.info("wiki %s skipped, adds/changes dump already" " complete", self.wiki.db_name) return STATUS_GOOD if not self.flags["dryrun"]: lock = MiscDumpLock(self.args["config"], self.wiki.date, self.wiki.db_name) # if lock is stale, remove it lock.remove_if_stale(self.wiki.config.lock_stale) # try to get the lock ourselves if not lock.get_lock(): log.info( "wiki %s skipped, wiki is locked," " another process should be doing the job", self.wiki.db_name ) return STATUS_TODO self.dumper.set_lockinfo(lock) dumps_dirs = MiscDumpDirs(self.wiki.config, self.wiki.db_name) dumps_dirs.cleanup_old_dumps(self.wiki.date) log.info("Doing run for wiki: %s", self.wiki.db_name) try: result = self.dumper.run() if not result: return STATUS_FAILED if not self.flags["dryrun"]: output_files, expected = self.dumper.get_output_files() if not md5sums(self.wiki, self.wiki.config.fileperms, output_files, expected): return STATUS_FAILED status_info.set_status("done:" + self.dumper.get_steps_done()) lock.unlock_if_owner() if self.flags["do_index"]: index = Index(self.args) index.do_all_wikis() except Exception as ex: log.warning("error from dump run" " for wiki %s", self.wiki.db_name, exc_info=ex) if not self.flags["dryrun"]: lock.unlock_if_owner() return STATUS_FAILED log.info("Success! Wiki %s %s dump complete.", self.wiki.db_name, self.args["dumptype"]) return STATUS_GOOD
def is_most_recent_run(self): ''' return True if there are no runs on any wikis more recent than the current run date, otherwise False ''' for wikiname in self.args['config'].all_wikis_list: dumps_dirs = MiscDumpDirs(self.args['config'], wikiname, self.log) dirs = dumps_dirs.get_misc_dumpdirs() if not dirs or dirs[-1] > self.args['date']: return False return True
def do_one_wiki(self, wikiname, date=None): ''' collect the text strings for one wiki to be inserted into the index.html file ''' if not skip_wiki(wikiname, self.args['config']): dumps_dirs = MiscDumpDirs(self.args['config'], wikiname, self.log) if not exists(self.dumpdir.get_dumpdir_no_date(wikiname)): self.log.info("No dump for wiki %s", wikiname) return None if date is not None: dump_date = date else: dump_date = dumps_dirs.get_latest_dump_date(True) if not dump_date: self.log.info("No dump for wiki %s", wikiname) return None other_runs_text = "other runs: %s<br />" % make_link(wikiname, wikiname) try: wiki = Wiki(self.args['config'], wikiname) wiki.set_date(dump_date) files_text = self.get_files_text(wiki) stat_text = self.get_stat_text(dump_date, wikiname) except Exception as ex: self.log.warning("Error encountered, no information available" " for wiki %s", wikiname, exc_info=ex) return ("<strong>%s</strong> Error encountered," " no information available | %s" % (wikiname, other_runs_text)) try: wikiname_text = "<strong>%s</strong>" % wikiname wiki_info = (" ".join([entry for entry in [wikiname_text, stat_text] if entry is not None]) + "<br />") wiki_info = (wiki_info + " " + "\n ".join(files_text)) wiki_info = wiki_info + "\n " + other_runs_text except Exception as ex: self.log.warning("Error encountered formatting information" " for wiki %s", wikiname, exc_info=ex) return ("Error encountered formatting information" " for wiki %s" % wikiname) return wiki_info return None