Ejemplo n.º 1
0
 def get_prev_incrdate(self, date, dumpok=False, revidok=False):
     '''
     find the most recent incr dump before the
     specified date
     if "dumpok" is True, find most recent dump that completed successfully
     if "revidok" is True, find most recent dump that has a populated maxrevid.txt file
     '''
     previous = None
     old = self.dirs.get_misc_dumpdirs()
     if old:
         for dump in old:
             if dump == date:
                 return previous
             else:
                 if dumpok:
                     status_info = StatusInfo(self.wiki.config, dump, self.wiki.db_name)
                     if status_info.get_status(dump) == "done":
                         previous = dump
                 elif revidok:
                     max_revid_file = MaxRevIDFile(self.wiki.config, dump, self.wiki.db_name)
                     if exists(max_revid_file.get_path()):
                         revid = FileUtils.read_file(max_revid_file.get_path().rstrip())
                         if int(revid) > 0:
                             previous = dump
                 else:
                     previous = dump
     return previous
    def do_one_wiki(self):
        """
        run dump of specified type for one wiki, for given date
        unless it is among the wikis we skip, has already been run
        for the date, or some other process has the lock and is
        therefore presumably already dumping it
        """
        if not skip_wiki(self.wiki.db_name, self.wiki.config):

            dumpdir = MiscDumpDir(self.args["config"], self.args["date"])
            if not exists(dumpdir.get_dumpdir(self.wiki.db_name)):
                os.makedirs(dumpdir.get_dumpdir(self.wiki.db_name))

            status_info = StatusInfo(self.args["config"], self.wiki.date, self.wiki.db_name)
            status = status_info.get_status()
            if status == "done:all" and not self.flags["forcerun"]:
                log.info("wiki %s skipped, adds/changes dump already" " complete", self.wiki.db_name)
                return STATUS_GOOD

            if not self.flags["dryrun"]:
                lock = MiscDumpLock(self.args["config"], self.wiki.date, self.wiki.db_name)

                # if lock is stale, remove it
                lock.remove_if_stale(self.wiki.config.lock_stale)

                # try to get the lock ourselves
                if not lock.get_lock():
                    log.info(
                        "wiki %s skipped, wiki is locked," " another process should be doing the job", self.wiki.db_name
                    )
                    return STATUS_TODO

                self.dumper.set_lockinfo(lock)
                dumps_dirs = MiscDumpDirs(self.wiki.config, self.wiki.db_name)
                dumps_dirs.cleanup_old_dumps(self.wiki.date)

            log.info("Doing run for wiki: %s", self.wiki.db_name)

            try:
                result = self.dumper.run()
                if not result:
                    return STATUS_FAILED

                if not self.flags["dryrun"]:
                    output_files, expected = self.dumper.get_output_files()
                    if not md5sums(self.wiki, self.wiki.config.fileperms, output_files, expected):
                        return STATUS_FAILED
                    status_info.set_status("done:" + self.dumper.get_steps_done())
                    lock.unlock_if_owner()

                if self.flags["do_index"]:
                    index = Index(self.args)
                    index.do_all_wikis()
            except Exception as ex:
                log.warning("error from dump run" " for wiki %s", self.wiki.db_name, exc_info=ex)
                if not self.flags["dryrun"]:
                    lock.unlock_if_owner()
                return STATUS_FAILED
        log.info("Success!  Wiki %s %s dump complete.", self.wiki.db_name, self.args["dumptype"])
        return STATUS_GOOD