def do_one_wiki(self):
"""
run dump of specified type for one wiki, for given date
unless it is among the wikis we skip, has already been run
for the date, or some other process has the lock and is
therefore presumably already dumping it
"""
if not skip_wiki(self.wiki.db_name, self.wiki.config):
dumpdir = MiscDumpDir(self.args["config"], self.args["date"])
if not exists(dumpdir.get_dumpdir(self.wiki.db_name)):
os.makedirs(dumpdir.get_dumpdir(self.wiki.db_name))
status_info = StatusInfo(self.args["config"], self.wiki.date, self.wiki.db_name)
status = status_info.get_status()
if status == "done:all" and not self.flags["forcerun"]:
log.info("wiki %s skipped, adds/changes dump already" " complete", self.wiki.db_name)
return STATUS_GOOD
if not self.flags["dryrun"]:
lock = MiscDumpLock(self.args["config"], self.wiki.date, self.wiki.db_name)
# if lock is stale, remove it
lock.remove_if_stale(self.wiki.config.lock_stale)
# try to get the lock ourselves
if not lock.get_lock():
log.info(
"wiki %s skipped, wiki is locked," " another process should be doing the job", self.wiki.db_name
)
return STATUS_TODO
self.dumper.set_lockinfo(lock)
dumps_dirs = MiscDumpDirs(self.wiki.config, self.wiki.db_name)
dumps_dirs.cleanup_old_dumps(self.wiki.date)
log.info("Doing run for wiki: %s", self.wiki.db_name)
try:
result = self.dumper.run()
if not result:
return STATUS_FAILED
if not self.flags["dryrun"]:
output_files, expected = self.dumper.get_output_files()
if not md5sums(self.wiki, self.wiki.config.fileperms, output_files, expected):
return STATUS_FAILED
status_info.set_status("done:" + self.dumper.get_steps_done())
lock.unlock_if_owner()
if self.flags["do_index"]:
index = Index(self.args)
index.do_all_wikis()
except Exception as ex:
log.warning("error from dump run" " for wiki %s", self.wiki.db_name, exc_info=ex)
if not self.flags["dryrun"]:
lock.unlock_if_owner()
return STATUS_FAILED
log.info("Success! Wiki %s %s dump complete.", self.wiki.db_name, self.args["dumptype"])
return STATUS_GOOD
def dump_html(self):
'''
dump HTML-formated revision content from RESTBase
for the given wiki and date
'''
dumpdir = MiscDumpDir(self.wiki.config, self.wiki.date)
outputdir = dumpdir.get_dumpdir(self.wiki.db_name, self.wiki.date)
htmlfile = HTMLFile(self.wiki.config, self.wiki.date, self.wiki.db_name)
outputfile = htmlfile.get_filename(self.args['ns'])
# /usr/bin/nodejs ./bin/dump_wiki --domain en.wikipedia.org --ns 0 \
# --apiURL http://en.wikipedia.org/w/api.php \
# --dataBase /srv/www/htmldumps/dumps/20160826/en.wikipedia.org.articles.ns0.sqlite3
domain = self.get_domain_from_wikidbname()
# FIXME: the nodejs wrapper which will do the compress etc stuff for one wiki is
# not yet written
command = [self.wiki.config.nodejs]
command.append(self.wiki.config.scriptpath)
command.extend(["--domain", domain, "--ns", self.args['ns'],
"--apiURL", "http://%s/w/api.php" % domain,
"--dataBase", os.path.join(outputdir, outputfile),
"--wiki=%s" % self.wiki.db_name,
"--output=gzip:%s" % os.path.join(outputdir, outputfile)])
if self.dryrun:
print("would run command for html dump:", command)
else:
success = RunSimpleCommand.run_with_no_output(
command, shell=False,
timeout=self.get_lock_timeout_interval(),
timeout_callback=self.periodic_callback)
if not success:
self.log.warning("error producing html files for wiki %s", self.wiki.db_name)
return False
return True
def dump_revs(self):
'''
dump revision content corresponding to previously-dumped
stubs (revision metadata)
'''
if not self.steps['revs']['run']:
return True
dumpdir = MiscDumpDir(self.wiki.config, self.wiki.date)
outputdir = dumpdir.get_dumpdir(self.wiki.db_name, self.wiki.date)
revsfile = RevsFile(self.wiki.config, self.wiki.date, self.wiki.db_name)
outputfile = revsfile.get_filename()
script_command = MultiVersion.mw_script_as_array(self.wiki.config,
"dumpTextPass.php")
command = [self.wiki.config.php]
command.extend(script_command)
stubfile = StubFile(self.wiki.config, self.wiki.date, self.wiki.db_name)
stuboutputfile = stubfile.get_filename()
command.extend(["--wiki=%s" % self.wiki.db_name,
"--stub=gzip:%s" % os.path.join(outputdir, stuboutputfile),
"--quiet",
"--spawn=%s" % self.wiki.config.php,
"--output=bzip2:%s" % os.path.join(outputdir, outputfile)])
if self.dryrun:
print "would run command for revs dump:", command
else:
log.info("running with no output: " + " ".join(command))
success = RunSimpleCommand.run_with_no_output(
command, shell=False, timeout=self.get_lock_timeout_interval(),
timeout_callback=self.periodic_callback)
if not success:
log.warning("error producing revision text files"
" for wiki %s", self.wiki.db_name)
return False
return True
def dump_stub(self, start_revid, end_revid):
'''
dump stubs (metadata) for revs from start_revid
up to but not including end_revid
'''
if not self.steps['stubs']['run']:
return True
dumpdir = MiscDumpDir(self.wiki.config, self.wiki.date)
outputdir = dumpdir.get_dumpdir(self.wiki.db_name, self.wiki.date)
stubfile = StubFile(self.wiki.config, self.wiki.date, self.wiki.db_name)
outputfile = stubfile.get_filename()
script_command = MultiVersion.mw_script_as_array(self.wiki.config,
"dumpBackup.php")
command = [self.wiki.config.php]
command.extend(script_command)
command.extend(["--wiki=%s" % self.wiki.db_name, "--stub", "--quiet",
"--output=gzip:%s" % os.path.join(outputdir, outputfile),
"--revrange", "--revstart=%s" % start_revid,
"--revend=%s" % end_revid])
if self.dryrun:
print "would run command for stubs dump:", command
else:
log.info("running with no output: " + " ".join(command))
success = RunSimpleCommand.run_with_no_output(
command, shell=False, timeout=self.get_lock_timeout_interval(),
timeout_callback=self.periodic_callback)
if not success:
log.warning("error producing stub files for wiki %s", self.wiki.db_name)
return False
return True
def get_outputfile_indextxt(self, filenames_tocheck, expected, wikiname, dump_date):
'''
generate and return a list of text strings that provide a
link to the given files, along with filename, size and date.
if the file does not exist, it will be silently excluded from
the list.
the expected list is a list of filenames that are expected to
be produced by the dump; currently no errors are generated
on this basis but this may change in the future.
'''
dirinfo = MiscDumpDir(self.args['config'], dump_date)
path = dirinfo.get_dumpdir(wikiname)
output_fileinfo = {}
for filename in filenames_tocheck:
output_fileinfo[filename] = FileUtils.file_info(os.path.join(path, filename))
files_text = []
filenames = sorted(output_fileinfo.keys())
for filename in filenames:
file_date, file_size = output_fileinfo[filename]
self.log.info("output file %s for %s %s %s",
filename, wikiname, safe(file_date), safe(file_size))
if filename in expected and file_date is None:
# may do more with this sort of error in the future
# for now, just get stats on the other files
continue
if file_date:
files_text.append(
"%s: %s (size %s)<br />"
% (make_link(
os.path.join(
wikiname, dump_date,
filename),
os.path.basename(filename)), file_date, file_size))
return files_text
def get_output_files(self):
dumpdir = MiscDumpDir(self.wiki.config, self.wiki.date)
outputdir = dumpdir.get_dumpdir(self.wiki.db_name, self.wiki.date)
htmlfile = HTMLFile(self.wiki.config, self.wiki.date, self.wiki.db_name)
filenames = [htmlfile.get_filename(self.args['ns'])]
return [os.path.join(outputdir, filename) for filename in filenames]
