def __init__(self, args):
"""
pass a dict of the standard args
(config, date, dumptype, args)
"""
self.args = args
self.indexfile = IndexFile(self.args["config"])
self.dumpdir = MiscDumpDir(self.args["config"])
def __init__(self, dryrun, args, log):
'''
pass a dict of the standard args
(config, date, dumptype, args)
'''
self.args = args
self.dryrun = dryrun
self.log = log
self.indexfile = IndexFile(self.args['config'])
self.dumpdir = MiscDumpDir(self.args['config'])
class Index():
'''
generate index.html page containing information for the dump
run of the specified date for all wikis
'''
def __init__(self, dryrun, args, log):
'''
pass a dict of the standard args
(config, date, dumptype, args)
'''
self.args = args
self.dryrun = dryrun
self.log = log
self.indexfile = IndexFile(self.args['config'])
self.dumpdir = MiscDumpDir(self.args['config'])
def do_all_wikis(self):
'''
generate index.html file for all wikis for the given date.
This should only be done if the run is the most recent, otherwise
your nice fresh index.html file willbe replaced with old data.
'''
text = ""
for wikiname in self.args['config'].all_wikis_list:
result = self.do_one_wiki(wikiname)
if result:
self.log.info("result for wiki %s is %s", wikiname, result)
text = text + "<li>" + result + "</li>\n"
index_text = (self.args['config'].read_template(self.args['config'].indextmpl)
% {"items": text})
if self.dryrun:
print("would write {path} with index text".format(path=self.indexfile.get_path()))
else:
FileUtils.write_file_in_place(self.indexfile.get_path(),
index_text, self.args['config'].fileperms)
def get_outputfile_indextxt(self, filenames_tocheck, expected, wikiname, dump_date):
'''
generate and return a list of text strings that provide a
link to the given files, along with filename, size and date.
if the file does not exist, it will be silently excluded from
the list.
the expected list is a list of filenames that are expected to
be produced by the dump; currently no errors are generated
on this basis but this may change in the future.
'''
dirinfo = MiscDumpDir(self.args['config'], dump_date)
path = dirinfo.get_dumpdir(wikiname)
output_fileinfo = {}
for filename in filenames_tocheck:
output_fileinfo[filename] = FileUtils.file_info(os.path.join(path, filename))
files_text = []
filenames = sorted(output_fileinfo.keys())
for filename in filenames:
file_date, file_size = output_fileinfo[filename]
self.log.info("output file %s for %s %s %s",
filename, wikiname, safe(file_date), safe(file_size))
if filename in expected and file_date is None:
# may do more with this sort of error in the future
# for now, just get stats on the other files
continue
if file_date:
files_text.append(
"%s: %s (size %s)<br />"
% (make_link(
os.path.join(
wikiname, dump_date,
filename),
os.path.basename(filename)), file_date, file_size))
return files_text
def get_stat_text(self, dump_date, wikiname):
'''
generate and return the text string describing
the status of the dump of the wiki for the given date
'''
stat = StatusFile(self.args['config'], dump_date, wikiname)
stat_contents = FileUtils.read_file(stat.get_path())
self.log.info("status for %s %s", wikiname, safe(stat_contents))
if stat_contents:
stat_text = "(%s)" % (stat_contents)
else:
stat_text = None
return stat_text
def get_files_text(self, wiki):
'''
given wiki object, return the list of links and descriptions
for the output files for that wiki of the current dump type
and date
'''
dump_class = MiscDumpFactory.get_dumper(self.args['dumptype'])
dumper = dump_class(wiki, self.log, False, self.args['args'])
output_files, expected = dumper.get_output_files()
files_text = self.get_outputfile_indextxt(output_files, expected,
wiki.db_name, wiki.date)
md5file = MD5File(wiki.config, wiki.date, wiki.db_name)
md5file_text = self.get_outputfile_indextxt(
[md5file.get_filename()], [], wiki.db_name, wiki.date)
files_text.extend(md5file_text)
return files_text
def do_one_wiki(self, wikiname, date=None):
'''
collect the text strings for one wiki to be inserted into
the index.html file
'''
if not skip_wiki(wikiname, self.args['config']):
dumps_dirs = MiscDumpDirs(self.args['config'], wikiname, self.log)
if not exists(self.dumpdir.get_dumpdir_no_date(wikiname)):
self.log.info("No dump for wiki %s", wikiname)
return None
if date is not None:
dump_date = date
else:
dump_date = dumps_dirs.get_latest_dump_date(True)
if not dump_date:
self.log.info("No dump for wiki %s", wikiname)
return None
other_runs_text = "other runs: %s<br />" % make_link(wikiname, wikiname)
try:
wiki = Wiki(self.args['config'], wikiname)
wiki.set_date(dump_date)
files_text = self.get_files_text(wiki)
stat_text = self.get_stat_text(dump_date, wikiname)
except Exception as ex:
self.log.warning("Error encountered, no information available"
" for wiki %s", wikiname, exc_info=ex)
return ("<strong>%s</strong> Error encountered,"
" no information available | %s" % (wikiname, other_runs_text))
try:
wikiname_text = "<strong>%s</strong>" % wikiname
wiki_info = (" ".join([entry for entry in [wikiname_text, stat_text]
if entry is not None]) + "<br />")
wiki_info = (wiki_info + " " + "\n ".join(files_text))
wiki_info = wiki_info + "\n " + other_runs_text
except Exception as ex:
self.log.warning("Error encountered formatting information"
" for wiki %s", wikiname, exc_info=ex)
return ("Error encountered formatting information"
" for wiki %s" % wikiname)
return wiki_info
return None
