def _create_opt_parser(self):
option_parser = ReportScript._create_opt_parser(self)
option_parser.add_option('--urlminversion',
type='int',
help='minimum version [%default]')
return option_parser
def _create_opt_parser(self):
option_parser = ReportScript._create_opt_parser(self)
option_parser.add_option("--agreement", type="float", help="minimum acceptable agreement [%default]")
option_parser.add_option(
"--problemsonly", action="store_true", help="restrict listing to troubled archival units [%default]"
)
option_parser.add_option(
"--replserver",
dest="replserver",
type="int",
help="mark as error archival units those with less replications on given servers [%default]",
)
option_parser.add_option(
"--repl",
type="int",
help="mark as error archival units those with less total known replications [%default]",
)
return option_parser
def check_opts(self):
ReportScript.check_opts(self, logopts=True)
self.require_auids()
def __init__(self, argv0):
ReportScript.__init__(self, argv0, "$Revision$", self.__class__.MYCONFIGS)
def process(self):
log.info("---")
log.info("Start Processing")
if self.options.explainheaders:
print ReportScript.explainheaders(self)
return
if self.options.dryrun:
return
opts = self.options.__dict__
print "# COMMAND", self.options._COMMAND
for key in ["repl", "replserver", "agreement"]:
if opts.has_key(key):
print "#", key, opts.get(key)
print "# "
if self.options.problemsonly:
print "# listing only archival units that appear to have a problem"
print "# "
masterAuIds = self.collectMasterAuIdInstances()
print self.report_preamble()
headers = self.options.reportheaders
print "\t".join(headers)
for mauid in masterAuIds:
auids = LockssCacheAuId.objects.filter(Q(auId=mauid.auId), self.get_Q_caches())
repl = mauid.replication()
replserver = auids.count()
prof = {
"auid": mauid.auId,
"repl": repl,
"lowrepl": repl < self.options.repl,
"replserver": replserver,
"lowreplserver": replserver < self.options.replserver,
"nLowAgree": 0,
"reportDate": {"min": None, "max": None},
"nausummary": None,
"missingauinfo": False,
"sizeMB": {"min": None, "max": None, "avg": None, "sum": None},
"diskMB": {"min": None, "max": None, "avg": None, "sum": None},
"agree": {"min": None, "max": None, "avg": None, "sum": None},
"caches": [],
}
nagree = 0
nlausum = 0
for au in auids:
lau = au.getlocksscacheausummary()
if lau:
nlausum = nlausum + 1
if lau.agreement < self.options.agreement:
prof["nLowAgree"] = prof["nLowAgree"] + 1
delta, prof["agree"] = self.add_value(lau.agreement, "agree", prof["agree"])
nagree = nagree + delta
delta, prof["sizeMB"] = self.add_value(lau.contentSizeMB(), "sizeMB", prof["sizeMB"])
assert delta == 1, "%s sizeMB: %s" % (str(lau), str(prof["sizeMB"]))
delta, prof["diskMB"] = self.add_value(lau.diskUsageMB, "sizeMB", prof["diskMB"])
assert delta == 1, "%s diskMB: %s" % (str(lau), str(prof["diskMB"]))
delta, prof["reportDate"] = self.add_value(lau.reportDate, "reportDate", prof["reportDate"])
assert delta == 1, "%s reportDare: %s" % (str(lau), str(prof["reportDate"]))
else:
prof["missingauinfo"] = True
prof["caches"].append(au.cache.name)
prof["caches"] = sorted(prof["caches"])
prof["nausummary"] = nlausum
if nagree > 0:
prof["agree"]["avg"] = prof["agree"]["sum"] / nagree
if nlausum > 0:
prof["sizeMB"]["avg"] = prof["sizeMB"]["sum"] / nlausum
prof["diskMB"]["avg"] = prof["diskMB"]["sum"] / nlausum
if (not self.options.problemsonly) or prof["lowrepl"] or prof["lowreplserver"] or prof["nLowAgree"] > 0:
vals = []
for f in headers:
keys = f.split(" ")
if len(keys) == 1:
v = prof[f]
else:
v = prof[keys[0]][keys[1]]
if v.__class__ == float:
v = "%.2f" % v
elif v.__class__ == list:
v = "\t".join(sorted(v))
vals.append(str(v))
print "\t".join(vals)
log.info("Stop Processing")
def process(self):
log.info("---")
log.info("Start Processing")
if (self.options.dryrun):
return
if (self.options.explainheaders):
print ReportScript.explainheaders(self);
return;
print "# COMMAND", self.options._COMMAND;
print "# ";
if (self.options.urlminversion > 1):
print "# listing only urls with a version number greater equal than ", self.options.urlminversion
print "# ";
masterAuIds = self.collectMasterAuIdInstances()
print self.report_preamble();
headers = self.options.reportheaders;
fields = [];
if ('reportDate' in headers):
fields.append("urlReport")
if ("auId" in headers or "cache" in headers ):
fields.append("urlReport__auId")
if ("cache" in headers ):
fields.append("urlReport__auId__cache")
relatedfields = ",".join(fields);
caches = None;
if (self.options.serverlist):
caches = self.options.cachelist;
for mauid in masterAuIds:
print "\n# AUID %s" % mauid.auId
mauidRepl = None;
urlKnownRepl = None;
qu = self.get_url_query(mauid, self.options.urlminversion, caches);
urls = [];
if (fields):
urls = Url.objects.filter(qu).order_by('name').select_related(relatedfields);
else:
urls = Url.objects.filter(qu).order_by('name');
if (urls):
print "\t".join(headers)
for u in urls:
vals = [];
for h in headers:
try:
if (h == 'name'):
vals.append(u.name)
elif (h == 'size'):
vals.append(str(u.size))
elif (h == 'version'):
vals.append(str(u.version))
elif (h == 'auId'):
vals.append(u.urlReport.auId.auId)
elif (h == 'plugin'):
vals.append(mauid.plugin)
elif (h == 'baseUrl'):
vals.append(mauid.baseUrl)
elif (h == 'extraParams'):
vals.append(mauid.extraParams)
elif (h == 'reportDate'):
vals.append(str(u.urlReport.reportDate))
elif (h == 'cache'):
vals.append(str(u.urlReport.auId.cache));
elif (h == 'urlRepl'):
repl = str(Url.objects.filter(name=u.name).count());
vals.append(str(repl))
elif (h == 'repl'):
if (not mauidRepl):
mauidRepl = str(mauid.replication())
vals.append(mauidRepl)
elif (h == 'urlKnownRepl'):
if (not urlKnownRepl):
# number of cache aus for which we have a urlReport i
qu = Q(auId__auId__exact=mauid.auId)
urlKnownRepl = str(UrlReport.objects.filter(qu).count())
vals.append(urlKnownRepl)
else:
raise LockssError("Unknown report header %s" % h)
except Exception as e:
vals.append("#Error url_id=%d %s" % (u.id, str(e)));
print "\t".join(vals)
log.info("Stop Processing")
def __init__(self, arg0):
ReportScript.__init__(self, arg0, '$Revision$', Report_urlprofile.MYCONFIGS)