def load(cache, auId, doUrls, expire, trials, sleep, timeout):
act = Action.GETAUSUMMARY
if (doUrls):
act = act + "," + Action.GETURLLIST
log.info('Start %s: %s expire=%s ...' % (act, auId, str(expire)))
success = False
try:
for _ in range(trials):
try:
if (doUrls):
UrlReport.loadTry(cache.ui, auId, expire)
else:
LockssCacheAuSummary.__loadTry(cache.ui, auId, expire)
success = True
break
except urllib2.HTTPError as inst:
cache.reconnect(sleep,timeout)
except ExpatError:
log.error("XML Parser error; could not %s %s" % (auId, act))
success = False; # try again
if (not success):
log.error("exhausted trials for %s; could not load %s" % (auId, act))
except LockssError as inst:
log.warn("LockssException: %s" % inst) # output is scanned for the ERROR string
log.warn("could not digest %s for %s" % (act, auId.auId))
finally:
log.debug2('Stop %s: %s Success = %s ...' % (act, auId, success))
return success
def cleanList(lst):
if lst:
log.info("clean: %s" % str(lst[0].__class__))
for e in lst:
try:
e.full_clean()
except ValidationError as err:
print "%s.%d.DELETE %s" % (e.__class__.__name__, e.id, str(err))
e.delete()
def process(self):
log.info("ignoring all parameters; just cleaning out inconsistent entries in db tables")
Cleandb.cleanList(UrlReport.objects.all())
Cleandb.cleanList(Url.objects.all())
Cleandb.cleanList(LockssCacheAuId.objects.all())
Cleandb.cleanList(LockssCacheAuSummary.objects.all())
Cleandb.cleanList(LockssCrawlStatus.objects.all())
Cleandb.cleanList(UrlReport.objects.all())
Cleandb.cleanList(CacheCommPeer.objects.all())
def crawlstatus(self, auids):
self.getcrawlstatus(auids, self.options.dir, True) # retry until all of them have been retrieved
# now check which ones are still active
repeat = set()
for auid in auids:
if (LockssCrawlStatus.lastStatus(auid) != LockssCrawlStatus.DONE):
log.info("Crawl not done %s" % auid)
repeat.add(auid)
return repeat
def collectAuIdInstances(self, cache):
ids = [];
if (self.options.all):
ids = cache.locksscacheauid_set.all()
else:
ids = LockssCacheAuId.get(self.options.auids, self.options.auidprefixlist, cache)
log.info("#Matching AUIDS: %d" % len(ids))
for au in ids:
log.debug2("Matching AUIDS: %s" % au.auId)
return ids
def collectMasterAuIdInstances(self):
ids = [];
if (self.options.all):
ids = MasterAuId.objects.all()
else:
ids = MasterAuId.get(self.options.auids, self.options.auidprefixlist)
log.info("#Matching MASTER AUIDS: %d" % len(ids))
for au in ids:
log.info("Matching MASTER AUIDS: %s" % au.auId)
return ids
def mkdir(self, action, server):
dirname = "%s/%s" % (self.options.dir, server)
if not os.path.exists(dirname):
try:
os.makedirs(dirname,0777)
log.info("created output directory %s" % dirname)
except:
log.error("Could not create %s" % dirname)
return None
else:
log.debug2("using output directory %s" % dirname)
return dirname
def loadTry(ui, auId, datafresh):
'''
create/update urlReport
and associated related urls (deleting existing urls)
'''
urlReport = None
reportDate = datetime.utcnow().replace(tzinfo=pytz.UTC)
try:
# check for existing au summary
urlReport = UrlReport.objects.get(auId = auId)
if ((reportDate - urlReport.reportDate) < datafresh):
log.debug2("uptodate UrlReport available - not querying cache")
return urlReport
except UrlReport.DoesNotExist:
pass;
log.info("get UrlReport %s" % auId)
(reportDate, urllist) = LockssCacheAuSummary.loadTryIt(ui, auId, True);
if (not urllist):
raise LockssError("%s on cache %s reports 0 urls" % (str(auId), auId.cache.name));
if (not urlReport):
urlReport = UrlReport.objects.create(reportDate=reportDate, auId=auId);
else:
Url.objects.filter(urlReport=urlReport).delete()
urlReport.reportDate = reportDate;
urlReport.save();
print "urlReport ", urlReport;
try:
for url in urllist:
# work only on urls with real content
if (not url.has_key('NodeContentSize') or url['NodeContentSize'] == '-'):
continue
u = { 'urlReport' : urlReport }
u['name'] = url[u'NodeName']
u['childCount'] = url[u'NodeChildCount'] if url.has_key(u'NodeChildCount') else 0
u['treeSize'] = url[u'NodeTreeSize'] if url.has_key(u'NodeTreeSize') else 0
u['size'] = url[u'NodeContentSize'] if url.has_key(u'NodeContentSize') else 0
u['version'] = url[u'NodeVersion'] if url.has_key(u'NodeVersion') else 0
if url.has_key(u'NodeVersion'):
if (url[u'NodeVersion'] == '1' ):
u['version'] = 1
else:
u['version'] = url[u'NodeVersion']['value']
lurl = Url.objects.create(**u)
lurl.save()
log.debug2("Url %s " % lurl.name)
except Exception as e:
urlReport.delete(); # deletes dependent urls
raise LockssError("Failed to read Url Info for %s %s\nException %s" %
(auId.cache.name, str(auId), str(e)))
def _read_configuration(self, filename):
'''Process configuration file; handles nesting if necessary'''
if (filename):
filename = os.path.expanduser(filename)
while filename not in self.read_configurations:
self.read_configurations.add( filename )
if (filename):
fname = self.configuration.read( filename )
if (fname):
log.info("Read Config File '%s'" % filename)
else:
log.info("Could not read Config File '%s'" % filename)
if (self.configuration.has_option(LockssScript.PARAMSECTION, 'config')):
filename = self.configuration.get( LockssScript.PARAMSECTION, 'config' )
self._read_configuration(filename)
def printcsv(f, auids, sort, hdrs, sep):
if (not auids):
log.info('NOOP %s: No auids for %s' % (Action.PRTAUSUMMARY, f.name) )
return
cache = auids[0].cache
log.info('Start %s: Cache %s File %s ...' % (Action.PRTAUSUMMARY, cache, f.name))
f.write(sep.join(hdrs) + "\n")
# build query
qu = Q()
for auid in auids:
qu = qu | Q(auId = auid.id)
sums = LockssCacheAuSummary.objects.filter(qu).order_by(sort)
for sm in sums:
f.write(sm.csv(hdrs, sep) + "\n")
f.close()
log.debug2('Stop %s: Cache %s File %s ...' % (Action.PRTAUSUMMARY, cache, f.name))
def __loadTry(ui, auId, datafresh):
'''
deleting existing LockssCacheAuSummary
create new summary by reading status info from cache
'''
lockssSummary = None
try:
# check for existing au summary
lockssSummary = LockssCacheAuSummary.objects.get(auId = auId)
if ((datetime.utcnow().replace(tzinfo=pytz.UTC) - lockssSummary.reportDate) < datafresh):
log.debug2("uptodate LockssCacheAuSummary available - not querying cache")
return lockssSummary
except:
pass;
log.info("get LockssCacheAuSummary %s" % auId);
LockssCacheAuSummary.loadTryIt(ui, auId, False);
def process(self):
for server in self.options.cachelist:
self.cache = self.get_cache(server.domain, server.port,
True, self.options.username, self.options.password)
auids = self.collectAuIdInstances(self.cache)
if (len(auids) > self.options.max):
msg = "cowardly refusing to start %d crawls on %s" % (len(auids), self.cache)
raise RuntimeError, msg
for auId in auids:
log.info('Request Crawl: %s %s ' % (self.cache, auId))
try:
self.cache.ui.startCrawl(auId.getLockssAu())
log.info("started crawl: %s" % auId)
except LockssError as e:
log.error( "no crawl for %s (%s)" % (auId, e));
def load(cache, trials, sleep, timeout):
act = Action.GETCOMMPEERS
log.info("Start %s:..." % (act))
success = False
try:
for _ in range(trials):
try:
CacheCommPeer.__loadTry(cache.ui, cache)
success = True
break
except urllib2.HTTPError as inst:
cache.reconnect(sleep, timeout)
if not success:
log.error("exhausted trials; could not %s %s" % (act, cache))
except Exception as inst:
log.error("could not digest comm peer status %s (%s)" % (str(inst), traceback.format_exc()))
finally:
log.debug("Stop %s: Success %s..." % (act, success))
return success
def load(cache, auId, trials, sleep,timeout):
success = False
log.debug2('Start %s: %s ...' % (Action.GETCRAWLSTATUS, auId))
try:
log.info('get %s: %s ...' % (Action.GETCRAWLSTATUS, auId))
for i in range(trials):
try:
LockssCrawlStatus.__loadTry(cache.ui, auId)
success = True
break
except urllib2.HTTPError as inst:
cache.reconnect(sleep,timeout)
log.error("exhausted trials for %s, could not load crawlstatus" % (auId))
except LockssError as inst:
log.warn("LockssException: %s" % inst) # output is scanned for the ERROR string
log.warn("could not digest %s for %s" % (Action.GETCRAWLSTATUS, auId.auId))
finally:
log.debug2('Stop %s: %s Success = %s ...' % (Action.GETCRAWLSTATUS, auId, success))
return success
def printcsv(f, auids, sort, limit, hdrs, sep):
if (not auids):
log.info('NOOP %s: No auids for %s' % (Action.PRTCRAWLSTATUS, f.name) )
return
cache = auids[0].cache
log.info('Start %s: Cache %s File %s ...' % (Action.PRTCRAWLSTATUS, cache, f.name))
f.write(sep.join(hdrs) + "\n")
if (limit > 0):
crawls = []
for auid in auids:
crawls = crawls + LockssCrawlStatus.recents(auid, limit)
crawls = sorted(crawls, key=lambda crawl:crawl.__dict__.get(sort))
else:
qu = Q()
for auid in auids:
qu = qu | Q(auId = auid.id)
crawls = LockssCrawlStatus.objects.filter(qu).order_by(sort)
for st in crawls:
f.write(st.csv(hdrs,sep) + "\n")
log.debug2('Stop %s: File %s ...' % (Action.PRTCRAWLSTATUS, f.name))
def process(self):
log.info("---")
log.info("Start Processing")
if (self.options.dryrun):
return
options = self.options
for server in self.options.serverlist:
self.cache = self.get_cache(server[0], server[1],
True, options.username, options.password)
log.info(Action.GETCOMMPEERS + " cache: " + self.cache.domain)
self.getcommpeers(options.dir, options.noquit)
LockssCacheCommPeer.printcsv(sys.stdout, self.cache)
log.info("Stop Processing")
def process(self):
log.info("--- Start Processing")
options = self.options
server = self.options.cachelist[0];
self.cache = self.get_cache(server.domain, server.port,
True, options.username,
options.password)
if (not self.options.dryrun):
success = LockssCacheAuId.load(self.cache)
if (not success):
log.error('Exiting: could not load auids from cache %s' % (self.cache))
raise RuntimeError, 'could not load auids from cache'
auids = self.collectAuIdInstances(self.cache)
if (self.options.dryrun):
return
repeat = auids
while (repeat):
# do a first round until all auids are found to conform
# do not retry auids that are found to be ok
while (repeat):
repeat = self.crawlstatus(repeat)
if (repeat):
self.sleep(options.pause)
# time has expired, try on original auids to see
# whether they all still conforming
log.info("Retrying all auids to recheck")
repeat = self.crawlstatus(auids)
if (repeat):
self.sleep(options.pause)
log.info("No active crawls; printing info about most recent crawls")
f = open(options.dir + "/AuCrawlStatus.tsv", 'w')
LockssCrawlStatus.printcsv(f, auids, options.crawlsort, 1, options.crawlheaders, "\t")
f.close()
log.info("--- Stop Processing")
def printcsv(folder, auids, orderby, hdrs, sep, minrev = 1):
'''
print url reports for all given auids including urls that have a
version at least as great as minrev, which defaults to 1
'''
if (not auids):
log.info('NOOP %s: No auids to print to %s' % (Action.PRTURLLIST, folder) )
return
for auid in auids:
urls = []
try:
if (orderby == 'minversion' or orderby == 'replication'):
urls = auid.urlreport.url_set.filter(version__gte=minrev).all()
else:
urls = auid.urlreport.url_set.filter(version__gte=minrev).order_by(orderby).all()
ext = ".tsv"
if (sep == ","):
ext = ".csv"
f = open(folder + "/" + auid.auId + ext, 'w')
if (urls.count() == 0):
log.info("NOOP %s: file %s No Urls with version >= %s" % (Action.PRTURLLIST, f.name, minrev))
log.info('Start %s: file %s version %s' % (Action.PRTURLLIST, f.name, minrev))
try:
reportDate = auid.urlreport.reportDate
f.write("ReportDate\t%s\nIncluding Urls with version >= %s\n\n" % (str(reportDate), minrev))
f.write(sep.join(hdrs) + "\n")
for url in urls:
f.write(url.csv(hdrs, sep) + "\n")
log.debug2('Stop %s: file %s version %s' % (Action.PRTURLLIST, f.name, minrev))
f.close()
except IndexError:
log.info("NOOP %s: file %s No Urls at all" % (Action.PRTURLLIST, f.name))
except ObjectDoesNotExist:
log.warn('Start %s: No UrlReport for %s at %s' %
(Action.PRTURLLIST, auid, auid.cache.name))
def log_opts(self):
'''
log all int, float, str options
except for password or strings with '\n'
also log auids and auiprefix lists
'''
options = self.options
log.info( "COMMAND = %s" % self.options._COMMAND)
log.info( "CWD = %s" % self.options._CWD)
for e in options.__dict__:
if (not e.startswith('_') and
not e == self.AUIDS and
not e == 'password' and
not e == self.AUIDPREFIXLIST and
not e == self.SERVERLIST):
v = self.options.__dict__.get(e)
if (not v):
continue ;
if (v.__class__ == str) :
if (v.find('\n') == -1):
log.info( "%s = %s" % (e, v))
elif (v.__class__ == int or v.__class__ == float):
log.info( "%s = %s" % (e, str(v)))
elif (v.__class__ != list):
log.info( "%s = %s" % (e, str(v)))
if (self.options.cachelist):
for c in sorted(self.options.cachelist):
log.info( "SERVER = %s" % c)
else:
for s in sorted(options.serverlist):
log.info( "SERVER = %s:%s" % (s[0], s[1]))
for i in sorted(options.auids):
log.info( "AUIDS = %s" % i)
for i in options.auidprefixlist:
log.info( "AUIDPREFIX = %s" % i)
def log_opts(self):
LockssScript.log_opts(self)
for a in self.options.serverlist:
log.info("SERVER = %s", a)
def process_server(self, server):
'''
if dryrun collect matching auids and log.info them
otherwise perform all requested actions
'''
log.info("------ Start Processing %s" % server)
options = self.options
try:
self.cache = self.get_cache(server.domain, server.port,
options.need_credentials, options.username,
options.password)
if (not options.dryrun):
if (Action.GETAUIDLIST in options.action):
success = LockssCacheAuId.load(self.cache)
if (not success):
log.error('Exiting: could not load auids from cache %s' % (self.cache))
raise RuntimeError, 'could not load auids from cache'
auids = self.collectAuIdInstances(self.cache)
if (options.dryrun):
return
if (Action.PRTAUIDLIST in options.action):
f = self.open_file(self.cache.name, Action.PRTAUIDLIST)
# TODO get all auids for server
if (f):
LockssCacheAuId.printcsv(f, auids, "\t")
f.close()
if (Action.GETREPOSSPACE in options.action):
self.getreposspace()
if (Action.PRTREPOSSPACE in options.action):
f = self.open_file(self.cache.name, Action.PRTREPOSSPACE)
if (f):
RepositorySpace.printcsv(f, [ self.cache ], "\t")
f.close()
# actions below needs auids to operate on
if (not auids):
log.info("no matching auids");
return;
doUrls = Action.GETURLLIST in options.action
success = None
if (Action.GETAUSUMMARY in options.action):
self.getausummaries(auids, options.dir, doUrls, options.expire, options.noquit)
if (Action.PRTAUSUMMARY in options.action):
f = self.open_file(self.cache.name, Action.PRTAUSUMMARY)
if (f):
LockssCacheAuSummary.printcsv(f, auids, options.ausummarysort, options.ausummaryheaders, "\t")
f.close()
if (Action.PRTURLLIST in options.action):
dr = self.mkdir(options.action, self.cache.name)
if (dr):
UrlReport.printcsv("%s/%s" % (self.options.dir, server.name), #dir,
auids, options.urlsort, options.urlheaders, '\t', options.urlminversion)
if (Action.GETCRAWLSTATUS in options.action):
self.getcrawlstatus(auids, options.dir, options.noquit)
if (Action.PRTCRAWLSTATUS in options.action):
f = self.open_file(self.cache.name, Action.PRTCRAWLSTATUS)
if (f):
LockssCrawlStatus.printcsv(f, auids, options.crawlsort, options.ncrawllimit, options.crawlheaders, "\t")
f.close()
if (Action.GETCOMMPEERS in options.action):
self.getcommpeers(options.dir, options.noquit)
if (Action.PRTCOMMPEERS in options.action):
f = self.open_file(self.cache.name, Action.PRTCOMMPEERS)
if (f):
# TODO LockssCacheCommPeer.printcsv(f, self.cache)
f.close()
except LockssError as e:
log.error("EXCEPTION %s" % str(e))
finally:
log.debug2("------ Stop Processing %s" % server)
def process(self):
log.info("---")
log.info("Start Processing")
if (self.options.dryrun):
return
opts = self.options.__dict__
print "# COMMAND", self.options._COMMAND;
for key in ['ausetname', 'auseturl']:
if (opts.has_key(key)):
print "#", key, opts.get(key);
print "# ";
caches = self.get_caches();
print "# SERVER\t", "\n# SERVER\t".join([str(c) for c in caches]);
print "# ";
# TODO deal with serverlist - aka restrict to aus on given servers
fields = ["auid", "repl", "reportDate", "cache", "agree",
"sizeMB", "diskMB", "repository",
"#urls", "avg_version", "min_version", "max_version",
"best_copy"]
print "\t".join(fields);
for auid in self.options.auids:
# TODO restrict to servers in options.serverlist
auids = LockssCacheAuId.objects.filter(Q(auId=auid) ,self.get_Q_caches() )
log.info("AUID: %s" % "\t".join([auid, "#matches=%d" % auids.count()]));
profiles = []
for au in auids:
prof = {
'auid' : auid,
'repl' : au.replication(),
'reportDate' : "",
'cache' : au.cache.name,
'agree' : "",
'repository' : "",
'#urls' : "",
'avg_version' : "",
'min_version' : "",
'max_version' : "",
'sizeMB': "",
'diskMB': "",
"best_copy" : False
}
lausum = LockssCacheAuSummary.objects.filter(auId = au);
if (not lausum.exists()):
log.warn("No AuSummary Info for " + str(au))
else:
lausum = lausum[0];
if (lausum.agreement != None) : prof['agree'] = lausum.agreement
prof['repository'] = lausum.repository
prof['sizeMB'] = lausum.contentSizeMB()
prof['diskMB'] = lausum.diskUsageMB
prof['reportDate'] = lausum.reportDate
urlReport = UrlReport.objects.filter(auId = au);
if (not urlReport.exists()):
log.warn("No Url Info for " + str(au))
prof['nurls'] = 0
else:
report = urlReport[0];
urls = report.url_set;
version_info = urls.aggregate(Avg('version'), Min('version'), Max('version'))
prof['avg_version'] = version_info['version__avg'];
prof['min_version'] = version_info['version__min'];
prof['max_version'] = version_info['version__max'];
profiles.append(prof)
# find the au that has the max agreement and if there are multiple
# the one/one of the ones with the max avg_version number
# and designate as best_copy
#
# find max avg_version number
if (not [] == profiles):
max_agree = max(v['agree'] for v in profiles)
# find all that have that max_agree value
candidates = []
for prof in profiles:
if (prof['agree'] == max_agree):
candidates.append(prof)
max_version = max(v['avg_version'] for v in candidates)
for candidate in profiles:
if (candidate['avg_version'] == max_version):
candidate['best_copy'] = 'True'
break
assert(candidate)
for prof in profiles:
vals = [];
for f in fields:
v = prof[f]
if (v.__class__ == float):
vals.append("%.2f" % v)
else:
vals.append(str(v))
print "\t".join(vals)
for i in range(0,3):
print "";
log.info("Stop Processing")
def log_opts(self):
LockssScript.log_opts(self)
if (self.options.action):
for a in self.options.action:
log.info("ACTION = %s", a)
def sleep(s):
log.info("Sleeping %s sec ..." % s)
time.sleep(s)
def process(self):
log.info("---")
log.info("Start Processing")
if (self.options.dryrun):
return
if (self.options.explainheaders):
print ReportScript.explainheaders(self);
return;
print "# COMMAND", self.options._COMMAND;
print "# ";
if (self.options.urlminversion > 1):
print "# listing only urls with a version number greater equal than ", self.options.urlminversion
print "# ";
masterAuIds = self.collectMasterAuIdInstances()
print self.report_preamble();
headers = self.options.reportheaders;
fields = [];
if ('reportDate' in headers):
fields.append("urlReport")
if ("auId" in headers or "cache" in headers ):
fields.append("urlReport__auId")
if ("cache" in headers ):
fields.append("urlReport__auId__cache")
relatedfields = ",".join(fields);
caches = None;
if (self.options.serverlist):
caches = self.options.cachelist;
for mauid in masterAuIds:
print "\n# AUID %s" % mauid.auId
mauidRepl = None;
urlKnownRepl = None;
qu = self.get_url_query(mauid, self.options.urlminversion, caches);
urls = [];
if (fields):
urls = Url.objects.filter(qu).order_by('name').select_related(relatedfields);
else:
urls = Url.objects.filter(qu).order_by('name');
if (urls):
print "\t".join(headers)
for u in urls:
vals = [];
for h in headers:
try:
if (h == 'name'):
vals.append(u.name)
elif (h == 'size'):
vals.append(str(u.size))
elif (h == 'version'):
vals.append(str(u.version))
elif (h == 'auId'):
vals.append(u.urlReport.auId.auId)
elif (h == 'plugin'):
vals.append(mauid.plugin)
elif (h == 'baseUrl'):
vals.append(mauid.baseUrl)
elif (h == 'extraParams'):
vals.append(mauid.extraParams)
elif (h == 'reportDate'):
vals.append(str(u.urlReport.reportDate))
elif (h == 'cache'):
vals.append(str(u.urlReport.auId.cache));
elif (h == 'urlRepl'):
repl = str(Url.objects.filter(name=u.name).count());
vals.append(str(repl))
elif (h == 'repl'):
if (not mauidRepl):
mauidRepl = str(mauid.replication())
vals.append(mauidRepl)
elif (h == 'urlKnownRepl'):
if (not urlKnownRepl):
# number of cache aus for which we have a urlReport i
qu = Q(auId__auId__exact=mauid.auId)
urlKnownRepl = str(UrlReport.objects.filter(qu).count())
vals.append(urlKnownRepl)
else:
raise LockssError("Unknown report header %s" % h)
except Exception as e:
vals.append("#Error url_id=%d %s" % (u.id, str(e)));
print "\t".join(vals)
log.info("Stop Processing")
def process(self):
log.info("---")
log.info("Start Processing")
if self.options.explainheaders:
print ReportScript.explainheaders(self)
return
if self.options.dryrun:
return
opts = self.options.__dict__
print "# COMMAND", self.options._COMMAND
for key in ["repl", "replserver", "agreement"]:
if opts.has_key(key):
print "#", key, opts.get(key)
print "# "
if self.options.problemsonly:
print "# listing only archival units that appear to have a problem"
print "# "
masterAuIds = self.collectMasterAuIdInstances()
print self.report_preamble()
headers = self.options.reportheaders
print "\t".join(headers)
for mauid in masterAuIds:
auids = LockssCacheAuId.objects.filter(Q(auId=mauid.auId), self.get_Q_caches())
repl = mauid.replication()
replserver = auids.count()
prof = {
"auid": mauid.auId,
"repl": repl,
"lowrepl": repl < self.options.repl,
"replserver": replserver,
"lowreplserver": replserver < self.options.replserver,
"nLowAgree": 0,
"reportDate": {"min": None, "max": None},
"nausummary": None,
"missingauinfo": False,
"sizeMB": {"min": None, "max": None, "avg": None, "sum": None},
"diskMB": {"min": None, "max": None, "avg": None, "sum": None},
"agree": {"min": None, "max": None, "avg": None, "sum": None},
"caches": [],
}
nagree = 0
nlausum = 0
for au in auids:
lau = au.getlocksscacheausummary()
if lau:
nlausum = nlausum + 1
if lau.agreement < self.options.agreement:
prof["nLowAgree"] = prof["nLowAgree"] + 1
delta, prof["agree"] = self.add_value(lau.agreement, "agree", prof["agree"])
nagree = nagree + delta
delta, prof["sizeMB"] = self.add_value(lau.contentSizeMB(), "sizeMB", prof["sizeMB"])
assert delta == 1, "%s sizeMB: %s" % (str(lau), str(prof["sizeMB"]))
delta, prof["diskMB"] = self.add_value(lau.diskUsageMB, "sizeMB", prof["diskMB"])
assert delta == 1, "%s diskMB: %s" % (str(lau), str(prof["diskMB"]))
delta, prof["reportDate"] = self.add_value(lau.reportDate, "reportDate", prof["reportDate"])
assert delta == 1, "%s reportDare: %s" % (str(lau), str(prof["reportDate"]))
else:
prof["missingauinfo"] = True
prof["caches"].append(au.cache.name)
prof["caches"] = sorted(prof["caches"])
prof["nausummary"] = nlausum
if nagree > 0:
prof["agree"]["avg"] = prof["agree"]["sum"] / nagree
if nlausum > 0:
prof["sizeMB"]["avg"] = prof["sizeMB"]["sum"] / nlausum
prof["diskMB"]["avg"] = prof["diskMB"]["sum"] / nlausum
if (not self.options.problemsonly) or prof["lowrepl"] or prof["lowreplserver"] or prof["nLowAgree"] > 0:
vals = []
for f in headers:
keys = f.split(" ")
if len(keys) == 1:
v = prof[f]
else:
v = prof[keys[0]][keys[1]]
if v.__class__ == float:
v = "%.2f" % v
elif v.__class__ == list:
v = "\t".join(sorted(v))
vals.append(str(v))
print "\t".join(vals)
log.info("Stop Processing")
