def test_alert_email_mtcaller(self):
"""
Generate an e-mail alert and verify that it was sent (this is where we
use 'monkey patching'). For this case, caller is ''.
"""
self.dbgfunc()
fakesmtp.inbox = []
logfile = self.tmpdir('alert_email.log')
targets = "[email protected], [email protected], [email protected]"
payload = 'this is an e-mail alert'
sender = 'hpssic@' + util.hostname(long=True)
cfg = CrawlConfig.CrawlConfig()
cfg.add_section('crawler')
cfg.add_section('alerts')
cfg.set('crawler', 'logpath', logfile)
cfg.set('alerts', 'email', targets)
CrawlConfig.log(logpath=logfile, close=True)
x = Alert.Alert(caller='', msg=payload, cfg=cfg)
m = fakesmtp.inbox[0]
self.expected(targets, ', '.join(m.to_address))
self.expected(m.from_address, sender)
self.expected_in('sent mail to', util.contents(logfile))
self.expected_in(payload, m.fullmessage)
def test_alert_shell_nospec(self):
"""
Generate a shell alert and verify that it ran. With no '%s' in the
shell alert string, no message should be offered for formatting.
"""
self.dbgfunc()
logfile = self.tmpdir('alert_shell.log')
outfile = self.tmpdir('alert_shell.out')
runfile = self.tmpdir('runme')
f = open(runfile, 'w')
f.write("#!/bin/bash\n")
f.write("echo \"ALERT: $*\" > %s\n" % outfile)
f.close()
os.chmod(runfile,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP |
stat.S_IROTH | stat.S_IXOTH)
cfg = CrawlConfig.CrawlConfig()
cfg.add_section('crawler')
cfg.add_section('AlertTest')
cfg.add_section('alert_section')
cfg.set('crawler', 'logpath', logfile)
cfg.set('AlertTest', 'alerts', 'alert_section')
cfg.set('alert_section', 'shell', runfile)
CrawlConfig.log(logpath=logfile, close=True)
x = Alert.Alert(caller='AlertTest', msg='this is a test message',
cfg=cfg)
expected = "ran: '%s'" % runfile
self.expected_in(expected, util.contents(logfile))
self.assertPathPresent(outfile)
def test_alert_shell_nospec(self):
"""
Generate a shell alert and verify that it ran. With no '%s' in the
shell alert string, no message should be offered for formatting.
"""
self.dbgfunc()
logfile = self.tmpdir('alert_shell.log')
outfile = self.tmpdir('alert_shell.out')
runfile = self.tmpdir('runme')
f = open(runfile, 'w')
f.write("#!/bin/bash\n")
f.write("echo \"ALERT: $*\" > %s\n" % outfile)
f.close()
os.chmod(
runfile, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP
| stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
cfg = CrawlConfig.CrawlConfig()
cfg.add_section('crawler')
cfg.add_section('AlertTest')
cfg.add_section('alert_section')
cfg.set('crawler', 'logpath', logfile)
cfg.set('AlertTest', 'alerts', 'alert_section')
cfg.set('alert_section', 'shell', runfile)
CrawlConfig.log(logpath=logfile, close=True)
x = Alert.Alert(caller='AlertTest',
msg='this is a test message',
cfg=cfg)
expected = "ran: '%s'" % runfile
self.expected_in(expected, util.contents(logfile))
self.assertPathPresent(outfile)
def test_alert_email_mtcaller(self):
"""
Generate an e-mail alert and verify that it was sent (this is where we
use 'monkey patching'). For this case, caller is ''.
"""
self.dbgfunc()
fakesmtp.inbox = []
logfile = self.tmpdir('alert_email.log')
targets = "[email protected], [email protected], [email protected]"
payload = 'this is an e-mail alert'
sender = 'hpssic@' + util.hostname(long=True)
cfg = CrawlConfig.CrawlConfig()
cfg.add_section('crawler')
cfg.add_section('alerts')
cfg.set('crawler', 'logpath', logfile)
cfg.set('alerts', 'email', targets)
CrawlConfig.log(logpath=logfile, close=True)
x = Alert.Alert(caller='', msg=payload,
cfg=cfg)
m = fakesmtp.inbox[0]
self.expected(targets, ', '.join(m.to_address))
self.expected(m.from_address, sender)
self.expected_in('sent mail to', util.contents(logfile))
self.expected_in(payload, m.fullmessage)
def main(cfg):
"""
Plugin example
"""
try:
msg = cfg.get('example', 'message')
except ConfigParser.NoOptionError:
msg = 'No message in configuration'
CrawlConfig.log('EXAMPLE: This is plugin EXAMPLE saying "%s"' % msg)
def main(cfg):
"""
Migration Purge Record Ager (mpra) reads the database tables BFMIGRREC and
BFPURGEREC and reports migration and purge records that are older than the
age specified in the configuration.
"""
if cfg is None:
cfg = CrawlConfig.get_config()
age = cfg.get_time('mpra', 'age')
end = time.time() - age
start = mpra_lib.mpra_fetch_recent("migr")
#
# If the configured age has been moved back in time, so that end is before
# start, we need to reset and start scanning from the beginning of time.
#
if end < start:
start = 0
CrawlConfig.log("migr recs after %d (%s) before %d (%s)" %
(start, util.ymdhms(start), end, util.ymdhms(end)))
result = mpra_lib.age("migr", start=start, end=end, mark=True)
CrawlConfig.log("found %d migration records in the range" % result)
rval = result
start = mpra_lib.mpra_fetch_recent("purge")
CrawlConfig.log("Looking for expired purge locks")
result = mpra_lib.xplocks(mark=True)
CrawlConfig.log("found %d expired purge locks" % result)
rval += result
return rval
def test_html_report(self):
"""
Try running 'html report > filename' and verify that 1) no traceback
occurs and 2) something is actually written to the output file.
"""
self.dbgfunc()
cfpath = self.tmpdir("crawl.cfg")
cfg = CrawlConfig.add_config()
cfg.crawl_write(open(cfpath, 'w'))
cmd = "html report --config %s" % cfpath
CrawlConfig.log(cmd, close=True)
result = pexpect.run(cmd)
if "HPSS Unavailable" in result:
pytest.skip("HPSS Unavailable")
self.validate_report(result)
def test_html_report(self):
"""
Try running 'html report > filename' and verify that 1) no traceback
occurs and 2) something is actually written to the output file.
"""
self.dbgfunc()
cfpath = self.tmpdir("crawl.cfg")
cfg = CrawlConfig.add_config()
cfg.crawl_write(open(cfpath, 'w'))
cmd = "html report --config %s" % cfpath
CrawlConfig.log(cmd, close=True)
result = pexpect.run(cmd)
if "HPSS Unavailable" in result:
pytest.skip("HPSS Unavailable")
self.validate_report(result)
def main(cfg):
"""
Generate an html-formatted report and store it at the designated location
"""
CrawlConfig.log("html_plugin starting")
fpath = cfg.get('html', 'output_path')
rpt = html_lib.get_html_report(cfg=cfg)
npath = fpath + '.new'
opath = fpath + '.old'
with open(npath, 'w') as out:
out.write(rpt)
if os.path.exists(fpath):
os.rename(fpath, opath)
os.rename(npath, fpath)
CrawlConfig.log("html_plugin finished")
def test_maybe_update_hsi_cant(muh_prep, tmpdir):
"""
If we don't have write permission on the target, then even if we should
update, we can't. In this case, should log a message.
"""
pytest.dbgfunc()
lp = tmpdir.join('crawl.test.log')
rf = test_maybe_update_hsi_cant
path = ":".join([rf.bin.strpath, rf.hsihome])
with U.tmpenv('PATH', path):
CrawlConfig.log(logpath=lp.strpath, close=True)
hpss.maybe_update_hsi()
c = rf.file.read()
assert 'not changed' in c
assert os.path.exists(lp.strpath)
c = lp.read()
assert MSG.hsi_wrap_ood in c
CrawlConfig.log(close=True)
def main(cfg):
"""
HSI demo
"""
CrawlConfig.log("hsi-demo: sending output to hsi.out")
hsi_prompt = "]:"
S = pexpect.spawn("/opt/public/bin/hsi")
S.logfile = f = open("hsi.out", 'a')
S.expect(hsi_prompt)
S.sendline("ls")
S.expect(hsi_prompt)
S.sendline("quit")
S.expect(pexpect.EOF)
S.logfile.close()
S.close()
def test_alert_log(self):
"""
Generate a log alert and verify that the message was written to the
correct log file.
"""
self.dbgfunc()
logfile = self.tmpdir('alert_log.log')
cfg = CrawlConfig.CrawlConfig()
cfg.add_section('crawler')
cfg.add_section('AlertTest')
cfg.add_section('alert_section')
cfg.set('crawler', 'logpath', logfile)
cfg.set('AlertTest', 'alerts', 'alert_section')
cfg.set('alert_section', 'log', "%s")
CrawlConfig.log(logpath=logfile, close=True)
x = Alert.Alert(caller='AlertTest', msg='this is a test message',
cfg=cfg)
self.expected_in('this is a test message', util.contents(logfile))
def test_alert_log(self):
"""
Generate a log alert and verify that the message was written to the
correct log file.
"""
self.dbgfunc()
logfile = self.tmpdir('alert_log.log')
cfg = CrawlConfig.CrawlConfig()
cfg.add_section('crawler')
cfg.add_section('AlertTest')
cfg.add_section('alert_section')
cfg.set('crawler', 'logpath', logfile)
cfg.set('AlertTest', 'alerts', 'alert_section')
cfg.set('alert_section', 'log', "%s")
CrawlConfig.log(logpath=logfile, close=True)
x = Alert.Alert(caller='AlertTest',
msg='this is a test message',
cfg=cfg)
self.expected_in('this is a test message', util.contents(logfile))
def test_alert_use_other(self):
"""
A use directive sends us to another config section where we generate a
log alert and verify that the message was written to the correct log
file.
"""
self.dbgfunc()
logfile = self.tmpdir('alert_use.log')
cfg = CrawlConfig.CrawlConfig()
cfg.add_section('crawler')
cfg.add_section('AlertTest')
cfg.add_section('alert_section')
cfg.add_section('other_section')
cfg.set('crawler', 'logpath', logfile)
cfg.set('AlertTest', 'alerts', 'alert_section')
cfg.set('alert_section', 'use', "other_section")
cfg.set('other_section', 'log', "%s")
CrawlConfig.log(logpath=logfile, close=True)
payload = 'this is a test message from %s' % util.my_name()
x = Alert.Alert(caller='AlertTest', msg=payload, cfg=cfg)
self.expected_in(payload, util.contents(logfile))
def test_dlog(self):
"""
Test method dlog on daemon object
"""
lfname = self.tmpdir('daemon.dlog.log')
lf = CrawlConfig.log(logpath=lfname)
a = daemon.Daemon(self.tmpdir("daemon_pid"), logger=lf)
logmsg = "testing the dlog method of %s" % a
a.dlog(logmsg)
self.assertTrue(
logmsg in util.contents(lfname), "Expected '%s' in '%s'" %
(logmsg, util.line_quote(util.contents(lfname))))
def test_alert_use_other(self):
"""
A use directive sends us to another config section where we generate a
log alert and verify that the message was written to the correct log
file.
"""
self.dbgfunc()
logfile = self.tmpdir('alert_use.log')
cfg = CrawlConfig.CrawlConfig()
cfg.add_section('crawler')
cfg.add_section('AlertTest')
cfg.add_section('alert_section')
cfg.add_section('other_section')
cfg.set('crawler', 'logpath', logfile)
cfg.set('AlertTest', 'alerts', 'alert_section')
cfg.set('alert_section', 'use', "other_section")
cfg.set('other_section', 'log', "%s")
CrawlConfig.log(logpath=logfile, close=True)
payload = 'this is a test message from %s' % util.my_name()
x = Alert.Alert(caller='AlertTest', msg=payload,
cfg=cfg)
self.expected_in(payload, util.contents(logfile))
def test_alert_email_defcfg(self):
"""
Generate an e-mail alert using the default config and verify that it
was sent (this is where we use 'monkey patching').
"""
self.dbgfunc()
fakesmtp.inbox = []
CrawlConfig.add_config(close=True)
# with U.tmpenv('CRAWL_CONF', 'hpssic_test.cfg'):
with U.tmpenv('CRAWL_CONF', None):
logfile = self.tmpdir('alert_email.log')
targets = "[email protected], [email protected]"
payload = 'this is an e-mail alert'
sender = 'hpssic@' + util.hostname(long=True)
CrawlConfig.log(logpath=logfile, close=True)
x = Alert.Alert(caller='cv', msg=payload)
m = fakesmtp.inbox[0]
self.expected(', '.join(m.to_address), targets)
self.expected(m.from_address, sender)
self.expected_in('sent mail to', util.contents(logfile))
self.expected_in(payload, m.fullmessage)
def test_dlog(self):
"""
Test method dlog on daemon object
"""
lfname = self.tmpdir('daemon.dlog.log')
lf = CrawlConfig.log(logpath=lfname)
a = daemon.Daemon(self.tmpdir("daemon_pid"), logger=lf)
logmsg = "testing the dlog method of %s" % a
a.dlog(logmsg)
self.assertTrue(logmsg in util.contents(lfname),
"Expected '%s' in '%s'" %
(logmsg,
util.line_quote(util.contents(lfname))))
def test_alert_email_defcfg(self):
"""
Generate an e-mail alert using the default config and verify that it
was sent (this is where we use 'monkey patching').
"""
self.dbgfunc()
fakesmtp.inbox = []
CrawlConfig.add_config(close=True)
# with U.tmpenv('CRAWL_CONF', 'hpssic_test.cfg'):
with U.tmpenv('CRAWL_CONF', None):
logfile = self.tmpdir('alert_email.log')
targets = "[email protected], [email protected]"
payload = 'this is an e-mail alert'
sender = 'hpssic@' + util.hostname(long=True)
CrawlConfig.log(logpath=logfile, close=True)
x = Alert.Alert(caller='cv', msg=payload)
m = fakesmtp.inbox[0]
self.expected(', '.join(m.to_address), targets)
self.expected(m.from_address, sender)
self.expected_in('sent mail to', util.contents(logfile))
self.expected_in(payload, m.fullmessage)
def main(cfg):
"""
This plugin will generate a report and send it to the designated e-mail
address(es).
"""
rval = 0
try:
if cfg is None:
cfg = CrawlConfig.get_config()
subject = "%s %s" % (cfg.get('rpt', 'subject'),
time.strftime("%Y.%m%d %H:%M:%S",
time.localtime()))
CrawlMail.send(sender=cfg.get('rpt', 'sender'),
to='rpt.recipients',
subj=subject,
msg=rpt_lib.get_report())
except Exception as e:
rval = 1
CrawlConfig.log("Failure in rpt_lib: '%s'" % str(e))
return rval
def main(cfg):
"""
This plugin will generate a report and send it to the designated e-mail
address(es).
"""
rval = 0
try:
if cfg is None:
cfg = CrawlConfig.get_config()
subject = "%s %s" % (cfg.get(
'rpt',
'subject'), time.strftime("%Y.%m%d %H:%M:%S", time.localtime()))
CrawlMail.send(sender=cfg.get('rpt', 'sender'),
to='rpt.recipients',
subj=subject,
msg=rpt_lib.get_report())
except Exception as e:
rval = 1
CrawlConfig.log("Failure in rpt_lib: '%s'" % str(e))
return rval
def main(cfg):
"""
Tape Copy Checker retrieves the necessary information from the DB2 database
to find files where the number of copies stored may not match the number
called for by the COS.
"""
# retrieve configuration items as needed
how_many = int(cfg.get_d(tcc_lib.sectname(), 'operations', 10))
CrawlConfig.log("tape-copy-checker: firing up for %d items" % how_many)
# retrieve COS info
cosinfo = tcc_lib.get_cos_info()
# check for priority file(s)
pri_glob = cfg.get_d(tcc_lib.sectname(), 'priority', '')
if pri_glob != '':
if 0 < tcc_priority(pri_glob, cosinfo):
return
# get the nsobject_id of the next bitfile to process from mysql
next_nsobj_id = tcc_lib.get_next_nsobj_id(cfg)
CrawlConfig.log("next nsobject id = %d" % next_nsobj_id)
# fetch the next N bitfiles from DB2
CrawlConfig.log("looking for nsobject ids between %d and %d" %
(next_nsobj_id, next_nsobj_id + how_many - 1))
try:
bfl = tcc_lib.get_bitfile_set(int(next_nsobj_id), how_many)
except U.HpssicError as e:
bfl = []
pass
CrawlConfig.log("got %d bitfiles" % len(bfl))
errcount = 0
if len(bfl) == 0:
for oid in range(next_nsobj_id, next_nsobj_id + how_many):
tcc_lib.record_checked_ids(cfg, oid, oid, 1, 0)
if cfg.getboolean(tcc_lib.sectname(), 'verbose'):
CrawlConfig.log("Object %d is not complete" % oid)
errcount += 1
else:
# for each bitfile, if it does not have the right number of copies,
# report it
for bf in bfl:
correct = 1
error = 0
if bf['SC_COUNT'] != cosinfo[bf['BFATTR_COS_ID']]:
tcc_lib.tcc_report(bf, cosinfo)
correct = 0
error = 1
CrawlConfig.log("%s %s %d != %d" %
(bf['OBJECT_ID'], tcc_lib.hexstr(bf['BFID']),
bf['SC_COUNT'], cosinfo[bf['BFATTR_COS_ID']]))
elif cfg.getboolean(tcc_lib.sectname(), 'verbose'):
CrawlConfig.log("%s %s %d == %d" %
(bf['OBJECT_ID'], tcc_lib.hexstr(bf['BFID']),
bf['SC_COUNT'], cosinfo[bf['BFATTR_COS_ID']]))
last_obj_id = int(bf['OBJECT_ID'])
tcc_lib.record_checked_ids(cfg, last_obj_id, last_obj_id, correct,
error)
errcount += error
CrawlConfig.log("last nsobject in range: %d" % last_obj_id)
return errcount
def main(cfg):
"""
Main entry point for the cv plugin
"""
# Get stuff we need -- the logger object, dataroot, etc.
CrawlConfig.log("firing up")
plugdir = cfg.get('crawler', 'plugin-dir')
dataroot = util.csv_list(cfg.get(plugin_name, 'dataroot'))
odds = cfg.getfloat(plugin_name, 'odds')
n_ops = int(cfg.get(plugin_name, 'operations'))
# Initialize our statistics
(t_checksums, t_matches, t_failures) = get_stats()
(checksums, matches, failures) = (0, 0, 0)
# Fetch the list of HPSS objects that we're looking at from the
# database
try:
clist = Checkable.Checkable.get_list(prob=odds, rootlist=dataroot)
except CrawlDBI.DBIerror as e:
if any([util.rgxin(msg, str(e))
for msg in ["no such table: checkables",
"Table '.*' doesn't exist"]]):
CrawlConfig.log("calling ex_nihilo")
Checkable.Checkable.ex_nihilo(dataroot=dataroot)
clist = Checkable.Checkable.get_list(prob=odds)
else:
raise
except StandardError as e:
if 'Please call .ex_nihilo()' in str(e):
CrawlConfig.log("calling ex_nihilo")
Checkable.Checkable.ex_nihilo(dataroot=dataroot)
clist = Checkable.Checkable.get_list(prob=odds)
else:
raise
# We're going to process n_ops things in the HPSS namespace
for op in range(n_ops):
# if the list from the database is empty, there's nothing to do
if 0 < len(clist):
# but it's not, so grab the first item and check it
item = clist.pop(0)
CrawlConfig.log("[%d] checking %s" % (item.rowid, item))
ilist = item.check()
# Expected outcomes that check can return:
# list of Checkables: read dir or checksummed files (may be empty)
# Alert: checksum verify failed
# 'access denied': unaccessible directory
# 'matched': a checksum was verified
# 'checksummed': file was checksummed
# 'skipped': file was skipped
# 'unavailable': HPSS is temporarily unavailable
# StandardError: invalid Checkable type (not 'f' or 'd')
#
if type(ilist) == str:
if ilist == "access denied":
CrawlConfig.log("dir %s not accessible" % item.path)
# clist.remove(item)
elif ilist == "matched":
matches += 1
CrawlConfig.log("%s checksums matched" % item.path)
elif ilist == "checksummed":
# checksums += 1
CrawlConfig.log("%s checksummed" % item.path)
elif ilist == "skipped":
CrawlConfig.log("%s skipped" % item.path)
elif ilist == "unavailable":
CrawlConfig.log("HPSS is not available")
break
else:
CrawlConfig.log("unexpected string returned " +
"from Checkable: '%s'" % ilist)
elif type(ilist) == list:
CrawlConfig.log("in %s, found:" % item)
for n in ilist:
CrawlConfig.log(">>> %s" % str(n))
if 'f' == n.type and n.checksum != 0:
CrawlConfig.log(".. previously checksummed")
# checksums += 1
elif isinstance(ilist, Checkable.Checkable):
CrawlConfig.log("Checkable returned - file checksummed" +
" - %s, %s" % (ilist.path, ilist.checksum))
# checksums += 1
elif isinstance(ilist, Alert.Alert):
CrawlConfig.log("Alert generated: '%s'" %
ilist.msg())
failures += 1
else:
CrawlConfig.log("unexpected return val from " +
"Checkable.check: %s: %r" %
(type(ilist), ilist))
# Report the statistics in the log
# ** For checksums, we report the current total minus the previous
# ** For matches and failures, we counted them up during the iteration
# ** See the description of get_stats for why we don't store total
# checksums
p_checksums = t_checksums
t_matches += matches
t_failures += failures
cv_lib.update_stats((t_matches, t_failures))
(t_checksums, t_matches, t_failures) = get_stats()
CrawlConfig.log("files checksummed: %d; " % (t_checksums - p_checksums) +
"checksums matched: %d; " % matches +
"failures: %d" % failures)
CrawlConfig.log("totals checksummed: %d; " % t_checksums +
"matches: %d; " % t_matches +
"failures: %d" % t_failures)
# Report the dimension data in the log
d = Dimension.Dimension(name='cos')
t = Dimension.Dimension(name='cart')
CrawlConfig.log(d.report())
CrawlConfig.log(t.report())
return failures
def main(cfg):
"""
Main entry point for the cv plugin
"""
# Get stuff we need -- the logger object, dataroot, etc.
CrawlConfig.log("firing up")
plugdir = cfg.get('crawler', 'plugin-dir')
dataroot = util.csv_list(cfg.get(plugin_name, 'dataroot'))
odds = cfg.getfloat(plugin_name, 'odds')
n_ops = int(cfg.get(plugin_name, 'operations'))
# Initialize our statistics
(t_checksums, t_matches, t_failures) = get_stats()
(checksums, matches, failures) = (0, 0, 0)
# Fetch the list of HPSS objects that we're looking at from the
# database
try:
clist = Checkable.Checkable.get_list(prob=odds, rootlist=dataroot)
except CrawlDBI.DBIerror as e:
if any([
util.rgxin(msg, str(e)) for msg in
["no such table: checkables", "Table '.*' doesn't exist"]
]):
CrawlConfig.log("calling ex_nihilo")
Checkable.Checkable.ex_nihilo(dataroot=dataroot)
clist = Checkable.Checkable.get_list(prob=odds)
else:
raise
except StandardError as e:
if 'Please call .ex_nihilo()' in str(e):
CrawlConfig.log("calling ex_nihilo")
Checkable.Checkable.ex_nihilo(dataroot=dataroot)
clist = Checkable.Checkable.get_list(prob=odds)
else:
raise
# We're going to process n_ops things in the HPSS namespace
for op in range(n_ops):
# if the list from the database is empty, there's nothing to do
if 0 < len(clist):
# but it's not, so grab the first item and check it
item = clist.pop(0)
CrawlConfig.log("[%d] checking %s" % (item.rowid, item))
ilist = item.check()
# Expected outcomes that check can return:
# list of Checkables: read dir or checksummed files (may be empty)
# Alert: checksum verify failed
# 'access denied': unaccessible directory
# 'matched': a checksum was verified
# 'checksummed': file was checksummed
# 'skipped': file was skipped
# 'unavailable': HPSS is temporarily unavailable
# StandardError: invalid Checkable type (not 'f' or 'd')
#
if type(ilist) == str:
if ilist == "access denied":
CrawlConfig.log("dir %s not accessible" % item.path)
# clist.remove(item)
elif ilist == "matched":
matches += 1
CrawlConfig.log("%s checksums matched" % item.path)
elif ilist == "checksummed":
# checksums += 1
CrawlConfig.log("%s checksummed" % item.path)
elif ilist == "skipped":
CrawlConfig.log("%s skipped" % item.path)
elif ilist == "unavailable":
CrawlConfig.log("HPSS is not available")
break
else:
CrawlConfig.log("unexpected string returned " +
"from Checkable: '%s'" % ilist)
elif type(ilist) == list:
CrawlConfig.log("in %s, found:" % item)
for n in ilist:
CrawlConfig.log(">>> %s" % str(n))
if 'f' == n.type and n.checksum != 0:
CrawlConfig.log(".. previously checksummed")
# checksums += 1
elif isinstance(ilist, Checkable.Checkable):
CrawlConfig.log("Checkable returned - file checksummed" +
" - %s, %s" % (ilist.path, ilist.checksum))
# checksums += 1
elif isinstance(ilist, Alert.Alert):
CrawlConfig.log("Alert generated: '%s'" % ilist.msg())
failures += 1
else:
CrawlConfig.log("unexpected return val from " +
"Checkable.check: %s: %r" %
(type(ilist), ilist))
# Report the statistics in the log
# ** For checksums, we report the current total minus the previous
# ** For matches and failures, we counted them up during the iteration
# ** See the description of get_stats for why we don't store total
# checksums
p_checksums = t_checksums
t_matches += matches
t_failures += failures
cv_lib.update_stats((t_matches, t_failures))
(t_checksums, t_matches, t_failures) = get_stats()
CrawlConfig.log("files checksummed: %d; " % (t_checksums - p_checksums) +
"checksums matched: %d; " % matches +
"failures: %d" % failures)
CrawlConfig.log("totals checksummed: %d; " % t_checksums +
"matches: %d; " % t_matches + "failures: %d" % t_failures)
# Report the dimension data in the log
d = Dimension.Dimension(name='cos')
t = Dimension.Dimension(name='cart')
CrawlConfig.log(d.report())
CrawlConfig.log(t.report())
return failures
def main(cfg):
"""
Tape Copy Checker retrieves the necessary information from the DB2 database
to find files where the number of copies stored may not match the number
called for by the COS.
"""
# retrieve configuration items as needed
how_many = int(cfg.get_d(tcc_lib.sectname(), 'operations', 10))
CrawlConfig.log("tape-copy-checker: firing up for %d items" % how_many)
# retrieve COS info
cosinfo = tcc_lib.get_cos_info()
# check for priority file(s)
pri_glob = cfg.get_d(tcc_lib.sectname(), 'priority', '')
if pri_glob != '':
if 0 < tcc_priority(pri_glob, cosinfo):
return
# get the nsobject_id of the next bitfile to process from mysql
next_nsobj_id = tcc_lib.get_next_nsobj_id(cfg)
CrawlConfig.log("next nsobject id = %d" % next_nsobj_id)
# fetch the next N bitfiles from DB2
CrawlConfig.log("looking for nsobject ids between %d and %d"
% (next_nsobj_id, next_nsobj_id+how_many-1))
try:
bfl = tcc_lib.get_bitfile_set(int(next_nsobj_id),
how_many)
except U.HpssicError as e:
bfl = []
pass
CrawlConfig.log("got %d bitfiles" % len(bfl))
errcount = 0
if len(bfl) == 0:
for oid in range(next_nsobj_id, next_nsobj_id+how_many):
tcc_lib.record_checked_ids(cfg, oid, oid, 1, 0)
if cfg.getboolean(tcc_lib.sectname(), 'verbose'):
CrawlConfig.log("Object %d is not complete" % oid)
errcount += 1
else:
# for each bitfile, if it does not have the right number of copies,
# report it
for bf in bfl:
correct = 1
error = 0
if bf['SC_COUNT'] != cosinfo[bf['BFATTR_COS_ID']]:
tcc_lib.tcc_report(bf, cosinfo)
correct = 0
error = 1
CrawlConfig.log("%s %s %d != %d" %
(bf['OBJECT_ID'],
tcc_lib.hexstr(bf['BFID']),
bf['SC_COUNT'],
cosinfo[bf['BFATTR_COS_ID']]))
elif cfg.getboolean(tcc_lib.sectname(), 'verbose'):
CrawlConfig.log("%s %s %d == %d" %
(bf['OBJECT_ID'],
tcc_lib.hexstr(bf['BFID']),
bf['SC_COUNT'],
cosinfo[bf['BFATTR_COS_ID']]))
last_obj_id = int(bf['OBJECT_ID'])
tcc_lib.record_checked_ids(cfg,
last_obj_id,
last_obj_id,
correct,
error)
errcount += error
CrawlConfig.log("last nsobject in range: %d" % last_obj_id)
return errcount
