def main(cfg):
"""
Migration Purge Record Ager (mpra) reads the database tables BFMIGRREC and
BFPURGEREC and reports migration and purge records that are older than the
age specified in the configuration.
"""
if cfg is None:
cfg = CrawlConfig.get_config()
age = cfg.get_time('mpra', 'age')
end = time.time() - age
start = mpra_lib.mpra_fetch_recent("migr")
#
# If the configured age has been moved back in time, so that end is before
# start, we need to reset and start scanning from the beginning of time.
#
if end < start:
start = 0
CrawlConfig.log("migr recs after %d (%s) before %d (%s)" %
(start, util.ymdhms(start), end, util.ymdhms(end)))
result = mpra_lib.age("migr", start=start, end=end, mark=True)
CrawlConfig.log("found %d migration records in the range" % result)
rval = result
start = mpra_lib.mpra_fetch_recent("purge")
CrawlConfig.log("Looking for expired purge locks")
result = mpra_lib.xplocks(mark=True)
CrawlConfig.log("found %d expired purge locks" % result)
rval += result
return rval
def test_ctor_reset_atime_cfg_true(self):
"""
If reset_atime is specified in the config as True, it should be True
"""
cf_name = self.tmpdir(util.my_name() + ".cfg")
# write out a config file with no reset_atime spec
self.write_cfg_file(cf_name, self.cfg_d)
# make the test config the default
CrawlConfig.get_config(cfname=cf_name, reset=True)
# get an hpss.HSI object and check its reset_atime attribute
h = hpss.HSI(connect=False)
self.expected(True, h.reset_atime)
CrawlConfig.get_config(reset=True, soft=True)
def test_ctor_reset_atime_call_true(self):
"""
If reset_atime is specified in the call as True, it should be True,
even if it's specified as False in the config
"""
cf_name = self.tmpdir(util.my_name() + ".cfg")
# write out a config file with no reset_atime spec
cfg = copy.deepcopy(self.cfg_d)
cfg['cv']['reset_atime'] = 'no'
self.write_cfg_file(cf_name, cfg)
# make the test config the default
CrawlConfig.get_config(cfname=cf_name, reset=True)
# get an hpss.HSI object and check its reset_atime attribute
h = hpss.HSI(connect=False, reset_atime=True)
self.expected(True, h.reset_atime)
CrawlConfig.get_config(reset=True, soft=True)
def test_ctor_reset_atime_default(self):
"""
If reset_atime is not specified in the config or argument list, it
should default to False
"""
cf_name = self.tmpdir(util.my_name() + ".cfg")
# write out a config file with no reset_atime spec
cd = copy.deepcopy(self.cfg_d)
del cd['cv']['reset_atime']
self.write_cfg_file(cf_name, cd)
# make the test config the default
CrawlConfig.get_config(cfname=cf_name, reset=True)
# get an hpss.HSI object and check its reset_atime attribute
h = hpss.HSI(connect=False)
self.expected(False, h.reset_atime)
CrawlConfig.get_config(reset=True, soft=True)
def tcc_priority(globspec, cosinfo):
"""
Handle any files matching globspec. Return the number of files processed.
"""
rval = 0
cfg = CrawlConfig.get_config()
pri_compdir = cfg.get_d(tcc_lib.sectname(), 'completed', '/tmp')
for filepath in glob.glob(globspec):
tcc_lib.check_file(filepath, verbose=False, plugin=True)
cpath = U.pathjoin(pri_compdir, U.basename(filepath))
os.rename(filepath, cpath)
return rval
def tcc_priority(globspec, cosinfo):
"""
Handle any files matching globspec. Return the number of files processed.
"""
rval = 0
cfg = CrawlConfig.get_config()
pri_compdir = cfg.get_d(tcc_lib.sectname(), 'completed', '/tmp')
for filepath in glob.glob(globspec):
tcc_lib.check_file(filepath, verbose=False, plugin=True)
cpath = U.pathjoin(pri_compdir, U.basename(filepath))
os.rename(filepath, cpath)
return rval
def check_hash_algorithm(self, cf_stem, alg, checkfor=None):
"""
With hash_algorithm = *alg* in config
"""
if checkfor is None:
checkfor = alg
# generate a config file and make it the default config
cf_name = self.tmpdir(cf_stem + ".cfg")
cd = copy.deepcopy(self.cfg_d)
if alg == '(none)':
del cd['cv']['hash_algorithm']
else:
cd['cv']['hash_algorithm'] = alg
self.write_cfg_file(cf_name, cd)
CrawlConfig.get_config(cfname=cf_name, reset=True)
# Get an hsi object
testfile = self.plist[1]
try:
h = hpss.HSI()
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
# if necessary, delete any hash on the test file
result = h.hashlist(testfile)
if "(none)" not in result:
h.hashdelete(testfile)
# generate a hash on the test file
h.hashcreate(testfile)
# verify that the hash created is of the proper type
result = h.hashlist(testfile)
self.expected_in(checkfor, result)
def main(cfg):
"""
This plugin will generate a report and send it to the designated e-mail
address(es).
"""
rval = 0
try:
if cfg is None:
cfg = CrawlConfig.get_config()
subject = "%s %s" % (cfg.get('rpt', 'subject'),
time.strftime("%Y.%m%d %H:%M:%S",
time.localtime()))
CrawlMail.send(sender=cfg.get('rpt', 'sender'),
to='rpt.recipients',
subj=subject,
msg=rpt_lib.get_report())
except Exception as e:
rval = 1
CrawlConfig.log("Failure in rpt_lib: '%s'" % str(e))
return rval
def main(cfg):
"""
This plugin will generate a report and send it to the designated e-mail
address(es).
"""
rval = 0
try:
if cfg is None:
cfg = CrawlConfig.get_config()
subject = "%s %s" % (cfg.get(
'rpt',
'subject'), time.strftime("%Y.%m%d %H:%M:%S", time.localtime()))
CrawlMail.send(sender=cfg.get('rpt', 'sender'),
to='rpt.recipients',
subj=subject,
msg=rpt_lib.get_report())
except Exception as e:
rval = 1
CrawlConfig.log("Failure in rpt_lib: '%s'" % str(e))
return rval
last_obj_id,
last_obj_id,
correct,
error)
errcount += error
CrawlConfig.log("last nsobject in range: %d" % last_obj_id)
return errcount
# -----------------------------------------------------------------------------
def tcc_priority(globspec, cosinfo):
"""
Handle any files matching globspec. Return the number of files processed.
"""
rval = 0
cfg = CrawlConfig.get_config()
pri_compdir = cfg.get_d(tcc_lib.sectname(), 'completed', '/tmp')
for filepath in glob.glob(globspec):
tcc_lib.check_file(filepath, verbose=False, plugin=True)
cpath = U.pathjoin(pri_compdir, U.basename(filepath))
os.rename(filepath, cpath)
return rval
# -----------------------------------------------------------------------------
if __name__ == '__main__':
main(CrawlConfig.get_config())
"failures: %d" % t_failures)
# Report the dimension data in the log
d = Dimension.Dimension(name='cos')
t = Dimension.Dimension(name='cart')
CrawlConfig.log(d.report())
CrawlConfig.log(t.report())
return failures
# -----------------------------------------------------------------------------
def get_stats():
"""
Return the number of files checksummed, checksums matched, and checksums
failed.
Matches and failures are stored in the cvstats table but total checksum
count is retrieved from the checkables table by counting records with
checksum = 1. This avoids discrepancies where the checksum count in cvstats
might get out of synch with the records in checkables.
"""
checksums = cv_lib.get_checksum_count()
(matches, failures) = cv_lib.get_match_fail_count()
return(checksums, matches, failures)
# -----------------------------------------------------------------------------
if __name__ == '__main__':
cfg = CrawlConfig.get_config()
main(cfg)
"matches: %d; " % t_matches + "failures: %d" % t_failures)
# Report the dimension data in the log
d = Dimension.Dimension(name='cos')
t = Dimension.Dimension(name='cart')
CrawlConfig.log(d.report())
CrawlConfig.log(t.report())
return failures
# -----------------------------------------------------------------------------
def get_stats():
"""
Return the number of files checksummed, checksums matched, and checksums
failed.
Matches and failures are stored in the cvstats table but total checksum
count is retrieved from the checkables table by counting records with
checksum = 1. This avoids discrepancies where the checksum count in cvstats
might get out of synch with the records in checkables.
"""
checksums = cv_lib.get_checksum_count()
(matches, failures) = cv_lib.get_match_fail_count()
return (checksums, matches, failures)
# -----------------------------------------------------------------------------
if __name__ == '__main__':
cfg = CrawlConfig.get_config()
main(cfg)
last_obj_id = int(bf['OBJECT_ID'])
tcc_lib.record_checked_ids(cfg, last_obj_id, last_obj_id, correct,
error)
errcount += error
CrawlConfig.log("last nsobject in range: %d" % last_obj_id)
return errcount
# -----------------------------------------------------------------------------
def tcc_priority(globspec, cosinfo):
"""
Handle any files matching globspec. Return the number of files processed.
"""
rval = 0
cfg = CrawlConfig.get_config()
pri_compdir = cfg.get_d(tcc_lib.sectname(), 'completed', '/tmp')
for filepath in glob.glob(globspec):
tcc_lib.check_file(filepath, verbose=False, plugin=True)
cpath = U.pathjoin(pri_compdir, U.basename(filepath))
os.rename(filepath, cpath)
return rval
# -----------------------------------------------------------------------------
if __name__ == '__main__':
main(CrawlConfig.get_config())