def main(cfg):
"""
Migration Purge Record Ager (mpra) reads the database tables BFMIGRREC and
BFPURGEREC and reports migration and purge records that are older than the
age specified in the configuration.
"""
if cfg is None:
cfg = CrawlConfig.get_config()
age = cfg.get_time('mpra', 'age')
end = time.time() - age
start = mpra_lib.mpra_fetch_recent("migr")
#
# If the configured age has been moved back in time, so that end is before
# start, we need to reset and start scanning from the beginning of time.
#
if end < start:
start = 0
CrawlConfig.log("migr recs after %d (%s) before %d (%s)" %
(start, util.ymdhms(start), end, util.ymdhms(end)))
result = mpra_lib.age("migr", start=start, end=end, mark=True)
CrawlConfig.log("found %d migration records in the range" % result)
rval = result
start = mpra_lib.mpra_fetch_recent("purge")
CrawlConfig.log("Looking for expired purge locks")
result = mpra_lib.xplocks(mark=True)
CrawlConfig.log("found %d expired purge locks" % result)
rval += result
return rval
def test_hashverify_atime_reset(self):
"""
1) Use file /home/tpb/hic_test/hashable1
2) hashcreate on it
3) Set the access time into the past (touch -a t yyyymmddHHMM.SS ...)
4) hashverify with atime reset turned on
5) Get the access time -- it should be the same as was set in step 3
"""
try:
filename = self.plist[0]
h = hpss.HSI(reset_atime=True)
# give it a hash
h.hashcreate(filename)
# set the access time into the past
past = util.epoch("2001.0203 04:05:06")
h.touch(filename, when=past)
# hashverify
h.hashverify(filename)
# check the atime -- it should be old
atime = h.access_time(filename)
self.expected(util.ymdhms(past), util.ymdhms(atime))
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_hashcreate_atime_reset(self):
"""
1) use file /home/tpb/hic_test/hashable1
2) if it has a hash, delete it
3) Set the access time into the past (touch -a t yyyymmddHHMM.SS ...)
4) hashcreate with atime reset turned on (set when opening HSI)
5) Get the access time -- it should be the same as was set in step 2
"""
try:
filename = self.plist[0]
h = hpss.HSI(reset_atime=True)
# delete the file's hash if it has one
hash = h.hashlist(filename)
if "(none)" not in hash:
h.hashdelete(filename)
# set the atime into the past
past = util.epoch("2001.0203 04:05:06")
h.touch(filename, when=past)
# create a hash
h.hashcreate(filename)
# check the atime -- it should be way in the past
atime = h.access_time(filename)
self.expected(util.ymdhms(past), util.ymdhms(atime))
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def validate(days):
daylen = 24 * 3600
low = U.daybase(time.time()) + days * daylen
high = low + daylen
r = U.day_offset(days) + 30 # 30 seconds into the day
self.assertTrue(
low <= r and r < high, "Expected %s <= %s < %s" %
(U.ymdhms(low), U.ymdhms(r), U.ymdhms(high)))
def validate(days):
daylen = 24 * 3600
low = U.daybase(time.time()) + days * daylen
high = low + daylen
r = U.day_offset(days) + 30 # 30 seconds into the day
self.assertTrue(low <= r and r < high,
"Expected %s <= %s < %s" %
(U.ymdhms(low), U.ymdhms(r), U.ymdhms(high)))
def test_hashverify_atime_no_reset(self):
"""
1) Create a file
2) hashcreate on it
3) Set the access time into the past (touch -a t yyyymmddHHMM.SS ...)
4) hashverify with atime reset turned OFF
5) Get the access time -- it should be near the present
6) remove the file
"""
try:
filename = self.plist[0]
h = hpss.HSI(reset_atime=False)
h.hashcreate(filename)
# set the access time into the past
past = util.epoch("2001.0203 04:05:06")
h.touch(filename, when=past)
# hashverify
h.hashverify(filename)
# check the atime -- it should be recent
atime = h.access_time(filename)
delta = time.time() - atime
self.assertTrue(delta < 60,
"Expected a recent time, got '%s'" %
util.ymdhms(atime))
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_hashcreate_atime_no_reset(self):
"""
1) use file /home/tpb/hic_test/hashable1
2) delete the file's hash if it has one
3) Set the access time into the past (touch -a t yyyymmddHHMM.SS ...)
4) hashcreate with atime reset turned OFF
5) Get the access time -- it should be near the present
"""
try:
filename = self.plist[0]
h = hpss.HSI(reset_atime=False)
# delete the file's hash if it has one
hash = h.hashlist(filename)
if "(none)" not in hash:
h.hashdelete(filename)
# set the file's atime into the past
past = util.epoch("2001.0203 04:05:06")
h.touch(filename, when=past)
# give the file a hash
h.hashcreate(filename)
# check the atime -- it should be recent
atime = h.access_time(filename)
delta = time.time() - atime
self.assertTrue(delta < 10,
"Expected a recent time, got '%s'" %
util.ymdhms(atime))
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))