def test_hashcreate_atime_no_reset(self):
"""
1) use file /home/tpb/hic_test/hashable1
2) delete the file's hash if it has one
3) Set the access time into the past (touch -a t yyyymmddHHMM.SS ...)
4) hashcreate with atime reset turned OFF
5) Get the access time -- it should be near the present
"""
try:
filename = self.plist[0]
h = hpss.HSI(reset_atime=False)
# delete the file's hash if it has one
hash = h.hashlist(filename)
if "(none)" not in hash:
h.hashdelete(filename)
# set the file's atime into the past
past = util.epoch("2001.0203 04:05:06")
h.touch(filename, when=past)
# give the file a hash
h.hashcreate(filename)
# check the atime -- it should be recent
atime = h.access_time(filename)
delta = time.time() - atime
self.assertTrue(delta < 10,
"Expected a recent time, got '%s'" %
util.ymdhms(atime))
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_hashverify_atime_no_reset(self):
"""
1) Create a file
2) hashcreate on it
3) Set the access time into the past (touch -a t yyyymmddHHMM.SS ...)
4) hashverify with atime reset turned OFF
5) Get the access time -- it should be near the present
6) remove the file
"""
try:
filename = self.plist[0]
h = hpss.HSI(reset_atime=False)
h.hashcreate(filename)
# set the access time into the past
past = util.epoch("2001.0203 04:05:06")
h.touch(filename, when=past)
# hashverify
h.hashverify(filename)
# check the atime -- it should be recent
atime = h.access_time(filename)
delta = time.time() - atime
self.assertTrue(delta < 60,
"Expected a recent time, got '%s'" %
util.ymdhms(atime))
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_hashdelete_ok_list(self):
"""
If hashdelete get a list argument, it should work
"""
try:
plist = self.plist + [self.hdir + "/hashnot"]
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
# make sure the hashables all have a checksum stored
x = h.hashlist(self.plist)
for path in self.plist:
if util.rgxin("\(?none\)? %s" % path, x):
h.hashcreate(path)
# run hashdelete on the list
result = h.hashdelete(plist)
h.quit()
# verify the results
self.expected_in("hashdelete", result)
for path in self.plist:
self.expected_in("hash deleted: md5 %s" % path, result)
exp = "\(?none\)? %s/hashnot" % self.hdir
self.assertTrue(exp not in result,
"'%s' not expected in %s" %
(exp,
util.line_quote(result)))
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_hashlist_ok_str(self):
"""
If hashlist gets a string argument, it should work
"""
try:
paths = self.paths + " %s/hashnot" % self.hdir
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
# make sure the hashables all have a checksum stored
x = h.hashlist(self.plist)
for path in self.plist:
if util.rgxin("\(?none\)? %s" % path, x):
h.hashcreate(path)
# run the test payload
result = h.hashlist(paths)
h.quit()
self.expected_in("hashlist", result)
for path in self.paths.split():
exp = "\(?md5\)? %s" % path
self.expected_in(exp, result)
exp = "\(?none\)? %s/hashnot" % self.hdir
self.expected_in(exp, result)
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_hashdelete_ok_str(self):
"""
If hashdelete gets a string argument, it should work
"""
try:
paths = self.paths + " %s/hashnot" % self.hdir
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
# make sure the hashables all have a checksum stored
x = h.hashlist(self.plist)
for path in self.plist:
if util.rgxin("\(?none\)? %s" % path, x):
h.hashcreate(path)
# run hashdelete on the string
result = h.hashdelete(paths)
h.quit()
# verify the results
self.expected_in("hashdelete", result)
for path in self.paths.split():
exp = "hash deleted: \(?md5\)? %s" % path
self.expected_in(exp, result)
exp = "hash deleted: \(?md5\)? %s/hashnot" % self.hdir
self.assertFalse(util.rgxin(exp, result),
"'%s' not expected in %s" %
(exp, util.line_quote(result)))
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_hashverify_ok_glob(self):
"""
Issue "hashverify" in hsi on a wildcard, return results
"""
self.dbgfunc()
try:
glop = "/home/tpb/hic_test/hash*"
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
# make sure the hashables all have a checksum stored
x = h.hashlist(self.plist)
for path in self.plist:
if "\(?none\)? %s" % path in x:
h.hashcreate(path)
# run hashverify on the glob path
result = h.hashverify(glop)
h.quit()
# verify the results: we should see "OK" for the hashables and an
# error for hashnot
self.expected_in("hashverify", result)
for path in self.plist:
exp = "%s: \(?md5\)? OK" % path
self.expected_in(exp, result)
exp = "hashnot failed: no valid checksum found"
self.expected_in(exp, result)
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_hashverify_ok_list(self):
"""
Issue "hashverify" in hsi, return results
"""
self.dbgfunc()
try:
plist = self.plist + ["%s/hashnot" % self.hdir]
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
# make sure the hashables all have a checksum stored
x = h.hashlist(self.plist)
for path in self.plist:
if "\(?none\)? %s" % path in x:
h.hashcreate(path)
# run hashverify on the list
result = h.hashverify(plist)
h.quit()
# verify the results: we should see "OK" for the hashables and an
# error for hashnot
self.expected_in("hashverify", result)
for path in self.plist:
self.expected_in("%s: \(?md5\)? OK" % path, result)
self.expected_in("hashnot failed: no valid checksum found",
result)
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_hashcreate_atime_reset(self):
"""
1) use file /home/tpb/hic_test/hashable1
2) if it has a hash, delete it
3) Set the access time into the past (touch -a t yyyymmddHHMM.SS ...)
4) hashcreate with atime reset turned on (set when opening HSI)
5) Get the access time -- it should be the same as was set in step 2
"""
try:
filename = self.plist[0]
h = hpss.HSI(reset_atime=True)
# delete the file's hash if it has one
hash = h.hashlist(filename)
if "(none)" not in hash:
h.hashdelete(filename)
# set the atime into the past
past = util.epoch("2001.0203 04:05:06")
h.touch(filename, when=past)
# create a hash
h.hashcreate(filename)
# check the atime -- it should be way in the past
atime = h.access_time(filename)
self.expected(util.ymdhms(past), util.ymdhms(atime))
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_hashverify_atime_reset(self):
"""
1) Use file /home/tpb/hic_test/hashable1
2) hashcreate on it
3) Set the access time into the past (touch -a t yyyymmddHHMM.SS ...)
4) hashverify with atime reset turned on
5) Get the access time -- it should be the same as was set in step 3
"""
try:
filename = self.plist[0]
h = hpss.HSI(reset_atime=True)
# give it a hash
h.hashcreate(filename)
# set the access time into the past
past = util.epoch("2001.0203 04:05:06")
h.touch(filename, when=past)
# hashverify
h.hashverify(filename)
# check the atime -- it should be old
atime = h.access_time(filename)
self.expected(util.ymdhms(past), util.ymdhms(atime))
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_unavailable(self):
"""
If HPSS is down, the HSI constructor should throw an exception. And in
this case, we don't need to call h.quit() since the connection never
got completed.
"""
h = hpss.HSI(connect=False, unavailable=True)
self.assertRaisesMsg(hpss.HSIerror,
MSG.hpss_unavailable,
h.connect)
def test_chdir_noarg(self):
"""
Change dir in HPSS to a non directory
"""
try:
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
self.assertRaisesMsg(TypeError,
"chdir() takes exactly 2 arguments",
h.chdir)
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_ctor_attrs(self):
"""
Verify that a newly created HSI object has the right attributes
"""
a = hpss.HSI(connect=False)
for attr in ['prompt', 'verbose', 'xobj', 'lscos', 'quit', 'connect',
'chdir', 'lsP', 'hashcreate', 'hashlist', 'hashverify',
'hashdelete']:
self.assertTrue(hasattr(a, attr),
"Expected %s to have attribute '%s'" %
(a, attr))
def test_lsP_str(self):
"""
Issue "ls -P foo bar" in hic_test, validate results
"""
try:
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
result = h.lsP(self.paths)
for path in self.plist:
self.expected_in("FILE\s+%s" % util.basename(path), result)
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_lsP_argnone(self):
"""
Issue "ls -P" in /home/tpb/hic_test with no arg, validate result
"""
try:
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
h.chdir("/home/tpb/hic_test")
result = h.lsP()
for path in self.plist:
self.expected_in("FILE\s+%s" % util.basename(path), result)
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_chdir_unicode(self):
"""
Unicode argument to chdir should work
"""
try:
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
ucdir = unicode("hic_test")
result = h.chdir(ucdir)
exp = "/home/tpb/hic_test"
self.expected_in(exp, result)
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_hashlist_argnone(self):
"""
If hashlist gets no argument, it should throw an exception
"""
try:
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
self.assertRaisesMsg(TypeError,
"hashlist() takes exactly 2 arguments",
h.hashlist)
h.quit()
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_hashverify_argnone(self):
"""
Issue "hashverify" in hsi, return results
"""
try:
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
self.assertRaisesMsg(TypeError,
"hashverify() takes exactly 2 arguments",
h.hashverify)
h.quit()
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_ctor_no_cv_section(self):
"""
If there is no cv section in the config, reset_atime and hash_algorithm
should take on their default values.
"""
self.dbgfunc()
cfg = copy.deepcopy(self.cfg_d)
del cfg['cv']
zcfg = CrawlConfig.add_config(close=True, dct=cfg)
self.assertFalse(zcfg.has_section('cv'))
h = hpss.HSI(connect=False)
self.expected(False, h.reset_atime)
self.expected(None, h.hash_algorithm)
def test_lsP_argbad(self):
"""
Issue "ls -P" with non-string, non-list arg, expect exception
"""
try:
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
self.assertRaisesMsg(hpss.HSIerror,
"lsP: Invalid argument",
h.lsP,
19)
h.quit()
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_hashverify_argbad(self):
"""
If hashverify gets an invalid argument (not a str or list), it should
throw an exception
"""
try:
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
self.assertRaisesMsg(hpss.HSIerror,
"hashverify: Invalid argument",
h.hashverify,
32)
h.quit()
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_hashcreate_ok_str(self):
"""
Issue "hashcreate" in hsi with a string argument, return results
"""
try:
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
result = h.hashcreate(self.paths)
h.quit()
self.expected_in("hashcreate", result)
for path in self.paths.split():
exp = "\(?md5\)? %s" % path
self.expected_in(exp, result)
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_chdir_ok(self):
"""
Successful change dir in HPSS
"""
self.dbgfunc()
try:
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
result = h.chdir("hic_test")
h.quit()
exp = "/home/tpb/hic_test"
self.expected_in(exp, result)
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_lscos(self):
"""
Issue "lscos", check result
"""
try:
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
result = h.lscos()
h.quit()
self.expected_in("3003 Disk Big Backups", result)
self.expected_in("5081 Disk X-Small", result)
self.expected_in("6001 Disk Small", result)
self.expected_in("6054 Disk Large_T", result)
self.expected_in("6057 Disk X-Large_T 2-Copy", result)
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_chdir_perm(self):
"""
Change dir in HPSS to a non-accessible directory
"""
try:
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
result = h.chdir("cli_test/unreadable")
h.quit()
exp = "hpss_Chdir: Access denied"
self.assertTrue(exp in result,
"Expected '%s' in %s" %
(exp, util.line_quote(result)))
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_chdir_notdir(self):
"""
Change dir in HPSS to a non directory
"""
try:
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
result = h.chdir("hic_test/crawler.tar")
h.quit()
exp = "Not a directory"
self.assertTrue(exp in result,
"Expected '%s' in %s" %
(exp, util.line_quote(result)))
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_ctor_reset_atime_cfg_true(self):
"""
If reset_atime is specified in the config as True, it should be True
"""
cf_name = self.tmpdir(util.my_name() + ".cfg")
# write out a config file with no reset_atime spec
self.write_cfg_file(cf_name, self.cfg_d)
# make the test config the default
CrawlConfig.get_config(cfname=cf_name, reset=True)
# get an hpss.HSI object and check its reset_atime attribute
h = hpss.HSI(connect=False)
self.expected(True, h.reset_atime)
CrawlConfig.get_config(reset=True, soft=True)
def test_hashverify_ok_unicode(self):
"""
Issue "hashverify" in hsi with a unicode arg, return results
"""
self.dbgfunc()
try:
paths = unicode(self.paths + " %s/hashnot" % self.hdir)
h = hpss.HSI(verbose=("verbose" in testhelp.testargs()))
result = h.hashverify(paths)
h.quit()
self.expected_in("hashverify", result)
for path in self.plist:
self.expected_in("%s: \(?md5\)? OK" % path, result)
self.expected_in("hashnot failed: no valid checksum found",
result)
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
def test_ctor_reset_atime_default(self):
"""
If reset_atime is not specified in the config or argument list, it
should default to False
"""
cf_name = self.tmpdir(util.my_name() + ".cfg")
# write out a config file with no reset_atime spec
cd = copy.deepcopy(self.cfg_d)
del cd['cv']['reset_atime']
self.write_cfg_file(cf_name, cd)
# make the test config the default
CrawlConfig.get_config(cfname=cf_name, reset=True)
# get an hpss.HSI object and check its reset_atime attribute
h = hpss.HSI(connect=False)
self.expected(False, h.reset_atime)
CrawlConfig.get_config(reset=True, soft=True)
def test_ctor_reset_atime_call_true(self):
"""
If reset_atime is specified in the call as True, it should be True,
even if it's specified as False in the config
"""
cf_name = self.tmpdir(util.my_name() + ".cfg")
# write out a config file with no reset_atime spec
cfg = copy.deepcopy(self.cfg_d)
cfg['cv']['reset_atime'] = 'no'
self.write_cfg_file(cf_name, cfg)
# make the test config the default
CrawlConfig.get_config(cfname=cf_name, reset=True)
# get an hpss.HSI object and check its reset_atime attribute
h = hpss.HSI(connect=False, reset_atime=True)
self.expected(True, h.reset_atime)
CrawlConfig.get_config(reset=True, soft=True)
def check_hash_algorithm(self, cf_stem, alg, checkfor=None):
"""
With hash_algorithm = *alg* in config
"""
if checkfor is None:
checkfor = alg
# generate a config file and make it the default config
cf_name = self.tmpdir(cf_stem + ".cfg")
cd = copy.deepcopy(self.cfg_d)
if alg == '(none)':
del cd['cv']['hash_algorithm']
else:
cd['cv']['hash_algorithm'] = alg
self.write_cfg_file(cf_name, cd)
CrawlConfig.get_config(cfname=cf_name, reset=True)
# Get an hsi object
testfile = self.plist[1]
try:
h = hpss.HSI()
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
# if necessary, delete any hash on the test file
result = h.hashlist(testfile)
if "(none)" not in result:
h.hashdelete(testfile)
# generate a hash on the test file
h.hashcreate(testfile)
# verify that the hash created is of the proper type
result = h.hashlist(testfile)
self.expected_in(checkfor, result)
