def test_load_new(self):
"""
With the database and checkables table in place, create a new Dimension
that is not in the table. Calling load() on it should be a no-op -- the
object should not be stored to the database and its contents should not
be changed.
"""
self.dbgfunc()
# reboot the database and call persist() to create the table without
# adding any data
U.conditional_rm(self.dbname())
CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name()))
Checkable.Checkable.ex_nihilo()
ignore = Dimension(name='foobar')
# get a Dimension object that is not in the table
test = Dimension(name='notindb')
# make a copy of the object for reference (not just a handle to the
# same ojbect)
ref = copy.deepcopy(test)
# call load(), which should be a no op
test.load()
# verify that the object didn't change
self.expected(ref.name, test.name)
self.expected(ref.sampsize, test.sampsize)
self.expected(ref.p_sum, test.p_sum)
self.expected(ref.s_sum, test.s_sum)
def test_ctor_attrs(self):
"""
Verify that a newly created Dimension object has the following
attributes:
- name (string)
- sampsize (small float value, e.g., 0.005)
- p_sum (empty dict)
- s_sum (empty dict)
- methods
> sum_total
> load
"""
dimname = 'cos'
CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name()))
Checkable.Checkable.ex_nihilo()
a = Dimension(name=dimname, sampsize=0.005)
for attr in [
'name',
'sampsize',
'p_sum',
's_sum',
'sum_total',
'load',
]:
self.assertTrue(
hasattr(a, attr),
"Object %s does not have expected attribute %s" % (a, attr))
def test_sum_total(self):
"""
Return the sum of all the 'count' values in either the p_sum or s_sum
dictionary.
"""
CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name()))
Checkable.Checkable.ex_nihilo()
a = Dimension(name='sum_total')
a.p_sum = {
'6001': {
'count': 2,
'pct': 50.0
},
'5081': {
'count': 2,
'pct': 50.0
}
}
a.s_sum = {
'6001': {
'count': 2,
'pct': 40.0
},
'5081': {
'count': 3,
'pct': 60.0
}
}
self.expected(4, a.sum_total())
self.expected(4, a.sum_total(dict=a.p_sum))
self.expected(5, a.sum_total(which='s'))
self.expected(5, a.sum_total(dict=a.s_sum))
def test_load_new(self):
"""
With the database and checkables table in place, create a new Dimension
that is not in the table. Calling load() on it should be a no-op -- the
object should not be stored to the database and its contents should not
be changed.
"""
self.dbgfunc()
# reboot the database and call persist() to create the table without
# adding any data
U.conditional_rm(self.dbname())
CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name()))
Checkable.Checkable.ex_nihilo()
ignore = Dimension(name='foobar')
# get a Dimension object that is not in the table
test = Dimension(name='notindb')
# make a copy of the object for reference (not just a handle to the
# same ojbect)
ref = copy.deepcopy(test)
# call load(), which should be a no op
test.load()
# verify that the object didn't change
self.expected(ref.name, test.name)
self.expected(ref.sampsize, test.sampsize)
self.expected(ref.p_sum, test.p_sum)
self.expected(ref.s_sum, test.s_sum)
def test_ctor_attrs(self):
"""
Verify that a newly created Dimension object has the following
attributes:
- name (string)
- sampsize (small float value, e.g., 0.005)
- p_sum (empty dict)
- s_sum (empty dict)
- methods
> sum_total
> load
"""
dimname = 'cos'
CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name()))
Checkable.Checkable.ex_nihilo()
a = Dimension(name=dimname,
sampsize=0.005)
for attr in ['name',
'sampsize',
'p_sum',
's_sum',
'sum_total',
'load',
]:
self.assertTrue(hasattr(a, attr),
"Object %s does not have expected attribute %s" %
(a, attr))
def test_repr(self):
"""
Method __repr__ should return <Dimension(name='foo')>.
"""
CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name()))
exp = "Dimension(name='foo')"
a = eval(exp)
self.expected(exp, a.__repr__())
def test_repr(self):
"""
Method __repr__ should return <Dimension(name='foo')>.
"""
CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name()))
exp = "Dimension(name='foo')"
a = eval(exp)
self.expected(exp, a.__repr__())
def test_load_already(self):
"""
With the database and a checkables table in place and records in the
table, calling load() on a Dimension should load the information from
the table into the object. However, it should only count records where
last_check <> 0.
"""
self.dbgfunc()
U.conditional_rm(self.dbname())
CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name()))
Checkable.Checkable.ex_nihilo()
chk = Checkable.Checkable
testdata = [
chk(rowid=1, path="/abc/001", type='f', cos='6001', checksum=0,
last_check=0),
chk(rowid=2, path="/abc/002", type='f', cos='6002', checksum=0,
last_check=5),
chk(rowid=3, path="/abc/003", type='f', cos='6003', checksum=1,
last_check=0),
chk(rowid=4, path="/abc/004", type='f', cos='6001', checksum=1,
last_check=17),
chk(rowid=5, path="/abc/005", type='f', cos='6002', checksum=0,
last_check=0),
chk(rowid=6, path="/abc/006", type='f', cos='6003', checksum=0,
last_check=8),
chk(rowid=7, path="/abc/007", type='f', cos='6001', checksum=0,
last_check=0),
chk(rowid=8, path="/abc/008", type='f', cos='6002', checksum=0,
last_check=19),
chk(rowid=9, path="/abc/009", type='f', cos='6003', checksum=0,
last_check=0),
]
# insert some test data into the table
for t in testdata:
t.persist()
# get a default Dimension with the same name as the data in the table
q = Dimension(name='cos')
# this should load the data from the table into the object
q.load()
# verify the loaded data in the object
self.expected('cos', q.name)
self.assertTrue('6001' in q.p_sum.keys(),
"Expected '6001' in p_sum.keys()")
self.assertTrue('6002' in q.p_sum.keys(),
"Expected '6001' in p_sum.keys()")
self.assertTrue('6003' in q.p_sum.keys(),
"Expected '6003' in p_sum.keys()")
self.assertTrue('6001' in q.s_sum.keys(),
"Expected '6001' in s_sum.keys()")
self.assertTrue('6002' in q.s_sum.keys(),
"Expected '6002' in s_sum.keys()")
self.assertTrue('6003' in q.s_sum.keys(),
"Expected '6003' in s_sum.keys()")
def test_ctor_defaults(self):
"""
A new Dimension with only the name specified should have the right
defaults.
"""
dimname = 'cos'
CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name()))
a = Dimension(name=dimname)
self.expected(dimname, a.name)
self.expected(0.01, a.sampsize)
self.expected({}, a.p_sum)
self.expected({}, a.s_sum)
def test_ctor_defaults(self):
"""
A new Dimension with only the name specified should have the right
defaults.
"""
dimname = 'cos'
CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name()))
a = Dimension(name=dimname)
self.expected(dimname, a.name)
self.expected(0.01, a.sampsize)
self.expected({}, a.p_sum)
self.expected({}, a.s_sum)
def test_ctor_bad_attr(self):
"""
Attempting to create a Dimension with attrs that are not in the
settable list should get an exception.
"""
dimname = 'bad_attr'
CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name()))
got_exception = False
self.assertRaisesMsg(StandardError,
"Attribute 'catl' is not valid",
Dimension, name=dimname, catl=[1, 2, 3])
self.assertRaisesMsg(StandardError,
"Attribute 'aardvark' is not valid",
Dimension, name=dimname, aardvark='Fanny Brice')
def test_sum_total(self):
"""
Return the sum of all the 'count' values in either the p_sum or s_sum
dictionary.
"""
CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name()))
Checkable.Checkable.ex_nihilo()
a = Dimension(name='sum_total')
a.p_sum = {'6001': {'count': 2, 'pct': 50.0},
'5081': {'count': 2, 'pct': 50.0}
}
a.s_sum = {'6001': {'count': 2, 'pct': 40.0},
'5081': {'count': 3, 'pct': 60.0}
}
self.expected(4, a.sum_total())
self.expected(4, a.sum_total(dict=a.p_sum))
self.expected(5, a.sum_total(which='s'))
self.expected(5, a.sum_total(dict=a.s_sum))
def test_ctor_bad_attr(self):
"""
Attempting to create a Dimension with attrs that are not in the
settable list should get an exception.
"""
dimname = 'bad_attr'
CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name()))
got_exception = False
self.assertRaisesMsg(StandardError,
"Attribute 'catl' is not valid",
Dimension,
name=dimname,
catl=[1, 2, 3])
self.assertRaisesMsg(StandardError,
"Attribute 'aardvark' is not valid",
Dimension,
name=dimname,
aardvark='Fanny Brice')
def test_ctor_no_cv_section(self):
"""
If there is no cv section in the config, reset_atime and hash_algorithm
should take on their default values.
"""
self.dbgfunc()
cfg = copy.deepcopy(self.cfg_d)
del cfg['cv']
zcfg = CrawlConfig.add_config(close=True, dct=cfg)
self.assertFalse(zcfg.has_section('cv'))
h = hpss.HSI(connect=False)
self.expected(False, h.reset_atime)
self.expected(None, h.hash_algorithm)
def test_alert_email_defcfg(self):
"""
Generate an e-mail alert using the default config and verify that it
was sent (this is where we use 'monkey patching').
"""
self.dbgfunc()
fakesmtp.inbox = []
CrawlConfig.add_config(close=True)
# with U.tmpenv('CRAWL_CONF', 'hpssic_test.cfg'):
with U.tmpenv('CRAWL_CONF', None):
logfile = self.tmpdir('alert_email.log')
targets = "[email protected], [email protected]"
payload = 'this is an e-mail alert'
sender = 'hpssic@' + util.hostname(long=True)
CrawlConfig.log(logpath=logfile, close=True)
x = Alert.Alert(caller='cv', msg=payload)
m = fakesmtp.inbox[0]
self.expected(', '.join(m.to_address), targets)
self.expected(m.from_address, sender)
self.expected_in('sent mail to', util.contents(logfile))
self.expected_in(payload, m.fullmessage)
def test_alert_email_defcfg(self):
"""
Generate an e-mail alert using the default config and verify that it
was sent (this is where we use 'monkey patching').
"""
self.dbgfunc()
fakesmtp.inbox = []
CrawlConfig.add_config(close=True)
# with U.tmpenv('CRAWL_CONF', 'hpssic_test.cfg'):
with U.tmpenv('CRAWL_CONF', None):
logfile = self.tmpdir('alert_email.log')
targets = "[email protected], [email protected]"
payload = 'this is an e-mail alert'
sender = 'hpssic@' + util.hostname(long=True)
CrawlConfig.log(logpath=logfile, close=True)
x = Alert.Alert(caller='cv', msg=payload)
m = fakesmtp.inbox[0]
self.expected(', '.join(m.to_address), targets)
self.expected(m.from_address, sender)
self.expected_in('sent mail to', util.contents(logfile))
self.expected_in(payload, m.fullmessage)
def test_html_report(self):
"""
Try running 'html report > filename' and verify that 1) no traceback
occurs and 2) something is actually written to the output file.
"""
self.dbgfunc()
cfpath = self.tmpdir("crawl.cfg")
cfg = CrawlConfig.add_config()
cfg.crawl_write(open(cfpath, 'w'))
cmd = "html report --config %s" % cfpath
CrawlConfig.log(cmd, close=True)
result = pexpect.run(cmd)
if "HPSS Unavailable" in result:
pytest.skip("HPSS Unavailable")
self.validate_report(result)
def test_html_report(self):
"""
Try running 'html report > filename' and verify that 1) no traceback
occurs and 2) something is actually written to the output file.
"""
self.dbgfunc()
cfpath = self.tmpdir("crawl.cfg")
cfg = CrawlConfig.add_config()
cfg.crawl_write(open(cfpath, 'w'))
cmd = "html report --config %s" % cfpath
CrawlConfig.log(cmd, close=True)
result = pexpect.run(cmd)
if "HPSS Unavailable" in result:
pytest.skip("HPSS Unavailable")
self.validate_report(result)
def setUp(self):
"""
Set up for each test: create the mpra table with prefix 'test', touch
the mpra test file in the tmpdir
"""
super(MpraResetTest, self).setUp()
if pexpect.which("mpra"):
self.cmd = "mpra"
elif os.path.exists("bin/mpra"):
self.cmd = "bin/mpra"
else:
raise HpssicError("mpra command not found")
self.rptname = self.tmpdir("mpra_report.txt")
self.cfgname = self.tmpdir("mpra_test.cfg")
self.cfg = CrawlConfig.add_config(close=True, dct=self.cfg_dict())
self.cfg.crawl_write(open(self.cfgname, 'w'))
dbschem.make_table("mpra", cfg=self.cfg)
U.touch(self.rptname)
def test_get_html_report(self):
"""
Call html_lib.get_html_report() directly
"""
self.dbgfunc()
c = CrawlConfig.add_config()
db = CrawlDBI.DBI(dbtype="crawler")
dbschem.drop_table(table="lscos")
self.expected(False, db.table_exists(table="lscos"))
try:
result = html_lib.get_html_report('')
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
self.expected(True, db.table_exists(table="lscos"))
db.close()
self.validate_report(result)
def test_get_html_report(self):
"""
Call html_lib.get_html_report() directly
"""
self.dbgfunc()
c = CrawlConfig.add_config()
db = CrawlDBI.DBI(dbtype="crawler")
dbschem.drop_table(table="lscos")
self.expected(False, db.table_exists(table="lscos"))
try:
result = html_lib.get_html_report('')
except hpss.HSIerror as e:
if MSG.hpss_unavailable in str(e):
pytest.skip(str(e))
self.expected(True, db.table_exists(table="lscos"))
db.close()
self.validate_report(result)
def setUp(self):
"""
Set up for tests
"""
super(HtmlTest, self).setUp()
data = {
'checkables': [
"/log/2007/05/27/logfile01_200705270654,f,6001,X1701700," +
"media,0,1410363023,0,0",
"/log/2007/05/27/logfile01_200705271304,f,6001,X1701700," +
"media,0,1410363023,0,0",
"/log/2007/05/27/logfile02_200705270424,f,6001,X1701700," +
"media,0,1410363024,0,0",
"/log/2007/05/27/logfile02_200705270847,f,6001,X1701700," +
"media,0,1410363024,0,0",
"/log/2007/05/27/logfile02_200705271716,f,6001,X1701700," +
"media,0,1410363025,0,0",
"/log/2007/02/27/logfile01_200702270958,f,6001,X1701700," +
"media,0,0,0,0",
"/log/2007/02/27/logfile01_200702271707,f,6001,X1701700," +
"media,0,0,0,0",
"/log/2007/02/27/logfile01_200702272150,f,6001,X1701700," +
"media,0,0,0,0",
"/log/2007/02/27/logfile02_200702270431,f,6001,X1701700," +
"media,0,0,0,0",
"/log/2007/02/27/logfile02_200702271427,f,6001,X1701700," +
"media,0,0,0,0",
"/log/2007/02/27/logfile02_200702271800,f,6001,X1701700," +
"media,0,0,0,0",
],
'mpra': [
"migr,1403126855,0,1335302855,0",
"purge,1403126872,0,0,0",
"migr,1403127173,1335302855,1335303172,0",
"purge,1403127187,0,0,0",
"migr,1403127488,1335303172,1335303487,0",
"purge,1403127502,0,0,0",
"migr,1403127802,1335303487,1335303802,0",
"purge,1403127818,0,0,0",
"migr,1403128118,1335303802,1335304118,0",
"purge,1403128142,0,0,0",
"migr,1403128443,1335304118,1335304443,0",
"purge,1403128458,0,0,0",
],
'tcc_data': [
"1398454713,1,5,1,0",
"1398454968,1,5,1,0",
"1398455089,1,5,1,0",
"1398455209,1,5,1,0",
"1398455329,1,5,1,0",
"1398455440,1,5,1,0",
"1398455449,1,5,1,0",
"1398455570,1,5,1,0",
"1398455690,1,5,1,0",
"1398455810,1,5,1,0",
"1398455930,1,5,1,0",
"1398456051,1,5,1,0",
"1398456171,1,5,1,0",
"1398456244,6,10,1,0",
"1398456291,1,5,1,0",
"1398456395,6,10,1,0",
"1398456403,11,15,1,0",
"1398456428,16,20,1,0",
"1398456437,1,5,1,0",
"1398456443,6,10,1,0",
"1398456474,11,15,1,0",
"1398456504,16,20,1,0",
"1398456534,1,5,1,0",
"1398456564,6,10,1,0",
"1398456594,11,15,1,0",
]
}
cfg = CrawlConfig.add_config(close=True,
filename='hpssic_sqlite_test.cfg')
cfg.set('dbi-crawler', 'dbname', self.tmpdir("test.db"))
cfg.set('crawler', 'logpath', self.tmpdir("crawl.log"))
db = CrawlDBI.DBI(dbtype='crawler')
for table in ['checkables', 'mpra', 'tcc_data']:
dbschem.make_table(table)
db.close()
db = CrawlDBI.DBI(dbtype='crawler')
for table in ['checkables', 'mpra', 'tcc_data']:
fld_list = [
x[1] for x in db.describe(table=table) if x[1] != 'rowid'
]
db.insert(table=table,
fields=fld_list,
data=[x.split(',') for x in data[table]])
db.close()
def setUp(self):
"""
Set up for tests
"""
super(HtmlTest, self).setUp()
data = {'checkables':
["/log/2007/05/27/logfile01_200705270654,f,6001,X1701700," +
"media,0,1410363023,0,0",
"/log/2007/05/27/logfile01_200705271304,f,6001,X1701700," +
"media,0,1410363023,0,0",
"/log/2007/05/27/logfile02_200705270424,f,6001,X1701700," +
"media,0,1410363024,0,0",
"/log/2007/05/27/logfile02_200705270847,f,6001,X1701700," +
"media,0,1410363024,0,0",
"/log/2007/05/27/logfile02_200705271716,f,6001,X1701700," +
"media,0,1410363025,0,0",
"/log/2007/02/27/logfile01_200702270958,f,6001,X1701700," +
"media,0,0,0,0",
"/log/2007/02/27/logfile01_200702271707,f,6001,X1701700," +
"media,0,0,0,0",
"/log/2007/02/27/logfile01_200702272150,f,6001,X1701700," +
"media,0,0,0,0",
"/log/2007/02/27/logfile02_200702270431,f,6001,X1701700," +
"media,0,0,0,0",
"/log/2007/02/27/logfile02_200702271427,f,6001,X1701700," +
"media,0,0,0,0",
"/log/2007/02/27/logfile02_200702271800,f,6001,X1701700," +
"media,0,0,0,0",
],
'mpra':
["migr,1403126855,0,1335302855,0",
"purge,1403126872,0,0,0",
"migr,1403127173,1335302855,1335303172,0",
"purge,1403127187,0,0,0",
"migr,1403127488,1335303172,1335303487,0",
"purge,1403127502,0,0,0",
"migr,1403127802,1335303487,1335303802,0",
"purge,1403127818,0,0,0",
"migr,1403128118,1335303802,1335304118,0",
"purge,1403128142,0,0,0",
"migr,1403128443,1335304118,1335304443,0",
"purge,1403128458,0,0,0",
],
'tcc_data':
["1398454713,1,5,1,0",
"1398454968,1,5,1,0",
"1398455089,1,5,1,0",
"1398455209,1,5,1,0",
"1398455329,1,5,1,0",
"1398455440,1,5,1,0",
"1398455449,1,5,1,0",
"1398455570,1,5,1,0",
"1398455690,1,5,1,0",
"1398455810,1,5,1,0",
"1398455930,1,5,1,0",
"1398456051,1,5,1,0",
"1398456171,1,5,1,0",
"1398456244,6,10,1,0",
"1398456291,1,5,1,0",
"1398456395,6,10,1,0",
"1398456403,11,15,1,0",
"1398456428,16,20,1,0",
"1398456437,1,5,1,0",
"1398456443,6,10,1,0",
"1398456474,11,15,1,0",
"1398456504,16,20,1,0",
"1398456534,1,5,1,0",
"1398456564,6,10,1,0",
"1398456594,11,15,1,0",
]
}
cfg = CrawlConfig.add_config(close=True,
filename='hpssic_sqlite_test.cfg')
cfg.set('dbi-crawler', 'dbname', self.tmpdir("test.db"))
cfg.set('crawler', 'logpath', self.tmpdir("crawl.log"))
db = CrawlDBI.DBI(dbtype='crawler')
for table in ['checkables', 'mpra', 'tcc_data']:
dbschem.make_table(table)
db.close()
db = CrawlDBI.DBI(dbtype='crawler')
for table in ['checkables', 'mpra', 'tcc_data']:
fld_list = [x[1] for x in db.describe(table=table)
if x[1] != 'rowid']
db.insert(table=table,
fields=fld_list,
data=[x.split(',') for x in data[table]])
db.close()
def test_load_already(self):
"""
With the database and a checkables table in place and records in the
table, calling load() on a Dimension should load the information from
the table into the object. However, it should only count records where
last_check <> 0.
"""
self.dbgfunc()
U.conditional_rm(self.dbname())
CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name()))
Checkable.Checkable.ex_nihilo()
chk = Checkable.Checkable
testdata = [
chk(rowid=1,
path="/abc/001",
type='f',
cos='6001',
checksum=0,
last_check=0),
chk(rowid=2,
path="/abc/002",
type='f',
cos='6002',
checksum=0,
last_check=5),
chk(rowid=3,
path="/abc/003",
type='f',
cos='6003',
checksum=1,
last_check=0),
chk(rowid=4,
path="/abc/004",
type='f',
cos='6001',
checksum=1,
last_check=17),
chk(rowid=5,
path="/abc/005",
type='f',
cos='6002',
checksum=0,
last_check=0),
chk(rowid=6,
path="/abc/006",
type='f',
cos='6003',
checksum=0,
last_check=8),
chk(rowid=7,
path="/abc/007",
type='f',
cos='6001',
checksum=0,
last_check=0),
chk(rowid=8,
path="/abc/008",
type='f',
cos='6002',
checksum=0,
last_check=19),
chk(rowid=9,
path="/abc/009",
type='f',
cos='6003',
checksum=0,
last_check=0),
]
# insert some test data into the table
for t in testdata:
t.persist()
# get a default Dimension with the same name as the data in the table
q = Dimension(name='cos')
# this should load the data from the table into the object
q.load()
# verify the loaded data in the object
self.expected('cos', q.name)
self.assertTrue('6001' in q.p_sum.keys(),
"Expected '6001' in p_sum.keys()")
self.assertTrue('6002' in q.p_sum.keys(),
"Expected '6001' in p_sum.keys()")
self.assertTrue('6003' in q.p_sum.keys(),
"Expected '6003' in p_sum.keys()")
self.assertTrue('6001' in q.s_sum.keys(),
"Expected '6001' in s_sum.keys()")
self.assertTrue('6002' in q.s_sum.keys(),
"Expected '6002' in s_sum.keys()")
self.assertTrue('6003' in q.s_sum.keys(),
"Expected '6003' in s_sum.keys()")