def test_load_new(self): """ With the database and checkables table in place, create a new Dimension that is not in the table. Calling load() on it should be a no-op -- the object should not be stored to the database and its contents should not be changed. """ self.dbgfunc() # reboot the database and call persist() to create the table without # adding any data U.conditional_rm(self.dbname()) CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name())) Checkable.Checkable.ex_nihilo() ignore = Dimension(name='foobar') # get a Dimension object that is not in the table test = Dimension(name='notindb') # make a copy of the object for reference (not just a handle to the # same ojbect) ref = copy.deepcopy(test) # call load(), which should be a no op test.load() # verify that the object didn't change self.expected(ref.name, test.name) self.expected(ref.sampsize, test.sampsize) self.expected(ref.p_sum, test.p_sum) self.expected(ref.s_sum, test.s_sum)
def test_ctor_attrs(self): """ Verify that a newly created Dimension object has the following attributes: - name (string) - sampsize (small float value, e.g., 0.005) - p_sum (empty dict) - s_sum (empty dict) - methods > sum_total > load """ dimname = 'cos' CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name())) Checkable.Checkable.ex_nihilo() a = Dimension(name=dimname, sampsize=0.005) for attr in [ 'name', 'sampsize', 'p_sum', 's_sum', 'sum_total', 'load', ]: self.assertTrue( hasattr(a, attr), "Object %s does not have expected attribute %s" % (a, attr))
def test_sum_total(self): """ Return the sum of all the 'count' values in either the p_sum or s_sum dictionary. """ CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name())) Checkable.Checkable.ex_nihilo() a = Dimension(name='sum_total') a.p_sum = { '6001': { 'count': 2, 'pct': 50.0 }, '5081': { 'count': 2, 'pct': 50.0 } } a.s_sum = { '6001': { 'count': 2, 'pct': 40.0 }, '5081': { 'count': 3, 'pct': 60.0 } } self.expected(4, a.sum_total()) self.expected(4, a.sum_total(dict=a.p_sum)) self.expected(5, a.sum_total(which='s')) self.expected(5, a.sum_total(dict=a.s_sum))
def test_load_new(self): """ With the database and checkables table in place, create a new Dimension that is not in the table. Calling load() on it should be a no-op -- the object should not be stored to the database and its contents should not be changed. """ self.dbgfunc() # reboot the database and call persist() to create the table without # adding any data U.conditional_rm(self.dbname()) CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name())) Checkable.Checkable.ex_nihilo() ignore = Dimension(name='foobar') # get a Dimension object that is not in the table test = Dimension(name='notindb') # make a copy of the object for reference (not just a handle to the # same ojbect) ref = copy.deepcopy(test) # call load(), which should be a no op test.load() # verify that the object didn't change self.expected(ref.name, test.name) self.expected(ref.sampsize, test.sampsize) self.expected(ref.p_sum, test.p_sum) self.expected(ref.s_sum, test.s_sum)
def test_ctor_attrs(self): """ Verify that a newly created Dimension object has the following attributes: - name (string) - sampsize (small float value, e.g., 0.005) - p_sum (empty dict) - s_sum (empty dict) - methods > sum_total > load """ dimname = 'cos' CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name())) Checkable.Checkable.ex_nihilo() a = Dimension(name=dimname, sampsize=0.005) for attr in ['name', 'sampsize', 'p_sum', 's_sum', 'sum_total', 'load', ]: self.assertTrue(hasattr(a, attr), "Object %s does not have expected attribute %s" % (a, attr))
def test_repr(self): """ Method __repr__ should return <Dimension(name='foo')>. """ CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name())) exp = "Dimension(name='foo')" a = eval(exp) self.expected(exp, a.__repr__())
def test_repr(self): """ Method __repr__ should return <Dimension(name='foo')>. """ CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name())) exp = "Dimension(name='foo')" a = eval(exp) self.expected(exp, a.__repr__())
def test_load_already(self): """ With the database and a checkables table in place and records in the table, calling load() on a Dimension should load the information from the table into the object. However, it should only count records where last_check <> 0. """ self.dbgfunc() U.conditional_rm(self.dbname()) CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name())) Checkable.Checkable.ex_nihilo() chk = Checkable.Checkable testdata = [ chk(rowid=1, path="/abc/001", type='f', cos='6001', checksum=0, last_check=0), chk(rowid=2, path="/abc/002", type='f', cos='6002', checksum=0, last_check=5), chk(rowid=3, path="/abc/003", type='f', cos='6003', checksum=1, last_check=0), chk(rowid=4, path="/abc/004", type='f', cos='6001', checksum=1, last_check=17), chk(rowid=5, path="/abc/005", type='f', cos='6002', checksum=0, last_check=0), chk(rowid=6, path="/abc/006", type='f', cos='6003', checksum=0, last_check=8), chk(rowid=7, path="/abc/007", type='f', cos='6001', checksum=0, last_check=0), chk(rowid=8, path="/abc/008", type='f', cos='6002', checksum=0, last_check=19), chk(rowid=9, path="/abc/009", type='f', cos='6003', checksum=0, last_check=0), ] # insert some test data into the table for t in testdata: t.persist() # get a default Dimension with the same name as the data in the table q = Dimension(name='cos') # this should load the data from the table into the object q.load() # verify the loaded data in the object self.expected('cos', q.name) self.assertTrue('6001' in q.p_sum.keys(), "Expected '6001' in p_sum.keys()") self.assertTrue('6002' in q.p_sum.keys(), "Expected '6001' in p_sum.keys()") self.assertTrue('6003' in q.p_sum.keys(), "Expected '6003' in p_sum.keys()") self.assertTrue('6001' in q.s_sum.keys(), "Expected '6001' in s_sum.keys()") self.assertTrue('6002' in q.s_sum.keys(), "Expected '6002' in s_sum.keys()") self.assertTrue('6003' in q.s_sum.keys(), "Expected '6003' in s_sum.keys()")
def test_ctor_defaults(self): """ A new Dimension with only the name specified should have the right defaults. """ dimname = 'cos' CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name())) a = Dimension(name=dimname) self.expected(dimname, a.name) self.expected(0.01, a.sampsize) self.expected({}, a.p_sum) self.expected({}, a.s_sum)
def test_ctor_defaults(self): """ A new Dimension with only the name specified should have the right defaults. """ dimname = 'cos' CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name())) a = Dimension(name=dimname) self.expected(dimname, a.name) self.expected(0.01, a.sampsize) self.expected({}, a.p_sum) self.expected({}, a.s_sum)
def test_ctor_bad_attr(self): """ Attempting to create a Dimension with attrs that are not in the settable list should get an exception. """ dimname = 'bad_attr' CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name())) got_exception = False self.assertRaisesMsg(StandardError, "Attribute 'catl' is not valid", Dimension, name=dimname, catl=[1, 2, 3]) self.assertRaisesMsg(StandardError, "Attribute 'aardvark' is not valid", Dimension, name=dimname, aardvark='Fanny Brice')
def test_sum_total(self): """ Return the sum of all the 'count' values in either the p_sum or s_sum dictionary. """ CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name())) Checkable.Checkable.ex_nihilo() a = Dimension(name='sum_total') a.p_sum = {'6001': {'count': 2, 'pct': 50.0}, '5081': {'count': 2, 'pct': 50.0} } a.s_sum = {'6001': {'count': 2, 'pct': 40.0}, '5081': {'count': 3, 'pct': 60.0} } self.expected(4, a.sum_total()) self.expected(4, a.sum_total(dict=a.p_sum)) self.expected(5, a.sum_total(which='s')) self.expected(5, a.sum_total(dict=a.s_sum))
def test_ctor_bad_attr(self): """ Attempting to create a Dimension with attrs that are not in the settable list should get an exception. """ dimname = 'bad_attr' CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name())) got_exception = False self.assertRaisesMsg(StandardError, "Attribute 'catl' is not valid", Dimension, name=dimname, catl=[1, 2, 3]) self.assertRaisesMsg(StandardError, "Attribute 'aardvark' is not valid", Dimension, name=dimname, aardvark='Fanny Brice')
def test_ctor_no_cv_section(self): """ If there is no cv section in the config, reset_atime and hash_algorithm should take on their default values. """ self.dbgfunc() cfg = copy.deepcopy(self.cfg_d) del cfg['cv'] zcfg = CrawlConfig.add_config(close=True, dct=cfg) self.assertFalse(zcfg.has_section('cv')) h = hpss.HSI(connect=False) self.expected(False, h.reset_atime) self.expected(None, h.hash_algorithm)
def test_alert_email_defcfg(self): """ Generate an e-mail alert using the default config and verify that it was sent (this is where we use 'monkey patching'). """ self.dbgfunc() fakesmtp.inbox = [] CrawlConfig.add_config(close=True) # with U.tmpenv('CRAWL_CONF', 'hpssic_test.cfg'): with U.tmpenv('CRAWL_CONF', None): logfile = self.tmpdir('alert_email.log') targets = "[email protected], [email protected]" payload = 'this is an e-mail alert' sender = 'hpssic@' + util.hostname(long=True) CrawlConfig.log(logpath=logfile, close=True) x = Alert.Alert(caller='cv', msg=payload) m = fakesmtp.inbox[0] self.expected(', '.join(m.to_address), targets) self.expected(m.from_address, sender) self.expected_in('sent mail to', util.contents(logfile)) self.expected_in(payload, m.fullmessage)
def test_alert_email_defcfg(self): """ Generate an e-mail alert using the default config and verify that it was sent (this is where we use 'monkey patching'). """ self.dbgfunc() fakesmtp.inbox = [] CrawlConfig.add_config(close=True) # with U.tmpenv('CRAWL_CONF', 'hpssic_test.cfg'): with U.tmpenv('CRAWL_CONF', None): logfile = self.tmpdir('alert_email.log') targets = "[email protected], [email protected]" payload = 'this is an e-mail alert' sender = 'hpssic@' + util.hostname(long=True) CrawlConfig.log(logpath=logfile, close=True) x = Alert.Alert(caller='cv', msg=payload) m = fakesmtp.inbox[0] self.expected(', '.join(m.to_address), targets) self.expected(m.from_address, sender) self.expected_in('sent mail to', util.contents(logfile)) self.expected_in(payload, m.fullmessage)
def test_html_report(self): """ Try running 'html report > filename' and verify that 1) no traceback occurs and 2) something is actually written to the output file. """ self.dbgfunc() cfpath = self.tmpdir("crawl.cfg") cfg = CrawlConfig.add_config() cfg.crawl_write(open(cfpath, 'w')) cmd = "html report --config %s" % cfpath CrawlConfig.log(cmd, close=True) result = pexpect.run(cmd) if "HPSS Unavailable" in result: pytest.skip("HPSS Unavailable") self.validate_report(result)
def test_html_report(self): """ Try running 'html report > filename' and verify that 1) no traceback occurs and 2) something is actually written to the output file. """ self.dbgfunc() cfpath = self.tmpdir("crawl.cfg") cfg = CrawlConfig.add_config() cfg.crawl_write(open(cfpath, 'w')) cmd = "html report --config %s" % cfpath CrawlConfig.log(cmd, close=True) result = pexpect.run(cmd) if "HPSS Unavailable" in result: pytest.skip("HPSS Unavailable") self.validate_report(result)
def setUp(self): """ Set up for each test: create the mpra table with prefix 'test', touch the mpra test file in the tmpdir """ super(MpraResetTest, self).setUp() if pexpect.which("mpra"): self.cmd = "mpra" elif os.path.exists("bin/mpra"): self.cmd = "bin/mpra" else: raise HpssicError("mpra command not found") self.rptname = self.tmpdir("mpra_report.txt") self.cfgname = self.tmpdir("mpra_test.cfg") self.cfg = CrawlConfig.add_config(close=True, dct=self.cfg_dict()) self.cfg.crawl_write(open(self.cfgname, 'w')) dbschem.make_table("mpra", cfg=self.cfg) U.touch(self.rptname)
def test_get_html_report(self): """ Call html_lib.get_html_report() directly """ self.dbgfunc() c = CrawlConfig.add_config() db = CrawlDBI.DBI(dbtype="crawler") dbschem.drop_table(table="lscos") self.expected(False, db.table_exists(table="lscos")) try: result = html_lib.get_html_report('') except hpss.HSIerror as e: if MSG.hpss_unavailable in str(e): pytest.skip(str(e)) self.expected(True, db.table_exists(table="lscos")) db.close() self.validate_report(result)
def test_get_html_report(self): """ Call html_lib.get_html_report() directly """ self.dbgfunc() c = CrawlConfig.add_config() db = CrawlDBI.DBI(dbtype="crawler") dbschem.drop_table(table="lscos") self.expected(False, db.table_exists(table="lscos")) try: result = html_lib.get_html_report('') except hpss.HSIerror as e: if MSG.hpss_unavailable in str(e): pytest.skip(str(e)) self.expected(True, db.table_exists(table="lscos")) db.close() self.validate_report(result)
def setUp(self): """ Set up for tests """ super(HtmlTest, self).setUp() data = { 'checkables': [ "/log/2007/05/27/logfile01_200705270654,f,6001,X1701700," + "media,0,1410363023,0,0", "/log/2007/05/27/logfile01_200705271304,f,6001,X1701700," + "media,0,1410363023,0,0", "/log/2007/05/27/logfile02_200705270424,f,6001,X1701700," + "media,0,1410363024,0,0", "/log/2007/05/27/logfile02_200705270847,f,6001,X1701700," + "media,0,1410363024,0,0", "/log/2007/05/27/logfile02_200705271716,f,6001,X1701700," + "media,0,1410363025,0,0", "/log/2007/02/27/logfile01_200702270958,f,6001,X1701700," + "media,0,0,0,0", "/log/2007/02/27/logfile01_200702271707,f,6001,X1701700," + "media,0,0,0,0", "/log/2007/02/27/logfile01_200702272150,f,6001,X1701700," + "media,0,0,0,0", "/log/2007/02/27/logfile02_200702270431,f,6001,X1701700," + "media,0,0,0,0", "/log/2007/02/27/logfile02_200702271427,f,6001,X1701700," + "media,0,0,0,0", "/log/2007/02/27/logfile02_200702271800,f,6001,X1701700," + "media,0,0,0,0", ], 'mpra': [ "migr,1403126855,0,1335302855,0", "purge,1403126872,0,0,0", "migr,1403127173,1335302855,1335303172,0", "purge,1403127187,0,0,0", "migr,1403127488,1335303172,1335303487,0", "purge,1403127502,0,0,0", "migr,1403127802,1335303487,1335303802,0", "purge,1403127818,0,0,0", "migr,1403128118,1335303802,1335304118,0", "purge,1403128142,0,0,0", "migr,1403128443,1335304118,1335304443,0", "purge,1403128458,0,0,0", ], 'tcc_data': [ "1398454713,1,5,1,0", "1398454968,1,5,1,0", "1398455089,1,5,1,0", "1398455209,1,5,1,0", "1398455329,1,5,1,0", "1398455440,1,5,1,0", "1398455449,1,5,1,0", "1398455570,1,5,1,0", "1398455690,1,5,1,0", "1398455810,1,5,1,0", "1398455930,1,5,1,0", "1398456051,1,5,1,0", "1398456171,1,5,1,0", "1398456244,6,10,1,0", "1398456291,1,5,1,0", "1398456395,6,10,1,0", "1398456403,11,15,1,0", "1398456428,16,20,1,0", "1398456437,1,5,1,0", "1398456443,6,10,1,0", "1398456474,11,15,1,0", "1398456504,16,20,1,0", "1398456534,1,5,1,0", "1398456564,6,10,1,0", "1398456594,11,15,1,0", ] } cfg = CrawlConfig.add_config(close=True, filename='hpssic_sqlite_test.cfg') cfg.set('dbi-crawler', 'dbname', self.tmpdir("test.db")) cfg.set('crawler', 'logpath', self.tmpdir("crawl.log")) db = CrawlDBI.DBI(dbtype='crawler') for table in ['checkables', 'mpra', 'tcc_data']: dbschem.make_table(table) db.close() db = CrawlDBI.DBI(dbtype='crawler') for table in ['checkables', 'mpra', 'tcc_data']: fld_list = [ x[1] for x in db.describe(table=table) if x[1] != 'rowid' ] db.insert(table=table, fields=fld_list, data=[x.split(',') for x in data[table]]) db.close()
def setUp(self): """ Set up for tests """ super(HtmlTest, self).setUp() data = {'checkables': ["/log/2007/05/27/logfile01_200705270654,f,6001,X1701700," + "media,0,1410363023,0,0", "/log/2007/05/27/logfile01_200705271304,f,6001,X1701700," + "media,0,1410363023,0,0", "/log/2007/05/27/logfile02_200705270424,f,6001,X1701700," + "media,0,1410363024,0,0", "/log/2007/05/27/logfile02_200705270847,f,6001,X1701700," + "media,0,1410363024,0,0", "/log/2007/05/27/logfile02_200705271716,f,6001,X1701700," + "media,0,1410363025,0,0", "/log/2007/02/27/logfile01_200702270958,f,6001,X1701700," + "media,0,0,0,0", "/log/2007/02/27/logfile01_200702271707,f,6001,X1701700," + "media,0,0,0,0", "/log/2007/02/27/logfile01_200702272150,f,6001,X1701700," + "media,0,0,0,0", "/log/2007/02/27/logfile02_200702270431,f,6001,X1701700," + "media,0,0,0,0", "/log/2007/02/27/logfile02_200702271427,f,6001,X1701700," + "media,0,0,0,0", "/log/2007/02/27/logfile02_200702271800,f,6001,X1701700," + "media,0,0,0,0", ], 'mpra': ["migr,1403126855,0,1335302855,0", "purge,1403126872,0,0,0", "migr,1403127173,1335302855,1335303172,0", "purge,1403127187,0,0,0", "migr,1403127488,1335303172,1335303487,0", "purge,1403127502,0,0,0", "migr,1403127802,1335303487,1335303802,0", "purge,1403127818,0,0,0", "migr,1403128118,1335303802,1335304118,0", "purge,1403128142,0,0,0", "migr,1403128443,1335304118,1335304443,0", "purge,1403128458,0,0,0", ], 'tcc_data': ["1398454713,1,5,1,0", "1398454968,1,5,1,0", "1398455089,1,5,1,0", "1398455209,1,5,1,0", "1398455329,1,5,1,0", "1398455440,1,5,1,0", "1398455449,1,5,1,0", "1398455570,1,5,1,0", "1398455690,1,5,1,0", "1398455810,1,5,1,0", "1398455930,1,5,1,0", "1398456051,1,5,1,0", "1398456171,1,5,1,0", "1398456244,6,10,1,0", "1398456291,1,5,1,0", "1398456395,6,10,1,0", "1398456403,11,15,1,0", "1398456428,16,20,1,0", "1398456437,1,5,1,0", "1398456443,6,10,1,0", "1398456474,11,15,1,0", "1398456504,16,20,1,0", "1398456534,1,5,1,0", "1398456564,6,10,1,0", "1398456594,11,15,1,0", ] } cfg = CrawlConfig.add_config(close=True, filename='hpssic_sqlite_test.cfg') cfg.set('dbi-crawler', 'dbname', self.tmpdir("test.db")) cfg.set('crawler', 'logpath', self.tmpdir("crawl.log")) db = CrawlDBI.DBI(dbtype='crawler') for table in ['checkables', 'mpra', 'tcc_data']: dbschem.make_table(table) db.close() db = CrawlDBI.DBI(dbtype='crawler') for table in ['checkables', 'mpra', 'tcc_data']: fld_list = [x[1] for x in db.describe(table=table) if x[1] != 'rowid'] db.insert(table=table, fields=fld_list, data=[x.split(',') for x in data[table]]) db.close()
def test_load_already(self): """ With the database and a checkables table in place and records in the table, calling load() on a Dimension should load the information from the table into the object. However, it should only count records where last_check <> 0. """ self.dbgfunc() U.conditional_rm(self.dbname()) CrawlConfig.add_config(close=True, dct=self.cfg_dict(U.my_name())) Checkable.Checkable.ex_nihilo() chk = Checkable.Checkable testdata = [ chk(rowid=1, path="/abc/001", type='f', cos='6001', checksum=0, last_check=0), chk(rowid=2, path="/abc/002", type='f', cos='6002', checksum=0, last_check=5), chk(rowid=3, path="/abc/003", type='f', cos='6003', checksum=1, last_check=0), chk(rowid=4, path="/abc/004", type='f', cos='6001', checksum=1, last_check=17), chk(rowid=5, path="/abc/005", type='f', cos='6002', checksum=0, last_check=0), chk(rowid=6, path="/abc/006", type='f', cos='6003', checksum=0, last_check=8), chk(rowid=7, path="/abc/007", type='f', cos='6001', checksum=0, last_check=0), chk(rowid=8, path="/abc/008", type='f', cos='6002', checksum=0, last_check=19), chk(rowid=9, path="/abc/009", type='f', cos='6003', checksum=0, last_check=0), ] # insert some test data into the table for t in testdata: t.persist() # get a default Dimension with the same name as the data in the table q = Dimension(name='cos') # this should load the data from the table into the object q.load() # verify the loaded data in the object self.expected('cos', q.name) self.assertTrue('6001' in q.p_sum.keys(), "Expected '6001' in p_sum.keys()") self.assertTrue('6002' in q.p_sum.keys(), "Expected '6001' in p_sum.keys()") self.assertTrue('6003' in q.p_sum.keys(), "Expected '6003' in p_sum.keys()") self.assertTrue('6001' in q.s_sum.keys(), "Expected '6001' in s_sum.keys()") self.assertTrue('6002' in q.s_sum.keys(), "Expected '6002' in s_sum.keys()") self.assertTrue('6003' in q.s_sum.keys(), "Expected '6003' in s_sum.keys()")