def test_reset_errors_on_successful_normalization(self): """ Test that error state is removed from hpfeed entries on successful normalizations. """ #the entry we are going to manipulate o = ObjectId() db = MongoClient('localhost', 27017)[self.dbname] db['hpfeed'].insert( {'_id': o, 'channel': 'channel3', 'ident': 'ident3', 'payload': 'payload3', 'timestamp': datetime.utcnow(), 'normalized': False, 'last_error': 'Some error', 'last_error_timestamp': datetime.now()}) sut = mnemodb.MnemoDB(self.dbname) #insert dummy url entry sut.insert_normalized([{'url': {'url': '/dummy/url'}}], o) #retrieve the item from the database r = db['hpfeed'].find_one({'_id': o}) #check that stats is as expected self.assertTrue(r['normalized']) self.assertNotIn('last_error', r) self.assertNotIn('last_error_timestamp', r)
def test_set_errors(self): """ Test that error state is correctly set in the specified hpfeed entries. """ #the entry we are going to manipulate o = ObjectId() insert_items = [ {'channel': 'channel1', 'ident': 'ident1', 'payload': 'payload1', 'timestamp': datetime.utcnow(), 'normalized': True}, {'_id': o, 'channel': 'channel2', 'ident': 'ident2', 'payload': 'payload2', 'timestamp': datetime.utcnow(), 'normalized': True}, {'channel': 'channel3', 'ident': 'ident3', 'payload': 'payload3', 'timestamp': datetime.utcnow(), 'normalized': True} ] db = MongoClient('localhost', 27017)[self.dbname] for item in insert_items: db['hpfeed'].insert(item) sut = mnemodb.MnemoDB(self.dbname) #set a single item in error state dt = datetime.now() sut.hpfeed_set_errors([ {'_id': o, 'last_error': "Some error message", 'last_error_timestamp': dt} ]) #retrieve it again and check if the error state was set. r = db['hpfeed'].find_one({'_id': o}) self.assertEqual('Some error message', r['last_error']) self.assertAlmostEqual(dt, r['last_error_timestamp'], delta=timedelta(seconds=1))
def test_insert_dorks(self): """ Test insertion into the dorks collection. """ sut = mnemodb.MnemoDB(self.dbname) insert_items = [ {'dork': {'type': 'inurl', 'content': '/somedork.php', 'count': 1, 'timestamp': datetime.now()}}, {'dork': {'type': 'inurl', 'content': '/somedork.php', 'count': 1, 'timestamp': datetime.now()}}, {'dork': {'type': 'inurl', 'content': '/otherdork.php', 'count': 1, 'timestamp': datetime.now()}}, ] sut.insert_normalized(insert_items, ObjectId()) db = MongoClient('localhost', 27017)[self.dbname] #we expect two entries in the database db_entries = db['dork'].find().count() self.assertEqual(2, db_entries) result_one = db['dork'].find_one({'content': '/somedork.php'}) self.assertIn('lasttime', result_one) self.assertEqual(2, result_one['count']) self.assertEqual('inurl', result_one['type']) result_one = db['dork'].find_one({'content': '/otherdork.php'}) self.assertIn('lasttime', result_one) self.assertEqual(1, result_one['count']) self.assertEqual('inurl', result_one['type'])
def test_insert_hpfeed(self): """ Test correct insertion of raw hpfeed entry into mongodb. """ sut = mnemodb.MnemoDB(self.dbname) #ident, channel, payload insert_items = [ ('ident1', 'testchannel1', 'payload1'), ('ident2', 'testchannel2', 'payload2'), ('ident3', 'testchannel3', 'payload3') ] for item in insert_items: sut.insert_hpfeed(item[0], item[1], item[2]) #Check that the hpfeed collection has the correct amount of items db = MongoClient('localhost', 27017)[self.dbname] actual = list(db.hpfeed.find()) #TODO: assert content of actual self.assertEqual(len(insert_items), len(actual))
def test_reset(self): """ Check if all normalized collections get dropped and that normalized is set to False in all hpfeed entries. """ db = MongoClient('localhost', 27017)[self.dbname] #prepare and insert dummy values directly into the hpfeed collection insert_items = [ {'channel': 'channel1', 'ident': 'ident1', 'payload': 'payload1', 'timestamp': datetime.utcnow(), 'normalized': True}, {'channel': 'channel2', 'ident': 'ident2', 'payload': 'payload2', 'timestamp': datetime.utcnow(), 'normalized': True}, {'channel': 'channel3', 'ident': 'ident3', 'payload': 'payload3', 'timestamp': datetime.utcnow(), 'normalized': True, 'last_error': "Some error", 'last_error_timestamp': datetime.now()} ] for item in insert_items: db['hpfeed'].insert(item) #create a few dummy collection that we expect to get dropped db['somecollection1'].insert({'something': 'something'}) db['somecollection2'].insert({'something': 'something'}) sut = mnemodb.MnemoDB(self.dbname) #This is the function we are testing sut.reset_normalized() #has normalized collections been removed self.assertNotIn('somecollection1', db.collection_names()) self.assertNotIn('somecollection2', db.collection_names()) #has all normalized been set to True self.assertEquals(0, db['hpfeed'].find({'normalized': True}).count()) #has last_error attribute been removed self.assertEquals(0, db['hpfeed'].find({'last_error': {'$exists': 1}}).count()) #has last_error_timestamp attribute been removed self.assertEquals(0, db['hpfeed'].find({'last_error_timestamp': {'$exists': 1}}).count())
default=False, help='Do not enable the webapi.') args = parser.parse_args() c = parse_config(args.config_file) git_ref = "Unknown" if os.path.isfile('.git/refs/heads/master'): with open('.git/refs/heads/master', 'r') as f: git_ref = f.readline().rstrip() logger.info('Starting mnemosyne. (Git: {0})'.format(git_ref)) greenlets = {} db = mnemodb.MnemoDB(c['mongo_db']) webapi = None hpfriends_puller = None normalizer = None if args.reset: print 'Renormalization (reset) of a large database can take several days.' answer = raw_input('Write YES if you want to continue: ') if answer == 'YES': db.reset_normalized() else: print 'Aborting' sys.exit(0) if not args.no_feedpuller:
args = parser.parse_args() c = parse_config(args.config_file) git_ref = "Unknown" if os.path.isfile('.git/refs/heads/master'): with open('.git/refs/heads/master', 'r') as f: git_ref = f.readline().rstrip() logger.info('Starting mnemosyne. (Git: {0})'.format(git_ref)) greenlets = {} if c['mongo_auth']: db = mnemodb.MnemoDB(c['mongo_host'], c['mongo_port'], c['mongo_db'], c['mongo_user'], c['mongo_password'], c['mongo_auth_mechanism']) else: db = mnemodb.MnemoDB(c['mongo_host'], c['mongo_port'], c['mongo_db']) webapi = None hpfriends_puller = None normalizer = None if args.reset: print 'Renormalization (reset) of a large database can take several days.' answer = raw_input('Write YES if you want to continue: ') if answer == 'YES': db.reset_normalized() else: print 'Aborting'
) args = parser.parse_args() c = parse_config(args.config_file) git_ref = "Unknown" if os.path.isfile('.git/refs/heads/master'): with open('.git/refs/heads/master', 'r') as f: git_ref = f.readline().rstrip() logger.info('Starting mnemosyne. (Git: {0})'.format(git_ref)) greenlets = {} db = mnemodb.MnemoDB(host=c['mongo_host'], port=c['mongo_port'], database_name=c['mongo_db'], indexttl=c['mongo_indexttl']) hpfriends_puller = None normalizer = None if args.reset: print( 'Renormalization (reset) of a large database can take several days.' ) answer = input('Write YES if you want to continue: ') if answer == 'YES': db.reset_normalized() else: print('Aborting') sys.exit(0)