def test_import_filter_by_date_all(self): # First get the date of the last item results, latest_date = SyndicationModularInput.get_feed("http://feeds.feedburner.com/456bereastreet", return_latest_date=True) # Try it and see if we get zero results results = SyndicationModularInput.get_feed("http://feeds.feedburner.com/456bereastreet", include_later_than=latest_date) self.assertGreaterEqual(len(results), 0)
def test_import_filter_by_date(self): # First get the date of the last item results, latest_date = SyndicationModularInput.get_feed("http://127.0.0.1:8888/atom_example.xml", return_latest_date=True) # Back off the date by a second latest_datetime = datetime.fromtimestamp(time.mktime(latest_date)) latest_date_minus_second = latest_datetime - timedelta(seconds=1) latest_date_minus_second_struct = latest_date_minus_second.timetuple() # Try it and see if we get one result results = SyndicationModularInput.get_feed("http://127.0.0.1:8888/atom_example.xml", include_later_than=latest_date_minus_second_struct) self.assertGreaterEqual(len(results), 1)
def test_get_auth_handler_none(self): auth_handler = SyndicationModularInput.get_auth_handler( "http://127.0.0.1:8888/rss_example.xml", username="******", password="******") self.assertEquals(auth_handler, None)
def test_get_auth_handler(self): auth_handler = SyndicationModularInput.get_auth_handler( "http://127.0.0.1:8888/auth/rss_example.xml", username="******", password="******") self.assertIsInstance(auth_handler, urllib2.HTTPBasicAuthHandler)
def test_basic_auth_rss(self): username = '******' password = '******' results = SyndicationModularInput.get_feed("http://127.0.0.1:8888/auth/rss_example.xml", username="******", password="******") self.assertEqual(len(results), 2)
def test_flatten_dict(self): dict = {'list': {'one': 'uno', 'two': 'dos'}} d = SyndicationModularInput.flatten(dict) self.assertEquals(len(d), 2) self.assertEquals(d['list.one'], 'uno')
def test_import_return_latest_date(self): results, latest_date = SyndicationModularInput.get_feed( "http://feeds.feedburner.com/456bereastreet", return_latest_date=True) self.assertGreaterEqual(len(results), 10) self.assertIsNotNone(latest_date)
def test_flatten_int(self): dict = {'int': 1} d = SyndicationModularInput.flatten(dict) self.assertEquals(len(d), 1) self.assertEquals(d['int'], 1)
def test_flatten_none(self): dict = {'none': None} d = SyndicationModularInput.flatten(dict) self.assertEquals(len(d), 1) self.assertEquals(d['none'], None)
def handle_results(self, results, session_key, in_preview): results = SyndicationModularInput.get_feed(self.url, self.username, self.password, clean_html=self.clean_html, logger=self.logger) self.output_results(results)
def test_flatten_time(self): t = time.strptime("9 Feb 15", "%d %b %y") dict = {'time': t} d = SyndicationModularInput.flatten(dict) self.assertEquals(d['time'], '2015-02-09T00:00:00Z')
def test_flatten_boolean(self): dict = {'TrueDat': True, 'FalseDat': False} d = SyndicationModularInput.flatten(dict) self.assertEquals(len(d), 2) self.assertEquals(d['TrueDat'], True) self.assertEquals(d['FalseDat'], False)
def test_flatten_list(self): dict = {'list': ['first', 'second']} d = SyndicationModularInput.flatten(dict) self.assertEquals(len(d), 2) self.assertEquals(d['list.0'], 'first') self.assertEquals(d['list.1'], 'second')
def test_flatten_dict_sort(self): # https://lukemurphey.net/issues/2039 d = { 'list': { '3' : 'tres', '1' : 'uno', '2' : 'dos', '5' : 'cinco', '4' : 'cuatro', '6' : 'seis' } } d = SyndicationModularInput.flatten(d, sort=True) self.assertEquals(len(d), 6) keys = list(d.keys()) self.assertEquals(keys[0], 'list.1') self.assertEquals(keys[1], 'list.2') self.assertEquals(keys[2], 'list.3') self.assertEquals(keys[3], 'list.4') self.assertEquals(keys[4], 'list.5') self.assertEquals(keys[5], 'list.6')
def test_get_realm_and_auth_type_invalid(self): auth_realm, auth_type = SyndicationModularInput.get_realm_and_auth_type("http://127.0.0.1:8888/invalid_auth/rss_example.xml", username="******", password="******") self.assertEquals(auth_realm, None) self.assertEquals(auth_type, None)
def test_cleanup_html_rss(self): # https://lukemurphey.net/issues/2038 results = SyndicationModularInput.get_feed("http://127.0.0.1:8888/rss_with_html.xml", clean_html=True) self.assertEqual(len(results), 3) self.assertEqual(results[2]['content.0.value'][:120], " 1. **Introduction**\n\nIt seems that Google Chrome extensions have become quite the tool for banking\nmalware fraudsters.")
def test_import_rss_public(self): results = SyndicationModularInput.get_feed("http://answers.splunk.com/feed/questions.rss") self.assertGreaterEqual(len(results), 10)
def test_import_return_latest_date(self): results, latest_date = SyndicationModularInput.get_feed("http://127.0.0.1:8888/atom_example.xml", return_latest_date=True) self.assertGreaterEqual(len(results), 0) self.assertIsNotNone(latest_date)
def test_import_atom(self): results = SyndicationModularInput.get_feed("http://127.0.0.1:8888/atom_example.xml") self.assertGreaterEqual(len(results), 1)
def test_import_atom_public(self): results = SyndicationModularInput.get_feed("http://currentworldnews.blogspot.com/atom.xml") self.assertGreaterEqual(len(results), 10)