def test_import_filter_by_date_all(self):
# First get the date of the last item
results, latest_date = SyndicationModularInput.get_feed("http://feeds.feedburner.com/456bereastreet", return_latest_date=True)
# Try it and see if we get zero results
results = SyndicationModularInput.get_feed("http://feeds.feedburner.com/456bereastreet", include_later_than=latest_date)
self.assertGreaterEqual(len(results), 0)
def test_import_filter_by_date(self):
# First get the date of the last item
results, latest_date = SyndicationModularInput.get_feed("http://127.0.0.1:8888/atom_example.xml", return_latest_date=True)
# Back off the date by a second
latest_datetime = datetime.fromtimestamp(time.mktime(latest_date))
latest_date_minus_second = latest_datetime - timedelta(seconds=1)
latest_date_minus_second_struct = latest_date_minus_second.timetuple()
# Try it and see if we get one result
results = SyndicationModularInput.get_feed("http://127.0.0.1:8888/atom_example.xml", include_later_than=latest_date_minus_second_struct)
self.assertGreaterEqual(len(results), 1)
def test_get_auth_handler_none(self):
auth_handler = SyndicationModularInput.get_auth_handler(
"http://127.0.0.1:8888/rss_example.xml",
username="******",
password="******")
self.assertEquals(auth_handler, None)
def test_get_auth_handler(self):
auth_handler = SyndicationModularInput.get_auth_handler(
"http://127.0.0.1:8888/auth/rss_example.xml",
username="******",
password="******")
self.assertIsInstance(auth_handler, urllib2.HTTPBasicAuthHandler)
def test_basic_auth_rss(self):
username = '******'
password = '******'
results = SyndicationModularInput.get_feed("http://127.0.0.1:8888/auth/rss_example.xml", username="******", password="******")
self.assertEqual(len(results), 2)
def test_flatten_dict(self):
dict = {'list': {'one': 'uno', 'two': 'dos'}}
d = SyndicationModularInput.flatten(dict)
self.assertEquals(len(d), 2)
self.assertEquals(d['list.one'], 'uno')
def test_import_return_latest_date(self):
results, latest_date = SyndicationModularInput.get_feed(
"http://feeds.feedburner.com/456bereastreet",
return_latest_date=True)
self.assertGreaterEqual(len(results), 10)
self.assertIsNotNone(latest_date)
def test_flatten_int(self):
dict = {'int': 1}
d = SyndicationModularInput.flatten(dict)
self.assertEquals(len(d), 1)
self.assertEquals(d['int'], 1)
def test_flatten_none(self):
dict = {'none': None}
d = SyndicationModularInput.flatten(dict)
self.assertEquals(len(d), 1)
self.assertEquals(d['none'], None)
def handle_results(self, results, session_key, in_preview):
results = SyndicationModularInput.get_feed(self.url,
self.username,
self.password,
clean_html=self.clean_html,
logger=self.logger)
self.output_results(results)
def test_flatten_time(self):
t = time.strptime("9 Feb 15", "%d %b %y")
dict = {'time': t}
d = SyndicationModularInput.flatten(dict)
self.assertEquals(d['time'], '2015-02-09T00:00:00Z')
def test_flatten_boolean(self):
dict = {'TrueDat': True, 'FalseDat': False}
d = SyndicationModularInput.flatten(dict)
self.assertEquals(len(d), 2)
self.assertEquals(d['TrueDat'], True)
self.assertEquals(d['FalseDat'], False)
def test_flatten_list(self):
dict = {'list': ['first', 'second']}
d = SyndicationModularInput.flatten(dict)
self.assertEquals(len(d), 2)
self.assertEquals(d['list.0'], 'first')
self.assertEquals(d['list.1'], 'second')
def test_flatten_dict_sort(self):
# https://lukemurphey.net/issues/2039
d = {
'list': {
'3' : 'tres',
'1' : 'uno',
'2' : 'dos',
'5' : 'cinco',
'4' : 'cuatro',
'6' : 'seis'
}
}
d = SyndicationModularInput.flatten(d, sort=True)
self.assertEquals(len(d), 6)
keys = list(d.keys())
self.assertEquals(keys[0], 'list.1')
self.assertEquals(keys[1], 'list.2')
self.assertEquals(keys[2], 'list.3')
self.assertEquals(keys[3], 'list.4')
self.assertEquals(keys[4], 'list.5')
self.assertEquals(keys[5], 'list.6')
def test_get_realm_and_auth_type_invalid(self):
auth_realm, auth_type = SyndicationModularInput.get_realm_and_auth_type("http://127.0.0.1:8888/invalid_auth/rss_example.xml", username="******", password="******")
self.assertEquals(auth_realm, None)
self.assertEquals(auth_type, None)
def test_cleanup_html_rss(self):
# https://lukemurphey.net/issues/2038
results = SyndicationModularInput.get_feed("http://127.0.0.1:8888/rss_with_html.xml", clean_html=True)
self.assertEqual(len(results), 3)
self.assertEqual(results[2]['content.0.value'][:120], " 1. **Introduction**\n\nIt seems that Google Chrome extensions have become quite the tool for banking\nmalware fraudsters.")
def test_import_rss_public(self):
results = SyndicationModularInput.get_feed("http://answers.splunk.com/feed/questions.rss")
self.assertGreaterEqual(len(results), 10)
def test_import_return_latest_date(self):
results, latest_date = SyndicationModularInput.get_feed("http://127.0.0.1:8888/atom_example.xml", return_latest_date=True)
self.assertGreaterEqual(len(results), 0)
self.assertIsNotNone(latest_date)
def test_import_atom(self):
results = SyndicationModularInput.get_feed("http://127.0.0.1:8888/atom_example.xml")
self.assertGreaterEqual(len(results), 1)
def test_import_atom_public(self):
results = SyndicationModularInput.get_feed("http://currentworldnews.blogspot.com/atom.xml")
self.assertGreaterEqual(len(results), 10)