def main():
d = date.today()
food = build_feed('food', UrlProtocol(FoodFile()), FoodParser())
who = build_feed('who', UrlProtocol(WhoFiles()), WhoParser())
results = go('health', [who, food], start=date(2011,1,1), end=d)
for feed, feed_result in results.iteritems():
print 'feed={0}, count={1}'.format(feed, feed_result['count'])
for item in feed_result['obj'].updater.data_items:
print item
def test_go_no_results(self):
from pull import build_feed, SkipProtocol, go, ErrorForAllFeeds
import datetime
def build_files(*args):
return []
criteria = MyFiles(builder=build_files)
feed = build_feed('test', SkipProtocol(criteria))
d = datetime.date.today()
self.assertRaises(ErrorForAllFeeds,
lambda: go('test_feed', [feed], d))
def main():
d = date.today()
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(name)s %(message)s')
food = build_feed('food', UrlProtocol(FoodFile(), httpDebugLevel=9), FoodParser())
who = build_feed('who', UrlProtocol(WhoFiles()), WhoParser())
who.get_logger().setLevel(logging.DEBUG)
results = go('health', [who, food], start=date(2011,1,1), end=d)
for feed, feed_result in results.iteritems():
print 'feed={0}, count={1}'.format(feed, feed_result['count'])
for item in feed_result['obj'].updater.data_items:
print item
def test_go(self):
from pull import build_feed, SkipProtocol, go, ErrorForAllFeeds
import datetime
def build_files(*args):
return ['x', 'y']
criteria = MyFiles(builder=build_files)
feed = build_feed('test', SkipProtocol(criteria))
d = datetime.date.today()
results = go('test_feed', [feed], d)
self.assert_(results['test']['count']==2, 'should have 2 results')
self.assertEqual(results['test']['obj'].updater.data_items,
['x', 'y'], 'updater should have stored what was input')
def test_go_fetch_error(self):
def build_files(cache_location, start, end):
return [('you_wont_find_this', 'x')]
criteria = MyFiles(builder=build_files)
import datetime
from pull import UrlProtocol, build_feed, go, ErrorForAllFeeds
protocol = UrlProtocol(criteria)
feed = build_feed('test', protocol)
from pull import ErrorForAllRequests
d = datetime.date.today()
self.assertRaises(ErrorForAllFeeds, lambda: go('test_feed', [feed], d))
def test_go_with_parse(self):
from pull import build_feed, SkipProtocol, go
import datetime
def build_files(*args):
return ['x', 'y']
from pull import Parser
class MyParser(Parser):
def parse(self, file_path):
return ['x1', 'y1']
criteria = MyFiles(builder=build_files)
feed = build_feed('test', SkipProtocol(criteria), parser=MyParser())
d = datetime.date.today()
results = go('test_feed', [feed], d)
self.assert_(results['test']['count']==2, 'should have 2 results')
self.assertEqual(results['test']['obj'].updater.data_items,
['x1', 'y1'], 'updater should have stored what was input')
def test_go_some_feeds_failed(self):
def build_bad_files(*args):
return [('you_wont_find_this', 'x')]
def build_ok_files(*args):
return [
("http://www.who.int/csr/don/archive/year/2012/en/index.html",
'/tmp/delme')]
import datetime
from pull import UrlProtocol, build_feed, go, ErrorForSomeFeeds
badfeed = build_feed('badfeed',
UrlProtocol(MyFiles(builder=build_bad_files)))
okfeed = build_feed('okfeed',
UrlProtocol(MyFiles(builder=build_ok_files)))
d = datetime.date.today()
self.assertRaises(ErrorForSomeFeeds, lambda: go('test_feed',
[badfeed, okfeed], d))