Пример #1
0
def main():    
    d = date.today()    
    food = build_feed('food', UrlProtocol(FoodFile()), FoodParser())
    who = build_feed('who', UrlProtocol(WhoFiles()), WhoParser())
    results = go('health', [who, food], start=date(2011,1,1), end=d)
    for feed, feed_result in results.iteritems():
        print 'feed={0}, count={1}'.format(feed, feed_result['count'])
        for item in feed_result['obj'].updater.data_items:
            print item 
Пример #2
0
def main():    
    d = date.today()    
    logging.basicConfig(level=logging.DEBUG,
                        format='%(asctime)s %(levelname)s %(name)s %(message)s')
    food = build_feed('food', UrlProtocol(FoodFile(), httpDebugLevel=9), FoodParser())
    who = build_feed('who', UrlProtocol(WhoFiles()), WhoParser())
    who.get_logger().setLevel(logging.DEBUG)
    results = go('health', [who, food], start=date(2011,1,1), end=d)
    for feed, feed_result in results.iteritems():
        print 'feed={0}, count={1}'.format(feed, feed_result['count'])
        for item in feed_result['obj'].updater.data_items:
            print item 
Пример #3
0
 def test_build(self):
     from pull import build_feed, SkipProtocol
     def build_files():
         return []
     criteria = MyFiles(builder=build_files)
     feed = build_feed('test', SkipProtocol(criteria)) 
     self.assert_(feed.name=='test', 'name not set')      
Пример #4
0
 def test_go_some_feeds_failed(self):
     def build_bad_files(*args):
         return [('you_wont_find_this', 'x')]
     
     def build_ok_files(*args):
         return [
             ("http://www.who.int/csr/don/archive/year/2012/en/index.html",
             '/tmp/delme')]     
         
     import datetime
     from pull import UrlProtocol, build_feed, go, ErrorForSomeFeeds
     badfeed = build_feed('badfeed',
                          UrlProtocol(MyFiles(builder=build_bad_files)))
     okfeed = build_feed('okfeed',
                         UrlProtocol(MyFiles(builder=build_ok_files)))
   
     d = datetime.date.today()
     self.assertRaises(ErrorForSomeFeeds, lambda: go('test_feed',
         [badfeed, okfeed], d))    
Пример #5
0
 def test_go_no_results(self):
     from pull import build_feed, SkipProtocol, go, ErrorForAllFeeds
     import datetime
     def build_files(*args):
         return []
     criteria = MyFiles(builder=build_files)
     feed = build_feed('test', SkipProtocol(criteria)) 
     d = datetime.date.today()
     
     self.assertRaises(ErrorForAllFeeds,
                       lambda: go('test_feed', [feed], d))
Пример #6
0
 def test_go(self):
     from pull import build_feed, SkipProtocol, go, ErrorForAllFeeds
     import datetime
     def build_files(*args):
         return ['x', 'y']
     criteria = MyFiles(builder=build_files)
     feed = build_feed('test', SkipProtocol(criteria)) 
     d = datetime.date.today()
     results = go('test_feed', [feed], d)
     self.assert_(results['test']['count']==2, 'should have 2 results')
     self.assertEqual(results['test']['obj'].updater.data_items,
        ['x', 'y'], 'updater should have stored what was input')              
Пример #7
0
 def test_go_fetch_error(self):
     def build_files(cache_location, start, end):
         return [('you_wont_find_this', 'x')]     
         
     criteria = MyFiles(builder=build_files) 
     import datetime
     from pull import UrlProtocol, build_feed, go, ErrorForAllFeeds
     protocol = UrlProtocol(criteria)
     feed = build_feed('test', protocol)
     from pull import ErrorForAllRequests
     d = datetime.date.today()
     self.assertRaises(ErrorForAllFeeds, lambda: go('test_feed', [feed], d))
Пример #8
0
 def test_go_with_parse(self):
     from pull import build_feed, SkipProtocol, go
     import datetime
     def build_files(*args):
         return ['x', 'y']
     
     from pull import Parser
     class MyParser(Parser):
         def parse(self, file_path):
             return ['x1', 'y1']
 
     criteria = MyFiles(builder=build_files)
     feed = build_feed('test', SkipProtocol(criteria), parser=MyParser()) 
     d = datetime.date.today()
     results = go('test_feed', [feed], d)
     self.assert_(results['test']['count']==2, 'should have 2 results')
     self.assertEqual(results['test']['obj'].updater.data_items,
        ['x1', 'y1'], 'updater should have stored what was input')