Beispiel #1
0
    def handle(self, *args, **options):
        HERE = os.getcwd()
        print "Working directory is", HERE

        print "Adding latest events..."
        from ebdata.scrapers.us.ma.boston.events.retrieval import main as events_main
        events_main()

        print "Adding latest news..."
        from ebdata.scrapers.general.georss.retrieval import main as news_main
        news_main([
            "http://search.boston.com/search/api?q=*&sort=-articleprintpublicationdate&subject=massachusetts&scope=bonzai"
        ])
        # more feeds from Joel. Local blog news:
        news_main([
            "http://search.boston.com/search/api?q=*&sort=-articleprintpublicationdate&scope=blogs&count=250&subject=massachusetts&format=atom"
        ])

        print "Adding police reports..."
        from ebdata.scrapers.us.ma.boston.police_reports.retrieval import main as pr_main
        pr_main()

        print " Adding building permits..."
        from ebdata.scrapers.us.ma.boston.building_permits.retrieval import PermitScraper
        PermitScraper().update()

        print "Adding GeoReport issues..."
        from ebdata.scrapers.general.open311.georeportv2 import main as georeport_main
        georeport_main([
            '--html-url-template=http://seeclickfix.com/issues/{id}',
            '--days-prior=6', 'http://seeclicktest.com/boston/open311/v2'
        ])

        # TODO: fix traceback:  ebdata.blobs.scrapers.NoSeedYet: You need to add a Seed with the URL 'http://www.cityofboston.gov/news/
        #echo Adding press releases...
        #python everyblock/everyblock/cities/boston/city_press_releases/retrieval.py || die

        print "Adding flickr photos..."
        from ebdata.scrapers.general.flickr import flickr_retrieval
        flickr_retrieval.main()

        print "Updating aggregates, see ebpub/README.txt..."
        from ebpub.db.bin.update_aggregates import update_all_aggregates
        update_all_aggregates()

        print """
    def handle(self, *args, **options):
        HERE = os.getcwd()
        print "Working directory is", HERE

        print "Adding latest events..."
        from ebdata.scrapers.us.ma.boston.events.retrieval import main as events_main
        events_main()

        print "Adding latest news..."
        from ebdata.scrapers.general.georss.retrieval import main as news_main
        news_main(["http://search.boston.com/search/api?q=*&sort=-articleprintpublicationdate&subject=massachusetts&scope=bonzai"])
        # more feeds from Joel. Local blog news:
        # ... this one times out
        #news_main(["http://search.boston.com/search/api?q=*&sort=-articleprintpublicationdate&scope=blogs&count=250&subject=massachusetts&format=atom"])


        print "Adding police reports..."
        from ebdata.scrapers.us.ma.boston.police_reports.retrieval import main as pr_main
        pr_main()

        print " Adding building permits..."
        from ebdata.scrapers.us.ma.boston.building_permits.retrieval import PermitScraper
        PermitScraper().update()


        print "Adding GeoReport issues..."
        from ebdata.scrapers.general.open311.georeportv2 import main as georeport_main
        georeport_main(['--html-url-template=http://seeclickfix.com/issues/{id}',
                        '--days-prior=6',
                        'http://seeclicktest.com/boston/open311/v2'])


        # TODO: fix traceback:  ebdata.blobs.scrapers.NoSeedYet: You need to add a Seed with the URL 'http://www.cityofboston.gov/news/
        #echo Adding press releases...
        #python everyblock/everyblock/cities/boston/city_press_releases/retrieval.py || die

        print "Adding flickr photos..."
        from ebdata.scrapers.general.flickr import flickr_retrieval
        flickr_retrieval.main(['--days=3'])

        print "Updating aggregates, see ebpub/README.txt..."
        from ebpub.db.bin.update_aggregates import update_all_aggregates
        update_all_aggregates()

        print """
def do_events(**kwargs):
    from obdemo.scrapers.add_events import main
    return main()
def do_news(**kwargs):
    from obdemo.scrapers.add_news import main
    main(["http://search.boston.com/search/api?q=*&sort=-articleprintpublicationdate&subject=massachusetts&scope=bonzai&count=400"])
    main(["http://search.boston.com/search/api?q=*&sort=-articleprintpublicationdate&scope=blogs&count=400&subject=massachusetts&format=atom"])
def do_flickr(**kwargs):
    from ebdata.scrapers.general.flickr.flickr_retrieval import main
    return main()