def run_river_spider(startDate, endDate, **kwargs): water_crawlerProcess = CrawlerProcess(settings) water_crawlerProcess.install() water_crawlerProcess.configure() spider = RiverSpider(sys.argv[1], sys.argv[2]) water_crawlerProcess.crawl(spider) try: water_crawlerProcess.start() water_crawlerProcess.stop() water_crawlerProcess.uninstall() except Exception as e: print e
def run_water_spider(startDate, endDate, **kwargs): water_crawlerProcess = CrawlerProcess(settings) water_crawlerProcess.install() water_crawlerProcess.configure() spider = WaterSpider("8735180", sys.argv[1], sys.argv[2]) water_crawlerProcess.crawl(spider) try: water_crawlerProcess.start() water_crawlerProcess.stop() water_crawlerProcess.uninstall() except Exception as e: print e
def run_wind_spider(startDate, endDate, **kwargs): wind_crawlerProcess = CrawlerProcess(settings) wind_crawlerProcess.install() wind_crawlerProcess.configure() spider2 = WindSpider("dpia1", sys.argv[1], sys.argv[2]) wind_crawlerProcess.crawl(spider2) try: wind_crawlerProcess.start() wind_crawlerProcess.stop() wind_crawlerProcess.uninstall() except Exception as e: print e
def runscrapy(stationID, startDate, endDate, **kwargs): crawlerProcess = CrawlerProcess(settings) crawlerProcess.install() crawlerProcess.configure() spider = Spider(stationID, startDate, endDate) crawlerProcess.crawl(spider) try: crawlerProcess.start() crawlerProcess.stop() crawlerProcess.uninstall() except Exception as e: print e