def run_river_spider(startDate, endDate, **kwargs):
water_crawlerProcess = CrawlerProcess(settings)
water_crawlerProcess.install()
water_crawlerProcess.configure()
spider = RiverSpider(sys.argv[1], sys.argv[2])
water_crawlerProcess.crawl(spider)
try:
water_crawlerProcess.start()
water_crawlerProcess.stop()
water_crawlerProcess.uninstall()
except Exception as e:
print e
def run_water_spider(startDate, endDate, **kwargs):
water_crawlerProcess = CrawlerProcess(settings)
water_crawlerProcess.install()
water_crawlerProcess.configure()
spider = WaterSpider("8735180", sys.argv[1], sys.argv[2])
water_crawlerProcess.crawl(spider)
try:
water_crawlerProcess.start()
water_crawlerProcess.stop()
water_crawlerProcess.uninstall()
except Exception as e:
print e
def run_wind_spider(startDate, endDate, **kwargs):
wind_crawlerProcess = CrawlerProcess(settings)
wind_crawlerProcess.install()
wind_crawlerProcess.configure()
spider2 = WindSpider("dpia1", sys.argv[1], sys.argv[2])
wind_crawlerProcess.crawl(spider2)
try:
wind_crawlerProcess.start()
wind_crawlerProcess.stop()
wind_crawlerProcess.uninstall()
except Exception as e:
print e
def run_wind_spider(startDate, endDate, **kwargs):
wind_crawlerProcess = CrawlerProcess(settings)
wind_crawlerProcess.install()
wind_crawlerProcess.configure()
spider2 = WindSpider("dpia1", sys.argv[1], sys.argv[2])
wind_crawlerProcess.crawl(spider2)
try:
wind_crawlerProcess.start()
wind_crawlerProcess.stop()
wind_crawlerProcess.uninstall()
except Exception as e:
print e
def runscrapy(stationID, startDate, endDate, **kwargs):
crawlerProcess = CrawlerProcess(settings)
crawlerProcess.install()
crawlerProcess.configure()
spider = Spider(stationID, startDate, endDate)
crawlerProcess.crawl(spider)
try:
crawlerProcess.start()
crawlerProcess.stop()
crawlerProcess.uninstall()
except Exception as e:
print e
def runscrapy(stationID, startDate, endDate, **kwargs):
crawlerProcess = CrawlerProcess(settings)
crawlerProcess.install()
crawlerProcess.configure()
spider = Spider(stationID, startDate, endDate)
crawlerProcess.crawl(spider)
try:
crawlerProcess.start()
crawlerProcess.stop()
crawlerProcess.uninstall()
except Exception as e:
print e
