class CoordinatesURLTranslatorTest(unittest.TestCase):
def setUp(self):
self.translator = CoordinatesURLTranslator()
def tearDown(self):
pass
@patch("__builtin__.open")
def testGetURLsFromCoordinatesCSV(self, mockOpen):
mockOpen.return_value = io.BytesIO(b"4.681951,-74.050877\n4.680840,-74.046985")
urls = self.translator.getURLsFromCoordinatesCSV("coordinates.csv")
expectedURLs = ["http://www.domiciliosbogota.com/buscar?lat=4.681951&lng=-74.050877",
"http://www.domiciliosbogota.com/buscar?lat=4.680840&lng=-74.046985"]
self.assertEquals(urls, expectedURLs)
def testGetLatitudeFromUrl(self):
url = "http://www.domiciliosbogota.com/buscar?lat=4.681951&lng=-74.050877"
latitude = self.translator.getLatitude(url)
expectedLatitude = 4.681951
self.assertEquals(latitude, expectedLatitude)
def testGetLongitudeFromUrl(self):
url = "http://www.domiciliosbogota.com/buscar?lat=4.681951&lng=-74.050877"
longitude = self.translator.getLongitude(url)
expectedLongitude = -74.050877
self.assertEquals(longitude, expectedLongitude)
class LocationSpider(scrapy.Spider):
name = "LocationSpider"
allowed_domains = ["domiciliosbogota.com"]
coordinatesURLTranslator = CoordinatesURLTranslator()
start_urls = coordinatesURLTranslator.getURLsFromCoordinatesCSV("coordinates.csv")
def parse(self, response):
sel = Selector(response)
locations = Locations()
locations["restaurantIDs"] = sel.xpath('//a/@data-id').extract()
locations["coordinates"] = {}
locations["coordinates"]["longitude"] = self.coordinatesURLTranslator.getLongitude(response.url)
locations["coordinates"]["latitude"] = self.coordinatesURLTranslator.getLatitude(response.url)
return locations
def setUp(self):
self.translator = CoordinatesURLTranslator()