def parse_market(self, response):
''' Parse a market '''
# load Maize Vendors.
maizeresp = urlopen('http://yumbli.herokuapp.com/api/v1/allkitchens/?format=json')
vendors = json.loads(maizeresp.read().decode('utf8'))
maizevendors = {}
for v in vendors:
maizevendors[v['name'].lower()] = v['id']
item = StreetFoodDatTimeItem()
market = json.loads(response.text)
market_detail = market["MarketDetail"]["Market"]["Market"]
market_events = market["MarketDetail"]["Events"]
# Market Address.
market_address = market_detail["address"].strip()
market_city = market_detail["city"].strip()
full_address = "{} {}".format(market_address, market_city)
# Market location.
market_latitude = market_detail['latitude']
market_longitude = market_detail['longitude']
# geolocation = "{} {}".format(market_latitude, market_longitude)
# Add data to item.
item['address'] = full_address
# Parse market events.
for event in market_events:
start_datetime, end_datetime = tools.get_start_end_datetime(event['Event'])
item['start_datetime'] = start_datetime
item['end_datetime'] = end_datetime
# Parse vendors of event.
for vendor in event['Vendors']:
vendor_name = vendor['name']
item['VendorName'] = vendor_name
# randlongpos = random.randint(-150, 150) / 1000000
# randlatpos = random.randint(-200, 200) / 1000000
# item['latitude'] = abs(float(market_latitude)) + randlatpos
# abs then *-1 b/c off the grid has some wrong values
# item['longitude'] = abs(float(market_longitude))*-1 + randlongpos
item['latitude'] = basic_tools.mix_location(market_latitude)
item['longitude'] = basic_tools.mix_location(market_longitude)
if vendor_name and vendor_name.lower() in maizevendors.keys():
item['maize_status'] = 'found'
item['maize_id'] = maizevendors[vendor_name.lower()]
else:
item['maize_status'] = 'not found'
item['maize_id'] = 'n/a'
yield item
def parse_dinner_vendors(self, response):
# ---- Dinner time vendors ---- #
address = tools.get_address(response)
vendors_xp = '//h2[contains(text(), "Tonight\'s Dinner Time Vendors")]'
vendors = response.xpath(vendors_xp)
start_dtime, end_dtime = tools.get_time(response, dinner=True)
# Vendors list block.
vblock_xp = './parent::div/parent::div/following-sibling::div'
vendors_block = vendors.xpath(vblock_xp)[0]
vlist_xp = ".//div[contains(@class, 'summary-item\n')]"
for vendor in vendors_block.xpath(vlist_xp):
item = StreetFoodDatTimeItem()
vendor_name = tools.get_vendor_name(vendor)
item['VendorName'] = vendor_name
item['address'] = address
item['latitude'] = basic_tools.mix_location('37.769782')
item['longitude'] = basic_tools.mix_location('-122.411848')
item['start_datetime'] = start_dtime
item['end_datetime'] = end_dtime
item['maize_id'] = basic_tools.maize_api_search(
self.maize_vendors, vendor_name)
yield item
def parse_dinner_vendors(self, response):
# ---- Dinner time vendors ---- #
address = tools.get_address(response)
vendors_xp = '//h2[contains(text(), "Tonight\'s Dinner Time Vendors")]'
vendors = response.xpath(vendors_xp)
start_dtime, end_dtime = tools.get_time(response, dinner=True)
# Vendors list block.
vblock_xp = './parent::div/parent::div/following-sibling::div'
vendors_block = vendors.xpath(vblock_xp)[0]
vlist_xp = ".//div[contains(@class, 'summary-item\n')]"
for vendor in vendors_block.xpath(vlist_xp):
item = StreetFoodDatTimeItem()
vendor_name = tools.get_vendor_name(vendor)
item['VendorName'] = vendor_name
item['address'] = address
item['latitude'] = basic_tools.mix_location('37.769782')
item['longitude'] = basic_tools.mix_location('-122.411848')
item['start_datetime'] = start_dtime
item['end_datetime'] = end_dtime
item['maize_id'] = basic_tools.maize_api_search(self.maize_vendors,
vendor_name)
yield item
def make_item(self, vendor_name):
item = StreetFoodDatTimeItem()
item['VendorName'] = vendor_name
item['address'] = "425 2nd street San Francisco, CA"
item['latitude'] = basic_tools.mix_location('37.783711')
item['longitude'] = basic_tools.mix_location('-122.394375')
item['start_datetime'] = gf_start_time()
item['end_datetime'] = gf_end_time()
item['maize_id'] = basic_tools.maize_api_search(self.maize_vendors,
vendor_name)
return item
def make_item(self, vendor_name):
item = StreetFoodDatTimeItem()
item['VendorName'] = vendor_name
item['address'] = "425 2nd street San Francisco, CA"
item['latitude'] = basic_tools.mix_location('37.783711')
item['longitude'] = basic_tools.mix_location('-122.394375')
item['start_datetime'] = gf_start_time()
item['end_datetime'] = gf_end_time()
item['maize_id'] = basic_tools.maize_api_search(
self.maize_vendors, vendor_name)
return item
def make_item(self, vendor_name, vendor_date):
start_time = vendor_date.replace(hour=11)
end_time = vendor_date.replace(hour=14)
item = StreetFoodDatTimeItem()
item['VendorName'] = vendor_name
item['address'] = "450 Mission St San Francisco, CA"
item['latitude'] = basic_tools.mix_location('37.79021200')
item['longitude'] = basic_tools.mix_location('-122.39725000')
item['start_datetime'] = str(start_time)
item['end_datetime'] = str(end_time)
item['maize_id'] = basic_tools.maize_api_search(
self.maize_vendors, vendor_name)
return item
def make_item(self, vendor_name, vendor_date):
start_time = vendor_date.replace(hour=11)
end_time = vendor_date.replace(hour=14)
item = StreetFoodDatTimeItem()
item['VendorName'] = vendor_name
item['address'] = "450 Mission St San Francisco, CA"
item['latitude'] = basic_tools.mix_location('37.79021200')
item['longitude'] = basic_tools.mix_location('-122.39725000')
item['start_datetime'] = str(start_time)
item['end_datetime'] = str(end_time)
item['maize_id'] = basic_tools.maize_api_search(self.maize_vendors,
vendor_name)
return item
def make_item(self, vendor_name, last_event):
address = last_event.get("place").get("name")
latitude, longitude = get_geolocation(address,
self.here_app_id,
self.here_app_code)
item = StreetFoodDatTimeItem()
item['VendorName'] = vendor_name
item['address'] = address
item['latitude'] = basic_tools.mix_location(latitude)
item['longitude'] = basic_tools.mix_location(longitude)
item['start_datetime'] = last_event.get('start_time')
item['end_datetime'] = last_event.get('end_time')
item['maize_id'] = basic_tools.maize_api_search(self.maize_vendors,
vendor_name)
return item
def make_item(self, vendor_name, last_event):
event_location = last_event.get("place").get('location', {})
address = last_event.get("place").get("name")
if 'longitude' in event_location and 'latitude' in event_location:
latitude = event_location['latitude']
longitude = event_location['longitude']
else:
latitude, longitude = get_geolocation(address, self.here_app_id,
self.here_app_code)
item = StreetFoodDatTimeItem()
item['VendorName'] = vendor_name
item['address'] = address
item['latitude'] = basic_tools.mix_location(latitude)
item['longitude'] = basic_tools.mix_location(longitude)
item['start_datetime'] = last_event.get('start_time')
item['end_datetime'] = last_event.get('end_time')
item['maize_id'] = basic_tools.maize_api_search(
self.maize_vendors, vendor_name)
return item
def parse_dinner_vendors(self, response):
address = tools.get_address(response)
start_dtime, end_dtime = tools.get_time(response, dinner=True)
vendors_node_xp = "//h2[contains(text(), 'Dinner Shift')]/\
parent::div/parent::div/following-sibling::div"
vendors = response.xpath(vendors_node_xp)[0]
for node in vendors.xpath(".//a[@class='summary-title-link']/text()"):
item = StreetFoodDatTimeItem()
vendor_name = node.extract()
item['VendorName'] = node.extract()
item['address'] = address
item['latitude'] = basic_tools.mix_location('37.770775')
item['longitude'] = basic_tools.mix_location('-122.391588')
item['start_datetime'] = start_dtime
item['end_datetime'] = end_dtime
item['maize_id'] = basic_tools.maize_api_search(
self.maize_vendors, vendor_name)
yield item
def parse_dinner_vendors(self, response):
address = tools.get_address(response)
start_dtime, end_dtime = tools.get_time(response, dinner=True)
vendors_node_xp = "//h2[contains(text(), 'Dinner Shift')]/\
parent::div/parent::div/following-sibling::div"
vendors = response.xpath(vendors_node_xp)[0]
for node in vendors.xpath(".//a[@class='summary-title-link']/text()"):
item = StreetFoodDatTimeItem()
vendor_name = node.extract()
item['VendorName'] = node.extract()
item['address'] = address
item['latitude'] = basic_tools.mix_location('37.770775')
item['longitude'] = basic_tools.mix_location('-122.391588')
item['start_datetime'] = start_dtime
item['end_datetime'] = end_dtime
item['maize_id'] = basic_tools.maize_api_search(self.maize_vendors,
vendor_name)
yield item
def parse_event(self, response):
# Load event data from response
event = json.loads(response.text)
event_detail = event['data']['event']
event_vendors = event_detail['services']
# load Maize Vendors.
maizeresp = urlopen(
'http://yumbli.herokuapp.com/api/v1/allkitchens/?format=json')
vendors = json.loads(maizeresp.read().decode('utf8'))
maizevendors = {}
for v in vendors:
maizevendors[v['name'].lower()] = v['id']
# Market details
market_detail = response.meta['market_detail']
if market_detail["address1"] is None:
market_detail["address1"] = ""
if market_detail["address2"] is None:
market_detail["address2"] = ""
# Build market full address
market_address = market_detail["address1"].strip(
) + market_detail["address2"].strip()
market_city = market_detail["city"].strip()
full_address = "{} {}".format(market_address, market_city)
# Market location.
market_latitude = market_detail['latitude']
market_longitude = market_detail['longitude']
# geolocation = "{} {}".format(market_latitude, market_longitude)
# Add data to item.
item = StreetFoodDatTimeItem()
item['address'] = full_address
# start_datetime, end_datetime = tools.get_start_end_datetime(event['Event'])
start_datetime = event_detail['startTime']
end_datetime = event_detail['endTime']
item['start_datetime'] = start_datetime
item['end_datetime'] = end_datetime
# Parse vendors of event.
for vendor in event_vendors:
vendor_name = vendor['vendor']['name']
item['VendorName'] = vendor_name
# randlongpos = random.randint(-150, 150) / 1000000
# randlatpos = random.randint(-200, 200) / 1000000
# item['latitude'] = abs(float(market_latitude)) + randlatpos
# abs then *-1 b/c off the grid has some wrong values
# item['longitude'] = abs(float(market_longitude))*-1 + randlongpos
item['latitude'] = basic_tools.mix_location(market_latitude)
item['longitude'] = basic_tools.mix_location(market_longitude)
if vendor_name and vendor_name.lower() in maizevendors.keys():
item['maize_status'] = 'found'
item['maize_id'] = maizevendors[vendor_name.lower()]
else:
item['maize_status'] = 'not found'
item['maize_id'] = 'n/a'
yield item
