def get_details(self, url=None):
try:
# time.sleep(5)
self.driver.get(self.url)
FIND_BY = self.dish_config['SELECTORS']['DISHESH']['FIND_BY']
VALUE = self.dish_config['SELECTORS']['DISHESH']['VALUE']
element = WebDriverWait(self.driver, 10).until(
EC.presence_of_element_located((By.CLASS_NAME, VALUE)))
if FIND_BY == 'class':
dishes = self.driver.find_elements_by_class_name(VALUE)
elif FIND_BY == 'id':
dishes = self.driver.find_elements_by_id(VALUE)
restaurant_obj = {
'city_code': self.city_code,
'name': "Masala Box",
'type': 'North Indian, South Indian',
'stars': 4.1,
'ratings': '100+ ratings',
'image':
'https://www.google.com/url?sa=i&source=images&cd=&ved=2ahUKEwju_pWC2N_mAhVOzzgGHe4dBvIQjRx6BAgBEAQ&url=https%3A%2F%2Fwww.masalabox.com%2F&psig=AOvVaw1h-_mKB5oipFia4-jSyvIU&ust=1577874845479690',
'opens_at': None,
'country': self.country,
'subzone': 'General',
'city': self.city,
'platform': 'MASALABOX',
'dishes': [],
'added_on': str(datetime.datetime.utcnow())
}
self.restaurant_obj = restaurant_obj
dish_obj = Dish(self.dish_config, self.restaurant_obj)
dish_obj.get_dishes(dishes)
print("OBJJJJJJJJJJJJJ++++", len(self.restaurant_obj['dishes']))
sort_key_info = restaurant_obj['platform'] + '__' + restaurant_obj[
'subzone'] + '__' + restaurant_obj['name'].strip().replace(
' ', '_')
restaurant_obj['sort_key_info'] = sort_key_info
restaurant_obj['stars'] = Decimal(str(
self.restaurant_obj['stars']))
# write to dynamodb
self.dynamodb_write_obj = DynamoDBWrite()
self.dynamodb_write_obj.dynamodb_write(self.restaurant_obj)
# write data to parquet in s3
restaurant_obj['stars'] = float(self.restaurant_obj['stars'])
self.write_to_s3_parquet_obj = WriteS3Parquet(self.city)
self.write_to_s3_parquet_obj.write_to_parquet(self.restaurant_obj)
except Exception as e:
print('++++++++NOT DONE', e)
self.driver.close()
def extract_dishes(self):
'''Extract all dishes from file and store to an
array as Dish objects'''
ArrayType = ctypes.py_object * (self._data['to'] - self._data['from'])
dishes_array = ArrayType()
print('lenghfgrawuiygfilu: ',len(self._data['hits']))
for i in range(len(self._data['hits'])):
recipe = self._data['hits'][i]['recipe']
name = recipe['label']
url = recipe['url']
image_url = recipe['image']
portions = int(recipe['yield'])
calories = recipe['calories']
weight = recipe['totalWeight']
ingredients = dict()
for ingredient in recipe['ingredients']:
ingredients[ingredient['text']] = ingredient['weight']
nutrients = recipe['totalNutrients']
daily_info = recipe['totalDaily']
diet_labels = recipe['dietLabels']
health_labels = recipe['healthLabels']
dishes_array[i] = Dish(name, url, image_url, portions, calories,
weight, ingredients, nutrients, daily_info,
diet_labels, health_labels)
return dishes_array
def __post_init__(self):
if self.base is None:
self.base = [18.16, 18.16]
if self.top_delta is None:
self.top_delta = [6, 4]
if self.size is None:
self.size = [1, 1]
if self.skew is None:
self.skew = [0, 0]
if self.tilt is None:
self.tilt = [0, 0]
self.dims = [self.base[i] * self.size[i] for i in [0, 1]]
self.top_dims = [self.dims[i] - self.top_delta[i] for i in [0, 1]]
dish_kwargs = {
'dish_type': self.dish_type,
'top_dims': self.top_dims,
'key_z': self.z
}
if self.dish_kwargs is not None:
dish_kwargs.update(self.dish_kwargs)
self.dish = Dish(**dish_kwargs)
# masks used to exclude anything not inside/outside the key
self.inside = s.difference()(s.translate([0, 0,
50])(s.cube([100, 100, 100],
center=True)),
self._shape(self.thickness, self.depth))
self.outside = s.difference()(s.cube([1000, 1000, 1000], center=True),
self.hull(self.thickness,
self.depth,
mod=2))
def __init__(self):
self.CONFIG_FILE = "jwt-config.json"
with open(self.CONFIG_FILE, 'r') as config_file:
self.data_set = json.load(config_file)
self.starter_config = self.data_set['SWIGGY']['CONFIG']['STARTER']
self.config = self.data_set['SWIGGY']['CONFIG']
self.dishes_data = []
self.restaurants_data = []
self.restaurants_obj = Restaurant(self.config)
self.city = os.getenv('CITY', "jaipur")
self.country = os.getenv('COUNTRY', "india")
self.city_code = self.city + '__' + self.country
self.dish_obj = Dish(self.config, self.city, self.country,
self.city_code)
def __init__(self, data=[], dishlistfile='tasteinp.json', history=20):
# self.flavourlist = ["salt", "sweet", "rich"]
self.segpercs = [0.5, 0.3, 0.2]
self.history = history
self.items = list()
if len(data) == 0:
with open(dishlistfile) as ifile:
for item in json.load(ifile):
self.items.append(Dish(item))
else:
for item in data:
self.items.append(Dish(item))
# Uncomment for testing
# self.historydata = random.sample(self.items, k=history)
self.flavourlist = self.items[0].tastedata.keys()
self.historydata = copy.deepcopy(self.items)
self.init_profile(self.historydata)
def scrape(self):
response = requests.get(self.link)
tree = html.fromstring(response.content)
image_url = tree.xpath('//figure/img/@src')
if not image_url:
return
self.link = image_url[0]
self.dish_array.append(
Dish('Click the restaurant link to check the weekly menu', ''))
def test_config():
d = Dish()
assert d.config == DEFAULT_CONFIG
d.config_from_dict({'PS1': 'uwu$ '})
assert d.config == {**DEFAULT_CONFIG, 'PS1': 'uwu$ '}
d.reset_config()
assert d.config == DEFAULT_CONFIG
def seed_dishes():
"""Seed dishes table with common dish names from Dianping and others."""
with open('seeds/dishes_seed.txt', 'rb') as f:
reader = csv.reader(f, delimiter='|')
for row in reader:
for i in range(len(row)):
row[i] = row[i].decode('utf-8')
row[i] = row[i].strip()
chin_name = row[0]
eng_name = row[1].lower()
if len(row) > 2:
desc = row[2]
dish = Dish(chin_name=chin_name, eng_name=eng_name, desc=desc)
else:
dish = Dish(chin_name=chin_name, eng_name=eng_name)
db_session.add(dish)
try:
db_session.commit()
except sqlalchemy.exc.IntegrityError, e:
db_session.rollback()
def __init__(self):
self.CONFIG_FILE = "jwt-config.json"
with open(self.CONFIG_FILE, 'r') as config_file:
self.data_set = json.load(config_file)
self.starter_config = self.data_set['ZOMATO']['CONFIG']['STARTER']
self.config = self.data_set['ZOMATO']['CONFIG']
self.dishes_data = []
self.restaurants_data = []
self.restaurants_obj = Restaurant(self.config)
self.city = os.getenv('CITY', "jaipur")
self.country = os.getenv('COUNTRY', "india")
self.city_code = self.city + '__' + self.country
self.dish_obj = Dish(self.config, self.city, self.country,
self.city_code)
self.options = Options()
self.options.headless = True
self.driver = webdriver.Chrome(options=self.options)
def test_prompt_generation():
d = Dish()
def uwu_start_tag(attrs):
result = ''
for key, val in attrs.items():
result += f'{key}:{val};'
return result
def uwu_end_tag():
return '[end]'
d.start_tag_handlers['uwu'] = uwu_start_tag
d.end_tag_handlers['uwu'] = uwu_end_tag
d.config['PS1'] = '<uwu one="two" three="four">owo</uwu>'
prompt = d.generate_prompt('PS1')
assert prompt == 'one:two;three:four;owo[end]'
from dish import Dish
dishes_string = input('What would you like? (comma required)\n')
dishes_list = dishes_string.title().split(',')
stripped_dishes_list = [i.strip() for i in dishes_list]
unique_dishes_list = list(set(stripped_dishes_list))
for unique_dishes in unique_dishes_list:
if unique_dishes != '':
print(Dish.to_print(unique_dishes))
def get_details(self):
try:
self.driver.get(self.url)
FIND_BY = self.starter_config['SELECTORS']['WAIT']['FIND_BY']
VALUE = self.starter_config['SELECTORS']['WAIT']['VALUE']
if FIND_BY == 'class':
element = WebDriverWait(self.driver, 10).until(
EC.visibility_of_element_located((By.CLASS_NAME, VALUE))
)
elif FIND_BY == 'css':
element = WebDriverWait(self.driver, 10).until(
EC.visibility_of_element_located((By.CSS_SELECTOR, VALUE))
)
elif FIND_BY == 'id':
element = WebDriverWait(self.driver, 10).until(
EC.visibility_of_element_located((By.ID, VALUE))
)
elif FIND_BY == 'tag':
element = WebDriverWait(self.driver, 10).until(
EC.visibility_of_element_located((By.TAG_NAME, VALUE))
)
FIND_BY = self.starter_config['SELECTORS']['ALL_REST']['FIND_BY']
VALUE = self.starter_config['SELECTORS']['ALL_REST']['VALUE']
if FIND_BY == 'css':
all_rest = self.driver.find_elements_by_css_selector(VALUE)
elif FIND_BY == 'class':
all_rest = self.driver.find_elements_by_class_name(VALUE)
RESTAURANT_TYPE_FIND_BY = self.restaurant_config['SELECTORS']['TYPE']['FIND_BY']
RESTAURANT_TYPE_VALUE = self.restaurant_config['SELECTORS']['TYPE']['VALUE']
NAME={
'FIND_BY': self.restaurant_config['SELECTORS']['NAME']['FIND_BY'],
'VALUE': self.restaurant_config['SELECTORS']['NAME']['VALUE']
}
STARS_RATINGS={
'FIND_BY': self.restaurant_config['SELECTORS']['STARS_RATINGS']['FIND_BY'],
'VALUE': self.restaurant_config['SELECTORS']['STARS_RATINGS']['VALUE']
}
restaurants_obj = []
print('+++++LENGTH',len(all_rest))
for restaurant in all_rest:
try:
anchor_tag = restaurant.find_element_by_tag_name("a")
reatuarnat_url = anchor_tag.get_attribute("href")
try:
if RESTAURANT_TYPE_FIND_BY == 'css':
type = restaurant.find_element_by_css_selector(RESTAURANT_TYPE_VALUE).get_attribute('textContent')
elif RESTAURANT_TYPE_FIND_BY == 'class':
type = restaurant.find_element_by_class_name(RESTAURANT_TYPE_VALUE).get_attribute('textContent')
except Exception as e:
type = None
try:
if STARS_RATINGS['FIND_BY'] == 'css':
stars_ratings = restaurant.find_element_by_css_selector(STARS_RATINGS['VALUE']).get_attribute('textContent')
elif STARS_RATINGS['FIND_BY'] == 'class':
stars_ratings = restaurant.find_element_by_class_name(STARS_RATINGS['VALUE']).get_attribute('textContent')
tokens = stars_ratings.split('(')
print('+++++++tokens',tokens)
stars = tokens[0]
ratings = tokens[1].split(')')[0]
except Exception as e:
print('++++++stars exce',e)
stars = Decimal(str(4.1))
ratings = None
try:
image = restaurant.find_element_by_tag_name('img').get_attribute("src")
except Exception as e:
image = None
if NAME['FIND_BY'] == 'css':
name = restaurant.find_element_by_css_selector(NAME['VALUE']).get_attribute('textContent')
elif NAME['FIND_BY'] == 'class':
name = restaurant.find_element_by_class_name(NAME['VALUE']).get_attribute('textContent')
restaurant = {
'type': type,
'name':name,
'url':reatuarnat_url,
'stars': stars,
'ratings':ratings,
'image':image
}
restaurants_obj.append(restaurant)
# print('+++++rest added',restaurant)
except Exception as e:
print("&&&&&&&&",e)
try:
self.driver.close()
except Exception as e:
print('++++++driver closing exception',e)
self.rest_obj = Restaurant(self.dish_config,self.restaurant_config,self.dishes_data,self.city_code,self.country,self.city)
self.dish_obj = Dish(self.dish_config)
self.get_restaurant_data(restaurants_obj)
# # # write to dynamodb
self.dynamodb_write_obj = DynamoDBBatchWrite()
self.dynamodb_write_obj.batch_write_to_ddb(self.dishes_data)
# # # # write data to parquet in s3
for restaurant_obj in self.dishes_data:
restaurant_obj['stars'] = float(restaurant_obj['stars'])
# # # print('++++++dishes data',json.dumps(self.dishes_data))
self.write_to_s3_parquet_obj = WriteS3Parquet(self.city)
self.write_to_s3_parquet_obj.write_to_parquet(self.dishes_data)
except Exception as e:
print('++++++++NOT DONE',e)
class Swiggy:
def __init__(self):
self.CONFIG_FILE = "jwt-config.json"
with open(self.CONFIG_FILE, 'r') as config_file:
self.data_set = json.load(config_file)
self.starter_config = self.data_set['SWIGGY']['CONFIG']['STARTER']
self.config = self.data_set['SWIGGY']['CONFIG']
self.dishes_data = []
self.restaurants_data = []
self.restaurants_obj = Restaurant(self.config)
self.city = os.getenv('CITY', "jaipur")
self.country = os.getenv('COUNTRY', "india")
self.city_code = self.city + '__' + self.country
self.dish_obj = Dish(self.config, self.city, self.country,
self.city_code)
def get_data(self, url=None):
print('+++++++IT WILL RUN')
URL = self.starter_config["URLS"]['jaipur']
r = requests.get(URL)
soup = BeautifulSoup(r.content, 'html5lib')
TAG = self.starter_config['SELECTORS']['WAIT']['TAG']
FIND_BY = self.starter_config['SELECTORS']['WAIT']['FIND_BY']
VALUE = self.starter_config['SELECTORS']['WAIT']['VALUE']
data = soup.find_all(TAG, attrs={FIND_BY: VALUE})
_list = []
for row in data:
link = row['href']
subzone = row.text
_list.append({
'subzone': subzone,
'link': "https://www.swiggy.com" + link
})
no_of_threads = 3
last_chunk = -1
subzone_batch_threads = []
length_of_subzones = len(_list)
chunk_size = int(length_of_subzones / no_of_threads)
print('************chunk size', chunk_size)
if length_of_subzones >= no_of_threads:
for i in range(no_of_threads):
batch = _list[i * chunk_size:(i + 1) * chunk_size]
# batch = _list[0:1]
subzone_batch_threads.append(
threading.Thread(target=self.get_restaurants_thread,
args=(batch, )))
subzone_batch_threads[-1].start()
last_chunk = i
last_chunk += 1
if no_of_threads * chunk_size < length_of_subzones:
batch = _list[last_chunk * chunk_size:length_of_subzones]
# batch = _list[0:1]
subzone_batch_threads.append(
threading.Thread(target=self.get_restaurants_thread,
args=(batch, )))
subzone_batch_threads[-1].start()
for thread in subzone_batch_threads:
print('\n\n**************JOINING***********')
thread.join()
else:
for subzone in _list:
self.restaurants_obj.get_restaurants(subzone['link'],
subzone['subzone'],
self.restaurants_data)
self.get_dishes()
print('++++DISHES LEN', len(self.dishes_data))
# load data to dynamodb
self.dynamodb_batch_write_obj = DynamoDBBatchWrite()
self.dynamodb_batch_write_obj.batch_write_to_ddb(self.dishes_data)
for dish_data in self.dishes_data:
dish_data['stars'] = float(dish_data['stars'])
# write data to parquet in s3
self.write_to_s3_parquet_obj = WriteS3Parquet(self.city)
self.write_to_s3_parquet_obj.write_to_parquet(self.dishes_data)
def get_restaurants_thread(self, subzones):
print('+++++=THREAD')
for subzone in subzones:
self.restaurants_obj.get_restaurants(subzone['link'],
subzone['subzone'],
self.restaurants_data)
def get_dishes(self):
no_of_threads = 3
last_chunk = -1
restaurant_batch_threads = []
length_of_restaurants_data = len(self.restaurants_data)
chunk_size = int(length_of_restaurants_data / no_of_threads)
for i in range(no_of_threads):
batch = self.restaurants_data[i * chunk_size:(i + 1) * chunk_size]
# batch = restaurants_data[0:2]
restaurant_batch_threads.append(
threading.Thread(target=self.get_dishes_thread,
args=(batch, )))
restaurant_batch_threads[-1].start()
last_chunk = i
last_chunk += 1
if no_of_threads * chunk_size < length_of_restaurants_data:
batch = self.restaurants_data[
last_chunk * chunk_size:length_of_restaurants_data]
# batch = restaurants_data[0:2]
restaurant_batch_threads.append(
threading.Thread(target=self.get_dishes_thread,
args=(batch, )))
restaurant_batch_threads[-1].start()
for thread in restaurant_batch_threads:
thread.join()
def get_dishes_thread(self, restaurants):
for restaurant in restaurants:
self.dish_obj.get_details(restaurant, self.dishes_data)
#matches = 0
for dish in dishes:
if(all(x in supplies for x in dish.getIngredients())):
#for i in dish.getIngredients():
# if i not in supplies:
# break
# else:
# matches +=1
# if matches == len(dish.getIngredients()):
# matches = 0
cookableDishes.append(dish)
return cookableDishes
#some sample dishes to get started
dishes = [Dish('baklava',['pistachios', 'crust', 'lots of butter']), Dish('salmon', ['salmon']), Dish('omelette', ['eggs'])]
supplies = []
prompt = True
while(prompt):
response = input('Enter 1 to add a dish, 2 to see all dishes, 3 to check supplies, 4 to delete a dish, and 5 to quit \n')
if response == '1':
name = input('Enter your dish name:\n')
ingredients = []
more = True
while(more):
res = input('Enter a dish ingredient, or 0 to exit:\n')
if res != '0':
ingredients.append(res)
else:
more= False
if __name__ == '__main__':
restaurant = Restaurant('Test', 'address')
print(restaurant)
orange = Product('Orange')
strawberry = Product('Strawberry')
chocolate = Product('Chocolate')
lemon = Product('Lemon')
milk = Product('Milk')
egg = Product('Egg')
tomato = Product('Tomato')
cucumber = Product('Cucumber')
print(orange)
dish1 = Dish('Dish1', [orange, chocolate, milk], 34.5, 10)
dish2 = Dish('Dish2', [lemon, strawberry, orange], 23, 5)
dish3 = Dish('Dish3', [egg, milk, chocolate], 50, 8)
dish4 = Dish('Dish4', [tomato, cucumber, orange], 30, 7)
print(dish1)
restaurant.menu = [dish1, dish2, dish3, dish4]
for dish in restaurant.menu:
print(dish)
disease1 = Disease('Citrus allergy', [orange, lemon])
disease2 = Disease('Strawberry allergy', [strawberry])
disease3 = Disease('Lactose intolerance', [milk, egg])
disease4 = Disease('Chocolate allergy', [chocolate])
print(disease1)
from ingredient import Ingredient, IngredientType
from dish import Dish
from meal import Meal
import logging
logging.basicConfig(filename='test.log', level=logging.DEBUG, format='%(asctime)s:%(name)s:%(levelname)s:%(message)s')
grnPepper = Ingredient('green pepper', IngredientType.VEGETABLE)
banana = Ingredient('banana', IngredientType.FRUIT)
strawberry = Ingredient('strawberry', IngredientType.FRUIT)
spinach = Ingredient('spinach', IngredientType.VEGETABLE)
sampleDish1 = Dish('Dish 1', [grnPepper, spinach], 'Cut 4oz of spinach and 1 oz of green pepprs. Place in pot with 16oz of water. Boil for one hour. Serve')
sampleDish2 = Dish('Dish 2', [strawberry, banana], 'Cut 4oz of strawberries and 2 oz of bananas. Place in bowl and toss.')
sampleMeal = Meal([sampleDish1, sampleDish2, banana])
logging.debug(sampleMeal.getMeal())
def get_details(self):
try:
self.driver.set_page_load_timeout(30)
self.driver.get(self.url)
print("+++++++++HEREEEEE coming")
try:
FIND_BY = self.starter_config['SELECTORS']['PRE_ORDER'][
'FIND_BY']
VALUE = self.starter_config['SELECTORS']['PRE_ORDER']['VALUE']
# if FIND_BY == 'class':
# self.driver.find_element_by_class_name(VALUE).click()
# elif FIND_BY == 'css':
# self.driver.find_element_by_css_selector(VALUE).click()
WebDriverWait(self.driver, 10).until(
EC.element_to_be_clickable(
(By.XPATH,
'//*[@id="hmodal"]/div/div/div[2]/button'))).click()
print("+++++++CLICKED")
except Exception as e:
print("))))))))ERROR", e)
FIND_BY = self.dish_config['SELECTORS']['DISHESH']['FIND_BY']
VALUE = self.dish_config['SELECTORS']['DISHESH']['VALUE']
# element = WebDriverWait(driver, 10).until(
# EC.visibility_of_element_located((By.CLASS_NAME, VALUE))
# )
if FIND_BY == 'class':
dishes = self.driver.find_elements_by_class_name(VALUE)
elif FIND_BY == 'css':
dishes = self.driver.find_elements_by_css_selector(VALUE)
print("+++++++++HEREEEEE")
restaurant_obj = {
'city_code': self.city_code,
'name': "Ready Bowl",
'type': 'North Indian, South Indian',
'stars': 4.1,
'ratings': '100+ ratings',
'image':
'https://www.google.com/url?sa=i&rct=j&q=&esrc=s&source=images&cd=&ved=2ahUKEwjo7r61s-TmAhX1zDgGHfJjAkoQjRx6BAgBEAQ&url=https%3A%2F%2Fwww.zomato.com%2Fbangalore%2Freadybowl-btm&psig=AOvVaw2OnHt5mwTrtZrmFtJTEItS&ust=1578036819956899',
'opens_at': None,
'country': self.country,
'subzone': 'General',
'city': self.city,
'platform': 'READYBOWL',
'dishes': [],
'added_on': str(datetime.datetime.utcnow())
}
self.restaurant_obj = restaurant_obj
dish_obj = Dish(self.dish_config, self.restaurant_obj, self.driver)
dish_obj.get_dishes(dishes)
self.driver.close()
sort_key_info = restaurant_obj['platform'] + '__' + restaurant_obj[
'subzone'] + '__' + restaurant_obj['name'].strip().replace(
' ', '_')
restaurant_obj['sort_key_info'] = sort_key_info
restaurant_obj['stars'] = Decimal(str(
self.restaurant_obj['stars']))
# # write to dynamodb
self.dynamodb_write_obj = DynamoDBWrite()
self.dynamodb_write_obj.dynamodb_write(self.restaurant_obj)
# # write data to parquet in s3
restaurant_obj['stars'] = float(self.restaurant_obj['stars'])
print("OBJJJJJJJJJJJJJ++++print", json.dumps(self.restaurant_obj))
self.write_to_s3_parquet_obj = WriteS3Parquet(self.city)
self.write_to_s3_parquet_obj.write_to_parquet(self.restaurant_obj)
except:
print('++++++++NOT DONE')
self.driver.close()
DISHES = []
config = configparser.ConfigParser()
config.read(global_vars.resturant_data_file)
TAX = config['internal-config'].getfloat('tax')
# if the section isn't internal then make a dish object from it
for section in config.sections():
sec_info = config[section]
if 'type' in sec_info and sec_info['type'] == 'dish':
name = section
cost = sec_info.getfloat('cost')
long_arg = sec_info.get('long_arg')
short_arg = sec_info.get('short_arg')
dish = Dish(name, cost, TAX)
dish.set_long_arg(long_arg)
dish.set_short_arg(short_arg)
for key in sec_info.keys():
dish.add_info(key, sec_info[key])
DISHES.append(dish)
total_cost = 0
for i in range(len(sys.argv)):
arg = sys.argv[i]
how_many_dishes = 1
if arg == '-h' or arg == '--help':
print("Usage: ")
print(
def main():
pygame.init()
display = (1900, 1000)
pygame.display.set_mode(display, DOUBLEBUF | OPENGL)
u = 0
#pygame.mixer.music.load("Experience.mp3")
#pygame.mixer.music.play()
gluPerspective(45, (display[0] / display[1]), 0.1, 100.0)
glTranslatef(0, 0, -70)
object_passed = False
i = 0
controllore = 0
s = 0
w = 0
camera_y = 0
rotazione = 0
while not object_passed:
if rotazione == 72 or rotazione == -72:
rotazione = 0
glClearColor(1, 1, 0.9, 0)
if i == 1:
for event in pygame.event.get():
if event.type == pygame.QUIT:
pygame.quit()
quit()
if event.type == pygame.KEYDOWN:
if event.key == pygame.K_LEFT:
glTranslatef(0.5, 0, 0)
if event.key == pygame.K_RIGHT:
glTranslatef(-0.5, 0, 0)
if event.key == pygame.K_UP:
glTranslatef(0, -1, 0)
if event.key == pygame.K_DOWN:
if camera_y < 1:
glTranslatef(0, 1, 0)
if event.type == pygame.MOUSEBUTTONDOWN:
if event.button == 4:
glTranslatef(0, 0, 1.0)
if event.button == 5:
glTranslatef(0, 0, -1.0)
keypress = pygame.key.get_pressed()
if keypress[pygame.K_a]:
rotazione = rotazione - 1
glRotate(5, 0, 0.5, 0) #ang,x,y,z
if keypress[pygame.K_w]:
if w < 35:
w = w + 1
glRotate(5, 0.5, 0, 0) #ang,x,y,z
if keypress[pygame.K_s]:
rotazione = rotazione + 1
glRotate(5, 0, -0.5, 0) #ang,x,y,z
if keypress[pygame.K_z]:
if w > 0:
w = w - 1
glRotate(5, -0.5, 0, 0) #ang,x,y,z
x = glGetDoublev(GL_MODELVIEW_MATRIX) #, modelviewMatrix)
camera_x = x[3][0]
camera_y = x[3][1]
camera_z = x[3][2]
#print('rotazione',rotazione)
# slowly move forward :
if camera_z >= 40 and i == 0:
glTranslatef(0, 0, 0.8)
else:
i = 1
controllore = 1
glTranslatef(0, 0, 0)
if (rotazione < 40 and rotazione > -1) or (rotazione < -49
and rotazione > -72):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
if controllore == 0:
u = 0
Ground()
if (rotazione > 21 and rotazione < 51) or (rotazione < -19
and rotazione > -52):
Skeleton(u)
glPushMatrix()
glRotate(270, 0, 1, 0)
glTranslated(-1.5, -1, 0)
Chair()
glPopMatrix()
glPushMatrix()
glRotate(120, 0, 1, 0)
glTranslated(-14, -9, 9)
Table()
glPopMatrix()
glPushMatrix()
glTranslated(13, -1, 7)
Dish()
glPopMatrix()
glPushMatrix()
glTranslated(18, -1, 9)
Glass()
glPopMatrix()
u = u + 1
if u < 371:
u = u
else:
u = 0
else:
glPushMatrix()
glRotate(270, 0, 1, 0)
glTranslated(-1.5, -1, 0)
Chair()
glPopMatrix()
Skeleton(u)
glPushMatrix()
glRotate(120, 0, 1, 0)
glTranslated(-14, -9, 9)
Table()
glPopMatrix()
glPushMatrix()
glTranslated(13, -1, 7)
Dish()
glPopMatrix()
glPushMatrix()
glTranslated(18, -1, 9)
Glass()
glPopMatrix()
u = u + 1
if u < 371:
u = u
else:
u = 0
else:
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
if controllore == 0:
u = 0
Ground()
if (rotazione > 21 and rotazione < 51) or (rotazione < -19
and rotazione > -52):
glPushMatrix()
glRotate(120, 0, 1, 0)
glTranslated(-14, -9, 9)
Table()
glPopMatrix()
glPushMatrix()
glTranslated(13, -1, 7)
Dish()
glPopMatrix()
glPushMatrix()
glTranslated(18, -1, 9)
Glass()
glPopMatrix()
Skeleton(u)
glPushMatrix()
glRotate(270, 0, 1, 0)
glTranslated(-1.5, -1, 0)
Chair()
glPopMatrix()
u = u + 1
if u < 371:
u = u
else:
u = 0
else:
glPushMatrix()
glRotate(270, 0, 1, 0)
glTranslated(-1.5, -1, 0)
Chair()
glPopMatrix()
glPushMatrix()
glRotate(120, 0, 1, 0)
glTranslated(-14, -9, 9)
Table()
glPopMatrix()
glPushMatrix()
glTranslated(13, -1, 7)
Dish()
glPopMatrix()
glPushMatrix()
glTranslated(18, -1, 9)
Glass()
glPopMatrix()
Skeleton(u)
u = u + 1
if u < 371:
u = u
else:
u = 0
#print(u)
pygame.display.flip()
if camera_z <= 0:
object_passed = True
def click_ctx():
shell = Dish()
ctx = click.Context(cli.main, obj=shell)
return ctx
def get_details(self,url):
try:
print('+++++getting restro page')
self.driver.get(self.url)
try:
# element = WebDriverWait(driver, 10).until(
# EC.visibility_of_element_located((By.CLASS_NAME, "menu-content-wrapper"))
# )
FIND_BY = self.dish_config['SELECTORS']['WAIT']['FIND_BY']
VALUE = self.dish_config['SELECTORS']['WAIT']['VALUE']
if FIND_BY == 'class':
element = WebDriverWait(self.driver, 10).until(
EC.visibility_of_element_located((By.CLASS_NAME, VALUE))
)
elif FIND_BY == 'id':
element = WebDriverWait(self.driver, 10).until(
EC.visibility_of_element_located((By.ID, VALUE))
)
elif FIND_BY == 'tag':
element = WebDriverWait(self.driver, 10).until(
EC.visibility_of_element_located((By.TAG_NAME, VALUE))
)
CTG_NAV = {
'FIND_BY': self.dish_config['SELECTORS']['CATEGORY_NAV']['FIND_BY'],
'VALUE' : self.dish_config['SELECTORS']['CATEGORY_NAV']['VALUE']
}
CATEGORIES = {
'FIND_BY': self.dish_config['SELECTORS']['CATEGORIES']['FIND_BY'],
'VALUE' : self.dish_config['SELECTORS']['CATEGORIES']['VALUE']
}
if CTG_NAV['FIND_BY'] == 'class':
ctg_nav = self.driver.find_element_by_class_name(CTG_NAV['VALUE'])
elif CTG_NAV['FIND_BY'] == 'id':
ctg_nav = self.driver.find_element_by_id(CTG_NAV['VALUE'])
elif CTG_NAV['FIND_BY'] == 'tag':
ctg_nav = self.driver.find_element_by_tag_name(CTG_NAV['VALUE'])
if CATEGORIES['FIND_BY'] == 'class':
categories = ctg_nav.find_elements_by_class_name(CATEGORIES['VALUE'])
elif CATEGORIES['FIND_BY'] == 'id':
categories = ctg_nav.find_element_by_id(CATEGORIES['VALUE'])
elif CATEGORIES['FIND_BY'] == 'tag':
categories = ctg_nav.find_elements_by_tag_name(CATEGORIES['VALUE'])
# ctg_nav = driver.find_element_by_class_name('menu-sidebar-items')
# categories = ctg_nav.find_elements_by_tag_name('li')
restaurant_obj = {
'city_code':self.city_code,
'name':"Box8",
'type':"North Indian, South Indian",
'stars':4.1,
'ratings':"100+ Ratings",
'image':"https://assets.box8.co.in/images/Box8.jpg",
'country':self.country,
'city':self.city,
'city':'Bangalore',
'subzone':'General',
'platform':'Box8',
'dishes':[],
'added_on': str(datetime.datetime.utcnow())
}
self.dish_obj = Dish(self.driver, self.dish_config)
for ctg in categories:
self.dish_obj.get_dishes(ctg,restaurant_obj)
print('+++++++',len(restaurant_obj['dishes']))
sort_key_info = restaurant_obj['platform']+'__'+restaurant_obj['subzone']+'__'+restaurant_obj['name'].strip().replace(' ','_')
restaurant_obj['sort_key_info'] = sort_key_info
restaurant_obj['stars'] = Decimal(str(restaurant_obj['stars']))
self.dynamodb_write_obj = DynamoDBWrite()
self.dynamodb_write_obj.dynamodb_write(restaurant_obj)
# write data to parquet in s3
restaurant_obj['stars'] = float(restaurant_obj['stars'])
self.write_to_s3_parquet_obj = WriteS3Parquet(self.city)
self.write_to_s3_parquet_obj.write_to_parquet(restaurant_obj)
except Exception as e:
print('++++++++NOT DONE',e)
self.driver.close()
except Exception as e:
print('+++++++Exception while contents of given url',e)
def get_details(self):
try:
self.driver.get(self.url)
# for i in range(3):
# try:
# self.driver.find_element_by_id("exitIntentEmail").send_keys("*****@*****.**")
# time.sleep(3)
# self.driver.find_element_by_class_name("submit").click()
# except:
# print("ERROR++++++++")
# for i in range(0):
# try:
# self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
# except:
# print("ERROR SCROLL++++++++")
# time.sleep(3)
# print("________", i)
# for i in range(3):
# try:
# self.driver.find_element_by_id("exitIntentEmail").send_keys("*****@*****.**")
# time.sleep(3)
# self.driver.find_element_by_class_name("submit").click()
# time.sleep(3)
# except:
# print("ERROR++++++++")
try:
time.sleep(3)
FIND_BY = self.dish_config['SELECTORS']['DISHES']['FIND_BY']
VALUE = self.dish_config['SELECTORS']['DISHES']['VALUE']
if FIND_BY == 'class':
dishes = self.driver.find_elements_by_class_name(VALUE)
elif FIND_BY == 'id':
dishes = self.driver.find_elements_by_id(VALUE)
except Exception as identifier:
print("Error************", identifier)
restaurant_obj = {
'city_code': self.city_code,
'name': "Fresh Menu",
'type':
'Mexican,Thai,Continental,Mediterranean,Indian,Chinese,Italian,American',
'stars': 4.1,
'ratings': '100+ ratings',
'image':
'https://www.google.com/url?sa=i&rct=j&q=&esrc=s&source=images&cd=&cad=rja&uact=8&ved=2ahUKEwjVhJHBsuTmAhUwwzgGHQaEDAMQjRx6BAgBEAQ&url=https%3A%2F%2Fwww.youtube.com%2Fchannel%2FUCtg2fATkg7hXYa5VzxzZzPg&psig=AOvVaw3cdHfc2BzRGirA_FABBvnJ&ust=1578036499721707',
'opens_at': None,
'country': self.country,
'subzone': 'General',
'city': self.city,
'platform': 'FRESHMENU',
'dishes': [],
'added_on': str(datetime.datetime.utcnow())
}
self.restaurant_obj = restaurant_obj
dish_obj = Dish(self.dish_config, self.restaurant_obj)
print("++++HEREEE")
dish_obj.get_dishes_url(dishes, self.dishes_url)
self.driver.close()
print("++++BACKKKKK", len(self.dishes_url))
dish_obj.get_description(self.dishes_url)
print("OBJJJJJJJJJJJJJ++++", len(self.restaurant_obj['dishes']))
sort_key_info = restaurant_obj['platform'] + '__' + restaurant_obj[
'subzone'] + '__' + restaurant_obj['name'].strip().replace(
' ', '_')
restaurant_obj['sort_key_info'] = sort_key_info
restaurant_obj['stars'] = Decimal(str(
self.restaurant_obj['stars']))
# # write to dynamodb
self.dynamodb_write_obj = DynamoDBWrite()
self.dynamodb_write_obj.dynamodb_write(self.restaurant_obj)
# write data to parquet in s3
restaurant_obj['stars'] = float(self.restaurant_obj['stars'])
self.write_to_s3_parquet_obj = WriteS3Parquet(self.city)
self.write_to_s3_parquet_obj.write_to_parquet(self.restaurant_obj)
except Exception as e:
print("+++++NOT DONE", e)
try:
self.driver.close()
except:
pass
class Zomato:
def __init__(self):
self.CONFIG_FILE = "jwt-config.json"
with open(self.CONFIG_FILE, 'r') as config_file:
self.data_set = json.load(config_file)
self.starter_config = self.data_set['ZOMATO']['CONFIG']['STARTER']
self.config = self.data_set['ZOMATO']['CONFIG']
self.dishes_data = []
self.restaurants_data = []
self.restaurants_obj = Restaurant(self.config)
self.city = os.getenv('CITY', "jaipur")
self.country = os.getenv('COUNTRY', "india")
self.city_code = self.city + '__' + self.country
self.dish_obj = Dish(self.config, self.city, self.country,
self.city_code)
def get_data(self, url=None):
URL = self.starter_config["URLS"][self.city]
print('+++++++++city url ff', self.city, URL)
headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:67.0) Gecko/20100101 Firefox/67.0'
}
r = requests.get(URL, headers=headers)
soup = BeautifulSoup(r.content, 'html5lib')
TAG = self.starter_config['SELECTORS']['WAIT']['TAG']
FIND_BY = self.starter_config['SELECTORS']['WAIT']['FIND_BY']
VALUE = self.starter_config['SELECTORS']['WAIT']['VALUE']
data = soup.find_all(TAG, attrs={FIND_BY: VALUE})
_list = []
for row in data:
link = row['href']
subzone = row.text.strip().split(' ')
index = subzone.index('')
subzone = subzone[0:index]
subzone = " ".join(subzone)
# print('++++++subzone',subzone)
_list.append({
'subzone':
subzone,
'link':
"https://www.zomato.com/" + self.city + "/delivery-in-" +
subzone.lower().replace(' ', '-') + "?ref_page=subzone"
})
print(" list of subzones ", len(_list))
# _list = _list[:1]
no_of_threads = 3
last_chunk = -1
subzone_batch_threads = []
length_of_subzones = len(_list)
chunk_size = int(length_of_subzones / no_of_threads)
print('************chunk size', chunk_size)
if length_of_subzones >= no_of_threads:
for i in range(no_of_threads):
batch = _list[i * chunk_size:(i + 1) * chunk_size]
# batch = _list[0:1]
subzone_batch_threads.append(
threading.Thread(target=self.get_restaurants_thread,
args=(batch, )))
subzone_batch_threads[-1].start()
last_chunk = i
last_chunk += 1
if no_of_threads * chunk_size < length_of_subzones:
batch = _list[last_chunk * chunk_size:length_of_subzones]
# batch = _list[0:1]
subzone_batch_threads.append(
threading.Thread(target=self.get_restaurants_thread,
args=(batch, )))
subzone_batch_threads[-1].start()
for thread in subzone_batch_threads:
print('\n\n**************JOINING***********')
thread.join()
else:
for subzone in _list:
self.restaurants_obj.get_restaurants(subzone['link'],
subzone['subzone'],
self.restaurants_data)
self.get_dishes()
print('++++DISHES LEN', len(self.dishes_data))
# # load data to dynamodb
self.dynamodb_batch_write_obj = DynamoDBBatchWrite()
self.dynamodb_batch_write_obj.batch_write_to_ddb(self.dishes_data)
for dish_data in self.dishes_data:
dish_data['stars'] = float(dish_data['stars'])
# # write data to parquet in s3
self.write_to_s3_parquet_obj = WriteS3Parquet(self.city)
self.write_to_s3_parquet_obj.write_to_parquet(self.dishes_data)
def get_restaurants_thread(self, subzones):
print('+++++=THREAD')
for subzone in subzones:
self.restaurants_obj.get_restaurants(subzone['link'],
subzone['subzone'],
self.restaurants_data)
def get_dishes(self):
no_of_threads = 2
last_chunk = -1
restaurant_batch_threads = []
# self.restaurants_data = self.restaurants_data[3:8]
length_of_restaurants_data = len(self.restaurants_data)
chunk_size = int(length_of_restaurants_data / no_of_threads)
for i in range(no_of_threads):
batch = self.restaurants_data[i * chunk_size:(i + 1) * chunk_size]
# batch = restaurants_data[0:2]
restaurant_batch_threads.append(
threading.Thread(target=self.get_dishes_thread,
args=(batch, )))
restaurant_batch_threads[-1].start()
last_chunk = i
last_chunk += 1
if no_of_threads * chunk_size < length_of_restaurants_data:
batch = self.restaurants_data[
last_chunk * chunk_size:length_of_restaurants_data]
# batch = restaurants_data[0:2]
restaurant_batch_threads.append(
threading.Thread(target=self.get_dishes_thread,
args=(batch, )))
restaurant_batch_threads[-1].start()
for thread in restaurant_batch_threads:
thread.join()
def get_dishes_thread(self, restaurants):
for restaurant in restaurants:
try:
options = Options()
options.headless = True
driver = webdriver.Firefox(options=options)
# driver.set_page_load_timeout(10)
self.dish_obj.get_details(driver, restaurant, self.dishes_data)
driver.close()
except Exception as e:
print('++++++error while getting dish', e)
try:
driver.close()
except:
pass