def fill_table(maxFoodCat, maxFoodPage):
"""
This function download food and inserting it into
the local database
Parameter: maxFoodCat = maximum food categories wished
maxFoodPage = maximum categories food page wished
Not all foods are looking for relevant information.
So we used a table containing the keys searched for
verification before extraction.
"""
requiredIndex = ("url", "nutrition_grade_fr",
"purchase_places", "manufacturing_places", "countries",
"ingredients_text", "product_name", "generic_name_fr",
"image_url", "image_small_url",
)
# geting categories objects from openfoodfact
categoriesUrl = "https://fr.openfoodfacts.org/categories.json"
r = requests.get(categoriesUrl)
categories = r.json()
# insert the categories in the DB
for i in range(maxFoodCat):
cat = Category()
cat.name = categories['tags'][i]['name']
cat.url = categories['tags'][i]['url']
cat.save()
# insert foods for each category
"""
A category may contain many page of foods, paginate by 20.
Then we save the differents url in a list tab to loop other
it and extract any foods that respect the constraint.
"""
foodPageUrlTab = list() # the different page url of aliments
foodPageUrlTab = [cat.url + '/' + str(ind) + '.json'
for ind in range(1, maxFoodPage + 1)]
# request each url of the table to find out the contains foods
for j in range(len(foodPageUrlTab)):
foodsUrl = foodPageUrlTab[j]
r2 = requests.get(foodsUrl)
foods = r2.json()
foodsName = list()
# each page contain 20 foods
for k in range(len(foods)):
# verify if the food object keys contain the
# required index
if present(requiredIndex, foods['products'][k]):
# then add them to the DB
food = Food()
fObject = foods['products'][k] # json food object
# fill in all the texts fields
food.category = cat
food.name = fObject['product_name']
food.nameFr = fObject['product_name_fr']
food.genericNameFr = fObject['generic_name_fr']
food.url = fObject['url']
food.nutritionGrade = fObject['nutrition_grade_fr']
food.manufacturingPlaces = fObject['manufacturing_places']
food.purchasePlaces = fObject['purchase_places']
food.countries = fObject['countries'].replace("en:", "")
food.ingredientsText = fObject['ingredients_text']
food.image_link = fObject['image_front_url']
# this section deals with uploading images and inserting
# them into the DB
# we save two kinds of images; small and normal size
# variables what store the the saved images directory path
#imageDirectory = 'media/image/' # for the normal size
#imageSmallDirectory = 'media/imageSmall/' # small
"""imageDirectory = 'media/image/'
# variables who rename the downloaded images
imageName = "{}.jpg".format(slugify(food.name))
imageSmallName = "{}-sm.jpg".format(slugify(food.name))
# download the two images with urllib librairy
# urllib.request.urlretrieve(imageUrl, localImagePath)
imagePath = imageDirectory + str(imageName)
imageSmallPath = imageDirectory + str(imageSmallName)
urllib.request.urlretrieve(fObject['image_url'],
imagePath)
urllib.request.urlretrieve(fObject['image_small_url'],
imageSmallPath)
# now we can fill the two imageFields
# with the downloaded images
food.image = File(open(imagePath, 'rb'))
food.imageSmall = File(open(imageSmallPath, 'rb'))"""
if food.name in foodsName:
pass
else:
foodsName.append(food.name)
food.save()
