def setUp(self): self.category_1 = Category(name="Pain") self.category_1.save() self.product_1 = Product( name="pain de mie", nutrition_grade_fr="a", traces="", allergens="", url="", id_openfoodfact=1, image_front_url="", image_front_small_url="", ingredients_text="", fat_100g="", salt_100g="", saturated_fat_100g="", sugars_100g="", ) self.product_1.save() self.product_1.category.add(self.category_1) self.product_2 = Product( name="pain", nutrition_grade_fr="b", traces="", allergens="", url="", id_openfoodfact=2, image_front_url="", image_front_small_url="", ingredients_text="", fat_100g="", salt_100g="", saturated_fat_100g="", sugars_100g="", ) self.product_2.save() self.product_2.category.add(self.category_1) self.user_test = create_user( { "username": "******", "email": "*****@*****.**", "password": "******", } ) self.user_test_2 = create_user( { "username": "******", "email": "*****@*****.**", "password": "******", } )
def handle(self, *args, **options): Category.objects.all().delete() Product.objects.all().delete() categories = openfoodfacts.facets.get_categories() print("Starting injection ...") progression = 0 for item in categories[:5]: enum = 0 cat_name = item['name'] new_cat = Category(name=cat_name) new_cat.save() new_cat_id = getattr(new_cat, 'id') products = openfoodfacts.products.get_all_by_category(cat_name) for food_item in products: try: food_name = food_item['product_name'] foot_nutri = food_item['nutriscore_score'] food_image = food_item['image_front_url'] food_url = food_item['url'] food_nutri_grade = food_item['nutriscore_grade'] new_product = Product(name=food_name, nutriscore=foot_nutri, image=food_image, nutriscore_grade=food_nutri_grade, url=food_url, category_id=new_cat_id) new_product.save() new_cat.products.add(new_product) enum += 1 progression += 1 print("Injecting -> " + str(progression) + "/3750") if enum == 15: break except BaseException: pass print("Failed to inject this product")
def fill_table(maxFoodCat, maxFoodPage): """ This function download food and inserting it into the local database Parameter: maxFoodCat = maximum food categories wished maxFoodPage = maximum categories food page wished Not all foods are looking for relevant information. So we used a table containing the keys searched for verification before extraction. """ requiredIndex = ("url", "nutrition_grade_fr", "purchase_places", "manufacturing_places", "countries", "ingredients_text", "product_name", "generic_name_fr", "image_url", "image_small_url", ) # geting categories objects from openfoodfact categoriesUrl = "https://fr.openfoodfacts.org/categories.json" r = requests.get(categoriesUrl) categories = r.json() # insert the categories in the DB for i in range(maxFoodCat): cat = Category() cat.name = categories['tags'][i]['name'] cat.url = categories['tags'][i]['url'] cat.save() # insert foods for each category """ A category may contain many page of foods, paginate by 20. Then we save the differents url in a list tab to loop other it and extract any foods that respect the constraint. """ foodPageUrlTab = list() # the different page url of aliments foodPageUrlTab = [cat.url + '/' + str(ind) + '.json' for ind in range(1, maxFoodPage + 1)] # request each url of the table to find out the contains foods for j in range(len(foodPageUrlTab)): foodsUrl = foodPageUrlTab[j] r2 = requests.get(foodsUrl) foods = r2.json() foodsName = list() # each page contain 20 foods for k in range(len(foods)): # verify if the food object keys contain the # required index if present(requiredIndex, foods['products'][k]): # then add them to the DB food = Food() fObject = foods['products'][k] # json food object # fill in all the texts fields food.category = cat food.name = fObject['product_name'] food.nameFr = fObject['product_name_fr'] food.genericNameFr = fObject['generic_name_fr'] food.url = fObject['url'] food.nutritionGrade = fObject['nutrition_grade_fr'] food.manufacturingPlaces = fObject['manufacturing_places'] food.purchasePlaces = fObject['purchase_places'] food.countries = fObject['countries'].replace("en:", "") food.ingredientsText = fObject['ingredients_text'] food.image_link = fObject['image_front_url'] # this section deals with uploading images and inserting # them into the DB # we save two kinds of images; small and normal size # variables what store the the saved images directory path #imageDirectory = 'media/image/' # for the normal size #imageSmallDirectory = 'media/imageSmall/' # small """imageDirectory = 'media/image/' # variables who rename the downloaded images imageName = "{}.jpg".format(slugify(food.name)) imageSmallName = "{}-sm.jpg".format(slugify(food.name)) # download the two images with urllib librairy # urllib.request.urlretrieve(imageUrl, localImagePath) imagePath = imageDirectory + str(imageName) imageSmallPath = imageDirectory + str(imageSmallName) urllib.request.urlretrieve(fObject['image_url'], imagePath) urllib.request.urlretrieve(fObject['image_small_url'], imageSmallPath) # now we can fill the two imageFields # with the downloaded images food.image = File(open(imagePath, 'rb')) food.imageSmall = File(open(imageSmallPath, 'rb'))""" if food.name in foodsName: pass else: foodsName.append(food.name) food.save()
def handle(self, *args, **options): if options["category"] is not None: if options["category"] == "up" or options["category"] == "update": api = api_openfoodfact.ApiOpenFoodFact() response = api.get_category() for category in response["tags"]: if category["products"] > 10: c = Category(name=category["name"]) c.save() print("update terminée") if options["category"] == "del": Category.objects.all().delete() print("Toutes les catégories sont supprimées") if "get" in options["category"]: list_category = options["category"].split(":") id = list_category[1] item = Category.objects.get(pk=id) print(item.name) if options["product"] is not None: if options["product"] == "up" or options["product"] == "update": api = api_openfoodfact.ApiOpenFoodFact() category_list = Category.objects.all() for c in category_list: response = api.product_requests_by_category(c.name, 1) for products in response["products"]: if "id" in products: id_validation = True else: id_validation = False if id_validation: # check if products exist try: item = Product.objects.get( id_openfoodfact=products["id"]) item.category.add(c) except: item = None if item is None: # Change "nutrition_grade_fr" if absent if "nutrition_grade_fr" not in products: products[ "nutrition_grade_fr"] = "Non applicable" # Change "allergens" type build en:egg in egg if "allergens" in products: allergens = [] text = products["allergens"].split(",") for allergen in text: allerg = list(allergen) del allerg[:3] allergens.append("".join(allerg)) products["allergens"] = ", ".join( allergens) # Change "traces" type build en:gluten in gluten if "traces" in products: traces = [] text = products["traces"].split(",") for trace in text: tra = list(trace) del tra[:3] traces.append("".join(tra)) products["traces"] = ", ".join(traces) # change nutriments if not exist: nutriments = {} if "nutriments" in products: if "fat_100g" in products["nutriments"]: nutriments["fat_100g"] = products[ "nutriments"]["fat_100g"] else: nutriments[ "fat_100g"] = "Non applicable" if "salt_100g" in products["nutriments"]: nutriments["salt_100g"] = products[ "nutriments"]["salt_100g"] else: nutriments[ "salt_100g"] = "Non applicable" if "saturated-fat_100g" in products[ "nutriments"]: nutriments[ "saturated-fat_100g"] = products[ "nutriments"][ "saturated-fat_100g"] else: nutriments[ "saturated-fat_100g"] = "Non applicable" if "sugars_100g" in products["nutriments"]: nutriments["sugars_100g"] = products[ "nutriments"]["sugars_100g"] else: nutriments[ "sugars_100g"] = "Non applicable" else: nutriments = { "fat_100g": "Non applicable", "salt_100g": "Non applicable", "saturated-fat_100g": "Non applicable", "sugars_100g": "Non applicable", } p = Product( name=products["product_name"], nutrition_grade_fr=products[ "nutrition_grade_fr"], traces=products["traces"], allergens=products["allergens"], url=products["url"], id_openfoodfact=products["id"], image_front_url=products[ "image_front_url"], image_front_small_url=products[ "image_front_small_url"], fat_100g=nutriments["fat_100g"], salt_100g=nutriments["salt_100g"], saturated_fat_100g=nutriments[ "saturated-fat_100g"], sugars_100g=nutriments["sugars_100g"], ingredients_text=products[ "ingredients_text"], ) p.save() p.category.add(c) print("update terminée") if options["product"] == "test": api = api_openfoodfact.ApiOpenFoodFact() response = api.product_requests_by_category( "Aliments et boissons à base de végétaux", 1) for key in response["products"][0].keys(): print(key, " : ", response["products"][0][key]) if options["product"] == "del": Product.objects.all().delete() print("Tous les produits sont supprimés") if "get" in options["product"]: list_products = options["product"].split(":") id = list_products[1] try: item = Product.objects.get(pk=id) except: item = None if item is not None: print(item.name) else: print(item)