def handle(self, *args, **options):
number_categories = options['c']
number_products = options['p']
api = OpenFoodFactsAPI(number_categories, number_products)
# Adding categories
print(f"Adding {number_categories} categories", end="")
for api_category in api.categories:
category = Category(name=api_category)
category.save()
print(".", end="")
# Adding products
print(
f"\nAdding {number_categories * number_products} products, it can take a little while..."
)
for api_product in api.get_products():
api_product['category'] = Category.objects.get(
name=api_product['category'])
product = Product(**api_product)
# Taking care of duplicates if any (sanity check even with a set() in api.get_products())
try:
product.save()
except IntegrityError: # Unicity constraint
continue
print(
f"\nSuccessfully added {len(Product.objects.all())} products and "
f"{len(Category.objects.all())} categories. Differences may be duplicates."
)
def test_check_fr_is_false(self):
prod = {}
api = OpenFoodFactsAPI(1, 1, ["Boissons"])
is_fr = api._check_product_is_fr(prod)
self.assertFalse(is_fr,
msg="A \"Foreign\" product is taken as a French one.")
def test_check_fr_is_true(self):
prod = json.loads(self.response_content_prod)['products'][0]
api = OpenFoodFactsAPI(1, 1, ["Boissons"])
is_fr = api._check_product_is_fr(prod)
self.assertTrue(is_fr,
msg="A French product is considerer as non-French.")
def test_get_products_bad_response_content_type_error(self):
self.mock_response.content = json.dumps("")
self.mock_response.status_code = 200
api = OpenFoodFactsAPI(1, 1, ['Boissons'])
with self.assertRaises(
OpenFoodFactsException,
msg="Don't raise Exception when bad API response"):
next(api.get_products())
def test_get_products_no_response_content(self):
self.mock_response.content = ""
self.mock_response.status_code = 200
api = OpenFoodFactsAPI(1, 1, ["Boissons"])
with self.assertRaises(
OpenFoodFactsException,
msg="Don't raise Exception when Status Code != 200"):
next(api.get_products())
def test_get_products_proper_yield(self, mock_is_fr):
self.mock_response.content = self.response_content_prod
self.mock_response.status_code = 200
api = OpenFoodFactsAPI(1, 1, ['Boissons'])
product = next(api.get_products())
self.assertEqual(
json.loads(self.response_content_prod)['products'][0]['code'],
product['code'])
def handle(self, *args, **options):
number_updated = 0
categories = Category.objects.all()
products = Product.objects.all()
if not products or not categories:
print("There is no product in the database. Please "
"do a 'python manage.py db_init -c X -p Y' first.")
api = OpenFoodFactsAPI(len(categories), len(products))
date_begin = dt.datetime.now()
date_fmt = date_begin.strftime("%A %d %B %Y at %H:%M:%S")
print(f"Beginning : {date_fmt}")
print("Updating products...")
for api_product in api.get_products():
# Looking if the product is already in the db
try:
db_product = products.get(pk=api_product['code'])
except Product.DoesNotExist:
# print(f"Product not found in the db. {api_product}")
# No corresponding product found. As we only update but not add, we pass
continue
# Updating the product
db_product_updated = False
for api_prod_attr in api_product.keys():
if api_prod_attr in {"code", "category"}:
continue
if hasattr(db_product, api_prod_attr) \
and getattr(db_product, api_prod_attr) != api_product[api_prod_attr]:
setattr(db_product, api_prod_attr,
api_product[api_prod_attr])
db_product_updated = True
if db_product_updated:
try:
db_product.save()
# One product has changed it's name to one already existing in the db
# with a different code...
except IntegrityError:
continue
number_updated += 1
time_end = dt.datetime.now()
time_end = time_end - date_begin
print(
f"Successfully updated {number_updated} products of the {len(products)} present "
f"in the database. Took {time_end.seconds} seconds.")
def test_get_products_continue_if_prod_not_french_proper_yield(
self, mock_is_fr):
self.mock_response.content = self.response_content_prod
self.mock_response.status_code = 200
api = OpenFoodFactsAPI(1, 1, ['Boissons'])
product = next(api.get_products())
# It yields the SECOND product
self.assertEqual(
json.loads(self.response_content_prod)['products'][1]['code'],
product['code'])
def test_get_products_continue_if_product_key_error(self, mock_is_fr):
self.mock_response.status_code = 200
# Adding that 3rd product at index 0 to get the KeyError and to yield something
products = json.loads(self.response_content_prod)
products['products'].insert(0, {"Key": "Value"})
self.mock_response.content = json.dumps(products)
api = OpenFoodFactsAPI(1, 1, ['Boissons'])
product = next(api.get_products())
self.assertEqual(
json.loads(self.response_content_prod)['products'][0]['code'],
product['code'])
def test_get_products_continue_if_product_name_too_long(self, mock_is_fr):
self.mock_response.status_code = 200
# Modifying first product to have a name > 150
products = json.loads(self.response_content_prod)
products['products'][0]['product_name_fr'] = "z" * 160
self.mock_response.content = json.dumps(products)
api = OpenFoodFactsAPI(1, 1, ['Boissons'])
product = next(api.get_products())
# It should yield the SECOND product
self.assertEqual(
json.loads(self.response_content_prod)['products'][1]['code'],
product['code'])
def test_get_categories_no_content(self):
self.mock_response.content = ""
self.mock_response.status_code = 200
with self.assertRaises(OpenFoodFactsException,
msg="No content doesn't raises Exception"):
OpenFoodFactsAPI(1, 1)
def test_get_products_continue_if_bad_nutriscore(self, mock_is_fr):
self.mock_response.status_code = 200
# Modifying first product to have a bad nutriscore
products = json.loads(self.response_content_prod)
products['products'][0][
'nutriscore_score'] = "string and not int -- fr"
self.mock_response.content = json.dumps(products)
api = OpenFoodFactsAPI(1, 1, ['Boissons'])
product = next(api.get_products())
# It should yield the SECOND product
self.assertEqual(
json.loads(self.response_content_prod)['products'][1]['code'],
product['code'])
def test_get_categories_bad_response_content_type_error(self):
self.mock_response.content = json.dumps({"tags": 1})
self.mock_response.status_code = 200
with self.assertRaises(
OpenFoodFactsException,
msg="Bad response.content doesn't raise Exception"):
OpenFoodFactsAPI(1, 1)
def test_get_categories_status_code_not_200(self):
self.mock_response.content = self.response_content_cat
self.mock_response.status_code = 404
with self.assertRaises(
OpenFoodFactsException,
msg="Status code != 200 doesn't raises Exception"):
OpenFoodFactsAPI(1, 1)
def test_get_categories_len(self):
self.mock_response.status_code = 200
self.mock_response.content = self.response_content_cat
api = OpenFoodFactsAPI(2, 2)
self.assertEqual(2, len(api.categories))
self.assertIn("Boissons", api.categories)
self.assertIn("Charcuterie", api.categories)
def test_get_products_continue_if_product_name_already_registered(
self, mock_is_fr):
self.mock_response.status_code = 200
# Adding a third product so that theirs names are X = X != Y
products = json.loads(self.response_content_prod)
new_prod = copy.deepcopy(products['products'][1])
new_prod['product_name_fr'] = "New name to get yielded"
new_prod['code'] = 987654321
products['products'].append(new_prod)
self.mock_response.content = json.dumps(products)
api = OpenFoodFactsAPI(1, 2, ['Boissons'])
# It should yield the 1st AND 3rd product as the second as the same name as the first
products = []
for prod in api.get_products():
products.append(prod)
self.assertEqual(new_prod['code'], products[1]['code'])