def __init__(self, xfile, catfile='cats.json'):
self.filename = xfile
self.catman = CategoryManager(catfile)
self.expenses = list()
self.figures = list()
def __init__(self, database):
""" DatabaseUpdater constructor.
Creates instances of table manager classes.
Runs _run() method.
"""
self.product_manager = ProductManager(database)
self.category_manager = CategoryManager(database)
self.product_category_manager = ProductCategoryManager(database)
self.store_manager = StoreManager(database)
self.product_store_manager = ProductStoreManager(database)
self._run(database)
def __init__(self, database):
""" DatabaseFiller constructor.
For each category and for each nutrition grade, retrieves
corresponding products from Open Food Facts API and adds them to
local database.
"""
# self.database = database
# Creates instances of table manager classes
self.product_manager = ProductManager(database)
self.category_manager = CategoryManager(database)
self.product_category_manager = ProductCategoryManager(database)
self.store_manager = StoreManager(database)
self.product_store_manager = ProductStoreManager(database)
for nutrition_grade in nutrition_grades:
for category in tag_categories:
products = self._get_products(category, nutrition_grade)
self._fill_db(products)
class DatabaseUpdater:
""" Sets DatabaseUpdater class.
Consists of 14 private methods :
- __init__()
- _run()
- _get_products_codes()
- _get_OFF_product()
- _update_new_information()
- _update_categories_information()
- _get_local_product_categories_information()
- _remove_obsolete_categories()
- _add_new_categories()
- _update_stores_information()
- _get_local_product_stores_information()
- _remove_obsolete_stores()
- _add_new_stores()
- _save_update_date()
"""
def __init__(self, database):
""" DatabaseUpdater constructor.
Creates instances of table manager classes.
Runs _run() method.
"""
self.product_manager = ProductManager(database)
self.category_manager = CategoryManager(database)
self.product_category_manager = ProductCategoryManager(database)
self.store_manager = StoreManager(database)
self.product_store_manager = ProductStoreManager(database)
self._run(database)
def _run(self, database):
""" Manages database update.
For each product of local database, checks information &
updates if necessary.
Saves update date.
"""
codes = self._get_products_codes(database)
for i in range(len(codes.all())):
try:
self._get_OFF_product(codes, i)
local_product = self.product_manager.\
select_product_information(self.OFF_code)
self._update_new_information(local_product)
self._update_categories_information(local_product)
self._update_stores_information(local_product)
except KeyError as e:
print('Aïe, KeyError : ', e, file=open('print_log.txt', 'a'))
self._save_update_date()
def _get_products_codes(self, database):
""" Manages product codes retrieving.
Returns 'codes' object containing product id for all products of
the local database.
"""
codes = database.query('''SELECT Product.product_id FROM Product''')
return codes
def _get_OFF_product(self, codes, i):
""" Manages product information retrieving.
For a given product code, collects product information from Open
Food Facts API using get() method from request library.
"""
response = get(f'''https://fr.openfoodfacts.org/api/v0/product/\
{codes[i]['product_id']}.json''')
data = response.json()
OFF_product = data['product']
self.OFF_code = OFF_product['code']
self.OFF_name = OFF_product['product_name'].strip().capitalize()
self.OFF_description = OFF_product['generic_name'].capitalize()
OFF_brands = (OFF_product['brands']).split(',')
self.OFF_brand = OFF_brands[0].capitalize()
OFF_categories_to_strip = (OFF_product['categories']).split(',')
self.OFF_categories = []
for category in OFF_categories_to_strip:
self.OFF_categories.append(category.strip().capitalize())
OFF_stores_to_strip = (OFF_product['stores']).split(',')
self.OFF_stores = []
for store in OFF_stores_to_strip:
self.OFF_stores.append(store.strip().capitalize())
self.OFF_nutrition_grade = OFF_product['nutrition_grades'].lower()
def _update_new_information(self, local_product):
""" Manages information updating, except categories and stores.
Compares product information from local database vs. from Open
Food Facts API.
Updates information when needed.
"""
if local_product[0]['name'] != self.OFF_name:
self.product_manager.update_name(self.OFF_name, self.OFF_code)
print('"name" updated !', file=open('print_log.txt', 'a'))
if local_product[0]['description'] != self.OFF_description:
self.product_manager.update_description(self.OFF_description,
self.OFF_code)
print('"description" updated !', file=open('print_log.txt', 'a'))
if local_product[0]['brand'] != self.OFF_brand:
self.product_manager.update_brand(self.OFF_brand, self.OFF_code)
print('"brand" updated !', file=open('print_log.txt', 'a'))
if local_product[0]['nutrition_grade'] != self.OFF_nutrition_grade:
self.product_manager.update_nutrition_grade(
self.OFF_nutrition_grade, self.OFF_code)
print('"nutrition_grade" updated !',
file=open('print_log.txt', 'a'))
def _update_categories_information(self, local_product):
""" Manages categories information updating.
- Retrieves categories information for local product.
- Removes obsolete categories from local database.
- Adds new categories from Open Food Facts into local database.
"""
local_product_categories_id, local_product_categories_list = \
self._get_local_product_categories_information()
self._remove_obsolete_categories(local_product,
local_product_categories_id,
local_product_categories_list)
self._add_new_categories(local_product_categories_list)
def _get_local_product_categories_information(self):
""" Manages retrieving of categories information.
Returns categories id object & list of categories name for local
product.
"""
local_product_categories_id = self.product_category_manager.\
select_based_on_product_id(self.OFF_code)
local_product_categories_list = []
for i in range(len(local_product_categories_id.all())):
category_name = self.category_manager.select_based_on_id(
local_product_categories_id[i]['category_id'])
local_product_categories_list.append(category_name)
return local_product_categories_id, local_product_categories_list
def _remove_obsolete_categories(self, local_product,
local_product_categories_id,
local_product_categories_list):
""" Manages obsolete categories removing.
Removes categories from local database which don't appear
anymore in Open Food Facts platform.
"""
for i in range(len(local_product_categories_list)):
if local_product_categories_list[i] not in self.OFF_categories:
self.product_category_manager.delete(
self.OFF_code,
local_product_categories_id[i]['category_id'])
product_category = self.product_category_manager.\
select_based_on_category_id(
local_product_categories_id[i]['category_id'])
try:
category_id = product_category[0]['category_id']
# with open('print_log.txt', 'a') as f:
print(f'La catégorie {category_id} est associée à \
d\'autre(s) produit(s). On la conserve.',
file=open('print_log.txt', 'a'))
except IndexError:
# with open('print_log.txt', 'a') as f:
print('La catégorie n\'est associée à aucun autre \
produit. On la supprime',
file=open('print_log.txt', 'a'))
self.category_manager.delete(
local_product_categories_list[i])
def _add_new_categories(self, local_product_categories_list):
""" Manages new categories addition.
Adds new categories from Open Food Facts platform into local
database.
"""
for category in self.OFF_categories:
if category not in local_product_categories_list:
local_category = self.category_manager.\
select_based_on_name(category)
try:
local_category_name = local_category[0]['name']
self.product_category_manager.insert(
local_category_name, self.OFF_name)
except IndexError:
print('La catégorie n\'existe pas',
file=open('print_log.txt', 'a'))
self.category_manager.insert(category)
self.product_category_manager.insert(
category, self.OFF_name)
def _update_stores_information(self, local_product):
""" Manages stores information updating.
- Retrieves stores information for local product.
- Removes obsolete stores from local database.
- Adds new stores from Open Food Facts into local database.
"""
local_product_stores_id, local_product_stores_list = \
self._get_local_product_stores_information()
self._remove_obsolete_stores(local_product, local_product_stores_id,
local_product_stores_list)
self._add_new_stores(local_product_stores_list)
def _get_local_product_stores_information(self):
""" Manages retrieving of stores information.
Returns stores id object & list of stores name for local
product.
"""
local_product_stores_id = self.product_store_manager.\
select_based_on_product_id(self.OFF_code)
local_product_stores_list = []
for i in range(len(local_product_stores_id.all())):
store_name = self.store_manager.select_based_on_id(
local_product_stores_id[i]['store_id'])
local_product_stores_list.append(store_name)
return local_product_stores_id, local_product_stores_list
def _remove_obsolete_stores(self, local_product, local_product_stores_id,
local_product_stores_list):
""" Manages obsolete stores removing.
Removes stores from local database which don't appear
anymore in Open Food Facts platform.
"""
for i in range(len(local_product_stores_list)):
if local_product_stores_list[i] not in self.OFF_stores:
self.product_store_manager.delete(
self.OFF_code, local_product_stores_id[i]['store_id'])
product_store = self.product_store_manager.\
select_based_on_store_id(
local_product_stores_id[i]['store_id'])
try:
store_id = product_store[0]['store_id']
print(f'Le magasin {store_id} est associé à \
d\'autre(s) produit(s). On le conserve.',
file=open('print_log.txt', 'a'))
except IndexError:
print('Le magasin n\'est associé à aucun autre \
produit. On le supprime',
file=open('print_log.txt', 'a'))
self.store_manager.delete(local_product_stores_list[i])
def _add_new_stores(self, local_product_stores_list):
""" Manages new stores addition.
Adds new stores from Open Food Facts platform into local
database.
"""
for store in self.OFF_stores:
if store not in local_product_stores_list:
local_store = self.store_manager.select_based_on_name(store)
try:
local_store_name = local_store[0]['name']
self.product_store_manager.insert(local_store_name,
self.OFF_name)
except IndexError:
print('Le magasin n\'existe pas',
file=open('print_log.txt', 'a'))
self.store_manager.insert(store)
self.product_store_manager.insert(store, self.OFF_name)
def _save_update_date(self):
""" Manages saving of database update date.
Saves the date into an external 'last_update.txt' file.
"""
update_date = str(time())
with open('last_update.txt', "w") as f:
f.write(update_date)
def saving_categories(self, object_list):
category_manager = CategoryManager()
object_list = tuple(object_list)
category_manager.save(object_list)
class ExpenseReport:
"""Generates the expense report.
xfile: (str) the CSV file containing the expenses.
catfile: (str) the JSON file of the categories.
"""
def __init__(self, xfile, catfile='cats.json'):
self.filename = xfile
self.catman = CategoryManager(catfile)
self.expenses = list()
self.figures = list()
def generate_report(self):
self._categorize_expenses()
self._generate_graphs()
self._generate_pdf()
self._clean_graphs()
def _categorize_expenses(self):
logger.debug(f'Categorizing expenses from {self.filename}')
Expense = namedtuple(
'Expense',
['date', 'cat', 'subcat', 'expense', 'price']
)
date = ""
with open(self.filename, 'r') as file:
reader = DictReader(file)
for row in reader:
expense = row['Expense'].lower()
price = float(row['Price'])
# Reuse older date if not specified
if row['Date'].strip() != "":
date = row['Date']
query = self.catman.query(expense)
if query is None:
self.catman.add(expense, price)
query = self.catman.query(expense)
else:
self.catman.update(expense, price)
self.expenses.append(Expense(
date=date,
cat=query['cat'],
subcat=query['subcat'],
expense=expense,
price=price
))
logger.info(f'Successfully categorized expenses from {self.filename}')
def _generate_graphs(self):
logger.debug('Creating pie charts for overall and categorical expenses')
# Organize data for pie charts
overall = dict()
category = dict()
for x in self.expenses:
# Add expense to overall categories
if x.cat not in overall.keys():
overall[x.cat] = 0
overall[x.cat] += x.price
# Add expense to subcategories
if x.cat not in category.keys():
category[x.cat] = dict()
if x.subcat not in category[x.cat].keys():
category[x.cat][x.subcat] = 0
category[x.cat][x.subcat] += x.price
overall = {'overall': overall}
self._generate_pie_chart(overall)
self._generate_pie_chart(category)
logger.info('Successfully created all pie charts')
def _generate_pie_chart(self, data):
def pformat(pct, total):
value = pct / 100.0 * total
return f'${value:.2f}\n({pct:.0f}%)'
if len(data) == 1:
logger.debug('Creating single pie chart')
# Clear previous plot
plt.clf()
plt.cla()
fig, ax = plt.subplots(nrows=1, ncols=1, dpi=300, figsize=(6, 6))
for title, cats in data.items():
# Create pie chart
ax.pie(
cats.values(),
labels=cats.keys(),
autopct=lambda x: pformat(x, sum(cats.values()))
)
ax.set(title=f'{title.title()} Expenses')
# Keep track of created figures
filename = f'{title.replace(" ", "_")}.png'
plt.savefig(filename)
self.figures.append(filename)
else:
logger.debug('Creating sheet of pie charts')
# Clear previous plot
plt.clf()
plt.cla()
fig, ax = plt.subplots(nrows=3, ncols=2, dpi=300, figsize=(6, 9))
for indx, (cat, subcats) in enumerate(data.items()):
row = (indx % 6) // 2
col = indx % 2
# Create pie chart
ax[row, col].pie(
subcats.values(),
labels=subcats.keys(),
autopct=lambda x: pformat(x, sum(subcats.values()))
)
ax[row, col].set(title=f'{cat.title()} Expenses')
# Make new plot if all six figures are filled
if indx % 6 == 5:
logger.warning('All six pie charts are filled, creating a new sheet')
# Keep track of created figures
filename = f'category{indx // 6}.png'
plt.savefig(filename)
self.figures.append(filename)
# Clear previous plot
plt.clf()
plt.cla()
fig, ax = plt.subplots(nrows=2, ncols=3, dpi=300)
else:
logger.warning('Clearing empty pie charts')
# Clear empty plots
for i in range(indx + 1, (indx // 6 + 1) * 6):
row = (i % 6) // 2
col = i % 2
ax[row, col].axis('off')
# Save the last figure
filename = f'category{indx // 6}.png'
plt.savefig(filename)
self.figures.append(filename)
def _generate_pdf(self):
texgen = TexGenerator(self.filename[:-4] + '.tex')
texgen.add_header()
texgen.add_title(self.filename[:-4].replace('_', ' ').title())
texgen.add_section('Expense Charts')
for filename in self.figures:
texgen.add_figure(filename)
texgen.add_section('Expense Data')
# Reorganize data with date first
total = 0
data = dict()
for x in self.expenses:
if x.date not in data.keys():
data[x.date] = dict()
if x.cat not in data[x.date].keys():
data[x.date][x.cat] = dict()
if x.subcat not in data[x.date][x.cat].keys():
data[x.date][x.cat][x.subcat] = dict()
if x.expense not in data[x.date][x.cat][x.subcat].keys():
data[x.date][x.cat][x.subcat][x.expense] = 0
data[x.date][x.cat][x.subcat][x.expense] += x.price
total += x.price
texgen.add_table(data, total)
texgen.add_footer()
texgen.write()
texgen.compile()
def _clean_graphs(self):
logger.debug('Removing created graphs')
for filename in self.figures:
rmrf = subprocess.run(['rm', '-rf', filename])
if rmrf.returncode == 0:
logger.info('Successfully removed graphs')
else:
logger.error('Failed to remove graphs')
self.figures = list()
class DatabaseFiller:
""" Sets DatabaseFiller class.
Consists of 3 private methods :
- __init__()
- _get_products()
- _fill_db()
"""
def __init__(self, database):
""" DatabaseFiller constructor.
For each category and for each nutrition grade, retrieves
corresponding products from Open Food Facts API and adds them to
local database.
"""
# self.database = database
# Creates instances of table manager classes
self.product_manager = ProductManager(database)
self.category_manager = CategoryManager(database)
self.product_category_manager = ProductCategoryManager(database)
self.store_manager = StoreManager(database)
self.product_store_manager = ProductStoreManager(database)
for nutrition_grade in nutrition_grades:
for category in tag_categories:
products = self._get_products(category, nutrition_grade)
self._fill_db(products)
def _get_products(self, category, nutrition_grade):
""" Return a list of dictionnaries of products information.
Connects to Open Food Facts API via get() method from requests
library and sends requests using given criteria for categories
and nutrition grades.
json content from response object contains products information.
"""
criteria = {
'action': 'process',
'json': 1,
'countries': 'France',
'page_size': 100,
'page': 1,
'tagtype_0': 'categories',
'tag_contains_0': 'contains',
'tag_0': category,
'tagtype_1': 'nutrition_grades',
'tag_contains_1': 'contains',
'tag_1': nutrition_grade
}
response = get('https://fr.openfoodfacts.org/cgi/search.pl',
params=criteria)
data = response.json()
# 'products' is a list of dictionnaries
products = data['products']
return products
def _fill_db(self, products):
""" Manages database filling.
Checks for each product whether required data is available or
not.
If so, product is added to local database.
"""
for product in products:
try:
self.code = product['code']
self.name = product['product_name'].strip().capitalize()
self.description = product['generic_name'].capitalize()
brands = (product['brands']).split(',')
self.brand = brands[0].capitalize()
self.url = product['url']
categories_to_strip = (product['categories']).split(',')
self.categories = []
for category in categories_to_strip:
self.categories.append(category.strip().capitalize())
stores_to_strip = (product['stores']).split(',')
self.stores = []
for store in stores_to_strip:
self.stores.append(store.strip().capitalize())
self.nutrition_grade = product['nutrition_grades'].lower()
except KeyError:
print('Missing data', file=open('print_log.txt', 'a'))
if all([
self.code, self.name, self.description, self.brand,
self.url, self.nutrition_grade, self.categories[0],
self.stores[0]
]):
self.product_manager.insert(self.code, self.name,
self.description, self.brand,
self.url, self.nutrition_grade)
for category in self.categories:
self.category_manager.insert(category)
self.product_category_manager.insert(category, self.name)
for store in self.stores:
self.store_manager.insert(store)
self.product_store_manager.insert(store, self.name)