def p1_assortment_validator(self):
"""
This function validates the store assortment template.
It compares the OUTLET_ID (= store_number_1) and the products ean_code to the stores and products from the DB
:return: False in case of an error and True in case of a valid template
"""
raw_data = self.parse_assortment_template()
legal_template = True
invalid_inputs = {INVALID_STORES: [], INVALID_PRODUCTS: []}
valid_stores = self.store_data.loc[
self.store_data['store_number'].isin(raw_data[OUTLET_ID])]
if len(valid_stores) != len(raw_data[OUTLET_ID].unique()):
invalid_inputs[INVALID_STORES] = list(
set(raw_data[OUTLET_ID].unique()) -
set(valid_stores['store_number']))
Log.debug("The following stores don't exist in the DB: {}".format(
invalid_inputs[INVALID_STORES]))
legal_template = False
valid_product = self.all_products.loc[self.all_products[EAN_CODE].isin(
raw_data[EAN_CODE])]
if len(valid_product) != len(raw_data[EAN_CODE].unique()):
invalid_inputs[INVALID_PRODUCTS] = list(
set(raw_data[EAN_CODE].unique()) -
set(valid_product[EAN_CODE]))
Log.debug(
"The following products don't exist in the DB: {}".format(
invalid_inputs[INVALID_PRODUCTS]))
legal_template = False
return legal_template, invalid_inputs
def upload_assortment(self):
"""
This is the main function of the assortment.
It does the validation and then upload the assortment.
:return:
"""
Log.debug("Parsing and validating the assortment template")
is_valid, invalid_inputs = self.p1_assortment_validator()
Log.info("Assortment upload is started")
self.upload_store_assortment_file()
if not is_valid:
Log.warning("Errors were found during the template validation")
if invalid_inputs[INVALID_STORES]:
Log.warning("The following stores don't exist in the DB: {}"
"".format(invalid_inputs[INVALID_STORES]))
if invalid_inputs[INVALID_PRODUCTS]:
Log.warning("The following products don't exist in the DB: {}"
"".format(invalid_inputs[INVALID_PRODUCTS]))
Log.info("Assortment upload is finished")
def set_end_date_for_irrelevant_assortments(self, stores_list):
"""
This function sets an end_date to all of the irrelevant stores in the assortment.
:param stores_list: List of the stores from the assortment template
"""
Log.debug("Closing assortment for stores out of template")
irrelevant_stores = self.store_data.loc[
~self.store_data['store_number'].
isin(stores_list)]['store_fk'].unique().tolist()
current_assortment_stores = self.current_top_skus['store_fk'].unique(
).tolist()
stores_to_remove = list(
set(irrelevant_stores).intersection(
set(current_assortment_stores)))
for store in stores_to_remove:
query = [
self.get_store_deactivation_query(store, self.deactivate_date)
]
self.commit_results(query)
Log.debug("Assortment is closed for ({}) stores".format(
len(stores_to_remove)))
def upload_store_assortment_file(self):
raw_data = self.parse_assortment_template()
data = []
list_of_stores = raw_data[OUTLET_ID].unique().tolist()
if not self.partial_update:
self.set_end_date_for_irrelevant_assortments(list_of_stores)
Log.debug("Preparing assortment data for update")
store_counter = 0
for store in list_of_stores:
store_data = {}
store_products = raw_data.loc[raw_data[OUTLET_ID] ==
store][EAN_CODE].tolist()
store_data[store] = store_products
data.append(store_data)
store_counter += 1
if store_counter % 1000 == 0 or store_counter == len(
list_of_stores):
Log.debug("Assortment is prepared for {}/{} stores".format(
store_counter, len(list_of_stores)))
Log.debug("Updating assortment data in DB")
store_counter = 0
for store_data in data:
self.update_db_from_json(store_data)
if self.all_queries:
queries = self.merge_insert_queries(self.all_queries)
self.commit_results(queries)
self.all_queries = []
store_counter += 1
if store_counter % 1000 == 0 or store_counter == len(data):
Log.debug(
"Assortment is updated in DB for {}/{} stores".format(
store_counter, len(data)))
def update_db_from_json(self, data):
update_products = set()
missing_products = set()
store_number = data.keys()[0]
if store_number is None:
Log.debug("'{}' column or value is missing".format(STORE_NUMBER))
return
store_fk = self.get_store_fk(store_number)
if store_fk is None:
Log.debug(
'Store Number {} does not exist in DB'.format(store_number))
return
for key in data[store_number]:
validation = False
if isinstance(key, (float, int)):
validation = True
elif isinstance(key, (str, unicode)):
validation = True
if validation:
product_ean_code = str(key).split(',')[-1]
product_fk = self.get_product_fk(product_ean_code)
if product_fk is None:
missing_products.add(product_ean_code)
else:
update_products.add(product_fk)
if missing_products:
Log.debug(
'The following EAN Codes for Store Number {} do not exist in DB: {}.'
''.format(store_number, list(missing_products)))
queries = []
current_products = self.current_top_skus[
self.current_top_skus['store_fk'] ==
store_fk]['product_fk'].tolist()
products_to_deactivate = tuple(
set(current_products).difference(update_products))
products_to_activate = tuple(
set(update_products).difference(current_products))
if products_to_deactivate:
if len(products_to_deactivate) == 1:
queries.append(
self.get_deactivation_query(
store_fk, "(" + str(products_to_deactivate[0]) + ")",
self.deactivate_date))
else:
queries.append(
self.get_deactivation_query(store_fk,
tuple(products_to_deactivate),
self.deactivate_date))
for product_fk in products_to_activate:
queries.append(
self.get_activation_query(store_fk, product_fk,
self.activate_date))
self.all_queries.extend(queries)
Log.debug(
'Store Number {} - Products to update {}: Deactivated {}, Activated {}'
''.format(store_number, len(update_products),
len(products_to_deactivate), len(products_to_activate)))