def p1_assortment_validator(self): """ This function validates the store assortment template. It compares the OUTLET_ID (= store_number_1) and the products ean_code to the stores and products from the DB :return: False in case of an error and True in case of a valid template """ raw_data = self.parse_assortment_template() legal_template = True invalid_inputs = {INVALID_STORES: [], INVALID_PRODUCTS: []} valid_stores = self.store_data.loc[ self.store_data['store_number'].isin(raw_data[OUTLET_ID])] if len(valid_stores) != len(raw_data[OUTLET_ID].unique()): invalid_inputs[INVALID_STORES] = list( set(raw_data[OUTLET_ID].unique()) - set(valid_stores['store_number'])) Log.debug("The following stores don't exist in the DB: {}".format( invalid_inputs[INVALID_STORES])) legal_template = False valid_product = self.all_products.loc[self.all_products[EAN_CODE].isin( raw_data[EAN_CODE])] if len(valid_product) != len(raw_data[EAN_CODE].unique()): invalid_inputs[INVALID_PRODUCTS] = list( set(raw_data[EAN_CODE].unique()) - set(valid_product[EAN_CODE])) Log.debug( "The following products don't exist in the DB: {}".format( invalid_inputs[INVALID_PRODUCTS])) legal_template = False return legal_template, invalid_inputs
def upload_assortment(self): """ This is the main function of the assortment. It does the validation and then upload the assortment. :return: """ Log.debug("Parsing and validating the assortment template") is_valid, invalid_inputs = self.p1_assortment_validator() Log.info("Assortment upload is started") self.upload_store_assortment_file() if not is_valid: Log.warning("Errors were found during the template validation") if invalid_inputs[INVALID_STORES]: Log.warning("The following stores don't exist in the DB: {}" "".format(invalid_inputs[INVALID_STORES])) if invalid_inputs[INVALID_PRODUCTS]: Log.warning("The following products don't exist in the DB: {}" "".format(invalid_inputs[INVALID_PRODUCTS])) Log.info("Assortment upload is finished")
def set_end_date_for_irrelevant_assortments(self, stores_list): """ This function sets an end_date to all of the irrelevant stores in the assortment. :param stores_list: List of the stores from the assortment template """ Log.debug("Closing assortment for stores out of template") irrelevant_stores = self.store_data.loc[ ~self.store_data['store_number']. isin(stores_list)]['store_fk'].unique().tolist() current_assortment_stores = self.current_top_skus['store_fk'].unique( ).tolist() stores_to_remove = list( set(irrelevant_stores).intersection( set(current_assortment_stores))) for store in stores_to_remove: query = [ self.get_store_deactivation_query(store, self.deactivate_date) ] self.commit_results(query) Log.debug("Assortment is closed for ({}) stores".format( len(stores_to_remove)))
def upload_store_assortment_file(self): raw_data = self.parse_assortment_template() data = [] list_of_stores = raw_data[OUTLET_ID].unique().tolist() if not self.partial_update: self.set_end_date_for_irrelevant_assortments(list_of_stores) Log.debug("Preparing assortment data for update") store_counter = 0 for store in list_of_stores: store_data = {} store_products = raw_data.loc[raw_data[OUTLET_ID] == store][EAN_CODE].tolist() store_data[store] = store_products data.append(store_data) store_counter += 1 if store_counter % 1000 == 0 or store_counter == len( list_of_stores): Log.debug("Assortment is prepared for {}/{} stores".format( store_counter, len(list_of_stores))) Log.debug("Updating assortment data in DB") store_counter = 0 for store_data in data: self.update_db_from_json(store_data) if self.all_queries: queries = self.merge_insert_queries(self.all_queries) self.commit_results(queries) self.all_queries = [] store_counter += 1 if store_counter % 1000 == 0 or store_counter == len(data): Log.debug( "Assortment is updated in DB for {}/{} stores".format( store_counter, len(data)))
def update_db_from_json(self, data): update_products = set() missing_products = set() store_number = data.keys()[0] if store_number is None: Log.debug("'{}' column or value is missing".format(STORE_NUMBER)) return store_fk = self.get_store_fk(store_number) if store_fk is None: Log.debug( 'Store Number {} does not exist in DB'.format(store_number)) return for key in data[store_number]: validation = False if isinstance(key, (float, int)): validation = True elif isinstance(key, (str, unicode)): validation = True if validation: product_ean_code = str(key).split(',')[-1] product_fk = self.get_product_fk(product_ean_code) if product_fk is None: missing_products.add(product_ean_code) else: update_products.add(product_fk) if missing_products: Log.debug( 'The following EAN Codes for Store Number {} do not exist in DB: {}.' ''.format(store_number, list(missing_products))) queries = [] current_products = self.current_top_skus[ self.current_top_skus['store_fk'] == store_fk]['product_fk'].tolist() products_to_deactivate = tuple( set(current_products).difference(update_products)) products_to_activate = tuple( set(update_products).difference(current_products)) if products_to_deactivate: if len(products_to_deactivate) == 1: queries.append( self.get_deactivation_query( store_fk, "(" + str(products_to_deactivate[0]) + ")", self.deactivate_date)) else: queries.append( self.get_deactivation_query(store_fk, tuple(products_to_deactivate), self.deactivate_date)) for product_fk in products_to_activate: queries.append( self.get_activation_query(store_fk, product_fk, self.activate_date)) self.all_queries.extend(queries) Log.debug( 'Store Number {} - Products to update {}: Deactivated {}, Activated {}' ''.format(store_number, len(update_products), len(products_to_deactivate), len(products_to_activate)))