def commit_results(self, queries): """ This function commits the results into the DB in batches. query_num is the number of queires that were executed in the current batch After batch_size is reached, the function re-connects the DB and cursor. """ self.rds_conn.connect_rds() cursor = self.rds_conn.db.cursor() batch_size = 1000 query_num = 0 failed_queries = [] for query in queries: try: cursor.execute(query) # print query except Exception as e: Log.warning( 'Committing to DB failed to due to: {}. Query: {}'.format( e, query)) self.rds_conn.db.commit() failed_queries.append(query) self.rds_conn.connect_rds() cursor = self.rds_conn.db.cursor() continue if query_num > batch_size: self.rds_conn.db.commit() self.rds_conn.connect_rds() cursor = self.rds_conn.db.cursor() query_num = 0 query_num += 1 self.rds_conn.db.commit()
def menu_count(self): kpi_fk = self.get_kpi_fk_by_kpi_type(Consts.MENU_KPI_CHILD) parent_kpi = self.get_kpi_fk_by_kpi_type(Consts.TOTAL_MENU_KPI_SCORE) # we need to save a second set of KPIs with heirarchy for the mobile report kpi_fk_mr = self.get_kpi_fk_by_kpi_type(Consts.MENU_KPI_CHILD_MR) parent_kpi_mr = self.get_kpi_fk_by_kpi_type(Consts.TOTAL_MENU_KPI_SCORE_MR) if self.targets.empty: return try: menu_product_fks = [t for t in self.targets.product_fk.unique().tolist() if pd.notna(t)] except AttributeError: Log.warning('Menu Count targets are corrupt for this store') return filtered_scif = self.scif[self.scif['template_group'].str.contains('Menu')] present_menu_scif_sub_brands = filtered_scif.sub_brand.unique().tolist() passed_products = 0 for product_fk in menu_product_fks: result = 0 sub_brand = self.all_products['sub_brand'][self.all_products['product_fk'] == product_fk].iloc[0] custom_entity_df = self.custom_entity['pk'][self.custom_entity['name'] == sub_brand] if custom_entity_df.empty: custom_entity_pk = -1 else: custom_entity_pk = custom_entity_df.iloc[0] if sub_brand in present_menu_scif_sub_brands: result = 1 passed_products += 1 self.write_to_db(fk=kpi_fk_mr, numerator_id=product_fk, numerator_result=0, denominator_result=0, denominator_id=custom_entity_pk, result=result, score=0, identifier_parent=parent_kpi_mr, identifier_result=kpi_fk_mr, should_enter=True) self.write_to_db(fk=kpi_fk, numerator_id=product_fk, numerator_result=0, denominator_result=0, denominator_id=custom_entity_pk, result=result, score=0) target_products = len(menu_product_fks) self.write_to_db(fk=parent_kpi_mr, numerator_id=self.manufacturer_fk, numerator_result=0, denominator_result=0, denominator_id=self.store_id, result=passed_products, score=0, target=target_products, identifier_result=parent_kpi_mr) self.write_to_db(fk=parent_kpi, numerator_id=self.manufacturer_fk, numerator_result=0, denominator_result=0, denominator_id=self.store_id, result=passed_products, score=0, target=target_products)
def upload_assortment(self): """ This is the main function of the assortment. It does the validation and then upload the assortment. :return: """ Log.debug("Parsing and validating the assortment template") is_valid, invalid_inputs = self.p1_assortment_validator() Log.info("Assortment upload is started") self.upload_store_assortment_file() if not is_valid: Log.warning("Errors were found during the template validation") if invalid_inputs[INVALID_STORES]: Log.warning("The following stores don't exist in the DB: {}" "".format(invalid_inputs[INVALID_STORES])) if invalid_inputs[INVALID_PRODUCTS]: Log.warning("The following products don't exist in the DB: {}" "".format(invalid_inputs[INVALID_PRODUCTS])) Log.info("Assortment upload is finished")