class CCKHToolBox(CCKHConsts):
LEVEL1 = 1
LEVEL2 = 2
LEVEL3 = 3
def __init__(self, data_provider, output):
self.k_engine = BaseCalculationsScript(data_provider, output)
self.output = output
self.data_provider = data_provider
self.project_name = self.data_provider.project_name
self.session_uid = self.data_provider.session_uid
self.products = self.data_provider[Data.PRODUCTS]
self.all_products = self.data_provider[Data.ALL_PRODUCTS]
self.match_product_in_scene = self.data_provider[Data.MATCHES]
self.visit_date = self.data_provider[Data.VISIT_DATE]
self.session_info = self.data_provider[Data.SESSION_INFO]
self.scene_info = self.data_provider[Data.SCENES_INFO]
self.store_id = self.data_provider[Data.STORE_FK]
self.store_info = self.data_provider[Data.STORE_INFO]
self.store_type = self.store_info['store_type'].iloc[0]
self.scif = self.data_provider[Data.SCENE_ITEM_FACTS]
self.rds_conn = PSProjectConnector(self.project_name, DbUsers.CalculationEng)
self.general_tools = CCKHGENERALToolBox(self.data_provider, self.output)
self.template = CCKHTemplateConsts()
self.kpi_static_data = self.get_kpi_static_data()
self.kpi_results_queries = []
self.commonV2 = CommonV2(self.data_provider)
self.kpi_new_static_data = self.commonV2.get_new_kpi_static_data()
self.manufacturer = int(self.data_provider.own_manufacturer.param_value.values[0])
self.ps_data_provider = PsDataProvider(self.data_provider, self.output)
self.external_targets = self.ps_data_provider.get_kpi_external_targets()
self.assortment = Assortment(self.data_provider, self.output)
self.templates_info = self.external_targets[self.external_targets[CCKHTemplateConsts.TEMPLATE_OPERATION] ==
CCKHTemplateConsts.BASIC_SHEET]
self.visibility_info = self.external_targets[self.external_targets[CCKHTemplateConsts.TEMPLATE_OPERATION]
== CCKHTemplateConsts.VISIBILITY_SHEET]
self.cooler_info = self.external_targets[self.external_targets[CCKHTemplateConsts.TEMPLATE_OPERATION]
== CCKHTemplateConsts.COOLER_SHEET]
def get_kpi_static_data(self):
"""
This function extracts the static KPI data and saves it into one global data frame.
The data is taken from static.kpi / static.atomic_kpi / static.kpi_set.
"""
query = CCKHQueries.get_all_kpi_data()
kpi_static_data = pd.read_sql_query(query, self.rds_conn.db)
return kpi_static_data
def calculate_red_score(self):
"""
This function calculates the KPI results.
"""
scores_dict = {}
results_list_new_db = []
# assortments based calculations for availability
availability_kpi_dict, availability_score_dict = self.get_availability_kpi_data()
results_list_new_db.extend(availability_kpi_dict)
scores_dict.update(availability_score_dict)
# external target based calculations
final_main_child = self.templates_info[self.templates_info['Tested KPI Group'] == self.RED_SCORE].iloc[0]
all_kpi_dict, all_score_dict = self.get_all_kpi_data()
results_list_new_db.extend(all_kpi_dict)
scores_dict.update(all_score_dict)
# aggregation to calculate red score
max_points = sum([score[0] for score in scores_dict.values()])
actual_points = sum([score[1] for score in scores_dict.values()])
red_score = 0 if max_points == 0 else round((actual_points / float(max_points)) * 100, 2)
set_fk = self.kpi_static_data['kpi_set_fk'].values[0]
self.write_to_db_result(set_fk, (actual_points, max_points, red_score), level=self.LEVEL1)
results_list_new_db.append(self.get_new_kpi_dict(self.get_new_kpi_fk(final_main_child), red_score,
red_score, actual_points, max_points,
target=max_points,
weight=actual_points,
identifier_result=self.RED_SCORE,
identifier_parent=CCKHConsts.WEB_HIERARCHY))
results_list_new_db.append(self.get_new_kpi_dict(self.get_new_kpi_by_name(self.RED_SCORE), red_score,
red_score, actual_points, max_points,
target=max_points, weight=actual_points,
identifier_result=CCKHConsts.WEB_HIERARCHY))
self.commonV2.save_json_to_new_tables(results_list_new_db)
self.commonV2.commit_results_data()
def get_availability_kpi_data(self):
availability_results_list = []
scores_dict = {}
availability_assortment_df = self.assortment.calculate_lvl3_assortment()
if availability_assortment_df.empty:
Log.info("Availability KPI: session: {} does not have relevant assortments.".format(self.session_uid))
return [], {}
availability_kpi = self.kpi_new_static_data[self.kpi_new_static_data['type'].str.encode(
HelperConsts.UTF8) == self.template.AVAILABILITY_KPI_TYPE.encode(HelperConsts.UTF8)].iloc[0]
availability_new_kpi_fk = availability_kpi.pk
scores = []
# no need to validate_kpi_run because Availability is 1; seems it was added for the other KPIs
kpi_availability_group = availability_assortment_df.groupby('kpi_fk_lvl2')
for kpi_fk_lvl2, availability_kpi_df in kpi_availability_group:
score, result, threshold = self.calculate_availability(availability_kpi_df)
numerator, denominator, result_new_db = result, threshold, result
if score is not False:
if score is None:
points = 0
else:
points = 1
scores.append((points, score))
atomic_kpi_name = self.kpi_new_static_data[self.kpi_new_static_data['pk'] ==
kpi_fk_lvl2].iloc[0].type
Log.info('Save availability atomic kpi: {}'.format(atomic_kpi_name))
atomic_kpi = self.kpi_static_data[(self.kpi_static_data['kpi_name'].str.encode(HelperConsts.UTF8) ==
self.template.AVAILABILITY_KPI_TYPE.encode(HelperConsts.UTF8)) &
(self.kpi_static_data['atomic_kpi_name'] == atomic_kpi_name)]
atomic_fk = atomic_kpi.iloc[0].atomic_kpi_fk
self.write_to_db_result(atomic_fk, (score, result, threshold, points), level=self.LEVEL3)
child_name = atomic_kpi.atomic_kpi_name.iloc[0]
child_kpi_fk = self.get_new_kpi_by_name(child_name) # kpi fk from new tables
Log.info('Save availability for {} ID: {}'.format(child_name, child_kpi_fk))
availability_results_list.append(self.get_new_kpi_dict(child_kpi_fk, result_new_db, score,
numerator, denominator,
weight=points, target=denominator,
identifier_parent={
'kpi_fk': availability_new_kpi_fk},
))
max_points = sum([score[0] for score in scores])
actual_points = sum([score[0] * score[1] for score in scores])
percentage = 0 if max_points == 0 else round(
(actual_points / float(max_points)) * 100, 2)
kpi_fk = self.kpi_static_data[self.kpi_static_data['kpi_name'].str.encode(HelperConsts.UTF8) ==
self.template.AVAILABILITY_KPI_TYPE.encode(HelperConsts.UTF8)][
'kpi_fk'].values[0]
self.write_to_db_result(kpi_fk, (actual_points, max_points,
percentage), level=self.LEVEL2)
scores_dict[self.template.AVAILABILITY_KPI_TYPE] = (max_points, actual_points)
availability_results_list.append(self.get_new_kpi_dict(availability_new_kpi_fk, percentage, percentage,
actual_points, max_points,
target=max_points,
weight=actual_points,
identifier_result={
'kpi_fk': availability_new_kpi_fk},
identifier_parent=self.RED_SCORE))
return availability_results_list, scores_dict
def get_all_kpi_data(self):
results_list_new_db = []
scores_dict = {}
if self.templates_info.empty:
Log.info("All KPI: session: {} doesnt have relevant external targets".format(self.session_uid))
return [], {}
main_children = self.templates_info[self.templates_info[self.template.KPI_GROUP] == self.RED_SCORE]
for c in xrange(0, len(main_children)):
main_child = main_children.iloc[c]
main_child_kpi_fk = self.get_new_kpi_fk(main_child) # kpi fk from new tables
main_kpi_identifier = self.commonV2.get_dictionary(kpi_fk=main_child_kpi_fk)
if self.validate_store_type(main_child):
children = self.templates_info[self.templates_info
[self.template.KPI_GROUP].str.encode(HelperConsts.UTF8) ==
main_child[self.template.KPI_NAME].encode(HelperConsts.UTF8)]
scores = []
for i in xrange(len(children)):
child = children.iloc[i]
numerator, denominator, result_new_db, numerator_id = 0, 0, 0, None
kpi_weight = self.validate_kpi_run(child)
if kpi_weight is not False:
kpi_type = child[self.template.KPI_TYPE]
result = threshold = None
if kpi_type == self.SURVEY:
score, result, threshold, survey_answer_fk = self.check_survey(child)
threshold = None
numerator, denominator, result_new_db = 1, 1, score * 100
numerator_id = survey_answer_fk
elif kpi_type == self.SHARE_OF_SHELF:
score, result, threshold, result_new_db, numerator, denominator = \
self.calculate_share_of_shelf(child)
elif kpi_type == self.NUMBER_OF_SCENES:
scene_types = self.get_scene_types(child)
result = self.general_tools.calculate_number_of_scenes(
**{SCENE_TYPE_FIELD: scene_types})
numerator, denominator, result_new_db = result, 1, result
score = 1 if result >= 1 else 0
else:
Log.warning("KPI of type '{}' is not supported via assortments".format(kpi_type))
continue
if score is not False:
if score is None:
points = 0
else:
points = float(child[self.template.WEIGHT]
) if kpi_weight is True else kpi_weight
scores.append((points, score))
atomic_fk = self.get_atomic_fk(main_child, child)
self.write_to_db_result(
atomic_fk, (score, result, threshold, points), level=self.LEVEL3)
identifier_parent = main_kpi_identifier
child_name = '{}-{}'.format(child[self.template.TRANSLATION], 'Atomic') \
if main_child[self.template.KPI_NAME] == child[self.template.KPI_NAME] else child[
self.template.TRANSLATION]
child.set_value(self.template.TRANSLATION, child_name)
child_kpi_fk = self.get_new_kpi_fk(child) # kpi fk from new tables
results_list_new_db.append(self.get_new_kpi_dict(child_kpi_fk, result_new_db, score,
numerator, denominator,
weight=points, target=denominator,
identifier_parent=identifier_parent,
numerator_id=numerator_id))
max_points = sum([score[0] for score in scores])
actual_points = sum([score[0] * score[1] for score in scores])
percentage = 0 if max_points == 0 else round(
(actual_points / float(max_points)) * 100, 2)
kpi_name = main_child[self.template.TRANSLATION]
kpi_fk = self.kpi_static_data[self.kpi_static_data['kpi_name'].str.encode(HelperConsts.UTF8) ==
kpi_name.encode(HelperConsts.UTF8)]['kpi_fk'].values[0]
self.write_to_db_result(kpi_fk, (actual_points, max_points,
percentage), level=self.LEVEL2)
scores_dict[kpi_name] = (max_points, actual_points)
results_list_new_db.append(self.get_new_kpi_dict(main_child_kpi_fk, percentage, percentage,
actual_points, max_points,
target=max_points,
weight=actual_points,
identifier_result=main_kpi_identifier,
identifier_parent=self.RED_SCORE))
return results_list_new_db, scores_dict
def validate_store_type(self, params):
"""
This function checks whether or not a KPI is relevant for calculation, by the session's store type.
"""
validation = False
stores = params[self.template.STORE_TYPE]
if not stores:
validation = True
elif isinstance(stores, (str, unicode)):
if stores.upper() == self.template.ALL or self.store_type in stores.split(self.template.SEPARATOR):
validation = True
elif isinstance(stores, list):
if self.store_type in stores:
validation = True
return validation
def validate_kpi_run(self, params):
"""
This function checks whether or not a KPI Atomic needs to be calculated, based on a customized template.
"""
weight = params[self.template.WEIGHT]
if str(weight).isdigit():
validation = True
else:
kpi_group = params[self.template.KPI_GROUP]
if kpi_group == 'Visibility':
custom_template = self.visibility_info
elif kpi_group in ('Ambient Space', 'Cooler Space'):
custom_template = self.cooler_info
else:
return False
condition = (custom_template[self.template.KPI_NAME] == params[self.template.KPI_NAME])
if self.template.KPI_GROUP in custom_template.keys() and kpi_group != 'Visibility':
condition &= (custom_template[self.template.KPI_GROUP]
== params[self.template.KPI_GROUP])
kpi_data = custom_template[condition]
if kpi_data.empty:
return False
try:
weight = \
kpi_data[
kpi_data['store_type'].str.encode(HelperConsts.UTF8) == self.store_type.encode(
HelperConsts.UTF8)][
'Target'].values[0]
validation = float(weight)
except ValueError:
validation = False
except IndexError:
Log.warning("{kpi}: No matching external targets for this session: {sess}".format(
kpi=kpi_group,
sess=self.session_uid))
validation = False
return validation
def get_atomic_fk(self, pillar, params):
"""
This function gets an Atomic KPI's FK out of the template data.
"""
atomic_name = params[self.template.TRANSLATION]
kpi_name = pillar[self.template.TRANSLATION]
atomic_fk = self.kpi_static_data[(self.kpi_static_data['kpi_name'].str.encode(HelperConsts.UTF8) ==
kpi_name.encode(HelperConsts.UTF8)) & (
self.kpi_static_data['atomic_kpi_name'].str.encode(
HelperConsts.UTF8) == atomic_name.encode(HelperConsts.UTF8))][
'atomic_kpi_fk']
if atomic_fk.empty:
return None
return atomic_fk.values[0]
def get_new_kpi_fk(self, params):
"""
This function gets an KPI's FK from new kpi table 'static.kpi_level_2' out of the template data .
"""
kpi_name = params[self.template.TRANSLATION]
return self.get_new_kpi_by_name(kpi_name)
def get_new_kpi_by_name(self, kpi_name):
kpi_fk = self.kpi_new_static_data[self.kpi_new_static_data['type'].str.encode(HelperConsts.UTF8) ==
kpi_name.encode(HelperConsts.UTF8)]['pk']
if kpi_fk.empty:
return None
return kpi_fk.values[0]
def get_scene_types(self, params):
"""
This function extracts the relevant scene types (==additional_attribute_1) from the template.
"""
scene_types = params[self.template.SCENE_TYPE]
if not scene_types or (isinstance(scene_types, (str, unicode)) and scene_types.upper() == self.template.ALL):
return None
return scene_types
def calculate_availability(self, availability_kpi_df):
"""
This function calculates Availability typed Atomics from a customized template, and saves the results to the DB.
"""
all_targets = availability_kpi_df.target.unique()
if not all_targets:
return False, False, False
target = float(all_targets[0])
total_facings_count = availability_kpi_df.facings.sum()
score = 1 if total_facings_count >= target else 0
return score, total_facings_count, target
def check_survey(self, params):
"""
This function calculates Survey typed Atomics, and saves the results to the DB.
"""
survey_id = int(float(params[self.template.SURVEY_ID]))
target_answer = params[self.template.SURVEY_ANSWER]
survey_answer, survey_answer_fk = self.general_tools.get_survey_answer(survey_data=('question_fk', survey_id))
score = 1 if survey_answer == target_answer else 0
return score, survey_answer, target_answer, survey_answer_fk
def calculate_share_of_shelf(self, params):
"""
This function calculates Facings Share of Shelf typed Atomics, and saves the results to the DB.
"""
if params[self.template.SOS_NUMERATOR].startswith('~'):
sos_filters = {params[self.template.SOS_ENTITY]: (params[self.template.SOS_NUMERATOR][1:],
self.general_tools.EXCLUDE_FILTER)}
else:
sos_filters = {params[self.template.SOS_ENTITY]: params[self.template.SOS_NUMERATOR]}
general_filters = {}
scene_types = self.get_scene_types(params)
if isinstance(scene_types, (str, unicode)):
scene_types = scene_types.split(self.template.SEPARATOR)
if scene_types:
general_filters[SCENE_TYPE_FIELD] = scene_types
products_to_exclude = params[self.template.PRODUCT_TYPES_TO_EXCLUDE]
if products_to_exclude:
general_filters['product_type'] = (products_to_exclude.split(self.template.SEPARATOR),
self.general_tools.EXCLUDE_FILTER)
numerator_result = self.general_tools.calculate_availability(
**dict(sos_filters, **general_filters))
denominator_result = self.general_tools.calculate_availability(**general_filters)
if denominator_result == 0:
result = 0
else:
result = round((numerator_result / float(denominator_result)) * 100, 2)
if params[self.template.TARGET]:
target = float(params[self.template.TARGET]) * 100
score = 1 if result >= target else 0
else:
score = target = None
result_string = '{0}% ({1}/{2})'.format(result, int(numerator_result), int(denominator_result))
return score, result_string, target, result, numerator_result, denominator_result
def write_to_db_result(self, fk, score, level):
"""
This function creates the result data frame of every KPI (atomic KPI/KPI/KPI set),
and appends the insert SQL query into the queries' list, later to be written to the DB.
"""
attributes = self.create_attributes_dict(fk, score, level)
if level == self.LEVEL1:
table = KPS_RESULT
elif level == self.LEVEL2:
table = KPK_RESULT
elif level == self.LEVEL3:
table = KPI_RESULT
else:
return
query = insert(attributes, table)
self.kpi_results_queries.append(query)
def create_attributes_dict(self, fk, score, level):
"""
This function creates a data frame with all attributes needed for saving in KPI results tables.
"""
if level == self.LEVEL1:
score_2, score_3, score_1 = score
kpi_set_name = self.kpi_static_data[self.kpi_static_data['kpi_set_fk']
== fk]['kpi_set_name'].values[0]
attributes = pd.DataFrame([(kpi_set_name, self.session_uid, self.store_id, self.visit_date.isoformat(),
format(score_1, '.2f'), score_2, score_3, fk)],
columns=['kps_name', 'session_uid', 'store_fk', 'visit_date', 'score_1',
'score_2', 'score_3', 'kpi_set_fk'])
elif level == self.LEVEL2:
score_2, score_3, score = score
kpi_name = self.kpi_static_data[self.kpi_static_data['kpi_fk']
== fk]['kpi_name'].values[0]
attributes = pd.DataFrame([(self.session_uid, self.store_id, self.visit_date.isoformat(),
fk, kpi_name, score, score_2, score_3)],
columns=['session_uid', 'store_fk', 'visit_date', 'kpi_fk', 'kpk_name',
'score', 'score_2', 'score_3'])
elif level == self.LEVEL3:
score, result, threshold, weight = score
data = self.kpi_static_data[self.kpi_static_data['atomic_kpi_fk'] == fk]
atomic_kpi_name = data['atomic_kpi_name'].values[0]
kpi_fk = data['kpi_fk'].values[0]
kpi_set_name = self.kpi_static_data[self.kpi_static_data['atomic_kpi_fk']
== fk]['kpi_set_name'].values[0]
attributes = pd.DataFrame([(atomic_kpi_name, self.session_uid, kpi_set_name, self.store_id,
self.visit_date.isoformat(), datetime.utcnow().isoformat(),
score, kpi_fk, fk, threshold, result, weight)],
columns=['display_text', 'session_uid', 'kps_name', 'store_fk', 'visit_date',
'calculation_time', 'score', 'kpi_fk', 'atomic_kpi_fk', 'threshold',
'result', 'kpi_weight'])
else:
attributes = pd.DataFrame()
return attributes.to_dict()
@log_runtime('Saving to DB')
def commit_results_data(self):
"""
This function writes all KPI results to the DB, and commits the changes.
"""
cur = self.rds_conn.db.cursor()
delete_queries = CCKHQueries.get_delete_session_results_query(self.session_uid)
for query in delete_queries:
cur.execute(query)
for query in self.kpi_results_queries:
cur.execute(query)
self.rds_conn.db.commit()
def get_new_kpi_dict(self, kpi_fk, result, score, numerator_result, denominator_result,
score_after_action=0, weight=None, target=None, identifier_parent=None,
identifier_result=None, numerator_id=None):
"""
This function gets all kpi info and add the relevant numerator_id and denominator_id and return a dictionary
with the passed data.
:param kpi_fk: pk of kpi
:param result
:param score
:param numerator_result
:param denominator_result
:param weight
:param target
:param identifier_parent
:param identifier_result
:param numerator_id
:param score_after_action
:returns dict in format of db result
"""
numerator_id = self.manufacturer if numerator_id is None else numerator_id
denominator_id = self.store_id
return {'fk': kpi_fk,
SessionResultsConsts.NUMERATOR_ID: numerator_id,
SessionResultsConsts.DENOMINATOR_ID: denominator_id,
SessionResultsConsts.DENOMINATOR_RESULT: denominator_result,
SessionResultsConsts.NUMERATOR_RESULT: numerator_result,
SessionResultsConsts.RESULT: result, SessionResultsConsts.SCORE: score,
SessionResultsConsts.TARGET: target, SessionResultsConsts.WEIGHT: weight,
'identifier_parent': identifier_parent, 'identifier_result': identifier_result,
'score_after_actions': score_after_action, 'should_enter': True,
}
class PSAPAC_SAND3ToolBox:
# Gsk Japan kpis
# DEFAULT_TARGET = {ProductsConsts.BRAND_FK: [-1], 'shelves': ["1,2,3"], 'block_target': [80], 'brand_target': [100], 'position_target': [80]}
def __init__(self, data_provider, output):
self.output = output
self.data_provider = data_provider
self.common = Common(self.data_provider)
self.project_name = self.data_provider.project_name
self.session_uid = self.data_provider.session_uid
self.products = self.data_provider[Data.PRODUCTS]
self.all_products = self.data_provider[Data.ALL_PRODUCTS]
self.match_product_in_scene = self.data_provider[Data.MATCHES]
self.visit_date = self.data_provider[Data.VISIT_DATE]
self.session_info = self.data_provider[Data.SESSION_INFO]
self.scene_info = self.data_provider[Data.SCENES_INFO]
self.store_id = self.data_provider[Data.STORE_FK]
self.scif = self.data_provider[Data.SCENE_ITEM_FACTS]
self.rds_conn = PSProjectConnector(self.project_name,
DbUsers.CalculationEng)
self.kpi_static_data = self.common.get_kpi_static_data()
self.kpi_results_queries = []
self.set_up_template = pd.read_excel(os.path.join(
os.path.dirname(os.path.realpath(__file__)), '..', 'Data',
'gsk_set_up.xlsx'),
sheet_name='Functional KPIs',
keep_default_na=False)
self.gsk_generator = GSKGenerator(self.data_provider, self.output,
self.common, self.set_up_template)
self.blocking_generator = Block(self.data_provider)
self.assortment = self.gsk_generator.get_assortment_data_provider()
self.store_info = self.data_provider['store_info']
self.store_fk = self.data_provider[StoreInfoConsts.STORE_FK]
self.ps_data_provider = PsDataProvider(self.data_provider, self.output)
self.targets = self.ps_data_provider.get_kpi_external_targets(
key_fields=Consts.KEY_FIELDS, data_fields=Consts.DATA_FIELDS)
self.own_manufacturer = self.get_manufacturer
self.set_up_data = {
(Consts.PLN_BLOCK, Const.KPI_TYPE_COLUMN): Const.NO_INFO,
(Consts.POSITION_SCORE, Const.KPI_TYPE_COLUMN): Const.NO_INFO,
(Consts.ECAPS_FILTER_IDENT, Const.KPI_TYPE_COLUMN): Const.NO_INFO,
(Consts.PLN_MSL, Const.KPI_TYPE_COLUMN): Const.NO_INFO,
("GSK_PLN_LSOS_SCORE", Const.KPI_TYPE_COLUMN): Const.NO_INFO,
(Consts.POSM, Const.KPI_TYPE_COLUMN): Const.NO_INFO
}
@property
def get_manufacturer(self):
return int(self.data_provider.own_manufacturer[
self.data_provider.own_manufacturer['param_name'] ==
'manufacturer_id']['param_value'].iloc[0])
def main_calculation(self, *args, **kwargs):
"""
This function calculates the KPI results.Global functions and local functions
"""
# global kpis
assortment_store_dict = self.gsk_generator.availability_store_function(
)
self.common.save_json_to_new_tables(assortment_store_dict)
assortment_category_dict = self.gsk_generator.availability_category_function(
)
self.common.save_json_to_new_tables(assortment_category_dict)
assortment_subcategory_dict = self.gsk_generator.availability_subcategory_function(
)
self.common.save_json_to_new_tables(assortment_subcategory_dict)
linear_sos_dict = self.gsk_generator.gsk_global_linear_sos_by_sub_category_function(
)
self.common.save_json_to_new_tables(linear_sos_dict)
linear_sos_dict = self.gsk_generator.gsk_global_linear_sos_by_category_function(
)
self.common.save_json_to_new_tables(linear_sos_dict)
linear_sos_dict = self.gsk_generator.gsk_global_linear_sos_whole_store_function(
)
self.common.save_json_to_new_tables(linear_sos_dict)
# # local kpis
for kpi in Consts.KPI_DICT.keys():
self.gsk_generator.tool_box.extract_data_set_up_file(
kpi, self.set_up_data, Consts.KPI_DICT)
results_ecaps = self.gsk_ecaps_kpis()
self.common.save_json_to_new_tables(results_ecaps)
self.get_store_target() # choosing the policy
if self.targets.empty:
Log.warning('There is no target policy matching this store')
else:
results_compliance = self.gsk_compliance()
self.common.save_json_to_new_tables(results_compliance)
results_pos = self.gsk_pos_kpis()
self.common.save_json_to_new_tables(results_pos)
self.common.commit_results_data()
return
def position_shelf(self, brand_fk, policy, df):
"""
:param brand_fk :
:param policy : dictionary that contains {
'shelves':"1 ,2 ,4 ,5" (or any other string of numbers separate by ','),
'position_target': 80 (or any other percentage you want the score to
reach)
}
:param df: data frame that contains columns MatchesConsts.SHELF_NUMBER , "brand kf"
:returns tuple of (result,score,numerator,denominator)
result = number of products from brand_fk in shelves / number of products from brand_fk ,
score = if result reach position target 100 else 0 ,
numerator = number of products from brand_fk in shelves
denominator = number of products from brand_fk
"""
if (Consts.SHELVES
not in policy.keys()) or policy[Consts.SHELVES].empty:
Log.warning(
'This sessions have external targets but doesnt have value for shelves position'
)
return 0, 0, 0, 0, 0
if isinstance(policy[Consts.SHELVES].iloc[0], list):
shelf_from_bottom = [
int(shelf) for shelf in policy[Consts.SHELVES].iloc[0]
]
else:
shelf_from_bottom = [
int(shelf)
for shelf in policy[Consts.SHELVES].iloc[0].split(",")
]
threshold = policy[Consts.POSITION_TARGET].iloc[0]
brand_df = df[df[ProductsConsts.BRAND_FK] == brand_fk]
shelf_df = brand_df[brand_df[MatchesConsts.SHELF_NUMBER].isin(
shelf_from_bottom)]
numerator = shelf_df.shape[0]
denominator = brand_df.shape[0]
result = float(numerator) / float(denominator)
score = 1 if (result * 100) >= threshold else 0
return result, score, numerator, denominator, threshold
def lsos_score(self, brand, policy):
"""
:param brand : pk of brand
:param policy : dictionary of { 'brand_target' : lsos number you want to reach}
This function uses the lsos_in whole_store global calculation.
it takes the result of the parameter 'brand' according to the policy set target and results.
:return result,score,target
result : result of this brand lsos
score : result / brand_target ,
target : branf_target
"""
df = pd.merge(self.match_product_in_scene,
self.all_products[Const.PRODUCTS_COLUMNS],
how='left',
on=[MatchesConsts.PRODUCT_FK])
df = pd.merge(self.scif[Const.SCIF_COLUMNS],
df,
how='right',
right_on=[ScifConsts.SCENE_FK, ScifConsts.PRODUCT_FK],
left_on=[ScifConsts.SCENE_ID, ScifConsts.PRODUCT_FK])
if df.empty:
Log.warning('match_product_in_scene is empty ')
return 0, 0, 0
df = self.gsk_generator.tool_box.tests_by_template(
'GSK_PLN_LSOS_SCORE', df, self.set_up_data)
if df is None:
Log.warning('match_product_in_scene is empty ')
return 0, 0, 0
result = self.gsk_generator.tool_box.calculate_sos(
df, {ProductsConsts.BRAND_FK: brand}, {}, Const.LINEAR)[0]
target = policy['brand_target'].iloc[0]
score = float(result) / float(target)
return result, score, target
def brand_blocking(self, brand, policy):
"""
:param brand : pk of brand
:param policy : dictionary of { 'block_target' : number you want to reach}
:return result : 1 if there is a block answer set_up_data conditions else 0
"""
templates = self.set_up_data[(Const.SCENE_TYPE, Consts.PLN_BLOCK)]
template_name = {
ScifConsts.TEMPLATE_NAME: templates
} if templates else None # figure out which template name should I use
ignore_empty = False
# taking from params from set up info
stacking_param = False if not self.set_up_data[(
Const.INCLUDE_STACKING, Consts.PLN_BLOCK)] else True # false
population_parameters = {
ProductsConsts.BRAND_FK: [brand],
ProductsConsts.PRODUCT_TYPE: [ProductTypeConsts.SKU]
}
if self.set_up_data[(Const.INCLUDE_OTHERS, Consts.PLN_BLOCK)]:
population_parameters[ProductsConsts.PRODUCT_TYPE].append(
Const.OTHER)
if self.set_up_data[(Const.INCLUDE_IRRELEVANT, Consts.PLN_BLOCK)]:
population_parameters[ProductsConsts.PRODUCT_TYPE].append(
Const.IRRELEVANT)
if self.set_up_data[(Const.INCLUDE_EMPTY, Consts.PLN_BLOCK)]:
population_parameters[ProductsConsts.PRODUCT_TYPE].append(
Const.EMPTY)
else:
ignore_empty = True
if self.set_up_data[(Const.CATEGORY_INCLUDE,
Consts.PLN_BLOCK)]: # category_name
population_parameters[ProductsConsts.CATEGORY] = self.set_up_data[(
Const.CATEGORY_INCLUDE, Consts.PLN_BLOCK)]
if self.set_up_data[(Const.SUB_CATEGORY_INCLUDE,
Consts.PLN_BLOCK)]: # sub_category_name
population_parameters[
ProductsConsts.SUB_CATEGORY] = self.set_up_data[(
Const.SUB_CATEGORY_INCLUDE, Consts.PLN_BLOCK)]
# from Data file
target = float(policy['block_target'].iloc[0]) / float(100)
result = self.blocking_generator.network_x_block_together(
location=template_name,
population=population_parameters,
additional={
'minimum_block_ratio': target,
'calculate_all_scenes': True,
'ignore_empty': ignore_empty,
'include_stacking': stacking_param,
'check_vertical_horizontal': True,
'minimum_facing_for_block': 1
})
result.sort_values('facing_percentage', ascending=False, inplace=True)
score = 0 if result[result['is_block']].empty else 1
numerator = 0 if result.empty else result['block_facings'].iloc[0]
denominator = 0 if result.empty else result['total_facings'].iloc[0]
return score, target, numerator, denominator
def msl_assortment(self, kpi_fk, kpi_name):
"""
:param kpi_fk : name of level 3 assortment kpi
:param kpi_name: GSK_PLN_MSL_SCORE assortment , or GSK_ECAPS assortment
:return kpi_results : data frame of assortment products of the kpi, product's availability,
product details.
filtered by set up
"""
lvl3_assort, filter_scif = self.gsk_generator.tool_box.get_assortment_filtered(
self.set_up_data, kpi_name)
if lvl3_assort is None or lvl3_assort.empty:
return None
kpi_assortment_fk = self.common.get_kpi_fk_by_kpi_type(kpi_fk)
kpi_results = lvl3_assort[lvl3_assort['kpi_fk_lvl3'] ==
kpi_assortment_fk] # general assortment
kpi_results = pd.merge(kpi_results,
self.all_products[Const.PRODUCTS_COLUMNS],
how='left',
on=ProductsConsts.PRODUCT_FK)
kpi_results = kpi_results[kpi_results[
ProductsConsts.SUBSTITUTION_PRODUCT_FK].isnull()]
return kpi_results
def pln_ecaps_score(self, brand, assortment):
"""
:param brand : pk of desired brand
:param assortment : data frame of assortment products of the kpi, product's availability,
product details. filtered by set up
besides result of lvl2_assortment function writing level 3 assortment product presence
results
:return numerator : how many products available out of the granular groups
denominator : how many products in assortment groups
result : (numerator/denominator)*100
results : array of dictionary, each dict contains the result details
"""
identifier_parent = self.common.get_dictionary(
brand_fk=brand,
kpi_fk=self.common.get_kpi_fk_by_kpi_type(Consts.ECAP_ALL_BRAND))
results = []
kpi_ecaps_product = self.common.get_kpi_fk_by_kpi_type(
Consts.PRODUCT_PRESENCE)
ecaps_assortment_fk = self.common.get_kpi_fk_by_kpi_type(
Consts.PLN_ASSORTMENT_KPI)
if assortment.empty:
return 0, 0, 0, results
brand_results = assortment[assortment[ProductsConsts.BRAND_FK] ==
brand] # only assortment of desired brand
for result in brand_results.itertuples():
if (math.isnan(result.in_store)) | (result.kpi_fk_lvl3 !=
ecaps_assortment_fk):
score = self.gsk_generator.tool_box.result_value_pk(
Const.EXTRA)
result_num = 1
else:
score = self.gsk_generator.tool_box.result_value_pk(Const.OOS) if result.in_store == 0 else \
self.gsk_generator.tool_box.result_value_pk(Const.DISTRIBUTED)
result_num = result.in_store
last_status = self.gsk_generator.tool_box.get_last_status(
kpi_ecaps_product, result.product_fk)
# score = result.in_store * 100
results.append({
'fk': kpi_ecaps_product,
SessionResultsConsts.NUMERATOR_ID: result.product_fk,
SessionResultsConsts.DENOMINATOR_ID: self.store_fk,
SessionResultsConsts.DENOMINATOR_RESULT: 1,
SessionResultsConsts.NUMERATOR_RESULT: result_num,
SessionResultsConsts.RESULT: score,
SessionResultsConsts.SCORE: last_status,
'identifier_parent': identifier_parent,
'identifier_result': 1,
'should_enter': True
})
if 'total' not in self.assortment.LVL2_HEADERS or 'passes' not in self.assortment.LVL2_HEADERS:
self.assortment.LVL2_HEADERS.extend(['total', 'passes'])
lvl2 = self.assortment.calculate_lvl2_assortment(brand_results)
if lvl2.empty:
return 0, 0, 0, results # in case of no assortment return 0
result = round(
np.divide(float(lvl2.iloc[0].passes), float(lvl2.iloc[0].total)),
4)
return lvl2.iloc[0].passes, lvl2.iloc[0].total, result, results
def pln_msl_summary(self, brand, assortment):
"""
:param brand : pk of desired brand
:param assortment : data frame of assortment products of the kpi, product's availability,
product details. filtered by set up
:return numerator : how many products available out of the granular groups
denominator : how many products in assortment groups
result : (numerator/denominator)*100
results : array of dictionary, each dict contains the result details
"""
if assortment is None or assortment.empty:
return 0, 0, 0, 0
brand_results = assortment[assortment[ProductsConsts.BRAND_FK] ==
brand] # only assortment of desired brand
if 'total' not in self.assortment.LVL2_HEADERS or 'passes' not in self.assortment.LVL2_HEADERS:
self.assortment.LVL2_HEADERS.extend(['total', 'passes'])
lvl2 = self.assortment.calculate_lvl2_assortment(brand_results)
if lvl2.empty:
return 0, 0, 0, 0 # in case of no assortment return 0
result = round(
np.divide(float(lvl2.iloc[0].passes), float(lvl2.iloc[0].total)),
4)
return lvl2.iloc[0].passes, lvl2.iloc[0].total, result, lvl2.iloc[
0].assortment_group_fk
def get_store_target(self):
"""
Function checks which policies out of self.target are relevant to this store visit according to store
attributes.
"""
parameters_dict = {StoreInfoConsts.STORE_NUMBER_1: 'store_number'}
for store_param, target_param in parameters_dict.items():
if target_param in self.targets.columns:
if self.store_info[store_param][0] is None:
if self.targets.empty or self.targets[
self.targets[target_param] != ''].empty:
continue
else:
self.targets.drop(self.targets.index, inplace=True)
self.targets = self.targets[
(self.targets[target_param] ==
self.store_info[store_param][0].encode(HelperConsts.UTF8))
| (self.targets[target_param] == '')]
def gsk_compliance(self):
"""
Function calculate compliance score for each brand based on :
position score, brand-assortment score,
block score ,lsos score.
Also calculate compliance summary score - average of brands compliance scores
"""
results_df = []
df = self.scif
# kpis
kpi_block_fk = self.common.get_kpi_fk_by_kpi_type(Consts.PLN_BLOCK)
kpi_position_fk = self.common.get_kpi_fk_by_kpi_type(
Consts.POSITION_SCORE)
kpi_lsos_fk = self.common.get_kpi_fk_by_kpi_type(Consts.PLN_LSOS)
kpi_msl_fk = self.common.get_kpi_fk_by_kpi_type(Consts.PLN_MSL)
kpi_compliance_brands_fk = self.common.get_kpi_fk_by_kpi_type(
Consts.COMPLIANCE_ALL_BRANDS)
kpi_compliance_summary_fk = self.common.get_kpi_fk_by_kpi_type(
Consts.COMPLIANCE_SUMMARY)
identifier_compliance_summary = self.common.get_dictionary(
kpi_fk=kpi_compliance_summary_fk)
# targets
block_target = 0.25
posit_target = 0.25
lsos_target = 0.25
msl_target = 0.25
total_brand_score = 0
counter_brands = 0
# assortment_lvl3 msl df initialize
self.gsk_generator.tool_box.extract_data_set_up_file(
Consts.PLN_MSL, self.set_up_data, Consts.KPI_DICT)
assortment_msl = self.msl_assortment(Const.DISTRIBUTION,
Consts.PLN_MSL)
# set data frame to find position shelf
df_position_score = pd.merge(self.match_product_in_scene,
self.all_products,
on=ProductsConsts.PRODUCT_FK)
df_position_score = pd.merge(
self.scif[Const.SCIF_COLUMNS],
df_position_score,
how='right',
right_on=[ScifConsts.SCENE_FK, ProductsConsts.PRODUCT_FK],
left_on=[ScifConsts.SCENE_ID, ScifConsts.PRODUCT_FK])
df_position_score = self.gsk_generator.tool_box.tests_by_template(
Consts.POSITION_SCORE, df_position_score, self.set_up_data)
if not self.set_up_data[(Const.INCLUDE_STACKING,
Consts.POSITION_SCORE)]:
df_position_score = df_position_score if df_position_score is None else df_position_score[
df_position_score[MatchesConsts.STACKING_LAYER] == 1]
# calculate all brands if template doesnt require specific brand else only for specific brands
template_brands = self.set_up_data[(Const.BRANDS_INCLUDE,
Consts.PLN_BLOCK)]
brands = df[df[ProductsConsts.BRAND_NAME].isin(template_brands)][ProductsConsts.BRAND_FK].unique() if \
template_brands else df[ProductsConsts.BRAND_FK].dropna().unique()
for brand in brands:
policy = self.targets[self.targets[ProductsConsts.BRAND_FK] ==
brand]
if policy.empty:
Log.warning('There is no target policy matching brand'
) # adding brand name
return results_df
identifier_parent = self.common.get_dictionary(
brand_fk=brand, kpi_fk=kpi_compliance_brands_fk)
# msl_kpi
msl_numerator, msl_denominator, msl_result, msl_assortment_group = self.pln_msl_summary(
brand, assortment_msl)
msl_score = msl_result * msl_target
results_df.append({
'fk': kpi_msl_fk,
SessionResultsConsts.NUMERATOR_ID: brand,
SessionResultsConsts.DENOMINATOR_ID: self.store_fk,
SessionResultsConsts.DENOMINATOR_RESULT: msl_denominator,
SessionResultsConsts.NUMERATOR_RESULT: msl_numerator,
SessionResultsConsts.RESULT: msl_result,
SessionResultsConsts.SCORE: msl_score,
SessionResultsConsts.TARGET: msl_target,
SessionResultsConsts.CONTEXT_ID: msl_assortment_group,
'identifier_parent': identifier_parent,
'should_enter': True
})
# lsos kpi
lsos_numerator, lsos_result, lsos_denominator = self.lsos_score(
brand, policy)
lsos_result = 1 if lsos_result > 1 else lsos_result
lsos_score = lsos_result * lsos_target
results_df.append({
'fk': kpi_lsos_fk,
SessionResultsConsts.NUMERATOR_ID: brand,
SessionResultsConsts.DENOMINATOR_ID: self.store_fk,
SessionResultsConsts.DENOMINATOR_RESULT: lsos_denominator,
SessionResultsConsts.NUMERATOR_RESULT: lsos_numerator,
SessionResultsConsts.RESULT: lsos_result,
SessionResultsConsts.SCORE: lsos_score,
SessionResultsConsts.TARGET: lsos_target,
'identifier_parent': identifier_parent,
SessionResultsConsts.WEIGHT: lsos_denominator,
'should_enter': True
})
# block_score
block_result, block_benchmark, numerator_block, block_denominator = self.brand_blocking(
brand, policy)
block_score = round(block_result * block_target, 4)
results_df.append({
'fk':
kpi_block_fk,
SessionResultsConsts.NUMERATOR_ID:
brand,
SessionResultsConsts.DENOMINATOR_ID:
self.store_fk,
SessionResultsConsts.DENOMINATOR_RESULT:
block_denominator,
SessionResultsConsts.NUMERATOR_RESULT:
numerator_block,
SessionResultsConsts.RESULT:
block_result,
SessionResultsConsts.SCORE:
block_score,
SessionResultsConsts.TARGET:
block_target,
'identifier_parent':
identifier_parent,
'should_enter':
True,
SessionResultsConsts.WEIGHT: (block_benchmark * 100)
})
# position score
if df_position_score is not None:
position_result, position_score, position_num, position_den, position_benchmark = self.position_shelf(
brand, policy, df_position_score)
else:
position_result, position_score, position_num, position_den, position_benchmark = 0, 0, 0, 0, 0
position_score = round(position_score * posit_target, 4)
results_df.append({
'fk': kpi_position_fk,
SessionResultsConsts.NUMERATOR_ID: brand,
SessionResultsConsts.DENOMINATOR_ID: self.store_fk,
SessionResultsConsts.DENOMINATOR_RESULT: position_den,
SessionResultsConsts.NUMERATOR_RESULT: position_num,
SessionResultsConsts.RESULT: position_result,
SessionResultsConsts.SCORE: position_score,
SessionResultsConsts.TARGET: posit_target,
'identifier_parent': identifier_parent,
'should_enter': True,
SessionResultsConsts.WEIGHT: position_benchmark
})
# compliance score per brand
compliance_score = round(
position_score + block_score + lsos_score + msl_score, 4)
results_df.append({
'fk': kpi_compliance_brands_fk,
SessionResultsConsts.NUMERATOR_ID: self.own_manufacturer,
SessionResultsConsts.DENOMINATOR_ID: brand,
SessionResultsConsts.DENOMINATOR_RESULT: 1,
SessionResultsConsts.NUMERATOR_RESULT: compliance_score,
SessionResultsConsts.RESULT: compliance_score,
SessionResultsConsts.SCORE: compliance_score,
'identifier_parent': identifier_compliance_summary,
'identifier_result': identifier_parent,
'should_enter': True
})
# counter and sum updates
total_brand_score = round(total_brand_score + compliance_score, 4)
counter_brands = counter_brands + 1
if counter_brands == 0:
return results_df
# compliance summary
average_brand_score = round(total_brand_score / counter_brands, 4)
results_df.append({
'fk': kpi_compliance_summary_fk,
SessionResultsConsts.NUMERATOR_ID: self.own_manufacturer,
SessionResultsConsts.DENOMINATOR_ID: self.store_fk,
SessionResultsConsts.DENOMINATOR_RESULT: counter_brands,
SessionResultsConsts.NUMERATOR_RESULT: total_brand_score,
SessionResultsConsts.RESULT: average_brand_score,
SessionResultsConsts.SCORE: average_brand_score,
'identifier_result': identifier_compliance_summary
})
return results_df
def gsk_ecaps_kpis(self):
"""
Function calculate for each brand ecaps score, and for all brands together set ecaps summary score
:return
results_df : array of dictionary, each dict contains kpi's result details
"""
results_df = []
kpi_ecaps_brands_fk = self.common.get_kpi_fk_by_kpi_type(
Consts.ECAP_ALL_BRAND)
kpi_ecaps_summary_fk = self.common.get_kpi_fk_by_kpi_type(
Consts.ECAP_SUMMARY)
identifier_ecaps_summary = self.common.get_dictionary(
kpi_fk=kpi_ecaps_summary_fk)
total_brand_score = 0
assortment_display = self.msl_assortment(Consts.PLN_ASSORTMENT_KPI,
Consts.ECAPS_FILTER_IDENT)
if assortment_display is None or assortment_display.empty:
return results_df
template_brands = self.set_up_data[(Const.BRANDS_INCLUDE,
Consts.ECAPS_FILTER_IDENT)]
brands = assortment_display[assortment_display[ProductsConsts.BRAND_NAME].isin(template_brands)][
ProductsConsts.BRAND_FK].unique() if \
template_brands else assortment_display[ProductsConsts.BRAND_FK].dropna().unique()
for brand in brands:
numerator_res, denominator_res, result, product_presence_df = self.pln_ecaps_score(
brand, assortment_display)
results_df.extend(product_presence_df)
identifier_all_brand = self.common.get_dictionary(
brand_fk=brand,
kpi_fk=self.common.get_kpi_fk_by_kpi_type(
Consts.ECAP_ALL_BRAND))
results_df.append({
'fk': kpi_ecaps_brands_fk,
SessionResultsConsts.NUMERATOR_ID: self.own_manufacturer,
SessionResultsConsts.DENOMINATOR_ID: brand,
SessionResultsConsts.DENOMINATOR_RESULT: denominator_res,
SessionResultsConsts.NUMERATOR_RESULT: numerator_res,
SessionResultsConsts.RESULT: result,
SessionResultsConsts.SCORE: result,
'identifier_parent': identifier_ecaps_summary,
'identifier_result': identifier_all_brand,
'should_enter': True
})
total_brand_score = total_brand_score + result
if len(
brands
) > 0: # don't want to show result in case of there are no brands relevan to the template
result_summary = round(total_brand_score / len(brands), 4)
results_df.append({
'fk':
kpi_ecaps_summary_fk,
SessionResultsConsts.NUMERATOR_ID:
self.own_manufacturer,
SessionResultsConsts.DENOMINATOR_ID:
self.store_fk,
SessionResultsConsts.DENOMINATOR_RESULT:
len(brands),
SessionResultsConsts.NUMERATOR_RESULT:
total_brand_score,
SessionResultsConsts.RESULT:
result_summary,
SessionResultsConsts.SCORE:
result_summary,
'identifier_result':
identifier_ecaps_summary
})
return results_df
def gsk_pos_kpis(self):
"""
Function calculate POSM Distribution
:return
- results : array of dictionary, each dict contains kpi's result details
"""
results = []
OOS = 1
DISTRIBUTED = 2
self.gsk_generator.tool_box.extract_data_set_up_file(
Consts.POSM, self.set_up_data, Consts.KPI_DICT)
assortment_pos = self.msl_assortment(Consts.POSM_SKU, Consts.POSM)
kpi_gsk_pos_distribution_store_fk = self.common.get_kpi_fk_by_kpi_type(
Consts.GSK_POS_DISTRIBUTION_STORE)
kpi_gsk_pos_distribution_brand_fk = self.common.get_kpi_fk_by_kpi_type(
Consts.GSK_POS_DISTRIBUTION_BRAND)
kpi_gsk_pos_distribution_sku_fk = self.common.get_kpi_fk_by_kpi_type(
Consts.GSK_POS_DISTRIBUTION_SKU)
if assortment_pos is None or assortment_pos.empty:
Log.info(
"Assortment df is empty. GSK_POS_DISTRIBUTION Kpis are not calculated"
)
return results
# Calculate KPI : GSK_POS_DISTRIBUTION_STORE
assortment_pos['in_store'] = assortment_pos['in_store'].astype('int')
Log.info(
"Dropping duplicate product_fks accros multiple-granular groups")
Log.info("Before : {}".format(len(assortment_pos)))
assortment_pos = assortment_pos.drop_duplicates(
subset=[ProductsConsts.PRODUCT_FK])
Log.info("After : {}".format(len(assortment_pos)))
numerator_res = len(assortment_pos[assortment_pos['in_store'] == 1])
denominator_res = len(assortment_pos)
result = round(
(numerator_res /
float(denominator_res)), 4) if denominator_res != 0 else 0
results.append({
'fk': kpi_gsk_pos_distribution_store_fk,
SessionResultsConsts.NUMERATOR_ID: self.own_manufacturer,
SessionResultsConsts.DENOMINATOR_ID: self.store_fk,
SessionResultsConsts.NUMERATOR_RESULT: numerator_res,
SessionResultsConsts.DENOMINATOR_RESULT: denominator_res,
SessionResultsConsts.RESULT: result,
SessionResultsConsts.SCORE: result,
# 'identifier_parent': identifier_ecaps_summary,
'identifier_result': "Gsk_Pos_Distribution_Store",
'should_enter': True
})
# Calculate KPI: GSK_POS_DISTRIBUTION_BRAND
brands_group = assortment_pos.groupby([ProductsConsts.BRAND_FK])
for brand, assortment_pos_by_brand in brands_group:
numerator_res = len(assortment_pos_by_brand[
assortment_pos_by_brand['in_store'] == 1])
denominator_res = len(assortment_pos_by_brand)
result = round(
(numerator_res /
float(denominator_res)), 4) if denominator_res != 0 else 0
results.append({
'fk':
kpi_gsk_pos_distribution_brand_fk,
SessionResultsConsts.NUMERATOR_ID:
int(brand),
SessionResultsConsts.DENOMINATOR_ID:
self.store_fk,
SessionResultsConsts.NUMERATOR_RESULT:
numerator_res,
SessionResultsConsts.DENOMINATOR_RESULT:
denominator_res,
SessionResultsConsts.RESULT:
result,
SessionResultsConsts.SCORE:
result,
'identifier_parent':
"Gsk_Pos_Distribution_Store",
'identifier_result':
"Gsk_Pos_Distribution_Brand_" + str(int(brand)),
'should_enter':
True
})
for idx, each_product in assortment_pos_by_brand.iterrows():
product_fk = each_product[ProductsConsts.PRODUCT_FK]
result = 1 if int(each_product['in_store']) == 1 else 0
result_status = DISTRIBUTED if result == 1 else OOS
last_status = self.gsk_generator.tool_box.get_last_status(
kpi_gsk_pos_distribution_sku_fk, product_fk)
results.append({
'fk':
kpi_gsk_pos_distribution_sku_fk,
SessionResultsConsts.NUMERATOR_ID:
product_fk,
SessionResultsConsts.DENOMINATOR_ID:
self.store_fk,
SessionResultsConsts.NUMERATOR_RESULT:
result,
SessionResultsConsts.DENOMINATOR_RESULT:
1,
SessionResultsConsts.RESULT:
result_status,
SessionResultsConsts.SCORE:
last_status,
'identifier_parent':
"Gsk_Pos_Distribution_Brand_" + str(int(brand)),
'identifier_result':
"Gsk_Pos_Distribution_SKU_" + str(int(product_fk)),
'should_enter':
True
})
return results
class ToolBox(GlobalSessionToolBox):
def __init__(self, data_provider, output):
GlobalSessionToolBox.__init__(self, data_provider, output)
self.adjacency = Adjancency(data_provider)
self.block = Block(data_provider)
self.kpi_static_data = self.common.get_kpi_static_data()
self.ps_data_provider = PsDataProvider(data_provider)
self._scene_types = None
self.external_targets = self.ps_data_provider.get_kpi_external_targets(
)
@property
def scene_types(self):
if not self._scene_types:
self._scene_types = self.scif['template_fk'].unique().tolist()
return self._scene_types
def main_calculation(self):
custom_kpis = self.kpi_static_data[
(self.kpi_static_data['kpi_calculation_stage_fk'] == 3)
& (self.kpi_static_data['valid_from'] <= self.visit_date) &
((self.kpi_static_data['valid_until']).isnull() |
(self.kpi_static_data['valid_until'] >= self.visit_date))]
for kpi in custom_kpis.itertuples():
kpi_function = self.get_kpi_function_by_family_fk(
kpi.kpi_family_fk)
kpi_function(kpi.pk)
return
@run_for_every_scene_type
def calculate_presence(self, kpi_fk, template_fk=None):
config = self.get_external_target_data_by_kpi_fk(kpi_fk)
if config.empty or (template_fk is None):
return
result_df = self.scif[
self.scif[config.numerator_param].isin(config.numerator_value)
& (self.scif['template_fk'] == template_fk)]
numerator_id = self.get_brand_fk_from_brand_name(
config.numerator_value[0])
result = 0 if result_df.empty else 1
self.write_to_db(kpi_fk,
numerator_id=numerator_id,
denominator_id=template_fk,
result=result)
return
@run_for_every_scene_type
def calculate_shelf_location(self, kpi_fk, template_fk=None):
config = self.get_external_target_data_by_kpi_fk(kpi_fk)
shelf_location = config.shelf_location
if config.empty or (template_fk is None):
return
relevant_scene_fks = self.scif[
self.scif['template_fk'] ==
template_fk]['scene_fk'].unique().tolist()
relevant_matches = self.matches[self.matches['scene_fk'].isin(
relevant_scene_fks)]
shelves = relevant_matches.groupby(
'bay_number',
as_index=False)['shelf_number'].max()['shelf_number'].mean()
products_df = self.scif[
(self.scif[config.numerator_param].isin(config.numerator_value))
& (self.scif['template_fk'] == template_fk)]
products_list = products_df['product_fk'].unique().tolist()
if shelf_location == 'top':
shelf_matches = relevant_matches[
(relevant_matches['product_fk'].isin(products_list))
& (relevant_matches['shelf_number'] <= (shelves / 3))]
elif shelf_location == 'middle_bottom':
shelf_matches = relevant_matches[
(relevant_matches['product_fk'].isin(products_list))
& (relevant_matches['shelf_number'] > (shelves / 3))]
else:
shelf_matches = pd.DataFrame()
numerator_id = self.get_brand_fk_from_brand_name(
config.numerator_value[0])
result = 0 if shelf_matches.empty else 1
self.write_to_db(kpi_fk,
numerator_id=numerator_id,
denominator_id=template_fk,
result=result)
@run_for_every_scene_type
def calculate_blocking(self, kpi_fk, template_fk=None):
config = self.get_external_target_data_by_kpi_fk(kpi_fk)
if config.empty or (template_fk is None):
return
location = {'template_fk': template_fk}
blocks = self.block.network_x_block_together(
{config.numerator_param: config.numerator_value},
location,
additional={'check_vertical_horizontal': True})
if not blocks.empty:
blocks = blocks[blocks['is_block']]
orientation = config.orientation
if orientation and orientation is not pd.np.nan:
blocks = blocks[blocks['orientation'] == orientation]
numerator_id = self.get_brand_fk_from_brand_name(
config.numerator_value[0])
result = 0 if blocks.empty else 1
self.write_to_db(kpi_fk,
numerator_id=numerator_id,
denominator_id=template_fk,
result=result)
@run_for_every_scene_type
def calculate_adjacency(self, kpi_fk, template_fk=None):
config = self.get_external_target_data_by_kpi_fk(kpi_fk)
if config.empty or (template_fk is None):
return
location = {'template_fk': template_fk}
anchor_pks = \
self.scif[self.scif[config.anchor_param].isin(config.anchor_value)]['product_fk'].unique().tolist()
tested_pks = \
self.scif[self.scif[config.tested_param].isin(config.tested_value)]['product_fk'].unique().tolist()
# handle populations that are not mutually exclusive
tested_pks = [x for x in tested_pks if x not in anchor_pks]
population = {
'anchor_products': {
'product_fk': anchor_pks
},
'tested_products': {
'product_fk': tested_pks
}
}
# this function is only needed until the adjacency function is enhanced to not crash when an empty population
# is provided
if self.check_population_exists(population, template_fk):
try:
adj_df = self.adjacency.network_x_adjacency_calculation(
population, location, {
'minimum_facings_adjacent': 1,
'minimum_block_ratio': 0,
'minimum_facing_for_block': 1,
'include_stacking': True
})
except AttributeError:
Log.info(
"Error calculating adjacency for kpi_fk {} template_fk {}".
format(kpi_fk, template_fk))
return
if adj_df.empty:
result = 0
else:
result = 1 if not adj_df[adj_df['is_adj']].empty else 0
else:
result = 0
numerator_id = self.get_brand_fk_from_brand_name(
config.anchor_value[0])
self.write_to_db(kpi_fk,
numerator_id=numerator_id,
denominator_id=template_fk,
result=result)
return
@run_for_every_scene_type
def calculate_brand_facings(self, kpi_fk, template_fk=None):
relevant_scif = self.scif[self.scif['template_fk'] == template_fk]
denominator_results = relevant_scif.groupby(
'Customer Category', as_index=False)[[
'facings'
]].sum().rename(columns={'facings': 'denominator_result'})
numerator_result = relevant_scif.groupby(
['brand_fk', 'Customer Category'], as_index=False)[[
'facings'
]].sum().rename(columns={'facings': 'numerator_result'})
results = numerator_result.merge(denominator_results)
results['result'] = (results['numerator_result'] /
results['denominator_result'])
results['result'].fillna(0, inplace=True)
for index, row in results.iterrows():
relevant_perfetti_product_fk = self.get_product_fk_from_perfetti_category(
row['Customer Category'])
self.write_to_db(fk=kpi_fk,
numerator_id=row['brand_fk'],
denominator_id=relevant_perfetti_product_fk,
numerator_result=row['numerator_result'],
denominator_result=row['denominator_result'],
context_id=template_fk,
result=row['result'],
score=row['result'])
def get_kpi_function_by_family_fk(self, kpi_family_fk):
if kpi_family_fk == 19:
return self.calculate_presence
elif kpi_family_fk == 20:
return self.calculate_adjacency
elif kpi_family_fk == 21:
return self.calculate_blocking
elif kpi_family_fk == 22:
return self.calculate_shelf_location
elif kpi_family_fk == 23:
return self.calculate_brand_facings
def get_external_target_data_by_kpi_fk(self, kpi_fk):
return self.external_targets[self.external_targets['kpi_fk'] ==
kpi_fk].iloc[0]
def get_brand_fk_from_brand_name(self, brand_name):
return self.all_products[self.all_products['brand_name'] ==
brand_name]['brand_fk'].iloc[0]
def get_product_fk_from_perfetti_category(self, perfetti_category):
try:
return self.all_products[self.all_products['Customer Category'] ==
perfetti_category]['product_fk'].iloc[0]
except IndexError:
return None
def check_population_exists(self, population, template_fk):
relevant_scif = self.scif[self.scif['template_fk'] == template_fk]
anchor_scif = relevant_scif[relevant_scif['product_fk'].isin(
population['anchor_products']['product_fk'])]
tested_scif = relevant_scif[relevant_scif['product_fk'].isin(
population['tested_products']['product_fk'])]
if anchor_scif.empty or tested_scif.empty:
return False
else:
return True
class ALTRIAUSToolBox:
LEVEL1 = 1
LEVEL2 = 2
LEVEL3 = 3
def __init__(self, data_provider, output):
self.output = output
self.data_provider = data_provider
self.common = Common(self.data_provider)
self.common_v2 = CommonV2(self.data_provider)
self.project_name = self.data_provider.project_name
self.session_uid = self.data_provider.session_uid
self.products = self.data_provider[Data.PRODUCTS]
self.all_products = self.data_provider[Data.ALL_PRODUCTS]
self.match_product_in_scene = self.data_provider[Data.MATCHES]
self.visit_date = self.data_provider[Data.VISIT_DATE]
self.session_info = self.data_provider[Data.SESSION_INFO]
self.scene_info = self.data_provider[Data.SCENES_INFO]
self.store_id = self.data_provider[Data.STORE_FK]
self.scif = self.data_provider[Data.SCENE_ITEM_FACTS]
self.template_info = self.data_provider.all_templates
self.rds_conn = ProjectConnector(self.project_name,
DbUsers.CalculationEng)
self.ps_data_provider = PsDataProvider(self.data_provider)
self.match_product_in_probe_state_reporting = self.ps_data_provider.get_match_product_in_probe_state_reporting(
)
self.thresholds_and_results = {}
self.result_df = []
self.writing_to_db_time = datetime.timedelta(0)
self.kpi_results_queries = []
self.potential_products = {}
self.shelf_square_boundaries = {}
self.average_shelf_values = {}
self.kpi_static_data = self.common.get_kpi_static_data()
self.kpi_results_queries = []
self.all_template_data = parse_template(TEMPLATE_PATH, "KPI")
self.spacing_template_data = parse_template(TEMPLATE_PATH, "Spacing")
self.empty_data_template = parse_template(TEMPLATE_PATH, "Empty")
self.fixture_width_template = pd.read_excel(FIXTURE_WIDTH_TEMPLATE,
"Fixture Width",
dtype=pd.Int64Dtype())
self.facings_to_feet_template = pd.read_excel(FIXTURE_WIDTH_TEMPLATE,
"Conversion Table",
dtype=pd.Int64Dtype())
self.header_positions_template = pd.read_excel(FIXTURE_WIDTH_TEMPLATE,
"Header Positions")
self.flip_sign_positions_template = pd.read_excel(
FIXTURE_WIDTH_TEMPLATE, "Flip Sign Positions")
self.custom_entity_data = self.ps_data_provider.get_custom_entities(
1005)
self.ignore_stacking = False
self.facings_field = 'facings' if not self.ignore_stacking else 'facings_ign_stack'
self.INCLUDE_FILTER = 1
self.assortment = Assortment(self.data_provider,
output=self.output,
ps_data_provider=self.ps_data_provider)
self.store_assortment = self.assortment.get_lvl3_relevant_ass()
self.kpi_new_static_data = self.common.get_new_kpi_static_data()
try:
self.mpis = self.match_product_in_scene.merge(self.products, on='product_fk', suffixes=['', '_p']) \
.merge(self.scene_info, on='scene_fk', suffixes=['', '_s']) \
.merge(self.template_info, on='template_fk', suffixes=['', '_t'])
except KeyError:
Log.warning('MPIS cannot be generated!')
return
self.adp = AltriaDataProvider(self.data_provider)
self.active_kpis = self._get_active_kpis()
self.external_targets = self.ps_data_provider.get_kpi_external_targets(
)
self.survey_dot_com_collected_this_session = self._get_survey_dot_com_collected_value(
)
def main_calculation(self, *args, **kwargs):
"""
This function calculates the KPI results.
"""
self.calculate_signage_locations_and_widths('Cigarettes')
self.calculate_signage_locations_and_widths('Smokeless')
self.calculate_register_type()
self.calculate_age_verification()
self.calculate_juul_availability()
self.calculate_assortment()
self.calculate_vapor_kpis()
self.calculate_empty_brand()
self.calculate_facings_by_scene_type()
kpi_set_fk = 2
set_name = \
self.kpi_static_data.loc[self.kpi_static_data['kpi_set_fk'] == kpi_set_fk]['kpi_set_name'].values[0]
template_data = self.all_template_data.loc[
self.all_template_data['KPI Level 1 Name'] == set_name]
try:
if set_name and not set(
template_data['Scene Types to Include'].values[0].encode(
).split(', ')) & set(
self.scif['template_name'].unique().tolist()):
Log.info('Category {} was not captured'.format(
template_data['category'].values[0]))
return
except Exception as e:
Log.info(
'KPI Set {} is not defined in the template'.format(set_name))
for i, row in template_data.iterrows():
try:
kpi_name = row['KPI Level 2 Name']
if kpi_name in KPI_LEVEL_2_cat_space:
# scene_type = [s for s in row['Scene_Type'].encode().split(', ')]
kpi_type = row['KPI Type']
scene_type = row['scene_type']
if row['Param1'] == 'Category' or 'sub_category':
category = row['Value1']
if kpi_type == 'category_space':
kpi_set_fk = \
self.kpi_new_static_data.loc[self.kpi_new_static_data['type'] == kpi_type]['pk'].values[0]
self.calculate_category_space(
kpi_set_fk,
kpi_name,
category,
scene_types=scene_type)
except Exception as e:
Log.info('KPI {} calculation failed due to {}'.format(
kpi_name.encode('utf-8'), e))
continue
return
def _get_survey_dot_com_collected_value(self):
try:
sales_rep_fk = self.session_info['s_sales_rep_fk'].iloc[0]
except IndexError:
sales_rep_fk = 0
return int(sales_rep_fk) == 209050
def _get_active_kpis(self):
active_kpis = self.kpi_new_static_data[
(self.kpi_new_static_data['kpi_calculation_stage_fk'] == 3)
& (self.kpi_new_static_data['valid_from'] <= self.visit_date) &
((self.kpi_new_static_data['valid_until']).isnull() |
(self.kpi_new_static_data['valid_until'] >= self.visit_date))]
return active_kpis
def calculate_vapor_kpis(self):
category = 'Vapor'
relevant_scif = self.scif[self.scif['template_name'] ==
'JUUL Merchandising']
if relevant_scif.empty:
Log.info('No products found for {} category'.format(category))
return
relevant_scif = relevant_scif[
(relevant_scif['category'].isin([category, 'POS']))
& (relevant_scif['brand_name'] == 'Juul')]
if relevant_scif.empty:
return
relevant_product_pks = relevant_scif[
relevant_scif['product_type'] ==
'SKU']['product_fk'].unique().tolist()
relevant_scene_id = self.get_most_frequent_scene(relevant_scif)
product_mpis = self.mpis[
(self.mpis['product_fk'].isin(relevant_product_pks))
& (self.mpis['scene_fk'] == relevant_scene_id)]
if product_mpis.empty:
Log.info('No products found for {} category'.format(category))
return
self.calculate_total_shelves(product_mpis, category, product_mpis)
longest_shelf = \
product_mpis[product_mpis['shelf_number'] ==
self.get_longest_shelf_number(product_mpis,
max_shelves_from_top=999)].sort_values(by='rect_x',
ascending=True)
if longest_shelf.empty or longest_shelf.isnull().all().all():
Log.warning(
'The {} category items are in a non-standard location. The {} category will not be calculated.'
.format(category, category))
return
relevant_pos = pd.DataFrame()
self.calculate_fixture_width(relevant_pos, longest_shelf, category)
return
def calculate_assortment(self):
if self.scif.empty or self.store_assortment.empty:
Log.warning(
'Unable to calculate assortment: SCIF or store assortment is empty'
)
return
grouped_scif = self.scif.groupby('product_fk',
as_index=False)['facings'].sum()
assortment_with_facings = \
pd.merge(self.store_assortment, grouped_scif, how='left', on='product_fk')
assortment_with_facings.loc[:, 'facings'] = assortment_with_facings[
'facings'].fillna(0)
for product in assortment_with_facings.itertuples():
score = 1 if product.facings > 0 else 0
self.common_v2.write_to_db_result(
product.kpi_fk_lvl3,
numerator_id=product.product_fk,
denominator_id=product.assortment_fk,
numerator_result=product.facings,
result=product.facings,
score=score)
number_of_skus_present = len(
assortment_with_facings[assortment_with_facings['facings'] > 0])
score = 1 if number_of_skus_present > 0 else 0
kpi_fk = assortment_with_facings['kpi_fk_lvl2'].iloc[0]
assortment_group_fk = assortment_with_facings[
'assortment_group_fk'].iloc[0]
self.common_v2.write_to_db_result(
kpi_fk,
numerator_id=assortment_group_fk,
numerator_result=number_of_skus_present,
denominator_result=len(assortment_with_facings),
result=number_of_skus_present,
score=score)
def calculate_facings_by_scene_type(self):
kpi_name = 'FACINGS_BY_SCENE_TYPE'
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type(kpi_name)
if kpi_name not in self.active_kpis['type'].unique().tolist():
return
config = self.get_external_target_data_by_kpi_fk(kpi_fk)
product_types = config.product_type
template_names = config.template_name
relevant_mpis = self.mpis[
(self.mpis['product_type'].isin(product_types))
& (self.mpis['template_name'].isin(template_names))]
smart_attribute_data = \
self.adp.get_match_product_in_probe_state_values(relevant_mpis['probe_match_fk'].unique().tolist())
relevant_mpis = pd.merge(relevant_mpis,
smart_attribute_data,
on='probe_match_fk',
how='left')
relevant_mpis['match_product_in_probe_state_fk'].fillna(0,
inplace=True)
relevant_mpis = relevant_mpis.groupby(
['product_fk', 'template_fk', 'match_product_in_probe_state_fk'],
as_index=False)['scene_match_fk'].count()
relevant_mpis.rename(columns={'scene_match_fk': 'facings'},
inplace=True)
for row in relevant_mpis.itertuples():
self.common_v2.write_to_db_result(
kpi_fk,
numerator_id=row.product_fk,
denominator_id=row.template_fk,
context_id=row.match_product_in_probe_state_fk,
result=row.facings)
return
def calculate_register_type(self):
if self.survey_dot_com_collected_this_session:
return
relevant_scif = self.scif[
(self.scif['product_type'].isin(['POS', 'Other']))
& (self.scif['category'] == 'POS Machinery')]
if relevant_scif.empty:
result = 0
product_fk = 0
else:
result = 1
product_fk = relevant_scif['product_fk'].iloc[0]
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('Register Type')
self.common_v2.write_to_db_result(kpi_fk,
numerator_id=product_fk,
denominator_id=self.store_id,
result=result)
def calculate_age_verification(self):
if self.survey_dot_com_collected_this_session:
return
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('Age Verification')
relevant_scif = self.scif[self.scif['brand_name'].isin(
['Age Verification'])]
if relevant_scif.empty:
result = 0
product_fk = 0
self.common_v2.write_to_db_result(kpi_fk,
numerator_id=product_fk,
denominator_id=self.store_id,
result=result)
else:
result = 1
for product_fk in relevant_scif['product_fk'].unique().tolist():
self.common_v2.write_to_db_result(kpi_fk,
numerator_id=product_fk,
denominator_id=self.store_id,
result=result)
return
def calculate_empty_brand(self):
product_type = ['Empty']
for i, row in self.empty_data_template.iterrows():
kpi_name = row['KPI Level 2 Name']
param1 = row['Param1']
value1 = row['Value1']
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_name(kpi_name)
if value1.find(',') != -1:
value1 = value1.split(',')
for value in value1:
relevant_scif = self.scif[(self.scif[param1] == value) & (
self.scif['product_type'].isin(product_type))]
if relevant_scif.empty:
pass
else:
result = empty_facings = relevant_scif['facings'].sum()
brand_fk = relevant_scif['brand_fk'].iloc[0]
self.common_v2.write_to_db_result(
kpi_fk,
numerator_id=brand_fk,
numerator_result=empty_facings,
denominator_id=self.store_id,
result=result,
score=0)
def calculate_juul_availability(self):
relevant_scif = self.scif[(self.scif['brand_name'].isin(['Juul']))
& (self.scif['product_type'].isin(['POS']))]
juul_pos = relevant_scif['product_fk'].unique().tolist()
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('Juul POS Availability')
if not juul_pos:
return
result = 1
for product_fk in juul_pos:
self.common_v2.write_to_db_result(kpi_fk,
numerator_id=product_fk,
denominator_id=self.store_id,
result=result)
def calculate_category_space(self,
kpi_set_fk,
kpi_name,
category,
scene_types=None):
template = self.all_template_data.loc[
(self.all_template_data['KPI Level 2 Name'] == kpi_name)
& (self.all_template_data['Value1'] == category)]
kpi_template = template.loc[template['KPI Level 2 Name'] == kpi_name]
if kpi_template.empty:
return None
kpi_template = kpi_template.iloc[0]
values_to_check = []
filters = {
'template_name': scene_types,
'category': kpi_template['Value1']
}
if kpi_template['Value1'] in CATEGORIES:
category_att = 'category'
if kpi_template['Value1']:
values_to_check = self.all_products.loc[
self.all_products[category_att] ==
kpi_template['Value1']][category_att].unique().tolist()
for primary_filter in values_to_check:
filters[kpi_template['Param1']] = primary_filter
new_kpi_name = self.kpi_name_builder(kpi_name, **filters)
result = self.calculate_category_space_length(
new_kpi_name, **filters)
filters['Category'] = kpi_template['KPI Level 2 Name']
score = result
numerator_id = self.products['category_fk'][
self.products['category'] == kpi_template['Value1']].iloc[0]
self.common_v2.write_to_db_result(kpi_set_fk,
numerator_id=numerator_id,
numerator_result=999,
result=score,
score=score)
def calculate_category_space_length(self,
kpi_name,
threshold=0.5,
retailer=None,
exclude_pl=False,
**filters):
"""
:param threshold: The ratio for a bay to be counted as part of a category.
:param filters: These are the parameters which the data frame is filtered by.
:return: The total shelf width (in mm) the relevant facings occupy.
"""
try:
filtered_scif = self.scif[self.get_filter_condition(
self.scif, **filters)]
space_length = 0
bay_values = []
for scene in filtered_scif['scene_fk'].unique().tolist():
scene_matches = self.mpis[self.mpis['scene_fk'] == scene]
scene_filters = filters
scene_filters['scene_fk'] = scene
for bay in scene_matches['bay_number'].unique().tolist():
bay_total_linear = scene_matches.loc[
(scene_matches['bay_number'] == bay)
& (scene_matches['stacking_layer'] == 1) &
(scene_matches['status']
== 1)]['width_mm_advance'].sum()
scene_filters['bay_number'] = bay
tested_group_linear = scene_matches[
self.get_filter_condition(scene_matches,
**scene_filters)]
tested_group_linear_value = tested_group_linear[
'width_mm_advance'].sum()
if tested_group_linear_value:
bay_ratio = tested_group_linear_value / float(
bay_total_linear)
else:
bay_ratio = 0
if bay_ratio >= threshold:
category = filters['category']
max_facing = scene_matches.loc[
(scene_matches['bay_number'] == bay)
& (scene_matches['stacking_layer'] == 1
)]['facing_sequence_number'].max()
shelf_length = self.spacing_template_data.query(
'Category == "' + category + '" & Low <= "' +
str(max_facing) + '" & High >= "' +
str(max_facing) + '"')
shelf_length = int(shelf_length['Size'].iloc[-1])
bay_values.append(shelf_length)
space_length += shelf_length
except Exception as e:
Log.info('Linear Feet calculation failed due to {}'.format(e))
space_length = 0
return space_length
def get_filter_condition(self, df, **filters):
"""
:param df: The data frame to be filters.
:param filters: These are the parameters which the data frame is filtered by.
Every parameter would be a tuple of the value and an include/exclude flag.
INPUT EXAMPLE (1): manufacturer_name = ('Diageo', DIAGEOAUPNGROGENERALToolBox.INCLUDE_FILTER)
INPUT EXAMPLE (2): manufacturer_name = 'Diageo'
:return: a filtered Scene Item Facts data frame.
"""
if not filters:
return df['pk'].apply(bool)
if self.facings_field in df.keys():
filter_condition = (df[self.facings_field] > 0)
else:
filter_condition = None
for field in filters.keys():
if field in df.keys():
if isinstance(filters[field], tuple):
value, exclude_or_include = filters[field]
else:
value, exclude_or_include = filters[
field], self.INCLUDE_FILTER
if not value:
continue
if not isinstance(value, list):
value = [value]
if exclude_or_include == self.INCLUDE_FILTER:
condition = (df[field].isin(value))
elif exclude_or_include == self.EXCLUDE_FILTER:
condition = (~df[field].isin(value))
elif exclude_or_include == self.CONTAIN_FILTER:
condition = (df[field].str.contains(value[0], regex=False))
for v in value[1:]:
condition |= df[field].str.contains(v, regex=False)
else:
continue
if filter_condition is None:
filter_condition = condition
else:
filter_condition &= condition
else:
Log.warning('field {} is not in the Data Frame'.format(field))
return filter_condition
def kpi_name_builder(self, kpi_name, **filters):
"""
This function builds kpi name according to naming convention
"""
for filter in filters.keys():
if filter == 'template_name':
continue
kpi_name = kpi_name.replace('{' + filter + '}',
str(filters[filter]))
kpi_name = kpi_name.replace("'", "\'")
return kpi_name
def calculate_signage_locations_and_widths(self, category):
excluded_types = ['Other', 'Irrelevant', 'Empty']
relevant_scif = self.scif[self.scif['template_name'] ==
'Tobacco Merchandising Space']
if relevant_scif.empty:
Log.info('No products found for {} category'.format(category))
return
# need to include scene_id from previous relevant_scif
# also need to split this shit up into different categories, i.e. smokeless, cigarettes
# need to figure out how to deal with POS from smokeless being included with cigarette MPIS
# get relevant SKUs from the cigarettes category
relevant_scif = relevant_scif[relevant_scif['category'].isin(
[category, 'POS'])]
relevant_product_pks = relevant_scif[
relevant_scif['product_type'] ==
'SKU']['product_fk'].unique().tolist()
relevant_pos_pks = \
relevant_scif[(relevant_scif['product_type'] == 'POS') &
~(relevant_scif['brand_name'] == 'Age Verification') &
~(relevant_scif['product_name'] == 'General POS Other')]['product_fk'].unique().tolist()
other_product_and_pos_pks = \
relevant_scif[relevant_scif['product_type'].isin(excluded_types)]['product_fk'].tolist()
relevant_scene_id = self.get_most_frequent_scene(relevant_scif)
product_mpis = self.mpis[
(self.mpis['product_fk'].isin(relevant_product_pks))
& (self.mpis['scene_fk'] == relevant_scene_id)]
if product_mpis.empty:
Log.info('No products found for {} category'.format(category))
return
self.calculate_total_shelves(product_mpis, category)
longest_shelf = \
product_mpis[product_mpis['shelf_number'] ==
self.get_longest_shelf_number(product_mpis)].sort_values(by='rect_x', ascending=True)
if longest_shelf.empty or longest_shelf.isnull().all().all():
Log.warning(
'The {} category items are in a non-standard location. The {} category will not be calculated.'
.format(category, category))
return
# demarcation_line = longest_shelf['rect_y'].median() old method, had bugs due to longest shelf being lower
demarcation_line = product_mpis['rect_y'].min()
exclusion_line = -9999
excluded_mpis = self.mpis[
~(self.mpis['product_fk'].isin(relevant_pos_pks +
relevant_product_pks +
other_product_and_pos_pks))
& (self.mpis['rect_x'] < longest_shelf['rect_x'].max()) &
(self.mpis['rect_x'] > longest_shelf['rect_x'].min()) &
(self.mpis['scene_fk'] == relevant_scene_id) &
(self.mpis['rect_y'] < demarcation_line)]
# we need this line for when SCIF and MPIS don't match
excluded_mpis = excluded_mpis[~excluded_mpis['product_type'].
isin(excluded_types)]
if not excluded_mpis.empty:
exclusion_line = excluded_mpis['rect_y'].max()
# we need to get POS stuff that falls within the x-range of the longest shelf (which is limited by category)
# we also need to account for the fact that the images suck, so we're going to add/subtract 5% of the
# max/min values to allow for POS items that fall slightly out of the shelf length range
correction_factor = 0.05
correction_value = (longest_shelf['rect_x'].max() -
longest_shelf['rect_x'].min()) * correction_factor
pos_mpis = self.mpis[
(self.mpis['product_fk'].isin(relevant_pos_pks))
& (self.mpis['rect_x'] <
(longest_shelf['rect_x'].max() + correction_value)) &
(self.mpis['rect_x'] >
(longest_shelf['rect_x'].min() - correction_value)) &
(self.mpis['scene_fk'] == relevant_scene_id)]
# DO NOT SET TO TRUE WHEN DEPLOYING
# debug flag displays polygon_mask graph DO NOT SET TO TRUE WHEN DEPLOYING
# DO NOT SET TO TRUE WHEN DEPLOYING
relevant_pos = self.adp.get_products_contained_in_displays(
pos_mpis, y_axis_threshold=35, debug=False)
if relevant_pos.empty:
Log.warning(
'No polygon mask was generated for {} category - cannot compute KPIs'
.format(category))
# we need to attempt to calculate fixture width, even if there's no polygon mask
self.calculate_fixture_width(relevant_pos, longest_shelf, category)
return
relevant_pos = relevant_pos[[
'product_fk', 'product_name', 'left_bound', 'right_bound',
'center_x', 'center_y'
]]
relevant_pos = relevant_pos.reindex(
columns=relevant_pos.columns.tolist() +
['type', 'width', 'position'])
relevant_pos['width'] = \
relevant_pos.apply(lambda row: self.get_length_of_pos(row, longest_shelf, category), axis=1)
relevant_pos['type'] = \
relevant_pos['center_y'].apply(lambda x: 'Header' if exclusion_line < x < demarcation_line else 'Flip Sign')
relevant_pos = relevant_pos.sort_values(['center_x'], ascending=True)
relevant_pos = self.remove_duplicate_pos_tags(relevant_pos)
# generate header positions
if category == 'Cigarettes':
number_of_headers = len(
relevant_pos[relevant_pos['type'] == 'Header'])
if number_of_headers > len(
self.header_positions_template['Cigarettes Positions'].
dropna()):
Log.warning(
'Number of Headers for Cigarettes is greater than max number defined in template!'
)
elif number_of_headers > 0:
header_position_list = [
position.strip()
for position in self.header_positions_template[
self.header_positions_template['Number of Headers'] ==
number_of_headers]
['Cigarettes Positions'].iloc[0].split(',')
]
relevant_pos.loc[relevant_pos['type'] == 'Header',
['position']] = header_position_list
elif category == 'Smokeless':
relevant_pos = self.check_menu_scene_recognition(relevant_pos)
number_of_headers = len(
relevant_pos[relevant_pos['type'] == 'Header'])
if number_of_headers > len(
self.header_positions_template['Smokeless Positions'].
dropna()):
Log.warning(
'Number of Headers for Smokeless is greater than max number defined in template!'
)
elif number_of_headers > 0:
header_position_list = [
position.strip()
for position in self.header_positions_template[
self.header_positions_template['Number of Headers'] ==
number_of_headers]
['Smokeless Positions'].iloc[0].split(',')
]
relevant_pos.loc[relevant_pos['type'] == 'Header',
['position']] = header_position_list
relevant_pos = self.get_menu_board_items(relevant_pos,
longest_shelf, pos_mpis)
# generate flip-sign positions
if category == 'Cigarettes':
relevant_template = \
self.fixture_width_template.loc[(self.fixture_width_template['Fixture Width (facings)']
- len(longest_shelf)).abs().argsort()[:1]].dropna(axis=1)
locations = relevant_template.columns[2:].tolist()
right_bound = 0
longest_shelf_copy = longest_shelf.copy()
for location in locations:
if right_bound > 0:
left_bound = right_bound + 1
else:
left_bound = longest_shelf_copy.iloc[:relevant_template[
location].iloc[0]]['rect_x'].min()
right_bound = longest_shelf_copy.iloc[:relevant_template[
location].iloc[0]]['rect_x'].max()
if locations[-1] == location:
right_bound = right_bound + abs(right_bound * 0.05)
flip_sign_pos = relevant_pos[
(relevant_pos['type'] == 'Flip Sign')
& (relevant_pos['center_x'] > left_bound) &
(relevant_pos['center_x'] < right_bound)]
if flip_sign_pos.empty:
# add 'NO Flip Sign' product_fk
relevant_pos.loc[len(relevant_pos), ['position', 'product_fk', 'type']] = \
[location, NO_FLIP_SIGN_PK, 'Flip Sign']
else:
relevant_pos.loc[flip_sign_pos.index,
['position']] = location
longest_shelf_copy.drop(
longest_shelf_copy.iloc[:relevant_template[location].
iloc[0]].index,
inplace=True)
elif category == 'Smokeless':
# if there are no flip signs found, there are no positions to assign
number_of_flip_signs = len(
relevant_pos[relevant_pos['type'] == 'Flip Sign'])
if number_of_flip_signs > self.flip_sign_positions_template[
'Number of Flip Signs'].max():
Log.warning(
'Number of Flip Signs for Smokeless is greater than max number defined in template!'
)
elif number_of_flip_signs > 0:
flip_sign_position_list = [
position.strip()
for position in self.flip_sign_positions_template[
self.
flip_sign_positions_template['Number of Flip Signs'] ==
number_of_flip_signs]['Position'].iloc[0].split(',')
]
relevant_pos.loc[relevant_pos['type'] == 'Flip Sign',
['position']] = flip_sign_position_list
# store empty flip sign values
for location in ['Secondary', 'Tertiary']:
if location not in relevant_pos[
relevant_pos['type'] ==
'Flip Sign']['position'].tolist():
relevant_pos.loc[len(relevant_pos), ['position', 'product_fk', 'type']] = \
[location, NO_FLIP_SIGN_PK, 'Flip Sign']
relevant_pos = relevant_pos.reindex(
columns=relevant_pos.columns.tolist() + ['denominator_id'])
# this is a bandaid fix that should be removed -> 'F7011A7C-1BB6-4007-826D-2B674BD99DAE'
# removes POS items that were 'extra', i.e. more than max value in template
# only affects smokeless
relevant_pos.dropna(subset=['position'], inplace=True)
relevant_pos.loc[:,
['denominator_id']] = relevant_pos['position'].apply(
self.get_custom_entity_pk)
for row in relevant_pos.itertuples():
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_name(row.type)
self.common_v2.write_to_db_result(
kpi_fk,
numerator_id=row.product_fk,
denominator_id=row.denominator_id,
result=row.width,
score=row.width)
self.calculate_fixture_width(relevant_pos, longest_shelf, category)
return
def calculate_total_shelves(self,
longest_shelf,
category,
product_mpis=None):
category_fk = self.get_category_fk_by_name(category)
if product_mpis is None:
product_mpis = self.mpis[
(self.mpis['rect_x'] > longest_shelf['rect_x'].min())
& (self.mpis['rect_x'] < longest_shelf['rect_x'].max()) &
(self.mpis['scene_fk']
== longest_shelf['scene_fk'].fillna(0).mode().iloc[0])]
total_shelves = len(product_mpis['shelf_number'].unique())
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_name('Total Shelves')
self.common_v2.write_to_db_result(kpi_fk,
numerator_id=category_fk,
denominator_id=self.store_id,
result=total_shelves)
def calculate_fixture_width(self, relevant_pos, longest_shelf, category):
longest_shelf = longest_shelf[longest_shelf['stacking_layer'] == 1]
category_fk = self.get_category_fk_by_name(category)
# ======== old method for synchronizing fixture width to header data ========
# this is needed to remove intentionally duplicated 'Menu Board' POS 'Headers'
# relevant_pos = relevant_pos.drop_duplicates(subset=['position'])
# try:
# width = relevant_pos[relevant_pos['type'] == 'Header']['width'].sum()
# except KeyError:
# # needed for when 'width' doesn't exist
# width = 0
#
# if relevant_pos.empty or width == 0:
# correction_factor = 1 if category == 'Smokeless' else 2
# width = round(len(longest_shelf) + correction_factor / float(
# self.facings_to_feet_template[category + ' Facings'].iloc[0]))
# ======== old method for synchronizing fixture width to header data ========
width = len(longest_shelf) / float(
self.facings_to_feet_template[category + ' Facings'].iloc[0])
self.mark_tags_in_explorer(longest_shelf['probe_match_fk'].tolist(),
'Longest Shelf - ' + category)
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_name('Fixture Width')
self.common_v2.write_to_db_result(kpi_fk,
numerator_id=category_fk,
denominator_id=self.store_id,
result=width)
def mark_tags_in_explorer(self, probe_match_fk_list, mpipsr_name):
if not probe_match_fk_list:
return
try:
match_type_fk = \
self.match_product_in_probe_state_reporting[
self.match_product_in_probe_state_reporting['name'] == mpipsr_name][
'match_product_in_probe_state_reporting_fk'].values[0]
except IndexError:
Log.warning(
'Name not found in match_product_in_probe_state_reporting table: {}'
.format(mpipsr_name))
return
match_product_in_probe_state_values_old = self.common_v2.match_product_in_probe_state_values
match_product_in_probe_state_values_new = pd.DataFrame(columns=[
MATCH_PRODUCT_IN_PROBE_FK,
MATCH_PRODUCT_IN_PROBE_STATE_REPORTING_FK
])
match_product_in_probe_state_values_new[
MATCH_PRODUCT_IN_PROBE_FK] = probe_match_fk_list
match_product_in_probe_state_values_new[
MATCH_PRODUCT_IN_PROBE_STATE_REPORTING_FK] = match_type_fk
self.common_v2.match_product_in_probe_state_values = pd.concat([
match_product_in_probe_state_values_old,
match_product_in_probe_state_values_new
])
return
def get_category_fk_by_name(self, category_name):
return self.all_products[self.all_products['category'] ==
category_name]['category_fk'].iloc[0]
def check_menu_scene_recognition(self, relevant_pos):
mdis = self.adp.get_match_display_in_scene()
mdis = mdis[mdis['display_name'] == 'Menu POS']
if mdis.empty:
return relevant_pos
dummy_sku_for_menu_pk = 9282 # 'Other (Smokeless Tobacco)'
dummy_sky_for_menu_name = 'Menu POS (Scene Recognized Item)'
location_type = 'Header'
width = 1
center_x = mdis['x'].iloc[0]
center_y = mdis['y'].iloc[0]
relevant_pos.loc[len(relevant_pos), ['product_fk', 'product_name', 'center_x', 'center_y', 'type', 'width']] = \
[dummy_sku_for_menu_pk, dummy_sky_for_menu_name, center_x, center_y, location_type, width]
relevant_pos = relevant_pos.sort_values(
['center_x'], ascending=True).reset_index(drop=True)
return relevant_pos
def get_menu_board_items(self, relevant_pos, longest_shelf, pos_mpis):
# get the placeholder item
menu_board_dummy = relevant_pos[relevant_pos['product_name'] ==
'Menu POS (Scene Recognized Item)']
# if no placeholder, this function isn't relevant
if menu_board_dummy.empty:
return relevant_pos
center_x = menu_board_dummy['center_x'].iloc[0]
center_y = menu_board_dummy['center_y'].iloc[0]
position = menu_board_dummy['position'].iloc[0]
demarcation_line = longest_shelf['rect_y'].min()
upper_demarcation_line = center_y - (demarcation_line - center_y)
distance_in_facings = 2
try:
left_bound = longest_shelf[
longest_shelf['rect_x'] < center_x].sort_values(
by=['rect_x'],
ascending=False)['rect_x'].iloc[int(distance_in_facings) -
1]
except IndexError:
# if there are no POS items found to the left of the 'Menu POS' scene recognition tag, use the tag itself
# in theory this should never happen
left_bound = center_x
try:
right_bound = longest_shelf[
longest_shelf['rect_x'] > center_x].sort_values(
by=['rect_x'],
ascending=True)['rect_x'].iloc[int(distance_in_facings) -
1]
except IndexError:
# if there are no POS items found to the right of the 'Menu POS' scene recognition tag, use the tag itself
# this is more likely to happen for the right bound than the left bound
right_bound = center_x
pos_mpis = pos_mpis[(pos_mpis['rect_x'] > left_bound)
& (pos_mpis['rect_x'] < right_bound) &
(pos_mpis['rect_y'] > upper_demarcation_line) &
(pos_mpis['rect_y'] < demarcation_line)]
if pos_mpis.empty:
return relevant_pos
# remove the placeholder item
relevant_pos = relevant_pos[~(relevant_pos['product_name'] ==
'Menu POS (Scene Recognized Item)')]
location_type = 'Header'
width = 1
for row in pos_mpis.itertuples():
relevant_pos.loc[len(relevant_pos), ['product_fk', 'product_name', 'center_x',
'center_y', 'type', 'width', 'position']] = \
[row.product_fk, row.product_name, row.rect_x, row.rect_y, location_type, width, position]
return relevant_pos
@staticmethod
def remove_duplicate_pos_tags(relevant_pos_df):
duplicate_results = \
relevant_pos_df[relevant_pos_df.duplicated(subset=['left_bound', 'right_bound'], keep=False)]
duplicate_results_without_other = duplicate_results[
~duplicate_results['product_name'].str.contains('Other')]
results_without_duplicates = \
relevant_pos_df[~relevant_pos_df.duplicated(subset=['left_bound', 'right_bound'], keep=False)]
if duplicate_results_without_other.empty:
return relevant_pos_df[~relevant_pos_df.duplicated(
subset=['left_bound', 'right_bound'], keep='first')]
else:
results = pd.concat(
[duplicate_results_without_other, results_without_duplicates])
# we need to sort_index to fix the sort order to reflect center_x values
return results.drop_duplicates(
subset=['left_bound', 'right_bound']).sort_index()
def get_custom_entity_pk(self, name):
return self.custom_entity_data[self.custom_entity_data['name'] ==
name]['pk'].iloc[0]
def get_length_of_pos(self, row, longest_shelf, category):
width_in_facings = len(
longest_shelf[(longest_shelf['rect_x'] > row['left_bound']) &
(longest_shelf['rect_x'] < row['right_bound'])]) + 2
category_facings = category + ' Facings'
return self.facings_to_feet_template.loc[(
self.facings_to_feet_template[category_facings] -
width_in_facings).abs().argsort()[:1]]['POS Width (ft)'].iloc[0]
@staticmethod
def get_longest_shelf_number(relevant_mpis, max_shelves_from_top=3):
# returns the shelf_number of the longest shelf
try:
longest_shelf = \
relevant_mpis[relevant_mpis['shelf_number'] <= max_shelves_from_top].groupby('shelf_number').agg(
{'scene_match_fk': 'count'})['scene_match_fk'].idxmax()
except ValueError:
longest_shelf = pd.DataFrame()
return longest_shelf
@staticmethod
def get_most_frequent_scene(relevant_scif):
try:
relevant_scene_id = relevant_scif['scene_id'].fillna(
0).mode().iloc[0]
except IndexError:
relevant_scene_id = 0
return relevant_scene_id
def get_external_target_data_by_kpi_fk(self, kpi_fk):
return self.external_targets[self.external_targets['kpi_fk'] ==
kpi_fk].iloc[0]
def commit(self):
self.common_v2.commit_results_data()
class LIONJP_SANDToolBox:
LEVEL1 = 1
LEVEL2 = 2
LEVEL3 = 3
def __init__(self, data_provider, output):
self.output = output
self.data_provider = data_provider
self.common = Common(self.data_provider)
self.project_name = self.data_provider.project_name
self.session_uid = self.data_provider.session_uid
self.products = self.data_provider[Data.PRODUCTS]
self.all_products = self.data_provider[Data.ALL_PRODUCTS]
self.match_product_in_scene = self.data_provider[Data.MATCHES]
self.visit_date = self.data_provider[Data.VISIT_DATE]
self.session_info = self.data_provider[Data.SESSION_INFO]
self.scene_info = self.data_provider[Data.SCENES_INFO]
self.store_id = self.data_provider[Data.STORE_FK]
self.scif = self.data_provider[Data.SCENE_ITEM_FACTS]
self.rds_conn = PSProjectConnector(self.project_name,
DbUsers.CalculationEng)
self.kpi_static_data = self.common.get_kpi_static_data()
self.kpi_results_queries = []
self.ps_data_provider = PsDataProvider(self.data_provider, self.output)
self.targets = self.ps_data_provider.get_kpi_external_targets()
self.setup_file = "setup.xlsx"
self.kpi_sheet = self.get_setup(Consts.KPI_SHEET_NAME)
self.kpi_template_file = "kpi_template.xlsx"
self.kpi_template = self.get_kpi_template(Consts.KPI_CONFIG_SHEET)
def get_kpi_template(self, sheet_name):
kpi_template_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), '..', 'Data')
kpi_template_path = os.path.join(kpi_template_path,
self.kpi_template_file)
kpi_template = pd.read_excel(kpi_template_path,
sheet_name=sheet_name,
skiprows=1)
return kpi_template
def get_kpi_level_2_fk(self, kpi_level_2_type):
query = \
"""
SELECT pk FROM static.kpi_level_2
WHERE type = '{}';
""".format(kpi_level_2_type)
data = pd.read_sql_query(query, self.rds_conn.db)
return None if data.empty else data.values[0][0]
def get_kpi_name(self, kpi_level_2_fk):
query = \
"""
SELECT type kpi_name FROM static.kpi_level_2
WHERE pk = {};
""".format(kpi_level_2_fk)
data = pd.read_sql_query(query, self.rds_conn.db)
return None if data.empty else data.values[0][0]
def get_setup(self, sheet_name):
setup_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'..', 'Data')
setup_path = os.path.join(setup_path, self.setup_file)
setup = pd.read_excel(setup_path, sheet_name=sheet_name)
return setup
def main_calculation(self, *args, **kwargs):
"""
This function calculates the KPI results.
"""
try:
if self.kpi_sheet.empty:
Log.error("'kpi_list' sheet in setup file is empty.")
return
kpi_types = [
x.strip() for x in self.kpi_sheet[Consts.KPI_TYPE].unique()
]
for kpi_type in kpi_types:
kpis = self.kpi_sheet[self.kpi_sheet[Consts.KPI_TYPE] ==
kpi_type]
if kpi_type == Consts.FSOS:
self.main_sos_calculations(kpis)
elif kpi_type == Consts.ADJACENCY:
self.main_adjacency_calculations(kpis)
else:
Log.warning(
"KPI_TYPE:{kt} not found in setup=>kpi_list sheet.".
format(kt=kpi_type))
continue
self.common.commit_results_data()
return
except Exception as err:
Log.error(
"LionJP KPI calculation failed due to the following error: {}".
format(err))
def main_sos_calculations(self, kpis):
for row_number, row_data in kpis.iterrows():
if row_data[Consts.KPI_NAME] == Consts.FACINGS_IN_CELL_PER_PRODUCT:
self.calculate_facings_in_cell_per_product()
def calculate_facings_in_cell_per_product(self):
kpi_db = self.kpi_static_data[
(self.kpi_static_data[Consts.KPI_FAMILY] == Consts.PS_KPI_FAMILY)
& (self.kpi_static_data[Consts.KPI_NAME_DB] ==
Consts.FACINGS_IN_CELL_PER_PRODUCT)
& (self.kpi_static_data['delete_time'].isnull())]
if kpi_db.empty:
print("KPI Name:{} not found in DB".format(
Consts.FACINGS_IN_CELL_PER_PRODUCT))
else:
print("KPI Name:{} found in DB".format(
Consts.FACINGS_IN_CELL_PER_PRODUCT))
kpi_fk = kpi_db.pk.values[0]
match_prod_scene_data = self.match_product_in_scene.merge(
self.products,
how='left',
on='product_fk',
suffixes=('', '_prod'))
grouped_data = match_prod_scene_data.query(
'(stacking_layer==1) or (product_type=="POS")').groupby(
['scene_fk', 'bay_number', 'shelf_number', 'product_fk'])
for data_tup, scene_data_df in grouped_data:
scene_fk, bay_number, shelf_number, product_fk = data_tup
facings_count_in_cell = len(scene_data_df)
cur_template_fk = int(
self.scene_info[self.scene_info['scene_fk'] ==
scene_fk].get('template_fk'))
self.common.write_to_db_result(fk=kpi_fk,
numerator_id=product_fk,
denominator_id=self.store_id,
context_id=cur_template_fk,
numerator_result=bay_number,
denominator_result=shelf_number,
result=facings_count_in_cell,
score=scene_fk)
def main_adjacency_calculations(self, kpis):
for row_number, row_data in kpis.iterrows():
if row_data[
Consts.
KPI_NAME] == Consts.ADJACENCY_PRODUCT_GROUP_IN_SCENE_TYPE:
self.calculate_adjacency_per_scene(
Consts.ADJACENCY_PRODUCT_GROUP_IN_SCENE_TYPE)
else:
Log.warning(
"KPI_NAME:{kn} not found in setup=>kpi_list sheet.".format(
kn=row_data[Consts.KPI_NAME]))
@staticmethod
def exclude_and_include_filter(adj_config):
allowed_filters = []
exclude_filters = []
include_empty = adj_config['include_empty']
include_irrelevant = adj_config['include_irrelevant']
if include_empty == "exclude":
allowed_filters.append("Empty")
else:
exclude_filters.append("Empty")
if include_irrelevant == "exclude":
allowed_filters.append("Irrelevant")
else:
exclude_filters.append("Irrelevant")
exclude_filters.append("POS")
if len(allowed_filters) == 0:
allowed_products_filters = None
else:
allowed_products_filters = {"product_type": allowed_filters}
if len(exclude_filters) == 0:
exclude_products_filters = None
else:
exclude_products_filters = {"product_type": exclude_filters}
return exclude_products_filters, allowed_products_filters
def build_entity_groups(self, adj_config, scene_fk):
extra = []
# include_empty
if adj_config['include_empty'] == "exclude":
extra.append(Consts.GENERAL_EMPTY)
extra.append(Consts.EMPTY)
# include_irrelevant
if adj_config['include_irrelevant'] == "exclude":
extra.append(Consts.IRRELEVANT)
# include_others
if not pd.isnull(adj_config['include_other_ean_codes']):
include_others = tuple([
other.strip()
for other in adj_config['include_other_ean_codes'].split(",")
])
if len(include_others) != 0:
product_fks = self.get_product_fks(include_others)
extra.extend(product_fks)
entity_1_type = adj_config['entity_1_type']
entity_2_type = adj_config['entity_2_type']
entity_1_values = [
item.strip()
for item in adj_config['entity_1_values'].strip().split(",")
]
entity_2_values = [
item.strip()
for item in adj_config['entity_2_values'].strip().split(",")
]
entities = [{
"entity_name": "entity_1",
"entity_type": entity_1_type,
"entity_values": entity_1_values
}, {
"entity_name": "entity_2",
"entity_type": entity_2_type,
"entity_values": entity_2_values
}]
df_entities = pd.DataFrame()
for entity in entities:
entity_type = entity['entity_type']
entity_values = entity['entity_values']
if entity_type == "product":
df_entity = self.data_provider.all_products[[
'product_fk', 'product_ean_code'
]].copy()
df_entity = df_entity[[
'product_fk'
]][(df_entity['product_ean_code'].isin(entity_values))]
df_entity['entity'] = entity['entity_name']
elif entity_type == "brand":
df_entity = self.data_provider.all_products[[
'product_fk', 'brand_name'
]].copy()
df_entity = df_entity[[
'product_fk'
]][(df_entity['brand_name'].isin(entity_values))]
df_entity['entity'] = entity['entity_name']
elif entity_type == "category":
df_entity = self.data_provider.all_products[[
'product_fk', 'category_name'
]].copy()
df_entity = df_entity[[
'product_fk'
]][(df_entity['category_name'].isin(entity_values))]
df_entity['entity'] = entity['entity_name']
elif entity_type == "sub_category":
df_entity = self.data_provider.all_products[[
'product_fk', 'sub_category_name'
]].copy()
df_entity = df_entity[[
'product_fk'
]][(df_entity['sub_category_name'].isin(entity_values))]
df_entity['entity'] = entity['entity_name']
else:
Log.error("{} invalid entity_type".format(entity_type))
return pd.DataFrame(), pd.DataFrame(), pd.DataFrame()
df_entities = df_entities.append(df_entity)
df_entities = df_entities.reset_index(drop=True)
product_pks = list(df_entities['product_fk'].unique())
df_product_pks_in_scene = self.scif[
(self.scif['scene_id'] == scene_fk)
& (self.scif['item_id'].isin(product_pks)) &
(self.scif['facings_ign_stack'] > 0)]
if df_product_pks_in_scene.empty:
if is_debug:
Log.warning("Products:{} not in scene{}:scene_fk".format(
product_pks, scene_fk))
return pd.DataFrame(), pd.DataFrame(), pd.DataFrame()
else:
df_custom_matches = self.data_provider.matches.copy()
df_custom_matches = df_custom_matches.merge(df_entities,
on="product_fk")
df_custom_matches = df_custom_matches[
(df_custom_matches['scene_fk'] == scene_fk) &
# (df_custom_matches['face_count'] > 0) &
(df_custom_matches['stacking_layer'] == 1)]
blocking_percentage = adj_config['min_block_percentage']
if int(adj_config['min_overlap_percentage']) != 0:
overlap_percentage = int(
adj_config['min_overlap_percentage']) / 100.00
else:
overlap_percentage = 0.1
df = df_custom_matches[
(~df_custom_matches['product_fk'].isin(extra))
& (df_custom_matches['scene_fk'] == scene_fk)]
minimum_tags_per_entity = self.get_minimum_facings(
df, blocking_percentage)
population = {'entity': list(df_entities['entity'].unique())}
exclude_filter, allowed_products_filter = self.exclude_and_include_filter(
adj_config)
if adj_config['orientation'].lower() == "vertical":
direction = "DOWN"
elif adj_config['orientation'].lower() == "horizontal":
direction = "RIGHT"
else:
Log.warning(
"Invalid direction:{}. Resetting to default orientation RIGHT".
format(adj_config['orientation']))
direction = "RIGHT"
sequence_params = {
AdditionalAttr.DIRECTION: direction,
AdditionalAttr.EXCLUDE_FILTER: exclude_filter,
AdditionalAttr.CHECK_ALL_SEQUENCES: True,
AdditionalAttr.STRICT_MODE: False,
AdditionalAttr.REPEATING_OCCURRENCES: True,
AdditionalAttr.INCLUDE_STACKING: False,
AdditionalAttr.ALLOWED_PRODUCTS_FILTERS: allowed_products_filter,
AdditionalAttr.MIN_TAGS_OF_ENTITY: minimum_tags_per_entity,
AdditionalAttr.ADJACENCY_OVERLAP_RATIO: overlap_percentage
}
return population, sequence_params, df_custom_matches
@staticmethod
def get_minimum_facings(df, block_percentage):
facings = 0
if df.empty:
return facings
entities = df.groupby('entity')['product_fk'].count()
percentage = entities.apply(lambda entity_facings: entity_facings *
(block_percentage / 100.00))
if len(percentage) > 1:
facings = percentage.min()
else:
# when one of the entity is missing in the scene
facings = 0
if (facings > 0) and (facings < 1):
facings = 1
else:
facings = int(facings)
return facings
def get_custom_entity_fk(self, name):
query = """
SELECT pk FROM static.custom_entity
WHERE name='{}'
""".format(name)
data = pd.read_sql_query(query, self.rds_conn.db)
return None if data.empty else data.values[0][0]
def calculate_adjacency_per_scene(self, kpi_name):
allowed_entities = ["product", "brand", "category", "sub_category"]
kpi_config = self.kpi_template[self.kpi_template["kpi_name"] ==
kpi_name].copy()
kpi_config['visit_date'] = pd.Timestamp(self.visit_date)
kpi_config = kpi_config[kpi_config['visit_date'].between(
kpi_config['start_date'], kpi_config['end_date'], inclusive=True)]
kpi_level_2_fk = self.get_kpi_level_2_fk(kpi_name)
if kpi_config.empty:
message = "kpi_name:{} ".format(kpi_name)
message += " not found in static.kpi_level_2 table"
Log.warning(message)
return
for idx, adj_config in kpi_config.iterrows():
custom_entity_pk = self.get_custom_entity_fk(
adj_config['report_label'])
scene_types = [
x.strip() for x in adj_config['scene_type'].split(",")
]
df_scene_template = self.scif[[
'scene_fk', 'template_fk'
]][self.scif['template_name'].isin(scene_types)]
df_scene_template = df_scene_template.drop_duplicates()
for row_num, row_data in df_scene_template.iterrows():
scene_fk = row_data['scene_fk']
template_fk = row_data['template_fk']
if adj_config[
'entity_1_type'] in allowed_entities and adj_config[
'entity_2_type'] in allowed_entities:
location = {"scene_fk": scene_fk}
population, sequence_params, custom_matches = self.build_entity_groups(
adj_config, scene_fk)
if custom_matches.empty:
if is_debug:
Log.warning(
"scene_fk:{}, Custom Entities are not found in the scene"
.format(scene_fk))
continue
if sequence_params[AdditionalAttr.MIN_TAGS_OF_ENTITY] == 0:
continue
seq = Sequence(self.data_provider, custom_matches)
sequence_res = seq.calculate_sequence(
population, location, sequence_params)
result_count = len(sequence_res)
result = 1 if result_count > 0 else 0
score = result
if result > 0:
if is_debug:
Log.warning(
"scene_fk:{}, report_label:{}, result={}".
format(scene_fk, adj_config['report_label'],
result))
self.common.write_to_db_result(
fk=kpi_level_2_fk,
numerator_id=custom_entity_pk,
denominator_id=template_fk,
context_id=self.store_id,
numerator_result=result_count,
denominator_result=scene_fk,
result=result,
score=score,
target=sequence_params[
AdditionalAttr.MIN_TAGS_OF_ENTITY])
else:
Log.warning("Invalid entity:{}".format(
adj_config['entity']))
def get_product_fks(self, ean_codes):
product_pks = []
query = """
SELECT pk FROM static_new.product
WHERE ean_code in '{}'
""".format(ean_codes)
data = pd.read_sql_query(query, self.rds_conn.db)
if data.empty:
return product_pks
else:
product_pks = list(data['pk'].unique())
return product_pks
class ALTRIAUSToolBox:
def __init__(self, data_provider, output):
self.output = output
self.data_provider = data_provider
self.common = Common(self.data_provider)
self.common_v2 = CommonV2(self.data_provider)
self.project_name = self.data_provider.project_name
self.session_uid = self.data_provider.session_uid
self.products = self.data_provider[Data.PRODUCTS]
self.all_products = self.data_provider[Data.ALL_PRODUCTS]
self.match_product_in_scene = self.data_provider[Data.MATCHES]
self.visit_date = self.data_provider[Data.VISIT_DATE]
self.session_info = self.data_provider[Data.SESSION_INFO]
self.scene_info = self.data_provider[Data.SCENES_INFO]
self.store_id = self.data_provider[Data.STORE_FK]
self.scif = self.data_provider[Data.SCENE_ITEM_FACTS]
self.template_info = self.data_provider.all_templates
self.rds_conn = ProjectConnector(self.project_name, DbUsers.CalculationEng)
self.ps_data_provider = PsDataProvider(self.data_provider)
self.match_product_in_probe_state_reporting = self.ps_data_provider.get_match_product_in_probe_state_reporting()
self.kpi_results_queries = []
self.fixture_width_template = pd.read_excel(FIXTURE_WIDTH_TEMPLATE, "Fixture Width", dtype=pd.Int64Dtype())
self.facings_to_feet_template = pd.read_excel(FIXTURE_WIDTH_TEMPLATE, "Conversion Table", dtype=pd.Int64Dtype())
self.header_positions_template = pd.read_excel(FIXTURE_WIDTH_TEMPLATE, "Header Positions")
self.flip_sign_positions_template = pd.read_excel(FIXTURE_WIDTH_TEMPLATE, "Flip Sign Positions")
self.custom_entity_data = self.ps_data_provider.get_custom_entities(1005)
self.ignore_stacking = False
self.facings_field = 'facings' if not self.ignore_stacking else 'facings_ign_stack'
self.kpi_new_static_data = self.common.get_new_kpi_static_data()
try:
self.mpis = self.match_product_in_scene.merge(self.products, on='product_fk', suffixes=['', '_p']) \
.merge(self.scene_info, on='scene_fk', suffixes=['', '_s']) \
.merge(self.template_info, on='template_fk', suffixes=['', '_t'])
except KeyError:
Log.warning('MPIS cannot be generated!')
return
self.adp = AltriaDataProvider(self.data_provider)
self.active_kpis = self._get_active_kpis()
self.external_targets = self.ps_data_provider.get_kpi_external_targets()
self.survey_dot_com_collected_this_session = self._get_survey_dot_com_collected_value()
self._add_smart_attributes_to_mpis()
self.scene_graphs = {}
self.excessive_flipsigns = False
self.incorrect_tags_in_pos_areas = []
def _add_smart_attributes_to_mpis(self):
smart_attribute_data = \
self.adp.get_match_product_in_probe_state_values(self.mpis['probe_match_fk'].unique().tolist())
self.mpis = pd.merge(self.mpis, smart_attribute_data, on='probe_match_fk', how='left')
self.mpis['match_product_in_probe_state_fk'].fillna(0, inplace=True)
def main_calculation(self, *args, **kwargs):
"""
This function calculates the KPI results.
"""
self.generate_graph_per_scene()
for graph in self.scene_graphs.values():
self.calculate_fixture_and_block_level_kpis(graph)
self.calculate_register_type()
self.calculate_age_verification()
self.calculate_facings_by_scene_type()
self.calculate_session_flags()
return
def calculate_fixture_and_block_level_kpis(self, graph):
for node, node_data in graph.nodes(data=True):
if node_data['category'].value != 'POS':
self.calculate_fixture_width(node_data)
self.calculate_flip_sign(node, node_data, graph)
self.calculate_flip_sign_empty_space(node, node_data, graph)
self.calculate_flip_sign_locations(node_data)
self.calculate_total_shelves(node_data)
self.calculate_tags_by_fixture_block(node_data)
if node_data['block_number'].value == 1:
self.calculate_header(node, node_data, graph)
self.calculate_product_above_headers(node_data)
self.calculate_no_headers(node_data)
return
def generate_graph_per_scene(self):
relevant_scif = self.scif[self.scif['template_name'] == 'Tobacco Merchandising Space']
for scene in relevant_scif['scene_id'].unique().tolist():
agb = AltriaGraphBuilder(self.data_provider, scene)
self.scene_graphs[scene] = agb.get_graph()
if agb.incorrect_tags_in_pos_area:
self.incorrect_tags_in_pos_areas.extend(agb.incorrect_tags_in_pos_area)
if len(self.scene_graphs.keys()) > 1:
Log.warning("More than one Tobacco Merchandising Space scene detected. Results could be mixed!")
return
def calculate_fixture_width(self, node_data):
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('Fixture Width')
width = node_data['calculated_width_ft'].value
width = round(width)
fixture_number = node_data['fixture_number'].value
block_number = node_data['block_number'].value
category_fk = self.get_category_fk_by_name(node_data['category'].value)
self.common_v2.write_to_db_result(kpi_fk, numerator_id=category_fk, denominator_id=self.store_id,
numerator_result=block_number, denominator_result=fixture_number,
result=width)
def calculate_flip_sign(self, node, node_data, graph):
if node_data['category'].value == 'Cigarettes':
self.calculate_flip_signs_cigarettes(node, node_data, graph)
else:
self.calculate_flip_signs_non_cigarettes(node, node_data, graph)
return
def calculate_flip_signs_non_cigarettes(self, node, node_data, graph):
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('Flip Sign')
fixture_number = node_data['fixture_number'].value
block_number = node_data['block_number'].value
flip_signs_by_x_coord = {}
for neighbor in graph.neighbors(node):
neighbor_data = graph.nodes[neighbor]
if neighbor_data['category'].value != 'POS':
continue
if neighbor_data['pos_type'].value != 'Flip-Sign':
continue
center_x = neighbor_data['polygon'].centroid.coords[0][0]
flip_signs_by_x_coord[center_x] = neighbor_data
for i, pair in enumerate(sorted(flip_signs_by_x_coord.items())):
position = i+1
position_fk = self.get_custom_entity_pk(str(position))
if position_fk == 0 or position > 8:
self.excessive_flipsigns = True
Log.warning('More than 8 flip-sign positions found for a non-cigarettes block')
product_fk = pair[1]['product_fk'].value
width = pair[1]['calculated_width_ft'].value
implied_facings = pair[1]['width_of_signage_in_facings'].value
width = self.round_threshold(width)
self.common_v2.write_to_db_result(kpi_fk, numerator_id=product_fk, denominator_id=position_fk,
numerator_result=block_number, denominator_result=fixture_number,
result=width, score=implied_facings)
def calculate_flip_signs_cigarettes(self, node, node_data, graph):
width = node_data['calculated_width_ft'].value
fixture_bounds = node_data['polygon'].bounds
fixture_width_coord_units = abs(fixture_bounds[0] - fixture_bounds[2])
proportions_dict = self.get_flip_sign_position_proportions(round(width))
if not proportions_dict:
return
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('Flip Sign')
fixture_number = node_data['fixture_number'].value
block_number = node_data['block_number'].value
f_slot_boxes = {}
left_bound = fixture_bounds[0] # start proportional divisions on left side of fixture
for position, proportion in proportions_dict.items():
right_bound = left_bound + (fixture_width_coord_units * proportion)
# TODO update this to use min and max height of graph
f_slot_boxes[position] = box(left_bound, -9999, right_bound, 9999)
left_bound = right_bound
for position, slot_box in f_slot_boxes.items():
for neighbor in graph.neighbors(node):
neighbor_data = graph.nodes[neighbor]
try:
if neighbor_data['pos_type'].value != 'Flip-Sign':
continue
except KeyError:
continue
flip_sign_bounds = neighbor_data['polygon'].bounds
flip_sign_box = box(flip_sign_bounds[0], flip_sign_bounds[1], flip_sign_bounds[2], flip_sign_bounds[3])
overlap_ratio = flip_sign_box.intersection(slot_box).area / flip_sign_box.area
if overlap_ratio >= REQUIRED_FLIP_SIGN_FSLOT_OVERLAP_RATIO:
product_fk = neighbor_data['product_fk'].value
position_fk = self.get_custom_entity_pk(position)
flip_sign_width = neighbor_data['calculated_width_ft'].value
flip_sign_width = self.round_threshold(flip_sign_width)
implied_facings = neighbor_data['width_of_signage_in_facings'].value
self.common_v2.write_to_db_result(kpi_fk, numerator_id=product_fk, denominator_id=position_fk,
numerator_result=block_number, denominator_result=fixture_number,
result=flip_sign_width, score=implied_facings)
return
def calculate_flip_sign_empty_space(self, node, node_data, graph):
if node_data['category'].value != 'Cigarettes':
return
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('Empty Flip Sign Space')
fixture_number = node_data['fixture_number'].value
block_number = node_data['block_number'].value
fixture_width = node_data['calculated_width_ft'].value
fixture_width = self.round_threshold(fixture_width)
flip_sign_widths = []
for neighbor in graph.neighbors(node):
neighbor_data = graph.nodes[neighbor]
if neighbor_data['category'].value != 'POS':
continue
if neighbor_data['pos_type'].value != 'Flip-Sign':
continue
# exclude 'General POS Other'
if neighbor_data['product_fk'].value == 9304:
continue
neighbor_width = neighbor_data['calculated_width_ft'].value
neighbor_width = self.round_threshold(neighbor_width)
flip_sign_widths.append(neighbor_width)
empty_space = abs(fixture_width - sum(flip_sign_widths))
self.common_v2.write_to_db_result(kpi_fk, numerator_id=49, denominator_id=self.store_id,
numerator_result=block_number, denominator_result=fixture_number,
result=empty_space)
def calculate_flip_sign_locations(self, node_data):
if node_data['category'].value != 'Cigarettes':
return
fixture_number = node_data['fixture_number'].value
block_number = node_data['block_number'].value
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('Flip Sign Locations')
width = node_data['calculated_width_ft'].value
width = round(width)
proportions_dict = self.get_flip_sign_position_proportions(width)
if not proportions_dict:
return
for position, proportion in proportions_dict.items():
position_fk = self.get_custom_entity_pk(position)
self.common_v2.write_to_db_result(kpi_fk, numerator_id=49, denominator_id=position_fk,
numerator_result=block_number, denominator_result=fixture_number,
result=round(proportion * width))
return
def get_flip_sign_position_proportions(self, width):
relevant_template = self.fixture_width_template[self.fixture_width_template['Fixture Width (ft)'] == width]
if relevant_template.empty:
Log.error("Unable to save flip sign location data. {}ft does not exist as a defined width".format(width))
return None
# this could definitely be simpler, but I'm tired and don't want to use my brain
relevant_template[['F1', 'F2', 'F3', 'F4']] = relevant_template[['F1', 'F2', 'F3', 'F4']].fillna(0)
f_slots = [relevant_template['F1'].iloc[0], relevant_template['F2'].iloc[0],
relevant_template['F3'].iloc[0], relevant_template['F4'].iloc[0]]
f_slots = [i for i in f_slots if i != 0]
proportions_dict = {}
for i, slot in enumerate(f_slots):
proportions_dict["F{}".format(i+1)] = slot / float(sum(f_slots))
return proportions_dict
def calculate_total_shelves(self, node_data):
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('Total Shelves')
shelves = len(node_data['shelf_number'].values)
fixture_number = node_data['fixture_number'].value
block_number = node_data['block_number'].value
category_fk = self.get_category_fk_by_name(node_data['category'].value)
self.common_v2.write_to_db_result(kpi_fk, numerator_id=category_fk, denominator_id=self.store_id,
numerator_result=block_number, denominator_result=fixture_number,
result=shelves)
def calculate_tags_by_fixture_block(self, node_data):
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('FACINGS_BY_FIXTURE_BLOCK')
scene_match_fks = node_data['match_fk'].values
fixture_number = node_data['fixture_number'].value
block_number = node_data['block_number'].value
relevant_matches = self.mpis[self.mpis['scene_match_fk'].isin(scene_match_fks)]
relevant_matches = relevant_matches.groupby(['product_fk', 'shelf_number', 'match_product_in_probe_state_fk'],
as_index=False).agg({'scene_match_fk': 'min',
'probe_match_fk': 'count'})
relevant_matches.rename(columns={'probe_match_fk': 'facings'}, inplace=True)
for row in relevant_matches.itertuples():
self.common_v2.write_to_db_result(kpi_fk, numerator_id=row.product_fk,
denominator_id=row.match_product_in_probe_state_fk,
numerator_result=block_number, denominator_result=fixture_number,
result=row.facings, score=row.shelf_number,
context_id=row.scene_match_fk)
def calculate_header(self, node, node_data, graph):
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('Header')
parent_category = node_data['category'].value
fixture_number = node_data['fixture_number'].value
block_number = node_data['block_number'].value
if parent_category not in self.header_positions_template.columns.tolist():
return
headers_by_x_coord = {}
menu_board_items_by_x_coord = {}
for neighbor in graph.neighbors(node):
neighbor_data = graph.nodes[neighbor]
if neighbor_data['category'].value != 'POS':
continue
if neighbor_data['pos_type'].value != 'Header':
continue
center_x = neighbor_data['polygon'].centroid.coords[0][0]
if neighbor_data['in_menu_board_area'].value is True:
menu_board_items_by_x_coord[center_x] = neighbor_data
else:
headers_by_x_coord[center_x] = neighbor_data
number_of_headers = len(headers_by_x_coord.keys())
if menu_board_items_by_x_coord:
number_of_headers += 1
relevant_positions = \
self.header_positions_template[(self.header_positions_template['Number of Headers'] == number_of_headers)]
if relevant_positions.empty:
if number_of_headers == 0:
return
else:
Log.error("Too many headers ({}) found for one block ({}). Unable to calculate positions!".format(
number_of_headers, parent_category))
return
positions = relevant_positions[parent_category].iloc[0]
positions = [position.strip() for position in positions.split(',')]
for i, pair in enumerate(sorted(headers_by_x_coord.items())):
position_fk = self.get_custom_entity_pk(positions[i])
product_fk = pair[1]['product_fk'].value
width = pair[1]['calculated_width_ft'].value
width = self.round_threshold(width)
self.common_v2.write_to_db_result(kpi_fk, numerator_id=product_fk, denominator_id=position_fk,
numerator_result=block_number, denominator_result=fixture_number,
result=width, score=width)
if menu_board_items_by_x_coord:
for pair in menu_board_items_by_x_coord.items():
position_fk = self.get_custom_entity_pk(positions[-1])
product_fk = pair[1]['product_fk'].value
width = pair[1]['calculated_width_ft'].value
width = self.round_threshold(width)
# width = 1 # this is because there is no real masking for menu board items
self.common_v2.write_to_db_result(kpi_fk, numerator_id=product_fk, denominator_id=position_fk,
numerator_result=block_number, denominator_result=fixture_number,
result=width, score=width)
return
def calculate_product_above_headers(self, node_data):
try:
product_above_header = node_data['product_above_header'].value
except KeyError:
return
if not product_above_header:
return
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('Product Above Header')
fixture_number = node_data['fixture_number'].value
block_number = node_data['block_number'].value
self.common_v2.write_to_db_result(kpi_fk, numerator_id=49, denominator_id=self.store_id,
numerator_result=block_number, denominator_result=fixture_number,
result=1, score=1, context_id=fixture_number)
return
def calculate_no_headers(self, node_data):
try:
no_header = node_data['no_header'].value
except KeyError:
return
if not no_header:
return
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('No Header')
fixture_number = node_data['fixture_number'].value
block_number = node_data['block_number'].value
self.common_v2.write_to_db_result(kpi_fk, numerator_id=49, denominator_id=self.store_id,
numerator_result=block_number, denominator_result=fixture_number,
result=1, score=1, context_id=fixture_number)
return
def _get_active_kpis(self):
active_kpis = self.kpi_new_static_data[(self.kpi_new_static_data['kpi_calculation_stage_fk'] == 3) &
(self.kpi_new_static_data['valid_from'] <= self.visit_date) &
((self.kpi_new_static_data['valid_until']).isnull() |
(self.kpi_new_static_data['valid_until'] >= self.visit_date))]
return active_kpis
def calculate_facings_by_scene_type(self):
kpi_name = 'FACINGS_BY_SCENE_TYPE'
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type(kpi_name)
if kpi_name not in self.active_kpis['type'].unique().tolist():
return
config = self.get_external_target_data_by_kpi_fk(kpi_fk)
product_types = config.product_type
template_names = config.template_name
relevant_mpis = self.mpis[(self.mpis['product_type'].isin(product_types)) &
(self.mpis['template_name'].isin(template_names))]
relevant_mpis = relevant_mpis.groupby(['product_fk',
'template_fk',
'match_product_in_probe_state_fk'],
as_index=False)['scene_match_fk'].count()
relevant_mpis.rename(columns={'scene_match_fk': 'facings'}, inplace=True)
for row in relevant_mpis.itertuples():
self.common_v2.write_to_db_result(kpi_fk, numerator_id=row.product_fk, denominator_id=row.template_fk,
context_id=row.match_product_in_probe_state_fk, result=row.facings)
return
def calculate_register_type(self):
if self.survey_dot_com_collected_this_session:
return
relevant_scif = self.scif[(self.scif['product_type'].isin(['POS', 'Other'])) &
(self.scif['category'] == 'POS Machinery')]
if relevant_scif.empty:
result = 0
product_fk = 0
else:
result = 1
product_fk = relevant_scif['product_fk'].iloc[0]
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('Register Type')
self.common_v2.write_to_db_result(kpi_fk, numerator_id=product_fk, denominator_id=self.store_id,
result=result)
def calculate_age_verification(self):
if self.survey_dot_com_collected_this_session:
return
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('Age Verification')
relevant_scif = self.scif[self.scif['brand_name'].isin(['Age Verification'])]
if relevant_scif.empty:
result = 0
product_fk = 0
self.common_v2.write_to_db_result(kpi_fk, numerator_id=product_fk, denominator_id=self.store_id,
result=result)
else:
result = 1
for product_fk in relevant_scif['product_fk'].unique().tolist():
self.common_v2.write_to_db_result(kpi_fk, numerator_id=product_fk, denominator_id=self.store_id,
result=result)
return
def calculate_session_flags(self):
if self.excessive_flipsigns:
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('Excessive Flip Signs Detected')
self.common_v2.write_to_db_result(kpi_fk, numerator_id=49, denominator_id=self.store_id, result=1)
if self.incorrect_tags_in_pos_areas:
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('Invalid Product in Header Areas')
relevant_product_fks = self.mpis[self.mpis['scene_match_fk'].isin(self.incorrect_tags_in_pos_areas)][
'product_fk'].unique().tolist()
for product_fk in relevant_product_fks:
self.common_v2.write_to_db_result(kpi_fk, numerator_id=product_fk, denominator_id=self.store_id,
result=1)
def mark_tags_in_explorer(self, probe_match_fk_list, mpipsr_name):
if not probe_match_fk_list:
return
try:
match_type_fk = \
self.match_product_in_probe_state_reporting[
self.match_product_in_probe_state_reporting['name'] == mpipsr_name][
'match_product_in_probe_state_reporting_fk'].values[0]
except IndexError:
Log.warning('Name not found in match_product_in_probe_state_reporting table: {}'.format(mpipsr_name))
return
match_product_in_probe_state_values_old = self.common_v2.match_product_in_probe_state_values
match_product_in_probe_state_values_new = pd.DataFrame(columns=[MATCH_PRODUCT_IN_PROBE_FK,
MATCH_PRODUCT_IN_PROBE_STATE_REPORTING_FK])
match_product_in_probe_state_values_new[MATCH_PRODUCT_IN_PROBE_FK] = probe_match_fk_list
match_product_in_probe_state_values_new[MATCH_PRODUCT_IN_PROBE_STATE_REPORTING_FK] = match_type_fk
self.common_v2.match_product_in_probe_state_values = pd.concat([match_product_in_probe_state_values_old,
match_product_in_probe_state_values_new])
return
@staticmethod
def round_threshold(value, threshold=0.2):
return round(value - threshold + 0.5)
def _get_survey_dot_com_collected_value(self):
try:
sales_rep_fk = self.session_info['s_sales_rep_fk'].iloc[0]
except IndexError:
sales_rep_fk = 0
return int(sales_rep_fk) == 209050
def get_category_fk_by_name(self, category_name):
return self.all_products[self.all_products['category'] == category_name]['category_fk'].iloc[0]
def get_custom_entity_pk(self, name):
try:
return self.custom_entity_data[self.custom_entity_data['name'] == name]['pk'].iloc[0]
except IndexError:
Log.error('No custom entity found for {}'.format(name))
return 0
def get_external_target_data_by_kpi_fk(self, kpi_fk):
return self.external_targets[self.external_targets['kpi_fk'] == kpi_fk].iloc[0]
def commit(self):
self.common_v2.commit_results_data()
class ColdCutToolBox:
LEVEL1 = 1
LEVEL2 = 2
LEVEL3 = 3
def __init__(self, data_provider, output):
self.output = output
self.data_provider = data_provider
self.ps_data_provider = PsDataProvider(self.data_provider, self.output)
self.common = Common(self.data_provider)
self.project_name = self.data_provider.project_name
self.session_uid = self.data_provider.session_uid
self.products = self.data_provider[Data.PRODUCTS]
self.block = Block(data_provider)
self.all_products = self.data_provider[Data.ALL_PRODUCTS]
self.match_product_in_scene = self.data_provider[Data.MATCHES]
self.visit_date = self.data_provider[Data.VISIT_DATE]
self.session_info = self.data_provider[Data.SESSION_INFO]
self.scene_info = self.data_provider[Data.SCENES_INFO]
self.store_id = self.data_provider[Data.STORE_FK]
self.scif = self.data_provider[Data.SCENE_ITEM_FACTS]
self.rds_conn = self.ps_data_provider.rds_conn
self.kpi_static_data = self.common.get_kpi_static_data()
self.session_info = self.data_provider[Data.SESSION_INFO]
self.session_fk = self.session_info['pk'].values[0]
self.kpi_results_queries = []
self.kpi_static_queries = []
self.own_manufacturer_fk = int(
self.data_provider.own_manufacturer.param_value.values[0])
self.adjacency = BlockAdjacency(self.data_provider,
ps_data_provider=self.ps_data_provider,
common=self.common,
rds_conn=self.rds_conn)
self.eyelight = Eyelight(self.data_provider, self.common,
self.ps_data_provider)
self.merged_scif_mpis = self.match_product_in_scene.merge(
self.scif,
how='left',
left_on=['scene_fk', 'product_fk'],
right_on=['scene_fk', 'product_fk'])
self.targets = self.ps_data_provider.get_kpi_external_targets(
key_fields=[
"KPI Type", "Location: JSON", "Config Params: JSON",
"Dataset 1: JSON", "Dataset 2: JSON"
])
self.results_df = pd.DataFrame(columns=[
'kpi_name', 'kpi_fk', 'numerator_id', 'numerator_result',
'context_id', 'denominator_id', 'denominator_result', 'result',
'score'
])
self.custom_entity_table = self.get_kpi_custom_entity_table()
def main_calculation(self):
"""
This function calculates the KPI results.
"""
relevant_kpi_types = [
Consts.SOS, Consts.HORIZONTAL_SHELF_POSITION,
Consts.VERTICAL_SHELF_POSITION, Consts.BLOCKING, Consts.BLOCK_ADJ,
Consts.BLOCKING_ORIENTATION
]
targets = self.targets[self.targets[Consts.ACTUAL_TYPE].isin(
relevant_kpi_types)]
self._calculate_kpis_from_template(targets)
self.save_results_to_db()
return
def calculate_blocking(self, row, df):
if df.empty:
return None
additional_data = row['Config Params: JSON']
location_data = row['Location: JSON']
kpi_fk = row['kpi_fk']
population_data = row['Dataset 1: JSON']['include'][0]
result_dict_list = self._logic_for_blocking(kpi_fk, population_data,
location_data,
additional_data)
return result_dict_list
def calculate_blocking_adj(self, row, df):
result_dict_list = []
additional_data = row['Config Params: JSON']
location_data = row['Location: JSON']
kpi_fk = row['kpi_fk']
anchor_data = row['Dataset 1: JSON']['include'][0]
target_data = row['Dataset 2: JSON']['include'][0]
context_type = additional_data.get('context_type')
if context_type:
target_df = ParseInputKPI.filter_df(target_data, self.scif)
target_values = target_df[context_type].unique().tolist()
context_values = [
v for v in df[context_type].unique().tolist()
if v and pd.notna(v) and v in target_values
]
for context_value in context_values:
anchor_data.update({context_type: [context_value]})
target_data.update({context_type: [context_value]})
result_dict = self._logic_for_adj(
kpi_fk,
anchor_data,
target_data,
location_data,
additional_data,
eyelight_prefix='{}-'.format(context_value),
custom_entity=context_value)
result_dict_list.append(result_dict)
else:
result_dict = self._logic_for_adj(kpi_fk, anchor_data, target_data,
location_data, additional_data)
result_dict_list.append(result_dict)
return result_dict_list
def _logic_for_adj(self,
kpi_fk,
anchor_data,
target_data,
location_data,
additional_data,
custom_entity=None,
eyelight_prefix=None):
result = self.adjacency.evaluate_block_adjacency(
anchor_data,
target_data,
location=location_data,
additional=additional_data,
kpi_fk=kpi_fk,
eyelight_prefix=eyelight_prefix)
result_type_fk = Consts.CUSTOM_RESULT['Yes'] if result and pd.notna(
result) else Consts.CUSTOM_RESULT['No']
result_dict = {
'kpi_fk': kpi_fk,
'numerator_id': self.own_manufacturer_fk,
'denominator_id': self.store_id,
'numerator_result': 1 if result else 0,
'denominator_result': 1,
'result': result_type_fk
}
if custom_entity:
result_dict.update(
{'context_id': self.get_custom_entity_value(custom_entity)})
return result_dict
def _logic_for_blocking(self, kpi_fk, population_data, location_data,
additional_data):
result_dict_list = []
additional_data.update({'use_masking_only': True})
block = self.block.network_x_block_together(population=population_data,
location=location_data,
additional=additional_data)
for row in block.itertuples():
scene_match_fks = list(row.cluster.nodes[list(
row.cluster.nodes())[0]]['scene_match_fk'])
self.eyelight.write_eyelight_result(scene_match_fks, kpi_fk)
passed_block = block[block['is_block']]
if passed_block.empty:
numerator_result = 0
result_value = "No"
else:
numerator_result = 1
result_value = "Yes"
result_type_fk = Consts.CUSTOM_RESULT[result_value]
# numerator_id = df.custom_entity_fk.iloc[0]
result_dict = {
'kpi_fk': kpi_fk,
'numerator_id': self.own_manufacturer_fk,
'numerator_result': numerator_result,
'denominator_id': self.store_id,
'denominator_result': 1,
'result': result_type_fk
}
result_dict_list.append(result_dict)
return result_dict_list
def calculate_blocking_orientation(self, row, df):
if df.empty:
return
result_dict_list = []
additional_data = row['Config Params: JSON']
location_data = row['Location: JSON']
kpi_fk = row['kpi_fk']
population_data = row['Dataset 1: JSON']
if population_data:
population_data = population_data['include'][0]
else:
population_data = {}
additional_data.update({
'vertical_horizontal_methodology':
['bucketing', 'percentage_of_shelves'],
'shelves_required_for_vertical':
.8,
'check_vertical_horizontal':
True
})
numerator_type = additional_data.get('numerator_type')
if numerator_type:
numerator_values = [
v for v in df[numerator_type].unique().tolist()
if v and pd.notna(v)
]
for numerator_value in numerator_values:
population_data.update({numerator_type: [numerator_value]})
result_dict = self._logic_for_blocking_orientation(
kpi_fk, population_data, location_data, additional_data,
numerator_value)
result_dict_list.append(result_dict)
else:
result_dict = self._logic_for_blocking_orientation(
kpi_fk, population_data, location_data, additional_data)
result_dict_list.append(result_dict)
return result_dict_list
def _logic_for_blocking_orientation(self,
kpi_fk,
population_data,
location_data,
additional_data,
custom_entity=None):
additional_data.update({'use_masking_only': True})
block = self.block.network_x_block_together(population=population_data,
location=location_data,
additional=additional_data)
if custom_entity:
prefix = '{}-'.format(custom_entity)
numerator_id = self.get_custom_entity_value(custom_entity)
else:
prefix = None
numerator_id = self.own_manufacturer_fk
for row in block.itertuples():
scene_match_fks = list(row.cluster.nodes[list(
row.cluster.nodes())[0]]['scene_match_fk'])
self.eyelight.write_eyelight_result(scene_match_fks,
kpi_fk,
prefix=prefix)
passed_block = block[block['is_block']]
if passed_block.empty:
result_value = "Not Blocked"
else:
result_value = passed_block.orientation.iloc[0]
result = Consts.CUSTOM_RESULT[result_value]
result_dict = {
'kpi_fk': kpi_fk,
'numerator_id': numerator_id,
'numerator_result': 1 if result_value != 'Not Blocked' else 0,
'denominator_id': self.store_id,
'denominator_result': 1,
'result': result
}
return result_dict
def calculate_vertical_position(self, row, df):
result_dict_list = []
mpis = df # get this from the external target filter_df method thingy
scene_facings_df = mpis.groupby(['scene_fk', 'product_fk'],
as_index=False)['facings'].max()
scene_facings_df.rename(columns={'facings': 'scene_facings'},
inplace=True)
shelf_df = self.merged_scif_mpis.groupby(
['scene_fk', 'bay_number'],
as_index=False)['shelf_number_from_bottom'].max()
shelf_df.rename(columns={'shelf_number_from_bottom': 'shelf_count'},
inplace=True)
pre_sort_mpis = pd.merge(mpis,
scene_facings_df,
how='left',
on=['scene_fk', 'product_fk'])
scene_facings_df_sorted = pre_sort_mpis.sort_values('scene_facings')
mpis = scene_facings_df_sorted.drop_duplicates(
['scene_fk', 'product_fk'], keep="last")
mpis = pd.merge(mpis,
shelf_df,
how='left',
on=['scene_fk', 'bay_number'])
mpis['position'] = mpis.apply(self._calculate_vertical_position,
axis=1)
mpis['result_type_fk'] = mpis['position'].apply(
lambda x: Consts.CUSTOM_RESULT.get(x, 0))
mpis = mpis.groupby(['product_fk'],
as_index=False)['result_type_fk'].agg(
lambda x: pd.Series.mode(x).iat[0])
for result in mpis.itertuples():
custom_fk_result = result.result_type_fk
if type(custom_fk_result) == numpy.ndarray:
custom_fk_result = result.result_type_fk[0]
result_item = {
'kpi_fk': row.kpi_fk,
'numerator_id': result.product_fk,
'numerator_result': 1,
'denominator_id': self.store_id,
'denominator_result': 1,
'result': custom_fk_result,
'score': 0
}
result_dict_list.append(result_item)
return result_dict_list
def calculate_horizontal_position(self, row, df):
result_dict_list = []
mpis = df # get this from the external target filter_df method thingy
scene_facings_df = mpis.groupby(['scene_fk', 'product_fk'],
as_index=False)['facings'].max()
scene_facings_df.rename(columns={'facings': 'scene_facings'},
inplace=True)
pre_sort_mpis = pd.merge(mpis,
scene_facings_df,
how='left',
on=['scene_fk', 'product_fk'])
bay_df = pre_sort_mpis.groupby('scene_fk',
as_index=False)['bay_number'].max()
bay_df.rename(columns={'bay_number': 'bay_count'}, inplace=True)
mpis = pd.merge(pre_sort_mpis, bay_df, how='left', on='scene_fk')
mpis['position'] = mpis.apply(self._calculate_horizontal_position,
axis=1)
mpis['result_type_fk'] = mpis['position'].apply(
lambda x: Consts.CUSTOM_RESULT.get(x, 0))
mpis = mpis.groupby(['product_fk'],
as_index=False)['result_type_fk'].agg(
lambda x: pd.Series.mode(x).iat[0])
for result in mpis.itertuples():
custom_fk_result = result.result_type_fk
if type(custom_fk_result) == numpy.ndarray:
custom_fk_result = result.result_type_fk[0]
result_item = {
'kpi_fk': row.kpi_fk,
'numerator_id': result.product_fk,
'numerator_result': 1,
'denominator_id': self.store_id,
'denominator_result': 1,
'result': custom_fk_result,
'score': 0
}
result_dict_list.append(result_item)
return result_dict_list
@staticmethod
def _calculate_horizontal_position(row):
bay_count = row.bay_count
if bay_count == 1:
return 'Center'
factor = round(bay_count / float(3))
if row.bay_number <= factor:
return 'Left'
elif row.bay_number > (bay_count - factor):
return 'Right'
return 'Center'
@staticmethod
def _calculate_vertical_position(row):
shelf_number = str(row.shelf_number_from_bottom)
shelf_count = str(row.shelf_count)
shelf_count_pos_map = Consts.shelf_map[shelf_count]
pos_value = shelf_count_pos_map[shelf_number]
return pos_value
def calculate_facings_sos(self, row, df):
data_filter = {'population': row['Dataset 2: JSON']}
if 'include' not in data_filter['population'].keys():
data_filter['population'].update(
{'include': [{
'session_id': self.session_fk
}]})
data_filter.update({'location': row['Location: JSON']})
config_json = row['Config Params: JSON']
numerator_type = config_json['numerator_type']
df = ParseInputKPI.filter_df(data_filter, self.scif)
result_dict_list = self._logic_for_sos(row, df, numerator_type)
return result_dict_list
def _logic_for_sos(self, row, df, numerator_type):
result_list = []
facing_type = 'facings'
config_json = row['Config Params: JSON']
if 'include_stacking' in config_json:
if config_json['include_stacking']:
facing_type = 'facings_ign_stack'
for num_item in df[numerator_type].unique().tolist():
if num_item:
numerator_scif = df[df[numerator_type] == num_item]
else:
numerator_scif = df[df[numerator_type].isnull()]
num_item = 'None'
numerator_result = numerator_scif[facing_type].sum()
denominator_result = df[facing_type].sum()
custom_entity_fk = self.get_custom_entity_value(num_item)
sos_value = self.calculate_percentage_from_numerator_denominator(
numerator_result, denominator_result)
result_dict = {
'kpi_fk': row.kpi_fk,
'numerator_id': custom_entity_fk,
'numerator_result': numerator_result,
'denominator_id': self.store_id,
'denominator_result': denominator_result,
'result': sos_value
}
result_list.append(result_dict)
return result_list
def _get_calculation_function_by_kpi_type(self, kpi_type):
if kpi_type == Consts.SOS:
return self.calculate_facings_sos
elif kpi_type == Consts.HORIZONTAL_SHELF_POSITION:
return self.calculate_horizontal_position
elif kpi_type == Consts.VERTICAL_SHELF_POSITION:
return self.calculate_vertical_position
elif kpi_type == Consts.BLOCKING:
return self.calculate_blocking
elif kpi_type == Consts.BLOCK_ADJ:
return self.calculate_blocking_adj
elif kpi_type == Consts.BLOCKING_ORIENTATION:
return self.calculate_blocking_orientation
def _calculate_kpis_from_template(self, template_df):
for i, row in template_df.iterrows():
try:
calculation_function = self._get_calculation_function_by_kpi_type(
row[Consts.ACTUAL_TYPE])
row = self.apply_json_parser(row)
merged_scif_mpis = self._parse_json_filters_to_df(row)
result_data = calculation_function(row, merged_scif_mpis)
if result_data and isinstance(result_data, list):
for result in result_data:
self.results_df.loc[len(self.results_df),
result.keys()] = result
elif result_data and isinstance(result_data, dict):
self.results_df.loc[len(self.results_df),
result_data.keys()] = result_data
except Exception as e:
Log.error('Unable to calculate {}: {}'.format(
row[Consts.KPI_NAME], e))
def _parse_json_filters_to_df(self, row):
jsonv = row[(row.index.str.contains('JSON'))
& (~row.index.str.contains('Config Params')) &
(~row.index.str.contains('Dataset 2'))]
filter_json = jsonv[~jsonv.isnull()]
filtered_scif_mpis = self.merged_scif_mpis
for each_json in filter_json:
final_json = {
'population': each_json
} if ('include' or 'exclude') in each_json else each_json
filtered_scif_mpis = ParseInputKPI.filter_df(
final_json, filtered_scif_mpis)
if 'include_stacking' in row['Config Params: JSON'].keys():
including_stacking = row['Config Params: JSON'][
'include_stacking'][0]
filtered_scif_mpis[Consts.FINAL_FACINGS] = \
filtered_scif_mpis.facings if including_stacking == 'True' else filtered_scif_mpis.facings_ign_stack
filtered_scif_mpis = filtered_scif_mpis[
filtered_scif_mpis.stacking_layer == 1]
return filtered_scif_mpis
def apply_json_parser(self, row):
json_relevent_rows_with_parse_logic = row[
(row.index.str.contains('JSON')) & (row.notnull())].apply(
self.parse_json_row)
row = row[~row.index.isin(json_relevent_rows_with_parse_logic.index
)].append(
json_relevent_rows_with_parse_logic)
return row
def parse_json_row(self, item):
'''
:param item: improper json value (formatted incorrectly)
:return: properly formatted json dictionary
The function will be in conjunction with apply. The function will applied on the row(pandas series). This is
meant to convert the json comprised of improper format of strings and lists to a proper dictionary value.
'''
if item:
try:
container = self.prereq_parse_json_row(item)
except Exception as e:
container = None
Log.warning('{}: Unable to parse json for: {}'.format(e, item))
else:
container = None
return container
def save_results_to_db(self):
self.results_df.drop(columns=['kpi_name'], inplace=True)
self.results_df.rename(columns={'kpi_fk': 'fk'}, inplace=True)
self.results_df['result'].fillna(0, inplace=True)
self.results_df['score'].fillna(0, inplace=True)
results = self.results_df.to_dict('records')
for result in results:
result = simplejson.loads(simplejson.dumps(result,
ignore_nan=True))
self.common.write_to_db_result(**result)
@staticmethod
def prereq_parse_json_row(item):
'''
primarly logic for formatting the value of the json
'''
container = dict()
try:
container = ast.literal_eval(item)
except:
json_str = ",".join(item)
json_str_fixed = json_str.replace("'", '"')
container = json.loads(json_str_fixed)
return container
@staticmethod
def _get_numerator_and_denominator_type(config_param,
context_relevant=False):
numerator_type = config_param['numerator_type'][0]
denominator_type = config_param['denominator_type'][0]
if context_relevant:
context_type = config_param['context_type'][0]
return numerator_type, denominator_type, context_type
return numerator_type, denominator_type
@staticmethod
def calculate_percentage_from_numerator_denominator(
numerator_result, denominator_result):
try:
ratio = numerator_result / denominator_result
except Exception as e:
Log.error(e.message)
ratio = 0
if not isinstance(ratio, (float, int)):
ratio = 0
return round(ratio * 100, 2)
def get_kpi_custom_entity_table(self):
"""
:param entity_type: pk of entity from static.entity_type
:return: the DF of the static.custom_entity of this entity_type
"""
query = "SELECT pk, name, entity_type_fk FROM static.custom_entity;"
df = pd.read_sql_query(query, self.rds_conn.db)
return df
def get_custom_entity_value(self, value):
try:
custom_fk = self.custom_entity_table['pk'][
self.custom_entity_table['name'] == value].iloc[0]
return custom_fk
except IndexError:
Log.error('No custom entity found for: {}'.format(value))
return None
def commit_results(self):
self.common.commit_results_data()
class PsApacGSKAUSceneToolBox:
def __init__(self, data_provider, output, common):
self.output = output
self.data_provider = data_provider
self.common = common
self.project_name = self.data_provider.project_name
self.session_uid = self.data_provider.session_uid
self.products = self.data_provider[Data.PRODUCTS]
self.templates = self.data_provider[Data.TEMPLATES]
self.all_products = self.data_provider[Data.ALL_PRODUCTS]
self.scif = self.data_provider.scene_item_facts
self.match_product_in_scene = self.data_provider[Data.MATCHES]
self.visit_date = self.data_provider[Data.VISIT_DATE]
self.session_info = self.data_provider[Data.SESSION_INFO]
self.scene_info = self.data_provider[Data.SCENES_INFO]
self.store_info = self.data_provider[Data.STORE_INFO]
self.store_id = self.store_info.iloc[0].store_fk
self.store_type = self.data_provider.store_type
self.rds_conn = PSProjectConnector(self.project_name, DbUsers.CalculationEng)
self.kpi_static_data = self.common.get_kpi_static_data()
self.ps_data_provider = PsDataProvider(self.data_provider, self.output)
self.targets = self.ps_data_provider.get_kpi_external_targets()
self.match_display_in_scene = self.data_provider.match_display_in_scene
def calculate_display_compliance(self):
kpi_display_presence = self.kpi_static_data[
(self.kpi_static_data[KPI_TYPE_COL] == GSK_DISPLAY_PRESENCE)
& (self.kpi_static_data['delete_time'].isnull())]
kpi_display_sku_compliance = self.kpi_static_data[
(self.kpi_static_data[KPI_TYPE_COL] == GSK_DISPLAY_SKU_COMPLIANCE)
& (self.kpi_static_data['delete_time'].isnull())]
kpi_display_price_compliance = self.kpi_static_data[
(self.kpi_static_data[KPI_TYPE_COL] == GSK_DISPLAY_PRICE_COMPLIANCE)
& (self.kpi_static_data['delete_time'].isnull())]
kpi_display_bay_purity = self.kpi_static_data[
(self.kpi_static_data[KPI_TYPE_COL] == GSK_DISPLAY_BAY_PURITY)
& (self.kpi_static_data['delete_time'].isnull())]
kpi_display_presence_sku = self.kpi_static_data[
(self.kpi_static_data[KPI_TYPE_COL] == GSK_DISPLAY_PRESENCE_SKU)
& (self.kpi_static_data['delete_time'].isnull())]
secondary_display_targets = self.targets[
self.targets['kpi_fk'] == kpi_display_presence['pk'].iloc[0]]
# if no targets return
if secondary_display_targets.empty:
Log.warning('There is no target policy for calculating secondary display compliance.')
return False
else:
current_scene_fk = self.scene_info.iloc[0].scene_fk
display_per_sku_per_scene_calculated = False
target_matched = False
for idx, each_target in secondary_display_targets.iterrows():
if target_matched:
Log.info('The session: {sess} - scene: {scene} has matched one target '
'and won\'t run for another.'
.format(sess=self.session_uid, scene=current_scene_fk))
continue
# loop through each external target to fit the current store
has_posm_recognized = False
multi_posm_or_bay = False
mandatory_sku_compliance = False
optional_sku_compliance = False
price_compliance = False
is_scene_relevant = False
scene_relevant_targets = pd.DataFrame()
# check store relevance
store_relevant_targets = each_target[STORE_IDENTIFIERS].dropna()
_bool_store_check_df = self.store_info[list(store_relevant_targets.keys())] \
== store_relevant_targets.values
is_store_relevant = _bool_store_check_df.all(axis=None)
if is_store_relevant:
# check scene relevance
scene_relevant_targets = each_target[SCENE_IDENTIFIERS].dropna()
_bool_scene_check_df = self.scene_info[list(scene_relevant_targets.keys())] \
== scene_relevant_targets.values
is_scene_relevant = _bool_scene_check_df.all(axis=None)
if is_store_relevant and is_scene_relevant and not target_matched:
# calculate display compliance for the matched external targets
target_matched = True
Log.info('The session: {sess} - scene: {scene} is relevant for calculating '
'secondary display compliance.'
.format(sess=self.session_uid, scene=current_scene_fk))
posm_relevant_targets = each_target[POSM_IDENTIFIERS].dropna()
mandatory_eans = _sanitize_csv(posm_relevant_targets[MANDATORY_EANS_KEY])
optional_posm_eans = []
if OPTIONAL_EAN_KEY in posm_relevant_targets:
optional_posm_eans = _sanitize_csv(posm_relevant_targets[OPTIONAL_EAN_KEY])
# save detailed sku presence
posm_to_check = each_target[POSM_PK_KEY]
# FIND THE SCENES WHICH HAS THE POSM to check for multiposm or multibays
is_posm_absent = self.match_display_in_scene[
self.match_display_in_scene['display_fk'] == posm_to_check].empty
if is_posm_absent:
Log.info('The scene: {scene} is relevant but POSM {pos} is not present. '
'Save and start new scene.'
.format(scene=current_scene_fk, pos=posm_to_check))
# calculate display per sku -- POSM is absent
if not display_per_sku_per_scene_calculated:
display_per_sku_per_scene_calculated = self.save_display_presence_per_sku(
kpi=kpi_display_presence_sku,
numerator_result=0, # 0 posm not recognized
)
if len(self.match_product_in_scene['bay_number'].unique()) > 1 or \
len(self.match_display_in_scene) > 1:
Log.info(
'The scene: {scene} is relevant and multi_bay_posm is True. '
'Purity per bay is calculated and going to next scene.'
.format(scene=current_scene_fk, pos=posm_to_check))
multi_posm_or_bay = True
self.save_purity_per_bay(kpi_display_bay_purity)
self.save_display_compliance_data(
[
{'pk': kpi_display_presence.iloc[0].pk, 'result': int(has_posm_recognized),
'score': int(multi_posm_or_bay), 'numerator_id': posm_to_check,
'numerator_result': posm_to_check},
{'pk': kpi_display_sku_compliance.iloc[0].pk, 'result': float(mandatory_sku_compliance),
'score': float(optional_sku_compliance), 'denominator_id': posm_to_check,
'denominator_result': posm_to_check},
{'pk': kpi_display_price_compliance.iloc[0].pk, 'result': float(price_compliance),
'score': float(price_compliance), 'denominator_id': posm_to_check,
'denominator_result': posm_to_check},
]
)
continue
# this scene has the posm
Log.info('The scene: {scene} is relevant and POSM {pos} is present.'
.format(scene=current_scene_fk, pos=posm_to_check))
has_posm_recognized = True
# check if this scene has multi bays or multi posm
if len(self.match_product_in_scene['bay_number'].unique()) > 1 or \
len(self.match_display_in_scene) > 1:
# Its multi posm or bay -- only purity calc per bay is possible
Log.info('The scene: {scene} is relevant and POSM {pos} is present but multi_bay_posm is True. '
'Purity per bay is calculated and going to next scene.'
.format(scene=current_scene_fk, pos=posm_to_check))
multi_posm_or_bay = True
# calculate display per sku for multi posm/multi bay
if not display_per_sku_per_scene_calculated:
display_per_sku_per_scene_calculated = self.save_display_presence_per_sku(
kpi=kpi_display_presence_sku,
numerator_result=2, # 2 multi posm
)
self.save_display_compliance_data(
[
{'pk': kpi_display_presence.iloc[0].pk, 'result': int(has_posm_recognized),
'score': int(multi_posm_or_bay), 'numerator_id': posm_to_check,
'numerator_result': posm_to_check},
{'pk': kpi_display_sku_compliance.iloc[0].pk, 'result': float(mandatory_sku_compliance),
'score': float(optional_sku_compliance), 'denominator_id': posm_to_check,
'denominator_result': posm_to_check},
{'pk': kpi_display_price_compliance.iloc[0].pk, 'result': float(price_compliance),
'score': float(price_compliance), 'denominator_id': posm_to_check,
'denominator_result': posm_to_check},
]
)
self.save_purity_per_bay(kpi_display_bay_purity)
continue
Log.info('The scene: {scene} is relevant and POSM {pos} is present with only one bay.'
.format(scene=current_scene_fk, pos=posm_to_check))
# save purity per bay
self.save_purity_per_bay(kpi_display_bay_purity)
# calculate display per sku for ALL SUCCESS
if not display_per_sku_per_scene_calculated:
display_per_sku_per_scene_calculated = self.save_display_presence_per_sku(
kpi=kpi_display_presence_sku,
posm_to_check=posm_to_check,
numerator_result=1, # 1--one one posm
mandatory_eans=mandatory_eans,
optional_posm_eans=optional_posm_eans)
# calculate compliance
mandatory_sku_compliance = self.get_ean_presence_rate(mandatory_eans)
optional_sku_compliance = self.get_ean_presence_rate(optional_posm_eans)
if mandatory_sku_compliance:
price_compliance = self.get_price_presence_rate(mandatory_eans)
self.save_display_compliance_data(
[
{'pk': kpi_display_presence.iloc[0].pk, 'result': int(has_posm_recognized),
'score': int(multi_posm_or_bay), 'numerator_id': posm_to_check,
'numerator_result': posm_to_check},
{'pk': kpi_display_sku_compliance.iloc[0].pk, 'result': float(mandatory_sku_compliance),
'score': float(optional_sku_compliance), 'denominator_id': posm_to_check,
'denominator_result': posm_to_check},
{'pk': kpi_display_price_compliance.iloc[0].pk, 'result': float(price_compliance),
'score': float(price_compliance), 'denominator_id': posm_to_check,
'denominator_result': posm_to_check},
]
)
continue
else:
# the session/store is not part of the KPI targets
Log.info('The session: {sess} - scene: {scene}, the current kpi target [pk={t_pk}] '
'is not valid. Keep Looking...'
.format(sess=self.session_uid, scene=current_scene_fk,
t_pk=each_target.external_target_fk))
if scene_relevant_targets.empty:
# Store failed
Log.info("Store info is {curr_data} but target is {store_data}".format(
curr_data=self.store_info.iloc[0][list(store_relevant_targets.keys())].to_json(),
store_data=store_relevant_targets.to_json()
))
else:
Log.info("Scene info is {curr_data} but target is {store_data}".format(
curr_data=self.scene_info.iloc[0][list(scene_relevant_targets.keys())].to_json(),
store_data=scene_relevant_targets.to_json()
))
continue
else:
if not display_per_sku_per_scene_calculated:
# check if its secondary display type
if not self.templates.loc[(self.templates['template_group'] == 'Secondary display') &
(~self.templates['template_name'].isin(['Clipstrip', 'Hangsell']))].empty:
Log.info("Secondary Display => Session: {sess} - scene {scene} didn't qualify "
"any external targets.".format(sess=self.session_uid,
scene=self.scene_info.iloc[0].scene_fk,
))
display_per_sku_per_scene_calculated = self.save_display_presence_per_sku(
kpi=kpi_display_presence_sku,
numerator_result=0) # 0--posm not recognized
def calculate_layout_compliance(self):
current_scene_fk = self.scene_info.iloc[0].scene_fk
Log.info('Calculate Layout Compliance for session: {sess} - scene: {scene}'
.format(sess=self.session_uid, scene=current_scene_fk))
scene_layout_calc_obj = SceneLayoutComplianceCalc(scene_toolbox_obj=self)
scene_layout_calc_obj.calculate_all()
def get_ean_presence_rate(self, ean_list):
"""
This method takes the list of eans, checks availability in scif and returns percentage
of items among the input list; which are present
"""
Log.info('Calculate ean presence rate for : {scene}.'.format(scene=self.scene_info.iloc[0].scene_fk))
if not ean_list:
return 0.0
present_ean_count = len(self.scif[self.scif['product_ean_code'].isin(ean_list)])
return present_ean_count / float(len(ean_list)) * 100
def get_price_presence_rate(self, ean_list):
"""
This method takes ean list as input and returns percentage of eans which has price.
"""
Log.info('Calculate price presence rate for : {scene}.'.format(scene=self.scene_info.iloc[0].scene_fk))
if not ean_list:
return 0.0
scif_to_check = self.scif[self.scif['product_ean_code'].isin(ean_list)]
price_fields = ['median_price', 'median_promo_price']
present_price_count = 0
for idx, each_data in scif_to_check.iterrows():
if each_data[price_fields].apply(pd.notnull).any():
present_price_count += 1
return present_price_count / float(len(ean_list)) * 100
def save_purity_per_bay(self, kpi_bay_purity):
Log.info('Calculate purity per bay for : {scene}.'.format(scene=self.scene_info.iloc[0].scene_fk))
mpis_grouped_by_bay = self.match_product_in_scene.groupby(['bay_number'])
for bay_number, mpis in mpis_grouped_by_bay:
total_prod_in_bay_count = len(
mpis[mpis['product_fk'] != 0]
)
mpis_with_prod = mpis.merge(self.products, how='left', on=['product_fk'], suffixes=('', '_prod'))
gsk_prod_count = len(mpis_with_prod[mpis_with_prod['manufacturer_fk'] == 2])
if total_prod_in_bay_count:
purity = gsk_prod_count / float(total_prod_in_bay_count) * 100
Log.info('Save purity per bay for scene: {scene}; bay: {bay} & purity: {purity}.'
.format(scene=self.scene_info.iloc[0].scene_fk,
bay=bay_number,
purity=purity
))
if not bay_number or np.isnan(bay_number):
bay_number = -1
self.common.write_to_db_result(
fk=kpi_bay_purity.iloc[0].pk,
context_id=self.store_id,
numerator_id=self.store_id,
denominator_id=self.store_id,
numerator_result=self.store_id,
denominator_result=self.store_id,
result=purity,
score=bay_number,
by_scene=True,
)
return True
def save_display_compliance_data(self, data):
for each_kpi_data in data:
Log.info("Saving Display Compliance kpi_fk {pk} for session: {sess} - scene {scene}".format(
pk=each_kpi_data.get('pk'),
sess=self.session_uid,
scene=self.scene_info.iloc[0].scene_fk,
))
score = each_kpi_data.get('score')
result = each_kpi_data.get('result')
if not score or np.isnan(score):
score = -1
if not result or np.isnan(result):
result = 0
self.common.write_to_db_result(
fk=each_kpi_data.get('pk'),
numerator_id=each_kpi_data.get('numerator_id', self.store_id),
denominator_id=each_kpi_data.get('denominator_id', self.store_id),
numerator_result=each_kpi_data.get('numerator_result', self.store_id),
denominator_result=each_kpi_data.get('denominator_result', self.store_id),
result=result,
score=score,
context_id=self.store_id,
by_scene=True,
)
return True
def save_display_presence_per_sku(self, kpi, posm_to_check=None, numerator_result=None,
mandatory_eans=None, optional_posm_eans=None):
# This should be done once only per scene
current_scene_fk = self.scene_info.iloc[0].scene_fk
context_fk = self.scene_info.iloc[0].template_fk
if posm_to_check and (mandatory_eans or optional_posm_eans):
# the scene is valid as per external targets template
# PER SCENE which are secondary display
# save POSM as numerator; each product as denominator and template as context
Log.info('Calculate display presence per sku. The session: {sess} - scene: {scene} is valid.'
.format(sess=self.session_uid, scene=current_scene_fk))
numerator_id = posm_to_check
else:
# the scene doesn't have mandatory or optional eans
# PER SCENE which are secondary display
# save numerator as empty; each empty as denominator and template as context
Log.info('Calculate display presence per sku. The session: {sess} - scene: {scene} is invalid.'
.format(sess=self.session_uid, scene=current_scene_fk))
numerator_id = 0 # General Empty
# result => [ 0=Optional, 1=mandatory, 2=NA]
# numerator_result => [0-- posm not recognized; 1--one one posm; 2--multi posm]
for idx, each_row in self.scif.iterrows():
result = 2 # NA
if mandatory_eans and each_row.product_ean_code in mandatory_eans:
result = 1 # mandatory
elif optional_posm_eans and each_row.product_ean_code in optional_posm_eans:
result = 0 # optional
score = min(each_row.median_price, each_row.median_promo_price)
if not score or np.isnan(score):
score = -1
self.common.write_to_db_result(
fk=kpi.iloc[0].pk,
numerator_id=numerator_id, # its the POSM to check or {General Empty if not recognized or multi}
numerator_result=numerator_result, # whether POSM is 0, 1 or 2
denominator_id=each_row.item_id, # each product in scene
score=score, # -1 means not saved in DB
result=result, # 0-optional, 1-mandatory, 2- NA
context_id=context_fk, # template of scene
by_scene=True,
)
# Save All Displays in the scene
Log.info('Save all displays in the scene. The session: {sess} - scene: {scene}.'
.format(sess=self.session_uid, scene=current_scene_fk))
kpi_all_displays_in_scene = self.kpi_static_data[
(self.kpi_static_data[KPI_TYPE_COL] == GSK_DISPLAYS_ALL_IN_SCENE)
& (self.kpi_static_data['delete_time'].isnull())]
for idx, each_row in self.match_display_in_scene.iterrows():
self.common.write_to_db_result(
fk=kpi_all_displays_in_scene.iloc[0].pk,
numerator_id=each_row.display_fk, # the Display/POSM present in the scene
numerator_result=1, # no meaning
denominator_id=each_row.display_brand_fk, # display brand
score=1, # no meaning
result=1, # no meaning
context_id=self.store_id, # template of scene
by_scene=True,
)
return True
class SINOTHSceneToolBox:
def __init__(self, data_provider, output, common):
self.output = output
self.data_provider = data_provider
self.common = common
self.project_name = self.data_provider.project_name
self.session_uid = self.data_provider.session_uid
self.products = self.data_provider[Data.PRODUCTS]
self.templates = self.data_provider[Data.TEMPLATES]
self.all_products = self.data_provider[Data.ALL_PRODUCTS]
self.scif = self.data_provider.scene_item_facts
self.match_product_in_scene = self.data_provider[Data.MATCHES]
self.visit_date = self.data_provider[Data.VISIT_DATE]
self.session_info = self.data_provider[Data.SESSION_INFO]
self.scene_info = self.data_provider[Data.SCENES_INFO]
self.current_scene_fk = self.scene_info.iloc[0].scene_fk
self.store_info = self.data_provider[Data.STORE_INFO]
self.store_id = self.store_info.iloc[0].store_fk
self.store_type = self.data_provider.store_type
self.rds_conn = PSProjectConnector(self.project_name,
DbUsers.CalculationEng)
self.kpi_static_data = self.common.get_kpi_static_data()
self.ps_data_provider = PsDataProvider(self.data_provider, self.output)
self.targets = self.ps_data_provider.get_kpi_external_targets()
self.match_display_in_scene = self.data_provider.match_display_in_scene
self.scene_template_info = self.scif[[
'scene_fk', 'template_fk', 'template_name'
]].drop_duplicates()
self.kpi_template_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), '..',
TEMPLATE_PARENT_FOLDER, TEMPLATE_NAME)
self.kpi_template = pd.ExcelFile(self.kpi_template_path)
self.own_man_fk = OWN_DISTRIBUTOR_FK
self.kpi_template = pd.ExcelFile(self.kpi_template_path)
self.empty_prod_ids = self.all_products[
self.all_products.product_name.str.contains(
'empty', case=False)]['product_fk'].values
self.irrelevant_prod_ids = self.all_products[
self.all_products.product_name.str.contains(
'irrelevant', case=False)]['product_fk'].values
self.other_prod_ids = self.all_products[
self.all_products.product_name.str.contains(
'other', case=False)]['product_fk'].values
def filter_and_send_kpi_to_calc(self):
kpi_sheet = self.kpi_template.parse(KPI_NAMES_SHEET)
kpi_sheet[KPI_FAMILY_COL] = kpi_sheet[KPI_FAMILY_COL].fillna(
method='ffill')
kpi_details = self.kpi_template.parse(KPI_DETAILS_SHEET)
kpi_include_exclude = self.kpi_template.parse(KPI_INC_EXC_SHEET)
for index, kpi_sheet_row in kpi_sheet.iterrows():
if not is_nan(kpi_sheet_row[KPI_ACTIVE]):
if str(kpi_sheet_row[KPI_ACTIVE]).strip().lower() in [
'0.0', 'n', 'no'
]:
Log.warning("KPI :{} deactivated in sheet.".format(
kpi_sheet_row[KPI_NAME_COL]))
continue
kpi = self.kpi_static_data[
(self.kpi_static_data[KPI_TYPE_COL] ==
kpi_sheet_row[KPI_NAME_COL])
& (self.kpi_static_data['delete_time'].isnull())]
if kpi.empty or kpi.session_relevance.values[0] == 1:
Log.info(
"*** KPI Name:{name} not found in DB or is a SESSION LEVEL KPI for scene {scene} ***"
.format(name=kpi_sheet_row[KPI_NAME_COL],
scene=self.current_scene_fk))
continue
else:
Log.info(
"KPI Name:{name} found in DB for scene {scene}".format(
name=kpi_sheet_row[KPI_NAME_COL],
scene=self.current_scene_fk))
detail = kpi_details[kpi_details[KPI_NAME_COL] ==
kpi[KPI_TYPE_COL].values[0]]
# check for store types allowed
permitted_store_types = [
x.strip().lower()
for x in detail[STORE_POLICY].values[0].split(',')
if x.strip()
]
if self.store_info.store_type.iloc[0].lower(
) not in permitted_store_types:
if permitted_store_types and permitted_store_types[
0] != "all":
Log.warning(
"Not permitted store type - {type} for scene {scene}"
.format(type=kpi_sheet_row[KPI_NAME_COL],
scene=self.current_scene_fk))
continue
detail['pk'] = kpi['pk'].iloc[0]
# gather details
groupers, query_string = get_groupers_and_query_string(detail)
kpi_include_exclude = kpi_include_exclude[
kpi_include_exclude.kpi_name != ASSORTMENTS]
_include_exclude = kpi_include_exclude[
kpi_details[KPI_NAME_COL] == kpi[KPI_TYPE_COL].values[0]]
# gather include exclude
include_exclude_data_dict = get_include_exclude(
_include_exclude)
dataframe_to_process = self.get_sanitized_match_prod_scene(
include_exclude_data_dict)
# hack to cast all other than OWN_DISTRIBUTOR to non-sino
non_sino_index = dataframe_to_process[
OWN_CHECK_COL] != OWN_DISTRIBUTOR
dataframe_to_process.loc[non_sino_index,
OWN_CHECK_COL] = 'non-sino'
if kpi_sheet_row[KPI_FAMILY_COL] in [FSOS, SIMON]:
self.calculate_fsos(detail, groupers, query_string,
dataframe_to_process)
else:
Log.error(
"From project: {proj}. Unexpected kpi_family: {type}. Please check."
.format(type=kpi_sheet_row[KPI_FAMILY_COL],
proj=self.project_name))
pass
return True
def calculate_fsos(self, kpi, groupers, query_string,
dataframe_to_process):
Log.info("Calculate {name} for scene {scene}".format(
name=kpi.kpi_name.iloc[0], scene=self.current_scene_fk))
if query_string:
grouped_data_frame = dataframe_to_process.query(
query_string).groupby(groupers)
else:
grouped_data_frame = dataframe_to_process.groupby(groupers)
for group_id_tup, group_data in grouped_data_frame:
if type(group_id_tup) not in [tuple, list]:
# convert to a tuple
group_id_tup = group_id_tup,
param_id_map = dict(zip(groupers, group_id_tup))
# the hack! This casts the value of distributor to that in DB as custom entity.
if OWN_CHECK_COL in param_id_map:
distributor_name = param_id_map.pop(OWN_CHECK_COL)
if distributor_name == OWN_DISTRIBUTOR:
param_id_map[
OWN_CHECK_COL] = self.own_man_fk # as per custom entity
else:
param_id_map[
OWN_CHECK_COL] = OTHER_DISTRIBUTOR_FK # as per custom entity
param_id_map['manufacturer_fk'] = param_id_map[OWN_CHECK_COL]
# SET THE numerator, denominator and context
numerator_id = param_id_map.get(
PARAM_DB_MAP[kpi['numerator'].iloc[0]]['key'])
if numerator_id is None:
raise Exception(
"Numerator cannot be null. Check SinoTH KPIToolBox [calculate_fsos]."
)
denominator_id = get_parameter_id(
key_value=PARAM_DB_MAP[kpi['denominator'].iloc[0]]['key'],
param_id_map=param_id_map)
if denominator_id is None:
# because 0 is good; check None specifically
denominator_id = self.store_id
context_id = get_parameter_id(
key_value=PARAM_DB_MAP[kpi['context'].iloc[0]]['key'],
param_id_map=param_id_map)
if context_id is None:
# because 0 is good
context_id = self.store_id
if PARAM_DB_MAP[kpi['denominator'].iloc[0]]['key'] == 'store_fk':
denominator_df = dataframe_to_process
else:
denominator_df = dataframe_to_process.query(
'{key} == {value}'.format(
key=PARAM_DB_MAP[kpi['denominator'].iloc[0]]['key'],
value=denominator_id))
if not len(denominator_df):
Log.error(
"No denominator data for session {sess} and scene {scene} to calculate {name}"
.format(sess=self.session_uid,
name=kpi.kpi_name.iloc[0],
scene=self.current_scene_fk))
raise Exception(
"Denominator data cannot be null. Check SinoTH KPIToolBox [calculate_fsos]."
)
result = len(group_data) / float(len(denominator_df))
# its the parent. Save the identifier result.
self.common.write_to_db_result(
fk=kpi['pk'].iloc[0],
numerator_id=numerator_id,
denominator_id=denominator_id,
context_id=context_id,
result=result,
numerator_result=len(group_data),
denominator_result=len(denominator_df),
identifier_result="{}_{}_{}_{}".format(
kpi['kpi_name'].iloc[0],
kpi['pk'].iloc[0],
# numerator_id,
denominator_id,
context_id,
),
should_enter=True,
by_scene=True,
)
return True
def get_sanitized_match_prod_scene(self, include_exclude_data_dict):
scene_product_data = self.match_product_in_scene.merge(
self.products,
how='left',
on=['product_fk'],
suffixes=('', '_prod'))
sanitized_products_in_scene = scene_product_data.merge(
self.scene_template_info,
how='left',
on='scene_fk',
suffixes=('', '_scene'))
# flags
include_empty = include_exclude_data_dict.get('empty')
include_irrelevant = include_exclude_data_dict.get('irrelevant')
include_others = include_exclude_data_dict.get('others')
include_stacking = include_exclude_data_dict.get('stacking')
# list
scene_types_to_include = include_exclude_data_dict.get(
'scene_types_to_include', False)
categories_to_include = include_exclude_data_dict.get(
'categories_to_include', False)
brands_to_include = include_exclude_data_dict.get(
'brands_to_include', False)
ean_codes_to_include = include_exclude_data_dict.get(
'ean_codes_to_include', False)
# Start include items
if scene_types_to_include:
# list of scene types to include is present, otherwise all included
Log.info("Include template/scene type {}".format(
scene_types_to_include))
sanitized_products_in_scene = sanitized_products_in_scene[
sanitized_products_in_scene['template_name'].str.upper().isin([
x.upper() if type(x) in [unicode, str] else x
for x in scene_types_to_include
])]
if not include_stacking:
# exclude stacking if the flag is set
Log.info(
"Exclude stacking other than in layer 1 or negative stacking [menu]"
)
sanitized_products_in_scene = sanitized_products_in_scene.loc[
sanitized_products_in_scene['stacking_layer'] <= 1]
if categories_to_include:
# list of categories to include is present, otherwise all included
Log.info("Include categories {}".format(categories_to_include))
sanitized_products_in_scene = sanitized_products_in_scene[
sanitized_products_in_scene['category'].str.upper().isin([
x.upper() if type(x) in [unicode, str] else x
for x in categories_to_include
])]
if brands_to_include:
# list of brands to include is present, otherwise all included
Log.info("Include brands {}".format(brands_to_include))
sanitized_products_in_scene = sanitized_products_in_scene[
sanitized_products_in_scene['brand_name'].str.upper().isin([
x.upper() if type(x) in [unicode, str] else x
for x in brands_to_include
])]
if ean_codes_to_include:
# list of ean_codes to include is present, otherwise all included
Log.info("Include ean codes {}".format(ean_codes_to_include))
sanitized_products_in_scene = sanitized_products_in_scene[
sanitized_products_in_scene['product_ean_code'].str.upper(
).isin([
x.upper() if type(x) in [unicode, str] else x
for x in ean_codes_to_include
])]
product_ids_to_exclude = []
if not include_irrelevant:
# add product ids to exclude with irrelevant
product_ids_to_exclude.extend(self.irrelevant_prod_ids)
if not include_others:
# add product ids to exclude with others
product_ids_to_exclude.extend(self.other_prod_ids)
if not include_empty:
# add product ids to exclude with empty
product_ids_to_exclude.extend(self.empty_prod_ids)
if product_ids_to_exclude:
Log.info("Exclude product ids {}".format(product_ids_to_exclude))
sanitized_products_in_scene.drop(sanitized_products_in_scene[
sanitized_products_in_scene['product_fk'].isin(
product_ids_to_exclude)].index,
inplace=True)
return sanitized_products_in_scene
class PillarsSceneToolBox:
PROGRAM_TEMPLATE_PATH = os.path.join(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 'Data',
'CCUS_Templatev7_February2020.xlsx')
BITWISE_RECOGNIZER_SIZE = 6
RECOGNIZED_BY_POS = BITWISE_RECOGNIZER_SIZE - 1
RECOGNIZED_BY_SCENE_RECOGNITION = BITWISE_RECOGNIZER_SIZE - 2
RECOGNIZED_BY_QURI = BITWISE_RECOGNIZER_SIZE - 3
RECOGNIZED_BY_SURVEY = BITWISE_RECOGNIZER_SIZE - 4
def __init__(self, data_provider, output, common):
self.data_provider = data_provider
self.common = common
self.project_name = self.data_provider.project_name
self.session_uid = self.data_provider.session_uid
self.templates = self.data_provider[Data.TEMPLATES]
self.all_products = self.data_provider[Data.ALL_PRODUCTS]
self.match_product_in_scene = self.data_provider[Data.MATCHES]
empties = self.all_products[self.all_products['product_type'] ==
'Empty']['product_fk'].unique().tolist()
self.match_product_in_scene = self.match_product_in_scene[~(
self.match_product_in_scene['product_fk'].isin(empties))]
self.visit_date = self.data_provider[Data.VISIT_DATE]
self.scene_info = self.data_provider[Data.SCENES_INFO]
self.template_fk = self.templates['template_fk'].iloc[0]
self.scene_id = self.scene_info['scene_fk'][0]
self.store_id = self.data_provider[Data.STORE_INFO]['store_fk'][0]
# self.kpi_fk = self.common.get_kpi_fk_by_kpi_name(Const.POC)
self.all_brand = self.all_products[[
'brand_name', 'brand_fk'
]].drop_duplicates().set_index(u'brand_name')
self.displays_in_scene = self.data_provider.match_display_in_scene
self.ps_data_provider = PsDataProvider(self.data_provider, output)
# bit-like sequence to symoblize recognizing methods. Each 'bit' symbolize a recognition method.
self.bitwise_for_program_identifier_as_list = list(
"0" * self.BITWISE_RECOGNIZER_SIZE)
def is_scene_belong_to_program(self):
# Get template (from file or from external targets)
relevant_programs = self.get_programs(template=True)
for i in xrange(len(relevant_programs)):
# Get data for program from template:
current_program_data = relevant_programs.iloc[i]
program_name = current_program_data[
Const.PROGRAM_NAME_FIELD] # assumed to always be brand name!
program_brand_name_fk = self.get_brand_fk_from_name(program_name)
program_as_brand = current_program_data[
Const.PROGRAM_NAME_BY_BRAND]
program_as_brand_fk = self.get_brand_fk_from_name(program_as_brand)
program_as_display_brand = current_program_data[
Const.PROGRAM_NAME_BY_DISPLAY]
program_as_template = current_program_data[
Const.PROGRAM_NAME_BY_TEMPLATE]
survey_question_for_program = current_program_data[
Const.PROGRAM_NAME_BY_SURVEY_QUESTION]
program_as_survey_answer = current_program_data[
Const.PROGRAM_NAME_BY_SURVEY_ANSWER]
score = 0
# Checks if the scene was recognized as relevant program in one of possible recognition options:
self.bitwise_for_program_identifier_as_list[self.RECOGNIZED_BY_POS] = \
1 if self.found_program_products_by_brand(program_as_brand_fk) else 0
self.bitwise_for_program_identifier_as_list[self.RECOGNIZED_BY_SCENE_RECOGNITION] = \
1 if self.found_scene_program_by_display_brand(program_as_display_brand) else 0
self.bitwise_for_program_identifier_as_list[self.RECOGNIZED_BY_QURI] = \
1 if self.found_scene_program_by_quri(program_as_template) else 0
self.bitwise_for_program_identifier_as_list[self.RECOGNIZED_BY_SURVEY] = \
1 if self.found_scene_program_by_survey(survey_question_for_program, program_as_survey_answer) else 0
# convert list of bits to a string in order to convert to decimal in results:
bitwise_for_program_identifier_as_str = ''.join(
map(str, self.bitwise_for_program_identifier_as_list))
# convert string of binary-like to decimal value
method_recognized_in_bitwise = int(
bitwise_for_program_identifier_as_str, BINARY)
score = 1 if method_recognized_in_bitwise > 0 else 0
scene_kpi_fk = self.common.get_kpi_fk_by_kpi_name(
kpi_name=Const.SCENE_KPI_NAME)
self.common.write_to_db_result(fk=scene_kpi_fk,
numerator_id=program_brand_name_fk,
result=method_recognized_in_bitwise,
score=score,
by_scene=True,
denominator_id=self.store_id)
def get_programs(self, template=False):
"""
This function gets the relevant programs from template/ external targets list.
:param template: if True takes the template from Data folder in project.
Else, takes the kpi's external targets.
:return:
"""
if template:
programs = pd.read_excel(self.PROGRAM_TEMPLATE_PATH)
else:
programs = self.ps_data_provider.get_kpi_external_targets(
["Pillars Programs KPI"])
if programs.empty:
return programs
# Get only relevant programs to check
relevant_programs = programs.loc[
(programs['start_date'].dt.date <= self.visit_date)
& (programs['end_date'].dt.date >= self.visit_date)]
return relevant_programs
def get_brand_fk_from_name(self, brand_name):
if pd.isnull(brand_name):
return
fk = self.all_brand.loc[brand_name]
if not fk.empty:
fk = fk.values[0]
else:
fk = None
return fk
def found_program_products_by_brand(self, brand_fk=None, brand_name=None):
"""
This function can get brand either by fk or by name, with the assumption that in the
'customer' template there is brand name and in the db targets there is pk.
If none of the option was used, return False.
Otherwise return if there were product in this scene with the brand given.
"""
# checks if the scene's program was discovered by trax according to brand's recognized products
if pd.isnull(brand_name) and pd.isnull(brand_fk):
return False
brand_id = 'brand_fk' if brand_fk else 'brand_name'
brand_value = brand_fk if brand_fk else brand_name
pos_products_in_brand = self.all_products[
(self.all_products['product_type'] == 'POS')
& (self.all_products[brand_id] == brand_value
)]['product_fk'].unique().tolist()
program_products_in_scene = self.match_product_in_scene[(
self.match_product_in_scene['product_fk'].isin(
pos_products_in_brand))]
return len(program_products_in_scene) > 0
def found_scene_program_by_quri(self, template_name):
# checks if the scene's program was discovered by quri according to the template name
if pd.isnull(template_name):
return False
if self.templates['template_fk'].empty:
return False
return template_name in self.templates['template_name'].values
def found_scene_program_by_survey(
self, survey_question,
survey_answer): # TODO- complete after survey created
# Cannot complete until found relevant tables
if pd.isnull(survey_question) or pd.isnull(survey_answer):
return False
pass
def found_scene_program_by_display_brand(self, brand_name):
# checks if the scene's program was discovered by scene recognition
if pd.isnull(brand_name):
return False
if self.data_provider.match_display_in_scene.empty:
return False
display_in_brand = len(self.data_provider.match_display_in_scene.loc[
self.data_provider.match_display_in_scene['display_brand_name'] ==
brand_name])
return display_in_brand > 0
class MenuToolBox(GlobalSessionToolBox):
def __init__(self, data_provider, common):
GlobalSessionToolBox.__init__(self, data_provider, None)
# self.matches = self.get_filtered_matches()
self.store_number = self.store_info.store_number_1.iloc[0]
self.ps_data_provider = PsDataProvider(self.data_provider, self.output)
self.rds_conn = PSProjectConnector(self.project_name, DbUsers.CalcAdmin)
self.custom_entity = self.get_custom_entity()
self.targets = self.ps_data_provider.get_kpi_external_targets(kpi_fks=[6006],
key_fields=['store_number_1', 'product_fk'],
key_filters={'store_number_1': self.store_number})
self.common = common
def main_calculation(self):
"""This method calculates the entire Menu Brand KPIs set."""
self.menu_count()
def menu_count(self):
kpi_fk = self.get_kpi_fk_by_kpi_type(Consts.MENU_KPI_CHILD)
parent_kpi = self.get_kpi_fk_by_kpi_type(Consts.TOTAL_MENU_KPI_SCORE)
# we need to save a second set of KPIs with heirarchy for the mobile report
kpi_fk_mr = self.get_kpi_fk_by_kpi_type(Consts.MENU_KPI_CHILD_MR)
parent_kpi_mr = self.get_kpi_fk_by_kpi_type(Consts.TOTAL_MENU_KPI_SCORE_MR)
if self.targets.empty:
return
try:
menu_product_fks = [t for t in self.targets.product_fk.unique().tolist() if pd.notna(t)]
except AttributeError:
Log.warning('Menu Count targets are corrupt for this store')
return
filtered_scif = self.scif[self.scif['template_group'].str.contains('Menu')]
present_menu_scif_sub_brands = filtered_scif.sub_brand.unique().tolist()
passed_products = 0
for product_fk in menu_product_fks:
result = 0
sub_brand = self.all_products['sub_brand'][self.all_products['product_fk'] == product_fk].iloc[0]
custom_entity_df = self.custom_entity['pk'][self.custom_entity['name'] == sub_brand]
if custom_entity_df.empty:
custom_entity_pk = -1
else:
custom_entity_pk = custom_entity_df.iloc[0]
if sub_brand in present_menu_scif_sub_brands:
result = 1
passed_products += 1
self.write_to_db(fk=kpi_fk_mr, numerator_id=product_fk, numerator_result=0, denominator_result=0,
denominator_id=custom_entity_pk,
result=result, score=0, identifier_parent=parent_kpi_mr, identifier_result=kpi_fk_mr,
should_enter=True)
self.write_to_db(fk=kpi_fk, numerator_id=product_fk, numerator_result=0, denominator_result=0,
denominator_id=custom_entity_pk, result=result, score=0)
target_products = len(menu_product_fks)
self.write_to_db(fk=parent_kpi_mr, numerator_id=self.manufacturer_fk, numerator_result=0, denominator_result=0,
denominator_id=self.store_id,
result=passed_products, score=0, target=target_products, identifier_result=parent_kpi_mr)
self.write_to_db(fk=parent_kpi, numerator_id=self.manufacturer_fk, numerator_result=0, denominator_result=0,
denominator_id=self.store_id,
result=passed_products, score=0, target=target_products)
def get_custom_entity(self):
query = DiageoQueries.get_custom_entities_query()
query_result = pd.read_sql_query(query, self.rds_conn.db)
return query_result
class JRIJPToolBox:
LEVEL1 = 1
LEVEL2 = 2
LEVEL3 = 3
def __init__(self, data_provider, output):
self.output = output
self.data_provider = data_provider
self.common = Common(self.data_provider)
self.match_display_in_scene = self.data_provider.match_display_in_scene
self.project_name = self.data_provider.project_name
self.session_uid = self.data_provider.session_uid
self.products = self.data_provider[Data.PRODUCTS]
self.all_products = self.data_provider[Data.ALL_PRODUCTS]
self.scene_info = self.data_provider[Data.SCENES_INFO]
self.templates = self.data_provider[Data.TEMPLATES]
self.match_product_in_scene = self.data_provider[Data.MATCHES]
self.visit_date = self.data_provider[Data.VISIT_DATE]
self.session_info = self.data_provider[Data.SESSION_INFO]
self.scene_info = self.data_provider[Data.SCENES_INFO]
self.store_id = self.data_provider[Data.STORE_FK]
self.scif = self.data_provider[Data.SCENE_ITEM_FACTS]
self.rds_conn = PSProjectConnector(self.project_name, DbUsers.CalculationEng)
self.ps_data_provider = PsDataProvider(self.data_provider, self.output)
self.kpi_static_data = self.common.get_kpi_static_data()
self.kpi_results_queries = []
self.templates_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'Data')
self.excel_file_path = os.path.join(self.templates_path, 'Template.xlsx')
self.external_targets = self.ps_data_provider.get_kpi_external_targets(
kpi_operation_types=["Target Config"],
key_fields=["product_fks", "template_fks", "product_group_fk"],
data_fields=["stacking_exclude", "min_product_facing", "best_shelf_position", "group_facings_count"]
)
def main_calculation(self, *args, **kwargs):
"""
This function calculates the KPI results.
Important:
The name of the KPI is used to name the function to calculate it.
if kpi_name is *test_calc*; the function will be *calculate_test_calc*
"""
self.calculate_config_related()
self.parse_and_send_kpi_to_calc()
self.common.commit_results_data()
return
def calculate_config_related(self):
if self.external_targets.empty:
Log.info("Not calculating Config related KPIs for Canvas."
"External Targets empty while running session: {}".format(self.session_uid))
return True
product_presence_from_target_pk = self.kpi_static_data[
(self.kpi_static_data[KPI_FAMILY] == PS_KPI_FAMILY)
& (self.kpi_static_data[TYPE] == PRODUCT_PRESENCE_FROM_TARGET)
& (self.kpi_static_data['delete_time'].isnull())].iloc[0].pk
product_position_from_target_pk = self.kpi_static_data[
(self.kpi_static_data[KPI_FAMILY] == PS_KPI_FAMILY)
& (self.kpi_static_data[TYPE] == PRODUCT_POSITION_FROM_TARGET)
& (self.kpi_static_data['delete_time'].isnull())].iloc[0].pk
product_facing_from_target_pk = self.kpi_static_data[
(self.kpi_static_data[KPI_FAMILY] == PS_KPI_FAMILY)
& (self.kpi_static_data[TYPE] == PRODUCT_FACING_FROM_TARGET)
& (self.kpi_static_data['delete_time'].isnull())].iloc[0].pk
overall_result_from_target_pk = self.kpi_static_data[
(self.kpi_static_data[KPI_FAMILY] == PS_KPI_FAMILY)
& (self.kpi_static_data[TYPE] == OVERALL_RESULT_FROM_TARGET)
& (self.kpi_static_data['delete_time'].isnull())].iloc[0].pk
overall_score_from_target_pk = self.kpi_static_data[
(self.kpi_static_data[KPI_FAMILY] == PS_KPI_FAMILY)
& (self.kpi_static_data[TYPE] == OVERALL_SCORE_FROM_TARGET)
& (self.kpi_static_data['delete_time'].isnull())].iloc[0].pk
match_prod_in_scene_data = self.match_product_in_scene \
.merge(self.scene_info, on='scene_fk', suffixes=('', '_scene')) \
.merge(self.templates, on='template_fk', suffixes=('', '_template'))
self.external_targets.fillna('', inplace=True)
for index, each_target in self.external_targets.iterrows():
_each_target_dict = each_target.to_dict()
group_fk = _each_target_dict.get('product_group_fk')
product_fks = _each_target_dict.get('product_fks')
if type(product_fks) != list:
product_fks = [product_fks]
best_shelf_position = _each_target_dict.get('best_shelf_position')
if type(best_shelf_position) != list:
best_shelf_position = [best_shelf_position]
min_product_facing = _each_target_dict.get('min_product_facing', 0)
template_fks = _each_target_dict.get('template_fks')
if template_fks and type(template_fks) != list:
template_fks = [template_fks]
stacking_exclude = _each_target_dict.get('stacking_exclude')
min_group_product_facing = _each_target_dict.get('group_facings_count', 0)
# get mpis based on details
filtered_mpis = match_prod_in_scene_data
if template_fks:
filtered_mpis = filtered_mpis[match_prod_in_scene_data['template_fk'].isin(template_fks)]
if stacking_exclude == '1':
filtered_mpis = filtered_mpis[match_prod_in_scene_data['stacking_layer'] == 1]
product_presence_data = self.calculate_product_presence(
kpi_pk=product_presence_from_target_pk,
filtere_mpis=filtered_mpis,
group_fk=group_fk,
product_fks=product_fks,
min_product_facing=min_product_facing
)
product_position_data = self.calculate_product_position(
kpi_pk=product_position_from_target_pk,
filtere_mpis=filtered_mpis,
group_fk=group_fk,
product_fks=product_fks,
best_shelf_position=best_shelf_position
)
product_facings_data = self.calculate_product_facings(
kpi_pk=product_facing_from_target_pk,
filtere_mpis=filtered_mpis,
group_fk=group_fk,
product_fks=product_fks
)
self.calculate_overall_result_and_score(result_kpi_pk=overall_result_from_target_pk,
score_kpi_fk=overall_score_from_target_pk,
group_fk=group_fk,
product_presence_data=product_presence_data,
product_position_data=product_position_data,
product_facings_data=product_facings_data,
best_shelf_position=best_shelf_position,
min_group_product_facing=min_group_product_facing
)
pass
def calculate_product_presence(self, kpi_pk, filtere_mpis, group_fk, product_fks, min_product_facing):
data = {}
for each_product in product_fks:
prod_data_in_mpis = filtere_mpis[filtere_mpis['product_fk'] == each_product]
result = 0
if len(prod_data_in_mpis) >= int(min_product_facing):
result = 1
Log.info("Saving product presence for product: {product} as {result} in session {sess} in group: {group}"
.format(product=each_product,
result=result,
sess=self.session_uid,
group=group_fk
))
data[each_product] = result
self.common.write_to_db_result(fk=kpi_pk,
numerator_id=group_fk,
denominator_id=each_product,
context_id=self.all_products[self.all_products['product_fk']
==each_product].category_fk.iloc[0],
result=result,
score=result,
)
return data
def calculate_product_position(self, kpi_pk, filtere_mpis, group_fk, product_fks, best_shelf_position):
data = {}
for each_product in product_fks:
prod_data_in_mpis = filtere_mpis[filtere_mpis['product_fk'] == each_product]
result = 0 # best shelf from top
score = 0 # best shelf position from top in CONFIG?
if prod_data_in_mpis.empty:
Log.info("Position KPI => Product: {} not found in session: {} for group: {}".format(
each_product, self.session_uid, group_fk
))
result = 0
score = 0
else:
prod_data_in_mpis_sorted = prod_data_in_mpis.sort_values(by=['shelf_number'])
result = prod_data_in_mpis_sorted.iloc[0]['shelf_number'] # => presence_lowest_shelf
if result in [int(x) for x in best_shelf_position if x.strip()]:
score = 1
Log.info("Saving product position for product: {product} as"
" lowest={result}/is in config={score} in session"
" {sess} in group: {group}"
.format(product=each_product,
result=result,
score=score,
sess=self.session_uid,
group=group_fk
))
data[each_product] = (result, score)
self.common.write_to_db_result(fk=kpi_pk,
numerator_id=group_fk,
denominator_id=each_product,
context_id=self.all_products[self.all_products['product_fk']
==each_product].category_fk.iloc[0],
result=result,
score=score,
)
return data
def calculate_product_facings(self, kpi_pk, filtere_mpis, group_fk, product_fks):
data = {}
for each_product in product_fks:
prod_data_in_mpis = filtere_mpis[filtere_mpis['product_fk'] == each_product]
if prod_data_in_mpis.empty:
Log.info("Facings KPI => Product: {} not found in session: {} for group: {}".format(
each_product, self.session_uid, group_fk
))
result = 0
else:
result = len(prod_data_in_mpis)
Log.info("Saving product facings for product: {product} as {result} in session {sess} in group: {group}"
.format(product=each_product,
result=result,
sess=self.session_uid,
group=group_fk
))
data[each_product] = result
self.common.write_to_db_result(fk=kpi_pk,
numerator_id=group_fk,
denominator_id=each_product,
context_id=self.all_products[self.all_products['product_fk']
==each_product].category_fk.iloc[0],
result=result,
score=result,
)
return data
def calculate_overall_result_and_score(self, result_kpi_pk, score_kpi_fk, group_fk,
product_presence_data, product_position_data,
product_facings_data, best_shelf_position,
min_group_product_facing):
numerator_result = 1 in product_presence_data.values()
# product_position_data -> (min_shelf, is_in_config)
min_level_of_product = 0
if product_position_data:
_score_products_presence = filter(lambda x: x[1] == 1, product_position_data.values())
if _score_products_presence:
# find min result among the in config ones
_score_products_presence.sort(key=lambda x: x[0])
min_level_of_product = int(_score_products_presence[0][0])
else:
# score is all 0
_score_products_presence_present = filter(lambda x: x[0] != 0, product_position_data.values())
_score_products_presence_present.sort(key=lambda x: x[0])
if _score_products_presence_present:
min_level_of_product = _score_products_presence_present[0][0]
else:
Log.info("Product presence information is empty for any product in group = {group_fk}".format(
group_fk=group_fk
))
Log.info("Saving Overall Result for group: {group_fk} in session {sess}"
.format(group_fk=group_fk,
sess=self.session_uid,
))
self.common.write_to_db_result(fk=result_kpi_pk,
numerator_id=group_fk,
denominator_id=self.store_id,
context_id=self.store_id,
numerator_result=int(numerator_result), # bool to int
denominator_result=min_level_of_product,
result=sum(product_facings_data.values()), # bool to int
)
Log.info("Saving Overall Score for group: {group_fk} in session {sess}"
.format(group_fk=group_fk,
sess=self.session_uid,
))
is_in_best_shelf = False
if best_shelf_position and min_level_of_product:
if type(best_shelf_position) != list:
best_shelf_position = [best_shelf_position]
is_in_best_shelf = min_level_of_product in map(lambda x: int(x), best_shelf_position)
has_minumum_facings_per_config = False
if not min_group_product_facing or not min_group_product_facing.strip():
min_group_product_facing = 0
if sum(product_facings_data.values()) >= int(min_group_product_facing):
has_minumum_facings_per_config = True
self.common.write_to_db_result(fk=score_kpi_fk,
numerator_id=group_fk,
denominator_id=self.store_id,
context_id=self.store_id,
numerator_result=int(numerator_result), # bool to int
denominator_result=int(is_in_best_shelf), # bool to int
result=int(has_minumum_facings_per_config), # bool to int
score=int(all([numerator_result, is_in_best_shelf,
has_minumum_facings_per_config])) # bool to int
)
def parse_and_send_kpi_to_calc(self):
kpi_sheet = self.get_template_details(KPI_SHEET_NAME)
for index, kpi_sheet_row in kpi_sheet.iterrows():
kpi = self.kpi_static_data[(self.kpi_static_data[KPI_FAMILY] == PS_KPI_FAMILY)
& (self.kpi_static_data[TYPE] == kpi_sheet_row[KPI_TYPE])
& (self.kpi_static_data['delete_time'].isnull())]
if kpi.empty:
Log.info("KPI Name:{} not found in DB".format(kpi_sheet_row[KPI_NAME]))
else:
Log.info("KPI Name:{} found in DB".format(kpi_sheet_row[KPI_NAME]))
kpi_method_to_calc = getattr(self, 'calculate_{kpi}'.format(kpi=kpi_sheet_row[KPI_NAME].lower()), None)
if not kpi_method_to_calc:
Log.warning("Method not defined for KPI Name:{}.".format(kpi_sheet_row[KPI_NAME].lower()))
pass
kpi_fk = kpi.pk.values[0]
kpi_method_to_calc(kpi_fk)
def calculate_count_posm_per_scene(self, kpi_fk):
if self.match_display_in_scene.empty:
Log.info("No POSM detected at scene level for session: {}".format(self.session_uid))
return False
grouped_data = self.match_display_in_scene.groupby(['scene_fk', 'display_fk'])
for data_tup, scene_data_df in grouped_data:
scene_fk, display_fk = data_tup
posm_count = len(scene_data_df)
template_fk = self.scene_info[self.scene_info['scene_fk'] == scene_fk].get('template_fk')
if not template_fk.empty:
cur_template_fk = int(template_fk)
else:
Log.info("JRIJP: Scene ID {scene} is not complete and not found in scene Info.".format(
scene=scene_fk))
continue
self.common.write_to_db_result(fk=kpi_fk,
numerator_id=display_fk,
denominator_id=self.store_id,
context_id=cur_template_fk,
result=posm_count,
score=scene_fk)
def calculate_facings_in_cell_per_product(self, kpi_fk):
match_prod_scene_data = self.match_product_in_scene.merge(
self.products, how='left', on='product_fk', suffixes=('', '_prod'))
grouped_data = match_prod_scene_data.query(
'(stacking_layer==1) or (product_type=="POS")'
).groupby(
['scene_fk', 'bay_number', 'shelf_number', 'product_fk']
)
for data_tup, scene_data_df in grouped_data:
scene_fk, bay_number, shelf_number, product_fk = data_tup
facings_count_in_cell = len(scene_data_df)
cur_template_fk = int(self.scene_info[self.scene_info['scene_fk'] == scene_fk].get('template_fk'))
self.common.write_to_db_result(fk=kpi_fk,
numerator_id=product_fk,
denominator_id=self.store_id,
context_id=cur_template_fk,
numerator_result=bay_number,
denominator_result=shelf_number,
result=facings_count_in_cell,
score=scene_fk)
def get_template_details(self, sheet_name):
template = pd.read_excel(self.excel_file_path, sheetname=sheet_name)
return template
class SceneToolBox(GlobalSceneToolBox):
def __init__(self, data_provider, output):
GlobalSceneToolBox.__init__(self, data_provider, output)
self.ps_data_provider = PsDataProvider(self.data_provider, self.output)
self.rds_conn = PSProjectConnector(self.project_name, DbUsers.CalculationEng)
self.targets = self.ps_data_provider.get_kpi_external_targets(key_fields=['KPI Type'], data_fields=[
'Location: JSON',
'Config Params: JSON',
'Dataset 1: JSON'])
self.gold_zone_scene_location_kpi = ['Lobby/Entrance', 'Main Alley/Hot Zone', 'Gold Zone End Cap',
'Lobby/Main Entrance']
def main_function(self):
self.calculate_scene_location()
return
def calculate_scene_location(self):
scene_location_kpi_template = self.targets[self.targets[Consts.KPI_TYPE].isin(['Scene Location'])]
for i, row in scene_location_kpi_template.iterrows():
row = self.apply_json_parser(row)
return_holder = self._get_kpi_name_and_fk(row)
relevant_scif = self._parse_json_filters_to_df(row)
if relevant_scif.empty:
return
scene_store_area_df = self.get_store_area_df()
scene_store_area_df = scene_store_area_df[scene_store_area_df.scene_fk.isin(relevant_scif.scene_fk.unique())]
# scene_store_area_df['result'] = scene_store_area_df.name.apply(
# lambda x: 1 if x in self.gold_zone_scene_location_kpi else 0)
scene_store_area_df['result'] = np.in1d(scene_store_area_df.name.values,
self.gold_zone_scene_location_kpi) * 1
for store_area_row in scene_store_area_df.itertuples():
self.common.write_to_db_result(fk=return_holder[1], numerator_id=store_area_row.pk,
numerator_result=store_area_row.result, result=store_area_row.result,
denominator_id=self.store_id, denominator_result=1,
should_enter=True, by_scene=True)
def apply_json_parser(self, row):
json_relevent_rows_with_parse_logic = row[row.index.str.contains('JSON')].apply(self.parse_json_row)
row = row[~ row.index.isin(json_relevent_rows_with_parse_logic.index)].append(
json_relevent_rows_with_parse_logic)
return row
def parse_json_row(self, item):
'''
:param item: improper json value (formatted incorrectly)
:return: properly formatted json dictionary
The function will be in conjunction with apply. The function will applied on the row(pandas series). This is
meant to convert the json comprised of improper format of strings and lists to a proper dictionary value.
'''
if item:
container = self.prereq_parse_json_row(item)
else:
container = None
return container
@staticmethod
def prereq_parse_json_row(item):
'''
primarly logic for formatting the value of the json
'''
if isinstance(item, list):
container = OrderedDict()
for it in item:
# value = re.findall("[0-9a-zA-Z_]+", it)
value = re.findall("'([^']*)'", it)
if len(value) == 2:
for i in range(0, len(value), 2):
container[value[i]] = [value[i + 1]]
else:
if len(container.items()) == 0:
print('issue') # delete later
# raise error
# haven't encountered an this. So should raise an issue.
pass
else:
last_inserted_value_key = container.items()[-1][0]
container.get(last_inserted_value_key).append(value[0])
else:
container = eval(item)
return container
def get_store_area_df(self):
query = """
select st.pk, sst.scene_fk, st.name, sc.session_uid
from probedata.scene_store_task_area_group_items sst
join static.store_task_area_group_items st on st.pk=sst.store_task_area_group_item_fk
join probedata.scene sc on sc.pk=sst.scene_fk
where sc.delete_time is null and sc.session_uid = '{}' and sst.scene_fk = '{}';
""".format(self.session_uid, self.scene_info.scene_fk.iat[0])
df = pd.read_sql_query(query, self.rds_conn.db)
return df
def _get_kpi_name_and_fk(self, row):
kpi_name = row[Consts.KPI_NAME]
kpi_fk = self.common.get_kpi_fk_by_kpi_type(kpi_name)
output = [kpi_name, kpi_fk]
return output
def _parse_json_filters_to_df(self, row):
JSON = row[row.index.str.contains('JSON') & (~ row.index.str.contains('Config Params'))]
filter_JSON = JSON[~JSON.isnull()]
filtered_scif = self.scif
for each_JSON in filter_JSON:
final_JSON = {'population': each_JSON} if ('include' or 'exclude') in each_JSON else each_JSON
filtered_scif = ParseInputKPI.filter_df(final_JSON, filtered_scif)
if 'include_stacking' in row['Config Params: JSON'].keys():
including_stacking = row['Config Params: JSON']['include_stacking'][0]
filtered_scif[
Consts.FINAL_FACINGS] = filtered_scif.facings if including_stacking == 'True' else filtered_scif.facings_ign_stack
filtered_scif = filtered_scif[filtered_scif.stacking_layer == 1]
return filtered_scif
class StraussfritolayilUtil(UnifiedKPISingleton):
def __init__(self, output, data_provider):
super(StraussfritolayilUtil, self).__init__(data_provider)
self.output = output
self.common = Common(self.data_provider)
self.project_name = self.data_provider.project_name
self.session_uid = self.data_provider.session_uid
self.ps_data = PsDataProvider(self.data_provider, self.output)
self.products = self.data_provider[Data.PRODUCTS]
self.all_products = self.data_provider[Data.ALL_PRODUCTS]
self.scif = self.data_provider[Data.SCENE_ITEM_FACTS]
self.brand_mix_df = self.get_brand_mix_df()
self.add_sub_brand_to_scif()
self.add_brand_mix_to_scif()
self.match_probe_in_scene = self.ps_data.get_product_special_attribute_data(self.session_uid)
self.match_product_in_scene = self.data_provider[Data.MATCHES]
if not self.match_product_in_scene.empty:
self.match_product_in_scene = self.match_product_in_scene.merge(self.scif[Consts.RELEVENT_FIELDS],
on=["scene_fk", "product_fk"], how="left")
self.filter_scif_and_mpis_to_contain_only_primary_shelf()
else:
unique_fields = [ele for ele in Consts.RELEVENT_FIELDS if ele not in ["product_fk", "scene_fk"]]
self.match_product_in_scene = pd.concat([self.match_product_in_scene,
pd.DataFrame(columns=unique_fields)], axis=1)
self.match_product_in_scene_wo_hangers = self.exclude_special_attribute_products(df=self.match_product_in_scene)
self.visit_date = self.data_provider[Data.VISIT_DATE]
self.session_info = self.data_provider[Data.SESSION_INFO]
self.scene_info = self.data_provider[Data.SCENES_INFO]
self.store_info = self.data_provider[Data.STORE_INFO]
self.additional_attribute_2 = self.store_info[Consts.ADDITIONAL_ATTRIBUTE_2].values[0]
self.additional_attribute_3 = self.store_info[Consts.ADDITIONAL_ATTRIBUTE_3].values[0]
self.additional_attribute_4 = self.store_info[Consts.ADDITIONAL_ATTRIBUTE_4].values[0]
self.store_id = self.store_info['store_fk'].values[0] if self.store_info['store_fk'] is not None else 0
self.rds_conn = PSProjectConnector(self.project_name, DbUsers.CalculationEng)
self.toolbox = GENERALToolBox(self.data_provider)
self.kpi_external_targets = self.ps_data.get_kpi_external_targets(key_fields=Consts.KEY_FIELDS,
data_fields=Consts.DATA_FIELDS)
self.filter_external_targets()
self.assortment = Assortment(self.data_provider, self.output)
self.lvl3_assortment = self.set_updated_assortment()
self.own_manuf_fk = int(self.data_provider.own_manufacturer.param_value.values[0])
self.own_manufacturer_matches_wo_hangers = self.match_product_in_scene_wo_hangers[
self.match_product_in_scene_wo_hangers['manufacturer_fk'] == self.own_manuf_fk]
def set_updated_assortment(self):
assortment_result = self.assortment.get_lvl3_relevant_ass()
if assortment_result.empty:
return pd.DataFrame(columns=["kpi_fk_lvl2", "kpi_fk_lvl3"])
assortment_result = self.calculate_lvl3_assortment(assortment_result)
replacement_eans_df = pd.DataFrame([json_normalize(json.loads(js)).values[0] for js
in assortment_result['additional_attributes']])
replacement_eans_df.columns = [Consts.REPLACMENT_EAN_CODES]
replacement_eans_df = replacement_eans_df[Consts.REPLACMENT_EAN_CODES].apply(lambda row: [
x.strip() for x in str(row).split(",")] if row else None)
assortment_result = assortment_result.join(replacement_eans_df)
assortment_result['facings_all_products'] = assortment_result['facings'].copy()
assortment_result['facings_all_products_wo_hangers'] = assortment_result['facings_wo_hangers'].copy()
assortment_result = self.handle_replacment_products_row(assortment_result)
return assortment_result
def filter_external_targets(self):
self.kpi_external_targets = self.kpi_external_targets[
(self.kpi_external_targets[Consts.ADDITIONAL_ATTRIBUTE_2].str.encode("utf8").isin(
[None, self.additional_attribute_2.encode("utf8")])) &
(self.kpi_external_targets[Consts.ADDITIONAL_ATTRIBUTE_3].str.encode("utf8").isin(
[None, self.additional_attribute_3.encode("utf8")])) &
(self.kpi_external_targets[Consts.ADDITIONAL_ATTRIBUTE_4].str.encode("utf8").isin(
[None, self.additional_attribute_4.encode("utf8")]))]
def calculate_lvl3_assortment(self, assortment_result):
"""
:return: data frame on the sku level with the following fields:
['assortment_group_fk', 'assortment_fk', 'target', 'product_fk', 'in_store', 'kpi_fk_lvl1',
'kpi_fk_lvl2', 'kpi_fk_lvl3', 'group_target_date', 'assortment_super_group_fk',
'super_group_target', 'additional_attributes']. Indicates whether the product was in the store (1) or not (0).
"""
if assortment_result.empty:
return assortment_result
assortment_result['in_store_wo_hangers'] = assortment_result['in_store'].copy()
products_in_session = list(self.match_product_in_scene['product_fk'].values)
products_in_session_wo_hangers = list(self.match_product_in_scene_wo_hangers['product_fk'].values)
assortment_result.loc[assortment_result['product_fk'].isin(products_in_session), 'in_store'] = 1
assortment_result.loc[assortment_result['product_fk'].isin(products_in_session_wo_hangers),
'in_store_wo_hangers'] = 1
assortment_result['facings'] = 0
assortment_result['facings_wo_hangers'] = 0
product_assort = assortment_result['product_fk'].unique()
for sku in product_assort:
assortment_result.loc[assortment_result['product_fk'] == sku, 'facings'] = \
len(self.match_product_in_scene[self.match_product_in_scene['product_fk'] == sku])
assortment_result.loc[assortment_result['product_fk'] == sku, 'facings_wo_hangers'] = \
len(self.match_product_in_scene_wo_hangers[self.match_product_in_scene_wo_hangers['product_fk'] == sku])
return assortment_result
def handle_replacment_products_row(self, assortment_result):
additional_products_df = assortment_result[~assortment_result[Consts.REPLACMENT_EAN_CODES].isnull()]
products_in_session = set(self.match_product_in_scene['product_ean_code'].values)
products_in_session_wo_hangers = set(self.match_product_in_scene_wo_hangers['product_ean_code'].values)
for i, row in additional_products_df.iterrows():
replacement_products = row[Consts.REPLACMENT_EAN_CODES]
facings = len(self.match_product_in_scene[self.match_product_in_scene[
'product_ean_code'].isin(replacement_products)])
facings_wo_hangers = len(self.match_product_in_scene_wo_hangers[self.match_product_in_scene_wo_hangers[
'product_ean_code'].isin(replacement_products)])
assortment_result.loc[i, 'facings_all_products'] = facings + row['facings']
assortment_result.loc[i, 'facings_all_products_wo_hangers'] = facings_wo_hangers + row['facings_wo_hangers']
if row['in_store'] != 1:
for sku in replacement_products:
if sku in products_in_session:
product_df = self.all_products[self.all_products['product_ean_code'] == sku]['product_fk']
assortment_result.loc[i, 'product_fk'] = product_df.values[0]
assortment_result.loc[i, 'in_store'] = 1
break
if row['in_store_wo_hangers'] != 1:
for sku in replacement_products:
if sku in products_in_session_wo_hangers:
product_df = self.all_products[self.all_products['product_ean_code'] == sku]['product_fk']
assortment_result.loc[i, 'product_fk'] = product_df.values[0]
assortment_result.loc[i, 'in_store_wo_hangers'] = 1
break
return assortment_result
def filter_scif_and_mpis_to_contain_only_primary_shelf(self):
self.scif = self.scif[self.scif.location_type == Consts.PRIMARY_SHELF]
self.match_product_in_scene = self.match_product_in_scene[self.match_product_in_scene.location_type ==
Consts.PRIMARY_SHELF]
def add_sub_brand_to_scif(self):
sub_brand_df = self.ps_data.get_custom_entities_df(entity_type_name='Sub_Brand_Local')
sub_brand_df = sub_brand_df[['entity_name', 'entity_fk']].copy()
# sub_brand_df['entity_name'] = sub_brand_df['entity_name'].str.lower()
sub_brand_df.rename({'entity_fk': 'sub_brand_fk'}, axis='columns', inplace=True)
# delete duplicates by name and entity_type_fk to avoid recognition duplicates.
sub_brand_df.drop_duplicates(subset=['entity_name'], keep='first', inplace=True)
self.scif['Sub_Brand_Local'] = self.scif['Sub_Brand_Local'].fillna('no value')
self.scif = self.scif.merge(sub_brand_df, left_on='Sub_Brand_Local', right_on="entity_name", how="left")
self.scif['sub_brand_fk'].fillna(Consts.SUB_BRAND_NO_VALUE, inplace=True)
def get_brand_mix_df(self):
brand_mix_df = self.ps_data.get_custom_entities_df(entity_type_name='Brand_Mix')
brand_mix_df = brand_mix_df[['entity_name', 'entity_fk']].copy()
brand_mix_df.rename({'entity_fk': 'brand_mix_fk'}, axis='columns', inplace=True)
# delete duplicates by name and entity_type_fk to avoid recognition duplicates.
brand_mix_df.drop_duplicates(subset=['entity_name'], keep='first', inplace=True)
return brand_mix_df
def add_brand_mix_to_scif(self):
self.scif['Brand_Mix'] = self.scif['Brand_Mix'].fillna('no value')
self.scif = self.scif.merge(self.brand_mix_df, left_on='Brand_Mix', right_on="entity_name", how="left")
self.scif['brand_mix_fk'].fillna(Consts.BRAND_MIX_NO_VALUE, inplace=True)
@staticmethod
def calculate_sos_result(numerator, denominator):
if denominator == 0:
return 0
result = 100 * round((numerator / float(denominator)), 3)
return result
def exclude_special_attribute_products(self, df):
"""
Helper to exclude smart_attribute products
:return: filtered df without smart_attribute products
"""
if self.match_probe_in_scene.empty:
return df
smart_attribute_df = self.match_probe_in_scene[self.match_probe_in_scene['name'] == Consts.ADDITIONAL_DISPLAY]
if smart_attribute_df.empty:
return df
match_product_in_probe_fks = smart_attribute_df['match_product_in_probe_fk'].tolist()
df = df[~df['probe_match_fk'].isin(match_product_in_probe_fks)]
return df
class HEINZCRToolBox:
LVL3_HEADERS = ['assortment_group_fk', 'assortment_fk', 'target', 'product_fk',
'in_store', 'kpi_fk_lvl1', 'kpi_fk_lvl2', 'kpi_fk_lvl3', 'group_target_date',
'assortment_super_group_fk']
LVL2_HEADERS = ['assortment_group_fk', 'assortment_fk', 'target', 'passes', 'total',
'kpi_fk_lvl1', 'kpi_fk_lvl2', 'group_target_date']
LVL1_HEADERS = ['assortment_group_fk', 'target', 'passes', 'total', 'kpi_fk_lvl1']
ASSORTMENT_FK = 'assortment_fk'
ASSORTMENT_GROUP_FK = 'assortment_group_fk'
ASSORTMENT_SUPER_GROUP_FK = 'assortment_super_group_fk'
BRAND_VARIENT = 'brand_varient'
NUMERATOR = 'numerator'
DENOMINATOR = 'denominator'
DISTRIBUTION_KPI = 'Distribution - SKU'
OOS_SKU_KPI = 'OOS - SKU'
OOS_KPI = 'OOS'
def __init__(self, data_provider, output):
self.output = output
self.data_provider = data_provider
self.common = CommonV2 # remove later
self.common_v2 = CommonV2(self.data_provider)
self.project_name = self.data_provider.project_name
self.session_uid = self.data_provider.session_uid
self.products = self.data_provider[Data.PRODUCTS]
self.all_products = self.data_provider[Data.ALL_PRODUCTS]
self.match_product_in_scene = self.data_provider[Data.MATCHES]
self.visit_date = self.data_provider[Data.VISIT_DATE]
self.session_info = self.data_provider[Data.SESSION_INFO]
self.scene_info = self.data_provider[Data.SCENES_INFO]
self.store_id = self.data_provider[Data.STORE_FK]
self.scif = self.data_provider[Data.SCENE_ITEM_FACTS]
self.rds_conn = PSProjectConnector(self.project_name, DbUsers.CalculationEng)
self.kpi_results_queries = []
self.ps_data_provider = PsDataProvider(self.data_provider, self.output)
self.survey = Survey(self.data_provider, output=self.output, ps_data_provider=self.ps_data_provider,
common=self.common_v2)
self.store_sos_policies = self.ps_data_provider.get_store_policies()
self.labels = self.ps_data_provider.get_labels()
self.store_info = self.data_provider[Data.STORE_INFO]
self.store_info = self.ps_data_provider.get_ps_store_info(self.store_info)
self.country = self.store_info['country'].iloc[0]
self.current_date = datetime.now()
self.extra_spaces_template = pd.read_excel(Const.EXTRA_SPACES_RELEVANT_SUB_CATEGORIES_PATH)
self.store_targets = pd.read_excel(Const.STORE_TARGETS_PATH)
self.sub_category_weight = pd.read_excel(Const.SUB_CATEGORY_TARGET_PATH, sheetname='category_score')
self.kpi_weights = pd.read_excel(Const.SUB_CATEGORY_TARGET_PATH, sheetname='max_weight')
self.targets = self.ps_data_provider.get_kpi_external_targets()
self.store_assortment = PSAssortmentDataProvider(
self.data_provider).execute(policy_name=None)
self.supervisor_target = self.get_supervisor_target()
try:
self.sub_category_assortment = pd.merge(self.store_assortment,
self.all_products.loc[:, ['product_fk', 'sub_category',
'sub_category_fk']],
how='left', on='product_fk')
self.sub_category_assortment = \
self.sub_category_assortment[~self.sub_category_assortment['assortment_name'].str.contains(
'ASSORTMENT')]
self.sub_category_assortment = pd.merge(self.sub_category_assortment, self.sub_category_weight, how='left',
left_on='sub_category',
right_on='Category')
except KeyError:
self.sub_category_assortment = pd.DataFrame()
self.update_score_sub_category_weights()
try:
self.store_assortment_without_powerskus = \
self.store_assortment[self.store_assortment['assortment_name'].str.contains('ASSORTMENT')]
except KeyError:
self.store_assortment_without_powerskus = pd.DataFrame()
self.adherence_results = pd.DataFrame(columns=['product_fk', 'trax_average',
'suggested_price', 'into_interval', 'min_target', 'max_target',
'percent_range'])
self.extra_spaces_results = pd.DataFrame(
columns=['sub_category_fk', 'template_fk', 'count'])
self.powersku_scores = {}
self.powersku_empty = {}
self.powersku_bonus = {}
self.powersku_price = {}
self.powersku_sos = {}
def main_calculation(self, *args, **kwargs):
"""
This function calculates the KPI results.
"""
if self.scif.empty:
return
# these function must run first
# self.adherence_results = self.heinz_global_price_adherence(pd.read_excel(Const.PRICE_ADHERENCE_TEMPLATE_PATH,
# sheetname="Price Adherence"))
self.adherence_results = self.heinz_global_price_adherence(self.targets)
self.extra_spaces_results = self.heinz_global_extra_spaces()
self.set_relevant_sub_categories()
# this isn't relevant to the 'Perfect Score' calculation
self.heinz_global_distribution_per_category()
self.calculate_assortment()
self.calculate_powersku_assortment()
self.main_sos_calculation()
self.calculate_powersku_price_adherence()
self.calculate_perfect_store_extra_spaces()
self.check_bonus_question()
self.calculate_perfect_sub_category()
def calculate_assortment(self):
if self.store_assortment_without_powerskus.empty:
return
products_in_store = self.scif[self.scif['facings'] > 0]['product_fk'].unique().tolist()
pass_count = 0
total_kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('Distribution')
identifier_dict = self.common_v2.get_dictionary(kpi_fk=total_kpi_fk)
oos_kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('OOS')
oos_identifier_dict = self.common_v2.get_dictionary(kpi_fk=oos_kpi_fk)
for row in self.store_assortment_without_powerskus.itertuples():
result = 0
if row.product_fk in products_in_store:
result = 1
pass_count += 1
sku_kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('Distribution - SKU')
self.common_v2.write_to_db_result(sku_kpi_fk, numerator_id=row.product_fk, denominator_id=row.assortment_fk,
result=result, identifier_parent=identifier_dict, should_enter=True)
oos_result = 0 if result else 1
oos_sku_kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type('OOS - SKU')
self.common_v2.write_to_db_result(oos_sku_kpi_fk, numerator_id=row.product_fk,
denominator_id=row.assortment_fk,
result=oos_result, identifier_parent=oos_identifier_dict,
should_enter=True)
number_of_products_in_assortment = len(self.store_assortment_without_powerskus)
if number_of_products_in_assortment:
total_result = (pass_count / float(number_of_products_in_assortment)) * 100
oos_products = number_of_products_in_assortment - pass_count
oos_result = (oos_products / float(number_of_products_in_assortment)) * 100
else:
total_result = 0
oos_products = number_of_products_in_assortment
oos_result = number_of_products_in_assortment
self.common_v2.write_to_db_result(total_kpi_fk, numerator_id=Const.OWN_MANUFACTURER_FK,
denominator_id=self.store_id,
numerator_result=pass_count,
denominator_result=number_of_products_in_assortment,
result=total_result, identifier_result=identifier_dict)
self.common_v2.write_to_db_result(oos_kpi_fk, numerator_id=Const.OWN_MANUFACTURER_FK,
denominator_id=self.store_id,
numerator_result=oos_products,
denominator_result=number_of_products_in_assortment,
result=oos_result, identifier_result=oos_identifier_dict)
def calculate_powersku_assortment(self):
if self.sub_category_assortment.empty:
return 0
sub_category_kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type(Const.POWER_SKU_SUB_CATEGORY)
sku_kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type(Const.POWER_SKU)
target_kpi_weight = float(
self.kpi_weights['Score'][self.kpi_weights['KPIs'] == Const.KPI_WEIGHTS['POWERSKU']].iloc[
0])
kpi_weight = self.get_kpi_weight('POWERSKU')
products_in_session = self.scif[self.scif['facings'] > 0]['product_fk'].unique().tolist()
self.sub_category_assortment['in_session'] = \
self.sub_category_assortment.loc[:, 'product_fk'].isin(products_in_session)
# save PowerSKU results at SKU level
for sku in self.sub_category_assortment[
['product_fk', 'sub_category_fk', 'in_session', 'sub_category']].itertuples():
parent_dict = self.common_v2.get_dictionary(
kpi_fk=sub_category_kpi_fk, sub_category_fk=sku.sub_category_fk)
relevant_sub_category_df = self.sub_category_assortment[
self.sub_category_assortment['sub_category'] == sku.sub_category]
if relevant_sub_category_df.empty:
sub_category_count = 0
else:
sub_category_count = len(relevant_sub_category_df)
result = 1 if sku.in_session else 0
score = result * (target_kpi_weight / float(sub_category_count))
self.common_v2.write_to_db_result(sku_kpi_fk, numerator_id=sku.product_fk,
denominator_id=sku.sub_category_fk, score=score,
result=result, identifier_parent=parent_dict, should_enter=True)
# save PowerSKU results at sub_category level
aggregated_results = self.sub_category_assortment.groupby('sub_category_fk').agg(
{'in_session': 'sum', 'product_fk': 'count'}).reset_index().rename(
columns={'product_fk': 'product_count'})
aggregated_results['percent_complete'] = \
aggregated_results.loc[:, 'in_session'] / aggregated_results.loc[:, 'product_count']
aggregated_results['result'] = aggregated_results['percent_complete']
for sub_category in aggregated_results.itertuples():
identifier_dict = self.common_v2.get_dictionary(kpi_fk=sub_category_kpi_fk,
sub_category_fk=sub_category.sub_category_fk)
result = sub_category.result
score = result * kpi_weight
self.powersku_scores[sub_category.sub_category_fk] = score
self.common_v2.write_to_db_result(sub_category_kpi_fk, numerator_id=sub_category.sub_category_fk,
denominator_id=self.store_id,
identifier_parent=sub_category.sub_category_fk,
identifier_result=identifier_dict, result=result * 100, score=score,
weight=target_kpi_weight, target=target_kpi_weight,
should_enter=True)
def heinz_global_distribution_per_category(self):
relevant_stores = pd.DataFrame(columns=self.store_sos_policies.columns)
for row in self.store_sos_policies.itertuples():
policies = json.loads(row.store_policy)
df = self.store_info
for key, value in policies.items():
try:
df_1 = df[df[key].isin(value)]
except KeyError:
continue
if not df_1.empty:
stores = self.store_sos_policies[(self.store_sos_policies['store_policy'] == row.store_policy.encode('utf-8'))
& (
self.store_sos_policies[
'target_validity_start_date'] <= datetime.date(
self.current_date))]
if stores.empty:
relevant_stores = stores
else:
relevant_stores = relevant_stores.append(stores, ignore_index=True)
relevant_stores = relevant_stores.drop_duplicates(subset=['kpi', 'sku_name', 'target', 'sos_policy'],
keep='last')
for row in relevant_stores.itertuples():
sos_policy = json.loads(row.sos_policy)
numerator_key = sos_policy[self.NUMERATOR].keys()[0]
denominator_key = sos_policy[self.DENOMINATOR].keys()[0]
numerator_val = sos_policy[self.NUMERATOR][numerator_key]
denominator_val = sos_policy[self.DENOMINATOR][denominator_key]
target = row.target * 100
if numerator_key == 'manufacturer':
numerator_key = numerator_key + '_name'
if denominator_key == 'sub_category' \
and denominator_val.lower() != 'all' \
and json.loads(row.store_policy).get('store_type') \
and len(json.loads(row.store_policy).get('store_type')) == 1:
try:
denominator_id = self.all_products[self.all_products[denominator_key] == denominator_val][
denominator_key + '_fk'].values[0]
numerator_id = self.all_products[self.all_products[numerator_key] == numerator_val][
numerator_key.split('_')[0] + '_fk'].values[0]
# self.common.write_to_db_result_new_tables(fk=12, numerator_id=numerator_id,
# numerator_result=None,
# denominator_id=denominator_id,
# denominator_result=None,
# result=target)
self.common_v2.write_to_db_result(fk=12, numerator_id=numerator_id, numerator_result=None,
denominator_id=denominator_id, denominator_result=None,
result=target)
except Exception as e:
Log.warning(denominator_key + ' - - ' + denominator_val)
def calculate_perfect_store(self):
pass
def calculate_perfect_sub_category(self):
kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type(Const.PERFECT_STORE_SUB_CATEGORY)
parent_kpi = self.common_v2.get_kpi_fk_by_kpi_type(Const.PERFECT_STORE)
total_score = 0
sub_category_fk_list = []
kpi_type_dict_scores = [self.powersku_scores, self.powersku_empty, self.powersku_price,
self.powersku_sos]
for kpi_dict in kpi_type_dict_scores:
sub_category_fk_list.extend(kpi_dict.keys())
kpi_weight_perfect_store = 0
if self.country in self.sub_category_weight.columns.to_list():
kpi_weight_perfect_store = self.sub_category_weight[self.country][
self.sub_category_weight['Category'] == Const.PERFECT_STORE_KPI_WEIGHT]
if not kpi_weight_perfect_store.empty:
kpi_weight_perfect_store = kpi_weight_perfect_store.iloc[0]
unique_sub_cat_fks = list(dict.fromkeys(sub_category_fk_list))
sub_category_fks = self.sub_category_weight.sub_category_fk.unique().tolist()
relevant_sub_cat_list = [x for x in sub_category_fks if str(x) != 'nan']
# relevant_sub_cat_list = self.sub_category_assortment['sub_category_fk'][
# self.sub_category_assortment['Category'] != pd.np.nan].unique().tolist()
for sub_cat_fk in unique_sub_cat_fks:
if sub_cat_fk in relevant_sub_cat_list:
bonus_score = 0
try:
bonus_score = self.powersku_bonus[sub_cat_fk]
except:
pass
sub_cat_weight = self.get_weight(sub_cat_fk)
sub_cat_score = self.calculate_sub_category_sum(kpi_type_dict_scores, sub_cat_fk)
result = sub_cat_score
score = (result * sub_cat_weight) + bonus_score
total_score += score
self.common_v2.write_to_db_result(kpi_fk, numerator_id=sub_cat_fk,
denominator_id=self.store_id,
result=result, score=score,
identifier_parent=parent_kpi,
identifier_result=sub_cat_fk,
weight=sub_cat_weight * 100,
should_enter=True)
self.common_v2.write_to_db_result(parent_kpi, numerator_id=Const.OWN_MANUFACTURER_FK,
denominator_id=self.store_id,
result=total_score, score=total_score,
identifier_result=parent_kpi,
target=kpi_weight_perfect_store,
should_enter=True)
def main_sos_calculation(self):
relevant_stores = pd.DataFrame(columns=self.store_sos_policies.columns)
for row in self.store_sos_policies.itertuples():
policies = json.loads(row.store_policy)
df = self.store_info
for key, value in policies.items():
try:
if key != 'additional_attribute_3':
df1 = df[df[key].isin(value)]
except KeyError:
continue
if not df1.empty:
stores = \
self.store_sos_policies[(self.store_sos_policies['store_policy'].str.encode(
'utf-8') == row.store_policy.encode('utf-8')) &
(self.store_sos_policies['target_validity_start_date'] <= datetime.date(
self.current_date))]
if stores.empty:
relevant_stores = stores
else:
relevant_stores = relevant_stores.append(stores, ignore_index=True)
relevant_stores = relevant_stores.drop_duplicates(subset=['kpi', 'sku_name', 'target', 'sos_policy'],
keep='last')
results_df = pd.DataFrame(columns=['sub_category', 'sub_category_fk', 'score'])
sos_sub_category_kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type(Const.SOS_SUB_CATEGORY)
for row in relevant_stores.itertuples():
sos_policy = json.loads(row.sos_policy)
numerator_key = sos_policy[self.NUMERATOR].keys()[0]
denominator_key = sos_policy[self.DENOMINATOR].keys()[0]
numerator_val = sos_policy[self.NUMERATOR][numerator_key]
denominator_val = sos_policy[self.DENOMINATOR][denominator_key]
json_policy = json.loads(row.store_policy)
kpi_fk = row.kpi
# This is to assign the KPI to SOS_manufacturer_category_GLOBAL
if json_policy.get('store_type') and len(json_policy.get('store_type')) > 1:
kpi_fk = 8
if numerator_key == 'manufacturer':
numerator_key = numerator_key + '_name'
# we need to include 'Philadelphia' as a manufacturer for all countries EXCEPT Chile
if self.country == 'Chile':
numerator_values = [numerator_val]
else:
numerator_values = [numerator_val, 'Philadelphia']
else:
# if the numerator isn't 'manufacturer', we just need to convert the value to a list
numerator_values = [numerator_val]
if denominator_key == 'sub_category':
include_stacking_list = ['Nuts', 'DRY CHEESE', 'IWSN', 'Shredded', 'SNACK']
if denominator_val in include_stacking_list:
facings_field = 'facings'
else:
facings_field = 'facings_ign_stack'
else:
facings_field = 'facings_ign_stack'
if denominator_key == 'sub_category' and denominator_val.lower() == 'all':
# Here we are talkin on a KPI when the target have no denominator,
# the calculation should be done on Numerator only
numerator = self.scif[(self.scif[numerator_key] == numerator_val) &
(self.scif['location_type'] == 'Primary Shelf')
][facings_field].sum()
kpi_fk = 9
denominator = None
denominator_id = None
else:
numerator = self.scif[(self.scif[numerator_key].isin(numerator_values)) &
(self.scif[denominator_key] == denominator_val) &
(self.scif['location_type'] == 'Primary Shelf')][facings_field].sum()
denominator = self.scif[(self.scif[denominator_key] == denominator_val) &
(self.scif['location_type'] == 'Primary Shelf')][facings_field].sum()
try:
if denominator is not None:
denominator_id = self.all_products[self.all_products[denominator_key] == denominator_val][
denominator_key + '_fk'].values[0]
if numerator is not None:
numerator_id = self.all_products[self.all_products[numerator_key] == numerator_val][
numerator_key.split('_')[0] + '_fk'].values[0]
sos = 0
if numerator and denominator:
sos = np.divide(float(numerator), float(denominator)) * 100
score = 0
target = row.target * 100
if sos >= target:
score = 100
identifier_parent = None
should_enter = False
if denominator_key == 'sub_category' and kpi_fk == row.kpi:
# if this a sub_category result, save it to the results_df for 'Perfect Store' store
results_df.loc[len(results_df)] = [denominator_val, denominator_id, score / 100]
identifier_parent = self.common_v2.get_dictionary(kpi_fk=sos_sub_category_kpi_fk,
sub_category_fk=denominator_id)
should_enter = True
manufacturer = None
self.common_v2.write_to_db_result(kpi_fk, numerator_id=numerator_id, numerator_result=numerator,
denominator_id=denominator_id, denominator_result=denominator,
result=target, score=sos, target=target,
score_after_actions=manufacturer, identifier_parent=identifier_parent,
should_enter=should_enter)
except Exception as e:
Log.warning(denominator_key + ' - - ' + denominator_val)
# if there are no sub_category sos results, there's no perfect store information to be saved
if len(results_df) == 0:
return 0
# save aggregated results for each sub category
kpi_weight = self.get_kpi_weight('SOS')
for row in results_df.itertuples():
identifier_result = \
self.common_v2.get_dictionary(kpi_fk=sos_sub_category_kpi_fk,
sub_category_fk=row.sub_category_fk)
# sub_cat_weight = self.get_weight(row.sub_category_fk)
result = row.score
score = result * kpi_weight
self.powersku_sos[row.sub_category_fk] = score
# limit results so that aggregated results can only add up to 3
self.common_v2.write_to_db_result(sos_sub_category_kpi_fk,
numerator_id=row.sub_category_fk,
denominator_id=self.store_id,
result=row.score, score=score,
identifier_parent=row.sub_category_fk,
identifier_result=identifier_result,
weight=kpi_weight,
target=kpi_weight,
should_enter=True)
def calculate_powersku_price_adherence(self):
adherence_kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type(Const.POWER_SKU_PRICE_ADHERENCE)
adherence_sub_category_kpi_fk = \
self.common_v2.get_kpi_fk_by_kpi_type(Const.POWER_SKU_PRICE_ADHERENCE_SUB_CATEGORY)
if self.sub_category_assortment.empty:
return False
results = pd.merge(self.sub_category_assortment,
self.adherence_results, how='left', on='product_fk')
results['into_interval'].fillna(0, inplace=True)
for row in results.itertuples():
parent_dict = self.common_v2.get_dictionary(kpi_fk=adherence_sub_category_kpi_fk,
sub_category_fk=row.sub_category_fk)
score_value = 'Not Present'
in_session = row.in_session
if in_session:
if not pd.isna(row.trax_average) and row.suggested_price:
price_in_interval = 1 if row.into_interval == 1 else 0
if price_in_interval == 1:
score_value = 'Pass'
else:
score_value = 'Fail'
else:
score_value = 'No Price'
score = Const.PRESENCE_PRICE_VALUES[score_value]
self.common_v2.write_to_db_result(adherence_kpi_fk, numerator_id=row.product_fk,
denominator_id=row.sub_category_fk, result=row.trax_average,
score=score, target=row.suggested_price, numerator_result=row.min_target,
denominator_result=row.max_target,
weight=row.percent_range,
identifier_parent=parent_dict, should_enter=True)
aggregated_results = results.groupby('sub_category_fk').agg(
{'into_interval': 'sum', 'product_fk': 'count'}).reset_index().rename(
columns={'product_fk': 'product_count'})
aggregated_results['percent_complete'] = \
aggregated_results.loc[:, 'into_interval'] / aggregated_results.loc[:, 'product_count']
for row in aggregated_results.itertuples():
identifier_result = self.common_v2.get_dictionary(kpi_fk=adherence_sub_category_kpi_fk,
sub_category_fk=row.sub_category_fk)
kpi_weight = self.get_kpi_weight('PRICE')
result = row.percent_complete
score = result * kpi_weight
self.powersku_price[row.sub_category_fk] = score
self.common_v2.write_to_db_result(adherence_sub_category_kpi_fk, numerator_id=row.sub_category_fk,
denominator_id=self.store_id, result=result, score=score,
numerator_result=row.into_interval, denominator_result=row.product_count,
identifier_parent=row.sub_category_fk,
identifier_result=identifier_result,
weight=kpi_weight, target=kpi_weight,
should_enter=True)
def heinz_global_price_adherence(self, config_df):
config_df = config_df.sort_values(by=["received_time"], ascending=False).drop_duplicates(
subset=['start_date', 'end_date', 'ean_code', 'store_type'], keep="first")
if config_df.empty:
Log.warning("No external_targets data found - Price Adherence will not be calculated")
return self.adherence_results
self.match_product_in_scene.loc[self.match_product_in_scene['price'].isna(), 'price'] = \
self.match_product_in_scene.loc[self.match_product_in_scene['price'].isna(), 'promotion_price']
# =============== remove after updating logic to support promotional pricing ===============
results_df = self.adherence_results
my_config_df = \
config_df[config_df['store_type'].str.encode('utf-8') == self.store_info.store_type[0].encode('utf-8')]
products_in_session = self.scif['product_ean_code'].unique().tolist()
products_in_session = [ean for ean in products_in_session if ean is not pd.np.nan and ean is not None]
my_config_df = my_config_df[my_config_df['ean_code'].isin(products_in_session)]
for row in my_config_df.itertuples():
product_pk = \
self.all_products[self.all_products['product_ean_code']
== row.ean_code]['product_fk'].iloc[0]
mpisc_df_price = \
self.match_product_in_scene[(self.match_product_in_scene['product_fk'] == product_pk) |
(self.match_product_in_scene[
'substitution_product_fk'] == product_pk)]['price']
try:
suggested_price = float(row.suggested_price)
except Exception as e:
Log.error("Product with ean_code {} is not in the configuration file for customer type {}"
.format(row.ean_code, self.store_info.store_type[0].encode('utf-8')))
break
percentage_weight = int(row.percentage_weight)
upper_percentage = (100 + percentage_weight) / float(100)
lower_percentage = (100 - percentage_weight) / float(100)
min_price = suggested_price * lower_percentage
max_price = suggested_price * upper_percentage
percentage_sku = percentage_weight
into_interval = 0
prices_sum = 0
count = 0
trax_average = None
for price in mpisc_df_price:
if price and pd.notna(price):
prices_sum += price
count += 1
if prices_sum > 0:
trax_average = prices_sum / count
into_interval = 0
if not np.isnan(suggested_price):
if min_price <= trax_average <= max_price:
into_interval = 100
results_df.loc[len(results_df)] = [product_pk, trax_average,
suggested_price, into_interval / 100, min_price, max_price,
percentage_sku]
self.common_v2.write_to_db_result(10, numerator_id=product_pk,
numerator_result=suggested_price,
denominator_id=product_pk,
denominator_result=trax_average,
result=row.percentage_weight,
score=into_interval)
if trax_average:
mark_up = (np.divide(np.divide(float(trax_average), float(1.13)),
float(suggested_price)) - 1) * 100
self.common_v2.write_to_db_result(11, numerator_id=product_pk,
numerator_result=suggested_price,
denominator_id=product_pk,
denominator_result=trax_average,
score=mark_up,
result=mark_up)
return results_df
def calculate_perfect_store_extra_spaces(self):
extra_spaces_kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type(
Const.PERFECT_STORE_EXTRA_SPACES_SUB_CATEGORY)
sub_cats_for_store = self.relevant_sub_categories
if self.extra_spaces_results.empty:
pass
try:
relevant_sub_categories = [x.strip() for x in self.extra_spaces_template[
self.extra_spaces_template['country'].str.encode('utf-8') == self.country.encode('utf-8')][
'sub_category'].iloc[0].split(',')]
except IndexError:
Log.warning(
'No relevant sub_categories for the Extra Spaces KPI found for the following country: {}'.format(
self.country))
self.extra_spaces_results = pd.merge(self.extra_spaces_results,
self.all_products.loc[:, [
'sub_category_fk',
'sub_category']].dropna().drop_duplicates(),
how='left', on='sub_category_fk')
relevant_extra_spaces = \
self.extra_spaces_results[self.extra_spaces_results['sub_category'].isin(
relevant_sub_categories)]
kpi_weight = self.get_kpi_weight('EXTRA')
for row in relevant_extra_spaces.itertuples():
self.powersku_empty[row.sub_category_fk] = 1 * kpi_weight
score = result = 1
if row.sub_category_fk in sub_cats_for_store:
sub_cats_for_store.remove(row.sub_category_fk)
self.common_v2.write_to_db_result(extra_spaces_kpi_fk, numerator_id=row.sub_category_fk,
denominator_id=row.template_fk, result=result, score=score,
identifier_parent=row.sub_category_fk,
target=1, should_enter=True)
for sub_cat_fk in sub_cats_for_store:
result = score = 0
self.powersku_empty[sub_cat_fk] = 0
self.common_v2.write_to_db_result(extra_spaces_kpi_fk, numerator_id=sub_cat_fk,
denominator_id=0, result=result, score=score,
identifier_parent=sub_cat_fk,
target=1, should_enter=True)
def heinz_global_extra_spaces(self):
try:
supervisor = self.store_info['additional_attribute_3'][0]
store_target = -1
# for row in self.store_sos_policies.itertuples():
# policies = json.loads(row.store_policy)
# for key, value in policies.items():
# try:
# if key == 'additional_attribute_3' and value[0] == supervisor:
# store_target = row.target
# break
# except KeyError:
# continue
for row in self.supervisor_target.itertuples():
try:
if row.supervisor == supervisor:
store_target = row.target
break
except:
continue
except Exception as e:
Log.error("Supervisor target is not configured for the extra spaces report ")
raise e
results_df = self.extra_spaces_results
# limit to only secondary scenes
relevant_scif = self.scif[(self.scif['location_type_fk'] == float(2)) &
(self.scif['facings'] > 0)]
if relevant_scif.empty:
return results_df
# aggregate facings for every scene/sub_category combination in the visit
relevant_scif = \
relevant_scif.groupby(['scene_fk', 'template_fk', 'sub_category_fk'], as_index=False)['facings'].sum()
# sort sub_categories by number of facings, largest first
relevant_scif = relevant_scif.sort_values(['facings'], ascending=False)
# drop all but the sub_category with the largest number of facings for each scene
relevant_scif = relevant_scif.drop_duplicates(subset=['scene_fk'], keep='first')
for row in relevant_scif.itertuples():
results_df.loc[len(results_df)] = [row.sub_category_fk, row.template_fk, row.facings]
self.common_v2.write_to_db_result(13, numerator_id=row.template_fk,
numerator_result=row.facings,
denominator_id=row.sub_category_fk,
denominator_result=row.facings,
context_id=row.scene_fk,
result=store_target)
return results_df
def check_bonus_question(self):
bonus_kpi_fk = self.common_v2.get_kpi_fk_by_kpi_type(Const.BONUS_QUESTION_SUB_CATEGORY)
bonus_weight = self.kpi_weights['Score'][self.kpi_weights['KPIs'] == Const.KPI_WEIGHTS['Bonus']].iloc[0]
sub_category_fks = self.sub_category_weight.sub_category_fk.unique().tolist()
sub_category_fks = [x for x in sub_category_fks if str(x) != 'nan']
if self.survey.check_survey_answer(('question_fk', Const.BONUS_QUESTION_FK), 'Yes,yes,si,Si'):
result = 1
else:
result = 0
for sub_cat_fk in sub_category_fks:
sub_cat_weight = self.get_weight(sub_cat_fk)
score = result * sub_cat_weight
target_weight = bonus_weight * sub_cat_weight
self.powersku_bonus[sub_cat_fk] = score
self.common_v2.write_to_db_result(bonus_kpi_fk, numerator_id=sub_cat_fk,
denominator_id=self.store_id,
result=result, score=score, identifier_parent=sub_cat_fk,
weight=target_weight, target=target_weight,
should_enter=True)
def commit_results_data(self):
self.common_v2.commit_results_data()
def update_score_sub_category_weights(self):
all_sub_category_fks = self.all_products[['sub_category', 'sub_category_fk']].drop_duplicates()
self.sub_category_weight = pd.merge(self.sub_category_weight, all_sub_category_fks, left_on='Category',
right_on='sub_category',
how='left')
def get_weight(self, sub_category_fk):
weight_value = 0
if self.country in self.sub_category_weight.columns.to_list():
weight_df = self.sub_category_weight[self.country][
(self.sub_category_weight.sub_category_fk == sub_category_fk)]
if weight_df.empty:
return 0
weight_value = weight_df.iloc[0]
if pd.isna(weight_value):
weight_value = 0
weight = weight_value * 0.01
return weight
def get_kpi_weight(self, kpi_name):
weight = self.kpi_weights['Score'][self.kpi_weights['KPIs'] == Const.KPI_WEIGHTS[kpi_name]].iloc[0]
return weight
def get_supervisor_target(self):
supervisor_target = self.targets[self.targets['kpi_type'] == 'Extra Spaces']
return supervisor_target
def calculate_sub_category_sum(self, dict_list, sub_cat_fk):
total_score = 0
for item in dict_list:
try:
total_score += item[sub_cat_fk]
except:
pass
return total_score
def set_relevant_sub_categories(self):
if self.country in self.sub_category_weight.columns.to_list():
df = self.sub_category_weight[['Category', 'sub_category_fk', self.country]].dropna()
self.relevant_sub_categories = df.sub_category_fk.to_list()
else:
self.relevant_sub_categories = []
class GSKRUToolBox:
def __init__(self, data_provider, output):
self.output = output
self.data_provider = data_provider
self.project_name = self.data_provider.project_name
self.session_uid = self.data_provider.session_uid
self.products = self.data_provider[Data.PRODUCTS]
self.all_products = self.data_provider[Data.ALL_PRODUCTS]
self.match_product_in_scene = self.data_provider[Data.MATCHES]
self.visit_date = self.data_provider[Data.VISIT_DATE]
self.session_info = self.data_provider[Data.SESSION_INFO]
self.scene_info = self.data_provider[Data.SCENES_INFO]
self.store_id = self.data_provider[Data.STORE_FK]
self.scif = self.data_provider[Data.SCENE_ITEM_FACTS]
self.own_manufacturer_id = int(self.data_provider.own_manufacturer[
self.data_provider.own_manufacturer['param_name'] ==
'manufacturer_id']['param_value'].iloc[0])
self.rds_conn = PSProjectConnector(self.project_name,
DbUsers.CalculationEng)
self.common = Common(self.data_provider)
self.kpi_static_data = self.common.get_kpi_static_data()
self.ps_data_provider = PsDataProvider(self.data_provider, self.output)
self.store_type = self.ps_data_provider.session_info.store_type
self.store_channel = self.ps_data_provider.session_info.additional_attribute_11.encode(
'utf-8')
self.store_format = self.ps_data_provider.session_info.additional_attribute_12.encode(
'utf-8')
self.retailer_fk = self.ps_data_provider.session_info.retailer_fk
self.set_up_template = None
self.gsk_generator = None
self.core_range_targets = {}
self.set_up_data = LocalConsts.SET_UP_DATA
self.set_up_template = pd.read_excel(
os.path.join(os.path.dirname(os.path.realpath(__file__)), '..',
'Data', 'gsk_set_up.xlsx'),
sheet_name='Functional KPIs All Store',
keep_default_na=False)
self.gsk_generator = GSKGenerator(self.data_provider, self.output,
self.common, self.set_up_template)
def main_calculation(self, *args, **kwargs):
"""
This function calculates the KPI results.
"""
# Global KPIs
# All Store KPIs
assortment_store_dict = self.gsk_generator.availability_store_function(
custom_suffix='_Stacking_Included')
self.common.save_json_to_new_tables(assortment_store_dict)
assortment_category_dict = self.gsk_generator.availability_category_function(
custom_suffix='_Stacking_Included')
self.common.save_json_to_new_tables(assortment_category_dict)
assortment_subcategory_dict = self.gsk_generator.availability_subcategory_function(
custom_suffix='_Stacking_Included')
self.common.save_json_to_new_tables(assortment_subcategory_dict)
facings_sos_dict = self.gsk_generator.gsk_global_facings_sos_whole_store_function(
custom_suffix='_Stacking_Included',
fractional_facings_parameters=LocalConsts.
FRACTIONAL_FACINGS_PARAMETERS)
self.common.save_json_to_new_tables(facings_sos_dict)
facings_sos_dict = self.gsk_generator.gsk_global_facings_sos_by_category_function(
custom_suffix='_Stacking_Included',
fractional_facings_parameters=LocalConsts.
FRACTIONAL_FACINGS_PARAMETERS)
self.common.save_json_to_new_tables(facings_sos_dict)
facings_sos_dict = self.gsk_generator.gsk_global_facings_by_sub_category_function(
custom_suffix='_Stacking_Included',
fractional_facings_parameters=LocalConsts.
FRACTIONAL_FACINGS_PARAMETERS)
self.common.save_json_to_new_tables(facings_sos_dict)
linear_sos_dict = self.gsk_generator.gsk_global_linear_sos_whole_store_function(
)
self.common.save_json_to_new_tables(linear_sos_dict)
linear_sos_dict = self.gsk_generator.gsk_global_linear_sos_by_category_function(
)
self.common.save_json_to_new_tables(linear_sos_dict)
linear_sos_dict = self.gsk_generator.gsk_global_linear_sos_by_sub_category_function(
)
self.common.save_json_to_new_tables(linear_sos_dict)
# Main Shelf KPIs
self.rds_conn = PSProjectConnector(self.project_name,
DbUsers.CalculationEng)
self.set_up_template = pd.read_excel(
os.path.join(os.path.dirname(os.path.realpath(__file__)), '..',
'Data', 'gsk_set_up.xlsx'),
sheet_name='Functional KPIs Main Shelf',
keep_default_na=False)
self.gsk_generator.set_up_file = self.set_up_template
self.gsk_generator.tool_box.set_up_file = self.gsk_generator.set_up_file
self.gsk_generator.tool_box.set_up_data = LocalConsts.SET_UP_DATA.copy(
)
# self.gsk_generator = GSKGenerator(self.data_provider, self.output, self.common, self.set_up_template)
facings_sos_dict = self.gsk_generator.gsk_global_facings_sos_whole_store_function(
custom_suffix='_Stacking_Included_Main_Shelf',
fractional_facings_parameters=LocalConsts.
FRACTIONAL_FACINGS_PARAMETERS)
self.common.save_json_to_new_tables(facings_sos_dict)
facings_sos_dict = self.gsk_generator.gsk_global_facings_sos_by_category_function(
custom_suffix='_Stacking_Included_Main_Shelf',
fractional_facings_parameters=LocalConsts.
FRACTIONAL_FACINGS_PARAMETERS)
self.common.save_json_to_new_tables(facings_sos_dict)
facings_sos_dict = self.gsk_generator.gsk_global_facings_by_sub_category_function(
custom_suffix='_Stacking_Included_Main_Shelf',
fractional_facings_parameters=LocalConsts.
FRACTIONAL_FACINGS_PARAMETERS)
self.common.save_json_to_new_tables(facings_sos_dict)
linear_sos_dict = self.gsk_generator.gsk_global_linear_sos_whole_store_function(
custom_suffix='_Main_Shelf')
self.common.save_json_to_new_tables(linear_sos_dict)
linear_sos_dict = self.gsk_generator.gsk_global_linear_sos_by_category_function(
custom_suffix='_Main_Shelf')
self.common.save_json_to_new_tables(linear_sos_dict)
linear_sos_dict = self.gsk_generator.gsk_global_linear_sos_by_sub_category_function(
custom_suffix='_Main_Shelf')
self.common.save_json_to_new_tables(linear_sos_dict)
# Secondary Shelf KPIs
self.rds_conn = PSProjectConnector(self.project_name,
DbUsers.CalculationEng)
self.set_up_template = pd.read_excel(
os.path.join(os.path.dirname(os.path.realpath(__file__)), '..',
'Data', 'gsk_set_up.xlsx'),
sheet_name='Functional KPIs Secondary Shelf',
keep_default_na=False)
self.gsk_generator.set_up_file = self.set_up_template
self.gsk_generator.tool_box.set_up_file = self.gsk_generator.set_up_file
self.gsk_generator.tool_box.set_up_data = LocalConsts.SET_UP_DATA.copy(
)
# self.gsk_generator = GSKGenerator(self.data_provider, self.output, self.common, self.set_up_template)
facings_sos_dict = self.gsk_generator.gsk_global_facings_sos_whole_store_function(
custom_suffix='_Stacking_Included_Secondary_Shelf',
fractional_facings_parameters=LocalConsts.
FRACTIONAL_FACINGS_PARAMETERS)
self.common.save_json_to_new_tables(facings_sos_dict)
facings_sos_dict = self.gsk_generator.gsk_global_facings_sos_by_category_function(
custom_suffix='_Stacking_Included_Secondary_Shelf',
fractional_facings_parameters=LocalConsts.
FRACTIONAL_FACINGS_PARAMETERS)
self.common.save_json_to_new_tables(facings_sos_dict)
facings_sos_dict = self.gsk_generator.gsk_global_facings_by_sub_category_function(
custom_suffix='_Stacking_Included_Secondary_Shelf',
fractional_facings_parameters=LocalConsts.
FRACTIONAL_FACINGS_PARAMETERS)
self.common.save_json_to_new_tables(facings_sos_dict)
linear_sos_dict = self.gsk_generator.gsk_global_linear_sos_whole_store_function(
custom_suffix='_Secondary_Shelf')
self.common.save_json_to_new_tables(linear_sos_dict)
linear_sos_dict = self.gsk_generator.gsk_global_linear_sos_by_category_function(
custom_suffix='_Secondary_Shelf')
self.common.save_json_to_new_tables(linear_sos_dict)
linear_sos_dict = self.gsk_generator.gsk_global_linear_sos_by_sub_category_function(
custom_suffix='_Secondary_Shelf')
self.common.save_json_to_new_tables(linear_sos_dict)
# Local KPIs
self.rds_conn = PSProjectConnector(self.project_name,
DbUsers.CalculationEng)
self.set_up_template = pd.read_excel(
os.path.join(os.path.dirname(os.path.realpath(__file__)), '..',
'Data', 'gsk_set_up.xlsx'),
sheet_name='Functional KPIs Local',
keep_default_na=False)
self.gsk_generator.set_up_file = self.set_up_template
self.gsk_generator.tool_box.set_up_file = self.gsk_generator.set_up_file
self.gsk_generator.tool_box.set_up_data = LocalConsts.SET_UP_DATA.copy(
)
# self.gsk_generator = GSKGenerator(self.data_provider, self.output, self.common, self.set_up_template)
# SOA
soa_dict = self.gsk_soa_function()
self.common.save_json_to_new_tables(soa_dict)
# # Core Range Assortment - disabled until phase 2
# cra_dict = self.gsk_cra_function()
# self.common.save_json_to_new_tables(cra_dict)
self.common.commit_results_data()
return
def gsk_soa_function(self):
results = []
kpi_soa_fk = \
self.common.get_kpi_fk_by_kpi_type(LocalConsts.SOA_KPI)
kpi_soa_manufacturer_internal_target_fk = \
self.common.get_kpi_fk_by_kpi_type(LocalConsts.SOA_MANUFACTURER_INTERNAL_TARGET_KPI)
kpi_soa_manufacturer_external_target_fk = \
self.common.get_kpi_fk_by_kpi_type(LocalConsts.SOA_MANUFACTURER_EXTERNAL_TARGET_KPI)
kpi_soa_subcat_internal_target_fk = \
self.common.get_kpi_fk_by_kpi_type(LocalConsts.SOA_SUBCAT_INTERNAL_TARGET_KPI)
kpi_soa_subcat_external_target_fk = \
self.common.get_kpi_fk_by_kpi_type(LocalConsts.SOA_SUBCAT_EXTERNAL_TARGET_KPI)
identifier_internal = self.common.get_dictionary(
manufacturer_fk=self.own_manufacturer_id,
kpi_fk=kpi_soa_manufacturer_internal_target_fk)
identifier_external = self.common.get_dictionary(
manufacturer_fk=self.own_manufacturer_id,
kpi_fk=kpi_soa_manufacturer_external_target_fk)
targets = \
self.ps_data_provider.get_kpi_external_targets(kpi_fks=[kpi_soa_fk],
key_filters={'additional_attribute_11': self.store_channel,
'additional_attribute_12': self.store_format})
# if targets.empty:
# Log.warning('No SOA targets defined for this session')
# else:
self.gsk_generator.tool_box. \
extract_data_set_up_file(LocalConsts.SOA, self.set_up_data, LocalConsts.KPI_DICT)
df = self.gsk_generator.tool_box.tests_by_template(
LocalConsts.SOA, self.scif, self.set_up_data)
df, facings_column = self.df_filter_by_stacking(df, LocalConsts.SOA)
# Sub-Category
for sub_category_fk in df[
ScifConsts.SUB_CATEGORY_FK].unique().tolist():
numerator_result = len(
df[(df[ScifConsts.MANUFACTURER_FK] == self.own_manufacturer_id)
& (df[ScifConsts.SUB_CATEGORY_FK] == sub_category_fk)][
ScifConsts.PRODUCT_FK].unique().tolist())
denominator_result = len(
df[df[ScifConsts.SUB_CATEGORY_FK] == sub_category_fk][
ScifConsts.PRODUCT_FK].unique().tolist())
result = round(float(numerator_result) / float(denominator_result), 4) \
if numerator_result != 0 and denominator_result != 0 \
else 0
target = targets[targets['sub_category_fk'] ==
sub_category_fk]['internal_target'].values
target = float(target[0]) if len(target) > 0 else None
target = target / 100 if target else None
if target:
# score = 1 if result >= target else 0
score = round(result / target, 4)
else:
score = 0
results.append({
'fk': kpi_soa_subcat_internal_target_fk,
SessionResultsConsts.NUMERATOR_ID: self.own_manufacturer_id,
SessionResultsConsts.NUMERATOR_RESULT: numerator_result,
SessionResultsConsts.DENOMINATOR_ID: self.store_id,
SessionResultsConsts.DENOMINATOR_RESULT: denominator_result,
SessionResultsConsts.CONTEXT_ID: sub_category_fk,
SessionResultsConsts.RESULT: result,
SessionResultsConsts.TARGET: target,
SessionResultsConsts.SCORE: score,
'identifier_parent': identifier_internal,
'should_enter': True
})
self.core_range_targets.update({sub_category_fk: target})
target = targets[targets['sub_category_fk'] ==
sub_category_fk]['external_target'].values
target = float(target[0]) if len(target) > 0 else None
target = target / 100 if target else None
if target:
# score = 1 if result >= target else 0
score = round(result / target, 4)
else:
score = 0
results.append({
'fk': kpi_soa_subcat_external_target_fk,
SessionResultsConsts.NUMERATOR_ID: self.own_manufacturer_id,
SessionResultsConsts.NUMERATOR_RESULT: numerator_result,
SessionResultsConsts.DENOMINATOR_ID: self.store_id,
SessionResultsConsts.DENOMINATOR_RESULT: denominator_result,
SessionResultsConsts.CONTEXT_ID: sub_category_fk,
SessionResultsConsts.RESULT: result,
SessionResultsConsts.TARGET: target,
SessionResultsConsts.SCORE: score,
'identifier_parent': identifier_external,
'should_enter': True
})
# Manufacturer
numerator_result = len(
df[df[ScifConsts.MANUFACTURER_FK] == self.own_manufacturer_id][
ScifConsts.PRODUCT_FK].unique().tolist())
denominator_result = len(df[ScifConsts.PRODUCT_FK].unique().tolist())
result = round(float(numerator_result) / float(denominator_result), 4) \
if numerator_result != 0 and denominator_result != 0 \
else 0
target = targets[
targets['sub_category_fk'].isnull()]['internal_target'].values
target = float(target[0]) if len(target) > 0 else None
target = target / 100 if target else None
if target:
# score = 1 if result >= target else 0
score = round(result / target, 4)
else:
score = 0
results.append({
'fk': kpi_soa_manufacturer_internal_target_fk,
SessionResultsConsts.NUMERATOR_ID: self.own_manufacturer_id,
SessionResultsConsts.NUMERATOR_RESULT: numerator_result,
SessionResultsConsts.DENOMINATOR_ID: self.store_id,
SessionResultsConsts.DENOMINATOR_RESULT: denominator_result,
SessionResultsConsts.RESULT: result,
SessionResultsConsts.TARGET: target,
SessionResultsConsts.SCORE: score,
'identifier_result': identifier_internal,
'should_enter': True
})
target = targets[
targets['sub_category_fk'].isnull()]['external_target'].values
target = float(target[0]) if len(target) > 0 else None
target = target / 100 if target else None
if target:
# score = 1 if result >= target else 0
score = round(result / target, 4)
else:
score = 0
results.append({
'fk': kpi_soa_manufacturer_external_target_fk,
SessionResultsConsts.NUMERATOR_ID: self.own_manufacturer_id,
SessionResultsConsts.NUMERATOR_RESULT: numerator_result,
SessionResultsConsts.DENOMINATOR_ID: self.store_id,
SessionResultsConsts.DENOMINATOR_RESULT: denominator_result,
SessionResultsConsts.RESULT: result,
SessionResultsConsts.TARGET: target,
SessionResultsConsts.SCORE: score,
'identifier_result': identifier_external,
'should_enter': True
})
return results
def gsk_cra_function(self):
results = []
kpi_cra_fk = \
self.common.get_kpi_fk_by_kpi_type(LocalConsts.CRA_KPI)
kpi_cra_manufacturer_fk = \
self.common.get_kpi_fk_by_kpi_type(LocalConsts.CRA_MANUFACTURER_KPI)
kpi_cra_subcat_fk = \
self.common.get_kpi_fk_by_kpi_type(LocalConsts.CRA_SUBCAT_KPI)
kpi_cra_subcat_by_product_fk = \
self.common.get_kpi_fk_by_kpi_type(LocalConsts.CRA_SUBCAT_BY_PRODUCT_KPI)
identifier_manufacturer = self.common.get_dictionary(
manufacturer_fk=self.own_manufacturer_id,
kpi_fk=kpi_cra_manufacturer_fk)
total_cra_size_target = 0
total_cra_size_actual = 0
targets = \
self.ps_data_provider.get_kpi_external_targets(kpi_fks=[kpi_cra_fk],
key_filters={'additional_attribute_12': self.store_format,
'retailer_fk': self.retailer_fk})
if targets.empty:
Log.warning('No CRA targets defined for this session')
else:
self.gsk_generator.tool_box. \
extract_data_set_up_file(LocalConsts.CRA, self.set_up_data, LocalConsts.KPI_DICT)
df = self.gsk_generator.tool_box.tests_by_template(
LocalConsts.CRA, self.scif, self.set_up_data)
df, facings_column = self.df_filter_by_stacking(
df, LocalConsts.CRA)
df = df[df[ScifConsts.SUB_CATEGORY_FK].notnull()][
[ScifConsts.SUB_CATEGORY_FK, ScifConsts.PRODUCT_FK, facings_column]]\
.groupby([ScifConsts.SUB_CATEGORY_FK, ScifConsts.PRODUCT_FK]).agg({facings_column: 'sum'})\
.reset_index()
df = df.merge(
targets[['sub_category_fk', 'product_fk', 'priority']],
how='left',
left_on=[ScifConsts.SUB_CATEGORY_FK, ScifConsts.PRODUCT_FK],
right_on=['sub_category_fk', 'product_fk'])
df['unique_product_id'] = \
df.apply(lambda r:
'P' + str(r['priority']) if pd.notnull(r['priority']) else 'N' + str(r['product_fk']), axis=1)
# Sub-Category
target_subcat_fks = set(
targets['sub_category_fk'].unique().tolist()) & set(
self.core_range_targets.keys())
for sub_category_fk in target_subcat_fks:
identifier_subcat = self.common.get_dictionary(
manufacturer_fk=self.own_manufacturer_id,
sub_category_fk=sub_category_fk,
kpi_fk=kpi_cra_subcat_fk)
if sub_category_fk not in self.core_range_targets.keys():
numerator_result = denominator_result = result = score = 0
else:
subcat_size = len(
df[df[ScifConsts.SUB_CATEGORY_FK] == sub_category_fk]
['unique_product_id'].unique().tolist())
core_range_target = self.core_range_targets[
sub_category_fk]
cra_priority = round(
subcat_size *
core_range_target if core_range_target else 0)
cra_products_target = targets[
(targets['sub_category_fk'] == sub_category_fk)
& (targets['priority'] <= cra_priority)][[
'product_fk', 'priority'
]]
cra_products_actual = df[
(df[ScifConsts.SUB_CATEGORY_FK] == sub_category_fk)
& (df['priority'] <= cra_priority)][[
ScifConsts.PRODUCT_FK, facings_column
]]
cra_size_target = len(
targets[(targets['sub_category_fk'] == sub_category_fk)
& (targets['priority'] <= cra_priority)]
['priority'].unique().tolist())
cra_size_actual = len(
df[(df[ScifConsts.SUB_CATEGORY_FK] == sub_category_fk)
& (df['priority'] <= cra_priority)]
['priority'].unique().tolist())
if cra_size_target == 0:
numerator_result = denominator_result = result = score = 0
else:
# Product
for i, product in cra_products_target.iterrows():
numerator_result = \
cra_products_actual[cra_products_actual[ScifConsts.PRODUCT_FK] ==
product['product_fk']][facings_column].sum()
denominator_result = product['priority']
result = 1 if numerator_result else 0
score = result
results.append({
'fk':
kpi_cra_subcat_by_product_fk,
SessionResultsConsts.NUMERATOR_ID:
product['product_fk'],
SessionResultsConsts.NUMERATOR_RESULT:
numerator_result,
SessionResultsConsts.DENOMINATOR_ID:
self.own_manufacturer_id,
SessionResultsConsts.DENOMINATOR_RESULT:
denominator_result,
SessionResultsConsts.CONTEXT_ID:
sub_category_fk,
SessionResultsConsts.RESULT:
result,
SessionResultsConsts.SCORE:
score,
'identifier_parent':
identifier_subcat,
'should_enter':
True
})
numerator_result = cra_size_actual
denominator_result = cra_size_target
result = round(float(numerator_result) / float(denominator_result), 4) \
if numerator_result != 0 and denominator_result != 0 \
else 0
score = result
total_cra_size_target += cra_size_target
total_cra_size_actual += cra_size_actual
results.append({
'fk': kpi_cra_subcat_fk,
SessionResultsConsts.NUMERATOR_ID:
self.own_manufacturer_id,
SessionResultsConsts.NUMERATOR_RESULT: numerator_result,
SessionResultsConsts.DENOMINATOR_ID: self.store_id,
SessionResultsConsts.DENOMINATOR_RESULT:
denominator_result,
SessionResultsConsts.CONTEXT_ID: sub_category_fk,
SessionResultsConsts.RESULT: result,
SessionResultsConsts.SCORE: score,
'identifier_parent': identifier_manufacturer,
'identifier_result': identifier_subcat,
'should_enter': True
})
# Manufacturer
if target_subcat_fks:
numerator_result = total_cra_size_actual
denominator_result = total_cra_size_target
result = round(float(total_cra_size_actual) / float(total_cra_size_target), 4) \
if numerator_result != 0 and denominator_result != 0 \
else 0
score = result
results.append({
'fk': kpi_cra_manufacturer_fk,
SessionResultsConsts.NUMERATOR_ID:
self.own_manufacturer_id,
SessionResultsConsts.NUMERATOR_RESULT: numerator_result,
SessionResultsConsts.DENOMINATOR_ID: self.store_id,
SessionResultsConsts.DENOMINATOR_RESULT:
denominator_result,
SessionResultsConsts.RESULT: result,
SessionResultsConsts.SCORE: score,
'identifier_result': identifier_manufacturer,
'should_enter': True
})
return results
def df_filter_by_stacking(self, df, kpi_type):
include_stacking = self.set_up_data.get(
(GlobalConsts.INCLUDE_STACKING, kpi_type), True)
facings_column = ScifConsts.FACINGS
if not include_stacking:
facings_column = ScifConsts.FACINGS_IGN_STACK
df = df[df[facings_column] > 0]
return df, facings_column
class ToolBox(GlobalSessionToolBox):
def __init__(self, data_provider, output):
GlobalSessionToolBox.__init__(self, data_provider, output)
self.own_manufacturer_fk = int(self.data_provider.own_manufacturer.param_value.values[0])
self.parser = Parser
self.all_products = self.data_provider[Data.ALL_PRODUCTS]
self.assortment = Assortment(self.data_provider, self.output)
self.ps_data = PsDataProvider(self.data_provider, self.output)
self.kpi_external_targets = self.ps_data.get_kpi_external_targets(key_fields=Consts.KEY_FIELDS,
data_fields=Consts.DATA_FIELDS)
def main_calculation(self):
self.calculate_score_sos()
self.calculate_oos_and_distribution(assortment_type="Core")
self.calculate_oos_and_distribution(assortment_type="Launch")
self.calculate_oos_and_distribution(assortment_type="Focus")
self.calculate_hierarchy_sos(calculation_type='FACINGS')
self.calculate_hierarchy_sos(calculation_type='LINEAR')
@kpi_runtime()
def calculate_oos_and_distribution(self, assortment_type):
dis_numerator = total_facings = 0
oos_store_kpi_fk = self.common.get_kpi_fk_by_kpi_type(kpi_type=assortment_type + Consts.OOS)
oos_sku_kpi_fk = self.common.get_kpi_fk_by_kpi_type(kpi_type=assortment_type + Consts.OOS_SKU)
dis_store_kpi_fk = self.common.get_kpi_fk_by_kpi_type(kpi_type=assortment_type + Consts.DISTRIBUTION)
dis_cat_kpi_fk = self.common.get_kpi_fk_by_kpi_type(kpi_type=assortment_type + Consts.DISTRIBUTION_CAT)
dis_sku_kpi_fk = self.common.get_kpi_fk_by_kpi_type(kpi_type=assortment_type + Consts.DISTRIBUTION_SKU)
assortment_df = self.assortment.get_lvl3_relevant_ass()
assortment_df = assortment_df[assortment_df['kpi_fk_lvl3'] == dis_sku_kpi_fk]
product_fks = assortment_df['product_fk'].tolist()
categories = list(set(self.all_products[self.all_products['product_fk'].isin(product_fks)]['category_fk']))
categories_dict = dict.fromkeys(categories, (0, 0))
# sku level distribution
for sku in product_fks:
# 2 for distributed and 1 for oos
category_fk = self.all_products[self.all_products['product_fk'] == sku]['category_fk'].values[0]
product_df = self.scif[self.scif['product_fk'] == sku]
if product_df.empty:
categories_dict[category_fk] = map(sum, zip(categories_dict[category_fk], [0, 1]))
result = 1
facings = 0
# Saving OOS only if product wasn't in store
self.common.write_to_db_result(fk=oos_sku_kpi_fk, numerator_id=sku, denominator_id=category_fk,
result=result, numerator_result=result, denominator_result=result,
score=facings, identifier_parent=assortment_type + "_OOS",
should_enter=True)
else:
categories_dict[category_fk] = map(sum, zip(categories_dict[category_fk], [1, 1]))
result = 2
facings = product_df['facings'].values[0]
dis_numerator += 1
total_facings += facings
self.common.write_to_db_result(fk=dis_sku_kpi_fk, numerator_id=sku, denominator_id=category_fk,
result=result, numerator_result=result, denominator_result=result,
score=facings, should_enter=True,
identifier_parent=assortment_type + "_DIS_CAT_{}".format(str(category_fk)))
# category level distribution
for category_fk in categories_dict.keys():
cat_numerator, cat_denominator = categories_dict[category_fk]
cat_result = self.get_result(cat_numerator, cat_denominator)
self.common.write_to_db_result(fk=dis_cat_kpi_fk, numerator_id=category_fk,
denominator_id=self.store_id, result=cat_result, should_enter=True,
numerator_result=cat_numerator, denominator_result=cat_denominator,
score=cat_result, identifier_parent=assortment_type + "_DIS",
identifier_result=assortment_type + "_DIS_CAT_{}".format(str(category_fk)))
# store level oos and distribution
denominator = len(product_fks)
dis_result = self.get_result(dis_numerator, denominator)
oos_result = 1 - dis_result
oos_numerator = denominator - dis_numerator
self.common.write_to_db_result(fk=oos_store_kpi_fk, numerator_id=self.own_manufacturer_fk,
denominator_id=self.store_id, result=oos_result, numerator_result=oos_numerator,
denominator_result=denominator, score=total_facings,
identifier_result=assortment_type + "_OOS")
self.common.write_to_db_result(fk=dis_store_kpi_fk, numerator_id=self.own_manufacturer_fk,
denominator_id=self.store_id, result=dis_result, numerator_result=dis_numerator,
denominator_result=denominator, score=total_facings,
identifier_result=assortment_type + "_DIS")
def get_kpi_fks(self, kpis_list):
for kpi in kpis_list:
self.common.get_kpi_fk_by_kpi_type(kpi_type=kpi)
@staticmethod
def calculate_sos_res(numerator, denominator):
if denominator == 0:
return 0, 0, 0
result = round(numerator / float(denominator), 3)
return result, numerator, denominator
@kpi_runtime()
def calculate_score_sos(self):
relevant_template = self.kpi_external_targets[self.kpi_external_targets[ExternalTargetsConsts.OPERATION_TYPE]
== Consts.SOS_KPIS]
relevant_rows = relevant_template.copy()
lsos_score_kpi_fk = self.common.get_kpi_fk_by_kpi_type(Consts.LSOS_SCORE_KPI)
store_denominator = len(relevant_rows)
store_numerator = 0
for i, kpi_row in relevant_template.iterrows():
kpi_fk, num_type, num_value, deno_type, deno_value, target, target_range = kpi_row[Consts.RELEVANT_FIELDS]
numerator_filters, denominator_filters = self.get_num_and_den_filters(num_type, num_value, deno_type,
deno_value)
# Only straussil SKUs
numerator_filters['manufacturer_fk'] = self.own_manufacturer_fk
denominator_filters['manufacturer_fk'] = self.own_manufacturer_fk
numerator_df = self.parser.filter_df(conditions=numerator_filters, data_frame_to_filter=self.scif)
denominator_df = self.parser.filter_df(conditions=denominator_filters, data_frame_to_filter=self.scif)
numerator_result = numerator_df['gross_len_ign_stack'].sum()
denominator_result = denominator_df['gross_len_ign_stack'].sum()
lsos_result = self.get_result(numerator_result, denominator_result)
score = 1 if ((target - target_range) <= lsos_result <= (target + target_range)) else 0
store_numerator += score
self.common.write_to_db_result(fk=kpi_fk, numerator_id=self.own_manufacturer_fk,
denominator_id=self.store_id, should_enter=True, target=target,
numerator_result=numerator_result, denominator_result=denominator_result,
result=lsos_result, score=score, identifier_parent='LSOS_SCORE',
weight=target_range)
store_result = self.get_result(store_numerator, store_denominator)
self.common.write_to_db_result(fk=lsos_score_kpi_fk, numerator_id=self.own_manufacturer_fk,
denominator_id=self.store_id, should_enter=True, target=store_denominator,
numerator_result=store_numerator, denominator_result=store_denominator,
result=store_numerator, score=store_result, identifier_result='LSOS_SCORE')
@staticmethod
def get_num_and_den_filters(numerator_type, numerator_value, denominator_type, denominator_value):
if type(numerator_value) != list:
numerator_value = [numerator_value]
if type(denominator_value) != list:
denominator_value = [denominator_value]
numerator_filters = {numerator_type: numerator_value}
denominator_filters = {denominator_type: denominator_value}
return numerator_filters, denominator_filters
@staticmethod
def get_result(numerator, denominator):
if denominator == 0:
return 0
else:
return round(numerator / float(denominator), 4)
def calculate_hierarchy_sos(self, calculation_type):
brand_kpi_fk = self.common.get_kpi_fk_by_kpi_type(kpi_type=(calculation_type + Consts.SOS_BY_BRAND))
brand_category_kpi_fk = self.common.get_kpi_fk_by_kpi_type(kpi_type=(calculation_type +
Consts.SOS_BY_CAT_BRAND))
sku_kpi_fk = self.common.get_kpi_fk_by_kpi_type(kpi_type=(calculation_type + Consts.SOS_BY_CAT_BRAND_SKU))
calculation_param = "facings_ign_stack" if calculation_type == 'FACINGS' else "gross_len_ign_stack"
sos_df = self.parser.filter_df(conditions={'rlv_sos_sc': 1, 'product_type': ['SKU', 'Empty']},
data_frame_to_filter=self.scif)
# brand level sos
session_brands = set(sos_df['brand_fk'])
brand_den = sos_df[calculation_param].sum()
for brand_fk in session_brands:
filters = {'brand_fk': brand_fk}
brand_df = self.parser.filter_df(conditions=filters, data_frame_to_filter=sos_df)
if brand_df.empty:
continue
manufacturer_fk = brand_df['manufacturer_fk'].values[0]
brand_num = brand_df[calculation_param].sum()
if brand_num == 0:
continue
brand_res, brand_num, brand_den = self.calculate_sos_res(brand_num, brand_den)
self.common.write_to_db_result(fk=brand_kpi_fk, numerator_id=brand_fk,
denominator_id=manufacturer_fk,
result=brand_res, numerator_result=brand_num, denominator_result=brand_den,
score=brand_res,
identifier_result="{}_SOS_brand_{}".format(calculation_type, str(brand_fk)))
# brand-category level sos
brand_categories = set(self.parser.filter_df(conditions=filters,
data_frame_to_filter=sos_df)['category_fk'])
for category_fk in brand_categories:
cat_den = self.parser.filter_df(conditions={'category_fk': category_fk},
data_frame_to_filter=sos_df)[calculation_param].sum()
filters['category_fk'] = category_fk
category_df = self.parser.filter_df(conditions=filters, data_frame_to_filter=sos_df)
cat_num = category_df[calculation_param].sum()
if cat_num == 0:
continue
cat_res, cat_num, cat_den = self.calculate_sos_res(cat_num, cat_den)
self.common.write_to_db_result(fk=brand_category_kpi_fk, numerator_id=brand_fk,
context_id=manufacturer_fk,
denominator_id=category_fk, result=cat_res, numerator_result=cat_num,
should_enter=True, denominator_result=cat_den, score=cat_res,
identifier_parent="{}_SOS_brand_{}".format(calculation_type,
str(brand_fk)),
identifier_result="{}_SOS_cat_{}_brand_{}".format(calculation_type,
str(category_fk),
str(brand_fk)))
product_fks = set(self.parser.filter_df(conditions=filters, data_frame_to_filter=sos_df)['product_fk'])
for sku in product_fks:
filters['product_fk'] = sku
product_df = self.parser.filter_df(conditions=filters, data_frame_to_filter=sos_df)
sku_num = product_df[calculation_param].sum()
if sku_num == 0:
continue
sku_result, sku_num, sku_den = self.calculate_sos_res(sku_num, cat_num)
self.common.write_to_db_result(fk=sku_kpi_fk, numerator_id=sku, denominator_id=brand_fk,
result=sku_result, numerator_result=sku_num, should_enter=True,
denominator_result=cat_num, score=sku_num,
context_id=category_fk, weight=manufacturer_fk,
identifier_parent="{}_SOS_cat_{}_brand_{}".format(calculation_type,
str(category_fk),
str(brand_fk)))
del filters['product_fk']
del filters['category_fk']
class GSKSGToolBox:
KPI_DICT = {
"planogram": "planogram",
"secondary_display": "secondary_display",
"promo": "promo"
}
def __init__(self, data_provider, output):
self.output = output
self.data_provider = data_provider
self.common = Common(self.data_provider)
self.project_name = self.data_provider.project_name
self.session_uid = self.data_provider.session_uid
self.products = self.data_provider[Data.PRODUCTS]
self.all_products = self.data_provider[Data.ALL_PRODUCTS]
self.match_product_in_scene = self.data_provider[Data.MATCHES]
self.visit_date = self.data_provider[Data.VISIT_DATE]
self.session_info = self.data_provider[Data.SESSION_INFO]
self.scene_info = self.data_provider[Data.SCENES_INFO]
self.store_id = self.data_provider[Data.STORE_FK]
self.store_info = self.data_provider[Data.STORE_INFO]
self.scif = self.data_provider[Data.SCENE_ITEM_FACTS]
self.ps_data_provider = PsDataProvider(self.data_provider, self.output)
self.manufacturer_fk = None if self.data_provider[Data.OWN_MANUFACTURER]['param_value'].iloc[0] is None else \
int(self.data_provider[Data.OWN_MANUFACTURER]['param_value'].iloc[0])
self.set_up_template = pd.read_excel(os.path.join(
os.path.dirname(os.path.realpath(__file__)), '..', 'Data',
'gsk_set_up.xlsx'),
sheet_name='Functional KPIs',
keep_default_na=False)
self.gsk_generator = GSKGenerator(self.data_provider, self.output,
self.common, self.set_up_template)
self.targets = self.ps_data_provider.get_kpi_external_targets()
self.sequence = Sequence(self.data_provider)
self.set_up_data = {
('planogram', Const.KPI_TYPE_COLUMN): Const.NO_INFO,
('secondary_display', Const.KPI_TYPE_COLUMN): Const.NO_INFO,
('promo', Const.KPI_TYPE_COLUMN): Const.NO_INFO
}
def main_calculation(self):
"""
This function calculates the KPI results.
"""
# # global kpis in store_level
assortment_store_dict = self.gsk_generator.availability_store_function(
)
self.common.save_json_to_new_tables(assortment_store_dict)
facings_sos_dict = self.gsk_generator.gsk_global_facings_sos_whole_store_function(
)
self.common.save_json_to_new_tables(facings_sos_dict)
linear_sos_dict = self.gsk_generator.gsk_global_linear_sos_whole_store_function(
)
self.common.save_json_to_new_tables(linear_sos_dict)
# global kpis in category level & kpi results are used for orange score kpi
assortment_category_dict = self.gsk_generator.availability_category_function(
)
self.common.save_json_to_new_tables(assortment_category_dict)
fsos_category_dict = self.gsk_generator.gsk_global_facings_sos_by_category_function(
)
self.common.save_json_to_new_tables(fsos_category_dict)
# updating the set up dictionary for all local kpis
for kpi in self.KPI_DICT.keys():
self.gsk_generator.tool_box.extract_data_set_up_file(
kpi, self.set_up_data, self.KPI_DICT)
orange_score_dict = self.orange_score_category(
assortment_category_dict, fsos_category_dict)
self.common.save_json_to_new_tables(orange_score_dict)
self.common.commit_results_data()
score = 0
return score
def msl_compliance_score(self, category, categories_results_json,
cat_targets, parent_result_identifier):
results_list = []
msl_kpi_fk = self.common.get_kpi_fk_by_kpi_type(
Consts.MSL_ORANGE_SCORE)
msl_categories = self._filter_targets_by_kpi(cat_targets, msl_kpi_fk)
if category not in categories_results_json:
dst_result = 0
else:
dst_result = categories_results_json[category]
weight = msl_categories['msl_weight'].iloc[0]
score = dst_result * weight
result = score / weight
results_list.append({
'fk': msl_kpi_fk,
'numerator_id': category,
'denominator_id': self.store_id,
'denominator_result': 1,
'numerator_result': result,
'result': result,
'target': weight,
'score': score,
'identifier_parent': parent_result_identifier,
'should_enter': True
})
return score, results_list
def fsos_compliance_score(self, category, categories_results_json,
cat_targets, parent_result_identifier):
"""
This function return json of keys- categories and values - kpi result for category
:param cat_targets-targets df for the specific category
:param category: pk of category
:param categories_results_json: type of the desired kpi
:return category json : number-category_fk,number-result
"""
results_list = []
fsos_kpi_fk = self.common.get_kpi_fk_by_kpi_type(
Consts.FSOS_ORANGE_SCORE)
category_targets = self._filter_targets_by_kpi(cat_targets,
fsos_kpi_fk)
dst_result = categories_results_json[
category] if category in categories_results_json.keys() else 0
benchmark = category_targets['fsos_benchmark'].iloc[0]
weight = category_targets['fsos_weight'].iloc[0]
score = weight if dst_result >= benchmark else 0
result = score / weight
results_list.append({
'fk': fsos_kpi_fk,
'numerator_id': category,
'denominator_id': self.store_id,
'denominator_result': 1,
'numerator_result': result,
'result': result,
'target': weight,
'score': score,
'identifier_parent': parent_result_identifier,
'should_enter': True
})
return score, results_list
def extract_json_results_by_kpi(self, general_kpi_results, kpi_type):
"""
This function return json of keys and values. keys= categories & values = kpi result for category
:param general_kpi_results: list of json's , each json is a db result
:param kpi_type: type of the desired kpi
:return category json : number-category_fk,number-result
"""
kpi_fk = self.common.get_kpi_fk_by_kpi_type(kpi_type)
if general_kpi_results is None:
return {}
categories_results_json = self.extract_json_results(
kpi_fk, general_kpi_results)
return categories_results_json
@staticmethod
def extract_json_results(kpi_fk, general_kpi_results):
"""
This function created json of keys- categories and values - kpi result for category
:param kpi_fk: pk of the kpi you want to extract results from.
:param general_kpi_results: list of json's , each json is the db results
:return category json : number-category_fk,number-result
"""
category_json = {}
for row in general_kpi_results:
if row['fk'] == kpi_fk:
category_json[row[
DB.SessionResultsConsts.DENOMINATOR_ID]] = row[
DB.SessionResultsConsts.RESULT]
return category_json
def store_target(self):
"""
This function filters the external targets df , to the only df with policy that answer current session's store
attributes.
It search which store attributes defined the targets policy.
In addition it gives the targets flexibility to send "changed variables" , external targets need to save
store param+_key and store_val + _value , than this function search the store param to look for and which value
it need to have for this policy.
"""
target_columns = self.targets.columns
store_att = ['store_name', 'store_number', 'att']
store_columns = [
col for col in target_columns
if len([att for att in store_att if att in col]) > 0
]
for col in store_columns:
if self.targets.empty:
return
if 'key' in col:
value = col.replace('_key', '') + '_value'
if value not in store_columns:
continue
self.target_test(col, value)
store_columns.remove(value)
else:
if 'value' in col:
continue
self.target_test(col)
def target_test(self, store_param, store_param_val=None):
"""
:param store_param: string , store attribute . by this attribute will compare between targets policy and
current session
:param store_param_val: string , if not None the store attribute value the policy have
This function filters the targets to the only targets with a attributes that answer the current session's
store attributes
"""
store_param_val = store_param_val if store_param_val is not None else store_param
store_param = [
store_param
] if store_param_val is None else self.targets[store_param].unique()
for param in store_param:
if param is None:
continue
if self.store_info[param][0] is None:
if self.targets.empty:
return
else:
self.targets.drop(self.targets.index, inplace=True)
self.targets['target_match'] = self.targets[store_param_val].apply(
self.checking_param, store_info_col=param)
self.targets = self.targets[self.targets['target_match']]
def checking_param(self, df_param, store_info_col):
# x is self.targets[store_param_val]
if isinstance(df_param, list):
if self.store_info[store_info_col][0].encode(
GlobalConsts.HelperConsts.UTF8) in df_param:
return True
if isinstance(df_param, unicode):
if self.store_info[store_info_col][0].encode(
GlobalConsts.HelperConsts.UTF8
) == df_param or df_param == '':
return True
if isinstance(df_param, type(None)):
return True
return False
def display_distribution(self, display_name, category_fk, category_targets,
parent_identifier, kpi_name, parent_kpi_name,
scif_df):
"""
This Function sum facings of posm that it name contains substring (decided by external_targets )
if sum facings is equal/bigger than benchmark that gets weight.
:param display_name display name (in external targets this key contains relevant substrings)
:param category_fk
:param category_targets-targets df for the specific category
:param parent_identifier - result identifier for this kpi parent
:param kpi_name - kpi name
:param parent_kpi_name - this parent kpi name
:param scif_df - scif filtered by promo activation settings
"""
kpi_fk = self.common.get_kpi_fk_by_kpi_type(kpi_name +
Consts.COMPLIANCE_KPI)
results_list = []
identifier_result = self.common.get_dictionary(category_fk=category_fk,
kpi_fk=kpi_fk)
weight = category_targets['{}_weight'.format(parent_kpi_name)].iloc[0]
if scif_df is None:
results_list.append({
'fk': kpi_fk,
'numerator_id': category_fk,
'denominator_id': self.store_id,
'denominator_result': 1,
'numerator_result': 0,
'result': 0,
'score': 0,
'identifier_parent': parent_identifier,
'identifier_result': identifier_result,
'target': weight,
'should_enter': True
})
return 0, results_list
display_products = scif_df[(scif_df['product_type'] == 'POS')
& (scif_df['category_fk'] == category_fk)]
display_name = "{}_name".format(display_name.lower())
display_names = category_targets[display_name].iloc[0]
kpi_result = 0
# check's if display names (received from external targets) are string or array of strings
if isinstance(display_names, str) or isinstance(
display_names, unicode):
display_array = []
if len(display_names) > 0:
display_array.append(display_names)
display_names = display_array
# for each display name , search POSM that contains display name (= sub string)
for display in display_names:
current_display_prod = display_products[
display_products['product_name'].str.contains(display)]
display_sku_level = self.display_sku_results(
current_display_prod, category_fk, kpi_name)
kpi_result += current_display_prod['facings'].sum()
results_list.extend(display_sku_level)
benchmark = category_targets['{}_benchmark'.format(
parent_kpi_name)].iloc[0]
kpi_score = weight if kpi_result >= benchmark else 0
results_list.append({
'fk': kpi_fk,
'numerator_id': category_fk,
'denominator_id': self.store_id,
'denominator_result': 1,
'numerator_result': kpi_score,
'result': kpi_score,
'score': kpi_score,
'identifier_parent': parent_identifier,
'identifier_result': identifier_result,
'target': weight,
'should_enter': True
})
return kpi_score, results_list
def display_sku_results(self, display_data, category_fk, kpi_name):
"""
This Function create for each posm in display data db result with score of posm facings.
:param category_fk
:param display_data-targets df for the specific category
:param kpi_name - kpi name
"""
results_list = []
kpi_fk = self.common.get_kpi_fk_by_kpi_type(kpi_name +
Consts.SKU_LEVEL_LIST)
parent_kpi_fk = self.common.get_kpi_fk_by_kpi_type(
kpi_name + Consts.COMPLIANCE_KPI)
identifier_parent = self.common.get_dictionary(category_fk=category_fk,
kpi_fk=parent_kpi_fk)
display_names = display_data['item_id'].unique()
for display in display_names:
count = float(display_data[display_data['item_id'] == display]
['facings'].sum()) / float(100)
results_list.append({
'fk': kpi_fk,
'numerator_id': display,
'denominator_id': category_fk,
'denominator_result': 1,
'numerator_result': count,
'result': count,
'score': count,
'identifier_parent': identifier_parent,
'should_enter': True
})
return results_list
def assortment(self):
"""
This Function get relevant assortment based on filtered scif
"""
lvl3_assort, filter_scif = self.gsk_generator.tool_box.get_assortment_filtered(
self.set_up_data, "planogram")
return lvl3_assort, filter_scif
def msl_assortment(self, kpi_name):
"""
:param kpi_name : name of level 3 assortment kpi
:return kpi_results : data frame of assortment products of the kpi, product's availability,
product details.(reduce assortment products that are not available)
filtered by set up
"""
lvl3_assort, filtered_scif = self.assortment()
if lvl3_assort is None or lvl3_assort.empty:
return None
kpi_assortment_fk = self.common.get_kpi_fk_by_kpi_type(kpi_name)
kpi_results = lvl3_assort[lvl3_assort['kpi_fk_lvl3'] ==
kpi_assortment_fk] # general assortment
kpi_results = pd.merge(kpi_results,
self.all_products[Const.PRODUCTS_COLUMNS],
how='left',
on='product_fk')
# only distributed products
kpi_results = kpi_results[kpi_results['in_store'] == 1]
# filtering substitied products
kpi_results = kpi_results[
kpi_results['substitution_product_fk'].isnull()]
shelf_data = pd.merge(self.match_product_in_scene[[
'scene_fk', 'product_fk', 'shelf_number'
]],
filtered_scif[['scene_id', 'product_fk']],
how='right',
left_on=['scene_fk', 'product_fk'],
right_on=['scene_id', 'product_fk'
]) # why is this happening?
# merge assortment results with match_product_in_scene for shelf_number parameter
kpi_results = pd.merge(shelf_data,
kpi_results,
how='right',
on=['product_fk']) # also problematic
return kpi_results
def shelf_compliance(self, category, assortment_df, cat_targets,
identifier_parent):
"""
This function calculate how many assortment products available on specific shelves
:param category
:param cat_targets : targets df for the specific category
:param assortment_df :relevant assortment based on filtered scif
:param identifier_parent - result identifier for shelf compliance kpi parent .
"""
results_list = []
kpi_fk = self.common.get_kpi_fk_by_kpi_type(Consts.SHELF_COMPLIANCE)
category_targets = self._filter_targets_by_kpi(cat_targets, kpi_fk)
if assortment_df is not None:
assortment_cat = assortment_df[assortment_df['category_fk'] ==
category]
shelf_weight = category_targets['shelf_weight'].iloc[0]
benchmark = category_targets['shelf_benchmark'].iloc[0]
shelves = [
int(shelf) for shelf in
category_targets['shelf_number'].iloc[0].split(",")
]
shelf_df = assortment_cat[assortment_cat['shelf_number'].isin(
shelves)]
numerator = len(shelf_df['product_fk'].unique())
denominator = len(assortment_cat['product_fk'].unique())
result = float(numerator) / float(
denominator) if numerator and denominator != 0 else 0
score = shelf_weight if result >= benchmark else 0
else:
denominator, numerator, score, shelf_weight = 0, 0, 0, 0
result = float(numerator) / float(
denominator) if numerator and denominator != 0 else 0
shelf_weight = category_targets['shelf_weight'].iloc[0]
results_list.append({
'fk': kpi_fk,
'numerator_id': category,
'denominator_id': self.store_id,
'denominator_result': denominator,
'numerator_result': numerator,
'result': result,
'target': shelf_weight,
'score': score,
'identifier_parent': identifier_parent,
'should_enter': True
})
return score, results_list, shelf_weight
def planogram(self, category_fk, assortment, category_targets,
identifier_parent):
"""
This function sum sequence kpi and shelf compliance
:param category_fk
:param category_targets : targets df for the specific category
:param assortment :relevant assortment based on filtered scif
:param identifier_parent : result identifier for planogram kpi parent .
"""
results_list = []
kpi_fk = self.common.get_kpi_fk_by_kpi_type(Consts.PLN_CATEGORY)
identifier_result = self.common.get_dictionary(category_fk=category_fk,
kpi_fk=kpi_fk)
shelf_compliance_score, shelf_compliance_result, shelf_weight = self.shelf_compliance(
category_fk, assortment, category_targets, identifier_result)
results_list.extend(shelf_compliance_result)
sequence_kpi, sequence_weight = self._calculate_sequence(
category_fk, identifier_result)
planogram_score = shelf_compliance_score + sequence_kpi
planogram_weight = shelf_weight + sequence_weight
planogram_result = planogram_score / float(
planogram_weight) if planogram_weight else 0
results_list.append({
'fk': kpi_fk,
'numerator_id': category_fk,
'denominator_id': self.store_id,
'denominator_result': 1,
'numerator_result': planogram_score,
'result': planogram_result,
'target': planogram_weight,
'score': planogram_score,
'identifier_parent': identifier_parent,
'identifier_result': identifier_result,
'should_enter': True
})
return planogram_score, results_list
def _calculate_sequence(self, cat_fk, planogram_identifier):
"""
This method calculated the sequence KPIs using the external targets' data and sequence calculation algorithm.
"""
sequence_kpi_fk, sequence_sku_kpi_fk = self._get_sequence_kpi_fks()
sequence_targets = self._filter_targets_by_kpi(self.targets,
sequence_kpi_fk)
sequence_targets = sequence_targets.loc[sequence_targets.category_fk ==
cat_fk]
passed_sequences_score, total_sequence_weight = 0, 0
for i, sequence in sequence_targets.iterrows():
population, location, sequence_attributes = self._prepare_data_for_sequence_calculation(
sequence)
sequence_result = self.sequence.calculate_sequence(
population, location, sequence_attributes)
score = self._save_sequence_results_to_db(sequence_sku_kpi_fk,
sequence_kpi_fk,
sequence,
sequence_result)
passed_sequences_score += score
total_sequence_weight += sequence[SessionResultsConsts.WEIGHT]
self._save_sequence_main_level_to_db(sequence_kpi_fk,
planogram_identifier, cat_fk,
passed_sequences_score,
total_sequence_weight)
return passed_sequences_score, total_sequence_weight
@staticmethod
def _prepare_data_for_sequence_calculation(sequence_params):
"""
This method gets the relevant targets per sequence and returns the sequence params for calculation.
"""
population = {
ProductsConsts.PRODUCT_FK:
sequence_params[ProductsConsts.PRODUCT_FK]
}
location = {
TemplatesConsts.TEMPLATE_GROUP:
sequence_params[TemplatesConsts.TEMPLATE_GROUP]
}
additional_attributes = {
AdditionalAttr.STRICT_MODE: sequence_params['strict_mode'],
AdditionalAttr.INCLUDE_STACKING:
sequence_params['include_stacking'],
AdditionalAttr.CHECK_ALL_SEQUENCES: True
}
return population, location, additional_attributes
def _extract_target_params(self, sequence_params):
"""
This method extract the relevant category_fk and result value from the sequence parameters.
"""
numerator_id = sequence_params[ProductsConsts.CATEGORY_FK]
result_value = self.ps_data_provider.get_pks_of_result(
sequence_params['sequence_name'])
return numerator_id, result_value
def _save_sequence_main_level_to_db(self, kpi_fk, planogram_identifier,
cat_fk, sequence_score, total_weight):
"""
This method saves the top sequence level to DB.
"""
result = round(
(sequence_score / float(total_weight)), 2) if total_weight else 0
score = result * total_weight
self.common.write_to_db_result(fk=kpi_fk,
numerator_id=cat_fk,
numerator_result=sequence_score,
result=result,
denominator_id=self.store_id,
denominator_result=total_weight,
score=score,
weight=total_weight,
target=total_weight,
should_enter=True,
identifier_result=kpi_fk,
identifier_parent=planogram_identifier)
def _save_sequence_results_to_db(self, kpi_fk, parent_kpi_fk,
sequence_params, sequence_results):
"""
This method handles the saving of the SKU level sequence KPI.
:param kpi_fk: Sequence SKU kpi fk.
:param parent_kpi_fk: Total sequence score kpi fk.
:param sequence_params: A dictionary with sequence params for the external targets.
:param sequence_results: A DataFrame with the results that were received by the sequence algorithm.
:return: The score that was saved (0 or 100 * weight).
"""
category_fk, result_value = self._extract_target_params(
sequence_params)
num_of_sequences = len(sequence_results)
target, weight = sequence_params[
SessionResultsConsts.TARGET], sequence_params[
SessionResultsConsts.WEIGHT]
score = weight if len(sequence_results) >= target else 0
self.common.write_to_db_result(fk=kpi_fk,
numerator_id=category_fk,
numerator_result=num_of_sequences,
result=result_value,
denominator_id=self.store_id,
denominator_result=None,
score=score,
weight=weight,
parent_fk=parent_kpi_fk,
target=target,
should_enter=True,
identifier_parent=parent_kpi_fk,
identifier_result=(kpi_fk, category_fk))
return score
def _get_sequence_kpi_fks(self):
"""This method fetches the relevant sequence kpi fks"""
sequence_kpi_fk = self.common.get_kpi_fk_by_kpi_type(
Consts.SEQUENCE_KPI)
sequence_sku_kpi_fk = self.common.get_kpi_fk_by_kpi_type(
Consts.SEQUENCE_SKU_KPI)
return sequence_kpi_fk, sequence_sku_kpi_fk
def secondary_display(self, category_fk, cat_targets, identifier_parent,
scif_df):
"""
This function calculate secondary score - 0 or full weight if at least
one of it's child kpis equal to weight.
:param category_fk
:param cat_targets : targets df for the specific category
:param identifier_parent : result identifier for promo activation kpi parent .
:param scif_df : scif filtered by promo activation settings
"""
results_list = []
parent_kpi_name = 'display'
total_kpi_fk = self.common.get_kpi_fk_by_kpi_type(
Consts.DISPLAY_SUMMARY)
category_targets = self._filter_targets_by_kpi(cat_targets,
total_kpi_fk)
weight = category_targets['display_weight'].iloc[0]
result_identifier = self.common.get_dictionary(category_fk=category_fk,
kpi_fk=total_kpi_fk)
dispenser_score, dispenser_res = self.display_distribution(
Consts.DISPENSER_TARGET, category_fk, category_targets,
result_identifier, Consts.DISPENSERS, parent_kpi_name, scif_df)
counter_top_score, counter_top_res = self.display_distribution(
Consts.COUNTER_TOP_TARGET, category_fk, category_targets,
result_identifier, Consts.COUNTERTOP, parent_kpi_name, scif_df)
standee_score, standee_res = self.display_distribution(
Consts.STANDEE_TARGET, category_fk, category_targets,
result_identifier, Consts.STANDEE, parent_kpi_name, scif_df)
results_list.extend(dispenser_res)
results_list.extend(counter_top_res)
results_list.extend(standee_res)
display_score = weight if (dispenser_score == weight) or (
counter_top_score == weight) or (standee_score == weight) else 0
results_list.append({
'fk': total_kpi_fk,
'numerator_id': category_fk,
'denominator_id': self.store_id,
'denominator_result': 1,
'numerator_result': display_score,
'result': display_score,
'target': weight,
'score': display_score,
'identifier_parent': identifier_parent,
'identifier_result': result_identifier,
'should_enter': True
})
return results_list, display_score
def promo_activation(self, category_fk, cat_targets, identifier_parent,
scif_df):
"""
This function calculate promo activation score - 0 or full weight if at least
one of it's child kpis equal to weight.
:param category_fk
:param cat_targets : targets df for the specific category
:param identifier_parent : result identifier for promo activation kpi parent .
:param scif_df : scif filtered by promo activation settings
"""
total_kpi_fk = self.common.get_kpi_fk_by_kpi_type(Consts.PROMO_SUMMARY)
category_targets = self._filter_targets_by_kpi(cat_targets,
total_kpi_fk)
result_identifier = self.common.get_dictionary(category_fk=category_fk,
kpi_fk=total_kpi_fk)
results_list = []
parent_kpi_name = 'promo'
weight = category_targets['promo_weight'].iloc[0]
hang_shell_score, hang_shell_res = self.display_distribution(
Consts.HANGSELL, category_fk, category_targets, result_identifier,
Consts.HANGSELL_KPI, parent_kpi_name, scif_df)
top_shelf_score, top_shelf_res = self.display_distribution(
Consts.TOP_SHELF, category_fk, category_targets, result_identifier,
Consts.TOP_SHELF_KPI, parent_kpi_name, scif_df)
results_list.extend(hang_shell_res)
results_list.extend(top_shelf_res)
promo_score = weight if (hang_shell_score == weight) or (
top_shelf_score == weight) else 0
results_list.append({
'fk': total_kpi_fk,
'numerator_id': category_fk,
'denominator_id': self.store_id,
'denominator_result': 1,
'numerator_result': promo_score,
'result': promo_score,
'target': weight,
'score': promo_score,
'identifier_parent': identifier_parent,
'identifier_result': result_identifier,
'should_enter': True
})
return results_list, promo_score
@staticmethod
def _filter_targets_by_kpi(targets, kpi_fk):
""" This function filter all targets but targets which related to relevant kpi"""
filtered_targets = targets.loc[targets.kpi_fk == kpi_fk]
return filtered_targets
def orange_score_category(self, assortment_category_res,
fsos_category_res):
"""
This function calculate orange score kpi by category. Settings are based on external targets and set up file.
:param assortment_category_res : array of assortment results
:param fsos_category_res : array of facing sos by store results
"""
results_list = []
self.store_target()
if self.targets.empty:
return
total_kpi_fk = self.common.get_kpi_fk_by_kpi_type(
Consts.ORANGE_SCORE_COMPLIANCE)
fsos_json_global_results = self.extract_json_results_by_kpi(
fsos_category_res, Consts.GLOBAL_FSOS_BY_CATEGORY)
msl_json_global_results = self.extract_json_results_by_kpi(
assortment_category_res, Consts.GLOBAL_DST_BY_CATEGORY)
# scif after filtering it by set up file for each kpi
scif_secondary = self.gsk_generator.tool_box.tests_by_template(
'secondary_display', self.scif, self.set_up_data)
scif_promo = self.gsk_generator.tool_box.tests_by_template(
'promo', self.scif, self.set_up_data)
categories = self.targets[
DataProviderConsts.ProductsConsts.CATEGORY_FK].unique()
assortment = self.msl_assortment('Distribution - SKU')
for cat in categories:
orange_score_result_identifier = self.common.get_dictionary(
category_fk=cat, kpi_fk=total_kpi_fk)
cat_targets = self.targets[self.targets[
DataProviderConsts.ProductsConsts.CATEGORY_FK] == cat]
msl_score, msl_results = self.msl_compliance_score(
cat, msl_json_global_results, cat_targets,
orange_score_result_identifier)
fsos_score, fsos_results = self.fsos_compliance_score(
cat, fsos_json_global_results, cat_targets,
orange_score_result_identifier)
planogram_score, planogram_results = self.planogram(
cat, assortment, cat_targets, orange_score_result_identifier)
secondary_display_res, secondary_score = self.secondary_display(
cat, cat_targets, orange_score_result_identifier,
scif_secondary)
promo_activation_res, promo_score = self.promo_activation(
cat, cat_targets, orange_score_result_identifier, scif_promo)
compliance_category_score = promo_score + secondary_score + fsos_score + msl_score + planogram_score
results_list.extend(msl_results + fsos_results +
planogram_results + secondary_display_res +
promo_activation_res)
results_list.append({
'fk':
total_kpi_fk,
'numerator_id':
self.manufacturer_fk,
'denominator_id':
cat,
'denominator_result':
1,
'numerator_result':
compliance_category_score,
'result':
compliance_category_score,
'score':
compliance_category_score,
'identifier_result':
orange_score_result_identifier
})
return results_list
class CaseCountCalculator(GlobalSessionToolBox):
"""This class calculates the Case Count SKU KPI set.
It uses display tags and sub-products in order to calculate results per target and sum
all of them in order to calculate the main Case Count"""
def __init__(self, data_provider, common):
GlobalSessionToolBox.__init__(self, data_provider, None)
self.filtered_mdis = self._get_filtered_match_display_in_scene()
self.store_number_1 = self.store_info.store_number_1[0]
self.filtered_scif = self._get_filtered_scif()
self.ps_data_provider = PsDataProvider(data_provider)
self.target = self._get_case_count_targets()
self.matches = self.get_filtered_matches()
self.excluded_product_fks = self._get_excluded_product_fks()
self.adj_graphs_per_scene = {}
self.common = common
def _get_case_count_targets(self):
"""
This method fetches the relevant targets for the case count
"""
case_count_kpi_fk = self.get_kpi_fk_by_kpi_type(
Consts.TOTAL_CASES_STORE_KPI)
targets = self.ps_data_provider.get_kpi_external_targets(
kpi_fks=[case_count_kpi_fk],
data_fields=[Src.TARGET],
key_fields=[Sc.PRODUCT_FK, 'store_number_1'],
key_filters={'store_number_1': self.store_number_1})
targets = targets.loc[targets.store_number_1 == self.store_number_1][[
Pc.PRODUCT_FK, Src.TARGET
]]
return dict(zip(targets[Pc.PRODUCT_FK], targets[Src.TARGET]))
def _get_filtered_match_display_in_scene(self):
""" This method filters match display in scene - it saves only "close" and "open" tags"""
mdis = self.data_provider.match_display_in_scene.loc[
self.data_provider.match_display_in_scene.display_name.str.
contains(Consts.RELEVANT_DISPLAYS_SUFFIX)]
return mdis
def main_case_count_calculations(self):
"""This method calculates the entire Case Count KPIs set."""
if not (self.filtered_scif.empty or self.matches.empty):
try:
if not self.filtered_mdis.empty:
self._prepare_data_for_calculation()
self._generate_adj_graphs()
facings_res = self._calculate_display_size_facings()
sku_cases_res = self._count_number_of_cases()
unshoppable_cases_res = self._non_shoppable_case_kpi()
implied_cases_res = self._implied_shoppable_cases_kpi()
else:
facings_res = self._calculate_display_size_facings()
sku_cases_res = self._count_number_of_cases()
unshoppable_cases_res = []
implied_cases_res = []
sku_cases_res = self._remove_nonshoppable_cases_from_shoppable_cases(
sku_cases_res, unshoppable_cases_res)
total_res = self._calculate_total_cases(sku_cases_res +
implied_cases_res +
unshoppable_cases_res)
placeholder_res = self._generate_placeholder_results(
facings_res, sku_cases_res, unshoppable_cases_res,
implied_cases_res)
self._save_results_to_db(facings_res + sku_cases_res +
unshoppable_cases_res +
implied_cases_res + total_res +
placeholder_res)
self._calculate_total_score_level_res(total_res)
except Exception as err:
Log.error(
"DiageoUS Case Count calculation failed due to the following error: {}"
.format(err))
def _calculate_total_score_level_res(self, total_res_sku_level_results):
"""This method gets the Total Cases SKU level results and aggregates them in order to create the
mobile store level result"""
result, kpi_fk = 0, self.get_kpi_fk_by_kpi_type(
Consts.TOTAL_CASES_STORE_KPI)
for res in total_res_sku_level_results:
if res.get(Pc.PRODUCT_FK, 0) in self.target.keys():
result += res.get(Src.RESULT, 0)
self.common.write_to_db_result(fk=kpi_fk,
numerator_id=int(self.manufacturer_fk),
result=result,
denominator_id=self.store_id,
identifier_result=kpi_fk)
def _calculate_total_cases(self, kpi_results):
""" This method sums # of cases per brand and Implied Shoppable Cases KPIs
and saves the main result to the DB"""
kpi_fk = self.get_kpi_fk_by_kpi_type(Consts.TOTAL_CASES_SKU_KPI)
total_results_per_sku, results_list = Counter(), list()
for res in kpi_results:
total_results_per_sku[res[Pc.PRODUCT_FK]] += res[Src.RESULT]
# save products with targets
for product_fk, target in self.target.iteritems():
result = total_results_per_sku.pop(product_fk, 0)
results_list.append({
Pc.PRODUCT_FK: product_fk,
Src.RESULT: result,
'fk': kpi_fk,
Src.TARGET: target
})
# save products with no targets
for product_fk in list(
set(total_results_per_sku.keys()) -
set(self.excluded_product_fks)):
result = total_results_per_sku.get(product_fk, 0)
results_list.append({
Pc.PRODUCT_FK: product_fk,
Src.RESULT: result,
'fk': kpi_fk,
Src.TARGET: 0
})
return results_list
def _generate_placeholder_results(self, facings_res, sku_cases_res,
unshoppable_cases_res,
implied_cases_res):
"""This method creates KPI results with zeroes to give the illusion that all KPIs were calculated"""
placeholder_res = []
total_res = facings_res + sku_cases_res + unshoppable_cases_res + implied_cases_res
total_product_fks = list(set([res[Sc.PRODUCT_FK]
for res in total_res]))
for kpi_fk, res_list in [
(self.get_kpi_fk_by_kpi_type(Consts.FACINGS_KPI), facings_res),
(self.get_kpi_fk_by_kpi_type(Consts.SHOPPABLE_CASES_KPI),
sku_cases_res),
(self.get_kpi_fk_by_kpi_type(Consts.NON_SHOPPABLE_CASES_KPI),
unshoppable_cases_res),
(self.get_kpi_fk_by_kpi_type(Consts.IMPLIED_SHOPPABLE_CASES_KPI),
implied_cases_res)
]:
res_product_fks = [res[Sc.PRODUCT_FK] for res in res_list]
for missing_product_fk in [
fk for fk in total_product_fks if fk not in res_product_fks
]:
placeholder_res.append({
Pc.PRODUCT_FK: missing_product_fk,
Src.RESULT: 0,
'fk': kpi_fk
})
return placeholder_res
@staticmethod
def _remove_nonshoppable_cases_from_shoppable_cases(
sku_cases_res, unshoppable_cases_res):
unshopable_cases_dict = \
{res[Pc.PRODUCT_FK]: res[Src.RESULT] for res in unshoppable_cases_res if res[Src.RESULT] > 0}
new_sku_cases_res = []
for res in sku_cases_res:
if res[Pc.PRODUCT_FK] in unshopable_cases_dict.keys():
res[Src.RESULT] = res[Src.RESULT] - unshopable_cases_dict[res[
Pc.PRODUCT_FK]]
new_sku_cases_res.append(res)
return new_sku_cases_res
def _save_results_to_db(self, results_list):
"""This method saves the KPI results to DB"""
total_cases_sku_fk = int(
self.get_kpi_fk_by_kpi_type(Consts.TOTAL_CASES_SKU_KPI))
total_cases_store_fk = int(
self.get_kpi_fk_by_kpi_type(Consts.TOTAL_CASES_STORE_KPI))
for res in results_list:
kpi_fk = int(res.get('fk'))
parent_id = '{}_{}'.format(
int(res[Pc.PRODUCT_FK]), total_cases_sku_fk
) if kpi_fk != total_cases_sku_fk else total_cases_store_fk
kpi_id = '{}_{}'.format(int(res[Pc.PRODUCT_FK]), kpi_fk)
result, target = res.get(Src.RESULT), res.get(Src.TARGET)
score = 1 if target is not None and result >= target else 0
self.common.write_to_db_result(fk=kpi_fk,
denominator_id=res[Pc.PRODUCT_FK],
result=result,
score=score,
target=target,
identifier_result=kpi_id,
identifier_parent=parent_id,
should_enter=True)
def _prepare_data_for_calculation(self):
"""This method prepares the data for the case count calculation. Connection between the display
data and the tagging data."""
closest_tag_to_display_df = self._calculate_closest_product_to_display(
)
closest_tag_to_display_df = self._remove_items_outside_maximum_distance(
closest_tag_to_display_df)
self._add_displays_the_closet_product_fk(closest_tag_to_display_df)
self._add_matches_display_data(closest_tag_to_display_df)
self._remove_extra_tags_from_case_tags()
def _remove_extra_tags_from_case_tags(self):
"""This method ensures that if a display is associated with a 'case' tag that no other SKU tags can be
paired to the same display"""
case_product_fks = \
self.scif[self.scif[Sc.SKU_TYPE].isin(['case', 'Case', 'CASE'])][Sc.PRODUCT_FK].unique().tolist()
display_fks_with_cases = self.matches[
(self.matches[Consts.DISPLAY_IN_SCENE_FK].notna())
& (self.matches[Sc.PRODUCT_FK].isin(case_product_fks))][
Consts.DISPLAY_IN_SCENE_FK].unique().tolist()
self.matches.loc[(
(self.matches[Consts.DISPLAY_IN_SCENE_FK].
isin(display_fks_with_cases)) &
(~self.matches[Sc.PRODUCT_FK].isin(case_product_fks))),
Consts.DISPLAY_IN_SCENE_FK] = pd.np.nan
return
@staticmethod
def _remove_items_outside_maximum_distance(closest_tag_to_display_df):
"""This method removes SKU tags that are too far away from the display tag and limits the number
of possible tags"""
max_number_of_tags = 4
max_distance_from_display_tag = 20000
closest_tag_to_display_df.sort_values(
by=['display_in_scene_fk', 'minimum_distance'], inplace=True)
closest_tag_to_display_df = \
closest_tag_to_display_df[closest_tag_to_display_df['minimum_distance'] < max_distance_from_display_tag]
# we need to limit each display to only being associated with up to 4 tags
closest_tag_to_display_df = closest_tag_to_display_df.groupby(
'display_in_scene_fk').head(max_number_of_tags)
return closest_tag_to_display_df
def _calculate_closest_product_to_display(self):
"""This method calculates the closest tag for each display and returns a DataFrame with the results"""
matches = self.matches[self.matches[Mc.SCENE_FK].isin(
self._get_scenes_with_relevant_displays())]
matches = matches[Consts.RLV_FIELDS_FOR_MATCHES_CLOSET_DISPLAY_CALC]
mdis = self.filtered_mdis[
Consts.RLV_FIELDS_FOR_DISPLAY_IN_SCENE_CLOSET_TAG_CALC]
closest_display_data = matches.apply(
lambda row: self._apply_closet_point_logic_on_row(row, mdis, 'pk'),
axis=1)
closest_display_data = pd.DataFrame(
closest_display_data.values.tolist())
return closest_display_data
def _calculate_display_size_facings(self):
"""This method calculates the number of facings for SKUs with the relevant SKU Type
only in scenes that have displays"""
filtered_scif = self.filtered_scif.loc[(
self.filtered_scif[Sc.SKU_TYPE].isin(Consts.FACINGS_SKU_TYPES))]
filtered_scif = filtered_scif.loc[filtered_scif[Sc.TAGGED] > 0][[
Pc.SUBSTITUTION_PRODUCT_FK, Sc.TAGGED
]]
results_df = filtered_scif.groupby(
Pc.SUBSTITUTION_PRODUCT_FK, as_index=False).sum().rename(
{
Sc.TAGGED: Src.RESULT,
Sc.SUBSTITUTION_PRODUCT_FK: Sc.PRODUCT_FK
},
axis=1)
results_df = results_df.merge(pd.DataFrame(
{Pc.PRODUCT_FK: self.target.keys()}),
how='outer',
on=Pc.PRODUCT_FK)
results_df = results_df.assign(
fk=self.get_kpi_fk_by_kpi_type(Consts.FACINGS_KPI))
results_df = results_df.fillna(0)
return results_df.to_dict('records')
def _get_results_for_branded_other_cases(self):
"""This method identifies branded other cases and attempts to distribute them across the products in the
open case most closely above the branded other case"""
results = []
for scene_fk in self._get_scenes_with_relevant_displays():
adj_g = self.adj_graphs_per_scene[scene_fk]
branded_other_cases = self.matches[
(self.matches['product_type'] == 'Other')
& (self.matches[Consts.MPIPS_FK] == Consts.PACK_FK) &
(self.matches['scene_fk'] == scene_fk)]
branded_other_match_fks = branded_other_cases.scene_match_fk.tolist(
)
for node, node_data in adj_g.nodes(data=True):
node_scene_match_fks = list(node_data['match_fk'].values)
if any(match in node_scene_match_fks
for match in branded_other_match_fks):
results.append(self._find_open_case_above(node, adj_g))
return results
def _find_open_case_above(self, node, adj_g):
node_data = adj_g.nodes[node]
root_brand_name = list(node_data['brand_name'].values)
paths = self._get_relevant_path_for_calculation(adj_g)
paths = [path for path in paths if node in path]
activated = False
case_data = None
result = list(node_data['substitution_product_fk'].values)
for case in paths[0]:
if case == node:
activated = True
if not activated:
continue # we haven't reached the brand-other case in the path yet
if not self._get_case_status(adj_g.nodes[case]):
continue # the case is closed, and we haven't reached the first open case yet
else:
case_data = adj_g.nodes[case]
break
if case_data:
case_brand_names = list(case_data['brand_name'].values)
if any(brand in root_brand_name for brand in case_brand_names):
result = list(case_data['substitution_product_fk'].values)
return result
class TWEGAUSceneToolBox:
def __init__(self, data_provider, output, common):
self.output = output
self.data_provider = data_provider
self.common = common
self.templates_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), '..', 'Data')
self.excel_file_path = os.path.join(self.templates_path,
'Template.xlsx')
self.project_name = self.data_provider.project_name
self.session_uid = self.data_provider.session_uid
self.products = self.data_provider[Data.PRODUCTS]
self.templates = self.data_provider[Data.TEMPLATES]
self.all_products = self.data_provider[Data.ALL_PRODUCTS]
self.scif = self.data_provider.scene_item_facts
self.match_product_in_scene = self.data_provider[Data.MATCHES]
self.visit_date = self.data_provider[Data.VISIT_DATE]
self.session_info = self.data_provider[Data.SESSION_INFO]
self.scene_info = self.data_provider[Data.SCENES_INFO]
self.store_info = self.data_provider[Data.STORE_INFO]
self.store_id = self.store_info.iloc[0].store_fk
self.store_type = self.data_provider.store_type
self.rds_conn = PSProjectConnector(self.project_name,
DbUsers.CalculationEng)
self.kpi_static_data = self.common.get_kpi_static_data()
self.ps_data_provider = PsDataProvider(self.data_provider, self.output)
self.targets = self.ps_data_provider.get_kpi_external_targets()
self.empty_product_ids = self.all_products.query(
'product_name.str.contains("empty", case=False) or'
' product_name.str.contains("irrelevant", case=False)',
engine='python')['product_fk'].values
def _get_zone_based_data(self, kpi, kpi_sheet_row, denominator_row):
# generate scene max shelf max bay map
zone_number = kpi_sheet_row[ZONE_NAME]
shelves_policy_from_top = [
int(x.strip())
for x in str(kpi_sheet_row[SHELF_POLICY_FROM_TOP]).split(',')
if x.strip()
]
permitted_shelves = [
int(x.strip())
for x in str(kpi_sheet_row[NUMBER_OF_SHELVES]).split(',')
if x.strip()
]
Log.info(
"Calulating for zone {z} with shelf policy {sh} and permitted shelves {psh}"
.format(z=zone_number,
sh=shelves_policy_from_top,
psh=permitted_shelves))
unique_manufacturer_products_count = 0
# DENOMINATOR
if not denominator_row.empty:
denominator_filters, denominator_filter_string = get_filter_string_per_row(
denominator_row,
DENOMINATOR_FILTER_ENTITIES,
)
unique_manufacturer_products_count = len(
self.scif.query(denominator_filter_string).product_fk.unique())
if not is_nan(kpi_sheet_row[STORE_TYPE]):
if bool(kpi_sheet_row[STORE_TYPE].strip()
) and kpi_sheet_row[STORE_TYPE].strip().lower() != 'all':
Log.info("Check the store types in excel...")
permitted_store_types = [
x.strip() for x in kpi_sheet_row[STORE_TYPE].split(',')
if x.strip()
]
if self.store_info.store_type.values[
0] not in permitted_store_types:
Log.info(
"Store type = {st} not permitted for session {ses}...".
format(st=self.store_info.store_type.values[0],
ses=self.session_uid))
return []
filters, filter_string = get_filter_string_per_row(
kpi_sheet_row,
ZONE_NUMERATOR_FILTER_ENTITIES,
additional_filters=ZONE_ADDITIONAL_FILTERS_PER_COL,
)
Log.info("Store filters = {ft} and filter_string = {fts}...".format(
ft=filters, fts=filter_string))
# combined tables
match_product_df = pd.merge(self.match_product_in_scene,
self.products,
how='left',
left_on=['product_fk'],
right_on=['product_fk'])
scene_template_df = pd.merge(self.scene_info,
self.templates,
how='left',
left_on=['template_fk'],
right_on=['template_fk'])
product_scene_join_data = pd.merge(match_product_df,
scene_template_df,
how='left',
left_on=['scene_fk'],
right_on=['scene_fk'])
if filters:
filtered_grouped_scene_items = product_scene_join_data.query(filter_string) \
.groupby(filters, as_index=False)
else:
# dummy structure without filters
filtered_grouped_scene_items = [
('', product_scene_join_data.query(filter_string))
]
# get the scene_id's worth getting data from
scene_data_map = defaultdict(list)
for each_group_by_manf_templ in filtered_grouped_scene_items:
# append scene to group by
scene_type_grouped_by_scene = each_group_by_manf_templ[1].groupby(
SCENE_FK)
for scene_id, scene_data in scene_type_grouped_by_scene:
exclude_items = False
valid_bay_numbers = self.get_valid_bay_numbers(
scene_id, permitted_shelves)
if not valid_bay_numbers:
continue
scene_per_bay_number = scene_data.query(
'shelf_number in {shelves} and bay_number in {bays}'.
format(shelves=shelves_policy_from_top,
bays=valid_bay_numbers)).groupby(['bay_number'])
items_to_check_str = None
if not is_nan(kpi_sheet_row.exclude_include_policy):
match_exclude = exclude_re.search(
kpi_sheet_row.exclude_include_policy)
if not match_exclude:
match_only = only_re.search(
kpi_sheet_row.exclude_include_policy)
items_to_check_str = match_only.groups()[-1]
exclude_items = False
else:
items_to_check_str = match_exclude.groups()[-1]
exclude_items = True
# the deciding loop
# bay iterator
for bay_number, scene_data_per_bay in scene_per_bay_number:
total_products = [] # contain the total products per bay
if scene_data_per_bay.empty:
return {}
bay_number = bay_number
scene_data_per_bay_shelf = scene_data_per_bay.groupby(
'shelf_number')
denominator_entity_id = EXCEL_ENTITY_MAP[
kpi_sheet_row.denominator_fk]
denominator_data = getattr(
scene_data_per_bay,
EXCEL_DB_MAP[kpi_sheet_row.denominator_fk],
pd.Series())
if denominator_data.empty:
# find in self
denominator_id = getattr(
self, EXCEL_DB_MAP[kpi_sheet_row.denominator_fk],
None)
else:
denominator_id = denominator_data.unique()[0]
# shelf iterator
for shelf_id, shelf_data in scene_data_per_bay_shelf:
if items_to_check_str:
# exclude/include logic
last_shelf_number = str(
shelf_data.n_shelf_items.unique()[0])
shelf_filter = items_to_check_str.replace(
'N', last_shelf_number)
shelf_filter_string = '[{}]'.format(shelf_filter)
if exclude_items:
# exclude rows in `items_to_check_tuple`
required_shelf_items = shelf_data.drop(
shelf_data.query(
'facing_sequence_number in {filter_string}'
.format(
filter_string=shelf_filter_string
)).index.tolist())
else:
# it is include_only:
required_shelf_items = shelf_data.query(
'facing_sequence_number in {filter_string}'
.format(filter_string=shelf_filter_string))
product_ids = required_shelf_items.product_fk.tolist(
)
else:
product_ids = shelf_data.product_fk.tolist()
total_products.extend(product_ids)
# prod_count_map is per bay and shelf
scene_data_map[scene_id].append({
'fk':
int(kpi['pk']),
'store':
denominator_id,
'product_count_map':
Counter(total_products),
'bay_number':
bay_number,
'kpi_name':
kpi_sheet_row[KPI_NAME],
'zone_number':
zone_number,
'unique_manufacturer_products_count':
unique_manufacturer_products_count,
})
return scene_data_map
def get_template_details(self, sheet_name):
template = pd.read_excel(self.excel_file_path, sheetname=sheet_name)
return template
def calculate_zone_kpis(self):
Log.info("Scene based calculations for zone KPIs...")
zone_kpi_sheet = self.get_template_details(ZONE_KPI_SHEET)
name_grouped_zone_kpi_sheet = zone_kpi_sheet.groupby(KPI_TYPE)
for each_kpi in name_grouped_zone_kpi_sheet:
# ugly hack! -- this came as a requirement after the kpi was written
each_kpi_type = "SCENE_" + each_kpi[
0] # manipulate kpi name to fit into its scene based
kpi_sheet_rows = each_kpi[1]
denominator_row = pd.Series()
kpi = self.kpi_static_data[
(self.kpi_static_data[KPI_FAMILY] == PS_KPI_FAMILY)
& (self.kpi_static_data[TYPE] == each_kpi_type)
& (self.kpi_static_data['delete_time'].isnull())]
if kpi.empty:
Log.info("KPI Name:{} not found in DB".format(each_kpi_type))
else:
Log.info("KPI Name:{} found in DB".format(each_kpi_type))
if 'sku_all' not in each_kpi_type.lower():
# Skipping zone KPI's for Mobile Reports.
continue
list_of_zone_data = []
for idx, each_kpi_sheet_row in kpi_sheet_rows.iterrows():
zone_data = self._get_zone_based_data(
kpi,
each_kpi_sheet_row.T,
denominator_row=denominator_row)
if zone_data:
# empty when the row in sheet could not find any relevant data for zone
list_of_zone_data.append(zone_data)
# write for products
for each_scene_zone_map in list_of_zone_data:
for scene_id, bay_zone_list in each_scene_zone_map.iteritems(
):
for zone_data in bay_zone_list:
product_counter = zone_data['product_count_map']
for prod_id, count in product_counter.iteritems():
Log.info(
"Product_id is {pr} and count is {cn} on bay {bay}"
.format(pr=prod_id,
cn=count,
bay=zone_data['bay_number']))
if int(prod_id) not in self.empty_product_ids:
in_assort_sc_values = self.scif.query(
"item_id=={prod_id}".format(
prod_id=prod_id)).in_assort_sc
in_assort_sc = 0
if not in_assort_sc_values.empty:
if not is_nan(
in_assort_sc_values.values[0]):
in_assort_sc = int(
in_assort_sc_values.values[0])
self.common.write_to_db_result(
fk=int(zone_data['fk']),
numerator_id=int(
prod_id), # product ID
numerator_result=int(
zone_data['bay_number']),
# bay number comes as numerator
denominator_id=int(
zone_data['store']), # store ID
denominator_result=int(
scene_id
), # scene id comes as denominator
result=int(count), # save the count
score=in_assort_sc,
context_id=int(
zone_data['zone_number']),
by_scene=True)
def get_shelf_limit_for_scene(self, scene_id):
shelf_limit_per_scene_map = defaultdict(list)
scene_data = self.match_product_in_scene.loc[
self.match_product_in_scene['scene_fk'] == scene_id]
_bay_grouped_scene_data = scene_data.groupby('bay_number',
as_index=False)
for each_bay in _bay_grouped_scene_data:
bay_number = each_bay[0]
scene_data = each_bay[1]
if not scene_data.empty:
shelf_limit_per_scene_map[scene_id].append((bay_number, {
'max_shelf':
scene_data[SHELF_NUMBER].max(),
'min_shelf':
scene_data[SHELF_NUMBER].min()
}))
return shelf_limit_per_scene_map
def get_valid_bay_numbers(self, scene_id, permitted_shelves):
scene_max_min_map = self.get_shelf_limit_for_scene(scene_id)
bay_numbers = []
for scene_id, bay_shelf_map in scene_max_min_map.iteritems():
for each_map in bay_shelf_map:
_bay_number = each_map[0]
scene_max_min_map = each_map[1]
for each_permitted_shelf in permitted_shelves:
if scene_max_min_map['max_shelf'] == each_permitted_shelf:
bay_numbers.append(_bay_number)
return bay_numbers
