def __init__(self, data_provider, output, rds_conn=None, ignore_stacking=False, front_facing=False, **kwargs): self.k_engine = BaseCalculationsGroup(data_provider, output) self.rds_conn = rds_conn self.data_provider = data_provider self.project_name = self.data_provider.project_name self.session_uid = self.data_provider.session_uid self.scif = self.data_provider[Data.SCENE_ITEM_FACTS] self.match_product_in_scene = self.data_provider[Data.MATCHES] self.position_graphs = PositionGraphs(self.data_provider, rds_conn=self.rds_conn) self.all_products = self.data_provider[Data.ALL_PRODUCTS] self.survey_response = self.data_provider[Data.SURVEY_RESPONSES] self.scenes_info = self.data_provider[Data.SCENES_INFO].merge( self.data_provider[Data.ALL_TEMPLATES], how='left', on='template_fk', suffixes=['', '_y']) self.survey_response = self.data_provider[Data.SURVEY_RESPONSES] self.get_atts() self.ignore_stacking = ignore_stacking self.facings_field = 'facings' if not self.ignore_stacking else 'facings_ign_stack' self.front_facing = front_facing for data in kwargs.keys(): setattr(self, data, kwargs[data]) if self.front_facing: self.scif = self.scif[self.scif['front_face_count'] == 1] self._merge_matches_and_all_product()
def __init__(self, data_provider, output=None, ps_data_provider=None, common=None, rds_conn=None, front_facing=False, custom_scif=None, custom_matches=None, **kwargs): super(Block, self).__init__(data_provider, output, ps_data_provider, common, rds_conn, **kwargs) self._position_graphs = PositionGraphs(self.data_provider) self.front_facing = front_facing self.outliers_threshold = Default.outliers_threshold self.check_vertical_horizontal = Default.check_vertical_horizontal self.include_stacking = Default.include_stacking self.ignore_empty = Default.ignore_empty self.allowed_edge_type = Default.allowed_edge_type self.scif = data_provider.scene_item_facts if custom_scif is None else custom_scif self.matches = data_provider.matches if custom_matches is None else custom_matches self.adj_graphs_by_scene = {} self.masking_data = transform_maskings( retrieve_maskings( self.data_provider.project_name, self.data_provider.scenes_info['scene_fk'].to_list())) self.masking_data = self.masking_data.merge( self.matches[['probe_match_fk', 'scene_fk']], on=['probe_match_fk']) self.matches_df = self.matches.merge( self.data_provider.all_products_including_deleted, on='product_fk') self.matches_df = self.matches_df[ ~(self.matches_df['product_type'].isin(['POS'])) & (self.matches_df['stacking_layer'] > 0)] self.matches_df[Block.BLOCK_KEY] = None
class GOLD_PEAK_BLOCKGeneralToolBox: EXCLUDE_FILTER = 0 INCLUDE_FILTER = 1 CONTAIN_FILTER = 2 EXCLUDE_EMPTY = False INCLUDE_EMPTY = True STRICT_MODE = ALL = 1000 EMPTY = 'Empty' DEFAULT = 'Default' TOP = 'Top' BOTTOM = 'Bottom' def __init__(self, data_provider, output, rds_conn=None, ignore_stacking=False, front_facing=False, **kwargs): self.k_engine = BaseCalculationsGroup(data_provider, output) self.rds_conn = rds_conn self.data_provider = data_provider self.project_name = self.data_provider.project_name self.session_uid = self.data_provider.session_uid self.scif = self.data_provider[Data.SCENE_ITEM_FACTS] self.match_product_in_scene = self.data_provider[Data.MATCHES] self.position_graphs = PositionGraphs(self.data_provider, rds_conn=self.rds_conn) self.all_products = self.data_provider[Data.ALL_PRODUCTS] self.survey_response = self.data_provider[Data.SURVEY_RESPONSES] self.scenes_info = self.data_provider[Data.SCENES_INFO].merge( self.data_provider[Data.ALL_TEMPLATES], how='left', on='template_fk', suffixes=['', '_y']) self.survey_response = self.data_provider[Data.SURVEY_RESPONSES] self.get_atts() self.ignore_stacking = ignore_stacking self.facings_field = 'facings' if not self.ignore_stacking else 'facings_ign_stack' self.front_facing = front_facing for data in kwargs.keys(): setattr(self, data, kwargs[data]) if self.front_facing: self.scif = self.scif[self.scif['front_face_count'] == 1] self._merge_matches_and_all_product() def _merge_matches_and_all_product(self): """ This method merges the all product data with the match product in scene DataFrame """ self.match_product_in_scene = self.match_product_in_scene.merge( self.all_products, on='product_fk', how='left') def get_atts(self): query = GOLD_PEAK_BLOCKQueries.get_product_atts() product_att3 = pd.read_sql_query(query, self.rds_conn.db) self.scif = self.scif.merge(product_att3, how='left', left_on='product_ean_code', right_on='product_ean_code') def get_survey_answer(self, survey_data, answer_field=None): """ :param survey_data: 1) str - The name of the survey in the DB. 2) tuple - (The field name, the field value). For example: ('question_fk', 13) :param answer_field: The DB field from which the answer is extracted. Default is the usual hierarchy. :return: The required survey response. """ if not isinstance(survey_data, (list, tuple)): entity = 'question_text' value = survey_data else: entity, value = survey_data survey = self.survey_response[self.survey_response[entity] == value] if survey.empty: return None survey = survey.iloc[0] if answer_field is None or answer_field not in survey.keys(): answer_field = 'selected_option_text' if survey[ 'selected_option_text'] else 'number_value' survey_answer = survey[answer_field] return survey_answer def check_survey_answer(self, survey_text, target_answer): """ :param survey_text: 1) str - The name of the survey in the DB. 2) tuple - (The field name, the field value). For example: ('question_fk', 13) :param target_answer: The required answer/s for the KPI to pass. :return: True if the answer matches the target; otherwise - False. """ if not isinstance(survey_text, (list, tuple)): entity = 'question_text' value = survey_text else: entity, value = survey_text value = [value] if not isinstance(value, list) else value survey_data = self.survey_response[self.survey_response[entity].isin( value)] if survey_data.empty: Log.warning('Survey with {} = {} doesn\'t exist'.format( entity, value)) return None answer_field = 'selected_option_text' if not survey_data[ 'selected_option_text'].empty else 'number_value' target_answers = [target_answer ] if not isinstance(target_answer, (list, tuple)) else target_answer survey_answers = survey_data[answer_field].values.tolist() for answer in target_answers: if answer in survey_answers: return True return False def calculate_number_of_scenes(self, **filters): """ :param filters: These are the parameters which the Data frame is filtered by. :return: The number of scenes matching the filtered Scene Item Facts Data frame. """ if filters: if set(filters.keys()).difference(self.scenes_info.keys()): scene_data = self.scif[self.get_filter_condition( self.scif, **filters)] else: scene_data = self.scenes_info[self.get_filter_condition( self.scenes_info, **filters)] else: scene_data = self.scenes_info number_of_scenes = len(scene_data['scene_fk'].unique().tolist()) return number_of_scenes def calculate_availability(self, **filters): """ :param filters: These are the parameters which the Data frame is filtered by. :return: Total number of SKUs facings appeared in the filtered Scene Item Facts Data frame. """ if set(filters.keys()).difference(self.scif.keys()): filtered_df = self.match_product_in_scene[ self.get_filter_condition(self.match_product_in_scene, **filters)] else: filtered_df = self.scif[self.get_filter_condition( self.scif, **filters)] if self.facings_field in filtered_df.columns: availability = filtered_df[self.facings_field].sum() else: availability = len(filtered_df) return availability def calculate_assortment(self, assortment_entity='product_ean_code', minimum_assortment_for_entity=1, **filters): """ :param assortment_entity: This is the entity on which the assortment is calculated. :param minimum_assortment_for_entity: This is the number of assortment per each unique entity in order for it to be counted in the final assortment result (default is 1). :param filters: These are the parameters which the Data frame is filtered by. :return: Number of unique SKUs appeared in the filtered Scene Item Facts Data frame. """ if set(filters.keys()).difference(self.scif.keys()): filtered_df = self.match_product_in_scene[ self.get_filter_condition(self.match_product_in_scene, **filters)] else: filtered_df = self.scif[self.get_filter_condition( self.scif, **filters)] if minimum_assortment_for_entity == 1: assortment = len(filtered_df[assortment_entity].unique()) else: assortment = 0 for entity_id in filtered_df[assortment_entity].unique(): assortment_for_entity = filtered_df[ filtered_df[assortment_entity] == entity_id] if self.facings_field in filtered_df.columns: assortment_for_entity = assortment_for_entity[ self.facings_field].sum() else: assortment_for_entity = len(assortment_for_entity) if assortment_for_entity >= minimum_assortment_for_entity: assortment += 1 return assortment def calculate_share_of_shelf(self, sos_filters=None, include_empty=EXCLUDE_EMPTY, **general_filters): """ :param sos_filters: These are the parameters on which ths SOS is calculated (out of the general DF). :param include_empty: This dictates whether Empty-typed SKUs are included in the calculation. :param general_filters: These are the parameters which the general Data frame is filtered by. :return: The ratio of the Facings SOS. """ if include_empty == self.EXCLUDE_EMPTY and 'product_type' not in sos_filters.keys( ) + general_filters.keys(): general_filters['product_type'] = (self.EMPTY, self.EXCLUDE_FILTER) pop_filter = self.get_filter_condition(self.scif, **general_filters) subset_filter = self.get_filter_condition(self.scif, **sos_filters) try: ratio = self.k_engine.calculate_sos_by_facings( pop_filter=pop_filter, subset_filter=subset_filter) except: ratio = 0 if not isinstance(ratio, (float, int)): ratio = 0 return ratio def calculate_linear_share_of_shelf(self, sos_filters, include_empty=EXCLUDE_EMPTY, **general_filters): """ :param sos_filters: These are the parameters on which ths SOS is calculated (out of the general DF). :param include_empty: This dictates whether Empty-typed SKUs are included in the calculation. :param general_filters: These are the parameters which the general Data frame is filtered by. :return: The Linear SOS ratio. """ if include_empty == self.EXCLUDE_EMPTY: general_filters['product_type'] = (self.EMPTY, self.EXCLUDE_FILTER) numerator_width = self.calculate_share_space_length( **dict(sos_filters, **general_filters)) denominator_width = self.calculate_share_space_length( **general_filters) if denominator_width == 0: ratio = 0 else: ratio = numerator_width / float(denominator_width) return ratio def calculate_share_space_length(self, **filters): """ :param filters: These are the parameters which the Data frame is filtered by. :return: The total shelf width (in mm) the relevant facings occupy. """ filtered_matches = self.match_product_in_scene[ self.get_filter_condition(self.match_product_in_scene, **filters)] space_length = filtered_matches['width_mm_advance'].sum() return space_length def calculate_products_on_edge(self, min_number_of_facings=1, min_number_of_shelves=1, **filters): """ :param min_number_of_facings: Minimum number of edge facings for KPI to pass. :param min_number_of_shelves: Minimum number of different shelves with edge facings for KPI to pass. :param filters: This are the parameters which dictate the relevant SKUs for the edge calculation. :return: A tuple: (Number of scenes which pass, Total number of relevant scenes) """ filters, relevant_scenes = self.separate_location_filters_from_product_filters( **filters) if len(relevant_scenes) == 0: return 0, 0 number_of_edge_scenes = 0 for scene in relevant_scenes: edge_facings = pd.DataFrame( columns=self.match_product_in_scene.columns) matches = self.match_product_in_scene[ self.match_product_in_scene['scene_fk'] == scene] for shelf in matches['shelf_number'].unique(): shelf_matches = matches[matches['shelf_number'] == shelf] if not shelf_matches.empty: shelf_matches = shelf_matches.sort_values( by=['bay_number', 'facing_sequence_number']) edge_facings = edge_facings.append(shelf_matches.iloc[0]) if len(edge_facings) > 1: edge_facings = edge_facings.append( shelf_matches.iloc[-1]) edge_facings = edge_facings[self.get_filter_condition( edge_facings, **filters)] if len(edge_facings) >= min_number_of_facings \ and len(edge_facings['shelf_number'].unique()) >= min_number_of_shelves: number_of_edge_scenes += 1 return number_of_edge_scenes, len(relevant_scenes) def calculate_shelf_level_assortment(self, shelves, from_top_or_bottom=TOP, **filters): """ :param shelves: A shelf number (of type int or string), or a list of shelves (of type int or string). :param from_top_or_bottom: TOP for default shelf number (counted from top) or BOTTOM for shelf number counted from bottom. :param filters: These are the parameters which the Data frame is filtered by. :return: Number of unique SKUs appeared in the filtered condition. """ shelves = shelves if isinstance(shelves, list) else [shelves] shelves = [int(shelf) for shelf in shelves] if from_top_or_bottom == self.TOP: assortment = self.calculate_assortment(shelf_number=shelves, **filters) else: assortment = self.calculate_assortment( shelf_number_from_bottom=shelves, **filters) return assortment def calculate_eye_level_assortment(self, eye_level_configurations=DEFAULT, min_number_of_products=ALL, **filters): """ :param eye_level_configurations: A Data frame containing information about shelves to ignore (==not eye level) for every number of shelves in each bay. :param min_number_of_products: Minimum number of eye level unique SKUs for KPI to pass. :param filters: This are the parameters which dictate the relevant SKUs for the eye-level calculation. :return: A tuple: (Number of scenes which pass, Total number of relevant scenes) """ filters, relevant_scenes = self.separate_location_filters_from_product_filters( **filters) if len(relevant_scenes) == 0: return 0, 0 if eye_level_configurations == self.DEFAULT: if hasattr(self, 'eye_level_configurations'): eye_level_configurations = self.eye_level_configurations else: Log.error('Eye-level configurations are not set up') return False number_of_products = len(self.all_products[self.get_filter_condition( self.all_products, **filters)]['product_ean_code']) min_shelf, max_shelf, min_ignore, max_ignore = eye_level_configurations.columns number_of_eye_level_scenes = 0 for scene in relevant_scenes: eye_level_facings = pd.DataFrame( columns=self.match_product_in_scene.columns) matches = self.match_product_in_scene[ self.match_product_in_scene['scene_fk'] == scene] for bay in matches['bay_number'].unique(): bay_matches = matches[matches['bay_number'] == bay] number_of_shelves = bay_matches['shelf_number'].max() configuration = eye_level_configurations[ (eye_level_configurations[min_shelf] <= number_of_shelves) & (eye_level_configurations[max_shelf] >= number_of_shelves)] if not configuration.empty: configuration = configuration.iloc[0] else: configuration = {min_ignore: 0, max_ignore: 0} min_include = configuration[min_ignore] + 1 max_include = number_of_shelves - configuration[max_ignore] eye_level_shelves = bay_matches[ bay_matches['shelf_number'].between( min_include, max_include)] eye_level_facings = eye_level_facings.append(eye_level_shelves) eye_level_assortment = len( eye_level_facings[self.get_filter_condition( eye_level_facings, **filters)]['product_ean_code']) if min_number_of_products == self.ALL: min_number_of_products = number_of_products if eye_level_assortment >= min_number_of_products: number_of_eye_level_scenes += 1 return number_of_eye_level_scenes, len(relevant_scenes) def shelf_level_assortment(self, min_number_of_products, shelf_target, strict=True, **filters): filters, relevant_scenes = self.separate_location_filters_from_product_filters( **filters) if len(relevant_scenes) == 0: relevant_scenes = self.scif['scene_fk'].unique().tolist() number_of_products = len(self.all_products[self.get_filter_condition( self.all_products, **filters)]['product_ean_code']) result = 0 # Default score is FALSE for scene in relevant_scenes: eye_level_facings = pd.DataFrame( columns=self.match_product_in_scene.columns) matches = pd.merge(self.match_product_in_scene[ self.match_product_in_scene['scene_fk'] == scene], self.all_products, on=['product_fk']) for bay in matches['bay_number'].unique(): bay_matches = matches[matches['bay_number'] == bay] products_in_target_shelf = bay_matches[ (bay_matches['shelf_number'].isin(shelf_target)) & (bay_matches['product_ean_code'].isin(number_of_products))] eye_level_facings = eye_level_facings.append( products_in_target_shelf) eye_level_assortment = len( eye_level_facings[self.get_filter_condition( eye_level_facings, **filters)]['product_ean_code']) if eye_level_assortment >= min_number_of_products: result = 1 return result def calculate_product_sequence(self, sequence_filters, direction, empties_allowed=True, irrelevant_allowed=False, min_required_to_pass=STRICT_MODE, custom_graph=None, **general_filters): """ :param sequence_filters: One of the following: 1- a list of dictionaries, each containing the filters values of an organ in the sequence. 2- a tuple of (entity_type, [value1, value2, value3...]) in case every organ in the sequence is defined by only one filter (and of the same entity, such as brand_name, etc). :param direction: left/right/top/bottom - the direction of the sequence. :param empties_allowed: This dictates whether or not the sequence can be interrupted by Empty facings. :param irrelevant_allowed: This dictates whether or not the sequence can be interrupted by facings which are not in the sequence. :param min_required_to_pass: The number of sequences needed to exist in order for KPI to pass. If STRICT_MODE is activated, the KPI passes only if it has NO rejects. :param custom_graph: A filtered Positions graph - given in case only certain vertices need to be checked. :param general_filters: These are the parameters which the general Data frame is filtered by. :return: True if the KPI passes; otherwise False. """ if isinstance(sequence_filters, (list, tuple)) and isinstance(sequence_filters[0], (str, unicode)): sequence_filters = [{ sequence_filters[0]: values } for values in sequence_filters[1]] pass_counter = 0 reject_counter = 0 if not custom_graph: filtered_scif = self.scif[self.get_filter_condition( self.scif, **general_filters)] scenes = set(filtered_scif['scene_id'].unique()) for filters in sequence_filters: scene_for_filters = filtered_scif[self.get_filter_condition( filtered_scif, **filters)]['scene_id'].unique() scenes = scenes.intersection(scene_for_filters) if not scenes: Log.debug( 'None of the scenes include products from all types relevant for sequence' ) return True for scene in scenes: scene_graph = self.position_graphs.get(scene) scene_passes, scene_rejects = self.calculate_sequence_for_graph( scene_graph, sequence_filters, direction, empties_allowed, irrelevant_allowed) pass_counter += scene_passes reject_counter += scene_rejects if pass_counter >= min_required_to_pass: return True elif min_required_to_pass == self.STRICT_MODE and reject_counter > 0: return False else: scene_passes, scene_rejects = self.calculate_sequence_for_graph( custom_graph, sequence_filters, direction, empties_allowed, irrelevant_allowed) pass_counter += scene_passes reject_counter += scene_rejects if pass_counter >= min_required_to_pass or reject_counter == 0: return True else: return False def calculate_sequence_for_graph(self, graph, sequence_filters, direction, empties_allowed, irrelevant_allowed): """ This function checks for a sequence given a position graph (either a full scene graph or a customized one). """ pass_counter = 0 reject_counter = 0 # removing unnecessary edges filtered_scene_graph = graph.copy() edges_to_remove = filtered_scene_graph.es.select( direction_ne=direction) filtered_scene_graph.delete_edges( [edge.index for edge in edges_to_remove]) reversed_scene_graph = graph.copy() edges_to_remove = reversed_scene_graph.es.select( direction_ne=self._reverse_direction(direction)) reversed_scene_graph.delete_edges( [edge.index for edge in edges_to_remove]) vertices_list = [] for filters in sequence_filters: vertices_list.append( self.filter_vertices_from_graph(graph, **filters)) tested_vertices, sequence_vertices = vertices_list[0], vertices_list[ 1:] vertices_list = reduce(lambda x, y: x + y, sequence_vertices) sequences = [] for vertex in tested_vertices: previous_sequences = self.get_positions_by_direction( reversed_scene_graph, vertex) if previous_sequences and set(vertices_list).intersection( reduce(lambda x, y: x + y, previous_sequences)): reject_counter += 1 continue next_sequences = self.get_positions_by_direction( filtered_scene_graph, vertex) sequences.extend(next_sequences) sequences = self._filter_sequences(sequences) for sequence in sequences: all_products_appeared = True empties_found = False irrelevant_found = False full_sequence = False broken_sequence = False current_index = 0 previous_vertices = list(tested_vertices) for vertices in sequence_vertices: if not set(sequence).intersection(vertices): all_products_appeared = False break for vindex in sequence: vertex = graph.vs[vindex] if vindex not in vertices_list and vindex not in tested_vertices: if current_index < len(sequence_vertices): if vertex['product_type'] == self.EMPTY: empties_found = True else: irrelevant_found = True elif vindex in previous_vertices: pass elif vindex in sequence_vertices[current_index]: previous_vertices = list(sequence_vertices[current_index]) current_index += 1 else: broken_sequence = True if current_index == len(sequence_vertices): full_sequence = True if broken_sequence: reject_counter += 1 elif full_sequence: if not empties_allowed and empties_found: reject_counter += 1 elif not irrelevant_allowed and irrelevant_found: reject_counter += 1 elif all_products_appeared: pass_counter += 1 return pass_counter, reject_counter @staticmethod def _reverse_direction(direction): """ This function returns the opposite of a given direction. """ if direction == 'top': new_direction = 'bottom' elif direction == 'bottom': new_direction = 'top' elif direction == 'left': new_direction = 'right' elif direction == 'right': new_direction = 'left' else: new_direction = direction return new_direction def get_positions_by_direction(self, graph, vertex_index): """ This function gets a filtered graph (contains only edges of a relevant direction) and a Vertex index, and returns all sequences starting in it (until it gets to a dead end). """ sequences = [] edges = [graph.es[e] for e in graph.incident(vertex_index)] next_vertices = [edge.target for edge in edges] for vertex in next_vertices: next_sequences = self.get_positions_by_direction(graph, vertex) if not next_sequences: sequences.append([vertex]) else: for sequence in next_sequences: sequences.append([vertex] + sequence) return sequences @staticmethod def _filter_sequences(sequences): """ This function receives a list of sequences (lists of indexes), and removes sequences which can be represented by a shorter sequence (which is also in the list). """ if not sequences: return sequences sequences = sorted(sequences, key=lambda x: (x[-1], len(x))) filtered_sequences = [sequences[0]] for sequence in sequences[1:]: if sequence[-1] != filtered_sequences[-1][-1]: filtered_sequences.append(sequence) return filtered_sequences def calculate_non_proximity(self, tested_filters, anchor_filters, allowed_diagonal=False, **general_filters): """ :param tested_filters: The tested SKUs' filters. :param anchor_filters: The anchor SKUs' filters. :param allowed_diagonal: True - a tested SKU can be in a direct diagonal from an anchor SKU in order for the KPI to pass; False - a diagonal proximity is NOT allowed. :param general_filters: These are the parameters which the general Data frame is filtered by. :return: """ direction_data = [] if allowed_diagonal: direction_data.append({'top': (0, 1), 'bottom': (0, 1)}) direction_data.append({'right': (0, 1), 'left': (0, 1)}) else: direction_data.append({ 'top': (0, 1), 'bottom': (0, 1), 'right': (0, 1), 'left': (0, 1) }) is_proximity = self.calculate_relative_position(tested_filters, anchor_filters, direction_data, min_required_to_pass=1, **general_filters) return not is_proximity def calculate_relative_position(self, tested_filters, anchor_filters, direction_data, min_required_to_pass=1, **general_filters): """ :param tested_filters: The tested SKUs' filters. :param anchor_filters: The anchor SKUs' filters. :param direction_data: The allowed distance between the tested and anchor SKUs. In form: {'top': 4, 'bottom: 0, 'left': 100, 'right': 0} Alternative form: {'top': (0, 1), 'bottom': (1, 1000), ...} - As range. :param min_required_to_pass: The number of appearances needed to be True for relative position in order for KPI to pass. If all appearances are required: ==a string or a big number. :param general_filters: These are the parameters which the general Data frame is filtered by. :return: True if (at least) one pair of relevant SKUs fits the distance requirements; otherwise - returns False. """ filtered_scif = self.scif[self.get_filter_condition( self.scif, **general_filters)] tested_scenes = filtered_scif[self.get_filter_condition( filtered_scif, **tested_filters)]['scene_id'].unique() anchor_scenes = filtered_scif[self.get_filter_condition( filtered_scif, **anchor_filters)]['scene_id'].unique() relevant_scenes = set(tested_scenes).intersection(anchor_scenes) if relevant_scenes: pass_counter = 0 reject_counter = 0 for scene in relevant_scenes: scene_graph = self.position_graphs.get(scene) tested_vertices = self.filter_vertices_from_graph( scene_graph, **tested_filters) anchor_vertices = self.filter_vertices_from_graph( scene_graph, **anchor_filters) for tested_vertex in tested_vertices: for anchor_vertex in anchor_vertices: moves = {'top': 0, 'bottom': 0, 'left': 0, 'right': 0} path = scene_graph.get_shortest_paths(anchor_vertex, tested_vertex, output='epath') if path: path = path[0] for edge in path: moves[scene_graph.es[edge]['direction']] += 1 if self.validate_moves(moves, direction_data): pass_counter += 1 if isinstance( min_required_to_pass, int ) and pass_counter >= min_required_to_pass: return True else: reject_counter += 1 else: Log.debug('Tested and Anchor have no direct path') if pass_counter > 0 and reject_counter == 0: return True else: return False else: Log.debug('None of the scenes contain both anchor and tested SKUs') return False def filter_vertices_from_graph(self, graph, **filters): """ This function is given a graph and returns a set of vertices calculated by a given set of filters. """ vertices_indexes = None for field in filters.keys(): field_vertices = set() values = filters[field] if isinstance( filters[field], (list, tuple)) else [filters[field]] for value in values: vertices = [v.index for v in graph.vs.select(**{field: value})] field_vertices = field_vertices.union(vertices) if vertices_indexes is None: vertices_indexes = field_vertices else: vertices_indexes = vertices_indexes.intersection( field_vertices) vertices_indexes = vertices_indexes if vertices_indexes is not None else [ v.index for v in graph.vs ] if self.front_facing: front_facing_vertices = [ v.index for v in graph.vs.select(front_facing='Y') ] vertices_indexes = set(vertices_indexes).intersection( front_facing_vertices) return list(vertices_indexes) @staticmethod def validate_moves(moves, direction_data): """ This function checks whether the distance between the anchor and the tested SKUs fits the requirements. """ direction_data = direction_data if isinstance( direction_data, (list, tuple)) else [direction_data] validated = False for data in direction_data: data_validated = True for direction in moves.keys(): allowed_moves = data.get(direction, (0, 0)) min_move, max_move = allowed_moves if isinstance( allowed_moves, tuple) else (0, allowed_moves) if not min_move <= moves[direction] <= max_move: data_validated = False break if data_validated: validated = True break return validated @staticmethod def validate_block_moves(moves, direction_data): """ This function checks whether the distance between the anchor and the tested SKUs fits the requirements. """ direction_data = direction_data if isinstance( direction_data, (list, tuple)) else [direction_data] one_to_pass = {} for data in direction_data: for direction in data.keys(): allowed_moves = data.get(direction, (0, 0)) min_move, max_move = allowed_moves if isinstance( allowed_moves, tuple) else (0, allowed_moves) if min_move <= moves[direction] <= max_move: one_to_pass[direction] = 'T' if one_to_pass: if len(one_to_pass.values()) == 1: moves.pop(one_to_pass.keys()[0]) for direction in moves.keys(): min_move, max_move = (0, 0) if not min_move <= moves[direction] <= max_move: return False return True return False def get_product_unique_position_on_shelf(self, scene_id, shelf_number, include_empty=False, **filters): """ :param scene_id: The scene ID. :param shelf_number: The number of shelf in question (from top). :param include_empty: This dictates whether or not to include empties as valid positions. :param filters: These are the parameters which the unique position is checked for. :return: The position of the first SKU (from the given filters) to appear in the specific shelf. """ shelf_matches = self.match_product_in_scene[ (self.match_product_in_scene['scene_fk'] == scene_id) & (self.match_product_in_scene['shelf_number'] == shelf_number)] if not include_empty: filters['product_type'] = ('Empty', self.EXCLUDE_FILTER) if filters and shelf_matches[self.get_filter_condition( shelf_matches, **filters)].empty: Log.info( "Products of '{}' are not tagged in shelf number {}".format( filters, shelf_number)) return None shelf_matches = shelf_matches.sort_values( by=['bay_number', 'facing_sequence_number']) shelf_matches = shelf_matches.drop_duplicates( subset=['product_ean_code']) positions = [] for m in xrange(len(shelf_matches)): match = shelf_matches.iloc[m] match_name = 'Empty' if match[ 'product_type'] == 'Empty' else match['product_ean_code'] if positions and positions[-1] == match_name: continue positions.append(match_name) return positions def calculate_block_together(self, allowed_products_filters=None, include_empty=EXCLUDE_EMPTY, minimum_block_ratio=1, result_by_scene=False, vertical=False, horizontal=False, **filters): """ :param group_products: if we searching for group in block - this is the filter of the group inside the big block :param block_products: if we searching for group in block - this is the filter of the big block :param group: True if the kpi is for block of product inside a bigger block :param orphan: True if searching for orphan products 3 products away from the biggest block :param horizontal: True if the biggest block have to be at least 2X2 :param vertical: True :param allowed_products_filters: These are the parameters which are allowed to corrupt the block without failing it. :param include_empty: This parameter dictates whether or not to discard Empty-typed products. :param minimum_block_ratio: The minimum (block number of facings / total number of relevant facings) ratio in order for KPI to pass (if ratio=1, then only one block is allowed). :param result_by_scene: True - The result is a tuple of (number of passed scenes, total relevant scenes); False - The result is True if at least one scene has a block, False - otherwise. :param filters: These are the parameters which the blocks are checked for. :return: see 'result_by_scene' above. """ filters, relevant_scenes = self.separate_location_filters_from_product_filters( **filters) if len(relevant_scenes) == 0: if result_by_scene: return 0, 0 else: Log.debug( 'Block Together: No relevant SKUs were found for these filters {}' .format(filters)) return True number_of_blocked_scenes = 0 cluster_ratios = [] for scene in relevant_scenes: scene_graph = self.position_graphs.get(scene).copy() relevant_vertices = set( self.filter_vertices_from_graph(scene_graph, **filters)) if allowed_products_filters: allowed_vertices = self.filter_vertices_from_graph( scene_graph, **allowed_products_filters) else: allowed_vertices = set() if include_empty == self.EXCLUDE_EMPTY: empty_vertices = { v.index for v in scene_graph.vs.select(product_type='Empty') } allowed_vertices = set(allowed_vertices).union(empty_vertices) all_vertices = {v.index for v in scene_graph.vs} vertices_to_remove = all_vertices.difference( relevant_vertices.union(allowed_vertices)) scene_graph.delete_vertices(vertices_to_remove) # removing clusters including 'allowed' SKUs only clusters = [ cluster for cluster in scene_graph.clusters() if set(cluster).difference(allowed_vertices) ] new_relevant_vertices = self.filter_vertices_from_graph( scene_graph, **filters) for cluster in clusters: relevant_vertices_in_cluster = set(cluster).intersection( new_relevant_vertices) if len(new_relevant_vertices) > 0: cluster_ratio = len(relevant_vertices_in_cluster) / float( len(new_relevant_vertices)) else: cluster_ratio = 0 cluster_ratios.append(cluster_ratio) if cluster_ratio > minimum_block_ratio: all_vertices = {v.index for v in scene_graph.vs} non_cluster_vertices = all_vertices.difference(cluster) block_graph = scene_graph block_graph.delete_vertices(non_cluster_vertices) direction_data = {'top': (1, 1), 'bottom': (1, 1)} result_v = self.calculate_relative_in_block_per_graph( block_graph, direction_data) direction_data = {'left': (2, 2), 'right': (2, 2)} result_h = self.calculate_relative_in_block_per_graph( block_graph, direction_data) if result_h and result_v: return True if result_by_scene: return number_of_blocked_scenes, len(relevant_scenes) else: if minimum_block_ratio == 1: return False elif cluster_ratios: return False else: return False def calculate_relative_in_block_per_graph(self, scene_graph, direction_data, min_required_to_pass=1, sent_tested_vertices=None, sent_anchor_vertices=None): """ :param direction_data: The allowed distance between the tested and anchor SKUs. In form: {'top': 4, 'bottom: 0, 'left': 100, 'right': 0} Alternative form: {'top': (0, 1), 'bottom': (1, 1000), ...} - As range. :param min_required_to_pass: The number of appearances needed to be True for relative position in order for KPI to pass. If all appearances are required: ==a string or a big number. :param general_filters: These are the parameters which the general data frame is filtered by. :return: True if (at least) one pair of relevant SKUs fits the distance requirements; otherwise - returns False. """ pass_counter = 0 reject_counter = 0 if sent_anchor_vertices and sent_tested_vertices: tested_vertices = sent_tested_vertices anchor_vertices = sent_anchor_vertices else: tested_vertices = self.filter_vertices_from_graph(scene_graph) anchor_vertices = self.filter_vertices_from_graph(scene_graph) for tested_vertex in tested_vertices: for anchor_vertex in anchor_vertices: moves = {'top': 0, 'bottom': 0, 'left': 0, 'right': 0} path = scene_graph.get_shortest_paths(anchor_vertex, tested_vertex, output='epath') if path: path = path[0] for edge in path: moves[scene_graph.es[edge]['direction']] += 1 if self.validate_block_moves(moves, direction_data): pass_counter += 1 if isinstance( min_required_to_pass, int) and pass_counter >= min_required_to_pass: return True else: reject_counter += 1 if pass_counter > 0 and reject_counter == 0: return True else: return False def get_filter_condition(self, df, **filters): """ :param df: The Data frame to be filters. :param filters: These are the parameters which the Data frame is filtered by. Every parameter would be a tuple of the value and an include/exclude flag. INPUT EXAMPLE (1): manufacturer_name = ('Diageo', DIAGEOAUMSCGENERALToolBox.INCLUDE_FILTER) INPUT EXAMPLE (2): manufacturer_name = 'Diageo' :return: a filtered Scene Item Facts Data frame. """ if not filters: return df['pk'].apply(bool) if self.facings_field in df.keys(): filter_condition = (df[self.facings_field] > 0) else: filter_condition = None for field in filters.keys(): if field in df.keys(): if isinstance(filters[field], tuple): value, exclude_or_include = filters[field] else: value, exclude_or_include = filters[ field], self.INCLUDE_FILTER if not value: continue if not isinstance(value, list): value = [value] if exclude_or_include == self.INCLUDE_FILTER: condition = (df[field].isin(value)) elif exclude_or_include == self.EXCLUDE_FILTER: condition = (~df[field].isin(value)) elif exclude_or_include == self.CONTAIN_FILTER: condition = (df[field].str.contains(value[0], regex=False)) for v in value[1:]: condition |= df[field].str.contains(v, regex=False) else: continue if filter_condition is None: filter_condition = condition else: filter_condition &= condition else: Log.warning('field {} is not in the Data Frame'.format(field)) return filter_condition def separate_location_filters_from_product_filters(self, **filters): """ This function gets scene-item-facts filters of all kinds, extracts the relevant scenes by the location filters, and returns them along with the product filters only. """ location_filters = {} for field in filters.keys(): if field not in self.all_products.columns and field in self.scif.columns: location_filters[field] = filters.pop(field) relevant_scenes = self.scif[self.get_filter_condition( self.scif, **location_filters)]['scene_id'].unique() return filters, relevant_scenes @staticmethod def get_json_data(file_path, sheet_name=None, skiprows=0): """ This function gets a file's path and extract its content into a JSON. """ data = {} if sheet_name: sheet_names = [sheet_name] else: sheet_names = xlrd.open_workbook(file_path).sheet_names() for sheet_name in sheet_names: try: output = pd.read_excel(file_path, sheetname=sheet_name, skiprows=skiprows) except xlrd.biffh.XLRDError: Log.warning('Sheet name {} doesn\'t exist'.format(sheet_name)) return None output = output.to_json(orient='records') output = json.loads(output) for x in xrange(len(output)): for y in output[x].keys(): output[x][y] = unicode( '' if output[x][y] is None else output[x][y]).strip() if not output[x][y]: output[x].pop(y, None) data[sheet_name] = output if sheet_name: data = data[sheet_name] elif len(data.keys()) == 1: data = data[data.keys()[0]] return data