def __init__(self,
              data_provider,
              output,
              rds_conn=None,
              ignore_stacking=False,
              front_facing=False,
              **kwargs):
     self.k_engine = BaseCalculationsGroup(data_provider, output)
     self.rds_conn = rds_conn
     self.data_provider = data_provider
     self.project_name = self.data_provider.project_name
     self.session_uid = self.data_provider.session_uid
     self.scif = self.data_provider[Data.SCENE_ITEM_FACTS]
     self.match_product_in_scene = self.data_provider[Data.MATCHES]
     self.position_graphs = PositionGraphs(self.data_provider,
                                           rds_conn=self.rds_conn)
     self.all_products = self.data_provider[Data.ALL_PRODUCTS]
     self.survey_response = self.data_provider[Data.SURVEY_RESPONSES]
     self.scenes_info = self.data_provider[Data.SCENES_INFO].merge(
         self.data_provider[Data.ALL_TEMPLATES],
         how='left',
         on='template_fk',
         suffixes=['', '_y'])
     self.survey_response = self.data_provider[Data.SURVEY_RESPONSES]
     self.get_atts()
     self.ignore_stacking = ignore_stacking
     self.facings_field = 'facings' if not self.ignore_stacking else 'facings_ign_stack'
     self.front_facing = front_facing
     for data in kwargs.keys():
         setattr(self, data, kwargs[data])
     if self.front_facing:
         self.scif = self.scif[self.scif['front_face_count'] == 1]
     self._merge_matches_and_all_product()
Exemple #2
0
 def __init__(self,
              data_provider,
              output=None,
              ps_data_provider=None,
              common=None,
              rds_conn=None,
              front_facing=False,
              custom_scif=None,
              custom_matches=None,
              **kwargs):
     super(Block, self).__init__(data_provider, output, ps_data_provider,
                                 common, rds_conn, **kwargs)
     self._position_graphs = PositionGraphs(self.data_provider)
     self.front_facing = front_facing
     self.outliers_threshold = Default.outliers_threshold
     self.check_vertical_horizontal = Default.check_vertical_horizontal
     self.include_stacking = Default.include_stacking
     self.ignore_empty = Default.ignore_empty
     self.allowed_edge_type = Default.allowed_edge_type
     self.scif = data_provider.scene_item_facts if custom_scif is None else custom_scif
     self.matches = data_provider.matches if custom_matches is None else custom_matches
     self.adj_graphs_by_scene = {}
     self.masking_data = transform_maskings(
         retrieve_maskings(
             self.data_provider.project_name,
             self.data_provider.scenes_info['scene_fk'].to_list()))
     self.masking_data = self.masking_data.merge(
         self.matches[['probe_match_fk', 'scene_fk']],
         on=['probe_match_fk'])
     self.matches_df = self.matches.merge(
         self.data_provider.all_products_including_deleted, on='product_fk')
     self.matches_df = self.matches_df[
         ~(self.matches_df['product_type'].isin(['POS']))
         & (self.matches_df['stacking_layer'] > 0)]
     self.matches_df[Block.BLOCK_KEY] = None
class GOLD_PEAK_BLOCKGeneralToolBox:

    EXCLUDE_FILTER = 0
    INCLUDE_FILTER = 1
    CONTAIN_FILTER = 2
    EXCLUDE_EMPTY = False
    INCLUDE_EMPTY = True

    STRICT_MODE = ALL = 1000

    EMPTY = 'Empty'
    DEFAULT = 'Default'
    TOP = 'Top'
    BOTTOM = 'Bottom'

    def __init__(self,
                 data_provider,
                 output,
                 rds_conn=None,
                 ignore_stacking=False,
                 front_facing=False,
                 **kwargs):
        self.k_engine = BaseCalculationsGroup(data_provider, output)
        self.rds_conn = rds_conn
        self.data_provider = data_provider
        self.project_name = self.data_provider.project_name
        self.session_uid = self.data_provider.session_uid
        self.scif = self.data_provider[Data.SCENE_ITEM_FACTS]
        self.match_product_in_scene = self.data_provider[Data.MATCHES]
        self.position_graphs = PositionGraphs(self.data_provider,
                                              rds_conn=self.rds_conn)
        self.all_products = self.data_provider[Data.ALL_PRODUCTS]
        self.survey_response = self.data_provider[Data.SURVEY_RESPONSES]
        self.scenes_info = self.data_provider[Data.SCENES_INFO].merge(
            self.data_provider[Data.ALL_TEMPLATES],
            how='left',
            on='template_fk',
            suffixes=['', '_y'])
        self.survey_response = self.data_provider[Data.SURVEY_RESPONSES]
        self.get_atts()
        self.ignore_stacking = ignore_stacking
        self.facings_field = 'facings' if not self.ignore_stacking else 'facings_ign_stack'
        self.front_facing = front_facing
        for data in kwargs.keys():
            setattr(self, data, kwargs[data])
        if self.front_facing:
            self.scif = self.scif[self.scif['front_face_count'] == 1]
        self._merge_matches_and_all_product()

    def _merge_matches_and_all_product(self):
        """
        This method merges the all product data with the match product in scene DataFrame
        """
        self.match_product_in_scene = self.match_product_in_scene.merge(
            self.all_products, on='product_fk', how='left')

    def get_atts(self):

        query = GOLD_PEAK_BLOCKQueries.get_product_atts()
        product_att3 = pd.read_sql_query(query, self.rds_conn.db)
        self.scif = self.scif.merge(product_att3,
                                    how='left',
                                    left_on='product_ean_code',
                                    right_on='product_ean_code')

    def get_survey_answer(self, survey_data, answer_field=None):
        """
        :param survey_data:     1) str - The name of the survey in the DB.
                                2) tuple - (The field name, the field value). For example: ('question_fk', 13)
        :param answer_field: The DB field from which the answer is extracted. Default is the usual hierarchy.
        :return: The required survey response.
        """
        if not isinstance(survey_data, (list, tuple)):
            entity = 'question_text'
            value = survey_data
        else:
            entity, value = survey_data
        survey = self.survey_response[self.survey_response[entity] == value]
        if survey.empty:
            return None
        survey = survey.iloc[0]
        if answer_field is None or answer_field not in survey.keys():
            answer_field = 'selected_option_text' if survey[
                'selected_option_text'] else 'number_value'
        survey_answer = survey[answer_field]
        return survey_answer

    def check_survey_answer(self, survey_text, target_answer):
        """
        :param survey_text:     1) str - The name of the survey in the DB.
                                2) tuple - (The field name, the field value). For example: ('question_fk', 13)
        :param target_answer: The required answer/s for the KPI to pass.
        :return: True if the answer matches the target; otherwise - False.
        """
        if not isinstance(survey_text, (list, tuple)):
            entity = 'question_text'
            value = survey_text
        else:
            entity, value = survey_text
        value = [value] if not isinstance(value, list) else value
        survey_data = self.survey_response[self.survey_response[entity].isin(
            value)]
        if survey_data.empty:
            Log.warning('Survey with {} = {} doesn\'t exist'.format(
                entity, value))
            return None
        answer_field = 'selected_option_text' if not survey_data[
            'selected_option_text'].empty else 'number_value'
        target_answers = [target_answer
                          ] if not isinstance(target_answer,
                                              (list, tuple)) else target_answer
        survey_answers = survey_data[answer_field].values.tolist()
        for answer in target_answers:
            if answer in survey_answers:
                return True
        return False

    def calculate_number_of_scenes(self, **filters):
        """
        :param filters: These are the parameters which the Data frame is filtered by.
        :return: The number of scenes matching the filtered Scene Item Facts Data frame.
        """
        if filters:
            if set(filters.keys()).difference(self.scenes_info.keys()):
                scene_data = self.scif[self.get_filter_condition(
                    self.scif, **filters)]
            else:
                scene_data = self.scenes_info[self.get_filter_condition(
                    self.scenes_info, **filters)]
        else:
            scene_data = self.scenes_info
        number_of_scenes = len(scene_data['scene_fk'].unique().tolist())
        return number_of_scenes

    def calculate_availability(self, **filters):
        """
        :param filters: These are the parameters which the Data frame is filtered by.
        :return: Total number of SKUs facings appeared in the filtered Scene Item Facts Data frame.
        """
        if set(filters.keys()).difference(self.scif.keys()):
            filtered_df = self.match_product_in_scene[
                self.get_filter_condition(self.match_product_in_scene,
                                          **filters)]
        else:
            filtered_df = self.scif[self.get_filter_condition(
                self.scif, **filters)]
        if self.facings_field in filtered_df.columns:
            availability = filtered_df[self.facings_field].sum()
        else:
            availability = len(filtered_df)
        return availability

    def calculate_assortment(self,
                             assortment_entity='product_ean_code',
                             minimum_assortment_for_entity=1,
                             **filters):
        """
        :param assortment_entity: This is the entity on which the assortment is calculated.
        :param minimum_assortment_for_entity: This is the number of assortment per each unique entity in order for it
                                              to be counted in the final assortment result (default is 1).
        :param filters: These are the parameters which the Data frame is filtered by.
        :return: Number of unique SKUs appeared in the filtered Scene Item Facts Data frame.
        """
        if set(filters.keys()).difference(self.scif.keys()):
            filtered_df = self.match_product_in_scene[
                self.get_filter_condition(self.match_product_in_scene,
                                          **filters)]
        else:
            filtered_df = self.scif[self.get_filter_condition(
                self.scif, **filters)]
        if minimum_assortment_for_entity == 1:
            assortment = len(filtered_df[assortment_entity].unique())
        else:
            assortment = 0
            for entity_id in filtered_df[assortment_entity].unique():
                assortment_for_entity = filtered_df[
                    filtered_df[assortment_entity] == entity_id]
                if self.facings_field in filtered_df.columns:
                    assortment_for_entity = assortment_for_entity[
                        self.facings_field].sum()
                else:
                    assortment_for_entity = len(assortment_for_entity)
                if assortment_for_entity >= minimum_assortment_for_entity:
                    assortment += 1
        return assortment

    def calculate_share_of_shelf(self,
                                 sos_filters=None,
                                 include_empty=EXCLUDE_EMPTY,
                                 **general_filters):
        """
        :param sos_filters: These are the parameters on which ths SOS is calculated (out of the general DF).
        :param include_empty: This dictates whether Empty-typed SKUs are included in the calculation.
        :param general_filters: These are the parameters which the general Data frame is filtered by.
        :return: The ratio of the Facings SOS.
        """
        if include_empty == self.EXCLUDE_EMPTY and 'product_type' not in sos_filters.keys(
        ) + general_filters.keys():
            general_filters['product_type'] = (self.EMPTY, self.EXCLUDE_FILTER)
        pop_filter = self.get_filter_condition(self.scif, **general_filters)
        subset_filter = self.get_filter_condition(self.scif, **sos_filters)

        try:
            ratio = self.k_engine.calculate_sos_by_facings(
                pop_filter=pop_filter, subset_filter=subset_filter)
        except:
            ratio = 0

        if not isinstance(ratio, (float, int)):
            ratio = 0
        return ratio

    def calculate_linear_share_of_shelf(self,
                                        sos_filters,
                                        include_empty=EXCLUDE_EMPTY,
                                        **general_filters):
        """
        :param sos_filters: These are the parameters on which ths SOS is calculated (out of the general DF).
        :param include_empty: This dictates whether Empty-typed SKUs are included in the calculation.
        :param general_filters: These are the parameters which the general Data frame is filtered by.
        :return: The Linear SOS ratio.
        """
        if include_empty == self.EXCLUDE_EMPTY:
            general_filters['product_type'] = (self.EMPTY, self.EXCLUDE_FILTER)

        numerator_width = self.calculate_share_space_length(
            **dict(sos_filters, **general_filters))
        denominator_width = self.calculate_share_space_length(
            **general_filters)

        if denominator_width == 0:
            ratio = 0
        else:
            ratio = numerator_width / float(denominator_width)
        return ratio

    def calculate_share_space_length(self, **filters):
        """
        :param filters: These are the parameters which the Data frame is filtered by.
        :return: The total shelf width (in mm) the relevant facings occupy.
        """
        filtered_matches = self.match_product_in_scene[
            self.get_filter_condition(self.match_product_in_scene, **filters)]
        space_length = filtered_matches['width_mm_advance'].sum()
        return space_length

    def calculate_products_on_edge(self,
                                   min_number_of_facings=1,
                                   min_number_of_shelves=1,
                                   **filters):
        """
        :param min_number_of_facings: Minimum number of edge facings for KPI to pass.
        :param min_number_of_shelves: Minimum number of different shelves with edge facings for KPI to pass.
        :param filters: This are the parameters which dictate the relevant SKUs for the edge calculation.
        :return: A tuple: (Number of scenes which pass, Total number of relevant scenes)
        """
        filters, relevant_scenes = self.separate_location_filters_from_product_filters(
            **filters)
        if len(relevant_scenes) == 0:
            return 0, 0
        number_of_edge_scenes = 0
        for scene in relevant_scenes:
            edge_facings = pd.DataFrame(
                columns=self.match_product_in_scene.columns)
            matches = self.match_product_in_scene[
                self.match_product_in_scene['scene_fk'] == scene]
            for shelf in matches['shelf_number'].unique():
                shelf_matches = matches[matches['shelf_number'] == shelf]
                if not shelf_matches.empty:
                    shelf_matches = shelf_matches.sort_values(
                        by=['bay_number', 'facing_sequence_number'])
                    edge_facings = edge_facings.append(shelf_matches.iloc[0])
                    if len(edge_facings) > 1:
                        edge_facings = edge_facings.append(
                            shelf_matches.iloc[-1])
            edge_facings = edge_facings[self.get_filter_condition(
                edge_facings, **filters)]
            if len(edge_facings) >= min_number_of_facings \
                    and len(edge_facings['shelf_number'].unique()) >= min_number_of_shelves:
                number_of_edge_scenes += 1
        return number_of_edge_scenes, len(relevant_scenes)

    def calculate_shelf_level_assortment(self,
                                         shelves,
                                         from_top_or_bottom=TOP,
                                         **filters):
        """
        :param shelves: A shelf number (of type int or string), or a list of shelves (of type int or string).
        :param from_top_or_bottom: TOP for default shelf number (counted from top)
                                    or BOTTOM for shelf number counted from bottom.
        :param filters: These are the parameters which the Data frame is filtered by.
        :return: Number of unique SKUs appeared in the filtered condition.
        """
        shelves = shelves if isinstance(shelves, list) else [shelves]
        shelves = [int(shelf) for shelf in shelves]
        if from_top_or_bottom == self.TOP:
            assortment = self.calculate_assortment(shelf_number=shelves,
                                                   **filters)
        else:
            assortment = self.calculate_assortment(
                shelf_number_from_bottom=shelves, **filters)
        return assortment

    def calculate_eye_level_assortment(self,
                                       eye_level_configurations=DEFAULT,
                                       min_number_of_products=ALL,
                                       **filters):
        """
        :param eye_level_configurations: A Data frame containing information about shelves to ignore (==not eye level)
                                         for every number of shelves in each bay.
        :param min_number_of_products: Minimum number of eye level unique SKUs for KPI to pass.
        :param filters: This are the parameters which dictate the relevant SKUs for the eye-level calculation.
        :return: A tuple: (Number of scenes which pass, Total number of relevant scenes)
        """
        filters, relevant_scenes = self.separate_location_filters_from_product_filters(
            **filters)
        if len(relevant_scenes) == 0:
            return 0, 0
        if eye_level_configurations == self.DEFAULT:
            if hasattr(self, 'eye_level_configurations'):
                eye_level_configurations = self.eye_level_configurations
            else:
                Log.error('Eye-level configurations are not set up')
                return False
        number_of_products = len(self.all_products[self.get_filter_condition(
            self.all_products, **filters)]['product_ean_code'])
        min_shelf, max_shelf, min_ignore, max_ignore = eye_level_configurations.columns
        number_of_eye_level_scenes = 0
        for scene in relevant_scenes:
            eye_level_facings = pd.DataFrame(
                columns=self.match_product_in_scene.columns)
            matches = self.match_product_in_scene[
                self.match_product_in_scene['scene_fk'] == scene]
            for bay in matches['bay_number'].unique():
                bay_matches = matches[matches['bay_number'] == bay]
                number_of_shelves = bay_matches['shelf_number'].max()
                configuration = eye_level_configurations[
                    (eye_level_configurations[min_shelf] <= number_of_shelves)
                    &
                    (eye_level_configurations[max_shelf] >= number_of_shelves)]
                if not configuration.empty:
                    configuration = configuration.iloc[0]
                else:
                    configuration = {min_ignore: 0, max_ignore: 0}
                min_include = configuration[min_ignore] + 1
                max_include = number_of_shelves - configuration[max_ignore]
                eye_level_shelves = bay_matches[
                    bay_matches['shelf_number'].between(
                        min_include, max_include)]
                eye_level_facings = eye_level_facings.append(eye_level_shelves)
            eye_level_assortment = len(
                eye_level_facings[self.get_filter_condition(
                    eye_level_facings, **filters)]['product_ean_code'])
            if min_number_of_products == self.ALL:
                min_number_of_products = number_of_products
            if eye_level_assortment >= min_number_of_products:
                number_of_eye_level_scenes += 1
        return number_of_eye_level_scenes, len(relevant_scenes)

    def shelf_level_assortment(self,
                               min_number_of_products,
                               shelf_target,
                               strict=True,
                               **filters):
        filters, relevant_scenes = self.separate_location_filters_from_product_filters(
            **filters)
        if len(relevant_scenes) == 0:
            relevant_scenes = self.scif['scene_fk'].unique().tolist()
        number_of_products = len(self.all_products[self.get_filter_condition(
            self.all_products, **filters)]['product_ean_code'])
        result = 0  # Default score is FALSE
        for scene in relevant_scenes:
            eye_level_facings = pd.DataFrame(
                columns=self.match_product_in_scene.columns)
            matches = pd.merge(self.match_product_in_scene[
                self.match_product_in_scene['scene_fk'] == scene],
                               self.all_products,
                               on=['product_fk'])
            for bay in matches['bay_number'].unique():
                bay_matches = matches[matches['bay_number'] == bay]
                products_in_target_shelf = bay_matches[
                    (bay_matches['shelf_number'].isin(shelf_target)) &
                    (bay_matches['product_ean_code'].isin(number_of_products))]
                eye_level_facings = eye_level_facings.append(
                    products_in_target_shelf)
            eye_level_assortment = len(
                eye_level_facings[self.get_filter_condition(
                    eye_level_facings, **filters)]['product_ean_code'])
            if eye_level_assortment >= min_number_of_products:
                result = 1
        return result

    def calculate_product_sequence(self,
                                   sequence_filters,
                                   direction,
                                   empties_allowed=True,
                                   irrelevant_allowed=False,
                                   min_required_to_pass=STRICT_MODE,
                                   custom_graph=None,
                                   **general_filters):
        """
        :param sequence_filters: One of the following:
                        1- a list of dictionaries, each containing the filters values of an organ in the sequence.
                        2- a tuple of (entity_type, [value1, value2, value3...]) in case every organ in the sequence
                           is defined by only one filter (and of the same entity, such as brand_name, etc).
        :param direction: left/right/top/bottom - the direction of the sequence.
        :param empties_allowed: This dictates whether or not the sequence can be interrupted by Empty facings.
        :param irrelevant_allowed: This dictates whether or not the sequence can be interrupted by facings which are
                                   not in the sequence.
        :param min_required_to_pass: The number of sequences needed to exist in order for KPI to pass.
                                     If STRICT_MODE is activated, the KPI passes only if it has NO rejects.
        :param custom_graph: A filtered Positions graph - given in case only certain vertices need to be checked.
        :param general_filters: These are the parameters which the general Data frame is filtered by.
        :return: True if the KPI passes; otherwise False.
        """
        if isinstance(sequence_filters,
                      (list, tuple)) and isinstance(sequence_filters[0],
                                                    (str, unicode)):
            sequence_filters = [{
                sequence_filters[0]: values
            } for values in sequence_filters[1]]

        pass_counter = 0
        reject_counter = 0

        if not custom_graph:
            filtered_scif = self.scif[self.get_filter_condition(
                self.scif, **general_filters)]
            scenes = set(filtered_scif['scene_id'].unique())
            for filters in sequence_filters:
                scene_for_filters = filtered_scif[self.get_filter_condition(
                    filtered_scif, **filters)]['scene_id'].unique()
                scenes = scenes.intersection(scene_for_filters)
                if not scenes:
                    Log.debug(
                        'None of the scenes include products from all types relevant for sequence'
                    )
                    return True

            for scene in scenes:
                scene_graph = self.position_graphs.get(scene)
                scene_passes, scene_rejects = self.calculate_sequence_for_graph(
                    scene_graph, sequence_filters, direction, empties_allowed,
                    irrelevant_allowed)
                pass_counter += scene_passes
                reject_counter += scene_rejects

                if pass_counter >= min_required_to_pass:
                    return True
                elif min_required_to_pass == self.STRICT_MODE and reject_counter > 0:
                    return False

        else:
            scene_passes, scene_rejects = self.calculate_sequence_for_graph(
                custom_graph, sequence_filters, direction, empties_allowed,
                irrelevant_allowed)
            pass_counter += scene_passes
            reject_counter += scene_rejects

        if pass_counter >= min_required_to_pass or reject_counter == 0:
            return True
        else:
            return False

    def calculate_sequence_for_graph(self, graph, sequence_filters, direction,
                                     empties_allowed, irrelevant_allowed):
        """
        This function checks for a sequence given a position graph (either a full scene graph or a customized one).
        """
        pass_counter = 0
        reject_counter = 0

        # removing unnecessary edges
        filtered_scene_graph = graph.copy()
        edges_to_remove = filtered_scene_graph.es.select(
            direction_ne=direction)
        filtered_scene_graph.delete_edges(
            [edge.index for edge in edges_to_remove])

        reversed_scene_graph = graph.copy()
        edges_to_remove = reversed_scene_graph.es.select(
            direction_ne=self._reverse_direction(direction))
        reversed_scene_graph.delete_edges(
            [edge.index for edge in edges_to_remove])

        vertices_list = []
        for filters in sequence_filters:
            vertices_list.append(
                self.filter_vertices_from_graph(graph, **filters))
        tested_vertices, sequence_vertices = vertices_list[0], vertices_list[
            1:]
        vertices_list = reduce(lambda x, y: x + y, sequence_vertices)

        sequences = []
        for vertex in tested_vertices:
            previous_sequences = self.get_positions_by_direction(
                reversed_scene_graph, vertex)
            if previous_sequences and set(vertices_list).intersection(
                    reduce(lambda x, y: x + y, previous_sequences)):
                reject_counter += 1
                continue

            next_sequences = self.get_positions_by_direction(
                filtered_scene_graph, vertex)
            sequences.extend(next_sequences)

        sequences = self._filter_sequences(sequences)
        for sequence in sequences:
            all_products_appeared = True
            empties_found = False
            irrelevant_found = False
            full_sequence = False
            broken_sequence = False
            current_index = 0
            previous_vertices = list(tested_vertices)

            for vertices in sequence_vertices:
                if not set(sequence).intersection(vertices):
                    all_products_appeared = False
                    break

            for vindex in sequence:
                vertex = graph.vs[vindex]
                if vindex not in vertices_list and vindex not in tested_vertices:
                    if current_index < len(sequence_vertices):
                        if vertex['product_type'] == self.EMPTY:
                            empties_found = True
                        else:
                            irrelevant_found = True
                elif vindex in previous_vertices:
                    pass
                elif vindex in sequence_vertices[current_index]:
                    previous_vertices = list(sequence_vertices[current_index])
                    current_index += 1
                else:
                    broken_sequence = True

            if current_index == len(sequence_vertices):
                full_sequence = True

            if broken_sequence:
                reject_counter += 1
            elif full_sequence:
                if not empties_allowed and empties_found:
                    reject_counter += 1
                elif not irrelevant_allowed and irrelevant_found:
                    reject_counter += 1
                elif all_products_appeared:
                    pass_counter += 1
        return pass_counter, reject_counter

    @staticmethod
    def _reverse_direction(direction):
        """
        This function returns the opposite of a given direction.
        """
        if direction == 'top':
            new_direction = 'bottom'
        elif direction == 'bottom':
            new_direction = 'top'
        elif direction == 'left':
            new_direction = 'right'
        elif direction == 'right':
            new_direction = 'left'
        else:
            new_direction = direction
        return new_direction

    def get_positions_by_direction(self, graph, vertex_index):
        """
        This function gets a filtered graph (contains only edges of a relevant direction) and a Vertex index,
        and returns all sequences starting in it (until it gets to a dead end).
        """
        sequences = []
        edges = [graph.es[e] for e in graph.incident(vertex_index)]
        next_vertices = [edge.target for edge in edges]
        for vertex in next_vertices:
            next_sequences = self.get_positions_by_direction(graph, vertex)
            if not next_sequences:
                sequences.append([vertex])
            else:
                for sequence in next_sequences:
                    sequences.append([vertex] + sequence)
        return sequences

    @staticmethod
    def _filter_sequences(sequences):
        """
        This function receives a list of sequences (lists of indexes), and removes sequences which can be represented
        by a shorter sequence (which is also in the list).
        """
        if not sequences:
            return sequences
        sequences = sorted(sequences, key=lambda x: (x[-1], len(x)))
        filtered_sequences = [sequences[0]]
        for sequence in sequences[1:]:
            if sequence[-1] != filtered_sequences[-1][-1]:
                filtered_sequences.append(sequence)
        return filtered_sequences

    def calculate_non_proximity(self,
                                tested_filters,
                                anchor_filters,
                                allowed_diagonal=False,
                                **general_filters):
        """
        :param tested_filters: The tested SKUs' filters.
        :param anchor_filters: The anchor SKUs' filters.
        :param allowed_diagonal: True - a tested SKU can be in a direct diagonal from an anchor SKU in order
                                        for the KPI to pass;
                                 False - a diagonal proximity is NOT allowed.
        :param general_filters: These are the parameters which the general Data frame is filtered by.
        :return:
        """
        direction_data = []
        if allowed_diagonal:
            direction_data.append({'top': (0, 1), 'bottom': (0, 1)})
            direction_data.append({'right': (0, 1), 'left': (0, 1)})
        else:
            direction_data.append({
                'top': (0, 1),
                'bottom': (0, 1),
                'right': (0, 1),
                'left': (0, 1)
            })
        is_proximity = self.calculate_relative_position(tested_filters,
                                                        anchor_filters,
                                                        direction_data,
                                                        min_required_to_pass=1,
                                                        **general_filters)
        return not is_proximity

    def calculate_relative_position(self,
                                    tested_filters,
                                    anchor_filters,
                                    direction_data,
                                    min_required_to_pass=1,
                                    **general_filters):
        """
        :param tested_filters: The tested SKUs' filters.
        :param anchor_filters: The anchor SKUs' filters.
        :param direction_data: The allowed distance between the tested and anchor SKUs.
                               In form: {'top': 4, 'bottom: 0, 'left': 100, 'right': 0}
                               Alternative form: {'top': (0, 1), 'bottom': (1, 1000), ...} - As range.
        :param min_required_to_pass: The number of appearances needed to be True for relative position in order for KPI
                                     to pass. If all appearances are required: ==a string or a big number.
        :param general_filters: These are the parameters which the general Data frame is filtered by.
        :return: True if (at least) one pair of relevant SKUs fits the distance requirements; otherwise - returns False.
        """
        filtered_scif = self.scif[self.get_filter_condition(
            self.scif, **general_filters)]
        tested_scenes = filtered_scif[self.get_filter_condition(
            filtered_scif, **tested_filters)]['scene_id'].unique()
        anchor_scenes = filtered_scif[self.get_filter_condition(
            filtered_scif, **anchor_filters)]['scene_id'].unique()
        relevant_scenes = set(tested_scenes).intersection(anchor_scenes)

        if relevant_scenes:
            pass_counter = 0
            reject_counter = 0
            for scene in relevant_scenes:
                scene_graph = self.position_graphs.get(scene)
                tested_vertices = self.filter_vertices_from_graph(
                    scene_graph, **tested_filters)
                anchor_vertices = self.filter_vertices_from_graph(
                    scene_graph, **anchor_filters)
                for tested_vertex in tested_vertices:
                    for anchor_vertex in anchor_vertices:
                        moves = {'top': 0, 'bottom': 0, 'left': 0, 'right': 0}
                        path = scene_graph.get_shortest_paths(anchor_vertex,
                                                              tested_vertex,
                                                              output='epath')
                        if path:
                            path = path[0]
                            for edge in path:
                                moves[scene_graph.es[edge]['direction']] += 1
                            if self.validate_moves(moves, direction_data):
                                pass_counter += 1
                                if isinstance(
                                        min_required_to_pass, int
                                ) and pass_counter >= min_required_to_pass:
                                    return True
                            else:
                                reject_counter += 1
                        else:
                            Log.debug('Tested and Anchor have no direct path')
            if pass_counter > 0 and reject_counter == 0:
                return True
            else:
                return False
        else:
            Log.debug('None of the scenes contain both anchor and tested SKUs')
            return False

    def filter_vertices_from_graph(self, graph, **filters):
        """
        This function is given a graph and returns a set of vertices calculated by a given set of filters.
        """
        vertices_indexes = None
        for field in filters.keys():
            field_vertices = set()
            values = filters[field] if isinstance(
                filters[field], (list, tuple)) else [filters[field]]
            for value in values:
                vertices = [v.index for v in graph.vs.select(**{field: value})]
                field_vertices = field_vertices.union(vertices)
            if vertices_indexes is None:
                vertices_indexes = field_vertices
            else:
                vertices_indexes = vertices_indexes.intersection(
                    field_vertices)
        vertices_indexes = vertices_indexes if vertices_indexes is not None else [
            v.index for v in graph.vs
        ]
        if self.front_facing:
            front_facing_vertices = [
                v.index for v in graph.vs.select(front_facing='Y')
            ]
            vertices_indexes = set(vertices_indexes).intersection(
                front_facing_vertices)
        return list(vertices_indexes)

    @staticmethod
    def validate_moves(moves, direction_data):
        """
        This function checks whether the distance between the anchor and the tested SKUs fits the requirements.
        """
        direction_data = direction_data if isinstance(
            direction_data, (list, tuple)) else [direction_data]
        validated = False
        for data in direction_data:
            data_validated = True
            for direction in moves.keys():
                allowed_moves = data.get(direction, (0, 0))
                min_move, max_move = allowed_moves if isinstance(
                    allowed_moves, tuple) else (0, allowed_moves)
                if not min_move <= moves[direction] <= max_move:
                    data_validated = False
                    break
            if data_validated:
                validated = True
                break
        return validated

    @staticmethod
    def validate_block_moves(moves, direction_data):
        """
        This function checks whether the distance between the anchor and the tested SKUs fits the requirements.
        """
        direction_data = direction_data if isinstance(
            direction_data, (list, tuple)) else [direction_data]
        one_to_pass = {}
        for data in direction_data:
            for direction in data.keys():
                allowed_moves = data.get(direction, (0, 0))
                min_move, max_move = allowed_moves if isinstance(
                    allowed_moves, tuple) else (0, allowed_moves)
                if min_move <= moves[direction] <= max_move:
                    one_to_pass[direction] = 'T'
        if one_to_pass:
            if len(one_to_pass.values()) == 1:
                moves.pop(one_to_pass.keys()[0])
                for direction in moves.keys():
                    min_move, max_move = (0, 0)
                    if not min_move <= moves[direction] <= max_move:
                        return False
                return True
        return False

    def get_product_unique_position_on_shelf(self,
                                             scene_id,
                                             shelf_number,
                                             include_empty=False,
                                             **filters):
        """
        :param scene_id: The scene ID.
        :param shelf_number: The number of shelf in question (from top).
        :param include_empty: This dictates whether or not to include empties as valid positions.
        :param filters: These are the parameters which the unique position is checked for.
        :return: The position of the first SKU (from the given filters) to appear in the specific shelf.
        """
        shelf_matches = self.match_product_in_scene[
            (self.match_product_in_scene['scene_fk'] == scene_id)
            & (self.match_product_in_scene['shelf_number'] == shelf_number)]
        if not include_empty:
            filters['product_type'] = ('Empty', self.EXCLUDE_FILTER)
        if filters and shelf_matches[self.get_filter_condition(
                shelf_matches, **filters)].empty:
            Log.info(
                "Products of '{}' are not tagged in shelf number {}".format(
                    filters, shelf_number))
            return None
        shelf_matches = shelf_matches.sort_values(
            by=['bay_number', 'facing_sequence_number'])
        shelf_matches = shelf_matches.drop_duplicates(
            subset=['product_ean_code'])
        positions = []
        for m in xrange(len(shelf_matches)):
            match = shelf_matches.iloc[m]
            match_name = 'Empty' if match[
                'product_type'] == 'Empty' else match['product_ean_code']
            if positions and positions[-1] == match_name:
                continue
            positions.append(match_name)
        return positions

    def calculate_block_together(self,
                                 allowed_products_filters=None,
                                 include_empty=EXCLUDE_EMPTY,
                                 minimum_block_ratio=1,
                                 result_by_scene=False,
                                 vertical=False,
                                 horizontal=False,
                                 **filters):
        """
        :param group_products: if we searching for group in block - this is the filter of the group inside the big block
        :param block_products: if we searching for group in block - this is the filter of the big block
        :param group: True if the kpi is for block of product inside a bigger block
        :param orphan: True if searching for orphan products 3 products away from the biggest block
        :param horizontal: True if the biggest block have to be at least 2X2
        :param vertical: True
        :param allowed_products_filters: These are the parameters which are allowed to corrupt the block without failing it.
        :param include_empty: This parameter dictates whether or not to discard Empty-typed products.
        :param minimum_block_ratio: The minimum (block number of facings / total number of relevant facings) ratio
                                    in order for KPI to pass (if ratio=1, then only one block is allowed).
        :param result_by_scene: True - The result is a tuple of (number of passed scenes, total relevant scenes);
                                False - The result is True if at least one scene has a block, False - otherwise.
        :param filters: These are the parameters which the blocks are checked for.
        :return: see 'result_by_scene' above.
        """
        filters, relevant_scenes = self.separate_location_filters_from_product_filters(
            **filters)
        if len(relevant_scenes) == 0:
            if result_by_scene:
                return 0, 0
            else:
                Log.debug(
                    'Block Together: No relevant SKUs were found for these filters {}'
                    .format(filters))
                return True
        number_of_blocked_scenes = 0
        cluster_ratios = []
        for scene in relevant_scenes:
            scene_graph = self.position_graphs.get(scene).copy()

            relevant_vertices = set(
                self.filter_vertices_from_graph(scene_graph, **filters))
            if allowed_products_filters:
                allowed_vertices = self.filter_vertices_from_graph(
                    scene_graph, **allowed_products_filters)
            else:
                allowed_vertices = set()

            if include_empty == self.EXCLUDE_EMPTY:
                empty_vertices = {
                    v.index
                    for v in scene_graph.vs.select(product_type='Empty')
                }
                allowed_vertices = set(allowed_vertices).union(empty_vertices)

            all_vertices = {v.index for v in scene_graph.vs}
            vertices_to_remove = all_vertices.difference(
                relevant_vertices.union(allowed_vertices))
            scene_graph.delete_vertices(vertices_to_remove)
            # removing clusters including 'allowed' SKUs only
            clusters = [
                cluster for cluster in scene_graph.clusters()
                if set(cluster).difference(allowed_vertices)
            ]
            new_relevant_vertices = self.filter_vertices_from_graph(
                scene_graph, **filters)
            for cluster in clusters:
                relevant_vertices_in_cluster = set(cluster).intersection(
                    new_relevant_vertices)
                if len(new_relevant_vertices) > 0:
                    cluster_ratio = len(relevant_vertices_in_cluster) / float(
                        len(new_relevant_vertices))
                else:
                    cluster_ratio = 0
                cluster_ratios.append(cluster_ratio)
                if cluster_ratio > minimum_block_ratio:
                    all_vertices = {v.index for v in scene_graph.vs}
                    non_cluster_vertices = all_vertices.difference(cluster)
                    block_graph = scene_graph
                    block_graph.delete_vertices(non_cluster_vertices)
                    direction_data = {'top': (1, 1), 'bottom': (1, 1)}
                    result_v = self.calculate_relative_in_block_per_graph(
                        block_graph, direction_data)
                    direction_data = {'left': (2, 2), 'right': (2, 2)}
                    result_h = self.calculate_relative_in_block_per_graph(
                        block_graph, direction_data)
                    if result_h and result_v:
                        return True
        if result_by_scene:
            return number_of_blocked_scenes, len(relevant_scenes)
        else:
            if minimum_block_ratio == 1:
                return False
            elif cluster_ratios:
                return False
            else:
                return False

    def calculate_relative_in_block_per_graph(self,
                                              scene_graph,
                                              direction_data,
                                              min_required_to_pass=1,
                                              sent_tested_vertices=None,
                                              sent_anchor_vertices=None):
        """
        :param direction_data: The allowed distance between the tested and anchor SKUs.
                               In form: {'top': 4, 'bottom: 0, 'left': 100, 'right': 0}
                               Alternative form: {'top': (0, 1), 'bottom': (1, 1000), ...} - As range.
        :param min_required_to_pass: The number of appearances needed to be True for relative position in order for KPI
                                     to pass. If all appearances are required: ==a string or a big number.
        :param general_filters: These are the parameters which the general data frame is filtered by.
        :return: True if (at least) one pair of relevant SKUs fits the distance requirements; otherwise - returns False.
        """
        pass_counter = 0
        reject_counter = 0
        if sent_anchor_vertices and sent_tested_vertices:
            tested_vertices = sent_tested_vertices
            anchor_vertices = sent_anchor_vertices
        else:
            tested_vertices = self.filter_vertices_from_graph(scene_graph)
            anchor_vertices = self.filter_vertices_from_graph(scene_graph)
        for tested_vertex in tested_vertices:
            for anchor_vertex in anchor_vertices:
                moves = {'top': 0, 'bottom': 0, 'left': 0, 'right': 0}
                path = scene_graph.get_shortest_paths(anchor_vertex,
                                                      tested_vertex,
                                                      output='epath')
                if path:
                    path = path[0]
                    for edge in path:
                        moves[scene_graph.es[edge]['direction']] += 1
                    if self.validate_block_moves(moves, direction_data):
                        pass_counter += 1
                        if isinstance(
                                min_required_to_pass,
                                int) and pass_counter >= min_required_to_pass:
                            return True
                    else:
                        reject_counter += 1
        if pass_counter > 0 and reject_counter == 0:
            return True
        else:
            return False

    def get_filter_condition(self, df, **filters):
        """
        :param df: The Data frame to be filters.
        :param filters: These are the parameters which the Data frame is filtered by.
                       Every parameter would be a tuple of the value and an include/exclude flag.
                       INPUT EXAMPLE (1):   manufacturer_name = ('Diageo', DIAGEOAUMSCGENERALToolBox.INCLUDE_FILTER)
                       INPUT EXAMPLE (2):   manufacturer_name = 'Diageo'
        :return: a filtered Scene Item Facts Data frame.
        """
        if not filters:
            return df['pk'].apply(bool)
        if self.facings_field in df.keys():
            filter_condition = (df[self.facings_field] > 0)
        else:
            filter_condition = None
        for field in filters.keys():
            if field in df.keys():
                if isinstance(filters[field], tuple):
                    value, exclude_or_include = filters[field]
                else:
                    value, exclude_or_include = filters[
                        field], self.INCLUDE_FILTER
                if not value:
                    continue
                if not isinstance(value, list):
                    value = [value]
                if exclude_or_include == self.INCLUDE_FILTER:
                    condition = (df[field].isin(value))
                elif exclude_or_include == self.EXCLUDE_FILTER:
                    condition = (~df[field].isin(value))
                elif exclude_or_include == self.CONTAIN_FILTER:
                    condition = (df[field].str.contains(value[0], regex=False))
                    for v in value[1:]:
                        condition |= df[field].str.contains(v, regex=False)
                else:
                    continue
                if filter_condition is None:
                    filter_condition = condition
                else:
                    filter_condition &= condition
            else:
                Log.warning('field {} is not in the Data Frame'.format(field))

        return filter_condition

    def separate_location_filters_from_product_filters(self, **filters):
        """
        This function gets scene-item-facts filters of all kinds, extracts the relevant scenes by the location filters,
        and returns them along with the product filters only.
        """
        location_filters = {}
        for field in filters.keys():
            if field not in self.all_products.columns and field in self.scif.columns:
                location_filters[field] = filters.pop(field)
        relevant_scenes = self.scif[self.get_filter_condition(
            self.scif, **location_filters)]['scene_id'].unique()
        return filters, relevant_scenes

    @staticmethod
    def get_json_data(file_path, sheet_name=None, skiprows=0):
        """
        This function gets a file's path and extract its content into a JSON.
        """
        data = {}
        if sheet_name:
            sheet_names = [sheet_name]
        else:
            sheet_names = xlrd.open_workbook(file_path).sheet_names()
        for sheet_name in sheet_names:
            try:
                output = pd.read_excel(file_path,
                                       sheetname=sheet_name,
                                       skiprows=skiprows)
            except xlrd.biffh.XLRDError:
                Log.warning('Sheet name {} doesn\'t exist'.format(sheet_name))
                return None
            output = output.to_json(orient='records')
            output = json.loads(output)
            for x in xrange(len(output)):
                for y in output[x].keys():
                    output[x][y] = unicode(
                        '' if output[x][y] is None else output[x][y]).strip()
                    if not output[x][y]:
                        output[x].pop(y, None)
            data[sheet_name] = output
        if sheet_name:
            data = data[sheet_name]
        elif len(data.keys()) == 1:
            data = data[data.keys()[0]]
        return data