Exemplo n.º 1
0
    def test_crud_get_user_data_get_user_data(self):

        crud = CRUD(True)
        crud.add_new_user("*****@*****.**", "2020-11-16")

        # get_user_data
        trust1 = crud.get_user_data(1, 'Trust')
        self.assertEqual(trust1, 50)

        # get_user_data
        trust1 = crud.get_user_data(1, 'Trust')
        self.assertEqual(trust1, 50)
Exemplo n.º 2
0
    def test_crud_update_users_update_users(self):

        crud = CRUD(True)
        crud.add_new_user("*****@*****.**", "2020-11-16")

        # update_users
        crud.update_users(1, "Trust", 0)
        self.assertEqual(crud.get_user_data(1, 'Trust'), 0)

        # update_users
        crud.update_users(1, 'Trust', 100)
        self.assertEqual(crud.get_user_data(1, 'Trust'), 100)
Exemplo n.º 3
0
    def test_crud_remove_user_group_get_user_data(self):

        crud = CRUD(True)
        crud.add_new_user("*****@*****.**", "2020-11-16")

        # remove_user_group
        crud.remove_user_group(1, 'default')
        self.assertEqual(crud.get_user_data(1, 'Groups'), [])

        # get_user_data
        trust1 = crud.get_user_data(1, 'Trust')
        self.assertEqual(trust1, 50)
Exemplo n.º 4
0
    def test_crud_remove_user_group_update_users(self):

        crud = CRUD(True)
        crud.add_new_user("*****@*****.**", "2020-11-16")

        # remove_user_group
        crud.remove_user_group(1, 'default')
        self.assertEqual(crud.get_user_data(1, 'Groups'), [])

        # update_users
        crud.update_users(1, "Trust", 0)
        self.assertEqual(crud.get_user_data(1, 'Trust'), 0)
Exemplo n.º 5
0
    def test_get_user_data_Returns_false_for_invalid_field(self, mock_read_users_file):
        mock_read_users_file.return_value=self.users_data
        user_id=3
        field="Test"

        crud=CRUD()
        self.assertEqual(crud.get_user_data(user_id,field),False)
Exemplo n.º 6
0
 def test_get_user_data_Returns_correct_value_if_field_and_id_are_valid(
         self, mock_read_users_file):
     fieldValue = 0
     mock_read_users_file.return_value = self.users_data
     crud = CRUD()
     self.assertEqual(crud.get_user_data("1", "SpamN"), fieldValue)
     """Description: il faut utiliser le mock de fonction "read_groups_file",
Exemplo n.º 7
0
    def test_get_user_data_Returns_false_for_invalid_id(
            self, mock_read_users_file):

        mock_read_users_file.return_value = self.users_data
        crud = CRUD()
        self.assertFalse(crud.get_user_data("3", "name"))
        """Description: il faut utiliser le mock de fonction "read_groups_file",
Exemplo n.º 8
0
 def test_get_user_data_Returns_correct_value_if_field_and_id_are_valid(
     self, mock_read_users_file
 ):
     mock_read_users_file.return_value=self.users_data
     crud=CRUD()
     user_id="2"
     field="SpamN"
     self.assertEqual(crud.get_user_data(user_id,field),171)
Exemplo n.º 9
0
    def test_get_user_data_Returns_false_for_invalid_id(
            self, mock_read_users_file):

        testMessage = "Invalide"
        mock_read_users_file.return_value = self.users_data
        crud = CRUD()
        self.assertFalse(crud.get_user_data("10", "SpamN"), testMessage)
        """Description: il faut utiliser le mock de fonction "read_groups_file",
Exemplo n.º 10
0
 def test_get_group_data_Returns_correct_value_if_field_and_id_are_valid(
         self, mock_read_users_file, mock_read_groups_file):
     fieldValue = 50
     mock_read_users_file.return_value = self.users_data
     mock_read_groups_file.return_value = self.groups_data
     crud = CRUD()
     self.assertEqual(crud.get_user_data("1", "Trust"), fieldValue)
     """"""
     pass
Exemplo n.º 11
0
    def test_crud_add_new_user_update_users(self):

        crud = CRUD(True)

        # add_new_user
        crud.add_new_user("*****@*****.**", "2020-11-16")
        self.assertEqual(crud.read_users_file().get('1'), self.user_2)

        # update_users
        crud.update_users(1, "Trust", 0)
        self.assertEqual(crud.get_user_data(1, 'Trust'), 0)
Exemplo n.º 12
0
    def test_crud_add_new_user_get_user_data(self):

        crud = CRUD(True)

        # add_new_user
        crud.add_new_user("*****@*****.**", "2020-11-16")
        self.assertEqual(crud.read_users_file().get('1'), self.user_2)

        # get_user_data
        trust1 = crud.get_user_data(1, 'Trust')
        self.assertEqual(trust1, 50)
Exemplo n.º 13
0
    def test_crud_add_new_user_remove_user_group(self):

        crud = CRUD(True)

        # add_new_user
        crud.add_new_user("*****@*****.**", "2020-11-16")
        self.assertEqual(crud.read_users_file().get('1'), self.user_2)

        # remove_user_group
        crud.remove_user_group(1, 'default')
        self.assertEqual(crud.get_user_data(1, 'Groups'), [])
Exemplo n.º 14
0
    def test_crud_update_users_remove_user(self):

        crud = CRUD(True)
        crud.add_new_user("*****@*****.**", "2020-11-16")

        crud.update_users(1, "Trust", 0)
        self.assertEqual(crud.get_user_data(1, 'Trust'), 0)

        # remove_user
        self.assertEqual(crud.remove_user(1), True)
        self.assertEqual(crud.read_users_file(), {})
Exemplo n.º 15
0
    def test_crud_get_user_data_remove_user(self):

        crud = CRUD(True)
        crud.add_new_user("*****@*****.**", "2020-11-16")

        # get_user_data
        trust1 = crud.get_user_data(1, 'Trust')
        self.assertEqual(trust1, 50)

        # remove_user
        self.assertEqual(crud.remove_user(1), True)
        self.assertEqual(crud.read_users_file(), {})
Exemplo n.º 16
0
    def test_crud_remove_user_group_remove_user(self):

        crud = CRUD(True)
        crud.add_new_user("*****@*****.**", "2020-11-16")

        # remove_user_group
        crud.remove_user_group(1, 'default')
        self.assertEqual(crud.get_user_data(1, 'Groups'), [])

        # remove_user
        self.assertEqual(crud.remove_user(1), True)
        self.assertEqual(crud.read_users_file(), {})
Exemplo n.º 17
0
    def test_crud_remove_user_group_add_new_user(self):

        crud = CRUD(True)
        crud.add_new_user("*****@*****.**", "2020-11-16")

        # remove_user_group
        crud.remove_user_group(1, 'default')
        self.assertEqual(crud.get_user_data(1, 'Groups'), [])

        # add_new_user
        self.assertEqual(crud.add_new_user("*****@*****.**", "2020-11-16"), True)
        self.assertEqual(crud.read_users_file().get('2'), self.user)
Exemplo n.º 18
0
    def test_crud_get_user_data_add_new_user(self):

        crud = CRUD(True)
        crud.add_new_user("*****@*****.**", "2020-11-16")

        # get_user_data
        trust1 = crud.get_user_data(1, 'Trust')
        self.assertEqual(trust1, 50)

        # add_new_user
        self.assertEqual(crud.add_new_user("*****@*****.**", "2020-11-16"), True)
        self.assertEqual(crud.read_users_file().get('1'), self.user_2)
Exemplo n.º 19
0
class RENEGE:
    """Class pour realiser le filtrage du spam en utilisant vocabular.json file et
    CRUD et EmalAnalyze classes"""
    def __init__(self):
        self.email_file = "train_set.json"
        self.crud = CRUD()
        self.e_mail = EmailAnalyzer()

    def calculate_user_trust(self, user_id):
        #json data
        date_of_first_seen_message = self.crud.get_user_data(
            user_id, "Date_of_first_seen_message")
        date_of_last_seen_message = self.crud.get_user_data(
            user_id, "Date_of_last_seen_message")
        n_ham = self.crud.get_user_data(user_id, "HamN")
        n_spam = self.crud.get_user_data(user_id, "SpamN")
        user_name = self.crud.get_user_data(user_id, "name")
        groups = self.crud.read_groups_file()

        sum_trust = 0
        n_groups = 0

        # find nb of groups to which user belongs
        for group in groups.values():
            if user_name in group["List_of_members"]:
                sum_trust += group["Trust"]
                n_groups += 1

        trust1 = (date_of_last_seen_message *
                  n_ham) / (date_of_first_seen_message * (n_ham + n_spam))
        trust2 = 0
        # to avoid 'division by zero' error
        if n_groups != 0:
            trust2 = sum_trust / n_groups

        trust = (trust1 + trust2) / 2

        if trust2 < 50:
            trust = trust2
        if trust1 > 100:
            trust = 100

        if trust > 100:
            trust = 100
        elif trust < 0:
            trust = 0

        return trust

    def classify_emails(self):
        '''
        Description: fonction pour commencer l'analyse des e-mails.
        Sortie: bool, 'True' pour succes, 'False' dans le cas de failure.
        '''
        try:
            self.process_email(self.get_email())
            return True
        except Exception as e:
            print("Error!", e.__class__, "occurred.")
            raise e
            return False

    def process_email(self, new_emails):
        '''
        Description: fonction pour analyser chaque nouvel e-mail dans le 
        dictionare. Elle gere l'ajout des nouveux utilisateurs et/ou modification
        de l'information existante sur les utilisateurs et groupes. 
        Sortie: bool, 'True' pour succes, 'False' dans le cas de failure.
        '''
        emails = self.get_email()
        print("Processing emails")
        i = 0
        email_count = len(emails["dataset"])
        # Load emails
        for email in emails["dataset"]:
            i += 1
            print("\rEmail " + str(i) + "/" + str(email_count), end="")

            data = email["mail"]
            subject = data["Subject"]
            name = data["From"]
            date = data["Date"]
            body = data["Body"]
            is_spam = data["Spam"]

            # Get registered data
            user_id = -1
            try:
                user_id = self.crud.get_user_id(name)
            except RuntimeError:
                # Create the user
                if not self.crud.add_new_user(name, date):
                    return False

                user_id = self.crud.get_user_id(name)

            # Update user's emails info
            if is_spam == "true":
                if not self.update_user_info(user_id, date, 1, 0):
                    return False
            else:
                if not self.update_user_info(user_id, date, 0, 1):
                    return False

            # Update groups data
            groups = self.crud.get_user_data(user_id, "Groups")
            for group_name in groups:
                try:
                    group_id = self.crud.get_group_id(group_name)
                    if not self.update_group_info(group_id, user_id):
                        return False

                except RuntimeError:
                    return False

        print("\n")

        return True

    def update_user_info(self, user_id, new_user_date, new_email_spam,
                         new_email_ham):
        '''
        Description: fonction pour modifier l'information de utilisateur (date de dernier message arrive,
        numero de spam/ham, trust level, etc).
        Sortie: bool, 'True' pour succes, 'False' dans le cas de failure.
        '''

        # Update last / first seen date
        new_date = self.crud.convert_to_unix(new_user_date)
        if new_date > self.crud.get_user_data(user_id,
                                              "Date_of_last_seen_message"):
            if not self.crud.update_users(user_id, "Date_of_last_seen_message",
                                          new_user_date):
                return False
        elif new_date < self.crud.get_user_data(user_id,
                                                "Date_of_first_seen_message"):
            if not self.crud.update_users(
                    user_id, "Date_of_first_seen_message", new_user_date):
                return False

        # Update trust score
        spam_n = self.crud.get_user_data(user_id, "SpamN") + new_email_spam
        ham_n = self.crud.get_user_data(user_id, "HamN") + new_email_ham

        trust_lvl = 50
        if (spam_n + ham_n) != 0:
            trust_lvl = ham_n / (spam_n + ham_n) * 100
            if trust_lvl > 100:
                trust_lvl = 100

        if not self.crud.update_users(user_id, "SpamN", spam_n):
            return False

        if not self.crud.update_users(user_id, "HamN", ham_n):
            return False

        return self.crud.update_users(user_id, "Trust", trust_lvl)

    def update_group_info(self, group_id, user_id):
        '''
        Description: fonction pour modifier l'information de groupe dans lequel 
        l'utilisater est present (trust level, etc).
        Sortie: bool, 'True' pour succes, 'False' dans le cas de failure.
        '''
        try:
            # Get list of users and update it
            users_list = self.crud.get_groups_data(group_id, "List_of_members")
            user_name = self.crud.get_user_data(user_id, "name")
            if user_name not in users_list:
                users_list.append(user_name)

            # Get data for trust update
            user_count = len(users_list)
            trust_lvl = 0

            # Compute group's trust
            for user in users_list:
                curr_user_id = self.crud.get_user_id(user)
                trust_lvl += self.crud.get_user_data(curr_user_id, "Trust")

            if (trust_lvl > 100):
                trust_lvl = 100

            # Update the group with the new trust level and the new member list
            if self.crud.update_groups(group_id, "Trust", trust_lvl):
                return self.crud.update_groups(group_id, 'List_of_members',
                                               users_list)

            return False
        except RuntimeError:
            return False

    def get_user_email_list(self):
        '''
        Description: fonction pour creer le liste des e-mails (noms) 
        des utilisateurs uniques.
        Sortie: liste des uniques e-mails des utilisateurs
        '''
        emails = []
        for user in self.crud.users_data:
            emails.append(user["name"])
        return emails

    def get_email(self):
        '''
        Description: fonction pour lire le ficher json avec les mails et extraire les 
        donees necessaire.
        Sortie: dictionare de e-mails formate selon le JSON.
        '''
        with open(self.email_file) as email_file:
            return json.load(email_file)
Exemplo n.º 20
0
class RENEGE:
    """Class pour realiser le filtrage du spam en utilisant vocabular.json file et
    CRUD et EmalAnalyze classes"""

    def __init__(self):
        self.email_file = "800-mails.json"
        self.crud = CRUD()
        self.e_mail = EmailAnalyzer()

    def calculate_user_trust(self, user_id):
        #extracting json data
        date_of_first_seen_message = self.crud.get_user_data(user_id, "Date_of_first_seen_message")
        date_of_last_seen_message = self.crud.get_user_data(user_id, "Date_of_last_seen_message")
        n_ham = self.crud.get_user_data(user_id, "HamN")
        n_spam = self.crud.get_user_data(user_id, "SpamN")
        groups = self.crud.get_user_data(user_id, "Groups")

        #calculate the sum of trust values of all groups
        sum_trust = 0
        for group in groups :
            group_id = self.crud.get_group_id(group)
            sum_trust += self.crud.get_group_data(group_id, 'Trust')

        #now that we have all the needed vars, calculate trust1, trust2 and trust
        trust1 = (date_of_last_seen_message * n_ham) / (date_of_first_seen_message * (n_ham + n_spam))
        trust2 = sum_trust / len(groups)

        trust = (trust1 + trust2) / 2

        if trust2 < 50:
            trust = trust2
        if trust1 > 100:
            trust = 100

        #before returning a value, check if trust is between 0 and 100
        if trust < 0:
            trust = 0
        elif trust > 100:
            trust = 100
            
        return trust

    def classify_emails(self, calculation_mode, is_log_est, is_log_combo):
        '''
        fonction deja implemente
        Description: fonction pour commencer l'analyse des e-mails.
        Sortie: bool, 'True' pour succes, 'False' dans le cas de failure.
        '''
        try:
            self.process_email(self.get_email(), calculation_mode, is_log_est, is_log_combo)
            return True
        except Exception:
            raise Exception

    def process_email(self, new_emails, calculation_mode, is_log_est, is_log_combo):
        '''
        Description: fonction pour analyser chaque nouvel e-mail dans le 
        dictionare. Elle gere l'ajout des nouveux utilisateurs et/ou modification
        de l'information existante sur les utilisateurs et groupes.
        Sortie: bool, 'True' pour succes, 'False' dans le cas de failure.
        '''
        print(type(new_emails))
        emails = new_emails["dataset"]
        for email in emails:
            email_adr = email['mail']['From']
            date = email['mail']['Date']
            spam = email['mail']['Spam'] == 'true'
            subject = email['mail']['Subject']
            body = email['mail']['Body']
            user_id = self.crud.get_user_id(email_adr)

            if user_id:
                self.update_user_info(email_adr, date, spam)
            else:
                self.add_user_info(email_adr, date)
            
            # trust de l'utilisateur
            trust = self.crud.get_user_data(user_id, "Trust")
            # moyenne du trust de tous les groupes
            user_group = self.crud.get_user_data(user_id, "Groups")
            sum_trust = 0
            groups = self.crud.read_groups_file()
            for group in groups :
                if group['List_of_members'] == user_group:
                    sum_trust += group['Trust']
            avg_group_trust = sum_trust / len(groups)
            # difference de jours entre last_seen_msg et first_seen_msg
            user_activity = self.substract_dates(self.crud.get_user_data(email_adr, "Date_of_last_seen_message"), 
            self.crud.get_user_data(email_adr, "Date_of_first_seen_message"))

            if(calculation_mode == 1):
                spam = self.e_mail.is_spam_function_one(spam, user_activity, trust, avg_group_trust)
            elif(calculation_mode == 2):
                spam = self.e_mail.is_spam_function_two(spam, trust, avg_group_trust)
            elif(calculation_mode == 0):
                spam = self.e_mail.is_spam(subject, body, is_log_est, is_log_combo, 0)

            # mettre a jour l'utilisateur avec la nouvelle valeur de spam
            self.update_user_info(email_adr, date, spam)

        return True
    
    def substract_dates(self, last_seen_msg, first_seen_msg):
        # calcul de difference de jours entre le dernier et le premier message
        last_seen_msg = str(last_seen_msg).split("-")
        first_seen_msg = str(first_seen_msg).split("-")
        user_activity = date(int(last_seen_msg[0]), int(last_seen_msg[1]), int(last_seen_msg[2]))
                        - date(int(first_seen_msg[0]), int(first_seen_msg[1]), int(first_seen_msg[2]))
        # convertir la soustraction en int
        user_activity = int((str(user_activity).split(" "))[0])

        return user_activity
Exemplo n.º 21
0
class RENEGE:
    """Class pour realiser le filtrage du spam en utilisant vocabular.json file et
    CRUD et EmalAnalyze classes"""
    def __init__(self):
        self.email_file = "1000-mails.json"
        self.crud = CRUD()
        self.e_mail = EmailAnalyzer()

    def calculate_user_trust(self, user_id):
        '''
        Description: fonction a implementer pour la deuxieme partie du
        deuxieme travail pratique. Permet de calculer le trust d'un
        utilisateur specifique

        Sortie: int; Retourne la valeur du trust de l'utilisateur
        '''
        # calculation for Trust1
        nb_spam = self.crud.get_user_data(user_id, "SpamN")
        nb_ham = self.crud.get_user_data(user_id, "HamN")

        time_first_seen_message = self.crud.get_user_data(
            user_id, "Date_of_first_seen_message")
        time_last_seen_message = self.crud.get_user_data(
            user_id, "Date_of_last_seen_message")

        if (time_first_seen_message * (nb_ham + nb_spam)) == 0:
            print("trust1 not possible, division by zero")
            return

        trust1 = ((time_last_seen_message * nb_ham) /
                  (time_first_seen_message * (nb_ham + nb_spam)))

        # calculation for Trust2
        group_list = self.crud.read_groups_file()

        # in order to determine if user is part of a group, we need their email
        user_email = self.crud.get_user_data(user_id, "name")

        # number of groups that the user is in
        nb_group = 0
        user_trust_total = 0

        for group in group_list.values():
            if user_email in group["List_of_members"]:
                nb_group += 1
                user_trust_total += group["Trust"]

        trust2 = user_trust_total / nb_group

        # determining user's final trust value
        if trust2 < 50:
            return trust2
        elif trust1 > 100:
            return 100

        final_trust = (trust1 + trust2) / 2

        if final_trust < 0:
            final_trust = 0
        elif final_trust > 100:
            final_trust = 100

        return final_trust

    def classify_emails(self, spam_definition_mode):
        '''
        fonction deja implemente
        Description: fonction pour commencer l'analyse des e-mails.
        Sortie: bool, 'True' pour succes, 'False' dans le cas de failure.
        '''
        try:
            self.process_email(self.get_email(), spam_definition_mode)
            return True
        except Exception as e:
            print("Error!", e.__class__, "occurred.")
            return False

    def process_email(self, new_emails, mode):
        '''
        Description: fonction pour analyser chaque nouvel e-mail dans le 
        dictionare. Elle gere l'ajout des nouveux utilisateurs et/ou modification
        de l'information existante sur les utilisateurs et groupes. 
        Sortie: bool, 'True' pour succes, 'False' dans le cas de failure.
        '''
        list_of_email_addresses = self.get_user_email_list()

        for user in list_of_email_addresses:
            # date de defaut est l'epoch linux
            self.crud.add_new_user(user, "1970-01-01")

        self.crud.add_new_group('default', 50, list_of_email_addresses)

        is_spam = ''

        for email in new_emails["dataset"]:
            individual_email = email["mail"]
            email_address = individual_email["From"]
            date = time.mktime(
                datetime.datetime.strptime(individual_email["Date"],
                                           "%Y-%m-%d").timetuple())

            if mode == 0:
                is_spam = individual_email["Spam"]
            elif mode == 1:
                is_spam = self.is_spam1(email)
            elif mode == 2:
                is_spam = self.is_spam2(email)

            self.update_user_info(email_address, date, is_spam)

        self.update_group_info(list_of_email_addresses, 'default')

    def is_spam1(self, email):
        '''
        Description: fonction qui calcule la probabilité de spam selon l'équation 1 de
        l'énoncé, soit S = P ∗ (H ∗ T1 + T2) + H ∗ T2 ∗ ¬T3
        '''
        user_id = self.crud.get_user_id(email["From"])
        trust = self.crud.get_user_data(user_id, "Trust")
        first = self.crud.get_user_data(user_id, "Date_of_first_seen_message")
        last = self.crud.get_user_data(user_id, "Date_of_last_seen_message")

        p = email["Spam"] == "true"
        h = (float(last) - float(first)) / (60 * 60 * 24) > 31
        t1 = trust < 60
        t2 = trust < 70
        t3 = trust > 75

        return (p and ((h and t1) or t2)) or (h and t2 and not t3)

    def is_spam2(self, email):
        '''
        Description: fonction qui calcule la probabilité de spam selon l'équation 2 de
        l'énoncé, soit S = P + ¬T3 ∗ T2
        '''

        user_id = self.crud.get_user_id(email["From"])
        trust = self.crud.get_user_data(user_id, "Trust")

        p = email["Spam"] == "true"
        t2 = trust < 70
        t3 = trust > 75

        return p or (not t3 and t2)

    def update_user_info(self, new_user_email, new_user_date, new_email_spam):
        '''
        Description: fonction pour modifier l'information de utilisateur (date de dernier message arrive,
        numero de spam/ham, trust level, etc).
        Sortie: bool, 'True' pour succes, 'False' dans le cas de failure.
        '''

        user_id = self.crud.get_user_id(new_user_email)

        date_first_seen_message = self.crud.get_user_data(
            user_id, "Date_of_first_seen_message")
        date_last_seen_message = self.crud.get_user_data(
            user_id, "Date_of_last_seen_message")

        try:
            if new_user_date < date_first_seen_message:
                self.crud.update_users(user_id, "Date_of_first_seen_message",
                                       new_user_date)
            elif new_user_date > date_last_seen_message:
                self.crud.update_users(user_id, "Date_of_last_seen_message",
                                       new_user_date)

            # Mise a jour des spamN/hamN et trust des utilisateurs
            user_spamN = self.crud.get_user_data(user_id, "SpamN")
            user_hamN = self.crud.get_user_data(user_id, "HamN")
            if new_email_spam == "true":
                user_spamN = user_spamN + 1
                self.crud.update_users(user_id, "SpamN", user_spamN)
            else:
                user_hamN = user_hamN + 1
                self.crud.update_users(user_id, "HamN", user_hamN)

            total_msg = user_spamN + user_hamN
            self.crud.update_users(user_id, "Trust",
                                   (user_hamN / total_msg) * 100)
            return True
        except:
            return False

    def update_group_info(self, user_group_list, group_name):
        '''
        Description: fonction pour modifier l'information de groupe dans lequel 
        l'utilisater est present (trust level, etc).
        Sortie: bool, 'True' pour succes, 'False' dans le cas de failure.
        '''
        try:
            trust_sum = 0
            for user in user_group_list:
                user_id = self.crud.get_user_id(user)
                trust_sum += self.crud.get_user_data(user_id, "Trust")

            # Calcul de la nouvelle moyenne de trust des membres
            group_trust = round(trust_sum / len(user_group_list), 0)
            group_id = self.crud.get_group_id(group_name)
            self.crud.update_groups(group_id, "Trust", group_trust)
            return True
        except:
            return False

    def get_user_email_list(self):
        '''
        Description: fonction pour creer le liste des e-mails (noms) 
        des utilisateurs uniques.
        Sortie: liste des uniques e-mails des utilisateurs
        '''
        email_dict = self.get_email()
        existing_email_address = set()
        unique_emails = []

        for email in email_dict["dataset"]:
            individual_email = email["mail"]
            email_address = individual_email["From"]

            if email_address in existing_email_address:
                continue

            existing_email_address.add(email_address)
            unique_emails.append(email_address)

        return unique_emails

    def get_email(self):
        '''
        Description: fonction pour lire le ficher json avec les mails et extraire les 
        donees necessaire.
        Sortie: dictionare de e-mails formate selon le JSON.
        '''
        with open(self.email_file) as email_file:
            return json.load(email_file)
Exemplo n.º 22
0
class RENEGE:
    """Class pour realiser le filtrage du spam en utilisant vocabular.json file et
    CRUD et EmalAnalyze classes"""

    def __init__(self):
        self.email_file = "800-mails.json"
        self.crud = CRUD()
        self.e_mail = EmailAnalyzer()

    def calculate_user_trust(self, user_id):
        #extracting json data
        date_of_first_seen_message = self.crud.get_user_data(user_id, "Date_of_first_seen_message")
        date_of_last_seen_message = self.crud.get_user_data(user_id, "Date_of_last_seen_message")
        n_ham = self.crud.get_user_data(user_id, "HamN")
        n_spam = self.crud.get_user_data(user_id, "SpamN")
        groups = self.crud.get_user_data(user_id, "Groups")

        #calculate the sum of trust values of all groups
        sum_trust = 0
        for group in groups :
            group_id = self.crud.get_group_id(group)
            sum_trust += self.crud.get_group_data(group_id, 'Trust')

        #now that we have all the needed vars, calculate trust1, trust2 and trust
        trust1 = (date_of_last_seen_message * n_ham) / (date_of_first_seen_message * (n_ham + n_spam))
        trust2 = sum_trust / len(groups)

        trust = (trust1 + trust2) / 2

        if trust2 < 50:
            trust = trust2
        if trust1 > 100:
            trust = 100

        #before returning a value, check if trust is between 0 and 100
        if trust < 0:
            trust = 0
        elif trust > 100:
            trust = 100
            
        return trust


    def classify_emails(self):
        '''
        fonction deja implemente
        Description: fonction pour commencer l'analyse des e-mails.
        Sortie: bool, 'True' pour succes, 'False' dans le cas de failure.
        '''
        try:
            self.process_email(self.get_email())
            return True
        except Exception as e:
            raise Exception
            return False

    def process_email(self, new_emails):
        '''
        Description: fonction pour analyser chaque nouvel e-mail dans le 
        dictionare. Elle gere l'ajout des nouveux utilisateurs et/ou modification
        de l'information existante sur les utilisateurs et groupes.
        Sortie: bool, 'True' pour succes, 'False' dans le cas de failure.
        '''
        print(type(new_emails))
        emails = new_emails["dataset"]
        for email in emails:
            email_adr = email['mail']['From']
            date = email['mail']['Date']
            spam = email['mail']['Spam']
            spam = spam == 'true' and True or False

            if self.crud.get_user_id(email_adr):
                self.update_user_info(email_adr, date, spam)
            else:
                self.add_user_info(email_adr, date)

        return True

    def update_user_info(self, new_user_email, new_user_date, new_email_spam):
        '''
        Description: fonction pour modifier l'information de utilisateur (date de dernier message arrive,
        numero de spam/ham, trust level, etc).
        Sortie: bool, 'True' pour succes, 'False' dans le cas de failure.
        '''
        users_dict = self.crud.read_users_file()

        for key in users_dict:
            user = users_dict.get(key)
            if user.get("name") == new_user_email:
                self.crud.update_users(key, 'Date_of_first_seen_message', new_user_date)

                if new_email_spam:
                    spamN = user.get('SpamN')
                    spamN += 1
                    trust = user.get('Trust')
                    if trust > 0:
                        trust -= 1
                    self.crud.update_users(key, 'SpamN', spamN)
                    self.crud.update_users(key, 'Trust', trust)
                else:
                    hamN = user.get('HamN')
                    hamN += 1
                    trust = user.get('Trust')
                    if trust < 100:
                        trust += 1
                    self.crud.update_users(key, 'HamN', hamN)
                    self.crud.update_users(key, 'HamN', trust)
                break

        return True

    def add_user_info(self, user_email, user_date):
        self.crud.add_new_user(user_email, user_date)

    def update_group_info(self, user_group_list):
        '''
        Description: fonction pour modifier l'information de groupe dans lequel 
        l'utilisater est present (trust level, etc).
        Sortie: bool, 'True' pour succes, 'False' dans le cas de failure.
        '''
        groups = self.crud.read_groups_file()

        for group in groups:
            if group['List_of_members'] == user_group_list:
                group['Trust'] = self.crud.get_user_data(user_group_list[1], 'Trust')
                return True

        return False

    def get_user_email_list(self):
        '''
        Description: fonction pour creer le liste des e-mails (noms) 
        des utilisateurs uniques.
        Sortie: liste des uniques e-mails des utilisateurs
        '''
        emails = self.get_email()
        list_of_email_addresses = []
        for mail in emails['dataset']:
            list_of_email_addresses.append((mail['mail']['From']))

        return list_of_email_addresses

    def get_email(self):
        '''
        Description: fonction pour lire le ficher json avec les mails et extraire les 
        donees necessaire.
        Sortie: dictionare de e-mails formate selon le JSON.
        '''

        with open(self.email_file) as file:
            emails = json.load(file)

        return emails
Exemplo n.º 23
0
class TestCRUD(unittest.TestCase):
    def setUp(self):
        self.crud = CRUD()
        # c'est un exemple de données "mock" à utiliser comme "return value" de read_users_file
        self.users_data = {
            "1": {
                "name": "*****@*****.**",
                "Trust": 100,
                "SpamN": 0,
                "HamN": 20,
                "Date_of_first_seen_message": 1596848266.0,
                "Date_of_last_seen_message": 1596848266.0,
                "Groups": ["default"],
            },
            "2": {
                "name": "*****@*****.**",
                "Trust": 65.45454,
                "SpamN": 171,
                "HamN": 324,
                "Date_of_first_seen_message": 1596855166.0,
                "Date_of_last_seen_message": 1596855166.0,
                "Groups": ["default"],
            },
        }
        # c'est un exemple de données "mock" à utiliser comme "return value" de read_groups_file
        self.groups_data = {
            "1": {
                "name": "default",
                "Trust": 50,
                "List_of_members": ["*****@*****.**", "*****@*****.**"],
            },
            "2": {
                "name": "friends",
                "Trust": 90,
                "List_of_members": ["*****@*****.**"],
            },
        }

    def tearDown(self):
        pass

    @patch("crud.CRUD.read_users_file")
    def test_add_new_user_returns_false_if_username_and_date_are_not_given(
            self, mock_read_users_file
    ):
        """Description: il faut utiliser les mocks des fonctions "read_users_file",
        pour vérifier que le nom d'utilisateur est donné à la fonction
        """
        mock_read_users_file.return_value = self.users_data
        add_new_user_return_value_two_none = self.crud.add_new_user(None, None)
        self.assertFalse(add_new_user_return_value_two_none)

    @patch("crud.CRUD.read_users_file")
    def test_add_new_user_returns_false_if_only_date_is_given(
            self, mock_read_users_file
    ):
        """Description: il faut utiliser les mocks des fonctions "read_users_file",
        pour vérifier que le nom d'utilisateur est donné à la fonction
        """
        mock_read_users_file.return_value = self.users_data
        add_new_user_return_value_one_none = self.crud.add_new_user(None, '2020-01-01')
        self.assertFalse(add_new_user_return_value_one_none)

    @patch("crud.CRUD.read_users_file")
    def test_add_new_user_returns_false_if_only_name_is_given(
            self, mock_read_users_file
    ):
        """Description: il faut utiliser les mocks des fonctions "read_users_file",
        pour vérifier que le nom d'utilisateur est donné à la fonction
        """
        mock_read_users_file.return_value = self.users_data
        add_new_user_return_value_one_none = self.crud.add_new_user('*****@*****.**', None)
        self.assertFalse(add_new_user_return_value_one_none)

    @patch("crud.CRUD.read_users_file")
    def test_add_new_user_returns_true_if_name_and_date_are_given(
            self, mock_read_users_file
    ):
        """Description: il faut utiliser les mocks des fonctions "read_users_file",
        pour vérifier que le nom d'utilisateur est donné à la fonction
        """
        mock_read_users_file.return_value = self.users_data
        add_new_user_return_value_correct_vals = self.crud.add_new_user('*****@*****.**', '2020-01-01')
        self.assertTrue(add_new_user_return_value_correct_vals)

    @patch("crud.CRUD.read_groups_file")
    def test_add_new_group_returns_false_if_name_is_too_short(
            self, mock_read_groups_file
    ):
        """Description: il faut utiliser les mocks des fonctions "read_groups_file",pour tester que
        l'information a ajouter pour le groupe a étée formée correctement
        """
        mock_read_groups_file.return_value = self.groups_data
        add_new_group_no_name = self.crud.add_new_group("", 50, ['dummyUser1', 'dummyUser2'])

        self.assertFalse(add_new_group_no_name)

    @patch("crud.CRUD.read_groups_file")
    def test_add_new_group_returns_false_if_name_is_too_long(
            self, mock_read_groups_file
    ):
        """Description: il faut utiliser les mocks des fonctions "read_groups_file",pour tester que
        l'information a ajouter pour le groupe a étée formée correctement
        """
        dummy_long_name = ("a" * 65)
        mock_read_groups_file.return_value = self.groups_data
        add_new_group_long_name = self.crud.add_new_group(dummy_long_name, 50, ['dummyUser1', 'dummyUser2'])

        self.assertFalse(add_new_group_long_name)

    @patch("crud.CRUD.read_groups_file")
    def test_add_new_group_returns_false_if_trust_is_more_than_100(
            self, mock_read_groups_file
    ):
        """Description: il faut utiliser les mocks des fonctions "read_groups_file",pour tester que
        l'information a ajouter pour le groupe a étée formée correctement
        """
        mock_read_groups_file.return_value = self.groups_data
        add_new_group_tooBigTrust = self.crud.add_new_group("dummyName", 101, ['dummyUser1', 'dummyUser2'])

        self.assertFalse(add_new_group_tooBigTrust)

    @patch("crud.CRUD.read_groups_file")
    def test_add_new_group_returns_false_if_trust_is_less_than_0(
            self, mock_read_groups_file
    ):
        """Description: il faut utiliser les mocks des fonctions "read_groups_file",pour tester que
        l'information a ajouter pour le groupe a étée formée correctement
        """

        mock_read_groups_file.return_value = self.groups_data
        add_new_group_too_small_trust = self.crud.add_new_group("dummyName", -1, ['dummyUser1', 'dummyUser2'])

        self.assertFalse(add_new_group_too_small_trust)

    @patch("crud.CRUD.read_groups_file")
    def test_add_new_group_returns_false_if_member_list_is_not_a_list(
            self, mock_read_groups_file
    ):
        """Description: il faut utiliser les mocks des fonctions "read_groups_file",pour tester que
        l'information a ajouter pour le groupe a étée formée correctement
        """
        mock_read_groups_file.return_value = self.groups_data
        add_new_group_not_good_member_list = self.crud.add_new_group("dummyName", 50, 'not a list')

        self.assertFalse(add_new_group_not_good_member_list)

    @patch("crud.CRUD.read_groups_file")
    def test_add_new_group_returns_true_if_values_are_correct(
            self, mock_read_groups_file
    ):
        """Description: il faut utiliser les mocks des fonctions "read_groups_file",pour tester que
        l'information a ajouter pour le groupe a étée formée correctement
        """

        mock_read_groups_file.return_value = self.groups_data
        add_new_group_correct_vals = self.crud.add_new_group("dummyName", 50, ['dummyUser1', 'dummyUser2'])
        self.assertTrue(add_new_group_correct_vals)


    @patch("crud.CRUD.read_users_file")
    def test_get_user_data_Returns_false_for_invalid_id(self, mock_read_users_file):
        """Description: il faut utiliser le mock de fonction "read_groups_file",
        (ou selon votre realisation) pour tester que false (ou bien une excepton)
        est returnee par la fonction si ID non-existant est utilisée
        il faut utiliser ".assertEqual()" ou ".assertFalse()"
        """
        dummyBadId = 10
        mock_read_users_file.return_value = self.users_data
        self.crud.number_of_users = 2
        get_user_data_return_value = self.crud.get_user_data(dummyBadId, 'name')
        self.assertIsNone(get_user_data_return_value)

    @patch("crud.CRUD.read_users_file")
    def test_get_user_data_Returns_false_for_invalid_field(self, mock_read_users_file):
        """Description: il faut utiliser le mock de fonction "read_groups_file",
        (ou selon votre realisation) pour tester que false (ou bien une excepton)
        est returnee par la fonction si champ non-existant est utilisée
        il faut utiliser ".assertEqual()" ou ".assertFalse()"
        """
        mock_read_users_file.return_value = self.users_data
        self.crud.number_of_users = 2
        get_user_data_return_value = self.crud.get_user_data(1, 'badField')
        self.assertIsNone(get_user_data_return_value)

    @patch("crud.CRUD.read_users_file")
    def test_get_user_data_Returns_correct_value_if_field_and_id_are_valid(
            self, mock_read_users_file
    ):
        """Description: il faut utiliser le mock de fonction "read_groups_file",
        (ou selon votre realisation) pour tester que une bonne valeur est fournie
        si champ est id valide sont utilisee
        il faut utiliser ".assertEqual()"
        """
        mock_read_users_file.return_value = self.users_data
        self.crud.number_of_users = 2
        get_user_data_return_value = self.crud.get_user_data(1, 'name')
        self.assertEqual(get_user_data_return_value, "*****@*****.**")

    @patch("crud.CRUD.read_groups_file")
    def test_get_group_data_Returns_false_for_invalid_id(self, mock_read_groups_file):
        """"""
        dummyBadId = 10
        mock_read_groups_file.return_value = self.groups_data
        self.crud.number_of_groups = 2
        get_group_return_value = self.crud.get_group_data(dummyBadId, 'name')
        self.assertIsNone(get_group_return_value)

    @patch("crud.CRUD.read_groups_file")
    def test_get_group_data_Returns_false_for_invalid_field(
            self, mock_read_groups_file
    ):
        """"""
        mock_read_groups_file.return_value = self.groups_data
        self.crud.number_of_groups = 2
        get_group_data_return_value = self.crud.get_group_data(1, 'badField')
        self.assertIsNone(get_group_data_return_value)

    @patch("crud.CRUD.read_groups_file")
    def test_get_group_data_Returns_correct_value_if_field_and_id_are_valid(
            self, mock_read_groups_file
    ):
        """"""
        mock_read_groups_file.return_value = self.groups_data
        self.crud.number_of_groups = 2
        get_group_data_return_value = self.crud.get_group_data(1, 'name')
        self.assertEqual(get_group_data_return_value, 'default')

    @patch("crud.CRUD.read_users_file")
    def test_get_user_id_Returns_None_for_invalid_user_name(
            self, mock_read_users_file
    ):
        mock_read_users_file.return_value = self.users_data
        get_user_id_return_value = self.crud.get_user_id("dummyName")
        self.assertIsNone(get_user_id_return_value)

    @patch("crud.CRUD.read_users_file")
    def test_get_user_id_Returns_id_for_valid_user_name(self, mock_read_users_file):
        mock_read_users_file.return_value = self.users_data
        get_user_id_return_value = self.crud.get_user_id("*****@*****.**")
        self.assertEqual('1', get_user_id_return_value)

    @patch("crud.CRUD.read_groups_file")
    def test_get_group_id_Returns_false_for_invalid_group_name(
            self, mock_read_groups_file
    ):
        mock_read_groups_file.return_value = self.groups_data
        get_group_id_return_value = self.crud.get_group_id("dummyName")
        self.assertIsNone(get_group_id_return_value)

    @patch("crud.CRUD.read_groups_file")
    def test_get_group_id_Returns_id_for_valid_group_name(self, mock_read_groups_file):
        mock_read_groups_file.return_value = self.groups_data
        get_group_id_return_value = self.crud.get_group_id("default")
        self.assertEqual(get_group_id_return_value, '1')

    @patch("crud.CRUD.read_users_file")
    def test_update_users_Returns_false_for_invalid_id(
            self, mock_read_users_file
    ):
        mock_read_users_file.return_value = self.users_data
        update_users_ret_val = self.crud.update_users('10', 'name', 'dummyName')
        self.assertFalse(update_users_ret_val)

    @patch("crud.CRUD.read_users_file")
    def test_update_users_Returns_false_for_invalid_field(
            self, mock_read_users_file
    ):
        mock_read_users_file.return_value = self.users_data
        update_users_ret_val = self.crud.update_users('1', 'invalid', 'dummyInvalid')
        self.assertFalse(update_users_ret_val)

    @patch("crud.CRUD.modify_users_file")
    @patch("crud.CRUD.read_users_file")
    def test_update_users_Passes_correct_data_to_modify_users_file(
            self, mock_read_users_file, mock_modify_users_file
    ):
        """Il faut utiliser les mocks pour 'read_users_file' et 'modify_users_file'
        (ou selon votre realisation)
        Il faut utiliser ".assert_called_once_with(expected_data)"
        """
        mock_read_users_file.return_value = self.users_data
        new_data = self.users_data
        new_data['1']['name'] = 'dummyName'
        self.crud.update_users('1', 'name', 'dummyName')
        mock_modify_users_file.assert_called_once_with(new_data)

    @patch("crud.CRUD.read_groups_file")
    def test_update_groups_Returns_false_for_invalid_id(
            self, mock_read_groups_file
    ):
        """Il faut utiliser les mocks pour 'read_groups_file' et 'modify_groups_file'
        (ou selon votre realisation)
        """
        mock_read_groups_file.return_value = self.groups_data
        update_groups_ret_val = self.crud.update_groups('10', 'name', 'dummyName')
        self.assertFalse(update_groups_ret_val)

    @patch("crud.CRUD.read_groups_file")
    def test_update_groups_Returns_false_for_invalid_field(
            self, mock_read_groups_file
    ):
        mock_read_groups_file.return_value = self.groups_data
        update_groups_ret_val = self.crud.update_groups('1', 'invalidField', 'dummyName')
        self.assertFalse(update_groups_ret_val)

    @patch("crud.CRUD.modify_groups_file")
    @patch("crud.CRUD.read_groups_file")
    def test_update_groups_Passes_correct_data_to_modify_groups_file(
            self, mock_read_groups_file, mock_modify_groups_file
    ):
        """Il faut utiliser les mocks pour 'read_groups_file' et 'modify_groups_file'
        (ou selon votre realisation)
        Il faut utiliser ".assert_called_once_with(expected_data)"
        """
        mock_read_groups_file.return_value = self.groups_data
        new_data = self.groups_data
        new_data['1']['name'] = 'dummyName'
        self.crud.update_groups('1', 'name', 'dummyName')
        mock_modify_groups_file.assert_called_once_with(new_data)

    @patch("crud.CRUD.read_users_file")
    def test_remove_user_Returns_false_for_invalid_id(
            self, mock_read_users_file
    ):
        mock_read_users_file.return_value = self.users_data
        remove_user_ret_val = self.crud.remove_user('10')
        self.assertFalse(remove_user_ret_val)

    @patch("crud.CRUD.modify_users_file")
    @patch("crud.CRUD.read_users_file")
    def test_remove_user_Passes_correct_value_to_modify_users_file(
            self, mock_read_users_file, mock_modify_users_file
    ):
        mock_read_users_file.return_value = self.users_data
        new_data = {"2": {
            "name": "*****@*****.**",
            "Trust": 65.45454,
            "SpamN": 171,
            "HamN": 324,
            "Date_of_first_seen_message": 1596855166.0,
            "Date_of_last_seen_message": 1596855166.0,
            "Groups": ["default"],
        }, }
        self.crud.remove_user('1')
        mock_modify_users_file.assert_called_once_with(new_data)

    @patch("crud.CRUD.read_users_file")
    def test_remove_user_group_Returns_false_for_invalid_id(
            self, mock_read_users_file,
    ):
        mock_read_users_file.return_value = self.users_data
        remove_user_group_ret_val = self.crud.remove_user_group('10', 'default')
        self.assertFalse(remove_user_group_ret_val)

    @patch("crud.CRUD.read_users_file")
    def test_remove_user_group_Returns_false_for_invalid_group(
            self, mock_read_users_file
    ):
        mock_read_users_file.return_value = self.users_data
        remove_user_group_ret_val = self.crud.remove_user_group('1', 'notValidGroup')
        self.assertFalse(remove_user_group_ret_val)

    @patch("crud.CRUD.modify_users_file")
    @patch("crud.CRUD.read_users_file")
    def test_remove_user_group_Passes_correct_value_to_modify_users_file(
            self, mock_read_users_file, mock_modify_users_file
    ):
        mock_read_users_file.return_value = self.users_data
        new_data = {
            "1": {
                "name": "*****@*****.**",
                "Trust": 100,
                "SpamN": 0,
                "HamN": 20,
                "Date_of_first_seen_message": 1596848266.0,
                "Date_of_last_seen_message": 1596848266.0,
                "Groups": [],
            },
            "2": {
                "name": "*****@*****.**",
                "Trust": 65.45454,
                "SpamN": 171,
                "HamN": 324,
                "Date_of_first_seen_message": 1596855166.0,
                "Date_of_last_seen_message": 1596855166.0,
                "Groups": ["default"],
            },
        }
        self.crud.remove_user_group('1', 'default')
        mock_modify_users_file.assert_called_once_with(new_data)

    @patch("crud.CRUD.read_groups_file")
    def test_remove_group_Returns_false_for_invalid_id(
            self, mock_read_groups_file
    ):
        mock_read_groups_file.return_value = self.groups_data
        remove_group_ret_val = self.crud.remove_group('10')
        self.assertFalse(remove_group_ret_val)

    @patch("crud.CRUD.modify_groups_file")
    @patch("crud.CRUD.read_groups_file")
    def test_remove_group_Passes_correct_value_to_modify_groups_file(
            self, mock_read_groups_file, mock_modify_groups_file
    ):
        mock_read_groups_file.return_value = self.groups_data
        new_data = {"2": {
            "name": "friends",
            "Trust": 90,
            "List_of_members": ["*****@*****.**"],
        }, }
        self.crud.remove_group('1')
        mock_modify_groups_file.assert_called_once_with(new_data)

    @patch("crud.CRUD.read_groups_file")
    def test_remove_group_member_Returns_false_for_invalid_id(
            self, mock_read_groups_file
    ):
        mock_read_groups_file.return_value = self.groups_data
        remove_group_member_ret_val = self.crud.remove_group_member('10', '*****@*****.**')
        self.assertFalse(remove_group_member_ret_val)

    @patch("crud.CRUD.read_groups_file")
    def test_remove_group_member_Returns_false_for_invalid_group_member(
            self, mock_read_groups_file
    ):
        mock_read_groups_file.return_value = self.groups_data
        remove_group_member_ret_val = self.crud.remove_group_member('1', 'invalidMember')
        self.assertFalse(remove_group_member_ret_val)

    @patch("crud.CRUD.modify_groups_file")
    @patch("crud.CRUD.read_groups_file")
    def test_remove_group_member_Passes_correct_value_to_modify_groups_file(
            self, mock_read_groups_file, mock_modify_groups_file
    ):
        mock_read_groups_file.return_value = self.groups_data
        self.crud.remove_group_member('1', '*****@*****.**')
        new_data =  {
            "1": {
                "name": "default",
                "Trust": 50,
                "List_of_members": ["*****@*****.**"],
            },
            "2": {
                "name": "friends",
                "Trust": 90,
                "List_of_members": ["*****@*****.**"],
            },
        }
        mock_modify_groups_file.assert_called_once_with(new_data)