def setup_libraries(self):
        # Usage:
        #       The request object is an HTTPResponse object in Django. It is able to be used
        #       like an dictionary to get http parameters, such as ?num_samples=X&facebook_Group=Y.
        #       We will try to get num_samples, facebook_group, and pusher_uuid. Otherwise
        #       we will raise a SuspiciousOperation if it fails. Afterwards get data from the facebook
        #       group, and then train our model, then use our model to determine sentiment of users.
        # Arguments:
        #       yelp_extractor      (YelpExtractor)      : a yelp data extractor that can extract yelp data
        #       facebook_extractor  (FacebookExtractor)  : a facebook extractor that can extract facebook data
        #       logistic_classifier (LogisticRegression) : a logistic regression class that can compute sentiment of a user
        # Return:
        #       None

        # Setup our Libraries
        self.yelp_extractor = YelpExtractor()
        self.facebook_extractor = FacebookExtractor()
        self.logistic_classifier = LogisticClassifier()
    def setUp(self,
              facebook_extractor=FacebookExtractor(),
              group_id="nyutass",
              num_messages=20):
        # Usage:
        #       Constructor for TestFacebookExtractor
        # Arguments:
        # 		facebook_extractor (object) : an object of our FacebookExtractor class
        #       group_id           (string) : a default group ID or group name for testing
        #       num_messages       (int)    : the number of messages to get

        self.facebook_extractor = facebook_extractor

        self.group_id = group_id

        self.num_messages = num_messages
    def setup_libraries(self):
        # Usage:
        #       The request object is an HTTPResponse object in Django. It is able to be used
        #       like an dictionary to get http parameters, such as ?num_samples=X&facebook_Group=Y.
        #       We will try to get num_samples, facebook_group, and pusher_uuid. Otherwise
        #       we will raise a SuspiciousOperation if it fails. Afterwards get data from the facebook
        #       group, and then train our model, then use our model to determine sentiment of users.
        # Arguments:
        #       yelp_extractor      (YelpExtractor)      : a yelp data extractor that can extract yelp data
        #       facebook_extractor  (FacebookExtractor)  : a facebook extractor that can extract facebook data
        #       logistic_classifier (LogisticRegression) : a logistic regression class that can compute sentiment of a user
        # Return:
        #       None

        # Setup our Libraries
        self.yelp_extractor = YelpExtractor()
        self.facebook_extractor = FacebookExtractor()
        self.logistic_classifier = LogisticClassifier()
class Train(View):
    # Usage:
    #   This is used to train a SFrame logistic regression model. And return the data using a pusher.
    #   The URL parameters should contain num_samples, facebook group name, and pusher UUID channel
    #   We will get the group data such as members from the facebook group name, and train the
    #   logistic regression model with num_samples of data from amazon, and yelp (each half of num_samples)

    def setup_libraries(self):
        # Usage:
        #       The request object is an HTTPResponse object in Django. It is able to be used
        #       like an dictionary to get http parameters, such as ?num_samples=X&facebook_Group=Y.
        #       We will try to get num_samples, facebook_group, and pusher_uuid. Otherwise
        #       we will raise a SuspiciousOperation if it fails. Afterwards get data from the facebook
        #       group, and then train our model, then use our model to determine sentiment of users.
        # Arguments:
        #       yelp_extractor      (YelpExtractor)      : a yelp data extractor that can extract yelp data
        #       facebook_extractor  (FacebookExtractor)  : a facebook extractor that can extract facebook data
        #       logistic_classifier (LogisticRegression) : a logistic regression class that can compute sentiment of a user
        # Return:
        #       None

        # Setup our Libraries
        self.yelp_extractor = YelpExtractor()
        self.facebook_extractor = FacebookExtractor()
        self.logistic_classifier = LogisticClassifier()

    def get(self, request):
        # Usage:
        #       The request object is an HTTPResponse object in Django. It is able to be used
        #       like an dictionary to get http parameters, such as ?num_samples=X&facebook_Group=Y.
        #       We will try to get num_samples, facebook_group, and pusher_uuid. Otherwise
        #       we will raise a SuspiciousOperation if it fails. Afterwards get data from the facebook
        #       group, and then train our model, then use our model to determine sentiment of users.
        # Arguments:
        #       request (object) : django request object
        # Return:
        #       Json (JsonResponse) : a Json object with status = done

        # Setup our libraries first
        self.setup_libraries()

        # get our url parameters
        num_samples, facebook_group = self.get_url_parameters(request)

        # use the group to create models for our facebook group
        facebook_message, facebook_count = self.get_facebook_group(facebook_group=facebook_group)

        # get our yelp reviews appended into a database, and we can get some message, and count
        yelp_message, yelp_exception, yelp_count = self.get_yelp_reviews(num_samples=math.ceil(num_samples/2))

        # set user_sentiment to None
        user_sentiment = None

        # set status to success, and message to None
        status = "Success"
        message = None

        # Need to test whether if we got any yelp data
        if YelpReview.objects.count() > 0:

            # train, and get our data for user's sentiment
            user_sentiment = self.compute_sentiment_analysis(model_iterators=[YelpReview.objects.all().iterator()],
                                                             num_samples=num_samples,
                                                             facebook_group=facebook_group,
                                                             facebook_user_model=FacebookUser,
                                                             facebook_post_model=FacebookPost)

        else:

            # There is no reviews, hence we will set a message
            status = "Failure"
            message = "There are no reviews"

        # Return a JsonResponse to update to the front-end that the status is success
        return JsonResponse({'status':status, 'message':message, 'yelp_message':yelp_message, 'yelp_exception':yelp_exception,
                             'yelp_count':yelp_count,'facebook_message':facebook_message, 'facebook_count':facebook_count,
                             'user_sentiment':user_sentiment})

    def get_url_parameters(self, request):
        # Usage:
        #       The request object is an HTTPResponse object in Django. It is able to be used
        #       like an dictionary to get http parameters, such as ?num_samples=X&facebook_Group=Y.
        #       We will try to get num_samples, facebook_group, and pusher_uuid. Otherwise
        #       we will return a HttpResponseBadRequest if it fails.
        # Arguments:
        #       request (object) : django request object
        # Return:
        #       num_samples    (int)    : number of samples to train with
        #       facebook_group (string) : facebook_group string, can be a number or a string

        # try to get a few url parameters, such as num_samples, facebook_group, pusher_uuid
        try:

            # num_samples should be an integer, and facebook_group are strings
            num_samples = int(request.GET.get('num_samples'))
            facebook_group = request.GET.get('facebook_group')

            # return the tuples of the url parameters
            return num_samples, facebook_group

        except KeyError:
            # If we fail to get any of the url parameters, we will raise an exception for Invalid url parameters
            raise SuspiciousOperation('Invalid url parameters')

        except Exception as e:
            # Fails due to unknown reasons
            raise SuspiciousOperation('Unknown Error:%s' % str(e))

    def get_facebook_group(self, facebook_group):
        # Usage:
        #       This function will utilize the facebook_extractor to get the data using the
        #       facebook graph api, and append the data into the facebook model.
        # Arguments:
        #       facebook_group (string)  : name of the facebook group
        # Return:
        #       message        (string) : Message of the current status
        #       count          (int)    : Number of users extracted from yelp

        # First, get or create a facebook_group
        facebook_group_model, facebook_group_model_created = FacebookGroup.objects.get_or_create(id=facebook_group,
                                                                                                 defaults=dict(id=facebook_group))

        # Loop through the messages using the facebook_extractor
        for data in self.facebook_extractor.distribute_get_posts(group=facebook_group):
            # Get or create a facebook user, since that user might exist in other public groups as well
            facebook_user, facebook_user_created = FacebookUser.objects.get_or_create(id=data["user_id"],
                                                                                      defaults=dict(username=data["user_name"],
                                                                                                    facebook_group=facebook_group_model))

            # Update or create the user's messages
            FacebookPost.objects.update_or_create(id=data["message_id"], post=data["message_text"],
                                                  facebook_user=facebook_user, defaults=dict(id=data["message_id"],
                                                                                             post=data["message_text"],
                                                                                             facebook_user=facebook_user))

        # Return a facebook extractor status, and the number of users in the group
        return "Successfully finished", FacebookUser.objects.filter(facebook_group=facebook_group).count()

    def get_yelp_reviews(self, num_samples):
        # Usage:
        #       This function will utilize the yelp_extractor to get the data using the
        #       yelp api, and append the data into the yelp model.
        # Arguments:
        #       num_samples (int)  : number of samples to get from yelp
        # Return:
        #       message     (string) : Message of the current status
        #       count       (int)    : Number of samples extracted from yelp

        # Get how many counts there are for yelp reviews before adding
        yelp_reviews_before = YelpReview.objects.count()

        # If there are less than reviews we needed, then we will add more into the database
        if yelp_reviews_before < num_samples:

            # distributed_find_reviews is a generator that will return reviews as a dictionary format
            # of dict(data, exception)
            for review in self.yelp_extractor.distributed_find_reviews(reviews_required=num_samples-yelp_reviews_before):

                # If there is an issue raised inside the generator
                if review["exception"]:

                    # Return the exception message, and how many count was added
                    return review["exception"].message, review["exception"].errors, abs(YelpReview.objects.count()-yelp_reviews_before)

                # Add a new YelpReview into the database
                YelpReview(id=review["data"]["id"], review_text=review["data"]["review"], star=review["data"]["star"]).save()

        # There was enough reviews in the first place
        else:

            # Return a message saying that extraction was not required, and the new reviews added was 0
            return "No need for extraction, already have enough reviews", None, 0

        # Return a success message, and how many count was added
        return "Yelp extractor completed successfully", None, abs(YelpReview.objects.count()-yelp_reviews_before)

    def compute_sentiment_analysis(self, model_iterators, num_samples, facebook_group, facebook_user_model, facebook_post_model):
        # Usage:
        #       This function will utilize the logistic classifier class to
        #       compute sentiment analysis
        # Arguments:
        #       model_iterators     (list)   : list of model ierators
        #                                      each model should have ID, Review, and Star
        #       num_samples         (int)    : the number of samples to train. These will be
        #                                      divided amongst the model_iterators.
        #       facebook_group      (string) : facebook group number or ID
        #       facebook_user_model (model)  : facebook user model
        #       facebook_post_model (model)  : facebook post model
        # Return:
        #       array of dictionary (array) : an array with dictionary user_id, mean_predicted_sentiment

        # Add models to the logistic classifier
        self.logistic_classifier.add_models(model_iterators, num_samples)

        # Train the models with the current models
        self.logistic_classifier.train_model()

        # Add Facebook model data to the logistic classifier
        self.logistic_classifier.add_facebook_model(facebook_group, facebook_user_model, facebook_post_model)

        # Predict the sentiment for all the facebook posts
        self.logistic_classifier.predict_sentiment()

        # Average out each user's sentiment
        self.logistic_classifier.average_user_sentiment()

        return [dict(user_id=sentiment["user_id"],
                     username=sentiment["username"],
                     mean_predicted_sentiment=sentiment["mean_predicted_sentiment"])
                for sentiment in self.logistic_classifier.get_sentiment()]
class Train(View):
    # Usage:
    #   This is used to train a SFrame logistic regression model. And return the data using a pusher.
    #   The URL parameters should contain num_samples, facebook group name, and pusher UUID channel
    #   We will get the group data such as members from the facebook group name, and train the
    #   logistic regression model with num_samples of data from amazon, and yelp (each half of num_samples)

    def setup_libraries(self):
        # Usage:
        #       The request object is an HTTPResponse object in Django. It is able to be used
        #       like an dictionary to get http parameters, such as ?num_samples=X&facebook_Group=Y.
        #       We will try to get num_samples, facebook_group, and pusher_uuid. Otherwise
        #       we will raise a SuspiciousOperation if it fails. Afterwards get data from the facebook
        #       group, and then train our model, then use our model to determine sentiment of users.
        # Arguments:
        #       yelp_extractor      (YelpExtractor)      : a yelp data extractor that can extract yelp data
        #       facebook_extractor  (FacebookExtractor)  : a facebook extractor that can extract facebook data
        #       logistic_classifier (LogisticRegression) : a logistic regression class that can compute sentiment of a user
        # Return:
        #       None

        # Setup our Libraries
        self.yelp_extractor = YelpExtractor()
        self.facebook_extractor = FacebookExtractor()
        self.logistic_classifier = LogisticClassifier()

    def get(self, request):
        # Usage:
        #       The request object is an HTTPResponse object in Django. It is able to be used
        #       like an dictionary to get http parameters, such as ?num_samples=X&facebook_Group=Y.
        #       We will try to get num_samples, facebook_group, and pusher_uuid. Otherwise
        #       we will raise a SuspiciousOperation if it fails. Afterwards get data from the facebook
        #       group, and then train our model, then use our model to determine sentiment of users.
        # Arguments:
        #       request (object) : django request object
        # Return:
        #       Json (JsonResponse) : a Json object with status = done

        # Setup our libraries first
        self.setup_libraries()

        # get our url parameters
        num_samples, facebook_group = self.get_url_parameters(request)

        # use the group to create models for our facebook group
        facebook_message, facebook_count = self.get_facebook_group(
            facebook_group=facebook_group)

        # get our yelp reviews appended into a database, and we can get some message, and count
        yelp_message, yelp_exception, yelp_count = self.get_yelp_reviews(
            num_samples=math.ceil(num_samples / 2))

        # set user_sentiment to None
        user_sentiment = None

        # set status to success, and message to None
        status = "Success"
        message = None

        # Need to test whether if we got any yelp data
        if YelpReview.objects.count() > 0:

            # train, and get our data for user's sentiment
            user_sentiment = self.compute_sentiment_analysis(
                model_iterators=[YelpReview.objects.all().iterator()],
                num_samples=num_samples,
                facebook_group=facebook_group,
                facebook_user_model=FacebookUser,
                facebook_post_model=FacebookPost)

        else:

            # There is no reviews, hence we will set a message
            status = "Failure"
            message = "There are no reviews"

        # Return a JsonResponse to update to the front-end that the status is success
        return JsonResponse({
            'status': status,
            'message': message,
            'yelp_message': yelp_message,
            'yelp_exception': yelp_exception,
            'yelp_count': yelp_count,
            'facebook_message': facebook_message,
            'facebook_count': facebook_count,
            'user_sentiment': user_sentiment
        })

    def get_url_parameters(self, request):
        # Usage:
        #       The request object is an HTTPResponse object in Django. It is able to be used
        #       like an dictionary to get http parameters, such as ?num_samples=X&facebook_Group=Y.
        #       We will try to get num_samples, facebook_group, and pusher_uuid. Otherwise
        #       we will return a HttpResponseBadRequest if it fails.
        # Arguments:
        #       request (object) : django request object
        # Return:
        #       num_samples    (int)    : number of samples to train with
        #       facebook_group (string) : facebook_group string, can be a number or a string

        # try to get a few url parameters, such as num_samples, facebook_group, pusher_uuid
        try:

            # num_samples should be an integer, and facebook_group are strings
            num_samples = int(request.GET.get('num_samples'))
            facebook_group = request.GET.get('facebook_group')

            # return the tuples of the url parameters
            return num_samples, facebook_group

        except KeyError:
            # If we fail to get any of the url parameters, we will raise an exception for Invalid url parameters
            raise SuspiciousOperation('Invalid url parameters')

        except Exception as e:
            # Fails due to unknown reasons
            raise SuspiciousOperation('Unknown Error:%s' % str(e))

    def get_facebook_group(self, facebook_group):
        # Usage:
        #       This function will utilize the facebook_extractor to get the data using the
        #       facebook graph api, and append the data into the facebook model.
        # Arguments:
        #       facebook_group (string)  : name of the facebook group
        # Return:
        #       message        (string) : Message of the current status
        #       count          (int)    : Number of users extracted from yelp

        # First, get or create a facebook_group
        facebook_group_model, facebook_group_model_created = FacebookGroup.objects.get_or_create(
            id=facebook_group, defaults=dict(id=facebook_group))

        # Loop through the messages using the facebook_extractor
        for data in self.facebook_extractor.distribute_get_posts(
                group=facebook_group):
            # Get or create a facebook user, since that user might exist in other public groups as well
            facebook_user, facebook_user_created = FacebookUser.objects.get_or_create(
                id=data["user_id"],
                defaults=dict(username=data["user_name"],
                              facebook_group=facebook_group_model))

            # Update or create the user's messages
            FacebookPost.objects.update_or_create(
                id=data["message_id"],
                post=data["message_text"],
                facebook_user=facebook_user,
                defaults=dict(id=data["message_id"],
                              post=data["message_text"],
                              facebook_user=facebook_user))

        # Return a facebook extractor status, and the number of users in the group
        return "Successfully finished", FacebookUser.objects.filter(
            facebook_group=facebook_group).count()

    def get_yelp_reviews(self, num_samples):
        # Usage:
        #       This function will utilize the yelp_extractor to get the data using the
        #       yelp api, and append the data into the yelp model.
        # Arguments:
        #       num_samples (int)  : number of samples to get from yelp
        # Return:
        #       message     (string) : Message of the current status
        #       count       (int)    : Number of samples extracted from yelp

        # Get how many counts there are for yelp reviews before adding
        yelp_reviews_before = YelpReview.objects.count()

        # If there are less than reviews we needed, then we will add more into the database
        if yelp_reviews_before < num_samples:

            # distributed_find_reviews is a generator that will return reviews as a dictionary format
            # of dict(data, exception)
            for review in self.yelp_extractor.distributed_find_reviews(
                    reviews_required=num_samples - yelp_reviews_before):

                # If there is an issue raised inside the generator
                if review["exception"]:

                    # Return the exception message, and how many count was added
                    return review["exception"].message, review[
                        "exception"].errors, abs(YelpReview.objects.count() -
                                                 yelp_reviews_before)

                # Add a new YelpReview into the database
                YelpReview(id=review["data"]["id"],
                           review_text=review["data"]["review"],
                           star=review["data"]["star"]).save()

        # There was enough reviews in the first place
        else:

            # Return a message saying that extraction was not required, and the new reviews added was 0
            return "No need for extraction, already have enough reviews", None, 0

        # Return a success message, and how many count was added
        return "Yelp extractor completed successfully", None, abs(
            YelpReview.objects.count() - yelp_reviews_before)

    def compute_sentiment_analysis(self, model_iterators, num_samples,
                                   facebook_group, facebook_user_model,
                                   facebook_post_model):
        # Usage:
        #       This function will utilize the logistic classifier class to
        #       compute sentiment analysis
        # Arguments:
        #       model_iterators     (list)   : list of model ierators
        #                                      each model should have ID, Review, and Star
        #       num_samples         (int)    : the number of samples to train. These will be
        #                                      divided amongst the model_iterators.
        #       facebook_group      (string) : facebook group number or ID
        #       facebook_user_model (model)  : facebook user model
        #       facebook_post_model (model)  : facebook post model
        # Return:
        #       array of dictionary (array) : an array with dictionary user_id, mean_predicted_sentiment

        # Add models to the logistic classifier
        self.logistic_classifier.add_models(model_iterators, num_samples)

        # Train the models with the current models
        self.logistic_classifier.train_model()

        # Add Facebook model data to the logistic classifier
        self.logistic_classifier.add_facebook_model(facebook_group,
                                                    facebook_user_model,
                                                    facebook_post_model)

        # Predict the sentiment for all the facebook posts
        self.logistic_classifier.predict_sentiment()

        # Average out each user's sentiment
        self.logistic_classifier.average_user_sentiment()

        return [
            dict(
                user_id=sentiment["user_id"],
                username=sentiment["username"],
                mean_predicted_sentiment=sentiment["mean_predicted_sentiment"])
            for sentiment in self.logistic_classifier.get_sentiment()
        ]