def  getClustersWithCat():
    feedback = ''
    str = ''
    try:
        category = request.args.get('category')

        str += 'got parameter category: %s\n' % category
        clusters = Cluster.query(Cluster.category == category).fetch()
        str += 'fetched the clusters'
        str = ''
        i = 1
        for c in clusters:
            str += 'cluster %d:\n' % i
            str += 'category: %s\n' % c.category


            for np in c.listNews:
                str += '\t%s\n' % np.title


            str += '\n'


            i+= 1

    except Exception as inst:
            feedback += 'Exception type: %s\n' % type(inst)
            feedback += 'Exception: %s\n' % inst.message

    str += feedback

    return Response(str, mimetype='text/plain')
def getClustersServer():

    category = request.args.get('category')

    str = ''

    clusters = Cluster.query(Cluster.category == category).fetch()

    str += 'Got category %s\n' % category

    if clusters is None:
        str = 'None clusters :/'
    else:
        i = 1

        str += 'Number of clusters %d\n' % len(clusters)

        for c in clusters:
            str += 'Cluster %d\n' % i

            for np in c.listNews:
                str += '\t%s\t%s\n' % (np.title, np.source_url)

            i += 1


    return Response(str, mimetype='text/plain')
def prepareClustering():
    logging.debug('preparing for refreshing..')
    ndb.delete_multi(Cluster.query().fetch(keys_only=True))

    getClusters()
    logging.debug('refreshed!')

    return ''
def getMYClusters():
    category = request.args.get('category')

    clusters = Cluster.query(Cluster.category == category).fetch()

    obj = {'listClusters' : [c.serialize() for c in clusters]}

    result = json.dumps(obj, ensure_ascii=True)

    return Response(result, mimetype='text/plain')
def getFilteredClustersDebug():
    reqParam = request.args.get('wantedSources')
    listWanted = []


    #vrednosta na parametarot wantedSources kje bide od oblik id1,id2


    if None != reqParam:
        listWanted = reqParam.split(',')

    # da se pretvorat vo integers
    for i in range(0, len(listWanted)):
        listWanted[i] = int(listWanted[i])

    result = 'Wanted sources: %s\n' % listWanted



    category = request.args.get('category')

    clusters = Cluster.query(Cluster.category == category).fetch()


    #tuka gi smestuvame samo klasterite shto se so filtrirani vesti
    # (ne zemame vesti od izvori sto korisnikot ne gi saka)
    clustersToReturn = []

    for c in clusters:
        listNews = []
        for np in c.listNews:
            if np.source_id in listWanted:
                listNews.append(np)
        #dokolku klasterot nema nitu edna vest posle filtriranjeto
        #ne go vkluchuvame bidejkji e prazen :)
        if len(listNews) > 0:
            c.listNews = listNews
            clustersToReturn.append(c)



    i = 1

    #debugging
    result += 'Number of clusters %d\n' % len(clusters)
    
    for c in clustersToReturn:
        result += 'Cluster %d\n' % i
        for np in c.listNews:
            result += '\t%s\t%s\n' % (np.title, np.source_url)
        i += 1



    return Response(result, mimetype='text/plain')
    def get_clusters(self):
        clusters = []

        with open(self.filename) as file:
            data = json.load(file)
            if data["clusters"]:
                clusters_json = data["clusters"]

                for cluster_json in clusters_json:
                    cluster = Cluster()
                    cluster.set_dimensions(self.get_dimensions())

                    if cluster_json["type"]:
                        type_found = False
                        for distribution in Distribution:
                            if distribution.name == cluster_json["type"]:
                                cluster.set_distribution(Distribution[distribution.name])
                                type_found = True
                        if not type_found:
                            print("Couldn't map distribution type. Defaulting to 'gaussian'")
                    else:
                        print("Couldn't find 'type' element for cluster")

                    if cluster_json["cardinality"]:
                        cardinality = int(cluster_json["cardinality"])
                        if 0 < cardinality <= 10:
                            cluster.set_cardinality(cardinality)
                        else:
                            print("Invalid cardinality. Has to be within 1 to 10. Defaulting to 5.")
                    else:
                        print("Couldn't find 'cardinality' element for cluster")

                    if cluster_json["density"]:
                        density = int(cluster_json["density"])
                        if 0 < density <= 10:
                            cluster.set_density(density)
                        else:
                            print("Invalid density. Has to be within 1 to 10. Defaulting to 5.")
                    else:
                        print("Couldn't find 'density' element for cluster")

                    clusters.append(cluster)
            else:
                print("Couldn't find 'clusters' element in input file")

        return clusters
def getMYNews():
    category = request.args.get('category')
    news = NewsPost.query().fetch()
    clusters = Cluster.query(Cluster.category == category).fetch()

    newNews = []

    for n in news:
        newObject = NewsPostClient(url = n.url,host_page = n.host_page,title = n.title, description = n.description)
        newNews.append(newObject)


    #result = str(byteify(newNews[0].serialize()))

    result = ''


    return Response(result, mimetype='application/javascript')
def getFilteredClusters():

    reqParam = request.args.get('wantedSources')

    listWanted = []

    if None != reqParam:
        listWanted = reqParam.split(',')


    for i in range(0, len(listWanted)):
        listWanted[i] = int(listWanted[i])

    category = request.args.get('category')

    clusters = Cluster.query(Cluster.category == category).fetch()

    clustersToReturn = []

    for c in clusters:
        listNews = []
        for np in c.listNews:
            if np.source_id in listWanted:
                listNews.append(np)

        if len(listNews) > 0:
            c.listNews = listNews
            clustersToReturn.append(c)


    # WE NEED TO SORT THE CLUSTERS BY SOME PARAMETER (MAYBE THE SIZE OF THE CLUSTER)

    obj = {'listClusters' : [c.serialize() for c in clustersToReturn]}

    result = json.dumps(obj, ensure_ascii=True)

    return Response(result, mimetype='text/plain')
def getClusters():
    feedback = ''
    str = ''
    try:
        newsPosts = crawler.take_all_news_posts()


        # utility dicts for majority voting with naive bayes

        fileToRead = open(naivebayes_classification.str_dict_word_in_cat)
        dict_words = Unpickler(fileToRead).load()
        fileToRead.close()

        fileToRead = open(naivebayes_classification.str_dict_cat_count)
        dict_cats = Unpickler(fileToRead).load()
        fileToRead.close()

        fileToRead = open(naivebayes_classification.str_dict_priors)
        dict_priors = Unpickler(fileToRead).load()
        fileToRead.close()

        feedback += 'took the newsposts \n'



        #return Response('%d' % counter, mimetype='text/plain')
        clusters, innerfeedback = clustering.cluster_news(newsPosts)

        feedback += '%s\n' % innerfeedback

        feedback += 'done the clustering\n'
        i = 0

        feedback += 'num of clusters: %d\n' % len(clusters)

        clusters = sorted(clusters, key=lambda x:-len(x.posts))
        for c in clusters:

            feedback += 'getting posts from cluster\n'
            newsInCluster = c.posts
            feedback += 'got the posts from cluster\n'

            str += 'cluster %d\n' % i

            #implementing the majority voting

            votes_cat = {}

            for np in newsInCluster:
                str += ' \t %s\n' % np.title
                category = test_classifications.get_NB_category(np.words,dict_words, dict_cats, dict_priors)
                votes_cat[category] = 1 + votes_cat.get(category, 0)

            maxVotes = 0
            maxCat = ''

            for cat in votes_cat:
                if votes_cat[cat] > maxVotes:
                    maxVotes = votes_cat[cat]
                    maxCat = cat

            feedback += '^^^ CLUSTER CATEGORY: %s with maxVotes: %d\n' % (maxCat, maxVotes)

            listNews = []

            feedback += ' number of posts in cluster %d\n' % len(c.posts)
            for np in  c.posts:

                feedback += 'trying to create NewsPostClient\n'
                feedback += 'title: %s \n' % np.title
                feedback += 'numWords: %d\n' % np.numWords
                feedback +=  'url: %s\n' % np.url

                newNews = NewsPostClient(url = np.url, host_page = np.host_page, title = np.title, numWords = np.numWords, source_id = np.source_id,
                                         source_url = np.source_url,
                                         img_url = np.img_url, description = np.description)

                feedback += 'created NewsPostClient'
                listNews.append(newNews)
                feedback += 'appended newNews\n'

            newCluster = Cluster(category = maxCat, listNews = listNews)
            newCluster.put()
            
            str += '\n'

            i += 1

        str += feedback
    except Exception as inst:
        feedback += 'Exception type: %s\n' % (type(inst))
        feedback += 'Exception: %s\n' % (inst.message)

    str += feedback
    return Response(str, mimetype='text/plain')
 def addCluster(self, r, x, y, v, theta, lambda0, color, is_point):
     self.clusters_list.append(
         Cluster(r, x, y, v, theta, lambda0, color, is_point))