Example #1
0
def get_top_sectors():

    synonyms = load_synonyms('./datasets/sinonimos.csv')
    synonyms1 = load_synonyms('./datasets/sinonimos2.csv')
    dictionary = load_words()
    stop_words = load_stop_words('./datasets/stop-words.txt')
    routes = load_routes('./datasets/routes.txt')

    counter = Counter()

    with open('counter.txt') as fp:
        counter = pickle.load(fp)

    topRoutes = set(counter.elements())

    sectorGraph = get_graph()

    listRoutes = list(topRoutes)

    topSectors = []

    for avenue in listRoutes:
        for (x, y) in sectorGraph.edges():
            routesEdge = sectorGraph.edge[x][y]['routes']
            for route in routesEdge:
                processedRoute = process_tweet(route, synonyms, synonyms1,
                                               dictionary, stop_words)

                if (processedRoute.find(avenue) > -1):
                    topSectors.append({'from': x, 'to': y})

    return json.dumps(topSectors)
Example #2
0
    def __init__(self, number, myWorldState, agType):
        super().__init__(number, myWorldState, agType)
        # the environment
        self.agOperatingSets = []
        self.number = number
        # self.news = np.zeros(common.dim+3) # contiene l'ultima notizia
        # [id-fonte, id-mittente, data, topics(dim)]

        if myWorldState != 0:
            self.myWorldState = myWorldState
        self.agType = agType

        self.state = np.zeros([common.dim])
        self.active = True
        #self.databaseCols = [
        #    'id-n', 'new', 'id-source', 'date-creation', 'relevance',
        #    'id-send', 'date-send', 'id-recive', 'date-recive'
        #]
        self.database = db.database()
        self.spreadState = 's'
        #print(LOG_LABEL, "agent", self.agType, "#", self.number,
        #"has been created")

        # =========================================================================================
        #
        # GRAPH CREATION
        #
        if graph.get_graph() == 0:
            graph.create_graph()  # if first agent create the graph
        common.G.add_node(self.number, agent=self)  # adds himself
        # create link only if you are only at the first step of the clock
        # and if you are the last user
        if common.cycle == 1 and len(common.G.nodes()) == common.N_AGENTS:
            graph.initialize_edges()  # if last creates edges
            graph.change_s_to_i()
Example #3
0
def get_top_sectors():
    
    synonyms = load_synonyms('./datasets/sinonimos.csv')
    synonyms1 = load_synonyms('./datasets/sinonimos2.csv')
    dictionary = load_words()
    stop_words = load_stop_words('./datasets/stop-words.txt')
    routes = load_routes('./datasets/routes.txt')

    counter = Counter()

    with open('counter.txt') as fp:
        counter = pickle.load(fp)

    topRoutes = set(counter.elements())

    sectorGraph = get_graph()

    listRoutes = list(topRoutes)

    topSectors = []

    for avenue in listRoutes:
        for (x, y) in sectorGraph.edges():
            routesEdge = sectorGraph.edge[x][y]['routes']
            for route in routesEdge:
                processedRoute = process_tweet(route, synonyms, synonyms1, dictionary, stop_words)

                if (processedRoute.find(avenue) > -1):
                    topSectors.append({'from': x, 'to': y})

    return json.dumps(topSectors)
Example #4
0
def find_path(origin, destination):

    queue = [(0, origin)]
   
    path = {}
    path[(0, origin)] = -1
   
    visitedNodes = set()
    graph = get_graph()

    now = datetime.now()
    now = datetime(2015,05,07,15,00)
    print "Current Date: %s\n" %(now)

    while queue:
        #Tuple: node = (cost, current sector)
        node = accumulatedCost, currentNode = heapq.heappop(queue) 
       
        print '----- Current Node: %s -----\n' %(currentNode)
        if currentNode == destination:
            path = list(build_path(path, node))
            path.reverse()
            yield path
            continue

        if currentNode in visitedNodes:
            continue

        visitedNodes.add(currentNode)

        accumulatedTime = now + timedelta(minutes=accumulatedCost)

        for successor in graph[currentNode]:
        
            actualScore = get_stream_score(currentNode, successor, now=now, spoof=True)
            print 'ACTUAL SCORE: %.2f' %(actualScore)
            
            before = accumulatedTime + timedelta(minutes=-10)
            after = accumulatedTime + timedelta(minutes=10)
            
            historicScore = get_historic_score(currentNode, successor,
                                     before.strftime('%H:%M:00'),
                                     after.strftime('%H:%M:00'))

            print 'HISTORIC SCORE: %.2f' %(historicScore)

            estimatedScore = (1-phi(accumulatedCost))*actualScore + phi(accumulatedCost)*historicScore

            #TODO change so that each node has a different traffic penalty function
            congestionValue = get_TravelTime(graph,currentNode,successor)*estimatedScore

            cost = accumulatedCost + get_TravelTime(graph,currentNode,successor) + congestionValue 
            
            path[(cost, successor)] = node
            print "ARC: %s -> %s ***** SCORE: %.2f, COST: %.2f\n" %(currentNode, successor, estimatedScore, cost)
            heapq.heappush(queue, (cost, successor))
Example #5
0
    def __init__(self, config):

        self.df = read_data(config)
        self.organs = config.ORGANS
        self.components = config.COMPONENTS
        self.co_in, self.co_in123, self.co_in45 = get_graph()
        self.get_graph_mapping()
        self.data_arrs = self.get_data_arrs()
        self.pca_est, self.transformed = self.get_estimators()
        self.cluster_torsos(neighbors=30)
Example #6
0
def related_tweets(source, dest, since_date=None, before_date=None):
    '''Returns all related tweets for `via`.'''
    graph = get_graph()
    if not graph.has_edge(source, dest):
        return
    keywords = graph[source][dest]['keywords']
    filters = or_(*[Tweet.text.ilike('%%%s%%' % x) for x in keywords])
    qs = Tweet.query.filter(filters)
    if since_date is not None:
        qs = qs.filter(cast(Tweet.created_at, Date) >= since_date)

    if before_date is not None:
        qs = qs.filter(cast(Tweet.created_at, Date) <= before_date)

    return qs
Example #7
0
def related_tweets(source, dest, since_date=None, before_date=None):
    '''Returns all related tweets for `via`.'''
    graph = get_graph()
    if not graph.has_edge(source, dest):
        return
    keywords = graph[source][dest]['keywords']
    filters = or_(*[Tweet.text.ilike('%%%s%%' % x) for x in keywords])
    qs = Tweet.query.filter(filters)
    if since_date is not None:
        qs = qs.filter(cast(Tweet.created_at, Date) >= since_date)

    if before_date is not None:
        qs = qs.filter(cast(Tweet.created_at, Date) <= before_date)

    return qs
Example #8
0
def display_click_data(clickData,fig_master,n_clicks, fig, fig_1,zoomed):
    ctx = dash.callback_context
    ctx = ctx.triggered[0]['prop_id']
    if ctx == 'network-graph.clickData' and clickData:
        based_on = [json.loads(clickData['points'][0]['customdata'])]
        curve_no = clickData['points'][0]['pointNumber']
        if zoomed and curve_no == 0:
            return fig_1, fig_master, 0
        cond_rec, recommendations, query, distances = get_recommendations(10, 10000,
        based_on = based_on, return_A=True)

        total = query + recommendations
        network_fig, colors = graph.get_graph(total, distances, n_query=1)
        return network_fig, fig, 1
    elif ctx in ['network-graph-master.figure','reset-button.n_clicks']:
        return fig_master, fig_master, 0
    else:
        return fig, fig_1, 0
Example #9
0
def update_recommendations(time_lim, no_papers, ref_trigger, *states):
    logging.warning(ref_trigger)
    if ref_trigger:
        cond_rec, recommendations, query, distances = get_recommendations(no_papers, time_lim, return_A=True)

        total = query + recommendations
        network_fig_, colors = graph.get_graph(total, distances, n_query=len(query))
        cond_rec_bm = [{
                'if': {
                    'filter_query': '{{id}} = {}'.format(r['id']) # matching rows of a hidden column with the id, `id`
                },
                'color': c,
                } for r, c in zip(total,colors)]
        if len(total):
            return cond_rec, get_authors_short(recommendations), network_fig_, cond_rec_bm
        else:
            return [],[{'title':'No bookmarks yet...','authors':''}], network_fig_, cond_rec_bm
    else:
        return states
Example #10
0
def welcome():

    # model = Net()
    # model.load_state_dict(torch.load(MODEL_PATH))
    # torch.no_grad()
    # model.eval()

    if request.method == 'POST':
        try:
            file = request.files['file']
        except:
            return render_template('welcome_page.html')
        if file:
            filename = file.filename

            file.save(os.path.join(UPLOAD_DIR, filename))
            #render graph
            graph_name = get_graph(UPLOAD_DIR + filename, model)

            return render_template('prediction.html', graph=graph_name)
    return render_template('welcome_page.html', title='Application')
Example #11
0
written by Rob Speer, Julian Chaidez
Common Sense Computing Group, Medialab
Massachusetts Institute of Technology
Fall 2011

"""

import graph
import flask
import urllib
import re
import sys
import simplejson
from werkzeug.contrib.cache import SimpleCache
app = flask.Flask(__name__)
concept_graph = graph.get_graph()

if len(sys.argv) == 1:
    root_url = 'http://conceptnet5.media.mit.edu/data'
else:
    root_url = sys.argv[1]

cache_dict = {
        'limit_timeout':60,
        'limit_amount':10000
        }

request_cache = SimpleCache(default_timeout = cache_dict['limit_timeout'])

no_leading_slash = ['http']
 def test_get_specified_k_graph(self):
     G = get_graph(self.points, k=2)
     self.assertEqual(list(G.neighbors(0)), [1, 2])
Example #13
0
from co_author_crawler import crawler
import graph

if __name__ == '__main__':
    nodes1 = []
    nodes2 = []
    edges = []

    graph.get_graph(nodes1, nodes2, edges)
    graph.get_graph_by_institution(nodes1, nodes2, edges)
Example #14
0
def report():
    word = request.get_json("word")
    if "num_page" in word:
        result = word.split('num_page')
        word = result[0]
        page = int(result[1])
    else:
        # 페이지 값 (디폴트값 = 1)
        page = request.args.get("page", 1, type=int)
    # 한 페이지 당 몇 개의 게시물을 출력할 것인가
    limit = 10

    # 그래프 이미지 가져오기
    graph = get_graph()

    if word:
        word = word.lower()
        existingNews = db.get(word)
        if existingNews:
            news = existingNews
            datas = news[(page - 1) * limit:limit * page]

            # 게시물의 총 개수 세기
            tot_count = len(news)
            # 마지막 페이지의 수 구하기
            last_page_num = math.ceil(tot_count / limit)  # 반드시 올림을 해줘야함
            # 페이지 블럭을 5개씩 표기
            block_size = 5
            # 현재 블럭의 위치 (첫 번째 블럭이라면, block_num = 0)
            block_num = int((page - 1) / block_size)
            # 현재 블럭의 맨 처음 페이지 넘버 (첫 번째 블럭이라면, block_start = 1, 두 번째 블럭이라면, block_start = 6)
            block_start = (block_size * block_num) + 1
            # 현재 블럭의 맨 끝 페이지 넘버 (첫 번째 블럭이라면, block_end = 5)
            block_end = block_start + (block_size - 1)
        else:
            # 뉴스 기사 가져오기
            naver_news = get_naver_news(word)
            hankyung_news = get_hankyung_news(word)
            news = naver_news + hankyung_news
            db[word] = news

            datas = news[(page - 1) * limit:limit * page]
            # 게시물의 총 개수 세기
            tot_count = len(news)
            # 마지막 페이지의 수 구하기
            last_page_num = math.ceil(tot_count / limit)  # 반드시 올림을 해줘야함
            # 페이지 블럭을 5개씩 표기
            block_size = 5
            # 현재 블럭의 위치 (첫 번째 블럭이라면, block_num = 0)
            block_num = int((page - 1) / block_size)
            # 현재 블럭의 맨 처음 페이지 넘버 (첫 번째 블럭이라면, block_start = 1, 두 번째 블럭이라면, block_start = 6)
            block_start = (block_size * block_num) + 1
            # 현재 블럭의 맨 끝 페이지 넘버 (첫 번째 블럭이라면, block_end = 5)
            block_end = block_start + (block_size - 1)

    else:  # 검색어를 입력하지 않은 경우 redirect시키기
        return redirect("/")
    return jsonify(
        render_template("report.html",
                        searchingBy=word,
                        resultNumber=len(news),
                        news=news,
                        datas=datas,
                        limit=limit,
                        page=page,
                        block_start=block_start,
                        block_end=block_end,
                        last_page_num=last_page_num,
                        graph=graph))
Example #15
0
                'if': {
                    'filter_query': '{{id}} = {}'.format(r['id']) # matching rows of a hidden column with the id, `id`
                },
                'color': 'tomato',
                'fontWeight': 'bold'
                } for r in recommendations]
    if return_A:
        return cond_rec, recommendations, query, distances
    else:
        return cond_rec, recommendations

cond_rec, recommendations, query, distances = get_recommendations(return_A=True)
display_columns = ['title', 'authors_short']
day = lambda i: '{:d} days ago'.format(abs(i)) if i != 0 else 'today'

network_fig, colors = graph.get_graph(query + recommendations, distances, n_query=len(query))
# =============== LAYOUT ===================

# external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']
# app = dash.Dash(__name__, external_stylesheets=external_stylesheets)
app = dash.Dash(
    __name__,
    meta_tags=[{"name": "viewport", "content": "width=device-width, initial-scale=1"}],
)
app.title = 'paper-scraper'
panel_color = '#161a28'
suffix_row = "_row"
suffix_button_id = "_button"
suffix_sparkline_graph = "_sparkline_graph"
suffix_count = "_count"
suffix_ooc_n = "_OOC_number"
#
#  For more details, try:\n
#  python3 classifier.py -h\n
#  python3 classifier.py train -h\n
#  python3 classifier.py test -h\n
#  example: python3 classifier.py -b 20 -s 17845 train ../data/ ../results/ -e 100
import graph
import dataio
import argument
import procedure
import logging as log
import tensorflow as tf
import numpy as np

logger = log.getLogger("classifier")
args = argument.args
dataio.save_command_line(args)

if args.seed is not None:
    np.random.seed(seed=args.seed)
    tf.set_random_seed(args.seed)

spectrums, labels = dataio.get_data(args)
data_tensors = dataio.get_data_tensors(args, spectrums, labels)
graph = graph.get_graph(args, data_tensors)
with tf.Session() as sess:
    procedure.initialize(sess, graph, args.test_or_train == 'test')
    output_data = procedure.run(sess, args, graph)
    dataio.save(sess, args, output_data)
logger.info("Success")
Example #17
0
    def bot_work(self):
        """Основной алгоритм работы бота"""
        # В цикле производится проверка longpool api
        room_name = ''

        for event in self.__longpoll.listen():
            # Событие "Добавление участника в группу"
            if event.type == VkBotEventType.GROUP_JOIN:
                self.__vk.messages.send(
                    user_id=event.obj.user_id,
                    random_id=get_random_id(),
                    keyboard=self.__rooms_keyboard.get_keyboard(),
                    message=
                    ('Привет, я помогу тебе узнать температуру, влажность и давление воздуха.'
                     +
                     '\nВоспользуйся клавиатурой для взаимодействия со мной.'),
                )

            # Событие "Новое сообщение"
            if event.type == VkBotEventType.MESSAGE_NEW:
                # Проверка на пустое сообщение
                if event.obj.text != '':
                    # Если сообщение пришло от пользователя
                    if event.from_user:
                        # Производится форматирование строки, удаляются знаки препинания
                        transtab = str.maketrans(
                            {key: None
                             for key in string.punctuation})
                        message_text = event.obj.text.translate(transtab)

                        # Если текст сообщения - это название комнаты
                        if message_text in self.__room_list:
                            room_name = message_text

                            self.__vk.messages.send(
                                user_id=event.obj.from_id,
                                random_id=get_random_id(),
                                keyboard=self.__function_keyboard.get_keyboard(
                                ),
                                message='Выберите функцию',
                            )

                        # Если текст сообщения - это название функции
                        elif message_text in self.__function_dict:
                            if room_name != '':
                                climate_data = self.__database.get_climate_data(
                                    room_name,
                                    self.__function_dict.get(message_text))
                                # Проверка данных для определения дальнейших действий
                                if not climate_data:
                                    # Если данных нет
                                    self.__vk.messages.send(
                                        user_id=event.obj.from_id,
                                        random_id=get_random_id(),
                                        keyboard=self.__rooms_keyboard.
                                        get_keyboard(),
                                        message=
                                        'Нет данных по этому помещению\nВыберите другое'
                                    )

                                else:
                                    # Если данные по помещению есть - проверяем полученные данные и определяем дальнейшие действия
                                    for cd in climate_data:
                                        if isinstance(climate_data.get(cd),
                                                      dict):
                                            print("dict")
                                            # Если функция требовала отчет - создаем график
                                            self.__vk.messages.send(
                                                user_id=event.obj.from_id,
                                                random_id=get_random_id(),
                                                message=
                                                'Минутку, надо найти фломастеры...',
                                            )

                                            title = room_name + ' - ' + message_text
                                            graph_file_name = graph.get_graph(
                                                climate_data,
                                                self.__measuring_system, title)

                                            if os.path.exists(graph_file_name):
                                                photo = self.__upload.photo_messages(
                                                    photos=graph_file_name)[0]

                                                self.__vk.messages.send(
                                                    user_id=event.obj.from_id,
                                                    random_id=get_random_id(),
                                                    keyboard=self.
                                                    __function_keyboard.
                                                    get_keyboard(),
                                                    message=
                                                    'Вот график за заданный интервал\nКрасивый, правда?',
                                                    attachment='photo{}_{}'.
                                                    format(
                                                        photo['owner_id'],
                                                        photo['id']))

                                            else:
                                                self.__vk.messages.send(
                                                    user_id=event.obj.from_id,
                                                    random_id=get_random_id(),
                                                    message=
                                                    ('Я нарисовал график, но кто-то его украл, пока я убирал фломастеры...\n'
                                                     +
                                                     'Простите, пожалуйста... :('
                                                     ),
                                                )

                                            break  # Прерываем цикл начатый для проверки

                                        elif isinstance(
                                                climate_data.get(cd), float
                                                or str):
                                            # Если функция требовала вывести последние данные
                                            msg_text = ''
                                            print(climate_data)
                                            for table in climate_data:
                                                if table == 'Температура':
                                                    msg_text += 'Температура: ' + str(
                                                        climate_data[table]
                                                    ) + ' °C \n'
                                                elif table == 'Влажность':
                                                    msg_text += 'Влажность: ' + str(
                                                        climate_data[table]
                                                    ) + ' %\n'
                                                elif table == 'Давление':
                                                    if self.__measuring_system == 'мм.рт.ст.':
                                                        msg_text += 'Давление: ' + str(
                                                            climate_data[table]
                                                        ) + ' мм.рт.ст.\n'
                                                    else:
                                                        msg_text += 'Давление: ' + str(
                                                            climate_data[table]
                                                        ) + ' гПа\n'
                                                else:
                                                    msg_text += str(
                                                        table) + ': ' + str(
                                                            climate_data[table]
                                                        )

                                            self.__vk.messages.send(
                                                user_id=event.obj.from_id,
                                                random_id=get_random_id(),
                                                keyboard=self.
                                                __function_keyboard.
                                                get_keyboard(),
                                                message=msg_text,
                                            )

                                            break  # Прерываем цикл начатый для проверки

                                        else:
                                            self.__vk.messages.send(
                                                user_id=event.obj.from_id,
                                                random_id=get_random_id(),
                                                keyboard=self.__rooms_keyboard.
                                                get_keyboard(),
                                                message=
                                                'Что-то пошло не так... :(\n Давайте попробуем сначала',
                                            )
                                            break  # Прерываем цикл начатый для проверки

                        elif message_text == 'Назад':
                            self.__vk.messages.send(
                                user_id=event.obj.from_id,
                                random_id=get_random_id(),
                                keyboard=self.__rooms_keyboard.get_keyboard(),
                                message='Выберите помещение',
                            )
                        else:
                            self.__vk.messages.send(
                                user_id=event.obj.from_id,
                                random_id=get_random_id(),
                                keyboard=self.__rooms_keyboard.get_keyboard(),
                                message=
                                'Простите, я не понимаю. Воспользуйтесь клавиатурой, пожалуйста',
                            )
 def test_get_graph_find_k(self):
     G = get_graph(self.points)
     self.assertEqual(G.degree[0], 2)
      print i, j, s_score
  return (highest_score, highest_num_cluster, highest_threshold)

def find_important(G, n):
  scores = nx.betweenness_centrality(G).items()
  l = [ ] 
  for (item, score) in scores:
    l.append((score, item))
  l.sort(reverse=True)
  important = []
  for i in range(n):
    important.append(l[i][1])
  return important
  
if __name__ == '__main__':
  g = graph.get_graph('/data/512/10000-twitter.gexf', limit=10000)
  top_subgraphs = graph.get_subgraphs(g, k=1)
  g = top_subgraphs[0]
  graph.display_graph(g)
  #t = find_best_params(g)
  #t2 = find_best_weighted_params(g)

  labels, score_1 = scluster(g, 4, 3)
  graph.display_graph_clusters(g, labels)

  important = find_important(g, 4)
  labels, score_2 = weighted_scluster(g, 4, 3, important)
  graph.display_graph_clusters(g, labels)

  print score_1, score_2
 def test_weights_are_as_expected(self):
     G = get_graph(self.points)
     actual_dist = G.get_edge_data(0, 2)['weight']
     self.assertEqual(3, actual_dist)