def get_top_sectors():
synonyms = load_synonyms('./datasets/sinonimos.csv')
synonyms1 = load_synonyms('./datasets/sinonimos2.csv')
dictionary = load_words()
stop_words = load_stop_words('./datasets/stop-words.txt')
routes = load_routes('./datasets/routes.txt')
counter = Counter()
with open('counter.txt') as fp:
counter = pickle.load(fp)
topRoutes = set(counter.elements())
sectorGraph = get_graph()
listRoutes = list(topRoutes)
topSectors = []
for avenue in listRoutes:
for (x, y) in sectorGraph.edges():
routesEdge = sectorGraph.edge[x][y]['routes']
for route in routesEdge:
processedRoute = process_tweet(route, synonyms, synonyms1,
dictionary, stop_words)
if (processedRoute.find(avenue) > -1):
topSectors.append({'from': x, 'to': y})
return json.dumps(topSectors)
def __init__(self, number, myWorldState, agType):
super().__init__(number, myWorldState, agType)
# the environment
self.agOperatingSets = []
self.number = number
# self.news = np.zeros(common.dim+3) # contiene l'ultima notizia
# [id-fonte, id-mittente, data, topics(dim)]
if myWorldState != 0:
self.myWorldState = myWorldState
self.agType = agType
self.state = np.zeros([common.dim])
self.active = True
#self.databaseCols = [
# 'id-n', 'new', 'id-source', 'date-creation', 'relevance',
# 'id-send', 'date-send', 'id-recive', 'date-recive'
#]
self.database = db.database()
self.spreadState = 's'
#print(LOG_LABEL, "agent", self.agType, "#", self.number,
#"has been created")
# =========================================================================================
#
# GRAPH CREATION
#
if graph.get_graph() == 0:
graph.create_graph() # if first agent create the graph
common.G.add_node(self.number, agent=self) # adds himself
# create link only if you are only at the first step of the clock
# and if you are the last user
if common.cycle == 1 and len(common.G.nodes()) == common.N_AGENTS:
graph.initialize_edges() # if last creates edges
graph.change_s_to_i()
def get_top_sectors():
synonyms = load_synonyms('./datasets/sinonimos.csv')
synonyms1 = load_synonyms('./datasets/sinonimos2.csv')
dictionary = load_words()
stop_words = load_stop_words('./datasets/stop-words.txt')
routes = load_routes('./datasets/routes.txt')
counter = Counter()
with open('counter.txt') as fp:
counter = pickle.load(fp)
topRoutes = set(counter.elements())
sectorGraph = get_graph()
listRoutes = list(topRoutes)
topSectors = []
for avenue in listRoutes:
for (x, y) in sectorGraph.edges():
routesEdge = sectorGraph.edge[x][y]['routes']
for route in routesEdge:
processedRoute = process_tweet(route, synonyms, synonyms1, dictionary, stop_words)
if (processedRoute.find(avenue) > -1):
topSectors.append({'from': x, 'to': y})
return json.dumps(topSectors)
def find_path(origin, destination):
queue = [(0, origin)]
path = {}
path[(0, origin)] = -1
visitedNodes = set()
graph = get_graph()
now = datetime.now()
now = datetime(2015,05,07,15,00)
print "Current Date: %s\n" %(now)
while queue:
#Tuple: node = (cost, current sector)
node = accumulatedCost, currentNode = heapq.heappop(queue)
print '----- Current Node: %s -----\n' %(currentNode)
if currentNode == destination:
path = list(build_path(path, node))
path.reverse()
yield path
continue
if currentNode in visitedNodes:
continue
visitedNodes.add(currentNode)
accumulatedTime = now + timedelta(minutes=accumulatedCost)
for successor in graph[currentNode]:
actualScore = get_stream_score(currentNode, successor, now=now, spoof=True)
print 'ACTUAL SCORE: %.2f' %(actualScore)
before = accumulatedTime + timedelta(minutes=-10)
after = accumulatedTime + timedelta(minutes=10)
historicScore = get_historic_score(currentNode, successor,
before.strftime('%H:%M:00'),
after.strftime('%H:%M:00'))
print 'HISTORIC SCORE: %.2f' %(historicScore)
estimatedScore = (1-phi(accumulatedCost))*actualScore + phi(accumulatedCost)*historicScore
#TODO change so that each node has a different traffic penalty function
congestionValue = get_TravelTime(graph,currentNode,successor)*estimatedScore
cost = accumulatedCost + get_TravelTime(graph,currentNode,successor) + congestionValue
path[(cost, successor)] = node
print "ARC: %s -> %s ***** SCORE: %.2f, COST: %.2f\n" %(currentNode, successor, estimatedScore, cost)
heapq.heappush(queue, (cost, successor))
def __init__(self, config):
self.df = read_data(config)
self.organs = config.ORGANS
self.components = config.COMPONENTS
self.co_in, self.co_in123, self.co_in45 = get_graph()
self.get_graph_mapping()
self.data_arrs = self.get_data_arrs()
self.pca_est, self.transformed = self.get_estimators()
self.cluster_torsos(neighbors=30)
def related_tweets(source, dest, since_date=None, before_date=None):
'''Returns all related tweets for `via`.'''
graph = get_graph()
if not graph.has_edge(source, dest):
return
keywords = graph[source][dest]['keywords']
filters = or_(*[Tweet.text.ilike('%%%s%%' % x) for x in keywords])
qs = Tweet.query.filter(filters)
if since_date is not None:
qs = qs.filter(cast(Tweet.created_at, Date) >= since_date)
if before_date is not None:
qs = qs.filter(cast(Tweet.created_at, Date) <= before_date)
return qs
def related_tweets(source, dest, since_date=None, before_date=None):
'''Returns all related tweets for `via`.'''
graph = get_graph()
if not graph.has_edge(source, dest):
return
keywords = graph[source][dest]['keywords']
filters = or_(*[Tweet.text.ilike('%%%s%%' % x) for x in keywords])
qs = Tweet.query.filter(filters)
if since_date is not None:
qs = qs.filter(cast(Tweet.created_at, Date) >= since_date)
if before_date is not None:
qs = qs.filter(cast(Tweet.created_at, Date) <= before_date)
return qs
def display_click_data(clickData,fig_master,n_clicks, fig, fig_1,zoomed):
ctx = dash.callback_context
ctx = ctx.triggered[0]['prop_id']
if ctx == 'network-graph.clickData' and clickData:
based_on = [json.loads(clickData['points'][0]['customdata'])]
curve_no = clickData['points'][0]['pointNumber']
if zoomed and curve_no == 0:
return fig_1, fig_master, 0
cond_rec, recommendations, query, distances = get_recommendations(10, 10000,
based_on = based_on, return_A=True)
total = query + recommendations
network_fig, colors = graph.get_graph(total, distances, n_query=1)
return network_fig, fig, 1
elif ctx in ['network-graph-master.figure','reset-button.n_clicks']:
return fig_master, fig_master, 0
else:
return fig, fig_1, 0
def update_recommendations(time_lim, no_papers, ref_trigger, *states):
logging.warning(ref_trigger)
if ref_trigger:
cond_rec, recommendations, query, distances = get_recommendations(no_papers, time_lim, return_A=True)
total = query + recommendations
network_fig_, colors = graph.get_graph(total, distances, n_query=len(query))
cond_rec_bm = [{
'if': {
'filter_query': '{{id}} = {}'.format(r['id']) # matching rows of a hidden column with the id, `id`
},
'color': c,
} for r, c in zip(total,colors)]
if len(total):
return cond_rec, get_authors_short(recommendations), network_fig_, cond_rec_bm
else:
return [],[{'title':'No bookmarks yet...','authors':''}], network_fig_, cond_rec_bm
else:
return states
def welcome():
# model = Net()
# model.load_state_dict(torch.load(MODEL_PATH))
# torch.no_grad()
# model.eval()
if request.method == 'POST':
try:
file = request.files['file']
except:
return render_template('welcome_page.html')
if file:
filename = file.filename
file.save(os.path.join(UPLOAD_DIR, filename))
#render graph
graph_name = get_graph(UPLOAD_DIR + filename, model)
return render_template('prediction.html', graph=graph_name)
return render_template('welcome_page.html', title='Application')
written by Rob Speer, Julian Chaidez
Common Sense Computing Group, Medialab
Massachusetts Institute of Technology
Fall 2011
"""
import graph
import flask
import urllib
import re
import sys
import simplejson
from werkzeug.contrib.cache import SimpleCache
app = flask.Flask(__name__)
concept_graph = graph.get_graph()
if len(sys.argv) == 1:
root_url = 'http://conceptnet5.media.mit.edu/data'
else:
root_url = sys.argv[1]
cache_dict = {
'limit_timeout':60,
'limit_amount':10000
}
request_cache = SimpleCache(default_timeout = cache_dict['limit_timeout'])
no_leading_slash = ['http']
def test_get_specified_k_graph(self):
G = get_graph(self.points, k=2)
self.assertEqual(list(G.neighbors(0)), [1, 2])
from co_author_crawler import crawler
import graph
if __name__ == '__main__':
nodes1 = []
nodes2 = []
edges = []
graph.get_graph(nodes1, nodes2, edges)
graph.get_graph_by_institution(nodes1, nodes2, edges)
def report():
word = request.get_json("word")
if "num_page" in word:
result = word.split('num_page')
word = result[0]
page = int(result[1])
else:
# 페이지 값 (디폴트값 = 1)
page = request.args.get("page", 1, type=int)
# 한 페이지 당 몇 개의 게시물을 출력할 것인가
limit = 10
# 그래프 이미지 가져오기
graph = get_graph()
if word:
word = word.lower()
existingNews = db.get(word)
if existingNews:
news = existingNews
datas = news[(page - 1) * limit:limit * page]
# 게시물의 총 개수 세기
tot_count = len(news)
# 마지막 페이지의 수 구하기
last_page_num = math.ceil(tot_count / limit) # 반드시 올림을 해줘야함
# 페이지 블럭을 5개씩 표기
block_size = 5
# 현재 블럭의 위치 (첫 번째 블럭이라면, block_num = 0)
block_num = int((page - 1) / block_size)
# 현재 블럭의 맨 처음 페이지 넘버 (첫 번째 블럭이라면, block_start = 1, 두 번째 블럭이라면, block_start = 6)
block_start = (block_size * block_num) + 1
# 현재 블럭의 맨 끝 페이지 넘버 (첫 번째 블럭이라면, block_end = 5)
block_end = block_start + (block_size - 1)
else:
# 뉴스 기사 가져오기
naver_news = get_naver_news(word)
hankyung_news = get_hankyung_news(word)
news = naver_news + hankyung_news
db[word] = news
datas = news[(page - 1) * limit:limit * page]
# 게시물의 총 개수 세기
tot_count = len(news)
# 마지막 페이지의 수 구하기
last_page_num = math.ceil(tot_count / limit) # 반드시 올림을 해줘야함
# 페이지 블럭을 5개씩 표기
block_size = 5
# 현재 블럭의 위치 (첫 번째 블럭이라면, block_num = 0)
block_num = int((page - 1) / block_size)
# 현재 블럭의 맨 처음 페이지 넘버 (첫 번째 블럭이라면, block_start = 1, 두 번째 블럭이라면, block_start = 6)
block_start = (block_size * block_num) + 1
# 현재 블럭의 맨 끝 페이지 넘버 (첫 번째 블럭이라면, block_end = 5)
block_end = block_start + (block_size - 1)
else: # 검색어를 입력하지 않은 경우 redirect시키기
return redirect("/")
return jsonify(
render_template("report.html",
searchingBy=word,
resultNumber=len(news),
news=news,
datas=datas,
limit=limit,
page=page,
block_start=block_start,
block_end=block_end,
last_page_num=last_page_num,
graph=graph))
'if': {
'filter_query': '{{id}} = {}'.format(r['id']) # matching rows of a hidden column with the id, `id`
},
'color': 'tomato',
'fontWeight': 'bold'
} for r in recommendations]
if return_A:
return cond_rec, recommendations, query, distances
else:
return cond_rec, recommendations
cond_rec, recommendations, query, distances = get_recommendations(return_A=True)
display_columns = ['title', 'authors_short']
day = lambda i: '{:d} days ago'.format(abs(i)) if i != 0 else 'today'
network_fig, colors = graph.get_graph(query + recommendations, distances, n_query=len(query))
# =============== LAYOUT ===================
# external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']
# app = dash.Dash(__name__, external_stylesheets=external_stylesheets)
app = dash.Dash(
__name__,
meta_tags=[{"name": "viewport", "content": "width=device-width, initial-scale=1"}],
)
app.title = 'paper-scraper'
panel_color = '#161a28'
suffix_row = "_row"
suffix_button_id = "_button"
suffix_sparkline_graph = "_sparkline_graph"
suffix_count = "_count"
suffix_ooc_n = "_OOC_number"
#
# For more details, try:\n
# python3 classifier.py -h\n
# python3 classifier.py train -h\n
# python3 classifier.py test -h\n
# example: python3 classifier.py -b 20 -s 17845 train ../data/ ../results/ -e 100
import graph
import dataio
import argument
import procedure
import logging as log
import tensorflow as tf
import numpy as np
logger = log.getLogger("classifier")
args = argument.args
dataio.save_command_line(args)
if args.seed is not None:
np.random.seed(seed=args.seed)
tf.set_random_seed(args.seed)
spectrums, labels = dataio.get_data(args)
data_tensors = dataio.get_data_tensors(args, spectrums, labels)
graph = graph.get_graph(args, data_tensors)
with tf.Session() as sess:
procedure.initialize(sess, graph, args.test_or_train == 'test')
output_data = procedure.run(sess, args, graph)
dataio.save(sess, args, output_data)
logger.info("Success")
def bot_work(self):
"""Основной алгоритм работы бота"""
# В цикле производится проверка longpool api
room_name = ''
for event in self.__longpoll.listen():
# Событие "Добавление участника в группу"
if event.type == VkBotEventType.GROUP_JOIN:
self.__vk.messages.send(
user_id=event.obj.user_id,
random_id=get_random_id(),
keyboard=self.__rooms_keyboard.get_keyboard(),
message=
('Привет, я помогу тебе узнать температуру, влажность и давление воздуха.'
+
'\nВоспользуйся клавиатурой для взаимодействия со мной.'),
)
# Событие "Новое сообщение"
if event.type == VkBotEventType.MESSAGE_NEW:
# Проверка на пустое сообщение
if event.obj.text != '':
# Если сообщение пришло от пользователя
if event.from_user:
# Производится форматирование строки, удаляются знаки препинания
transtab = str.maketrans(
{key: None
for key in string.punctuation})
message_text = event.obj.text.translate(transtab)
# Если текст сообщения - это название комнаты
if message_text in self.__room_list:
room_name = message_text
self.__vk.messages.send(
user_id=event.obj.from_id,
random_id=get_random_id(),
keyboard=self.__function_keyboard.get_keyboard(
),
message='Выберите функцию',
)
# Если текст сообщения - это название функции
elif message_text in self.__function_dict:
if room_name != '':
climate_data = self.__database.get_climate_data(
room_name,
self.__function_dict.get(message_text))
# Проверка данных для определения дальнейших действий
if not climate_data:
# Если данных нет
self.__vk.messages.send(
user_id=event.obj.from_id,
random_id=get_random_id(),
keyboard=self.__rooms_keyboard.
get_keyboard(),
message=
'Нет данных по этому помещению\nВыберите другое'
)
else:
# Если данные по помещению есть - проверяем полученные данные и определяем дальнейшие действия
for cd in climate_data:
if isinstance(climate_data.get(cd),
dict):
print("dict")
# Если функция требовала отчет - создаем график
self.__vk.messages.send(
user_id=event.obj.from_id,
random_id=get_random_id(),
message=
'Минутку, надо найти фломастеры...',
)
title = room_name + ' - ' + message_text
graph_file_name = graph.get_graph(
climate_data,
self.__measuring_system, title)
if os.path.exists(graph_file_name):
photo = self.__upload.photo_messages(
photos=graph_file_name)[0]
self.__vk.messages.send(
user_id=event.obj.from_id,
random_id=get_random_id(),
keyboard=self.
__function_keyboard.
get_keyboard(),
message=
'Вот график за заданный интервал\nКрасивый, правда?',
attachment='photo{}_{}'.
format(
photo['owner_id'],
photo['id']))
else:
self.__vk.messages.send(
user_id=event.obj.from_id,
random_id=get_random_id(),
message=
('Я нарисовал график, но кто-то его украл, пока я убирал фломастеры...\n'
+
'Простите, пожалуйста... :('
),
)
break # Прерываем цикл начатый для проверки
elif isinstance(
climate_data.get(cd), float
or str):
# Если функция требовала вывести последние данные
msg_text = ''
print(climate_data)
for table in climate_data:
if table == 'Температура':
msg_text += 'Температура: ' + str(
climate_data[table]
) + ' °C \n'
elif table == 'Влажность':
msg_text += 'Влажность: ' + str(
climate_data[table]
) + ' %\n'
elif table == 'Давление':
if self.__measuring_system == 'мм.рт.ст.':
msg_text += 'Давление: ' + str(
climate_data[table]
) + ' мм.рт.ст.\n'
else:
msg_text += 'Давление: ' + str(
climate_data[table]
) + ' гПа\n'
else:
msg_text += str(
table) + ': ' + str(
climate_data[table]
)
self.__vk.messages.send(
user_id=event.obj.from_id,
random_id=get_random_id(),
keyboard=self.
__function_keyboard.
get_keyboard(),
message=msg_text,
)
break # Прерываем цикл начатый для проверки
else:
self.__vk.messages.send(
user_id=event.obj.from_id,
random_id=get_random_id(),
keyboard=self.__rooms_keyboard.
get_keyboard(),
message=
'Что-то пошло не так... :(\n Давайте попробуем сначала',
)
break # Прерываем цикл начатый для проверки
elif message_text == 'Назад':
self.__vk.messages.send(
user_id=event.obj.from_id,
random_id=get_random_id(),
keyboard=self.__rooms_keyboard.get_keyboard(),
message='Выберите помещение',
)
else:
self.__vk.messages.send(
user_id=event.obj.from_id,
random_id=get_random_id(),
keyboard=self.__rooms_keyboard.get_keyboard(),
message=
'Простите, я не понимаю. Воспользуйтесь клавиатурой, пожалуйста',
)
def test_get_graph_find_k(self):
G = get_graph(self.points)
self.assertEqual(G.degree[0], 2)
print i, j, s_score
return (highest_score, highest_num_cluster, highest_threshold)
def find_important(G, n):
scores = nx.betweenness_centrality(G).items()
l = [ ]
for (item, score) in scores:
l.append((score, item))
l.sort(reverse=True)
important = []
for i in range(n):
important.append(l[i][1])
return important
if __name__ == '__main__':
g = graph.get_graph('/data/512/10000-twitter.gexf', limit=10000)
top_subgraphs = graph.get_subgraphs(g, k=1)
g = top_subgraphs[0]
graph.display_graph(g)
#t = find_best_params(g)
#t2 = find_best_weighted_params(g)
labels, score_1 = scluster(g, 4, 3)
graph.display_graph_clusters(g, labels)
important = find_important(g, 4)
labels, score_2 = weighted_scluster(g, 4, 3, important)
graph.display_graph_clusters(g, labels)
print score_1, score_2
def test_weights_are_as_expected(self):
G = get_graph(self.points)
actual_dist = G.get_edge_data(0, 2)['weight']
self.assertEqual(3, actual_dist)