for i, batch in enumerate(iterator_unlabeled):
inputs = {}
inputs['words'], inputs['length'] = batch.token
inputs['pos'] = batch.pos
inputs['ner'] = batch.ner
inputs['subj_pst'] = batch.subj_pst
inputs['obj_pst'] = batch.obj_pst
inputs['masks'] = torch.eq(batch.token[0], opt_p['vocab_pad_id'])
target = batch.relation
if args.cuda:
inputs = dict([(k, v.cuda()) for k, v in inputs.items()])
predictor.eval()
pred = predictor.predict(inputs)
pred = pred.data.cpu().numpy().tolist()
preds_p += pred
selector.eval()
pred = selector.predict(inputs)
pred = pred.data.cpu().numpy().tolist()
preds_s += pred
def arg_max(l):
bvl, bid = -1, -1
for k in range(len(l)):
if l[k] > bvl:
bvl = l[k]
bid = k
#print("pred_missing_tagspace", curr_pred_missing_tagspace)
if args.post:
curr_pred_missing_tagspace = set()
pred_missing_tagspace = sorted([tag2idx[untag] for untag in curr_pred_missing_tagspace])
pred_missing_tagspace = [pred_missing_tagspace] * len(filtered_gold_labels)
dataloader = build_dataloader(pred_tokens, filtered_gold_labels, 50, pred_missing_tagspace, 0, tag2idx, chr2idx, token2idx, train_args['caseless'], shuffle=False, drop_last=False)
# choose to eval the local prediction on the training brach
# unzip batch and group by sents
merge_batch = True
#pred_tags = predictor.predict(ner_model, dataloader, crf_idx, merge_batch=merge_batch, totag=False)
pred_tags = predictor.predict(ner_model, dataloader, crf_idx, args.pred_method, merge_batch=True, totag=True)
pred_tags = sorted(pred_tags, key=lambda item:item[0])
reorder_idx, pred_tags = zip(*pred_tags)
for i, re_idx in enumerate(sorted(reorder_idx)):
assert i == re_idx
if merge_batch:
assert len(pred_tokens) == len(pred_tags), "{} {}".format(len(pred_tokens), len(pred_tags))
else:
num_sample = sum(map(lambda t: len(t), dataloader))
assert num_sample == len(pred_tags), "{} {}".format(len(pred_tokens), len(pred_tags))
train_corpus = [train_args["train_file"][i].split("/")[-2] for i in crf2corpus[crf_idx]]
crf_training_corpus.append(train_corpus)
# store the prediction of current brach
local_pred_tags.append(pred_tags)
def planner(trajectory):
map_size = 12
step_resolution = 0.25
step_size = 10
obstacle_width = 10
obstacle_thickness = 5
agent_step = step_resolution * step_size
plt.figure()
currentAxis = plt.gca()
plt.axis([-map_size, map_size, -map_size, map_size])
plt.title("RRG Route Map for the Maze Problem")
plt.xlabel('X')
plt.ylabel('Y')
Node = namedtuple('Node', ['x', 'y'])
task_test = tasks[105]
predictor = Predictor(task=task_test,
checkpoint_path='../data/checkpoints/checkpoint_' +
task_test.task_name + '.pt')
_, obstacle_pos_prediction = predictor.predict(trajectory)
def is_valid_move(start_node, dir_x, dir_y):
global obstacle
end_x = start_node.x
end_y = start_node.y
start_step = calculate_step(start_node)
for s in range(step_size):
if (end_x < -map_size) or (end_x > map_size) or (
end_y < -map_size) or (end_y > map_size):
return False
for obstacle in walls:
if obstacle.interfere_node(end_x, end_y):
return False
if obstacle_range.interfere_node(end_x, end_y):
cur_step = start_step + s
if cur_step >= 100:
return False
obstacle = Area(obstacle_pos_prediction[0, cur_step, 0].item(),
0, obstacle_width, obstacle_thickness)
if obstacle.interfere_node(end_x, end_y):
return False
end_x += dir_x * step_resolution
end_y += dir_y * step_resolution
return True
def bfs(end_node, plot=False):
parents = {}
finished = set()
queue = deque()
queue.append(origin)
while queue:
parent = queue.popleft()
finished.add(parent)
if plot:
plt.plot([
parent.x,
], [
parent.y,
],
'o',
color='gray',
markersize=4)
for child in route_dict[parent]:
if child not in finished:
if plot:
plt.plot([parent.x, child.x], [parent.y, child.y],
color='gray')
if child not in queue:
queue.append(child)
parents[child] = parent
cur_node = end_node
path = [end_node]
while cur_node != origin:
cur_node = parents[cur_node]
path.append(cur_node)
path.reverse()
return path
def calculate_step(end_node):
cur_path = bfs(end_node=end_node, plot=False)
return (len(cur_path) - 1) * step_size
obstacle = Area(0, 0, obstacle_width, obstacle_thickness)
wall0 = Area(-8, 7.5, 12, 5)
wall1 = Area(8, -7.5, 12, 5)
walls = [wall0, wall1]
target = Area(-8, 11, 8, 2)
obstacle_range = Area(0, 0, map_size * 2 + 4, 5)
origin = Node(2, -11)
route_dict = {origin: set()}
done = False
last_node = None
while not done:
pos_rand_x = random.uniform(-map_size, map_size)
pos_rand_y = random.uniform(-map_size, map_size)
min_distance = float('inf')
n_nearest = None
for node in route_dict.keys():
distance = sqrt(
abs(node.x - pos_rand_x) + abs(node.y - pos_rand_y))
if min_distance > distance:
min_distance = distance
n_nearest = node
if min_distance > agent_step:
continue
direction_x, direction_y = choose_direction(n_nearest.x, n_nearest.y,
pos_rand_x, pos_rand_y)
if not is_valid_move(n_nearest, direction_x, direction_y):
continue
new_node_x = n_nearest.x + direction_x * agent_step
new_node_y = n_nearest.y + direction_y * agent_step
new_node = Node(new_node_x, new_node_y)
if new_node in route_dict:
continue
route_dict[new_node] = set()
route_dict[n_nearest].add(new_node)
route_dict[new_node].add(n_nearest)
for p in route_dict.keys():
direction_x, direction_y = choose_direction(
p.x, p.y, new_node.x, new_node.y)
if abs(p.x - new_node.x) == agent_step and abs(p.y - new_node.y) == agent_step \
and is_valid_move(p, direction_x, direction_y):
route_dict[p].add(new_node)
if target.interfere_node(new_node_x, new_node_y):
last_node = new_node
done = True
path = bfs(end_node=last_node, plot=True)
for milestone in path:
if obstacle_range.interfere_node(milestone.x, milestone.y):
danger_step = calculate_step(milestone)
if obstacle.interfere_node(milestone.x, milestone.y):
return None
obstacle = Area(obstacle_pos_prediction[0, danger_step, 0].item(),
0, obstacle_width, obstacle_thickness)
currentAxis.add_patch(
Rectangle((obstacle.min_x, obstacle.min_y),
obstacle.x_width,
obstacle.y_width,
fill=True,
facecolor="blue",
alpha=1))
policy = []
for i in range(len(path) - 1):
if path[i + 1].x < path[i].x:
policy_unit_x = -1
elif path[i + 1].x > path[i].x:
policy_unit_x = 1
else:
policy_unit_x = 0
if path[i + 1].y < path[i].y:
policy_unit_y = -1
elif path[i + 1].y > path[i].y:
policy_unit_y = 1
else:
policy_unit_y = 0
policy.append([policy_unit_x, policy_unit_y])
currentAxis.add_patch(
Rectangle((wall0.min_x, wall0.min_y),
wall0.x_width,
wall0.y_width,
fill=True,
facecolor="green",
alpha=1))
currentAxis.add_patch(
Rectangle((wall1.min_x, wall1.min_y),
wall1.x_width,
wall1.y_width,
fill=True,
facecolor="green",
alpha=1))
currentAxis.add_patch(
Rectangle((obstacle_range.min_x, obstacle_range.min_y),
obstacle_range.x_width,
obstacle_range.y_width,
fill=True,
facecolor="blue",
alpha=0.2))
currentAxis.add_patch(
Rectangle((target.min_x, target.min_y),
target.x_width,
target.y_width,
fill=True,
facecolor="purple",
alpha=0.5))
for i in range(1, len(path)):
plt.plot([
path[i].x,
], [
path[i].y,
], ' o', color='orange')
plt.plot([path[i - 1].x, path[i].x], [path[i - 1].y, path[i].y],
color='orange',
lw=3)
plt.plot([
origin.x,
], [
origin.y,
], 'rs', markersize=10)
plt.show()
return policy
class Bot:
def __init__(self, db, model_name):
group_token = os.environ['GROUP_TOKEN']
group_id = int(os.environ['GROUP_ID'])
service_token = os.environ['SERVICE_TOKEN']
app_id = int(os.environ['APP_ID'])
client_secret = os.environ['CLIENT_SECRET']
self.visited = set()
self.admin_pwd = os.environ['ADMIN_PWD']
self.new_cats = sorted([
'физика', 'математика', 'лингвистика', 'информатика', 'литература',
'химия', 'география', "психология", "обществознание", "история",
"музыка", "астрономия", "маркетинг", "биология", "спорт",
"искусство", "бизнес"
])
self.predictor = Predictor(model_name)
self.db = db
self.db_session = db.create_session()
self.group_session = vk_api.VkApi(token=group_token,
api_version='5.126')
self.service_session = vk_api.VkApi(app_id=app_id,
token=service_token,
client_secret=client_secret)
self.long_poll = VkBotLongPoll(self.group_session, group_id)
self.group_api = self.group_session.get_api()
self.service_api = self.service_session.get_api()
# For dataset filtering
self.latest_id = self.db_session.query(db.Groups.group_id).order_by(
db.Groups.group_id.desc()).first()
if self.latest_id is None:
self.latest_id = 0
else:
self.latest_id = self.latest_id[0]
def send_message(self,
user_id: int,
message: str,
keyboard: str = None) -> None:
"""
sends a message to user using method messages.send
(https://vk.com/dev/messages.send)
:param user_id: recipient user ID
:param message: message text
:param keyboard: json describing keyboard attached with message
:return: None
"""
self.group_api.messages.send(user_id=user_id,
random_id=get_random_id(),
message=message,
keyboard=keyboard)
print(f'<-- message {message[:30]}{"..." if len(message) > 30 else ""}'
f' to {user_id} has been sent')
def get_posts(self,
owner_id: int,
count: int = 1) -> Union[List[dict], dict]:
"""
gets posts from user's or group's wall using method wall.get
(https://vk.com/dev/wall.get)
:param owner_id: wall's owner ID
:param count: count of posts
:return: list of dictionaries of dictionary, describing post
"""
posts = self.service_api.wall.get(owner_id=owner_id, count=count)
print(f'group {owner_id} posts received')
try:
if len(posts['items']) > 1:
return posts['items']
else:
return posts['items'][0]
except IndexError:
print(f'error: {owner_id} {posts}')
def get_subscriptions(self, user_id: int, count=100) -> List[int]:
"""
gets user's subscriptions using method users.getSubscriptions
(https://vk.com/dev/users.getSubscriptions)
:param user_id: user ID
:param count: get random {count} groups
:return: list of numbers defining user IDs
"""
subscriptions = self.service_api.users.getSubscriptions(
user_id=user_id, extended=1)
print(f'received subscriptions from '
f'{"user" if user_id > 0 else "group"} {abs(user_id)}')
ids = [
i['id'] for i in subscriptions['items']
if not i['is_closed'] and 'type' in i and 'deactivated' not in i
]
return ids if len(ids) <= count else sample(ids, count)
def get_group_info(
self, group_id: int
) -> Union[Dict[str, Union[str, int]], List[Dict[str, Union[str, int]]]]:
"""
gets information about one or more groups using method groups.getById
(https://vk.com/dev/groups.getById)
:param group_id: group ID
:return: list of dictionaries of dictionary, describing information
about group
"""
info = self.service_api.groups.getById(group_id=group_id)
print(f'received info from {group_id}')
if len(info) == 1:
return info[0]
else:
return info
def listen(self) -> None:
"""
gets updates from server and handling them
:return: None
"""
for event in self.long_poll.listen():
if event.type == VkBotEventType.MESSAGE_NEW:
self.process_new_message(event)
def process_new_message(self, event):
from_id = event.object['message']['from_id']
cmd = event.object['message']['text']
print(f'--> {from_id} sent "{cmd}"')
payload = json.loads(event.object['message'].get('payload', '{}'))
if payload.get('button') == 'start_analysis':
self.command_start_analysis(from_id)
elif ('button' in payload
and 'show_recommendation' in payload['button']):
self.command_show_recommendation(from_id, payload)
elif ''.join(filter(str.isalpha, cmd.lower())) == self.admin_pwd:
self.command_admin(from_id)
elif ('button' in payload and 'dataset_filter' in payload['button']):
self.command_dataset_filter(from_id, payload)
else:
self.command_start(from_id)
def command_start(self, from_id):
keyboard = VkKeyboard(one_time=True)
keyboard.add_button('Начать анализ',
color=VkKeyboardColor.POSITIVE,
payload=json.dumps({'button': 'start_analysis'}))
msg = ('Здравствуйте, я - Виталя, бот-рекомендатор. Я помогу вам '
'определить ваши интересы и подскажу, где найти ещё больше '
'полезных групп ВКонтакте. Начнём анализ?')
user = self.db_session.query(self.db.UserStatuses).filter(
self.db.UserStatuses.user_id == from_id).first()
if user and user.subjects:
keyboard.add_button('Перейти к рекомендациям',
color=VkKeyboardColor.SECONDARY,
payload=json.dumps(
{'button': 'show_recommendation_1'}))
msg = ('С возвращением! Желаете провести анализ снова или '
'посмотреть, что я рекомендовал вам в прошлый раз?'
if from_id in self.visited else 'Нужно нажать на кнопку')
self.visited.add(from_id)
self.send_message(from_id, msg, keyboard.get_keyboard())
user_status = self.db_session.query(self.db.UserStatuses).filter(
self.db.UserStatuses.user_id == from_id).first()
if user_status:
user_status.status = 'started'
else:
self.db_session.add(
self.db.UserStatuses(user_id=from_id, status='started'))
print(f'=== user {from_id} added')
self.db_session.commit()
def command_start_analysis(self, from_id):
texts = []
try:
group_ids = self.get_subscriptions(from_id)
except vk_api.exceptions.ApiError:
message = 'Ваш профиль закрыт, я не могу увидеть подписки'
keyboard = VkKeyboard(one_time=True)
keyboard.add_button('Теперь профиль открыт, начать анализ',
color=VkKeyboardColor.POSITIVE,
payload=json.dumps(
{'button': 'start_analysis'}))
self.send_message(from_id, message, keyboard.get_keyboard())
return
message = ('Анализ может занять несколько минут. Пожалуйста, '
'подождите.')
self.send_message(from_id, message)
for _id in group_ids:
try:
posts = map(
itemgetter('text'),
filter(lambda x: not x['marked_as_ads'],
self.get_posts(-_id, 10)))
texts.append('\n'.join(posts))
except TypeError:
continue
prediction = list(map(itemgetter(0),
self.predictor.predict(texts)[:3]))
user_status = self.db_session.query(self.db.UserStatuses).filter(
self.db.UserStatuses.user_id == from_id).first()
user_status.subjects = '&'.join(prediction)
user_status.status = 'show_page'
user_status.page = 1
self.db_session.commit()
message = 'В ходе анализа было выявлено, что вас ' \
'интересуют следующие категории групп:\n'
message += '\n'.join([
f'{i}. {category.capitalize()}'
for i, category in enumerate(prediction, 1)
])
self.send_message(from_id, message)
keyboard = VkKeyboard(one_time=True)
keyboard.add_button('Начать анализ повторно',
color=VkKeyboardColor.SECONDARY,
payload=json.dumps({'button': 'start_analysis'}))
group_ids = self.db_session.query(self.db.GroupsIds).filter(
or_(self.db.GroupsIds.subject == prediction[0],
self.db.GroupsIds.subject == prediction[1],
self.db.GroupsIds.subject == prediction[2])).all()
if len(group_ids) > 0:
show_groups = group_ids[:10]
message = 'Страница 1:\n'
message += '\n'.join([
f'{i + 1}. {show_groups[i].name} -- '
f'https://vk.com/club{show_groups[i].group_id} '
for i in range(len(show_groups))
])
page_number = len(group_ids) // 10 + 1
keyboard.add_line()
keyboard.add_button(f'Страница {page_number}',
color=VkKeyboardColor.PRIMARY,
payload=json.dumps({
'button':
f'show_recommendation_{page_number}'
}))
keyboard.add_button(f'Страница 2',
color=VkKeyboardColor.PRIMARY,
payload=json.dumps(
{'button': f'show_recommendation_2'}))
else:
message = "Проанализировать ещё раз?"
self.send_message(from_id, message, keyboard.get_keyboard())
def command_show_recommendation(self, from_id, payload):
page = int(payload['button'].split('_')[2])
recommendation = self.db_session.query(self.db.UserStatuses).filter(
self.db.UserStatuses.user_id == from_id).first()
recommendation = recommendation.subjects.split('&')
group_ids = self.db_session.query(self.db.GroupsIds).filter(
or_(self.db.GroupsIds.subject == recommendation[0],
self.db.GroupsIds.subject == recommendation[1],
self.db.GroupsIds.subject == recommendation[2])).all()
show_groups = group_ids[(page - 1) * 10:page * 10]
message = f'Страница {page}:\n'
message += '\n'.join([
f'{i + 1}. {show_groups[i].name} -- '
f'https://vk.com/club{show_groups[i].group_id}'
for i in range(len(show_groups))
])
keyboard = VkKeyboard(one_time=True)
keyboard.add_button('Начать анализ повторно',
color=VkKeyboardColor.SECONDARY,
payload=json.dumps({'button': 'start_analysis'}))
keyboard.add_line()
page_number = page - 1 if page > 1 else len(group_ids) // 10 + 1
keyboard.add_button(f'Страница {page_number}',
color=VkKeyboardColor.PRIMARY,
payload=json.dumps({
'button':
f'show_recommendation_{page_number}'
}))
page_number = (page + 1) % (len(group_ids) // 10 + 1)
page_number = page_number or len(group_ids) // 10 + 1
keyboard.add_button(f'Страница {page_number}',
color=VkKeyboardColor.PRIMARY,
payload=json.dumps({
'button':
f'show_recommendation_{page_number}'
}))
self.send_message(from_id, message, keyboard.get_keyboard())
user_status = self.db_session.query(self.db.UserStatuses).filter(
self.db.UserStatuses.user_id == from_id).first()
user_status.status = 'show_page'
user_status.page = page
self.db_session.commit()
def command_admin(self, from_id):
print(f'*** {from_id} entered admin panel')
keyboard = VkKeyboard(one_time=True)
keyboard.add_button('Фильтровать датасет',
color=VkKeyboardColor.PRIMARY,
payload=json.dumps({'button': 'dataset_filter'}))
keyboard.add_button('Выйти',
color=VkKeyboardColor.NEGATIVE,
payload=json.dumps({'command': 'start'}))
msg = 'Вы вошли в панель администратора'
self.send_message(from_id, msg, keyboard.get_keyboard())
user_status = self.db_session.query(self.db.UserStatuses).filter(
self.db.UserStatuses.user_id == from_id).first()
if user_status:
user_status.status = 'admin'
else:
self.db_session.add(
self.db.UserStatuses(user_id=from_id, status='admin'))
self.db_session.commit()
def command_dataset_filter(self, from_id, payload):
user_status = self.db_session.query(self.db.UserStatuses).filter(
self.db.UserStatuses.user_id == from_id).first()
if user_status.status == 'admin':
if '#' in payload['button']:
_, gr_id, cat = payload['button'].split('#')
gr_id = int(gr_id)
if gr_id > self.latest_id:
self.latest_id = gr_id
cat = self.new_cats[int(cat)] if cat != '-1' else 'other'
old_group = self.db_session.query(self.db.GroupsIds).get(
self.latest_id)
self.db_session.add(
self.db.Groups(group_id=self.latest_id,
name=old_group.name,
subject=cat,
link=old_group.link))
msg = (f"{old_group.name} теперь относится к группе "
f"{cat.capitalize()}")
else:
msg = f'Группа {gr_id} уже была добавлена'
self.send_message(from_id, msg)
group = self.db_session.query(self.db.GroupsIds).order_by(
self.db.GroupsIds.group_id.asc()).filter(
self.db.GroupsIds.group_id > self.latest_id).first()
keyboard = VkKeyboard(one_time=True)
msg = ('К какой категории относится эта группа?\n'
f'https://vk.com/club{group.group_id}\n\n')
for i, cat in enumerate(self.new_cats):
keyboard.add_button(
cat.capitalize(),
color=VkKeyboardColor.SECONDARY,
payload=json.dumps({
'button':
f'dataset_filter#{group.group_id}#{self.new_cats.index(cat)}'
}))
if (i + 1) % 3 == 0:
keyboard.add_line()
if (i + 1) % 3 != 0:
keyboard.add_line()
keyboard.add_button('Ни к одной',
color=VkKeyboardColor.NEGATIVE,
payload=json.dumps({
'button':
f'dataset_filter#{group.group_id}#-1'
}))
keyboard.add_button('Завершить',
color=VkKeyboardColor.NEGATIVE,
payload=json.dumps({'command': 'start'}))
self.send_message(from_id, msg, keyboard.get_keyboard())
else:
keyboard = VkKeyboard(one_time=True)
keyboard.add_button('Начать анализ',
color=VkKeyboardColor.POSITIVE,
payload=json.dumps(
{'button': 'start_analysis'}))
msg = 'Начнём анализ?'
self.send_message(from_id, msg, keyboard.get_keyboard())
n_test = get_test_sample_size(X_test.shape[0], k=TEST_BATCH_SIZE)
X_test = X_test[:n_test, :]
y_test = y_test[:n_test, :]
users_test = X_test[:, 0].reshape(-1,1)
items_test = X_test[:, 1].reshape(-1,1)
y_test = y_test.reshape(-1,1)
predictor = Predictor(model=model, batch_size=TEST_BATCH_SIZE, users=users_test, items=items_test, y=y_test,
use_cuda=args.cuda, n_items=stats["n_items"])
preds = predictor.predict().reshape(-1,1)
output = pd.DataFrame(np.concatenate((users_test, preds, y_test), axis=1),
columns = ['user_id', 'pred', 'y_true'])
if args.task == "choice":
output, hit_ratio, ndcg = get_choice_eval_metrics(output, at_k=EVAL_K)
print("hit ratio: {:.4f}".format(hit_ratio))
print("ndcg: {:.4f}".format(ndcg))
else: