def makeRepostSteampunk(
): #Репостим стимпанк группу в конце дня, чтобы быть похожими на человека
print 'Парсим группу: 39009769'
with vk_api.VkRequestsPool(vk_session) as pool:
requests = pool.method('wall.get', {
'owner_id': '-39009769',
'count': '300'
})
for k in requests.result['items']:
try:
if k['copy_history'][0]['owner_id']:
owner_id = int(k['copy_history'][0]['owner_id'])
post_id = int(k['copy_history'][0]['id'])
except:
owner_id = int(k['owner_id'])
post_id = int(k['id'])
if not isReposted(owner_id, post_id):
if int(
re.search(r'\d{1,10}',
cmdline("cat reposted.txt | grep -c ''")).
group(0)) - todaycount < 149:
repname = 'wall' + str(owner_id) + '_' + str(post_id)
with vk_api.VkRequestsPool(vk_session) as pool:
requests = pool.method('wall.repost',
{'object': repname})
print 'owner_id ' + str(owner_id) + ' post_id ' + str(
post_id)
frep = open('reposted.txt', 'a+')
frep.write(str(owner_id) + ',' + str(post_id) + '\n')
frep.close()
break
def makegrouptable(self, catname, topit=True, alpha=1, gr_num=16):
self.catname = catname
with vk_api.VkRequestsPool(self.api) as pool:
groups = pool.method('groups.getCatalog',
values={'category_id': self.cat2id[catname]})
self.g_ids = list(map(lambda x: x['id'],
groups.result['items']))[:gr_num]
self.g_nam = list(
map(lambda x: x['screen_name'], groups.result['items']))[:gr_num]
self.id2group = dict(zip(self.g_ids, self.g_nam))
self.group2id = dict(zip(self.g_nam, self.g_ids))
for idd in self.g_ids:
self.id2cat[idd] = catname
print('Число подписчиков:')
self.n_subs = []
for name in self.group2id.keys():
with vk_api.VkRequestsPool(self.api) as pool:
subs = pool.method('groups.getMembers',
values={'group_id': self.group2id[name]})
print(name, subs.result['count'])
self.n_subs += [(self.group2id[name], subs.result['count'])]
self.dictsubs = dict(self.n_subs)
if topit:
tops = np.array(list(self.dictsubs.values())).mean() * alpha
else:
tops = 0
self.selected_groups = list(
filter(
lambda x: self.dictsubs[x] >= tops,
sorted(list(self.id2group.keys()),
key=lambda x: self.dictsubs[x],
reverse=True)))
self.oldtable = self.table.copy()
del (self.table)
self.table = ps.DataFrame()
self.table['category_name'] = [
catname for i in range(len(self.selected_groups))
]
self.table['category_id'] = [
int(self.cat2id[catname]) for i in range(len(self.selected_groups))
]
names = [self.id2group[i] for i in self.selected_groups]
self.table['group_name'] = names
self.table['group_id'] = [int(self.group2id[i]) for i in names]
self.table['n_subs'] = [
int(self.dictsubs[i]) for i in self.selected_groups
]
self.table = ps.concat([self.oldtable, self.table])
self.table = self.table.reset_index(drop=True)
def get_likes_list(
self
): # получает количесво лайков для каждого юзера со всех постов со стены
def count(request_dict):
"""возвращает количество лайков для каждого vk id"""
out = {}
for item in request_dict.values():
for man in item['items']:
name = ' '.join([man['first_name'], man['last_name']])
vk_id_ = man['id']
if vk_id_ in out:
out[vk_id_]['count'] += 1
else:
out[vk_id_] = {'name': name, 'count': 1}
return out
wall = VkUser.get_wall(self)
photos = VkUser.get_photos(self, parameter='items')
posts_ids = [item['id'] for item in wall]
photos_ids = [item['id'] for item in photos]
with vk_api.VkRequestsPool(session) as pool:
posts = pool.method_one_param('likes.getList',
key='item_id',
values=posts_ids,
default_values=dict(
owner_id=self.vk_id,
type='post',
extended=1))
with vk_api.VkRequestsPool(session) as pool:
photos = pool.method_one_param('likes.getList',
key='item_id',
values=photos_ids,
default_values=dict(
owner_id=self.vk_id,
type='photo',
extended=1))
posts = count(posts.result)
photos = count(photos.result)
out_dict = posts
pprint(out_dict)
for vk_id in photos:
if vk_id in out_dict:
out_dict[vk_id]['count'] += photos[vk_id]['count']
else:
out_dict[vk_id] = photos[vk_id]
return out_dict
def get_users_photo(self, user):
raiting_list = self.get_raiting_users(user)
random_top10 = random.sample(raiting_list, 10)
sorted_users = []
for user in random_top10:
params = {
'owner_id': int(*user.keys()),
'access_token': TOKEN,
'album_id': 'profile',
'extended': 1
}
with vk_api.VkRequestsPool(vk_session) as pool:
photos = pool.method('photos.get', params)
photos = photos.result
top_3_photo = []
for photo in photos['items']:
top_3_photo.append({
'id': photo['id'],
'likes': photo['likes']['count']
})
top_3_photo.sort(key=lambda dict: dict['likes'], reverse=True)
top_3_photo = top_3_photo[0:3]
sorted_users.append({
'user_id': int(*user.keys()),
'likes': top_3_photo
})
print(sorted_users)
return sorted_users
def _fetch_member_friends(self, user_subset):
if self.member_friends is not None:
return
log_method_begin()
members = [
member for member in self.members if member['id'] in user_subset
]
print('{} users to fetch'.format(len(members)))
pool_results = []
with vk_api.VkRequestsPool(self.vk_session) as pool:
for member in members:
pool_results.append(
(member['id'],
pool.method('friends.get', {
'user_id': member['id'],
'fields': 'photo'
})))
self.member_friends = defaultdict(list)
for member_id, friend_request in pool_results:
if friend_request.ok:
for friend in friend_request.result['items']:
if friend['id'] not in user_subset:
friend['is_member'] = False
self.member_friends[member_id].append(friend)
self._compress_users()
self._save_pickle('raw_users_data.member_friends', self.member_friends)
log_method_end()
def get_user_photos(self, count=False, custom_ids=None):
ids = self.ids
if custom_ids is not None:
ids = custom_ids
result = {}
with vk_api.VkRequestsPool(self.vk_session) as pool:
resp = pool.method_one_param('photos.getAll',
key='owner_id',
values=ids)
for id, photos in resp.result.iteritems():
counter = 0
if int(photos[u'count']) > 0:
if count:
counter += int(photos[u'count'])
result[id] = counter
continue
if not os.path.exists(str(id)):
os.makedirs(str(id))
for i in xrange(len(photos[u'items'])):
resp = ''
for key, val in photos[u'items'][i].iteritems():
if (key.find(u'photo') > -1):
url = val
resp = requests.get(url, stream=True)
self.__save_photo(resp, str(id) + '/' + str(i) + '.jpg')
if count:
return result
def downloadAttachmentMedia(attachment, vk_session):
type = attachment['type']
if type == 'video':
video_owner_id = attachment['video']['owner_id']
video_id = attachment['video']['id']
request_data = str(video_owner_id) + '_' + str(video_id)
with vk_api.VkRequestsPool(vk_session) as pool:
video_data = pool.method('video.get', {
'videos': request_data
})
video_url = str(video_data.result['items'][0]['player'])
new_video_url = video_url.replace('embed', 'watch')
downloadFile(new_video_url, 'asdasd.avi')
# html = requests.get(new_video_url).text
# soup = BeautifulSoup(html, 'lxml')
# print(soup)
# # video_base_url = soup.find('div', id='sas')
if type == 'audio':
pass
def main():
vk_session = vk_api.VkApi(MyVKData.LOGIN, MyVKData.GET_PASSWORD)
try:
vk_session.auth()
print('Ok')
except vk_api.AuthError as errror_msg:
print(errror_msg)
return
with vk_api.VkRequestsPool(vk_session) as pool:
posts = []
#for word in whiteWordPart:
#time.sleep(1 / 3)
finded_groups = pool.method(
'groups.search',
{
#'q': word
'q': 'лайк'
})
for post in newsfeed.items:
id_posts = post['id_posts']
group_id = post['group_id']
mass = {}
mass['id_posts'] = id_posts
mass['group_id'] = group_id
posts.append(mass)
def joinGroup(owner_id):
with vk_api.VkRequestsPool(vk_session) as pool:
if int(owner_id) < 0:
requests = pool.method('groups.join',
{'group_id': -int(owner_id)})
fjoin = open('joined.txt', 'a+')
fjoin.write(str(owner_id) + '\n')
fjoin.close()
def getName(vk_session, uid):
'''
Получает имя собеcедника, с помощью метода VK API
'''
with vk_api.VkRequestsPool(vk_session) as pool:
name = pool.method_one_param('users.get', key='user_id', values=(uid,))
return name.result[uid][0]['first_name']
def to_cache(self, input_):
global cached_ids
shortname = input_[input_.rfind("/")+1:]
if "public" in shortname:
shortname = shortname[6:]
print(input_)
r = self.vk_session.method("groups.getById", {"group_ids": shortname})
group_id = r[0]["id"]
if group_id not in self.cached_ids.keys():
"""Все сообщества подписчиков сообщества"""
users = vk_api.VkTools(self.vk_session).get_all("groups.getMembers", 100, {'group_id':group_id})['items']
k = int(len(users) / 1000)
if k:
users = users[::k]
data_for_clustering = []
users_publicpages = []
with vk_api.VkRequestsPool(self.vk_session) as pool:
for i in range(0, len(users), 1000):
batch = users[i:i+1000]
data_for_clustering.append(pool.method('users.get', {
"user_ids": batch,
"fields": "sex,age,education,universities,schools,interests,music,movies,bdate,city,country"
}))
for user_id in batch[:5]:
users_publicpages.append(pool.method('users.getSubscriptions', {
"user_id": user_id
<<<<<<< HEAD
=======
>>>>>>> f1cc8e4d061d67968fec0e3fefba831e4cf5253b
}))
u = []
try:
for x in data_for_clustering:
try:
u.append(x.result)
except:
pass
except:
pass
new_data_for_clustering = [x for x in u]
t = []
for user in users_publicpages:
try:
t.append(user.result['groups']['items'])
except:
pass
users_publicpages = t
wall50 = self.vk_session.method("wall.get", {"owner_id": -group_id, "filter": "owner", "count": 100})['items'][::2]
def categoriesinfo(self):
n_groups = 0
for name in self.cat2id.keys():
with vk_api.VkRequestsPool(self.api) as pool:
groups = pool.method('groups.getCatalog',
values={'category_id': self.cat2id[name]})
print(name, groups.result['count'])
n_groups += groups.result['count']
print('Всего', n_groups)
def get_user_stat(self):
friends = {}
with vk_api.VkRequestsPool(self.vk_session) as pool:
friends = pool.method_one_param(
'friends.get', # Метод
key='user_id', # Изменяющийся параметр
values=self.ids,
default_values={'fields': 'photo'})
return friends.result
def get_friends(list_ids):
# execute method api - получает список всех друзей по списку list_ids
with vk_api.VkRequestsPool(api) as pool:
friends = pool.method_one_param(
'friends.get', # Метод
key='user_id', # Изменяющийся параметр
values=list_ids,
# Параметры, которые будут в каждом запросе
default_values={'fields': 'deactivated'})
return friends.result
def __init__(self):
"""Авторизация ВКонтакте, инициализация сессии и инструментов"""
session = vk_api.VkApi(os.environ['VK_LOGIN'], os.environ['VK_PASSWORD'], app_id=5559651)
try:
session.authorization()
except vk_api.AuthorizationError as error_msg:
print(error_msg)
self.session = session
self.tools = vk_api.VkTools(session)
self.pool = vk_api.VkRequestsPool(session)
def get_members_friends(members_ids):
vk_session = vk_api.VkApi(fey_login, fey_password)
vk_session.authorization()
vk = vk_session.get_api()
# members_friends = {}
with vk_api.VkRequestsPool(vk_session) as pool:
members_friends = pool.method_one_param('friends.get',
key='user_id',
values=members_ids)
return members_friends
def vk_session_init(self): # Инициализация методов vk_api
self.vk_session = vk_api.VkApi(self.login, self.user_password)
self.vk_session.auth()
self.vk = self.vk_session.get_api()
self.vk_tools = vk_api.VkTools(self.vk_session)
self.vk_request_pool = vk_api.VkRequestsPool(self.vk_session)
self.vk_request_result = vk_api.requests_pool.RequestResult
self.vk_request_one_param_pool = vk_api.requests_pool
self.birth_year_person = datetime.datetime.strptime(
self.vk.account.getProfileInfo()['bdate'], '%d.%m.%Y').date().year
print('\t - инициализация vk api сессии')
def main():
""" Пример работы с VkRequestsPool """
login, password = '******', 'mypassword'
vk_session = vk_api.VkApi(login, password)
try:
vk_session.authorization()
except vk_api.AuthorizationError as error_msg:
print(error_msg)
return
"""
Позволяет выполнять до 25 запросов к API за один запрос.
Работает через метод execute.
"""
friends = {}
with vk_api.VkRequestsPool(vk_session) as pool:
for user_id in [1, 183433824]:
friends[user_id] = pool.method('friends.get', {'user_id': user_id})
print(friends)
# Same result
with vk_api.VkRequestsPool(vk_session) as pool:
friends = pool.method_one_param('friends.get',
key='user_id',
values=[1, 183433824])
print(friends)
with vk_api.VkRequestsPool(vk_session) as pool:
friends = pool.method('friends.get')
status = pool.method('status.get')
# Обратите внимание, что запрос выполняется после выхода из with
# и к результатам обратиться можно только после выхода из него
print(friends)
print(status)
async def deleter():
user = vk_api.VkApi(token=default_data["bp_token"])
while workers_state["deleter"]:
requests = user.method("friends.getRequests", {
"count": 1000,
"out": 1
})["items"]
with vk_api.VkRequestsPool(user) as p:
for i in requests:
p.method("friends.delete", {"user_id": i})
await asyncio.sleep(3600)
def isFake(owner_id):#True если фэйк (id>=0 или members_count<60)
if (owner_id<0):
with vk_api.VkRequestsPool(vk_session) as pool:
requests=pool.method('groups.getById', {
'group_id':-owner_id,
'fields':['members_count']
})
if (int(requests.result[0]['members_count'])>60):
return False
else:
print 'Fake: id'+ str(owner_id)+' count:'+ str(int(requests.result[0]['members_count']))
return True
def readWall(group_id):
print 'Парсим группу: ' + str(group_id)
with vk_api.VkRequestsPool(vk_session) as pool:
requests = pool.method('wall.get', {
'owner_id': group_id,
'count': '9'
})
for k in requests.result['items']:
try:
if k['copy_history'][0]['owner_id']:
description = unicode(
json.dumps(k['copy_history'][0]['text']),
'raw-unicode-escape').encode('utf-8')
owner_id = k['copy_history'][0]['owner_id']
post_id = k['copy_history'][0]['id']
except:
description = unicode(json.dumps(k['text']),
'raw-unicode-escape').encode('utf-8')
owner_id = k['owner_id']
post_id = k['id']
if (description.lower().find('лайк') > -1) or (
description.lower().find('Лайк') >
-1) or (description.lower().find('Репост') > -1) or (
description.lower().find('РЕПОСТ') >
-1) or (description.lower().find('друзьям') > -1) or (
description.lower().find('репост') >
-1) or (description.lower().find('поделит') > -1):
print description + '\n'
if not isStop(description):
if not isReposted(int(owner_id), int(post_id)):
print 'Репостим:\n' + description
makeRepost(owner_id, post_id)
for j in re.findall(r'wall-\d{1,10}_\d{1,10}',
description):
j_owner_id = re.search(r'-\d{1,10}', j).group(0)
j_id = re.search(r'\d{1,10}$', j).group(0)
if not isReposted(int(j_owner_id), int(j_id)):
print 'Репостим детей:\n'
makeRepost(int(j_owner_id), int(j_id))
if not isMember(int(j_owner_id)):
joinGroup(int(j_owner_id))
if not isMember(int(owner_id)):
joinGroup(int(owner_id))
for j in re.findall(r'club\d{1,10}', description):
j_owner_id = -int(
re.search(r'\d{1,10}', j).group(0))
if not isMember(int(j_owner_id)):
joinGroup(int(j_owner_id))
else:
print description
print "Нет слова Репостим"
def get_user_subscriptions(self, custom_ids=None):
subscriptions = {}
ids = self.ids
if custom_ids is not None:
ids = custom_ids
with vk_api.VkRequestsPool(self.vk_session) as pool:
resp = pool.method_one_param(
'users.getSubscriptions',
key='user_id', # Изменяющийся параметр
values=ids)
for id, sub in resp.result.iteritems():
if int(sub[u'groups'][u'count']) > 0:
subscriptions[id] = sub[u'groups'][u'items']
return subscriptions
def wallget(self, wall_id=-1, count=10):
with vk_api.VkRequestsPool(self.vk_session) as pool:
posts = pool.method('wall.get',
values={
'owner_id': wall_id,
'count': count,
'filter': 'owner'
})
if posts.ok and posts.ready:
self.last_type = 0
self.last = posts.result
return (True, posts.result)
else:
return (False, {})
async def friends():
user = vk_api.VkApi(token=default_data["bp_token"])
while workers_state["friends"]:
requests = user.method("friends.getRequests", {
"count": 1000,
"extended": 1
})["items"]
with vk_api.VkRequestsPool(user) as p:
for i in requests:
if "deactivated" not in i:
p.method("friends.add", {
"user_id": i["user_id"],
"follow": 0
})
await asyncio.sleep(120)
def get_checked_by_age(self, age_start, age_end):
checked = {}
with vk_api.VkRequestsPool(self.vk_session) as pool:
resp = pool.method_one_param('users.get',
key='user_ids',
values=self.ids,
default_values={'fields': 'bdate'})
for id, data in resp.result.items():
if u'bdate' in data[0]:
bdate = data[0][u'bdate']
checked[id] = re.findall(r'[0-9]{4}', bdate)
if (type(checked[id]) is list) and (len(checked[id]) > 0):
checked[id] = 2017 - int(checked[id][0])
return [
k for k, v in checked.items()
if (v >= age_start) and (v <= age_end)
]
def asm_wallget(self, wall_id=-1, count=10, offset=0):
"""Under-table function to get writings from wall VK."""
with vk_api.VkRequestsPool(self.api) as pool:
posts = pool.method(
'wall.get', values={
'owner_id': wall_id,
'count': count,
'filter': 'owner',
'offset': offset
}
)
if posts.ok and posts.ready:
self.last_type = 0
self.last = posts.result
return (True, posts.result)
else:
return (False, {})
def fetch_friends_of_friends(vk_session, friends):
user_vk_handles = dict()
with vk_api.VkRequestsPool(vk_session) as pool:
for friend in friends:
friend_id = friend['id']
user_vk_handles[friend_id] = pool.method(
'friends.get',
dict(user_id=friend_id,
count=5000,
fields='sex,bdate,education'))
friends_of_friends = []
for handle in user_vk_handles.values():
if handle.ok:
friends_of_friends.append(handle.result)
return friends_of_friends
def makeRepost(owner_id, post_id):
if len(open("reposted.txt").read().split())-todaycount<144*askDayNum():
repname='wall'+str(owner_id)+'_'+str(post_id)
with vk_api.VkRequestsPool(vk_session) as pool:
requests=pool.method('wall.repost', {
'object':repname
})
print 'owner_id '+str(owner_id)+' post_id '+str(post_id)
frep = open('reposted.txt','a+')
frep.write(str(owner_id)+','+str(post_id)+'\n')
frep.close()
time.sleep(120.0+random.random()*180.0)
else:
print 'На сегодня хватит'
while len(open("reposted.txt").read().split())-todaycount<149*askDayNum():
time.sleep(120.0+random.random()*200.0)
makeRepostSteampunk()
open("daycount","w").write(askDayCount()+1)
sys.exit(0)
def get_group_members(self, group_ids, offset=0):
members = []
while True:
with vk_api.VkRequestsPool(self.vk_session) as pool:
resp = pool.method_one_param(
'groups.getMembers',
key='group_id', # Изменяющийся параметр
values=group_ids,
default_values={'offset': offset})
group_ids = []
offset += 1000
for id, arr in resp.result.iteritems():
if arr[u'count'] > 0:
items = arr[u'items']
members = list(set(members + items))
if (arr[u'count'] - offset > 0):
group_ids.append(id)
if len(group_ids) == 0:
break
return members
def get_user_comments(self, count=False):
result = {}
with vk_api.VkRequestsPool(self.vk_session) as pool:
resp = pool.method_one_param('photos.getAllComments',
key='owner_id',
values=self.ids)
for id, comments in resp.result.iteritems():
counter = 0
if int(comments[u'count']) > 0:
if count:
counter += int(comments[u'count'])
result[id] = counter
continue
for item in comments[u'items']:
if item[u'from_id'] == id:
with open(str(id) + '/comments', 'ab') as out_file:
out_file.write('\r\n' +
item[u'text'].encode('utf-8') +
'\r\n')
if count:
return result
