def dump_fave_photo(dmp):
"""Фото
dmp: Dumper object
"""
folder = os.path.join('dump', 'photo', 'Понравившиеся')
os.makedirs(folder, exist_ok=True)
print('[получение понравившихся фото]')
photo = get_fave(dmp._vk, 'photos')
print('Сохранение понравившихся фото:')
if photo['count'] == 0:
print(' 0/0')
else:
print(' .../{}'.format(photo['count']), end='\r')
with Pool(dmp._settings['POOL_PROCESSES']) as pool:
res = pool.starmap(
copy_func(dmp._download),
zip(
itertools.repeat(dmp.__class__),
map(
lambda p: sorted(p['sizes'],
key=itemgetter('width', 'height'))[-1]
['url'], photo['items']), itertools.repeat(folder)))
print('\x1b[2K {}/{} (total: {})'.format(
sum(filter(None, res)), photo['count'],
len(next(os.walk(folder))[2])))
def dump_docs(dmp):
"""Документы
dmp: Dumper object
"""
folder = os.path.join('dump', 'docs')
os.makedirs(folder, exist_ok=True)
print('[получение списка документов]')
docs = dmp._vk.docs.get()
print('Сохраненние документов:')
if docs['count'] == 0:
print(' 0/0 (total: {})'.format(len(next(os.walk(folder))[2])))
else:
objs = []
for d in docs['items']:
objs.append({
'url': d['url'],
'name': d['title'] + '_' + str(d['id']),
'ext': d['ext']
})
print(' .../{}'.format(docs['count']), end='\r')
with Pool(dmp._settings['POOL_PROCESSES']) as pool:
res = pool.starmap(copy_func(dmp._download),
zip(itertools.repeat(dmp.__class__), objs, itertools.repeat(folder)))
print('\x1b[2K {}/{} (total: {})'.format(sum(filter(None, res)),
len(objs),
len(next(os.walk(folder))[2])))
def dump_fave_video(dmp):
"""Видео
dmp: Dumper object
"""
folder = os.path.join('dump', 'video', 'Понравившиеся')
os.makedirs(folder, exist_ok=True)
print('[получение понравившихся видео]')
video_ids = get_fave(dmp._vk, 'videos')
video = []
if video_ids:
for v in video_ids['items']:
video.append('{oid}_{id}{access_key}'.format(
oid=v['owner_id'],
id=v['id'],
access_key='_' + (v.get('access_key') or '')))
if video:
video = dmp._vk_tools.get_all(method='video.get',
max_count=200,
values={
'videos': ','.join(video),
'extended': 1
})
print('Сохранение понравившихся видео:')
if video['count'] == 0:
print(' 0/0')
else:
print(' .../{}'.format(video['count']), end='\r')
try:
with Pool(dmp._AVAILABLE_THREADS if dmp.
_settings['LIMIT_VIDEO_PROCESSES'] else dmp.
_settings['POOL_PROCESSES']) as pool:
res = pool.starmap(
copy_func(dmp._download_video),
zip(itertools.repeat(dmp.__class__), video['items'],
itertools.repeat(folder)))
print('\x1b[2K {}/{} (total: {})'.format(
sum([1 for i in res if i is True]), video['count'],
len(next(os.walk(folder))[2])))
except MaybeEncodingError:
print('\x1b[2K ???/{} (total: {})'.format(
video['count'], len(next(os.walk(folder))[2])))
def dump_photo(dmp):
"""Фото (по альбомам)
dmp: Dumper object
"""
os.makedirs(os.path.join('dump', 'photo'), exist_ok=True)
albums = dmp._vk.photos.getAlbums(need_system=1)
print('Сохранение фото:')
for al in albums['items']:
print(' Альбом "{}":'.format(al['title']))
folder = os.path.join('dump', 'photo', '_'.join(al['title'].split()))
os.makedirs(folder, exist_ok=True)
photo = dmp._vk_tools.get_all(method='photos.get',
max_count=1000,
values={
'album_id': al['id'],
'photo_sizes': 1
})
if photo['count'] == 0:
print(' 0/0 (total: {})'.format(len(next(os.walk(folder))[2])))
else:
print(' .../{}'.format(photo['count']), end='\r')
with Pool(dmp._settings['POOL_PROCESSES']) as pool:
res = pool.starmap(
copy_func(dmp._download),
zip(
itertools.repeat(dmp.__class__),
map(
lambda p: sorted(p['sizes'],
key=itemgetter('width', 'height'))
[-1]['url'], photo['items']),
itertools.repeat(folder)))
print('\x1b[2K {}/{} (total: {})'.format(
sum(filter(None, res)), photo['count'],
len(next(os.walk(folder))[2])))
def dump_audio(dmp):
"""Аудио
dmp: Dumper object
"""
print('[получение списка аудио]')
tracks = vk_api.audio.VkAudio(dmp._vk_session).get()
folder = os.path.join('dump', 'audio')
os.makedirs(folder, exist_ok=True)
print('Сохранение аудио:')
if len(tracks) == 0:
print(' 0/0 (total: {})'.format(len(next(os.walk(folder))[2])))
else:
audios = []
for a in tracks:
audios.append({
'url':
a['url'],
'name':
'{artist} - {title}_{id}'.format(artist=a['artist'],
title=a['title'],
id=a['id']),
'ext':
'mp3'
})
print(' .../{}'.format(len(tracks)), end='\r')
with Pool(dmp._settings['POOL_PROCESSES']) as pool:
res = pool.starmap(
copy_func(dmp._download),
zip(itertools.repeat(dmp.__class__), audios,
itertools.repeat(folder)))
print('\x1b[2K {}/{} (total: {})'.format(
sum(filter(None, res)), len(audios),
len(next(os.walk(folder))[2])))
def dump_video(dmp):
"""Видео (по альбомам)
dmp: Dumper object
"""
folder = os.path.join('dump', 'video')
os.makedirs(folder, exist_ok=True)
print('Сохранение видео:')
albums = dmp._vk_tools.get_all(method='video.getAlbums',
max_count=100,
values={'need_system': 1})
for al in albums['items']:
print(' Альбом "{}":'.format(al['title']))
folder = os.path.join('dump', 'video', '_'.join(al['title'].split()))
os.makedirs(folder, exist_ok=True)
video = dmp._vk_tools.get_all(method='video.get',
max_count=200,
values={'album_id': al['id']})
if video['count'] == 0:
print(' 0/0 (total: {})'.format(len(next(os.walk(folder))[2])))
else:
print(' .../{}'.format(len(video['items'])), end='\r')
with Pool(dmp._AVAILABLE_THREADS if dmp.
_settings['LIMIT_VIDEO_PROCESSES'] else dmp.
_settings['POOL_PROCESSES']) as pool:
res = pool.starmap(
copy_func(dmp._download_video),
zip(itertools.repeat(dmp.__class__), video['items'],
itertools.repeat(folder)))
print('\x1b[2K {}/{} (total: {})'.format(
sum(filter(None, res)), len(video['items']),
len(next(os.walk(folder))[2])))
def dump_attachments_only(dmp):
"""Вложения диалогов
dmp: Dumper object
"""
global users
folder = os.path.join('dump', 'dialogs')
os.makedirs(folder, exist_ok=True)
print('[получение диалогов...]')
print('\x1b[2K 0/???', end='\r')
conversations = dmp._vk_tools.get_all(method='messages.getConversations',
max_count=200,
values={
'extended':
1,
'fields':
'first_name, last_name, name'
})
print('\x1b[2K {}/{}'.format(len(conversations['items']),
conversations['count']))
print('[будет исключено диалогов: {}]'.format(len(dmp._EXCLUDED_DIALOGS)),
end='\n\n')
print('Сохранение диалогов:')
for con in conversations['items']:
did = con['conversation']['peer']['id']
if con['conversation']['peer']['type'] == 'user':
if did not in users:
users_add(dmp._vk, did)
dialog_name = users.get(did)['name']
elif con['conversation']['peer']['type'] == 'group':
if did not in users:
users_add(dmp._vk, did)
dialog_name = users.get(did)['name']
elif con['conversation']['peer']['type'] == 'chat':
dialog_name = con['conversation']['chat_settings']['title']
else:
dialog_name = r'{unknown}'
for c in dmp._INVALID_CHARS:
if c in dialog_name:
dialog_name = dialog_name.replace(
c, dmp._settings['REPLACE_CHAR'])
fn = '{}_{id}'.format('_'.join(dialog_name.split(' ')), id=did)
for n in os.listdir(folder):
if str(did) == n.split('.txt')[0].split('_')[-1]:
if dmp._settings['KEEP_DIALOG_NAMES']:
fn = n.split('.txt')[0]
else:
shutil.move(
os.path.join(folder, n),
os.path.join(
folder, '{}_{id}'.format(
'_'.join(dialog_name.split(' ')), id=did) +
('.txt' if '.txt' in n else '')))
print(' Диалог: {}{nfn}'.format(
dialog_name,
nfn=(' (as {})'.format(fn)
if ' '.join(fn.split('_')[:-1]) != dialog_name else '')))
if did in dmp._EXCLUDED_DIALOGS:
print(' [исключён]\n')
continue
at_folder = os.path.join(folder, fn)
os.makedirs(at_folder, exist_ok=True)
# PHOTO DUMP
print(' [получение фото]', end='\r')
photo = get_attachments(dmp._vk, did, 'photo')
if photo['count'] > 0:
af = os.path.join(at_folder, 'Фото')
os.makedirs(af, exist_ok=True)
print('\x1b[2K [сохранение фото]')
print(' .../{}'.format(photo['count']), end='\r')
with Pool(dmp._settings['POOL_PROCESSES']) as pool:
res = pool.starmap(
copy_func(dmp._download),
zip(
itertools.repeat(dmp.__class__),
map(
lambda t: sorted(t['attachment']['photo']['sizes'],
key=itemgetter('width', 'height')
)[-1]['url'], photo['items']),
itertools.repeat(af)))
print('\x1b[2K {}/{} (total: {})'.format(
sum(filter(None, res)), len(photo['items']),
len(next(os.walk(af))[2])))
del res
else:
print(' [фото отсутствуют]')
del photo
# VIDEO DUMP
print(' [получение видео]', end='\r')
video = get_attachments(dmp._vk, did, 'video')
if video['count'] > 0:
video_ids = []
for v in video['items']:
video_ids.append('{oid}_{id}{access_key}'.format(
oid=v['attachment']['video']['owner_id'],
id=v['attachment']['video']['id'],
access_key=('_' + v['attachment']['video']['access_key']
if 'access_key' in v['attachment']['video']
else '')))
video = dmp._vk_tools.get_all(method='video.get',
max_count=200,
values={
'videos': ','.join(video_ids),
'extended': 1
})
af = os.path.join(at_folder, 'Видео')
os.makedirs(af, exist_ok=True)
print('\x1b[2K [сохранение видео]')
print(' .../{}'.format(video['count']), end='\r')
try:
with Pool(dmp._AVAILABLE_THREADS if dmp.
_settings['LIMIT_VIDEO_PROCESSES'] else dmp.
_settings['POOL_PROCESSES']) as pool:
res = pool.starmap(
copy_func(dmp._download_video),
zip(itertools.repeat(dmp.__class__), video['items'],
itertools.repeat(af)))
print('\x1b[2K {}/{} (total: {})'.format(
sum(filter(None, res)), len(video['items']),
len(next(os.walk(af))[2])))
del res
except MaybeEncodingError:
print('\x1b[2K ???/{} (total: {})'.format(
len(video['items']), len(next(os.walk(af))[2])))
else:
print(' [видео отсутствуют]')
del video
# DOCS DUMP
print(' [получение документов]', end='\r')
docs = get_attachments(dmp._vk, did, 'doc')
if docs['count'] > 0:
af = os.path.join(at_folder, 'Документы')
os.makedirs(af, exist_ok=True)
print('\x1b[2K [сохранение документов]')
print(' .../{}'.format(docs['count']), end='\r')
with Pool(dmp._settings['POOL_PROCESSES']) as pool:
res = pool.starmap(
copy_func(dmp._download),
zip(itertools.repeat(dmp.__class__),
map(lambda t: t['attachment']['doc'], docs['items']),
itertools.repeat(af)))
print('\x1b[2K {}/{} (total: {})'.format(
sum(filter(None, res)), len(docs['items']),
len(next(os.walk(af))[2])))
del res
else:
print(' [документы отсутствуют]')
del docs
print()
with open('users.json', 'w', encoding='utf-8') as f:
json.dump(users, f, ensure_ascii=False, indent=4)
def dump_messages(dmp, **kwargs):
"""Сообщения
dmp: Dumper object
"""
global users
folder = os.path.join('dump', 'dialogs')
os.makedirs(folder, exist_ok=True)
print('[получение диалогов...]')
print('\x1b[2K 0/???', end='\r')
conversations = dmp._vk_tools.get_all(method='messages.getConversations',
max_count=200,
values={
'extended':
1,
'fields':
'first_name, last_name, name'
})
print('\x1b[2K {}/{}'.format(len(conversations['items']),
conversations['count']))
print('[будет исключено диалогов: {}]'.format(len(dmp._EXCLUDED_DIALOGS)),
end='\n\n')
print('Сохранение диалогов:')
for con in conversations['items']:
did = con['conversation']['peer']['id']
if con['conversation']['peer']['type'] == 'user':
if did not in users:
users_add(dmp._vk, did)
dialog_name = users.get(did)['name']
elif con['conversation']['peer']['type'] == 'group':
if did not in users:
users_add(dmp._vk, did)
dialog_name = users.get(did)['name']
elif con['conversation']['peer']['type'] == 'chat':
dialog_name = con['conversation']['chat_settings']['title']
else:
dialog_name = r'{unknown}'
for c in dmp._INVALID_CHARS:
if c in dialog_name:
dialog_name = dialog_name.replace(
c, dmp._settings['REPLACE_CHAR'])
fn = '{}_{id}'.format('_'.join(dialog_name.split(' ')), id=did)
for n in os.listdir(folder):
if str(did) == n.split('.txt')[0].split('_')[-1]:
if dmp._settings['KEEP_DIALOG_NAMES']:
fn = n.split('.txt')[0]
else:
shutil.move(
os.path.join(folder, n),
os.path.join(
folder, '{}_{id}'.format(
'_'.join(dialog_name.split(' ')), id=did) +
('.txt' if '.txt' in n else '')))
print(' Диалог: {}{nfn}'.format(
dialog_name,
nfn=(' (as {})'.format(fn)
if ' '.join(fn.split('_')[:-1]) != dialog_name else '')))
if did in dmp._EXCLUDED_DIALOGS:
print(' [исключён]\n')
continue
values = {
'peer_id': con['conversation']['peer']['id'],
'extended': 1,
'fields': 'first_name, last_name'
}
append = {
'use':
dmp._settings['DIALOG_APPEND_MESSAGES']
and os.path.exists(os.path.join(folder, f'{fn}.txt'))
}
try:
if append['use']:
with open(os.path.join(folder, f'{fn}.txt'), 'rb') as t:
t.seek(-2, 2)
while t.read(1) != b'\n':
t.seek(-2, 1)
last = t.readline().decode()
r = re.match('^\[last:[0-9]+\]$', last)
if r:
start_message_id = int(
re.search(r'\d+', r.group(0)).group(0))
values['start_message_id'] = start_message_id
t.seek(-len(last.encode('utf-8')) - 2, 1)
while True:
while t.read(1) != b'\n':
t.seek(-2, 1)
tmp = t.readline().decode()
r = re.match('^ {8}\[\d+ [а-я a-z]+ \d+\]$', tmp)
# TODO: получение last_id по последнему сообщению (???)
if r:
append['prev_date'] = re.search(
'\d+ [а-я a-z]+ \d+', r.group(0)).group(0)
break
else:
t.seek(-len(tmp.encode('utf-8')) - 2, 1)
else:
values['rev'] = 1
append['use'] = False
else:
values['rev'] = 1
except OSError:
values['rev'] = 1
append['use'] = False
print(' [кэширование]')
print('\x1b[2K 0/???', end='\r')
try:
history = dmp._vk_tools.get_all(method='messages.getHistory',
max_count=200,
values=values,
negative_offset=append['use'])
print('\x1b[2K {}/{}'.format(len(history['items']),
history['count']))
if len(history['items']) == 0:
print()
continue
except VkToolsException:
print('\x1b[2K 0/0\n')
continue
if append['use']:
def sortById(msg):
return msg['id']
history['items'].sort(key=sortById)
attachments = {
'photos': [],
'video_ids': [],
'docs': [],
'audio_messages': []
}
if append['use']:
tmp = ''
else:
f = open(os.path.join(folder, f'{fn}.txt'), 'w', encoding='utf-8')
count = len(history['items'])
print(' [сохранение сообщений]')
print(' {}/{}'.format(0, count), end='\r')
prev = None
prev_date = None
if append['use']:
prev_date = append['prev_date']
for i in range(count):
m = history['items'][i]
if m['from_id'] not in users:
users_add(dmp._vk, m['from_id'])
res = message_handler(dmp._vk, m)
date = time_handler(m['date'])
hold = ' ' * (users.get(m['from_id'])['length'] + 2)
msg = res['date'] + ' '
msg += hold if (prev and date and prev == m['from_id'] and prev_date == date) \
else users.get(m['from_id'])['name'] + ': '
if res['messages']:
msg += res['messages'][0] + '\n'
for r in res['messages'][1:]:
msg += hold + ' ' * 8 + r + '\n'
else:
msg += '\n'
for a in res['attachments']['audio_messages']:
if a not in attachments['audio_messages']:
attachments['audio_messages'].append(a)
if dmp._settings['SAVE_DIALOG_ATTACHMENTS']:
for tp in res['attachments']:
for a in res['attachments'][tp]:
if a not in attachments[tp]:
attachments[tp].append(a)
if prev_date != date:
if prev_date:
if append['use']:
tmp += '\n'
else:
f.write('\n')
if append['use']:
tmp += f' [{date}]\n'
else:
f.write(f' [{date}]\n')
prev_date = date
if append['use']:
tmp += msg
else:
f.write(msg)
prev = m['from_id']
print('\x1b[2K {}/{}'.format(i + 1, count), end='\r')
if append['use']:
import codecs
orig_file = os.path.join(folder, f'{fn}.txt')
tmp_file = os.path.join(folder, f'{fn}.new')
try:
with codecs.open(orig_file, 'r', encoding='utf-8') as fi,\
codecs.open(tmp_file, 'w', encoding='utf-8') as fo:
for line in fi:
if re.match('^\[last:[0-9]+\]$', line):
line = tmp + '[last:{}]\n'.format(
history['items'][-1]['id'])
fo.write(line)
os.remove(orig_file)
os.rename(tmp_file, orig_file)
except Exception:
os.remove(tmp_file)
else:
f.write('[last:{}]\n'.format(history['items'][-1]['id']))
f.close()
print()
if attachments['audio_messages']:
at_folder = os.path.join(folder, fn)
af = os.path.join(at_folder, 'Голосовые')
os.makedirs(af, exist_ok=True)
print(' [сохранение голосовых сообщений]')
print(' .../{}'.format(len(attachments['audio_messages'])),
end='\r')
with Pool(dmp._settings['POOL_PROCESSES']) as pool:
res = pool.starmap(
copy_func(dmp._download),
zip(itertools.repeat(dmp.__class__),
attachments['audio_messages'], itertools.repeat(af)))
print('\x1b[2K {}/{} (total: {})'.format(
sum(filter(None, res)), len(attachments['audio_messages']),
len(next(os.walk(af))[2])))
if dmp._settings['SAVE_DIALOG_ATTACHMENTS']:
at_folder = os.path.join(folder, fn)
os.makedirs(at_folder, exist_ok=True)
if attachments['photos']:
af = os.path.join(at_folder, 'Фото')
os.makedirs(af, exist_ok=True)
print(' [сохранение фото]')
print(' .../{}'.format(len(attachments['photos'])),
end='\r')
with Pool(dmp._settings['POOL_PROCESSES']) as pool:
res = pool.starmap(
copy_func(dmp._download),
zip(itertools.repeat(dmp.__class__),
attachments['photos'], itertools.repeat(af)))
print('\x1b[2K {}/{} (total: {})'.format(
sum(filter(None, res)), len(attachments['photos']),
len(next(os.walk(af))[2])))
if attachments['video_ids']:
af = os.path.join(at_folder, 'Видео')
os.makedirs(af, exist_ok=True)
videos = dmp._vk_tools.get_all(
method='video.get',
max_count=200,
values={
'videos': ','.join(attachments['video_ids']),
'extended': 1
})
print(' [сохранение видео]')
print(' .../{}'.format(len(videos['items'])), end='\r')
try:
with Pool(dmp._AVAILABLE_THREADS if dmp.
_settings['LIMIT_VIDEO_PROCESSES'] else dmp.
_settings['POOL_PROCESSES']) as pool:
res = pool.starmap(
copy_func(dmp._download_video),
zip(itertools.repeat(dmp.__class__),
videos['items'], itertools.repeat(af)))
print('\x1b[2K {}/{} (total: {})'.format(
sum(filter(None, res)), len(videos['items']),
len(next(os.walk(af))[2])))
except MaybeEncodingError:
print('\x1b[2K ???/{} (total: {})'.format(
len(videos['items']), len(next(os.walk(af))[2])))
if attachments['docs']:
af = os.path.join(at_folder, 'Документы')
os.makedirs(af, exist_ok=True)
print(' [сохранение документов]')
print(' .../{}'.format(len(attachments['docs'])),
end='\r')
with Pool(dmp._settings['POOL_PROCESSES']) as pool:
res = pool.starmap(
copy_func(dmp._download),
zip(itertools.repeat(dmp.__class__),
attachments['docs'], itertools.repeat(af)))
print('\x1b[2K {}/{} (total: {})'.format(
sum(filter(None, res)), len(attachments['docs']),
len(next(os.walk(af))[2])))
print()
with open('users.json', 'w', encoding='utf-8') as f:
json.dump(users, f, ensure_ascii=False, indent=4)
def update(self, dmp, **kwargs):
"""
Updates script and modules
dmp: Dumper obj
"""
import itertools
from multiprocessing import Pool
from modules.utils import copy_func
def apply_args_and_kwargs(fn, args, kwargs):
return fn(*args, **kwargs)
def starmap_with_kwargs(pool, fn, args_iter, kwargs_iter):
args_for_starmap = zip(itertools.repeat(fn), args_iter,
kwargs_iter)
return pool.starmap(apply_args_and_kwargs, args_for_starmap)
if kwargs.get('quite'):
print('Проверка на наличие обновлений...')
else:
self._clear()
self._print_center([
f'{NAME} [{VERSION}]', '', 'Проверка на наличие обновлений...'
],
color=['green', None, 'yellow'])
res = requests.get(
'https://api.github.com/repos/hikiko4ern/vk_dump/releases/latest'
).json()
queue = {}
if 'tag_name' in res:
cv = [int(i) for i in VERSION.split('.')]
nv = [int(i) for i in res['tag_name'].split('v')[1].split('.')]
if (nv[0] > cv[0]) or (nv[0] == cv[0] and nv[1] > cv[1]) or (
nv[0] == cv[0] and nv[1] == cv[1] and nv[2] > cv[2]):
if kwargs.get('quite'):
print('Найдена новая версия ({})'.format(res['tag_name']))
else:
self._print_center('Найдена новая версия ({})'.format(
res['tag_name']),
color='green',
mod='bold',
offset=-2)
for a in res['assets']:
if 'name' in a:
queue[a['name']] = a['browser_download_url']
with Pool(dmp._settings['POOL_PROCESSES']) as pool:
kw = {'force': True, 'text_mode': True}
q = [
queue[k] for k in queue.keys()
if (k != 'dump.py'
and os.path.exists(os.path.join('modules', k)))
]
rem = starmap_with_kwargs(
pool, copy_func(dmp._download),
zip(q, itertools.repeat('modules')),
itertools.repeat(kw))
if kwargs.get('quite'):
print('Обновлено модулей: {}'.format(sum(filter(None,
rem))))
else:
self._print_center('Обновлено модулей: {}'.format(
sum(filter(None, rem))),
color='green',
mod='bold',
offset=-3)
if queue.get('dump.py'):
if copy_func(dmp._download)(queue['dump.py'],
os.getcwd(),
force=True,
text_mode=True):
if kwargs.get('quite'):
print(
'Обновление успешно!\nПерезапустите программу вручную :3'
)
else:
self._print_center([
'Обновление успешно!',
'Перезапустите программу вручную :3'
],
color=['green', 'yellow'],
mod=['bold', 'bold'],
offset=-5)
raise SystemExit
else:
if kwargs.get('quite'):
print(
'Не удалось обновить\nСкачайте и замените dump.py вручную\nhttps://github.com/hikiko4ern/vk_dump/releases/latest'
)
else:
self._print_center([
'Не удалось обновить',
'Скачайте и замените dump.py вручную',
'https://github.com/hikiko4ern/vk_dump/releases/latest'
],
color=['red', 'yellow', None],
mod=['bold', 'bold', None],
offset=-3)
raise SystemExit
else:
if kwargs.get('quite'):
print('Обновлений не найдено')
def dump_fave_posts(dmp):
"""Вложения понравившихся постов (фото, видео, документы)
dmp: Dumper object
"""
folder_photo = os.path.join('dump', 'photo', 'Понравившиеся')
os.makedirs(folder_photo, exist_ok=True)
folder_video = os.path.join('dump', 'video', 'Понравившиеся')
os.makedirs(folder_video, exist_ok=True)
folder_docs = os.path.join('dump', 'docs', 'Понравившиеся')
os.makedirs(folder_docs, exist_ok=True)
print('[получение постов]')
posts = get_fave(dmp._vk, 'posts')
# from pprint import pprint
# print(type(posts))
# print(type(posts['items'][0]))
photo = []
video = []
docs = []
for p in posts['items']:
if 'attachments' in p:
for at in p['attachments']:
if at['type'] == 'photo':
at['photo']['sizes'].sort(
key=itemgetter('width', 'height'))
obj = {
'url': at['photo']['sizes'][-1]['url'],
'prefix': '{}_{}'.format(p['owner_id'], p['id'])
}
if 'access_key' in at['photo']:
obj['access_key'] = at['photo']['access_key']
photo.append(obj)
elif at['type'] == 'video':
video.append('{oid}_{id}{access_key}'.format(
oid=at['video']['owner_id'],
id=at['video']['id'],
access_key='_' +
(at['video'].get('access_key') or '')))
elif at['type'] == 'doc':
obj = {
'url': at['doc']['url'],
'prefix': '{}_{}'.format(p['owner_id'], p['id']),
'name': '{}_{}'.format(at['doc']['title'],
at['doc']['id']),
'ext': at['doc']['ext']
}
if 'access_key' in at['doc']:
obj['access_key'] = at['doc']['access_key']
docs.append(obj)
if video:
video = dmp._vk_tools.get_all(method='video.get',
max_count=200,
values={
'videos': ','.join(video),
'extended': 1
})
print('Сохранение ({} вложений из {} постов):'.format(
sum([len(photo), len(video), len(docs)]), len(posts['items'])))
if photo:
print(' [фото ({})]'.format(len(photo)))
with Pool(dmp._settings['POOL_PROCESSES']) as pool:
pool.starmap(
copy_func(dmp._download),
zip(itertools.repeat(dmp.__class__), photo,
itertools.repeat(folder_photo)))
try:
if video:
print(' [видео ({})]'.format(len(video['items'])))
with Pool(dmp._settings['POOL_PROCESSES']
if not dmp._settings['LIMIT_VIDEO_PROCESSES'] else dmp.
_AVAILABLE_THREADS) as pool:
pool.starmap(
copy_func(dmp._download_video),
zip(itertools.repeat(dmp.__class__), video['items'],
itertools.repeat(folder_video)))
except MaybeEncodingError:
None
if docs:
print(' [документы ({})]'.format(len(docs)))
with Pool(dmp._settings['POOL_PROCESSES']) as pool:
pool.starmap(
copy_func(dmp._download),
zip(itertools.repeat(dmp.__class__), docs,
itertools.repeat(folder_docs)))
