def __init__(self, ignore_existing_token: bool = False):
self.logger = LOGGER('config_vk_api', 'vk_api')
get_requests_db = GetRequestsToDB()
user_data_table_value = get_requests_db.get_records(
tb_name=get_requests_db.userdata,
one_record=True,
select=['access_token'])
token = user_data_table_value['access_token']
self.__additional_windows = AdditionalWindows
if ignore_existing_token is False:
if (token is None) or (token == DEFAULT_VALUE_FOR_BD):
token = self.get_token()
else:
token = self.get_token()
if (token is not None) or (token != DEFAULT_VALUE_FOR_BD):
is_donat = self.check_is_donat(token)
if is_donat is False:
token = None
self.token = token
if self.token is not None:
vk_session = vk_api.VkApi(token=self.token)
self.vk_tool = vk_api.tools.VkTools(vk_session)
if ignore_existing_token is True:
showinfo('Авторизовались', 'Вы удачно авторизовались!')
self.logger.info('Получен vk_tool и сам токен')
else:
self.logger.error('vk_tool не удалось получить')
self.vk_tool = None
del get_requests_db, user_data_table_value
def insert_many_values_into_get_requests(self, type_request: str,
count: int, response: List[dict],
time: float,
last_parse: int) -> None:
"""
Функция вставки данных в таблицу GET запросов к Vk в том случае.
Создана потому что Windows плохо работает если нужно большое
количество данных вставить в БД
:param type_request: тип парсинга
:param count: количество людей
:param response: результат выполнения парсинга
:param time: время парсинга
:param last_parse: возможен ли дальнейший парсинг по данным
:return:
"""
if count <= COUNT_MANY_INSERT:
# Если норм количество людей в записи
peoples = json.dumps(response, ensure_ascii=False)
self.insert_in_table(
tb_name=self.get_requests,
data=[type_request, count, peoples, time, last_parse])
else:
# Если слишком много людей в записи
self.insert_in_table( # Вставка заглушки в основную тб
tb_name=self.get_requests,
data=[type_request, count, REQUIRES_DATA, time, last_parse])
get_request_db = GetRequestsToDB()
pk = get_request_db.get_records(tb_name=self.get_requests,
select=['pk'],
one_record=True,
order='pk DESC')
get_request_db.connect_bd.close()
attachment_pk = int(pk['pk'])
slice_from = 0
slice_to = COUNT_MANY_INSERT + 1
while True:
peoples = response[slice_from:slice_to]
peoples = json.dumps(peoples, ensure_ascii=False)[1:-1]
self.insert_in_table(tb_name=self.additional_get_requests,
data=[attachment_pk, peoples])
if slice_to == count + 1:
break
slice_from = slice_to
if slice_to + COUNT_MANY_INSERT + 1 > count + 1:
slice_to = count + 1
else:
slice_to += COUNT_MANY_INSERT + 1
LOGGER.warning('Очистка данных')
del type_request, count, response, time, last_parse, peoples
gc.collect()
def __get_update_row__(self, tb_name: str, update_row: dict,
where: str) -> dict:
rows = self.columns[tb_name]
if len(rows) == len(update_row):
return update_row
get_requests_db = GetRequestsToDB()
values = get_requests_db.get_records(tb_name=tb_name,
where=where,
one_record=True)
for item in rows:
if item not in update_row:
update_row[item] = values[item]
return update_row
def __update_db__(attachment_pk: int, res_peoples: list, last_parse: int,
type_request: str) -> Union[None, int]:
"""
Функция добавления записи в базу
:param attachment_pk: к какому pk прикреплять
:param res_peoples: что нужно записать
:param last_parse: возможен ли дальнейший парсинг
:param type_request: тип запроса
:return: attachment_pk
"""
if len(res_peoples) == 0:
return
get_request_db = GetRequestsToDB()
update_request_db = UpdateRequestsToDB()
if attachment_pk is None:
update_request_db.insert_in_table(
tb_name=update_request_db.get_requests,
data=[type_request, 0, REQUIRES_DATA,
time_now(), last_parse])
attachment_pk = get_request_db.get_records(
tb_name=get_request_db.get_requests,
select=['pk'],
one_record=True,
order='pk DESC')
get_request_db.connect_bd.close()
attachment_pk = int(attachment_pk['pk'])
peoples = json.dumps(res_peoples, ensure_ascii=False)[1:-1]
update_request_db.insert_in_table(
tb_name=update_request_db.additional_get_requests,
data=[attachment_pk, peoples])
del peoples
del res_peoples
gc.collect()
return attachment_pk
def parsing_by_groups(self, widgets: dict) -> None:
"""
Парсинг по критериям
:param widgets: словарь {ключ: виджет}
:return:
"""
pk = widgets['entry_pk'].get().strip()
if bool(pk) is False:
showerror('Не выбрана запись',
ERROR_MSG['Parsing']['Dont_choose_pk'])
return
pk = int(pk)
need_country = widgets['var_need_country'].get()
need_city_region = widgets['var_need_city_region'].get()
var_city_region = widgets['var_city_region'].get()
cmb_city_region = widgets['cmb_city_region'].get()
need_followers = widgets['var_need_count_followers'].get()
need_data = widgets['var_need_old'].get()
need_relationship = widgets['var_need_relationship'].get()
need_political = widgets['var_need_political'].get()
need_life_main = widgets['var_need_life_main'].get()
need_people_main = widgets['var_need_people_main'].get()
need_smoking = widgets['var_need_smoking'].get()
need_alcohol = widgets['var_need_alcohol'].get()
need_entry_status = widgets['var_need_entry_status'].get()
need_entry_about = widgets['var_need_entry_about'].get()
need_last_seen = widgets['var_need_last_seen'].get()
sex = widgets['var_sex'].get()
send_message = widgets['var_can_send_message'].get()
online = widgets['var_only'].get()
photos = widgets['var_has_photo'].get()
deactivated = widgets['var_deactivate'].get()
lbl_progress = widgets['lbl_progress']
progressbar = widgets['progressbar']
followers_from, followers_to, data_from, data_to = 0, 0, 0, 0
relationship, last_seen_from, last_seen_to, political = 0, 0, 0, 0
people_main, life_main, smoking, alcohol, entry_status = 0, 0, 0, 0, ''
entry_about, country, city, cities = '', 0, 0, []
if (need_country == 0) and (need_city_region == 1):
need = askyesno('Не включена настройка',
ASKS['Parsing']['Dont_enable_country'])
if need is False:
return
need_city_region = 0
if need_city_region == 1 and cmb_city_region == 'Нажмите "Настройка"':
need = askyesno('Не сделана настройка',
ASKS['Parsing']['Dont_settings'])
if need is False:
return
need_city_region = 0
if need_followers == 1:
try:
followers_from = widgets['var_followers_from'].get()
followers_to = widgets['var_followers_to'].get()
except TclError:
showerror('Неверное значение',
ERROR_MSG['Parsing']['Validate_follower']['Not_int'])
return
if followers_from > followers_to:
showerror(
'Неверное значение',
ERROR_MSG['Validate_follower']['Not_int']['From_more_to'])
return
if (followers_from > FOLLOWERS_MAX) or (followers_to >
FOLLOWERS_MAX):
showerror(
'Неверное значение', ERROR_MSG['Validate_follower']
['Not_int']['Max_value'].format(FOLLOWERS_MAX))
return
if need_data == 1:
data_from = int(widgets['var_old_from'].get())
data_to = int(widgets['var_old_to'].get())
if data_from > data_to:
showerror('Неверное значение',
ERROR_MSG['Parsing']['Validate_old']['From_more_to'])
return
if need_relationship == 1:
relationship = widgets['var_relationship'].get()
relationship = STATUS_VK_PERSON[relationship]
if (online == 0) and (need_last_seen == 1):
try:
last_seen_from = widgets['var_last_seen_from'].get()
last_seen_to = widgets['var_last_seen_to'].get()
except TclError:
showerror(
'Неверное значение',
ERROR_MSG['Parsing']['Validate_last_seen']['Not_int'])
return
if last_seen_from > last_seen_to:
showerror(
'Неверное значение',
ERROR_MSG['Parsing']['Validate_last_seen']['From_more_to'])
return
if (last_seen_from > LAST_SEEN_MAX) or (last_seen_to >
LAST_SEEN_MAX):
showerror(
'Неверное значение',
ERROR_MSG['Parsing']['Validate_last_seen']['Max_value'])
return
if need_political == 1:
political = widgets['var_political'].get()
political = POLITICAL[political]
if need_people_main == 1:
people_main = widgets['var_people_main'].get()
people_main = PEOPLE_MAIN[people_main]
if need_life_main == 1:
life_main = widgets['var_life_main'].get()
life_main = LIFE_MAIN[life_main]
if need_smoking == 1:
smoking = widgets['var_smoking'].get()
smoking = SMOKING[smoking]
if need_alcohol == 1:
alcohol = widgets['var_alcohol'].get()
alcohol = ALCOHOL[alcohol]
if need_entry_status == 1:
entry_status = widgets['var_entry_status'].get().strip()
if entry_status == '':
showerror('Неверное значение',
ERROR_MSG['Parsing']['Validate_status']['Empty'])
return
if need_entry_about == 1:
entry_about = widgets['var_entry_about'].get().strip()
if entry_about == '':
showerror('Неверное значение',
ERROR_MSG['Parsing']['Validate_about']['Empty'])
return
if need_country == 1:
country = widgets['var_country'].get()
country = LIST_COUNTRIES[country]
if need_city_region == 1:
city_region = widgets['var_city_region'].get()
if city_region == 0:
city = widgets['cmb_city_region'].get()
city = self.additional_functions.cities['cities'][city]
else:
region = widgets['cmb_city_region'].get()
region = self.additional_functions.regions['regions'][region]
params = {
'country_id': country,
'region_id': region,
'need_all': 1
}
cities_res = GetRequestsToVkApi().get_all_object(
'database.getCities', **params)
cities = []
for item in cities_res['items']:
cities.append(item['id'])
lbl_progress.configure(text='Подождите...')
lbl_progress.update()
get_requests_db = GetRequestsToDB()
records = get_requests_db.get_records(
tb_name=get_requests_db.get_requests,
select=['response', 'count_people'],
where=f'pk={pk}',
one_record=True)
count_peoples = records['count_people']
step = PROGRESSBAR_MAX / count_peoples
pg_value = 0
records = records['response']
iteration = 0
result = []
lbl_text = f'Прогресс {iteration}/{count_peoples}. Подождите, ' \
'это может занять несколько минут... '
configure_progress_lbl(progressbar, lbl_progress, pg_value, lbl_text)
if records != REQUIRES_DATA:
pks = [json.loads(f'[{records}]')]
else:
pks = get_requests_db.get_records(
tb_name=get_requests_db.additional_get_requests,
select=['pk'],
where=f'pk_attachment={pk}',
dictionary=False)
pks = [item[0] for item in pks]
del records
gc.collect()
for pk in pks:
if type(pk) == str or type(pk) == int:
record = get_requests_db.get_records(
tb_name=get_requests_db.additional_get_requests,
select=['response'],
where=f'pk={pk}',
one_record=True)['response']
record = json.loads(f'[{record}]')
else:
record = pk
for item in record:
iteration += 1
lbl_text = f'Прогресс {iteration}/{count_peoples}. ' \
'Подождите, это может занять несколько минут... '
pg_value += step
configure_progress_lbl(progressbar, lbl_progress, pg_value,
lbl_text)
if deactivated == 1:
if item.get('deactivated'):
continue
else:
if item.get('deactivated'):
result.append(item['id'])
continue
if need_country == 1:
if item.get('country'):
if item['country']['id'] != country:
continue
else:
continue
if need_city_region == 1:
if var_city_region == 0:
if item.get('city'):
if item['city']['id'] != city:
continue
else:
continue
else:
if item.get('city'):
if item['city']['id'] not in cities:
continue
else:
continue
if need_followers == 1:
if item.get('followers_count'):
count = item['followers_count']
if followers_from > count > followers_to:
continue
else:
continue
if need_data == 1:
if item.get('bdate'):
bdate = item['bdate'].split('.')
if len(bdate) != 3:
pass
else:
year_now = int(datetime.datetime.now().year)
date_from = year_now - data_from
date_to = year_now - data_to
bdate = int(bdate[2])
if date_from <= bdate or bdate <= date_to:
del year_now, date_from, date_to, bdate
continue
del year_now, date_from, date_to, bdate
if need_relationship == 1:
if item.get('relation'):
if item['relation'] != relationship:
continue
else:
continue
if sex != 0:
if item.get('sex'):
if item['sex'] != sex:
continue
else:
continue
if send_message == 1:
if item.get('can_write_private_message'):
if item['can_write_private_message'] != 1:
continue
else:
continue
if online == 1:
if item.get('online'):
if item['online'] != 1:
continue
else:
continue
if (online == 0) and (need_last_seen == 1):
if item.get('online'):
time = time_now()
time_from = time - (last_seen_from * 24 * 60 * 60)
time_to = time - (last_seen_to * 24 * 60 * 60)
if (item['online'] < time_from) or (item['online'] >
time_to):
continue
else:
continue
if photos == 1:
if item.get('has_photo'):
if item['has_photo'] != 1:
continue
else:
continue
if (need_political == 1) or (need_life_main == 1) or (
need_people_main == 1) \
or (need_smoking == 1) or (need_alcohol == 1):
if item.get('personal'):
personal = item['personal']
if need_political == 1:
if personal.get('political'):
if personal['political'] != political:
continue
else:
continue
if need_life_main == 1:
if personal.get('life_main'):
if personal['life_main'] != life_main:
continue
else:
continue
if need_people_main == 1:
if personal.get('people_main'):
if personal['people_main'] != people_main:
continue
else:
continue
if need_smoking == 1:
if personal.get('smoking'):
if personal['smoking'] != smoking:
continue
else:
continue
if need_alcohol == 1:
if personal.get('alcohol'):
if personal['alcohol'] != alcohol:
continue
else:
continue
else:
continue
if need_entry_about == 1:
if item.get('about'):
if entry_about not in item['about']:
continue
else:
continue
if need_entry_status == 1:
if item.get('status'):
if entry_status not in item['status']:
continue
else:
continue
result.append(item['id'])
del pk, record
gc.collect()
lbl_progress.configure(text='Запись данных',
foreground=styles.NOTABLE_LABEL_FONT)
lbl_progress.update()
count = len(result)
if count == 0:
configure_progress_lbl(lbl=lbl_progress)
showinfo('Не найдено пользователей',
INFO_MSG['Parsing']['none_value_by_criteria'])
return
configure_progress_lbl(progressbar, lbl_progress, 0,
'Запись результатов', styles.NOTABLE_LABEL_FONT)
type_request = NAME_PARSING['by_criteria']
UpdateRequestsToDB().insert_many_values_into_get_requests(
type_request=type_request,
count=count,
response=result,
time=time_now(),
last_parse=0)
configure_progress_lbl(lbl=lbl_progress)
del result, pks, lbl_text
gc.collect()
class TreeViewWindow:
"""
Класс отвечающий за окно просмотра запросов
"""
def __init__(self,
method: str = 'view',
completion_name: str = None,
entry_pk: object = None):
"""
Запуск окна просмотра запросов
:param method: метод просмотра (view-обычный просмотр всех запросов)
:param completion_name: имя запроса default: None
:param entry_pk: виджет Entry для записи выбранного pk default: None
"""
self.method = method
self.completion_name = completion_name
self.entry_pk = entry_pk
self.select = ['pk', 'type_request', 'count_people', 'time_request']
self.get_requests_db = GetRequestsToDB()
self.values = []
self.window = Toplevel()
self.initialize_ui()
left_frame = ttk.Frame(self.window)
left_frame.grid(row=0, column=0, sticky='NSWE')
right_frame = ttk.Frame(self.window, padding=5)
right_frame.grid(row=0, column=1, sticky='NSWE')
self.tree_view = ttk.Treeview(left_frame,
columns=('Тип', 'Количество id', 'Дата'))
self.tree_view.heading('#0', text='Дата')
self.tree_view.heading('#1', text='Тип')
self.tree_view.heading('#2', text='Количество id')
self.tree_view.heading('#3', text='ID')
self.tree_view.column('#0', minwidth=200, width=200, stretch=0)
self.tree_view.column('#1', minwidth=200, width=220, stretch=0)
self.tree_view.column('#2',
minwidth=50,
width=130,
stretch=0,
anchor='center')
self.tree_view.column('#3',
minwidth=100,
width=100,
stretch=0,
anchor='center')
self.tree_view.grid(row=0,
column=0,
columnspan=4,
rowspan=3,
sticky='NSEW')
if method == 'view':
btn_clear = ttk.Button(right_frame, text='Очистить')
btn_delete = ttk.Button(right_frame, text='Удалить')
btn_download = ttk.Button(right_frame, text='Выгрузить txt')
btn_clear.grid(row=0, column=0, sticky='WE')
btn_delete.grid(row=1, column=0, pady=5, sticky='WE')
btn_download.grid(row=2, column=0, sticky='WE')
btn_clear.bind('<Button-1>', lambda event: self.clear_values())
btn_delete.bind('<Button-1>', lambda event: self.delete_value())
btn_download.bind('<Button-1>',
lambda event: self.download_on_txt())
else:
btn_choose = ttk.Button(right_frame, text='Выбрать')
btn_cancel = ttk.Button(right_frame,
text='Отмена',
command=lambda: self.window.destroy())
btn_choose.grid(row=0, column=0, sticky='SWE')
btn_cancel.grid(row=1, column=0, sticky='SWE', pady=5)
btn_cancel.bind('<Button-1>', lambda event: self.cancel())
btn_choose.bind('<Button-1>',
lambda event: self.choose_value_for_parsing())
self.tree_view.bind('<Double-Button-1>',
lambda event: self.choose_value_for_parsing())
self.window.rowconfigure(0, weight=1)
self.window.columnconfigure(0, weight=9)
self.window.columnconfigure(1, weight=1)
left_frame.rowconfigure(0, weight=1)
left_frame.columnconfigure(0, weight=1)
self.__get_records__()
self.completion_tree_view()
def initialize_ui(self):
"""
Инициализация окна
:return:
"""
FPVK = ImageTk.PhotoImage(
file=os.path.join(path_to_dir_ico, 'FPVK.ico'))
styles.set_global_style(self.window)
w = 800
h = 600
self.window.title('Основные запросы')
set_position_window_on_center(self.window, width=w, height=h)
self.window.tk.call('wm', 'iconphoto', self.window._w, FPVK)
def choose_value_for_parsing(self):
"""
Функция отвечающая за обработку двойного клика в случае выбора записи
:return:
"""
try:
pk = self.get_choose_value()['pk']
except IndexError:
return
self.entry_pk.delete(0, 'end')
self.entry_pk.insert('end', str(pk))
del self.values
self.window.destroy()
def cancel(self):
"""
Кнопка отмены выбора pk
:return:
"""
self.entry_pk.delete(0, 'end')
del self.values
self.window.destroy()
def get_choose_value(self):
"""
Получить объект выбранной записи
:return:
"""
if not self.tree_view.selection():
raise IndexError('не выбран элемент')
item = self.tree_view.selection()[0]
values = self.tree_view.item(item, option="values")
result = {'pk': values[2], 'method': values[0], 'count': values[1]}
return result
def completion_tree_view(self):
"""
Функция заполняет TreeView данным из BaseData
:return:
"""
if len(self.values) == 0:
return
if type(self.values) == dict:
self.values = [self.values]
index = 0
for item in self.values:
time = gmtime(int(item['time_request']))
data = strftime(FORMAT_DATE, time)
self.tree_view.insert('',
index=index,
text=data,
values=(item['type_request'],
item['count_people'], item['pk']))
index += 1
def clear_values(self):
"""
Очищает все записи в BD
:return:
"""
ask = askyesno('Очистка записей',
'Вы уверены, что хотите удалить все записи?')
if ask is True:
LOGGER.warning('Запрос на удаление таблицы GetRequestsApi')
delete_request_db = DeleteRequestsToDB()
delete_request_db.delete_all_records(
delete_request_db.get_requests)
delete_request_db.delete_all_records(
delete_request_db.additional_get_requests)
self.window.destroy()
def delete_value(self):
"""
Удаляет выбранную запись
:return:
"""
try:
values = self.get_choose_value()
pk = values['pk']
LOGGER.warning(
'Запрос на удаление элемента в таблице GetRequestsApi')
delete_request_db = DeleteRequestsToDB()
delete_request_db.delete_record_in_bd(
tb_name=delete_request_db.get_requests, where=f'pk={pk}')
delete_request_db.delete_record_in_bd(
tb_name=delete_request_db.additional_get_requests,
where=f'pk_attachment={pk}')
except IndexError as error:
if str(error) == 'не выбран элемент':
return
for row in self.tree_view.get_children():
self.tree_view.delete(row)
del self.values
gc.collect()
self.__get_records__()
self.completion_tree_view()
def download_on_txt(self):
"""
Выгрузка ID в txt формате
:return:
"""
try:
values = self.get_choose_value()
except IndexError:
return
pk = values['pk']
__values__ = self.get_requests_db.get_records(
tb_name=self.get_requests_db.get_requests,
select=['response', 'type_request', 'last_parse'],
where=f'pk={pk}',
one_record=True)
method = __values__['type_request']
last_parse = int(__values__['last_parse'])
result = []
need_pk = False
if __values__['response'] != REQUIRES_DATA:
results = __values__['response']
results = json.loads(f'[{results}]')
else:
__values__ = self.get_requests_db.get_records(
tb_name=self.get_requests_db.additional_get_requests,
select=['pk'],
where=f'pk_attachment={pk}',
dictionary=False)
results = [int(item[0]) for item in __values__]
need_pk = True
for pk in results:
if need_pk is False:
item = pk
else:
__item__ = self.get_requests_db.get_records(
tb_name=self.get_requests_db.additional_get_requests,
select=['response'],
where=f'pk={pk}',
one_record=True)
item = __item__['response']
item = json.loads(f'[{item}]')
if method == NAME_PARSING['by_groups']:
if last_parse == 1:
result += [str(i['id']) for i in item]
else:
result += [str(i) for i in item]
else:
result += [str(i) for i in item]
del pk, item
del results
result = '\n'.join(result)
directory = asksaveasfilename()
if directory in ['', ' ']:
del result
gc.collect()
return
if directory[-4:] != '.txt':
directory += '.txt'
with open(directory, 'w', encoding='utf-8') as file:
file.write(result)
del result
gc.collect()
def __get_records__(self):
if self.method == 'view':
values = self.get_requests_db.get_records(
select=self.select,
order='pk DESC',
tb_name=self.get_requests_db.get_requests,
)
self.values = [] if len(values) == 0 else values
else:
self.entry_pk = self.entry_pk
values = self.get_requests_db.get_records(
select=self.select,
order='pk DESC',
tb_name=self.get_requests_db.get_requests,
where=f' (type_request = "{self.completion_name}") '
'and (last_parse = 1)')
self.values = [] if len(values) == 0 else values
def __init__(self, window_preview):
"""
Создаёт превью и проверяет нужные настройки для программы, а также
запускает её
:param window_preview: объект окна превью
"""
self.logger = LOGGER('main', 'main')
png_preview_open, png_preview = self.preview_image_open()
self.preview_image_set(png_preview_open, png_preview, window_preview)
window_preview.update()
tracemalloc.start()
time.sleep(2)
MainDB()
get_requests_db = GetRequestsToDB()
settings = get_requests_db.get_records(
tb_name=get_requests_db.settings,
one_record=True,
select=['first_start', 'auto_update'])
first_start = settings['first_start']
auto_update = settings['auto_update']
if first_start == 1:
self.logger.info('Первый запуск')
window_preview.destroy()
done = AdditionalWindows().person_and_agreement_data()
if done is True:
update_requests_db = UpdateRequestsToDB()
update_requests_db.update_table(
tb_name=update_requests_db.settings,
update_row={'first_start': 0})
self.logger.warning('Очистка от лишних файлов в директории')
list_path = os.listdir(path)
if REPO_BRANCH_UPDATER in list_path:
rmtree(REPO_BRANCH_UPDATER, ignore_errors=True, onerror=None)
if REPO_BRANCH_VERSION in list_path:
rmtree(REPO_BRANCH_VERSION, ignore_errors=True, onerror=None)
if REPO_BRANCH_MASTER in list_path:
rmtree(REPO_BRANCH_MASTER, ignore_errors=True, onerror=None)
try:
self.logger.warning('Закрытие окна первью')
window_preview.destroy()
except TclError:
pass
del settings, first_start, list_path, window_preview
gc.collect()
self.logger.info('Создание задачи scheduler')
scheduler = BackgroundScheduler()
scheduler.start()
scheduler.add_job(__scheduler__, 'interval', minutes=1)
self.logger.info('Запуск приложения')
from windows import App
App(auto_update, OS)
self.logger.info('Закрытие приложения')