def test_save_single_groups_data_cli(monkeypatch):
group_tools = GroupTools()
monkeypatch.setattr('builtins.input', lambda _: single_group_name)
group_tools.save_single_groups_data_cli(url=success_test_page, raw_html_file=test_raw_html_path)
test_html_files = glob.glob(test_raw_html_path + '*_GroupONE')
assert len(test_html_files) == 1
shutil.rmtree(test_raw_html_path, ignore_errors=True)
def test_save_all_groups_data_gui():
group_tools = GroupTools()
group_tools.save_all_groups_data_gui(url=success_test_page,
raw_html_file=test_raw_html_path)
test_html_files = glob.glob(test_raw_html_path + '*_Group*')
assert len(test_html_files) == 2
shutil.rmtree(test_raw_html_path, ignore_errors=True)
def test_bad_file_join_multiple_groups_gui():
group_tools = GroupTools()
assert group_tools.join_multiple_groups_gui(test_local_groups_file,
groups_to_join_path=test_groups_to_join,
groups_failed_path=test_groups_failed,
timeout=timeout) is False
shutil.rmtree(test_groups_to_join, ignore_errors=True)
shutil.rmtree(test_groups_failed, ignore_errors=True)
def test_join_multiple_groups_gui():
if not os.path.exists(test_groups_to_join):
os.makedirs(test_groups_to_join)
with open(test_local_groups_path, 'w+') as fp:
fp.write(success_test_page + "\n")
fp.write(success_test_page + "\n")
fp.write(fail_test_page + "\n")
fp.close()
group_tools = GroupTools()
group_tools.join_multiple_groups_gui(test_local_groups_file,
groups_to_join_path=test_groups_to_join,
groups_failed_path=test_groups_failed,
timeout=timeout)
assert sum(1 for line in open(test_groups_failed + test_local_groups_file, 'r')) == 1
shutil.rmtree(test_groups_to_join, ignore_errors=True)
shutil.rmtree(test_groups_failed, ignore_errors=True)
def test_success_join_multiple_groups_cli(monkeypatch):
if not os.path.exists(test_groups_to_join):
os.makedirs(test_groups_to_join)
with open(test_local_groups_path, 'w+') as fp:
fp.write(success_test_page + "\n")
fp.write(success_test_page + "\n")
fp.write(fail_test_page + "\n")
fp.close()
group_tools = GroupTools()
monkeypatch.setattr('builtins.input', lambda _: test_local_groups_file)
group_tools.join_multiple_groups_cli(groups_to_join_path=test_groups_to_join,
groups_failed_path=test_groups_failed,
timeout=timeout)
assert sum(1 for line in open(test_groups_failed + test_local_groups_file, 'r')) == 1
shutil.rmtree(test_groups_to_join, ignore_errors=True)
shutil.rmtree(test_groups_failed, ignore_errors=True)
def test_remove_group_old_html():
if not os.path.exists(test_raw_html_path):
os.makedirs(test_raw_html_path)
with open(test_raw_html_path + '99999_' + single_group_name, 'w+') as fp:
fp.write("\n")
fp.close()
with open(test_raw_html_path + '00000_' + single_group_name, 'w+') as fp:
fp.write("\n")
fp.close()
group_tools = GroupTools()
group_tools.save_single_groups_data_gui(single_group_name,
url=success_test_page,
raw_html_file=test_raw_html_path)
test_html_files = glob.glob(test_raw_html_path + '*_GroupONE')
assert len(test_html_files) == 1
shutil.rmtree(test_raw_html_path, ignore_errors=True)
def test_back_join_multiple_groups_cli(monkeypatch):
group_tools = GroupTools()
monkeypatch.setattr('builtins.input', lambda _: next(test_multi_group_join_responses))
assert group_tools.join_multiple_groups_cli(groups_to_join_path=test_groups_to_join) is None
shutil.rmtree(test_groups_to_join, ignore_errors=True)
def test_join_group_gui():
group_tools = GroupTools()
assert group_tools.join_group_gui(success_test_page) is True
def test_back_join_group_cli(monkeypatch):
group_tools = GroupTools()
monkeypatch.setattr('builtins.input', lambda _: back_page)
assert group_tools.join_group_cli() is None
def test_fail_join_group_cli(monkeypatch):
group_tools = GroupTools()
monkeypatch.setattr('builtins.input', lambda _: fail_test_page)
assert group_tools.join_group_cli(timeout) is False
def main(test=False):
tool = GroupTools()
scrape = MultiProcessHtml()
system = SystemTools()
last_func_called = ""
while True:
print("\n" + 10 * "#" + " GROUP TOOLS MENU " + 10 * "#" +
"\nPlease choose one of the following options\n")
user_input = input(
"\njoin_group: joins a single WhatsApp group\n\n" +
"join_multiple_groups: joins all groups listed in a file\n\n" +
"single_group_data: retrieves and saves raw html containing single group messages\n\n"
+
"all_groups_data: retrieves and saves raw html containing group messages for all groups\n\n"
+ "print_groups: prints group list to the console\n\n" +
"back: return to main menu\n\nUSER INPUT: ")
print("")
if user_input == ("join_group"):
tool.join_group_cli()
last_func_called = "join_group"
elif user_input == ("join_multiple_groups"):
tool.join_multiple_groups_cli()
last_func_called = "join_multiple_groups"
elif user_input == ("single_group_data"):
tool.save_single_groups_data_cli()
last_func_called = "single_group_data"
elif user_input == ("all_groups_data"):
if test is False:
tool.save_all_groups_data_cli()
last_func_called = "all_groups_data"
elif user_input == ("print_groups"):
tool.print_groups()
last_func_called = "print_groups"
elif user_input == ("generate_csv"):
df = pd.DataFrame()
df = scrape.get_message_frame_all_groups(
system.get_processed_pck_list())
scrape.generate_message_summary_csv(df)
last_func_called = "generate_csv"
elif user_input == ("back"):
break
else:
print(
"\n\n******** PLEASE ENTER A VALID OPTION OR 'back' TO END PROGRAM.\n"
)
last_func_called = "invalid_input"
print_endpoints()
return last_func_called
def test__init__():
group_tools = GroupTools()
browser_type = BrowserTools()
assert type(group_tools.browser) == type(browser_type)
assert type(group_tools.group_elements) == list and len(group_tools.group_elements) == 0
assert type(group_tools.group_list) == list and len(group_tools.group_list) == 0
def test_print_groups():
group_tools = GroupTools()
assert len(group_tools.print_groups(success_test_page)) == 2
class WhatsScrapeGUI:
def __init__(self):
self.group_tools = GroupTools()
def make_home_menu(self):
layout = [
[
sg.Column([[
sg.Text('Welcome to the WhatsApp Web Scraping Tool',
key='-MESSAGE_1-',
font=['Comic', 30, 'bold'],
text_color='#ffffff',
background_color='#2e9688')
]],
background_color='#2e9688')
],
[
sg.Column([[
sg.Text('Please Choose One Of The Following Options!',
key='-MESSAGE_2-',
font=['Comic', 20, 'bold'],
text_color='#000000',
background_color='#d8dbd4')
]],
background_color='#d8dbd4',
justification='center',
pad=(50, 50))
],
[
sg.Column([[
sg.Button('Join Group', key='-JOIN_GROUP-'),
sg.Button('Join Multiple Groups',
key='-JOIN_MULTI_GROUP-'),
sg.Button('Save All Groups Data', key='-ALL_GROUPS-'),
sg.Button('Save Single Group Data', key='-SINGLE_GROUP-'),
sg.Button('Print Groups', key='-PRINT_GROUPS-'),
sg.Button('Generate CSV', key='-GENERATE_CSV-'),
sg.Button('Quit', key='-QUITER-')
]],
background_color='#d8dbd4',
justification='center')
]
]
return sg.Window('Window Title',
layout=layout,
background_color='#d8dbd4',
button_color=['#000000', '#2e9688'],
margins=[0, 0])
def make_print_groups_screen(self, calling_window):
group_list = self.group_tools.get_group_list()
group_text = ""
for group in group_list:
group_text += (group + "\n")
layout = [[
sg.Column([[
sg.Text('Your Joined Group List',
key='-MESSAGE_1-',
font=['Comic', 20, 'bold'],
text_color='#ffffff',
background_color='#2e9688')
]],
background_color='#2e9688')
],
[
sg.Column([[
sg.Multiline(default_text=group_text,
key='-OUTPUT_1-',
auto_size_text=True,
size=(20, 30))
]])
],
[
sg.Column([[sg.Button('Return', key='-QUITER-')]],
background_color='#d8dbd4',
justification='center')
]]
calling_window.close()
return sg.Window('Printed Groups',
layout=layout,
background_color='#d8dbd4',
button_color=['#000000', '#2e9688'],
margins=[0, 0])
def make_join_group_screen(self, calling_window):
layout = [[
sg.Column([[
sg.Text('JOIN A GROUP',
key='-MESSAGE_1-',
font=['Comic', 20, 'bold'],
text_color='#ffffff',
background_color='#2e9688')
]],
justification='center')
],
[
sg.Text('Please provide a group link to join: ',
key='-INPUT_1-',
size=(30, 1),
background_color='#d8dbd4',
font=['Comic', 12, 'bold'],
text_color='#000000'),
sg.InputText()
],
[
sg.Column([[
sg.Button('Return', key='-QUITER-'),
sg.Button('Submit', key='-SUBMIT-')
]],
background_color='#d8dbd4',
justification='center')
]]
calling_window.close()
return sg.Window('JOIN A GROUP',
layout=layout,
background_color='#d8dbd4',
button_color=['#000000', '#2e9688'],
margins=[0, 0])
def make_group_failure_screen(self, calling_window, group_link):
layout = [
[
sg.Column([[
sg.Text('GROUP FAIL NOTIFICATION',
key='-MESSAGE_1-',
font=['Comic', 20, 'bold'],
text_color='#ffffff',
background_color='#2e9688')
]],
justification='center')
],
[
sg.Column([[
sg.Text('This group link FAILED: "' + group_link + '"',
key='-MESSAGE_2-',
background_color='#d8dbd4',
font=['Comic', 12, 'bold'],
text_color='#000000')
]],
background_color='#d8dbd4',
justification='center')
],
[
sg.Column([[
sg.Text('Please ensure the link is valid.',
key='-MESSAGE_3-',
background_color='#d8dbd4',
font=['Comic', 12, 'bold'],
text_color='#000000')
]],
background_color='#d8dbd4',
justification='center')
],
[
sg.Column([[sg.Button('Return', key='-QUITER-')]],
background_color='#d8dbd4',
justification='center')
]
]
calling_window.close()
return sg.Window('GROUP FAIL',
layout=layout,
background_color='#d8dbd4',
button_color=['#000000', '#2e9688'],
margins=[0, 0])
def make_group_success_screen(self, calling_window, group_link):
layout = [
[
sg.Column([[
sg.Text('GROUP SUCCESS NOTIFICATION',
key='-MESSAGE_1-',
font=['Comic', 20, 'bold'],
text_color='#ffffff',
background_color='#2e9688')
]],
justification='center')
],
[
sg.Column([[
sg.Text('This group link SUCCEEDED: "' + group_link + '"',
key='-MESSAGE_2-',
background_color='#d8dbd4',
font=['Comic', 12, 'bold'],
text_color='#000000')
]],
background_color='#d8dbd4',
justification='center')
],
[
sg.Column([[sg.Button('Return', key='-QUITER-')]],
background_color='#d8dbd4',
justification='center')
]
]
calling_window.close()
return sg.Window('GROUP SUCCESS',
layout=layout,
background_color='#d8dbd4',
button_color=['#000000', '#2e9688'],
margins=[0, 0])
def make_join_multiple_groups_screen(self, calling_window):
layout = [
[
sg.Column([[
sg.Text('JOIN MULTIPLE GROUPS',
key='-MESSAGE_1-',
font=['Comic', 20, 'bold'],
text_color='#ffffff',
background_color='#2e9688')
]],
justification='center')
],
[
sg.Text(
'Please provide a filename in the "/scrape/group_files/groups_to_join/" directory: ',
key='-MESSAGE_2-',
background_color='#d8dbd4',
font=['Comic', 12, 'bold'],
text_color='#000000'),
sg.InputText()
],
[
sg.Column([[
sg.Button('Return', key='-QUITER-'),
sg.Button('Submit', key='-SUBMIT-')
]],
background_color='#d8dbd4',
justification='center')
]
]
calling_window.close()
return sg.Window('JOIN MULTIPLE GROUPS',
layout=layout,
background_color='#d8dbd4',
button_color=['#000000', '#2e9688'],
margins=[0, 0])
def make_multiple_group_failure_screen(self, calling_window, file_name):
layout = [
[
sg.Column([[
sg.Text('GROUP FAIL NOTIFICATION',
key='-MESSAGE_1-',
font=['Comic', 20, 'bold'],
text_color='#ffffff',
background_color='#2e9688')
]],
justification='center')
],
[
sg.Column([[
sg.Text('This group file FAILED: "' + file_name + '"',
key='-MESSAGE_2-',
background_color='#d8dbd4',
font=['Comic', 12, 'bold'],
text_color='#000000')
]],
background_color='#d8dbd4',
justification='center')
],
[
sg.Column([[
sg.Text('Please ensure the file exists.',
key='-MESSAGE_3-',
background_color='#d8dbd4',
font=['Comic', 12, 'bold'],
text_color='#000000')
]],
background_color='#d8dbd4',
justification='center')
],
[
sg.Column([[sg.Button('Return', key='-QUITER-')]],
background_color='#d8dbd4',
justification='center')
]
]
calling_window.close()
return sg.Window('GROUP FAIL',
layout=layout,
background_color='#d8dbd4',
button_color=['#000000', '#2e9688'],
margins=[0, 0])
def make_multiple_group_success_screen(self, calling_window, file_name):
failed_group_text = self.__get_failed_group_list(file_name)
layout = [
[
sg.Column([[
sg.Text('GROUP SUCCESS NOTIFICATION',
key='-MESSAGE_1-',
font=['Comic', 20, 'bold'],
text_color='#ffffff',
background_color='#2e9688')
]],
justification='center')
],
[
sg.Column([[
sg.Text('This group file SUCCEEDED: "' + file_name + '"',
key='-MESSAGE_2-',
background_color='#d8dbd4',
font=['Comic', 12, 'bold'],
text_color='#000000')
]],
background_color='#d8dbd4',
justification='center')
],
[
sg.Column([[
sg.Text('Group links in the TEXT box below FAILED:',
key='-MESSAGE_3-',
background_color='#d8dbd4',
font=['Comic', 12, 'bold'],
text_color='#000000')
]],
background_color='#d8dbd4',
justification='center')
],
[
sg.Column([[
sg.Multiline(default_text=failed_group_text,
key='-OUTPUT_1-',
auto_size_text=True,
size=(70, 30))
]])
],
[
sg.Column([[sg.Button('Return', key='-QUITER-')]],
background_color='#d8dbd4',
justification='center')
]
]
calling_window.close()
return sg.Window('GROUP FAIL',
layout=layout,
background_color='#d8dbd4',
button_color=['#000000', '#2e9688'],
margins=[0, 0])
def make_save_all_groups_data_screen(self, calling_window):
layout = [
[
sg.Column([[
sg.Text('SAVING ALL GROUPS DATA: *** PLEASE WAIT ***',
key='-MESSAGE_1-',
font=['Comic', 20, 'bold'],
text_color='#ffffff',
background_color='#2e9688')
]],
justification='center')
],
[
sg.Column([[
sg.Text(
'\tPLEASE WAIT for proccess to finish.\nYou will recieve a success response message when completed.',
key='-MESSAGE_2-',
background_color='#d8dbd4',
font=['Comic', 12, 'bold'],
text_color='#000000')
]],
background_color='#d8dbd4',
justification='center')
]
]
calling_window.close()
return sg.Window('SAVING ALL GROUPS DATA',
layout=layout,
background_color='#d8dbd4',
button_color=['#000000', '#2e9688'],
margins=[0, 0])
def make_save_single_group_data_screen(self, calling_window):
layout = [[
sg.Column([[
sg.Text('SAVE SINGLE GROUPS DATA',
key='-MESSAGE_1-',
font=['Comic', 20, 'bold'],
text_color='#ffffff',
background_color='#2e9688')
]],
justification='center')
],
[
sg.Text('Please provide WhatsApp Web GROUP NAME: ',
key='-MESSAGE_2-',
size=(38, 1),
background_color='#d8dbd4',
font=['Comic', 12, 'bold'],
text_color='#000000'),
sg.InputText()
],
[
sg.Column([[
sg.Button('Return', key='-QUITER-'),
sg.Button('Submit', key='-SUBMIT-')
]],
background_color='#d8dbd4',
justification='center')
]]
calling_window.close()
return sg.Window('SAVING ALL GROUPS DATA',
layout=layout,
background_color='#d8dbd4',
button_color=['#000000', '#2e9688'],
margins=[0, 0])
def make_save_group_data_fail_screen(self, calling_window):
layout = [
[
sg.Column([[
sg.Text('SAVE GROUP(S) DATA FAIL NOTIFICATION',
key='-MESSAGE_1-',
font=['Comic', 20, 'bold'],
text_color='#ffffff',
background_color='#2e9688')
]],
justification='center')
],
[
sg.Column([[
sg.Text(
'Please ensure you have a secure internet connection\nand your WhatsApp Web account is signed in.',
key='-MESSAGE_2-',
background_color='#d8dbd4',
font=['Comic', 12, 'bold'],
text_color='#000000')
]],
background_color='#d8dbd4',
justification='center')
],
[
sg.Column([[sg.Button('Return', key='-QUITER-')]],
background_color='#d8dbd4',
justification='center')
]
]
calling_window.close()
return sg.Window('SAVE GROUP(S) DATA FAIL NOTIFICATION',
layout=layout,
background_color='#d8dbd4',
button_color=['#000000', '#2e9688'],
margins=[0, 0])
def make_save_group_data_success_screen(self, calling_window):
layout = [
[
sg.Column([[
sg.Text('SAVE GROUP(S) DATA SUCCESS NOTIFICATION',
key='-MESSAGE_1-',
font=['Comic', 20, 'bold'],
text_color='#ffffff',
background_color='#2e9688')
]],
justification='center')
],
[
sg.Column([[
sg.Text('All your group data has been saved SUCCESSFULLY!',
key='-MESSAGE_2-',
background_color='#d8dbd4',
font=['Comic', 12, 'bold'],
text_color='#000000')
]],
background_color='#d8dbd4',
justification='center')
],
[
sg.Column([[sg.Button('Return', key='-QUITER-')]],
background_color='#d8dbd4',
justification='center')
]
]
calling_window.close()
return sg.Window('SAVE GROUP(S) DATA SUCCESS NOTIFICATION',
layout=layout,
background_color='#d8dbd4',
button_color=['#000000', '#2e9688'],
margins=[0, 0])
def make_generate_csv_screen(self, calling_window):
layout = [
[
sg.Column([[
sg.Text('GENERATE CSV SUCCESS NOTIFICATION',
key='-MESSAGE_1-',
font=['Comic', 20, 'bold'],
text_color='#ffffff',
background_color='#2e9688')
]],
justification='center')
],
[
sg.Column([[
sg.Text(
'A csv containing all message data from your groups has been generated.\nThe file is located in the "WhatsApp_scrap" main directory.',
key='-MESSAGE_2-',
background_color='#d8dbd4',
font=['Comic', 12, 'bold'],
text_color='#000000')
]],
background_color='#d8dbd4',
justification='center')
],
[
sg.Column([[sg.Button('Return', key='-QUITER-')]],
background_color='#d8dbd4',
justification='center')
]
]
calling_window.close()
return sg.Window('GENERATE CSV SUCCESS NOTIFICATION',
layout=layout,
background_color='#d8dbd4',
button_color=['#000000', '#2e9688'],
margins=[0, 0])
# *************** PRIVATE FUNCTIONS ***************************
def __get_failed_group_list(self, file_name):
failed_group_text = ""
with open("scrape/group_files/groups_failed/" + file_name,
'r') as failed_group_file:
while True:
failed_link = failed_group_file.readline()
if not failed_link:
break
else:
failed_group_text += (failed_link + "\n")
failed_group_file.close()
return failed_group_text
def test_success_join_group_cli(monkeypatch):
group_tools = GroupTools()
monkeypatch.setattr('builtins.input', lambda _: success_test_page)
assert group_tools.join_group_cli() is True
def __init__(self):
self.group_tools = GroupTools()
def main():
gui = WhatsScrapeGUI()
group_tools = GroupTools()
scrape = MultiProcessHtml()
system = SystemTools()
while True:
window = gui.make_home_menu()
event, values = window.read()
if event == '-QUITER-':
window.close()
exit(0)
elif event == '-PRINT_GROUPS-':
window = gui.make_print_groups_screen(window)
event, values = window.read()
if event == '-QUITER-':
window.close()
window = gui.make_home_menu()
elif event == '-JOIN_GROUP-':
window = gui.make_join_group_screen(window)
event, values = window.read()
if event == '-QUITER-':
window.close()
window = gui.make_home_menu()
elif event == '-SUBMIT-':
if group_tools.join_group_gui(values[0]) is False:
window = gui.make_group_failure_screen(window, values[0])
event, values = window.read()
if event == '-QUITER-':
window.close()
window = gui.make_home_menu()
else:
window = gui.make_group_success_screen(window, values[0])
event, values = window.read()
if event == '-QUITER-':
window.close()
window = gui.make_home_menu()
elif event == '-JOIN_MULTI_GROUP-':
window = gui.make_join_multiple_groups_screen(window)
event, values = window.read()
if event == '-QUITER-':
window.close()
window = gui.make_home_menu()
elif event == '-SUBMIT-':
if group_tools.join_multiple_groups_gui(values[0]) is False:
window = gui.make_multiple_group_failure_screen(window, values[0])
event, values = window.read()
if event == '-QUITER-':
window.close()
window = gui.make_home_menu()
else:
window = gui.make_multiple_group_success_screen(window, values[0])
event, values = window.read()
if event == '-QUITER-':
window.close()
window = gui.make_home_menu()
elif event == '-ALL_GROUPS-':
window = gui.make_save_all_groups_data_screen(window)
window.finalize()
if group_tools.save_all_groups_data_gui() is False:
window = gui.make_save_group_data_fail_screen(window)
event, values = window.read()
if event == '-QUITER-':
window.close()
window = gui.make_home_menu()
else:
window = gui.make_save_group_data_success_screen(window)
event, values = window.read()
if event == '-QUITER-':
window.close()
window = gui.make_home_menu()
elif event == '-SINGLE_GROUP-':
window = gui.make_save_single_group_data_screen(window)
event, values = window.read()
if event == '-QUITER-':
window.close()
window = gui.make_home_menu()
elif event == '-SUBMIT-':
if group_tools.save_single_groups_data_gui(values[0]) is False:
window = gui.make_group_failure_screen(window, values[0])
event, values = window.read()
if event == '-QUITER-':
window.close()
window = gui.make_home_menu()
else:
window = gui.make_save_group_data_success_screen(window)
event, values = window.read()
if event == '-QUITER-':
window.close()
window = gui.make_home_menu()
elif event == '-GENERATE_CSV-':
df = pd.DataFrame()
df = scrape.get_message_frame_all_groups(system.get_processed_pck_list())
scrape.generate_message_summary_csv(df)
window = gui.make_generate_csv_screen(window)
event, values = window.read()
if event == '-QUITER-':
window.close()
window = gui.make_home_menu()
else:
quit()