def handle_run_scraper(settings):
if not utils.validate_settings(settings, False):
click.echo(
utils.highlight(
"Run aborted. There are missing required settings."))
time.sleep(3)
return
scraper = App(settings, show_progress=True)
cleaner = utils.run_cleaner(settings)
errors = []
# run yields each source result, so we can clear the screen and start anew
_, source_name = cleaner()
for result in scraper.run():
if not result:
err = utils.highlight(
f"There was en error processing {source_name}", "red")
utils.pre_menu(settings, err)
errors.append(err)
time.sleep(2)
continue
_, source_name = cleaner()
# TODO: Find a way to show this message
out_folder = utils.highlight(os.path.abspath(settings.output_path))
end_desc = "You can find you data in : {}\n{}".format(
out_folder, "\n".join(errors))
utils.pre_menu(settings, "All Done!", end_desc)
if click.confirm(utils.fmt.format("Do you want to {exit:yellow}?"),
default=True):
return handle_exit(settings, True)
return False
def handle_output_type(settings):
utils.pre_menu(settings, "Change output Type", out_type_descr())
mi = utils.get_menuitems_for_handlers(manager.get_all_writers())
output_type_items = utils.get_menuitems_text(mi)
output_menu = TerminalMenu(
menu_entries=output_type_items,
cursor_index=utils.get_choice_index(mi, settings.output_type),
preview_command=utils.get_description_by_text(mi))
choice = output_menu.show()
try:
settings.output_type = mi[choice].v
except Exception:
pass
return False
def handle_csv_dialect(settings):
utils.pre_menu(settings, "Change CSV format", csv_dialect_desc())
# Get all the registered dialects, either on the csv module or in our
# manager, removing duplicates if necessary.
# As the manager is working now (21/5/1) the manger's list should be
# already included in the list from csv module, but this ensure consistency
csv_format_items = tuple(
sorted(
dict.fromkeys(
(*manager.get_dialects_list(), *csv.list_dialects()))))
csv_menu = TerminalMenu(menu_entries=csv_format_items,
cursor_index=csv_format_items.index(
settings.csv_out_dialect))
choice = csv_menu.show()
try:
settings.csv_out_dialect = csv_format_items[choice]
except Exception:
pass
return False
def handle_output_path(settings):
utils.pre_menu(settings, "Set output Path", out_path_descr())
path = click.prompt("Type your base path", default=settings.output_path)
settings.output_path = path
return False
def handle_output_filename(settings):
utils.pre_menu(settings, colored("Feature coming soon", "red"),
out_frmt_descr())
time.sleep(1)
return False
def handle_exit(settings, print_msg=True, save_on_exit=True):
save_on_exit and settings.to_file()
print_msg and utils.pre_menu(settings, "Goodbye!")
return True