def print_subreddits(parser, reddit, search_for): """ Print valid and invalid Subreddits. Calls previously defined private method: PrintSubs._find_subs() Parameters ---------- parser: ArgumentParser argparse ArgumentParser object reddit: Reddit object Reddit instance created by PRAW API credentials search_for: str String denoting Subreddits to scrape for Returns ------- subs: list List of valid Subreddits not_subs: list List of invalid Subreddits """ check_subs_spinner = Halo(color="white", text="Validating Subreddit(s).") print() check_subs_spinner.start() subs, not_subs = PrintSubs._find_subs(parser, reddit, search_for) check_subs_spinner.succeed("Finished Subreddit validation.") if subs: print(Fore.GREEN + Style.BRIGHT + "\nThe following Subreddits were found and will be scraped:") print(Fore.GREEN + Style.BRIGHT + "-" * 56) print(*subs, sep="\n") if not_subs: print( Fore.YELLOW + Style.BRIGHT + "\nThe following Subreddits were not found and will be skipped:" ) print(Fore.YELLOW + Style.BRIGHT + "-" * 60) print(*not_subs, sep="\n") logging.warning("Failed to validate the following Subreddits:") logging.warning("%s" % not_subs) logging.warning("Skipping.") logging.info("") if not subs: logging.critical("ALL SUBREDDITS FAILED VALIDATION.") Errors.n_title("Subreddits") logging.critical("NO SUBREDDITS LEFT TO SCRAPE.") logging.critical("ABORTING URS.\n") quit() return subs
def wrapper(*args): try: function(*args) except ValueError: Errors.e_title(f"INVALID {error}.") logging.critical(f"RECEIVED INVALID {error}.") logging.critical("ABORTING URS.\n") quit()
def wrapper(*args): try: return function(*args) except ValueError: Errors.n_title(reddit_object) logging.critical("NO %s LEFT TO SCRAPE." % reddit_object.upper()) logging.critical("ABORTING URS.\n") quit()
def wrapper(*args): try: function(*args) except FileNotFoundError: Errors.i_title("Invalid `scrapes` directory structure.") logging.critical("AN ERROR HAS OCCURED WHILE PROCESSING SCRAPE DATA.") logging.critical("Invalid `scrapes` directory structure.\n") logging.critical("ABORTING URS.\n") quit()
def wrapper(*args): try: function(*args) logging.info(LogExport._get_export_switch(args[0])) logging.info("") except Exception as e: Errors.ex_title(e) logging.critical("AN ERROR HAS OCCURRED WHILE EXPORTING SCRAPED DATA.") logging.critical("%s" % e) logging.critical("ABORTING URS.\n") quit()
def wrapper(parser, reddit): try: function(parser, reddit) logging.info("Successfully logged in as u/%s." % reddit.user.me()) logging.info("") except PrawcoreException as error: Errors.p_title(error) logging.critical("LOGIN FAILED.") logging.critical("PRAWCORE EXCEPTION: %s." % error) logging.critical("ABORTING URS.\n") parser.exit()
def wrapper(*args): try: return function(*args) except ValueError: Errors.i_title("Scrape data is not located within the `scrapes` directory.") logging.critical("AN ERROR HAS OCCURRED WHILE PROCESSING SCRAPE DATA.") logging.critical("Scrape data is not located within the `scrapes` directory.") logging.critical("ABORTING URS.\n") quit() except TypeError: Errors.i_title("Invalid file format. Try again with a valid JSON file.") logging.critical("AN ERROR HAS OCCURRED WHILE PROCESSING SCRAPE DATA.") logging.critical("Invalid file format.") logging.critical("ABORTING URS.\n") quit()
def wrapper(reddit): user_limits = function(reddit) logging.info("RATE LIMIT DISPLAYED.") logging.info("Remaining requests: %s" % int(user_limits["remaining"])) logging.info("Used requests: %s" % user_limits["used"]) logging.info("") if int(user_limits["remaining"]) == 0: Errors.l_title(convert_time(user_limits["reset_timestamp"])) logging.critical("RATE LIMIT REACHED. RATE LIMIT WILL RESET AT %s." % convert_time(user_limits["reset_timestamp"])) logging.critical("ABORTING URS.\n") quit() return user_limits
def validate_user(parser, reddit): """ Check if PRAW credentials are valid, then print rate limit PrettyTable. Parameters ---------- parser: ArgumentParser argparse ArgumentParser object reddit: Reddit object Reddit instance created by PRAW API credentials Returns ------- None """ login_spinner = Halo(color="white", text="Logging in.") login_spinner.start() try: redditor = reddit.user.me() login_spinner.succeed(Style.BRIGHT + Fore.GREEN + "Successfully logged in as u/%s." % redditor) print() Validation.print_rate_limit(reddit) logging.info("Successfully logged in as u/%s." % redditor) logging.info("") except PrawcoreException as error: login_spinner.fail(Style.BRIGHT + Fore.RED + "Failed to log in.") Errors.p_title(error) logging.critical("LOGIN FAILED.") logging.critical("PRAWCORE EXCEPTION: %s." % error) logging.critical("ABORTING URS.\n") parser.exit()
def validate(object_list, reddit, scraper_type): """ Check if Subreddit(s), Redditor(s), or submission(s) exist and catch PRAW exceptions. Log invalid Reddit objects to `urs.log` if applicable. Calls previously defined public method: Validation.check_existence() Parameters ---------- object_list: list List of Reddit objects to check reddit: Reddit object Reddit instance created by PRAW API credentials scraper_type: str String denoting the scraper type Returns ------- invalid: list List of invalid Reddit objects valid: list List of valid Reddit objects """ object_type = "submission" \ if scraper_type == "comments" \ else scraper_type.capitalize() check_status = Status( "Finished %s validation." % object_type, "Validating %s(s)" % object_type, "white" ) check_status.start() logging.info("Validating %s(s)..." % object_type) logging.info("") invalid, valid = Validation.check_existence(object_list, reddit, scraper_type) check_status.succeed() print() if invalid: warning_message = "The following %ss were not found and will be skipped:" % object_type print(Fore.YELLOW + Style.BRIGHT + warning_message) print(Fore.YELLOW + Style.BRIGHT + "-" * len(warning_message)) print(*invalid, sep = "\n") logging.warning("Failed to validate the following %ss:" % object_type) logging.warning("%s" % (invalid)) logging.warning("Skipping.") logging.info("") if not valid: logging.critical("ALL %sS FAILED VALIDATION." % object_type.upper()) Errors.n_title(object_type + "s") logging.critical("NO %sS LEFT TO SCRAPE." % object_type.upper()) logging.critical("ABORTING URS.\n") quit() return invalid, valid
def display_tree(search_date): """ Display the scrapes directory for a specific date. Calls previously defined private methods: DateTree._check_date_format() DateTree._create_directory_tree() DateTree._find_date_directory() Parameters ---------- search_date: str String denoting the date within the scrapes directory to search for Returns ------- None """ logging.info(f"Running tree command...") logging.info("") try: search_date = DateTree._check_date_format(search_date) find_dir_halo = Halo( color="white", text=f"Searching for {search_date} directory within `scrapes`." ) find_dir_halo.start() dir_exists = DateTree._find_date_directory(search_date) if dir_exists: find_dir_halo.succeed(text=f"URS was run on {search_date}.") date_dir = f"{Path(Path.cwd()).parents[0]}/scrapes/{search_date}" tree = Tree(f"[bold blue]scrapes/") dir_tree = tree.add(f"[bold blue]{search_date}") DateTree._create_directory_tree(date_dir, dir_tree) rich.print(tree) logging.info( f"Displayed directory tree for scrapes run on {search_date}." ) logging.info("") print() else: error_messsage = f"URS was not run on {search_date}." find_dir_halo.fail(Fore.RED + Style.BRIGHT + error_messsage) print() logging.critical(error_messsage) logging.critical("ABORTING URS.\n") quit() except TypeError: logging.critical("INVALID DATE FORMAT.") logging.critical("ABORTING URS.\n") Errors.e_title( "INVALID DATE FORMAT. ACCEPTED FORMATS: MM-DD-YYYY or MM/DD/YYYY." ) quit()