def __main__(): try: args = setup_parser().parse_args() conf_path = find_config() conf = get_config(args, conf_path) conf_r = conf["redditcurl"] conf_o = conf["oauth"] if conf_r.getboolean("silent"): prints = lambda x: None else: prints = print if conf_r.get("subreddits") == "": subreddits = [] else: subreddits = conf_r.get("subreddits").strip(',').casefold().split(',') prints("Downloading from {}".format(', '.join(subreddits))) r = praw.Reddit(user_agent="redditcurl") r.set_oauth_app_info(client_id=conf_o.get("clientid"), redirect_uri=conf_o.get("redirect"), client_secret="None") if not is_authenticated(conf_o): auth_url = r.get_authorize_url("state", OAUTH_SCOPES, True) print("Please visit {} to authorize access to your account history.".format(auth_url)) auth_code = input("Enter the code: ") access_information = r.get_access_information(auth_code) conf.read_dict({"oauth": {"refresh_token": access_information["refresh_token"], "access_token": access_information["access_token"]} }) with open(conf_path, "w") as conf_file: conf.write(conf_file) else: r.set_access_credentials(scope=OAUTH_SCOPES, access_token=conf_o.get("access_token"), refresh_token=conf_o.get("refresh_token")) prints("Refreshing access token.") r.refresh_access_information(conf_o.get("refresh_token")) # We don't save the new access_token here, since we refresh it every time the program is run prints("Getting data...") try: os.makedirs(conf_r.get("savedir")) except (FileExistsError): # If the save directory exists, we don't need to create it pass save_file = os.path.join(conf_r.get("savedir"), conf_r.get("savefile")) saved = manager.filter_new(r.user.get_saved(limit=None), save_file) prints("Starting to download, using {} processes.".format(conf_r.get("processes"))) downloaded = manager.download_submissions(saved, conf_r.get("savedir"), conf_r.getint("processes"), not conf_r.getboolean("notitles"), conf_r.getboolean("subfolders"), subreddits) prints("Processed {} urls.".format(len(downloaded))) remove = conf_r.getboolean("remove") success_count, fail_count, successful_downloads = count_success(downloaded, remove, prints, saved) prints("Updating saved files list.") manager.update_new(successful_downloads, save_file) prints("\nDownloading finished.") prints("Successful: {} \t Failed: {}".format(success_count, fail_count)) except (ConfigError) as err: print(err)
def test_filter_new(self): # First, create a set to test with manager.update_new(list(test_links.values())[:3], ".downloaded.gz") filtered_items = manager.filter_new(test_submissions, ".downloaded.gz") self.assertTrue(len(test_submissions) > 0) self.assertTrue(len(filtered_items) > 0) for filtered in filtered_items: self.assertTrue(filtered.url in original_urls) self.assertTrue(filtered.title in original_titles) self.assertTrue(filtered.subreddit.display_name in original_subreddits)
def test_write_existing(self): # Test with an existing file manager.update_new(list(test_links.values())[:3], ".downloaded.gz") manager.update_new(list(test_links.values())[3:], ".downloaded.gz") with gzip.open(".downloaded.gz", "rb") as file: self.assertEqual(list(test_links.values()), json.loads(file.read().decode("utf-8")))