def __main__():
try:
args = setup_parser().parse_args()
conf_path = find_config()
conf = get_config(args, conf_path)
conf_r = conf["redditcurl"]
conf_o = conf["oauth"]
if conf_r.getboolean("silent"):
prints = lambda x: None
else:
prints = print
if conf_r.get("subreddits") == "":
subreddits = []
else:
subreddits = conf_r.get("subreddits").strip(',').casefold().split(',')
prints("Downloading from {}".format(', '.join(subreddits)))
r = praw.Reddit(user_agent="redditcurl")
r.set_oauth_app_info(client_id=conf_o.get("clientid"),
redirect_uri=conf_o.get("redirect"),
client_secret="None")
if not is_authenticated(conf_o):
auth_url = r.get_authorize_url("state", OAUTH_SCOPES, True)
print("Please visit {} to authorize access to your account history.".format(auth_url))
auth_code = input("Enter the code: ")
access_information = r.get_access_information(auth_code)
conf.read_dict({"oauth":
{"refresh_token": access_information["refresh_token"],
"access_token": access_information["access_token"]}
})
with open(conf_path, "w") as conf_file:
conf.write(conf_file)
else:
r.set_access_credentials(scope=OAUTH_SCOPES,
access_token=conf_o.get("access_token"),
refresh_token=conf_o.get("refresh_token"))
prints("Refreshing access token.")
r.refresh_access_information(conf_o.get("refresh_token"))
# We don't save the new access_token here, since we refresh it every time the program is run
prints("Getting data...")
try:
os.makedirs(conf_r.get("savedir"))
except (FileExistsError):
# If the save directory exists, we don't need to create it
pass
save_file = os.path.join(conf_r.get("savedir"), conf_r.get("savefile"))
saved = manager.filter_new(r.user.get_saved(limit=None), save_file)
prints("Starting to download, using {} processes.".format(conf_r.get("processes")))
downloaded = manager.download_submissions(saved, conf_r.get("savedir"), conf_r.getint("processes"),
not conf_r.getboolean("notitles"),
conf_r.getboolean("subfolders"), subreddits)
prints("Processed {} urls.".format(len(downloaded)))
remove = conf_r.getboolean("remove")
success_count, fail_count, successful_downloads = count_success(downloaded, remove, prints, saved)
prints("Updating saved files list.")
manager.update_new(successful_downloads, save_file)
prints("\nDownloading finished.")
prints("Successful: {} \t Failed: {}".format(success_count, fail_count))
except (ConfigError) as err:
print(err)
def test_filter_new(self):
# First, create a set to test with
manager.update_new(list(test_links.values())[:3], ".downloaded.gz")
filtered_items = manager.filter_new(test_submissions, ".downloaded.gz")
self.assertTrue(len(test_submissions) > 0)
self.assertTrue(len(filtered_items) > 0)
for filtered in filtered_items:
self.assertTrue(filtered.url in original_urls)
self.assertTrue(filtered.title in original_titles)
self.assertTrue(filtered.subreddit.display_name in original_subreddits)
def test_write_existing(self):
# Test with an existing file
manager.update_new(list(test_links.values())[:3], ".downloaded.gz")
manager.update_new(list(test_links.values())[3:], ".downloaded.gz")
with gzip.open(".downloaded.gz", "rb") as file:
self.assertEqual(list(test_links.values()), json.loads(file.read().decode("utf-8")))