Esempio n. 1
0
def __main__():
    try:
        args = setup_parser().parse_args()
        conf_path = find_config()
        conf = get_config(args, conf_path)
        conf_r = conf["redditcurl"]
        conf_o = conf["oauth"]
        if conf_r.getboolean("silent"):
            prints = lambda x: None
        else:
            prints = print
        if conf_r.get("subreddits") == "":
            subreddits = []
        else:
            subreddits = conf_r.get("subreddits").strip(',').casefold().split(',')
            prints("Downloading from {}".format(', '.join(subreddits)))
        r = praw.Reddit(user_agent="redditcurl")
        r.set_oauth_app_info(client_id=conf_o.get("clientid"),
                             redirect_uri=conf_o.get("redirect"),
                             client_secret="None")
        if not is_authenticated(conf_o):
            auth_url = r.get_authorize_url("state", OAUTH_SCOPES, True)
            print("Please visit {} to authorize access to your account history.".format(auth_url))
            auth_code = input("Enter the code: ")
            access_information = r.get_access_information(auth_code)
            conf.read_dict({"oauth":
                            {"refresh_token": access_information["refresh_token"],
                             "access_token": access_information["access_token"]}
            })
            with open(conf_path, "w") as conf_file:
                conf.write(conf_file)
        else:
            r.set_access_credentials(scope=OAUTH_SCOPES,
                                     access_token=conf_o.get("access_token"),
                                     refresh_token=conf_o.get("refresh_token"))
            prints("Refreshing access token.")
            r.refresh_access_information(conf_o.get("refresh_token"))
            # We don't save the new access_token here, since we refresh it every time the program is run
        prints("Getting data...")
        try:
            os.makedirs(conf_r.get("savedir"))
        except (FileExistsError):
            # If the save directory exists, we don't need to create it
            pass
        save_file = os.path.join(conf_r.get("savedir"), conf_r.get("savefile"))
        saved = manager.filter_new(r.user.get_saved(limit=None), save_file)
        prints("Starting to download, using {} processes.".format(conf_r.get("processes")))
        downloaded = manager.download_submissions(saved, conf_r.get("savedir"), conf_r.getint("processes"),
                                                  not conf_r.getboolean("notitles"),
                                                  conf_r.getboolean("subfolders"), subreddits)
        prints("Processed {} urls.".format(len(downloaded)))
        remove = conf_r.getboolean("remove")
        success_count, fail_count, successful_downloads = count_success(downloaded, remove, prints, saved)
        prints("Updating saved files list.")
        manager.update_new(successful_downloads, save_file)
        prints("\nDownloading finished.")
        prints("Successful: {} \t Failed: {}".format(success_count, fail_count))
    except (ConfigError) as err:
        print(err)
Esempio n. 2
0
 def test_filter_new(self):
     # First, create a set to test with
     manager.update_new(list(test_links.values())[:3], ".downloaded.gz")
     filtered_items = manager.filter_new(test_submissions, ".downloaded.gz")
     self.assertTrue(len(test_submissions) > 0)
     self.assertTrue(len(filtered_items) > 0)
     for filtered in filtered_items:
         self.assertTrue(filtered.url in original_urls)
         self.assertTrue(filtered.title in original_titles)
         self.assertTrue(filtered.subreddit.display_name in original_subreddits)
Esempio n. 3
0
 def test_write_existing(self):
     # Test with an existing file
     manager.update_new(list(test_links.values())[:3], ".downloaded.gz")
     manager.update_new(list(test_links.values())[3:], ".downloaded.gz")
     with gzip.open(".downloaded.gz", "rb") as file:
         self.assertEqual(list(test_links.values()), json.loads(file.read().decode("utf-8")))