def test_check_subreddit_time_filter_applied_to_invalid_category(self):
parser = MakeArgs.make_scraper_args()
args = parser.parse_args(["--subreddit", ["test_subreddit", "h", "10", "year"]])
try:
Cli.CheckPRAWCli().check_subreddit(args)
assert False
except SystemExit:
assert True
def test_check_n_results_valid_n_results(self):
n_results = "10"
sub = ["test", "H", "10"]
try:
Cli.CheckPRAWCli()._check_n_results(n_results, sub)
assert True
except SystemExit:
assert False
def test_check_subreddit_with_time_filter_with_invalid_args(self):
parser = MakeArgs.make_scraper_args()
args = parser.parse_args(["--subreddit", ["test_subreddit", "s", "test", "asdf"]])
try:
Cli.CheckPRAWCli().check_subreddit(args)
assert False
except SystemExit:
assert True
def test_check_praw_cli_init_method_time_filters_instance_variable(self):
assert Cli.CheckPRAWCli()._time_filters == [
"all",
"day",
"hour",
"month",
"week",
"year"
]
def test_check_args_with_invalid_redditor_args(self):
parser = MakeArgs.make_scraper_args()
args = parser.parse_args(["--redditor", ["10", "test_redditor"]])
try:
Cli.CheckCli().check_args(args, parser)
assert False
except SystemExit:
assert True
def test_get_settings_with_redditor_args(self):
parser = MakeArgs.make_scraper_args()
args = parser.parse_args(["--redditor", ["test_redditor", "10"]])
master = {"test_redditor": None}
s_type = "redditor"
invalids = []
Cli.GetPRAWScrapeSettings().get_settings(args, invalids, master, s_type)
assert master == {"test_redditor": "10"}
def test_get_settings_with_subreddit_args(self):
parser = MakeArgs.make_scraper_args()
args = parser.parse_args(
["--subreddit", ["test_subreddit", "h", "10"]])
master = {"test_subreddit": []}
s_type = Global.s_t[0]
Cli.GetScrapeSettings().get_settings(args, master, s_type)
assert master == {"test_subreddit": [["h", "10", None]]}
def test_check_redditor_with_invalid_args(self):
parser = MakeArgs.make_scraper_args()
args = parser.parse_args(["--redditor", ["10", "test_redditor"]])
try:
Cli.CheckCli()._check_redditor(args)
assert False
except ValueError:
assert True
def test_add_flags_method_comments_flag(self):
test_subreddit_args = [["test_url", "10"]]
parser = MakeArgs.parser_for_testing_cli()
Cli.Parser()._add_flags(parser)
args = parser.parse_args("--comments test_url 10".split())
assert args.comments == test_subreddit_args
def test_create_list_from_subreddit_args(self):
parser = MakeArgs.make_scraper_args()
args = parser.parse_args(
["--subreddit", ["test_subreddit", "h", "10"]])
l_type = "subreddit"
assert Cli.GetScrapeSettings().create_list(args, l_type) == \
["test_subreddit"]
def test_add_flags_method_subreddit_flag(self):
test_subreddit_args = [["test_subreddit", "h", "10"]]
parser = MakeArgs.parser_for_testing_cli()
Cli.Parser()._add_flags(parser)
args = parser.parse_args("--subreddit test_subreddit h 10".split())
assert args.subreddit == test_subreddit_args
def test_check_wordcloud_len_is_greater_than_two(self):
parser = MakeArgs.make_scraper_args()
args = parser.parse_args(["--wordcloud", ["test_file", "png", "asdf"]])
try:
Cli.CheckAnalyticCli().check_wordcloud(args)
assert False
except SystemExit:
assert True
def test_get_settings_with_comments_args(self):
parser = MakeArgs.make_scraper_args()
args = parser.parse_args(["--comments", ["test_url", "2"]])
master = {"test_url": None}
s_type = "comments"
invalids = []
Cli.GetPRAWScrapeSettings().get_settings(args, invalids, master, s_type)
assert master == {"test_url": "2"}
def test_check_args_with_invalid_comments_args(self):
parser = MakeArgs.make_scraper_args()
args = parser.parse_args(["--comments", ["2", "test_url"]])
try:
Cli.CheckCli().check_args(args, parser)
assert False
except SystemExit:
assert True
def test_two_arg_settings_one_arg_redditor(self):
parser = MakeArgs.make_scraper_args()
args = parser.parse_args(["--redditor", ["test_redditor", "10"]])
master = {"test_redditor": None}
invalids = []
Cli.GetPRAWScrapeSettings()._two_arg_settings(args.redditor, invalids,
master)
assert master == {"test_redditor": "10"}
def test_two_arg_settings_one_arg_comments(self):
parser = MakeArgs.make_scraper_args()
args = parser.parse_args(["--comments", ["test_url", "2"]])
master = {"test_url": None}
invalids = []
Cli.GetPRAWScrapeSettings()._two_arg_settings(args.comments, invalids,
master)
assert master == {"test_url": "2"}
def test_add_analytics_method_wordcloud_flag(self):
test_subreddit_args = [["test_file"]]
parser = MakeArgs.parser_for_testing_cli()
Cli.Parser()._add_analytics(parser)
args = parser.parse_args("--wordcloud test_file".split())
assert args.wordcloud == test_subreddit_args
def test_add_analytics_method_frequencies_flag(self):
test_subreddit_args = [["test_file"]]
parser = MakeArgs.parser_for_testing_cli()
Cli.Parser()._add_analytics(parser)
args = parser.parse_args("--frequencies test_file".split())
assert args.frequencies == test_subreddit_args
def test_check_args_with_invalid_subreddit_args(self):
parser = MakeArgs.make_scraper_args()
args = parser.parse_args(["--subreddit", ["test_subreddit", "w", "asdf"]])
try:
Cli.CheckCli().check_args(args)
assert False
except SystemExit:
assert True
def test_parser_init_method_epilog_instance_variable(self):
epilog = r"""
Subreddit categories:
H,h selecting Hot category
N,n selecting New category
C,c selecting Controversial category (time filter available)
T,t selecting Top category (time filter available)
R,r selecting Rising category
S,s selecting Search category (time filter available)
Subreddit time filters:
all (default)
day
hour
month
week
year
EXAMPLES
Get the first 10 posts in r/askreddit in the Hot category and export to JSON:
$ ./Urs.py -r askreddit h 10 --json
Search for "United States of America" in r/worldnews and export to CSV:
$ ./Urs.py -r worldnews s "United States of America" --csv
You can apply a time filter when scraping Subreddit categories Controversial, Top, or Search:
(Scraping Search results from r/learnprogramming from the past month)
$ ./Urs.py -r learnprogramming s "python developer" month --json
Scraping 15 results from u/spez's Reddit account:
$ ./Urs.py -u spez 15 --json
Scraping 25 comments from this r/TIFU post:
(Returns a structured JSON file that includes down to third-level replies)
$ ./Urs.py -c https://www.reddit.com/r/tifu/comments/a99fw9/tifu_by_buying_everyone_an_ancestrydna_kit_and/ 25 --json
Scraping all comments from the same r/TIFU post:
(Returns an unstructured JSON file of all comments in level order, ie. top-level first, followed by second-level, then third-level, etc.)
$ ./Urs.py -c https://www.reddit.com/r/tifu/comments/a99fw9/tifu_by_buying_everyone_an_ancestrydna_kit_and/ 0 --json
You can scrape multiple items at once:
$ ./Urs.py -r askreddit h 15 -u spez 25 -c https://www.reddit.com/r/tifu/comments/a99fw9/tifu_by_buying_everyone_an_ancestrydna_kit_and/ 50 --json
You can also still use URS 1.0 (SUBREDDIT SCRAPING ONLY), but you cannot include this flag with any items besides export options:
$ ./Urs.py -b --csv
"""
assert Cli.Parser()._epilog == epilog
def test_add_flags_method_redditor_flag(self):
test_subreddit_args = [["test_redditor", "10"]]
parser = MakeArgs.parser_for_testing_cli()
Cli.Parser()._add_flags(parser)
args = parser.parse_args("--redditor test_redditor 10".split())
assert args.redditor == test_subreddit_args
def test_parse_args_method_redditor_and_csv_flags(self):
sys.argv = [sys.argv[0]]
input_args = ["--redditor", "test_redditor", "10", "--csv"]
for arg in input_args:
sys.argv.append(arg)
args, _ = Cli.Parser().parse_args()
assert args.redditor == [["test_redditor", "10"]]
assert args.csv == True
def test_parse_args_method_examples_flag_was_included(self):
sys.argv = [sys.argv[0]]
sys.argv.append("-e")
try:
_, _ = Cli.Parser().parse_args()
except SystemExit:
assert True
sys.argv = [sys.argv[0]]
def test_subreddit_settings_one_subreddit(self):
parser = MakeArgs.make_scraper_args()
args = parser.parse_args(
["--subreddit", ["test_subreddit", "h", "10"]])
master = {"test_subreddit": []}
invalids = []
Cli.GetPRAWScrapeSettings()._subreddit_settings(
args.subreddit, invalids, master)
assert master == {"test_subreddit": [["h", "10", None]]}
def test_parse_args_method_subreddit_and_json_flags(self):
input_args = ["--subreddit", "test_subreddit", "h", "10", "--json"]
for arg in input_args:
sys.argv.append(arg)
sys.argv = sys.argv[1:]
args, _ = Cli.Parser().parse_args()
assert args.subreddit == [["test_subreddit", "h", "10"]]
assert args.json == True
def test_check_analytic_cli_init_method_export_options_instance_variable(self):
assert Cli.CheckAnalyticCli()._export_options == [
"eps",
"jpeg",
"jpg",
"pdf",
"png",
"ps",
"rgba",
"tif",
"tiff"
]
def test_parser_init_method_usage_instance_variable(self):
usage = r"""$ Urs.py
[-h]
[-e]
[--check]
[-r <subreddit> <(h|n|c|t|r|s)> <n_results_or_keywords> [<optional_time_filter>]]
[--rules]
[-y]
[-u <redditor> <n_results>]
[-c <submission_url> <n_results>]
[--raw]
[-b]
[-f <file_path>]
[-wc <file_path> [<optional_export_format>]]
[--nosave]
[--csv]
"""
assert Cli.Parser()._usage == usage