def write(args, reddit, u_master):
"""
Get, sort, then write scraped Redditor information to CSV or JSON.
Calls previously defined public methods:
GetInteractions().get()
Write._determine_export()
Write._print_confirm()
Calls a public method from an external module:
NameFile().u_fname(limit, user)
Parameters
----------
args: Namespace
Namespace object containing all arguments that were defined in the CLI
reddit: Reddit object
Reddit instance created by PRAW API credentials
u_master: dict
Dictionary containing all scrape settings
Returns
-------
None
"""
for user, limit in u_master.items():
data = GetInteractions().get(limit, reddit, user)
f_name = NameFile().u_fname(limit, user)
Write._determine_export(args, data, f_name)
Write._print_confirm(args, user)
def test_r_fname_ignores_end_string_with_subreddit_args(self):
args = MakeArgs.parser_for_testing_export().parse_args(["--subreddit"])
cat_i = "S"
each_sub = ["s", "test", "all"]
sub = "askreddit"
assert NameFile().r_fname(args, cat_i, each_sub, sub) == "askreddit-search-'test'"
def test_r_fname_returns_non_plural_string_with_subreddit_args(self):
args = MakeArgs.parser_for_testing_export().parse_args(["--subreddit"])
cat_i = "H"
each_sub = ["h", 1, None]
sub = "askreddit"
assert NameFile().r_fname(args, cat_i, each_sub, sub) == "askreddit-hot-1-result"
def write(reddit, u_master):
"""
Get, sort, then write scraped Redditor information to CSV or JSON.
Calls a previously defined public method:
GetInteractions.get()
Calls a public method from an external module:
NameFile().u_fname()
Parameters
----------
reddit: Reddit object
Reddit instance created by PRAW API credentials
u_master: dict
Dictionary containing all scrape settings
Returns
-------
None
"""
for redditor, limit in u_master.items():
data = GetInteractions.get(limit, reddit, redditor)
f_name = NameFile().u_fname(limit, redditor)
Export.export(data, f_name, "json", "redditors")
print()
Halo().succeed(Style.BRIGHT + Fore.GREEN +
"JSON file for u/%s created." % redditor)
print()
def _write(args, cat_i, data, each_sub, sub):
"""
Write posts.
Calls previously defined private methods:
GetSortWrite._determine_export()
GetSortWrite._print_confirm()
Calls a method from an external module:
NameFile().r_fname()
Parameters
----------
args: Namespace
Namespace object containing all arguments that were defined in the CLI
cat_i: str
String denoting n_results returned or keywords searched for
data: dict
Dictionary containing scraped Subreddit submission data
each_sub: list
List of Subreddit scraping settings
sub: str
String denoting the Subreddit name
Returns
-------
None
"""
f_name = NameFile().r_fname(args, cat_i, each_sub, sub)
GetSortWrite._determine_export(args, data, f_name)
GetSortWrite._print_confirm(args, sub)
def test_get_raw_n_returns_search_filename_format_with_subreddit_args(self):
args = MakeArgs.parser_for_testing_export().parse_args(["--subreddit"])
cat_i = "S"
end = "result"
each_sub = ["s", "test", "all"]
sub = "askreddit"
assert NameFile()._get_raw_n(args, cat_i, end, each_sub, sub) == "askreddit-search-'test'"
def test_get_raw_n_returns_returns_filter_string_with_rules_included(self):
args = MakeArgs.parser_for_testing_export().parse_args(["--subreddit", "--rules"])
cat_i = "H"
end = "result"
each_sub = ["h", "1", "year"]
sub = "askreddit"
assert NameFile()._get_raw_n(args, cat_i, end, each_sub, sub) == "askreddit-hot-1-result-past-year-rules"
def test_get_raw_n_returns_category_filename_format_with_subreddit_args(self):
args = MakeArgs.parser_for_testing_export().parse_args(["--subreddit"])
cat_i = "H"
end = "result"
each_sub = ["h", "1", None]
sub = "askreddit"
assert NameFile()._get_raw_n(args, cat_i, end, each_sub, sub) == "askreddit-hot-1-result"
def write(args, c_master, reddit):
"""
Get, sort, then write scraped comments to CSV or JSON.
Calls previously defined public and private methods:
GetSort().get_sort()
Write._determine_export()
Write._make_json_skeleton()
Write._print_confirm()
Calls a public method from an external module:
NameFile().c_fname()
Parameters
----------
args: Namespace
Namespace object containing all arguments that were defined in the CLI
c_master: dict
Dictionary containing all scrape settings
reddit: Reddit object
Reddit instance created by PRAW API credentials
Returns
-------
None
"""
for url, limit in c_master.items():
submission = reddit.submission(url=url)
data = Write._make_json_skeleton(args, limit, submission, url)
data["data"]["comments"] = GetSort(args, submission,
url).get_sort(args, limit)
f_name = NameFile().c_fname(args, limit, submission.title)
Write._determine_export(args, data, f_name)
print()
Halo(color="green",
text=Style.BRIGHT + Fore.GREEN +
f"JSON file for '{submission.title}' comments created."
).succeed()
print()
def _write(args, cat_i, data, each_sub, sub):
"""
Write submissions to file.
Calls methods from external modules:
NameFile().r_fname()
Export.export()
Parameters
----------
args: Namespace
Namespace object containing all arguments that were defined in the CLI
cat_i: str
String denoting n_results returned or keywords searched for
data: dict
Dictionary containing scraped Subreddit submission data
each_sub: list
List of Subreddit scraping settings
sub: str
String denoting the Subreddit name
Returns
-------
None
"""
f_name = NameFile().r_fname(args, cat_i, each_sub, sub)
export_option = "json" \
if not args.csv \
else "csv"
Export.export(data, f_name, export_option, "subreddits")
print()
Halo(color="green",
text=Style.BRIGHT + Fore.GREEN +
f"{export_option.upper()} file for r/{sub} created.").succeed()
print()
def write(args, c_master, reddit):
"""
Get, sort, then write scraped comments to CSV or JSON.
Calls previously defined public and private methods:
GetSort().get_sort()
Write._determine_export()
Write._make_json_skeleton()
Write._print_confirm()
Calls a public method from an external module:
NameFile().c_fname()
Parameters
----------
args: Namespace
Namespace object containing all arguments that were defined in the CLI
c_master: dict
Dictionary containing all scrape settings
reddit: Reddit object
Reddit instance created by PRAW API credentials
Returns
-------
None
"""
for post, limit in c_master.items():
title = reddit.submission(url=post).title
data = Write._make_json_skeleton(limit, post, title)
data["data"].append(GetSort(post, reddit).get_sort(limit))
f_name = NameFile().c_fname(limit, title)
Write._determine_export(args, data, f_name)
Write._print_confirm(args, title)
def test_u_fname_returns_non_plural_string(self):
limit = 1
string = "test"
assert NameFile().u_fname(limit, string) == "test-1-result"
def test_check_len_string_is_shorter_than_50_char(self):
test_string = "poebtcjocweeooijhyltchjarvu"
cut_string = NameFile()._check_len(test_string)
assert test_string == cut_string
def test_c_fname_returns_non_plural_string_with_structured_format(self):
args = MakeArgs.parser_for_testing_export().parse_args()
limit = 1
string = "test"
assert NameFile().c_fname(args, limit, string) == "test-1-result"
def test_c_fname_returns_plural_string_with_raw_format(self):
args = MakeArgs.parser_for_testing_export().parse_args(["--raw"])
limit = 2
string = "test"
assert NameFile().c_fname(args, limit, string) == "test-2-results-raw"
def test_r_get_category_subreddit_arg_returns_zero(self):
assert NameFile()._r_get_category("S") == 0
def test_r_get_category_subreddit_arg_returns_one(self):
assert NameFile()._r_get_category("C") == 1
def test_r_category_second_switch(self):
for index, category in enumerate(short_cat[:5]):
assert NameFile()._r_category(category, 1) == categories[index]
def test_r_category_first_switch(self):
assert NameFile()._r_category("H", 0) == categories[5]
def test_fix(self):
name = "/t\\e?s%t*i:n|g<c@h!a#r$ac^t&e*(r)s>}{~+`="
fixed = "_t_e_s_t_i_n_g_c_h_a_r_ac_t_e__r_s_______"
assert fixed == NameFile()._fix(name)
def test_check_len_string_is_longer_than_50_char(self):
test_string = "fegskkxjsacxpusflfccqynqchbqdywvvjsmqmhaxyvhtipropqkzqstsxcx"
cut_string = NameFile()._check_len(test_string)
assert cut_string == "fegskkxjsacxpusflfccqynqchbqdywvvjsmqmhaxyvhtipr--"
def test_c_fname_returns_all_comments_with_structured_format(self):
args = MakeArgs.parser_for_testing_export().parse_args()
limit = 0
string = "test"
assert NameFile().c_fname(args, limit, string) == "test-all"
