def _determine_export(args, data, f_name):
"""
Export to either CSV or JSON.
Calls a public method from an external module:
Export.export()
Parameters
----------
args: Namespace
Namespace object containing all arguments that were defined in the CLI
data: dict
Dictionary containing all scraped data
f_name: str
String denoting the filename
Returns
-------
None
"""
export_option = eo[1] \
if not args.csv \
else eo[0]
Export.export(data, f_name, export_option, "comments")
def test_export_write_csv(self):
data = {
"this": [1, 2],
"is": [3, 4],
"a": [5, 6],
"test": [7, 8]
}
f_name = "export_write_csv_test"
f_type = "csv"
scrape = "subreddits"
Export.export(data, f_name, f_type, scrape)
with open(f"../scrapes/{date}/subreddits/export_write_csv_test.csv", "r", newline="", encoding = "utf-8") as test_csv:
reader = csv.reader(test_csv)
test_dict = dict((header, []) for header in next(reader))
for row in reader:
try:
for row_index, key in enumerate(test_dict.keys()):
test_dict[key].append(int(row[row_index]))
except IndexError:
continue
assert test_dict == data
def test_export_write_json(self):
data = {
"test_1": {
"this": 1,
"is": 1,
"a": 1,
"test": 1
},
"test_2": {
"this": 2,
"is": 2,
"a": 2,
"test": 2
}
}
f_name = "export_write_json_test"
f_type = "json"
scrape = "subreddits"
Export.export(data, f_name, f_type, scrape)
with open(f"../scrapes/{date}/subreddits/export_write_json_test.json", "r", encoding = "utf-8") as test_json:
test_dict = json.load(test_json)
assert test_dict == data
def write(reddit, u_master):
"""
Get, sort, then write scraped Redditor information to CSV or JSON.
Calls a previously defined public method:
GetInteractions.get()
Calls a public method from an external module:
NameFile().u_fname()
Parameters
----------
reddit: Reddit object
Reddit instance created by PRAW API credentials
u_master: dict
Dictionary containing all scrape settings
Returns
-------
None
"""
for redditor, limit in u_master.items():
data = GetInteractions.get(limit, reddit, redditor)
f_name = NameFile().u_fname(limit, redditor)
Export.export(data, f_name, "json", "redditors")
print()
Halo().succeed(Style.BRIGHT + Fore.GREEN +
"JSON file for u/%s created." % redditor)
print()
def _determine_export(args, data, f_name):
"""
Export either structured or raw comments.
Calls a public method from an external module:
Export.export()
Parameters
----------
args: Namespace
Namespace object containing all arguments that were defined in the CLI
data: dict
Dictionary containing all scraped data
f_name: str
String denoting the filename
Returns
-------
None
"""
if args.raw:
export_status = "Exporting %s comments in raw format." % data["scrape_settings"]["n_results"]
Halo().info(export_status)
logging.info(export_status)
Export.export(data, f_name, "json", "comments")
else:
export_status = "Exporting %s comments in structured format." % data["scrape_settings"]["n_results"]
Halo().info(export_status)
logging.info(export_status)
Export.write_structured_comments(data, f_name)
def _write(args, cat_i, data, each_sub, sub):
"""
Write submissions to file.
Calls methods from external modules:
NameFile().r_fname()
Export.export()
Parameters
----------
args: Namespace
Namespace object containing all arguments that were defined in the CLI
cat_i: str
String denoting n_results returned or keywords searched for
data: dict
Dictionary containing scraped Subreddit submission data
each_sub: list
List of Subreddit scraping settings
sub: str
String denoting the Subreddit name
Returns
-------
None
"""
f_name = NameFile().r_fname(args, cat_i, each_sub, sub)
export_option = "json" \
if not args.csv \
else "csv"
Export.export(data, f_name, export_option, "subreddits")
print()
Halo(color="green",
text=Style.BRIGHT + Fore.GREEN +
f"{export_option.upper()} file for r/{sub} created.").succeed()
print()
def test_export_write_csv(self):
data = {
"this": [1, 2],
"is": [3, 4],
"a": [5, 6],
"test": [7, 8]
}
f_name = "export_write_csv_test"
f_type = "csv"
scrape = "subreddits"
Export.export(data, f_name, f_type, scrape)
with open("../scrapes/%s/subreddits/export_write_csv_test.csv" % date, "r") as test_csv:
reader = csv.reader(test_csv)
test_dict = dict((header, []) for header in next(reader))
for row in reader:
for row_index, key in enumerate(test_dict.keys()):
test_dict[key].append(int(row[row_index]))
assert test_dict == data
def _determine_export(args, data, f_name):
"""
Export to either CSV or JSON.
Parameters
----------
args: Namespace
Namespace object containing all arguments that were defined in the CLI
data: dict
Dictionary containing scraped Subreddit submission data
f_name: str
String denoting the filename
Returns
-------
None
"""
export_option = eo[1] \
if not args.csv \
else eo[0]
Export.export(data, f_name, export_option, "subreddits")