def _make_livestream_dir(split_stream_info):
"""
Make the `livestream` directory within the `scrapes/[DATE]` directory.
Calls public methods from an external module:
InitializeDirectory.create_dirs()
Parameters
----------
split_stream_info: list
List containing stream information
Returns
-------
stream_directory: str
String denoting the path to the directory in which the stream is saved
"""
if split_stream_info[0] == "r":
sub_directory = "subreddits"
elif split_stream_info[0] == "u":
sub_directory = "redditors"
stream_directory = f"../scrapes/{date}/livestream/{sub_directory}"
InitializeDirectory.create_dirs(stream_directory)
return stream_directory
def write_structured_comments(data, f_name):
"""
Write structured comments to JSON by using the custom JSONEncoder class
with the `cls` parameter within `json.dumps()`.
Calls a method from an external module:
InitializeDirectory.create_dirs()
Parameters
----------
data: dict
Dictionary of scrape data
f_name: str
String denoting the filename
Returns
-------
None
"""
filename = Export._get_filename_extension(f_name, "json", "comments")
InitializeDirectory.create_dirs("/".join(filename.split("/")[:-1]))
with open(filename, "w", encoding = "utf-8") as results:
json.dump(data, results, indent = 4, cls = EncodeNode)
def test_create_dirs_method(self):
test_path = "../scrapes/test_dir/another_test_dir/a_final_dir"
InitializeDirectory.create_dirs(test_path)
assert True \
if os.path.isdir(test_path) \
else False
class LogMain():
"""
Decorator for logging URS runtime. Also handles KeyboardInterrupt and adds the
event to the log if applicable.
"""
### Set directory path and log format.
DIR_PATH = f"../scrapes/{date}"
LOG_FORMAT = "[%(asctime)s] [%(levelname)s]: %(message)s"
### Makes the `scrapes/[DATE]` directory in which the log and scraped files
### will be stored.
InitializeDirectory.create_dirs(DIR_PATH)
### Configure logging settings.
logging.basicConfig(filename=DIR_PATH + "/urs.log",
format=LOG_FORMAT,
level=logging.INFO)
@staticmethod
def master_timer(function):
"""
Wrapper for logging the amount of time it took to execute main(). Handle
KeyboardInterrupt if user cancels URS.
Parameters
----------
function: function()
Run method within the wrapper
Exceptions
----------
KeyboardInterrupt:
Raised if user cancels URS
Returns
-------
wrapper: function()
Return the wrapper method that runs the method passed into the
decorator
"""
def wrapper(*args):
logging.info("INITIALIZING URS.")
logging.info("")
start = time.time()
try:
function(*args)
except KeyboardInterrupt:
print(Style.BRIGHT + Fore.RED + "\n\nURS ABORTED BY USER.\n")
logging.warning("")
logging.warning("URS ABORTED BY USER.\n")
quit()
logging.info(
f"URS COMPLETED IN {time.time() - start:.2f} SECONDS.\n")
return wrapper
def get_scrape_type(scrape_file, tool):
"""
Get the name of the scrape-specific directory in which the data is stored
and create the directories within the `analytics` folder.
Parameters
----------
scrape_file: str
String denoting the filepath
tool: str
String denoting the tool type
Exceptions
----------
TypeError:
Raised if the file is not JSON or if the file resides in the `analytics`
directory
Returns
-------
analytics_dir: str
String denoting the path to the directory in which the analytical
data will be written
scrape_dir: str
String denoting the scrape-specific directory
"""
file_path = Path(scrape_file)
scrape_dir = list(file_path.parts)[file_path.parts.index("scrapes") +
2]
if file_path.name.split(".")[1] != "json" or scrape_dir == "analytics":
raise TypeError
split_analytics_dir = \
list(file_path.parts)[:file_path.parts.index("scrapes") + 2] + \
["analytics", tool] + \
list(file_path.parts)[file_path.parts.index("scrapes") + 2:-1]
analytics_dir = "/".join(split_analytics_dir)
InitializeDirectory.create_dirs(analytics_dir)
return analytics_dir, scrape_dir
def export(data, f_name, f_type, scrape):
"""
Write data to either CSV or JSON.
Calls a method from an external module:
InitializeDirectory.create_dirs()
Calls previously defined private and public methods:
Export._get_filename_extension()
Export.write_json()
Export.write_csv()
Parameters
----------
data: dict
Dictionary of scrape data
f_name: str
Filename
f_type: str
File type (.csv or .json)
scrape: str
Scrape type ("subreddits", "redditors", or "comments")
Returns
-------
None
"""
filename = Export._get_filename_extension(f_name, f_type, scrape)
InitializeDirectory.create_dirs("/".join(filename.split("/")[:-1]))
Export.write_json(data, filename) \
if f_type == "json" \
else Export.write_csv(data, filename)