def _make_livestream_dir(split_stream_info): """ Make the `livestream` directory within the `scrapes/[DATE]` directory. Calls public methods from an external module: InitializeDirectory.create_dirs() Parameters ---------- split_stream_info: list List containing stream information Returns ------- stream_directory: str String denoting the path to the directory in which the stream is saved """ if split_stream_info[0] == "r": sub_directory = "subreddits" elif split_stream_info[0] == "u": sub_directory = "redditors" stream_directory = f"../scrapes/{date}/livestream/{sub_directory}" InitializeDirectory.create_dirs(stream_directory) return stream_directory
def write_structured_comments(data, f_name): """ Write structured comments to JSON by using the custom JSONEncoder class with the `cls` parameter within `json.dumps()`. Calls a method from an external module: InitializeDirectory.create_dirs() Parameters ---------- data: dict Dictionary of scrape data f_name: str String denoting the filename Returns ------- None """ filename = Export._get_filename_extension(f_name, "json", "comments") InitializeDirectory.create_dirs("/".join(filename.split("/")[:-1])) with open(filename, "w", encoding = "utf-8") as results: json.dump(data, results, indent = 4, cls = EncodeNode)
def test_create_dirs_method(self): test_path = "../scrapes/test_dir/another_test_dir/a_final_dir" InitializeDirectory.create_dirs(test_path) assert True \ if os.path.isdir(test_path) \ else False
class LogMain(): """ Decorator for logging URS runtime. Also handles KeyboardInterrupt and adds the event to the log if applicable. """ ### Set directory path and log format. DIR_PATH = f"../scrapes/{date}" LOG_FORMAT = "[%(asctime)s] [%(levelname)s]: %(message)s" ### Makes the `scrapes/[DATE]` directory in which the log and scraped files ### will be stored. InitializeDirectory.create_dirs(DIR_PATH) ### Configure logging settings. logging.basicConfig(filename=DIR_PATH + "/urs.log", format=LOG_FORMAT, level=logging.INFO) @staticmethod def master_timer(function): """ Wrapper for logging the amount of time it took to execute main(). Handle KeyboardInterrupt if user cancels URS. Parameters ---------- function: function() Run method within the wrapper Exceptions ---------- KeyboardInterrupt: Raised if user cancels URS Returns ------- wrapper: function() Return the wrapper method that runs the method passed into the decorator """ def wrapper(*args): logging.info("INITIALIZING URS.") logging.info("") start = time.time() try: function(*args) except KeyboardInterrupt: print(Style.BRIGHT + Fore.RED + "\n\nURS ABORTED BY USER.\n") logging.warning("") logging.warning("URS ABORTED BY USER.\n") quit() logging.info( f"URS COMPLETED IN {time.time() - start:.2f} SECONDS.\n") return wrapper
def get_scrape_type(scrape_file, tool): """ Get the name of the scrape-specific directory in which the data is stored and create the directories within the `analytics` folder. Parameters ---------- scrape_file: str String denoting the filepath tool: str String denoting the tool type Exceptions ---------- TypeError: Raised if the file is not JSON or if the file resides in the `analytics` directory Returns ------- analytics_dir: str String denoting the path to the directory in which the analytical data will be written scrape_dir: str String denoting the scrape-specific directory """ file_path = Path(scrape_file) scrape_dir = list(file_path.parts)[file_path.parts.index("scrapes") + 2] if file_path.name.split(".")[1] != "json" or scrape_dir == "analytics": raise TypeError split_analytics_dir = \ list(file_path.parts)[:file_path.parts.index("scrapes") + 2] + \ ["analytics", tool] + \ list(file_path.parts)[file_path.parts.index("scrapes") + 2:-1] analytics_dir = "/".join(split_analytics_dir) InitializeDirectory.create_dirs(analytics_dir) return analytics_dir, scrape_dir
def export(data, f_name, f_type, scrape): """ Write data to either CSV or JSON. Calls a method from an external module: InitializeDirectory.create_dirs() Calls previously defined private and public methods: Export._get_filename_extension() Export.write_json() Export.write_csv() Parameters ---------- data: dict Dictionary of scrape data f_name: str Filename f_type: str File type (.csv or .json) scrape: str Scrape type ("subreddits", "redditors", or "comments") Returns ------- None """ filename = Export._get_filename_extension(f_name, f_type, scrape) InitializeDirectory.create_dirs("/".join(filename.split("/")[:-1])) Export.write_json(data, filename) \ if f_type == "json" \ else Export.write_csv(data, filename)