def create_and_save_thread(enum_url): """Returns correct subclass of CommentThread by parsing url and calling THREAD_TYPES.""" enum, (url, is_research) = enum_url filename = "CACHE/" + project + "_" + str(enum) + "_thread.p" def create_and_process(): """Helper-function to create , optionally save, and return thread""" thread_url = urlparse(url) thread_type = thread_url.netloc.split('.')[0].title() thread = THREAD_TYPES[thread_type](url, is_research) request_file = "CACHED_DATA/" + \ thread_url.netloc.split('.')[0] + \ ('_').join( thread_url.path.split('/')[:-1]) + '_req.p' if kwargs.get('delete_all', False): ac.handle_delete(filename) ac.handle_delete(request_file) if cache_it: ac.to_pickle(thread, filename) return thread if kwargs.get('use_cached', False): try: thread = joblib.load(filename) except (IOError, EOFError) as err: logging.warning("Could not load %s: %s", filename, err) ac.handle_delete(filename) thread = create_and_process() else: # not used_cached ac.handle_delete(filename) thread = create_and_process() assert isinstance(thread, CommentThread) return thread
def create_and_process(): """Helper-function to create , optionally save, and return thread""" thread_url = urlparse(url) thread_type = thread_url.netloc.split('.')[0].title() thread = THREAD_TYPES[thread_type](url, is_research) request_file = "CACHED_DATA/" + \ thread_url.netloc.split('.')[0] + \ ('_').join( thread_url.path.split('/')[:-1]) + '_req.p' if kwargs.get('delete_all', False): ac.handle_delete(filename) ac.handle_delete(request_file) if cache_it: ac.to_pickle(thread, filename) return thread