Ejemplo n.º 1
0
    def create_and_save_thread(enum_url):
        """Returns correct subclass of CommentThread by parsing
        url and calling THREAD_TYPES."""
        enum, (url, is_research) = enum_url
        filename = "CACHE/" + project + "_" + str(enum) + "_thread.p"

        def create_and_process():
            """Helper-function to create , optionally save,
            and return thread"""
            thread_url = urlparse(url)
            thread_type = thread_url.netloc.split('.')[0].title()
            thread = THREAD_TYPES[thread_type](url, is_research)
            request_file = "CACHED_DATA/" + \
                           thread_url.netloc.split('.')[0] + \
                           ('_').join(
                               thread_url.path.split('/')[:-1]) + '_req.p'
            if kwargs.get('delete_all', False):
                ac.handle_delete(filename)
                ac.handle_delete(request_file)
            if cache_it:
                ac.to_pickle(thread, filename)
            return thread

        if kwargs.get('use_cached', False):
            try:
                thread = joblib.load(filename)
            except (IOError, EOFError) as err:
                logging.warning("Could not load %s: %s", filename, err)
                ac.handle_delete(filename)
                thread = create_and_process()
        else:  # not used_cached
            ac.handle_delete(filename)
            thread = create_and_process()
        assert isinstance(thread, CommentThread)
        return thread
Ejemplo n.º 2
0
 def create_and_process():
     """Helper-function to create , optionally save,
     and return thread"""
     thread_url = urlparse(url)
     thread_type = thread_url.netloc.split('.')[0].title()
     thread = THREAD_TYPES[thread_type](url, is_research)
     request_file = "CACHED_DATA/" + \
                    thread_url.netloc.split('.')[0] + \
                    ('_').join(
                        thread_url.path.split('/')[:-1]) + '_req.p'
     if kwargs.get('delete_all', False):
         ac.handle_delete(filename)
         ac.handle_delete(request_file)
     if cache_it:
         ac.to_pickle(thread, filename)
     return thread