Example #1
0
def regist_filter(global_config, options):
    data = ldrfullfeed.load(global_config["filter.ldrfullfeed.path"])
    def fetch(context):
        index, feed = context
        if global_config.get("verbose"):
            print "fetching content for %d" % index

        queue = []
        for entry in feed["entries"]:
            content = get_content(entry)
            if entry.get("full_content"):
                pass
            elif entry.get(ad_filter.AD_FILTER_KEY) or content["type"] not in FETCH_TYPES:
                entry["full_content"] =  {"type": content["type"],
                                         "value": content["value"]}
            else:
                url = entry["link"]
                def get_xitem(fetch_url):
                    return ldrfullfeed.match(data, fetch_url)
                queue.append((entry, url, get_xitem, content["value"]))
                # jobs.append(gevent.spawn(merge, entry, url, get_xitem, content["value"]))
        def fetch():
            while len(queue):
                item = queue.pop()
                merge(*item)

        jobs = [gevent.spawn(fetch) for i in range(2)]
        gevent.joinall(jobs)
        
        if global_config.get("verbose"):
            print "fetched content for %d" % index
            
        return context

    return fetch
Example #2
0
def regist_filter(global_config, options):
    data = ldrfullfeed.load(global_config["filter.ldrfullfeed.path"])

    def fetch(context):
        index, feed = context
        if global_config.get("verbose"):
            print "fetching content for %d" % index

        queue = []
        for entry in feed["entries"]:
            content = get_content(entry)
            if entry.get("full_content"):
                pass
            elif entry.get(ad_filter.AD_FILTER_KEY
                           ) or content["type"] not in FETCH_TYPES:
                entry["full_content"] = {
                    "type": content["type"],
                    "value": content["value"]
                }
            else:
                url = entry["link"]

                def get_xitem(fetch_url):
                    return ldrfullfeed.match(data, fetch_url)

                queue.append((entry, url, get_xitem, content["value"]))
                # jobs.append(gevent.spawn(merge, entry, url, get_xitem, content["value"]))
        def fetch():
            while len(queue):
                item = queue.pop()
                merge(*item)

        jobs = [gevent.spawn(fetch) for i in range(2)]
        gevent.joinall(jobs)

        if global_config.get("verbose"):
            print "fetched content for %d" % index

        return context

    return fetch
Example #3
0
 def setUp(self):
     self.data = ldrfullfeed.load("fullfeed.json")
Example #4
0
 def setUp(self):
     self.data = ldrfullfeed.load("fullfeed.json")