def regist_filter(global_config, options):
data = ldrfullfeed.load(global_config["filter.ldrfullfeed.path"])
def fetch(context):
index, feed = context
if global_config.get("verbose"):
print "fetching content for %d" % index
queue = []
for entry in feed["entries"]:
content = get_content(entry)
if entry.get("full_content"):
pass
elif entry.get(ad_filter.AD_FILTER_KEY) or content["type"] not in FETCH_TYPES:
entry["full_content"] = {"type": content["type"],
"value": content["value"]}
else:
url = entry["link"]
def get_xitem(fetch_url):
return ldrfullfeed.match(data, fetch_url)
queue.append((entry, url, get_xitem, content["value"]))
# jobs.append(gevent.spawn(merge, entry, url, get_xitem, content["value"]))
def fetch():
while len(queue):
item = queue.pop()
merge(*item)
jobs = [gevent.spawn(fetch) for i in range(2)]
gevent.joinall(jobs)
if global_config.get("verbose"):
print "fetched content for %d" % index
return context
return fetch
def regist_filter(global_config, options):
data = ldrfullfeed.load(global_config["filter.ldrfullfeed.path"])
def fetch(context):
index, feed = context
if global_config.get("verbose"):
print "fetching content for %d" % index
queue = []
for entry in feed["entries"]:
content = get_content(entry)
if entry.get("full_content"):
pass
elif entry.get(ad_filter.AD_FILTER_KEY
) or content["type"] not in FETCH_TYPES:
entry["full_content"] = {
"type": content["type"],
"value": content["value"]
}
else:
url = entry["link"]
def get_xitem(fetch_url):
return ldrfullfeed.match(data, fetch_url)
queue.append((entry, url, get_xitem, content["value"]))
# jobs.append(gevent.spawn(merge, entry, url, get_xitem, content["value"]))
def fetch():
while len(queue):
item = queue.pop()
merge(*item)
jobs = [gevent.spawn(fetch) for i in range(2)]
gevent.joinall(jobs)
if global_config.get("verbose"):
print "fetched content for %d" % index
return context
return fetch
def setUp(self):
self.data = ldrfullfeed.load("fullfeed.json")
def setUp(self):
self.data = ldrfullfeed.load("fullfeed.json")