def process_new_links(period=media_period, force=False):
"""Fetches links from the last period and sets their media
properities. If force is True, it will fetch properities for links
even if the properties already exist"""
links = Link._query(Link.c._date > timeago(period),
sort=desc('_date'),
data=True)
results = {}
jobs = []
for link in fetch_things2(links):
if link.is_self or link.promoted:
continue
elif not force and (link.has_thumbnail or link.media_object):
continue
jobs.append(make_link_info_job(results, link, g.useragent))
#send links to a queue
wq = WorkQueue(jobs, num_workers=20, timeout=30)
wq.start()
wq.jobs.join()
#when the queue is finished, do the db writes in this thread
for link, info in results.items():
update_link(link, info[0], info[1])
def run():
"""Pull jobs from the queue, creates a job, and sends them to a
WorkQueue for processing."""
num_workers = g.num_query_queue_workers
wq = WorkQueue(num_workers = num_workers)
wq.start()
while True:
job = None
#limit the total number of jobs in the WorkQueue. we don't
#need to load the entire db queue right away (the db queue can
#get quite large).
if len(running) < 2 * num_workers:
with running_lock:
iden, pickled_cr = get_query()
if pickled_cr is not None:
if not iden in running:
running.add(iden)
job = make_query_job(iden, pickled_cr)
wq.add(job)
#if we didn't find a job, sleep before trying again
if not job:
time.sleep(1)
def process_new_links(period = media_period, force = False):
"""Fetches links from the last period and sets their media
properities. If force is True, it will fetch properities for links
even if the properties already exist"""
links = Link._query(Link.c._date > timeago(period), sort = desc('_date'),
data = True)
results = {}
jobs = []
for link in fetch_things2(links):
if link.is_self or link.promoted:
continue
elif not force and (link.has_thumbnail or link.media_object):
continue
jobs.append(make_link_info_job(results, link, g.useragent))
#send links to a queue
wq = WorkQueue(jobs, num_workers = 20, timeout = 30)
wq.start()
wq.jobs.join()
#when the queue is finished, do the db writes in this thread
for link, info in results.items():
update_link(link, info[0], info[1])
def run():
"""Pull jobs from the queue, creates a job, and sends them to a
WorkQueue for processing."""
num_workers = g.num_query_queue_workers
wq = WorkQueue(num_workers=num_workers)
wq.start()
while True:
job = None
#limit the total number of jobs in the WorkQueue. we don't
#need to load the entire db queue right away (the db queue can
#get quite large).
if len(running) < 2 * num_workers:
with running_lock:
iden, pickled_cr = get_query()
if pickled_cr is not None:
if not iden in running:
running.add(iden)
job = make_query_job(iden, pickled_cr)
wq.add(job)
#if we didn't find a job, sleep before trying again
if not job:
time.sleep(1)
