def process_new_links(period=media_period, force=False): """Fetches links from the last period and sets their media properities. If force is True, it will fetch properities for links even if the properties already exist""" links = Link._query(Link.c._date > timeago(period), sort=desc('_date'), data=True) results = {} jobs = [] for link in fetch_things2(links): if link.is_self or link.promoted: continue elif not force and (link.has_thumbnail or link.media_object): continue jobs.append(make_link_info_job(results, link, g.useragent)) #send links to a queue wq = WorkQueue(jobs, num_workers=20, timeout=30) wq.start() wq.jobs.join() #when the queue is finished, do the db writes in this thread for link, info in results.items(): update_link(link, info[0], info[1])
def run(): """Pull jobs from the queue, creates a job, and sends them to a WorkQueue for processing.""" num_workers = g.num_query_queue_workers wq = WorkQueue(num_workers=num_workers) wq.start() while True: job = None #limit the total number of jobs in the WorkQueue. we don't #need to load the entire db queue right away (the db queue can #get quite large). if len(running) < 2 * num_workers: with running_lock: iden, pickled_cr = get_query() if pickled_cr is not None: if not iden in running: running.add(iden) job = make_query_job(iden, pickled_cr) wq.add(job) #if we didn't find a job, sleep before trying again if not job: time.sleep(1)
def run(): """Pull jobs from the queue, creates a job, and sends them to a WorkQueue for processing.""" num_workers = g.num_query_queue_workers wq = WorkQueue(num_workers = num_workers) wq.start() while True: job = None #limit the total number of jobs in the WorkQueue. we don't #need to load the entire db queue right away (the db queue can #get quite large). if len(running) < 2 * num_workers: with running_lock: iden, pickled_cr = get_query() if pickled_cr is not None: if not iden in running: running.add(iden) job = make_query_job(iden, pickled_cr) wq.add(job) #if we didn't find a job, sleep before trying again if not job: time.sleep(1)
def process_new_links(period = media_period, force = False): """Fetches links from the last period and sets their media properities. If force is True, it will fetch properities for links even if the properties already exist""" links = Link._query(Link.c._date > timeago(period), sort = desc('_date'), data = True) results = {} jobs = [] for link in fetch_things2(links): if link.is_self or link.promoted: continue elif not force and (link.has_thumbnail or link.media_object): continue jobs.append(make_link_info_job(results, link, g.useragent)) #send links to a queue wq = WorkQueue(jobs, num_workers = 20, timeout = 30) wq.start() wq.jobs.join() #when the queue is finished, do the db writes in this thread for link, info in results.items(): update_link(link, info[0], info[1])