def pushLinkJobs():
"""
Cleanup old links in the links table.
Run this job periodically.
"""
jobManager = MinerJobManager()
linkManager = LinkManager()
if jobManager.count() > 100:
logging.info("Skipping. Too many jobs queued already!!")
return;
logging.info("Getting unprocessed links.")
links = linkManager.getUnprocessedLinks()
nLinks = 0;
for linkId in links:
processLinkJob = WorkerJob(
JOB_PROCESSLINK,
{ JOBARG_PROCESSLINK_LINKID : linkId})
jobManager.enqueueJob(processLinkJob)
logging.info(
"Process link job with jobId '%s' put for linkId: %s.",
processLinkJob.jobId,
linkId)
nLinks = nLinks + 1
logging.info("Number of process link jobs were: %i", nLinks)
def pushFeedJobs():
"""
Push feed processing jobs to job queue.
"""
jobManager = MinerJobManager()
feedManager = FeedManager()
if jobManager.count() > 50:
logging.info("Skipping. Too many jobs queued already!!")
return
logging.info("Getting stale feeds.")
staleFeeds = feedManager.getStaleFeeds()
nStaleFeeds = 0
for feed in staleFeeds:
processFeedJob = WorkerJob(JOB_PROCESSFEED,
{JOBARG_PROCESSFEED_FEEDID: feed})
jobManager.enqueueJob(processFeedJob)
logging.info("Process feed job put for feedId: %s. Job id: %s", feed,
processFeedJob.jobId)
nStaleFeeds = nStaleFeeds + 1
logging.info("Number of stale feeds are: %i", nStaleFeeds)