def acknowledge( self ):
stored_procedure( "acknowledge_job", self.current_job['domain'], self.current_job['url'] )
for url in self.discovered_urls:
new_url_scheme = urlsplit( url )[0]
if domain(url) == domain( self.current_job['url'] ) and new_url_scheme in ACCEPTABLE_SCHEMES:
self.message_stack.append( "ACCEPT) %s" % url )
self.queue( url )
else:
self.message_stack.append( "REJECT) %s" % url )
self.message_stack.append( "ACK'd) %s" % self.current_job['url'] )
def queue( self, url ):
j, d = self.current_job, ( int( self.current_job['depth'] ) + 1 )
stored_procedure( "queue_job", j['domain'], j['page_limit'], j['depth_limit'], url, d, j['url'], j['starting_point'], j['batch'], j['org_id'] )
def run():
LOG.info("starting katipo sweeper")
while True:
outcome = stored_procedure("release_holds")
LOG.info(outcome)
sleep(1)
