def acknowledge( self ): stored_procedure( "acknowledge_job", self.current_job['domain'], self.current_job['url'] ) for url in self.discovered_urls: new_url_scheme = urlsplit( url )[0] if domain(url) == domain( self.current_job['url'] ) and new_url_scheme in ACCEPTABLE_SCHEMES: self.message_stack.append( "ACCEPT) %s" % url ) self.queue( url ) else: self.message_stack.append( "REJECT) %s" % url ) self.message_stack.append( "ACK'd) %s" % self.current_job['url'] )
def queue( self, url ): j, d = self.current_job, ( int( self.current_job['depth'] ) + 1 ) stored_procedure( "queue_job", j['domain'], j['page_limit'], j['depth_limit'], url, d, j['url'], j['starting_point'], j['batch'], j['org_id'] )
def run(): LOG.info("starting katipo sweeper") while True: outcome = stored_procedure("release_holds") LOG.info(outcome) sleep(1)