Example #1
0
File: dev.py Project: zrthxn/Markup
 def sentence_checker(self, sentence, context_sentences):
     print 'Process webExtracter.sentence_checker running...'
     jobs = []
     scrapped_data = {}
     query = sentence
     for j in search(query, tld="co.in", num=5, stop=1, pause=2):
         q = Queue()
         z = webExtracter()
         p = Process(target=z.scrapper, args=(j, q))
         jobs.append(p)
         trusted_sources = [r'quora']
         for source in trusted_sources:
             if re.search(source, j):
                 start = time.clock()
                 p.start()
                 p.join()
                 print 'time taken for this process is ', time.clock()-start
                 scrapped_data[j] = q.get()
                 break
         else:
             p.start()
             p.join(2)
             scrapped_data[j] = q.get()
             #print 7
             if p.is_alive():
                 print "this shit is still running... let's kill it..."
                 p.terminate()
                 p.join()
                 scrapped_data[j] = 'KILLED'
     p = webExtracter()
     p.Data_Processing(scrapped_data, query, context_sentences)