def _test_it(self, company):
        from midas.scripts.fetch_crunchbase_companies import Fetcher
        from midas.compat import Queue

        q = Queue()
        q.put(company)
        cf = Fetcher(q)
        cf.daemon = True
        cf.start()
        return q
 def run(self):
     if self.args.quiet:
         log_level = logging.CRITICAL
     else:
         log_level = logging.INFO
     logging.basicConfig(level=log_level)
     cl = CompanyList(self.args.location)
     logging.info('Updating CompanyList')
     cl.update()
     q = Queue()
     for _ in range(self.args.num_threads):
         t = Fetcher(q)
         t.daemon = True
         t.start()
     for company in cl.list_not_local():
         q.put(company)
     q.join()
     return 0
Example #3
0
    def generate_missing_indicators(self):
        to_produce_q = Queue()
        self.out('Generating the following indicators:')
        for i in self.indicators:
            if not i.produced:
                to_produce_q.put(i)
                self.out(str(i))
        if not self.query_user_permission('Proceed?'):
            raise SystemExit('Canceled due to user interaction')

        threads = []
        for _ in range(min(self.num_threads, to_produce_q.qsize())):
            t = IndicatorUpdater(self.ids_to_samples, 
                                 to_produce_q,
                                 StreamAlexaIndicatorsCaller(self.cmd_path))
            t.start()
            threads.append(t)
        for t in threads:
            t.join()
        if any(imap(operator.attrgetter('failed'), threads)):
            raise Exception('At least one thread died!')
        to_produce_q.join()