def scrap(self,url): if url is not None and len(url) > 0: url_info = tldextract.extract(url) self.domain = url_info.domain print '[INFO] Scrapper::scrap, domain',self.domain tp = ThreadPool(max_workers=120) tp.add_job(threaded_function,[self.domain,url]) else: print '[ALARM] Scrapper:scrap, invalid url'
#!/usr/bin/python import Queue from ThreadPool import * from util import * hosts = get_hosts() queue = Queue.Queue() tp = ThreadPool(queue, len(hosts)) for hostname in hosts: tp.add_job(read_result, hostname) tp.wait_for_complete()
('InputSampleBitDepth', input_bit_depth), ('SampleBitDepth', sample_bit_depth), ] enc_other_cfg = [ ('QPIFrame', int(QP)), ('QPPFrame', int(QP) + 1), ('QPBFrame', int(QP) + 4), ('stdout', encoder_log), # ('err_log', err_log), # ('xxEnable', 1) ] enc_command = ' '.join( parse_param(enc_base_cfg) + parse_param(enc_other_cfg)) # 解码参数 dec_cfg = [ ('encoder', 'TAppDecoder'), ('-b', bin_stream), ('-o', dec_yuv), ('stdout', decoder_log), ] dec_command = ' '.join(parse_param(dec_cfg)) my_pool.add_job(job_func, enc_command, dec_command) pass my_pool.start() my_pool.wait_complete() print('finished!')