def CheckLinksInLibrary(self):
import thread_pool
import time
self.t_link_delay = int(control.setting('library.service.testlink.delay') or 0)
if self.t_link_delay == 0:
printuj_linka('[CheckLink]: START and STOP - Do not check library sources')
return
dirs,files = control.listDir(self.library_folder)
self.progressDialogBG = control.progressDialogBG
self.progressDialogBG.create('Sprawdzam \xc5\xbar\xc3\xb3d\xc5\x82a biblioteki cda.pl','Ilo\xc5\x9b\xc4\x87 pozycji: %d'%len(dirs))
printuj_linka('[CheckLink]: START Library Folders = %d'%len(dirs))
if not control.condVisibility('Window.IsVisible(infodialog)') and not control.condVisibility('Player.HasVideo'):
try:
control.infoDialog('Sprawdzam \xc5\xbar\xc3\xb3dla biblioteki cda.pl', time=3)
self.infoDialog = True
except:
self.infoDialog = False
pool = thread_pool.ThreadPool(self.fredy)
self.res={'checked':0,'skipped':0,'removed':0,'updated':0}
N = len(dirs)
for idx,folder_ in enumerate(dirs):
pool.add_task(self.check_remove, *(folder_,idx,N))
time.sleep(0.1)
pool.wait_completion()
printuj_linka('[CheckLink]: END Status checked:%d, skipped:%d, removed:%d, updated:%d'%(self.res['checked'],self.res['skipped'],self.res['removed'],self.res['updated']))
self.progressDialogBG.close()
if self.infoDialog == True:
control.infoDialog('Usuni\xc4\x99to :%d, Zaktualizowano :%d '%(self.res['removed'],self.res['updated']), time=30)
def main():
'''
Temporary test script for the auto discovery module.
'''
config = get_config()
dbname = config['dbname']
getIpConfig.cleartables(dbname);
# create the thread pool and the task_queue.
pool = thread_pool.ThreadPool(config['num_threads'])
# Now add tasks to the pool for all the ips in the iprange that we have
ip_ranges = config['ip_ranges']
for rng in ip_ranges:
if getIpConfig.compare_ips(rng[0],rng[1]) > 0:
raise Exception("invalid range specified : " + rng[0] + " to " + rng[1])
work_ip = rng[0]
while getIpConfig.compare_ips(work_ip,rng[1]) <= 0:
pool.add_task(getIpConfig.process,work_ip,ip_ranges,dbname)
work_ip = next_ip.next_ip(work_ip)
pool.wait()
getIpConfig.post_process(dbname)
switch_dbs(dbname)
def __init__(self, meta_configs, stanza_configs, job_factory):
"""
@meta_configs: a dict like object, implement dict.get/[] like
interfaces to get the value for a key. meta_configs shall at least
contain
{"server_uri": uri, "checkpoint_dir": dir, "session_key": key}
key/value pairs
@stanza_configs: a list like object containing a list of dict
like object. Each element shall implement dict.get/[] like interfaces
to get the value for a key. Each element in the list shall at least
contain
{"duration": an_duration_int} key/value pair
@job_factory: an object which creates jobs. Shall implement
create_job(stanza_config) interface and return an callable object
"""
import thread_pool as tp
import job_scheduler as js
import timer_queue as tq
self.settings = self._read_default_settings()
self.meta_configs = meta_configs
self.event_queue = Queue.Queue()
self.wakeup_queue = Queue.Queue()
self._set_event_queue(stanza_configs)
pool_size = self._get_pool_size(stanza_configs)
self.io_pool = tp.ThreadPool(pool_size)
self.cpu_pool = self._create_process_pool()
self.scheduler = js.JobScheduler(job_factory, stanza_configs)
self.event_reporter = threading.Thread(target=self._output_events)
self.event_reporter.daemon = True
self.timer_queue = tq.TimerQueue()
self._started = False
def _TestThreadPool(self, threads):
"""Tests pool with specified threads from end to end."""
pool = thread_pool.ThreadPool(threads)
self.actual_call_count = 0
expected_call_count = 10000
self.data = xrange(expected_call_count)
self.actual_result = 0
expected_result = sum(self.data)
stats_lock = threading.Lock()
def _Dummy(num):
stats_lock.acquire()
self.actual_call_count += 1
self.actual_result += num
stats_lock.release()
for data in xrange(expected_call_count):
pool.AddTask(_Dummy, data)
pool.WaitCompletion()
self.assertEqual(self.actual_call_count, expected_call_count)
self.assertEqual(self.actual_result, expected_result)
pool.Shutdown()
for thread in pool.threads:
self.assertFalse(thread.is_alive())
def __init__(self, url):
self.url = url
self.sess = requests.session()
self.hh = hackhttp.hackhttp(hackhttp.httpconpool())
self.tp = thread_pool.ThreadPool(500)
self.headers = headers_dict = {
'X-Forwarder-For': '192.168.1.1',
}
def __init__(self, device):
"""
Constructor.
@type device: Device
@param device: the device which owns this thread
"""
Thread.__init__(self, name="Device Thread %d" % device.device_id)
self.device = device
self.number_subthreads = 8
self.subthreads = thread_pool.ThreadPool(self.number_subthreads)
def run():
db = db_pool.get_db_pool(True) # 初始化数据库连接池
event_list = get_event_list(db) # 获取等待处理的发行列表
pool = thread_pool.ThreadPool(10) # 十条进程同时运算
for item in event_list:
print(item)
pool.put(pool_action, (db, item[0], item[1],), callback)
while True:
time.sleep(0.5)
if len(pool.generate_list) - len(pool.free_list) == 0:
print("Task finished! Closing...")
pool.close()
break
else:
# print("{} Threads still working.Wait.".format(len(pool.generate_list) - len(pool.free_list)))
pass
def TestThreadPoolExceptionHandler(self):
"""Tests thread pool with exceptions."""
self.exception_raised = False
def _ExceptionHandler(e):
"""Verify an exception is raised and that it's the correct one."""
self.assertTrue(isinstance(e, TypeError))
self.assertEqual(e[0], 'gsutil')
self.exception_raised = True
pool = thread_pool.ThreadPool(1, exception_handler=_ExceptionHandler)
def _Dummy():
raise TypeError('gsutil')
pool.AddTask(_Dummy)
pool.WaitCompletion()
pool.Shutdown()
self.assertTrue(self.exception_raised)
def init(self):
"""初始化根节点和crawler,parser,save类，\
分别用于抓取，分析和保存
"""
self.logger.info('init')
try:
self.logger.info("read config file:%s" % (self.conf))
self.config = ConfigParser.ConfigParser()
self.config.read(self.conf)
self.url_list_file = self.config.get("spider", "url_list_file")
self.output_dir = self.config.get("spider", "output_directory")
self.max_depth = self.config.getint("spider", "max_depth")
self.crawl_interval = self.config.getint("spider",
"crawl_interval")
self.crawl_timeout = self.config.getint("spider", "crawl_timeout")
self.target_url = self.config.get("spider", "target_url")
self.thread_count = self.config.getint("spider", "thread_count")
except ConfigParser.Error as e:
self.logger.warn(e)
return -1
#初始化url_queue,保存已经抓取的URL列表和去除重复
self.url_queue = self.init_url_queue()
if self.url_queue == -1:
self.logger.warn('init url queue fail,return...')
return -1
#初始化抓取类
self.logger.info('init crawler.')
page_crawler = webpage_crawler.WebpageCrawler(self.logger)
#初始化解析类
self.logger.info('init parser.')
page_parser = webpage_parser.WebpageParser(self.logger, self.max_depth)
#初始化保存类
self.logger.info('init saver.')
page_saver = webpage_saver.WebpageSaver(self.logger, self.target_url,
self.output_dir)
#初始化线程池
self.logger.info('init thread_pool.')
self.thread_pools = thread_pool.ThreadPool(self.logger, self.thread_count, page_crawler,\
page_parser, page_saver, self.url_queue)
return 0
def __init__(self, configs, job_scheduler, event_writer):
"""
@configs: a list like object containing a list of dict
like object. Each element shall implement dict.get/[] like interfaces
to get the value for a key.
@job_scheduler: schedulering the jobs. shall implement get_ready_jobs
@event_writer: write_events
"""
import thread_pool as tp
import timer_queue as tq
self.settings = self._read_default_settings()
self.configs = configs
self.event_writer = event_writer
self.wakeup_queue = Queue.Queue()
pool_size = self._get_pool_size()
self.io_pool = tp.ThreadPool(pool_size)
self.cpu_pool = None
self.scheduler = job_scheduler
self.timer_queue = tq.TimerQueue()
self._started = False
self._stopped = False
def run():
db_p = db_pool.get_db_pool(True)
pool = thread_pool.ThreadPool(30)
e_list = get_elist_to_be_done(db_p)
for item in e_list:
pool.put(get_history_check, (
db_p,
item[0],
item[1],
), callback)
# get_history_line(db_p, item[0], item[1])
while True:
time.sleep(2)
if len(pool.generate_list) - len(pool.free_list) == 0:
print("Task finished! Closing...")
pool.close()
break
else:
print("{} Threads still working.Wait.".format(
len(pool.generate_list) - len(pool.free_list)))
pass
def GetNewMovies(self):
from cdapl import searchCDA
import thread_pool
import time
self.check_setting = 'false'
url = 'https://www.cda.pl/video/show/cale_filmy_or_caly_film_or_lektor_or_pl_or_dubbing_or_napisy_or_fps_or_odc/p%d?duration=dlugie§ion=&quality=720p§ion=&s=date§ion='
if not control.condVisibility('Window.IsVisible(infodialog)') and not control.condVisibility('Player.HasVideo'):
try:
control.infoDialog('Szukam nowych film\xc3\xb3w ...', time=3)
self.infoDialog = True
except:
self.infoDialog = False
self.progressDialogBG = control.progressDialogBG
self.progressDialogBG.create('cda.pl','Szukam nowych film\xc3\xb3w ...')
items=[]
control.setSetting('library.service.last.run', datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
for stronkaX in range(int(control.setting('library.service.pages') or 1)):
stronkaX +=1
printuj_linka('[AddMovie]: Searching cda.pl, page [%d]'%stronkaX)
nowy_link,next=searchCDA(url%(stronkaX),False,False)
items.extend(nowy_link)
self.progressDialogBG.update(0,message='Znalaz\xc5\x82em pozycji [%d] '%(len(items)))
self.progressDialogBG.update(0,message='Indentyfikuje %d film\xc3\xb3w w %d w\xc4\x85tkach ...'%(len(items),self.fredy))
items = [x for x in items if x.get('code','')!= '']
printuj_linka('[AddMovie]: Found Total %d videos ... '%(len(items)))
pool = thread_pool.ThreadPool(self.fredy)
self.out=[]
N=len(items)
for idx,one in enumerate(items):
pool.add_task(self.dod_Movies, *(one,idx,N))
time.sleep(0.1)
pool.wait_completion()
printuj_linka('[AddMovie]: (After Threading) Found Total %d Movies ... '%(len(self.out)))
self.out.reverse()
self.progressDialogBG.close()
self.add2( self.out )
control.setSetting('library.service.last.run', datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
# url to search news
bing_news_base = "http://www.bing.com/news/search?q="
# url to get current trends
bing_hint_request_url = "http://api.bing.com/qsonhs.aspx?" \
"FORM=ASAPIW&mkt=en-US&type=cb&cb=sa_inst.apiCB&cp=0&q=&o=l+a+hc+p"
bing_hint_request_params = {
"FORM": "ASAPIW",
"mkt": "en-US",
"type": "cb",
"cb": "sa_inst.apiCB",
"cp": "0",
"q": "",
"o": "l a hc p"
}
m = hashlib.md5()
output_dir = "./Output/"
keyword_queue = set()
# store hashcode of title
covered_keyword = set()
# link_list = []
lock = threading.Lock()
# worker_captain = worker.WorkerCaptain()
pool = thread_pool.ThreadPool(max_crawler)
regex = re.compile('[%s]+' % re.escape(string.punctuation))
import thread_pool
class Student():
def run(self, *args, **kwargs):
print(args)
print('1234')
if __name__ == '__main__':
student = Student()
#student.run([student])
pool = thread_pool.ThreadPool(3)
requests = thread_pool.makeRequests(student.run, [[]])
pool.putRequest(requests[0])
pool.wait()
# if 0 != newpid:
# self.poll_connects(listen_fd)
for p in _processes:
p.join()
def poll_connects(self, listen_fd):
try:
epoll_fd = select.epoll()
epoll_fd.register(
listen_fd.fileno(), select.EPOLLIN | select.EPOLLET
| select.EPOLLERR | select.EPOLLHUP)
except select.error, e:
print e, vars.backtrace()
vars.logger.error(e)
tp = thread_pool.ThreadPool(4)
tp.start()
params = {}
def clear_fd(fd):
try:
_param = params[fd]
epoll_fd.unregister(fd)
_param["connection"].close()
except Exception, e:
print e, vars.backtrace()
pass
if fd in params:
del params[fd]
# try:
def tearDown(self):
thread_pool.thread_pool = thread_pool.ThreadPool()
def __init__(self):
self.work_queue = threadsafequeue.thread_safe_queue()
self.thread_pool = thread_pool.ThreadPool(self.work_queue)
self.socket = socket.socket()