class TaskPool(object): def __init__(self, limit, logger=None, **kwargs): self.limit = limit self.logger = logger or log.get_default_logger() self._pool = None def start(self): self._pool = ThreadPool(self.limit) def stop(self): self._pool.dismissWorkers(self.limit, do_join=True) def apply_async(self, target, args=None, kwargs=None, callbacks=None, errbacks=None, accept_callback=None, **compat): args = args or [] kwargs = kwargs or {} callbacks = callbacks or [] errbacks = errbacks or [] on_ready = curry(self.on_ready, callbacks, errbacks) self.logger.debug("ThreadPool: Apply %s (args:%s kwargs:%s)" % (target, args, kwargs)) req = WorkRequest(do_work, (target, args, kwargs, on_ready, accept_callback)) self._pool.putRequest(req) # threadpool also has callback support, # but for some reason the callback is not triggered # before you've collected the results. # Clear the results (if any), so it doesn't grow too large. self._pool._results_queue.queue.clear() return req def on_ready(self, callbacks, errbacks, ret_value): """What to do when a worker task is ready and its return value has been collected.""" if isinstance(ret_value, ExceptionInfo): if isinstance(ret_value.exception, (SystemExit, KeyboardInterrupt)): # pragma: no cover raise ret_value.exception [errback(ret_value) for errback in errbacks] else: [callback(ret_value) for callback in callbacks]
def gits_download(self, url, output="/tmp", threads=20): if not self.output: self.output = output results = self.index(url, output=output) if not results: return args = [((i[0], i[1]), {}) for i in self.giturls] # ... and build a WorkRequest object for each item in data requests = makeRequests(self.callback, args, self.print_result, self.handle_exception) main = ThreadPool(threads) for req in requests: main.putRequest(req) print "Work request #%s added." % req.requestID i = 0 while True: try: main.poll() print "Main thread working...", print "(active worker threads: %i)" % ( threading.activeCount()-1, ) if i == 10: print "**** Adding 3 more worker threads..." main.createWorkers(3) if i == 20: print "**** Dismissing 2 worker threads..." main.dismissWorkers(2) i += 1 except KeyboardInterrupt: print "**** Interrupted!" break except NoResultsPending: print "**** No pending results." break if main.dismissedWorkers: print "Joining all dismissed worker threads..." main.joinAllDismissedWorkers()
class TaskPool(object): def __init__(self, limit, logger=None, **kwargs): self.limit = limit self.logger = logger or log.get_default_logger() self._pool = None def start(self): self._pool = ThreadPool(self.limit) def stop(self): self._pool.dismissWorkers(self.limit, do_join=True) def apply_async(self, target, args=None, kwargs=None, callbacks=None, errbacks=None, accept_callback=None, **compat): args = args or [] kwargs = kwargs or {} callbacks = callbacks or [] errbacks = errbacks or [] on_ready = partial(self.on_ready, callbacks, errbacks) self.logger.debug("ThreadPool: Apply %s (args:%s kwargs:%s)" % ( target, args, kwargs)) req = WorkRequest(do_work, (target, args, kwargs, on_ready, accept_callback)) self._pool.putRequest(req) # threadpool also has callback support, # but for some reason the callback is not triggered # before you've collected the results. # Clear the results (if any), so it doesn't grow too large. self._pool._results_queue.queue.clear() return req def on_ready(self, callbacks, errbacks, ret_value): """What to do when a worker task is ready and its return value has been collected.""" if isinstance(ret_value, ExceptionInfo): if isinstance(ret_value.exception, ( SystemExit, KeyboardInterrupt)): # pragma: no cover raise ret_value.exception [errback(ret_value) for errback in errbacks] else: [callback(ret_value) for callback in callbacks]
def run_example(): num_workers = 3 pool = ThreadPool(num_workers) # This variable will tell us whether all threads worked or not. Stored in # an object (list) otherwise the inner definition cannot modify it. success = [True] # The exception handler is not required, but if it's not used the error # will be silent. def exc_handler(work_request, exc_info): mldb.log(traceback.format_tb(exc_info[2])) exception_type = exc_info[0] exception_message = exc_info[1] mldb.log(str(exception_type) + ': ' + str(exception_message)) success[0] = False # If there is an error, stop all threads as soon as possible pool.dismissWorkers(num_workers, do_join=True) # makeRequests takes, as a second argument, a list of tuples where the # first element is *args and the second one is **kwargs and where each # tuple represents a job to run. # # Here we schedule two jobs. requests = makeRequests(some_func, [(['patate'], {}), (['orange'], {})], exc_callback=exc_handler) [pool.putRequest(req) for req in requests] # pool.wait will raise an exception if an error occurs in an early jobs and # more jobs need to be run. It's ok, if there is an error we want to stop # anyway. pool.wait() # If the error occurred in one of the last jobs pool.wait will have worked # so we need to check it anyway. if not success[0]: mldb.log("An error occured") return # It is important (MLDBFB-470) to properly dismiss the workers pool.dismissWorkers(num_workers, do_join=True) mldb.log("Out of main thread")
def _run(self, urls, url, threads, num): pool = ThreadPool(threads) params = [] if url: for i in range(num): params.append(url) else: l = len(urls) for i in range(num): params.append(urls[i % l]) begin = time.time() requests = threadpool.makeRequests(self._get_url, params) [pool.putRequest(req) for req in requests] pool.wait() pool.dismissWorkers(threads) end = time.time() logging.info("all cost %f s" % (end - begin)) self._deal_results() pass
class TaskPool(BasePool): def on_start(self): self._pool = ThreadPool(self.limit) def on_stop(self): self._pool.dismissWorkers(self.limit, do_join=True) def on_apply(self, target, args=None, kwargs=None, callback=None, accept_callback=None, **_): req = WorkRequest(apply_target, (target, args, kwargs, callback, accept_callback)) self._pool.putRequest(req) # threadpool also has callback support, # but for some reason the callback is not triggered # before you've collected the results. # Clear the results (if any), so it doesn't grow too large. self._pool._results_queue.queue.clear() return req
def submitter(once=False, rses=[], mock=False, process=0, total_processes=1, total_threads=1, bulk=100, group_bulk=1, group_policy='rule', fts_source_strategy='auto', activities=None, sleep_time=600, max_sources=4, retry_other_fts=False): """ Main loop to submit a new transfer primitive to a transfertool. """ logging.info('Transfer submitter starting - process (%i/%i) threads (%i)' % (process, total_processes, total_threads)) try: scheme = config_get('conveyor', 'scheme') except NoOptionError: scheme = None try: failover_scheme = config_get('conveyor', 'failover_scheme') except NoOptionError: failover_scheme = None try: timeout = config_get('conveyor', 'submit_timeout') timeout = float(timeout) except NoOptionError: timeout = None try: bring_online = config_get('conveyor', 'bring_online') except NoOptionError: bring_online = 43200 try: max_time_in_queue = {} timelife_conf = config_get('conveyor', 'max_time_in_queue') timelife_confs = timelife_conf.split(",") for conf in timelife_confs: act, timelife = conf.split(":") max_time_in_queue[act.strip()] = int(timelife.strip()) except NoOptionError: max_time_in_queue = {} if 'default' not in max_time_in_queue: max_time_in_queue['default'] = 168 logging.debug("Maximum time in queue for different activities: %s" % max_time_in_queue) executable = ' '.join(sys.argv) hostname = socket.getfqdn() pid = os.getpid() hb_thread = threading.current_thread() heartbeat.sanity_check(executable=executable, hostname=hostname) hb = heartbeat.live(executable, hostname, pid, hb_thread) logging.info( 'Transfer submitter started - process (%i/%i) threads (%i/%i) timeout (%s)' % (process, total_processes, hb['assign_thread'], hb['nr_threads'], timeout)) threadPool = ThreadPool(total_threads) activity_next_exe_time = defaultdict(time.time) while not graceful_stop.is_set(): try: hb = heartbeat.live(executable, hostname, pid, hb_thread, older_than=3600) if activities is None: activities = [None] if rses: rse_ids = [rse['id'] for rse in rses] else: rse_ids = None for activity in activities: if activity_next_exe_time[activity] > time.time(): graceful_stop.wait(1) continue user_transfer = False if activity in USER_ACTIVITY and USER_TRANSFERS in ['cms']: logging.info("CMS user transfer activity") user_transfer = True logging.info( "%s:%s Starting to get transfer transfers for %s" % (process, hb['assign_thread'], activity)) ts = time.time() transfers = __get_transfers(process=process, total_processes=total_processes, thread=hb['assign_thread'], total_threads=hb['nr_threads'], failover_schemes=failover_scheme, limit=bulk, activity=activity, rses=rse_ids, schemes=scheme, mock=mock, max_sources=max_sources, bring_online=bring_online, retry_other_fts=retry_other_fts) record_timer( 'daemons.conveyor.transfer_submitter.get_transfers.per_transfer', (time.time() - ts) * 1000 / (len(transfers) if len(transfers) else 1)) record_counter( 'daemons.conveyor.transfer_submitter.get_transfers', len(transfers)) record_timer( 'daemons.conveyor.transfer_submitter.get_transfers.transfers', len(transfers)) logging.info( "%s:%s Got %s transfers for %s" % (process, hb['assign_thread'], len(transfers), activity)) # group transfers logging.info("%s:%s Starting to group transfers for %s" % (process, hb['assign_thread'], activity)) ts = time.time() grouped_jobs = bulk_group_transfer(transfers, group_policy, group_bulk, fts_source_strategy, max_time_in_queue) record_timer( 'daemons.conveyor.transfer_submitter.bulk_group_transfer', (time.time() - ts) * 1000 / (len(transfers) if len(transfers) else 1)) logging.info("%s:%s Starting to submit transfers for %s" % (process, hb['assign_thread'], activity)) for external_host in grouped_jobs: if not user_transfer: for job in grouped_jobs[external_host]: # submit transfers job_requests = makeRequests( submit_transfer, args_list=[((), { 'external_host': external_host, 'job': job, 'submitter': 'transfer_submitter', 'process': process, 'thread': hb['assign_thread'], 'timeout': timeout })]) [ threadPool.putRequest(job_req) for job_req in job_requests ] else: for user, jobs in grouped_jobs[ external_host].iteritems(): # submit transfers for job in jobs: job_requests = makeRequests( submit_transfer, args_list=[((), { 'external_host': external_host, 'job': job, 'submitter': 'transfer_submitter', 'process': process, 'thread': hb['assign_thread'], 'timeout': timeout, 'user_transfer_job': user_transfer })]) [ threadPool.putRequest(job_req) for job_req in job_requests ] threadPool.wait() if len(transfers) < group_bulk: logging.info( '%i:%i - only %s transfers for %s which is less than group bulk %s, sleep %s seconds' % (process, hb['assign_thread'], len(transfers), activity, group_bulk, sleep_time)) if activity_next_exe_time[activity] < time.time(): activity_next_exe_time[activity] = time.time( ) + sleep_time except: logging.critical( '%s:%s %s' % (process, hb['assign_thread'], traceback.format_exc())) if once: break logging.info('%s:%s graceful stop requested' % (process, hb['assign_thread'])) threadPool.dismissWorkers(total_threads, do_join=True) heartbeat.die(executable, hostname, pid, hb_thread) logging.info('%s:%s graceful stop done' % (process, hb['assign_thread'])) return
# [main.putRequest(req) for req in requests] # ...and wait for the results to arrive in the result queue # by using ThreadPool.wait(). This would block until results for # all work requests have arrived: # main.wait() # instead we can poll for results while doing something else: i = 0 while True: try: time.sleep(0.5) main.poll() print "Main thread working...", print "(active worker threads: %i)" % (threading.activeCount()-1, ) if i == 10: print "**** Adding 3 more worker threads..." main.createWorkers(3) if i == 20: print "**** Dismissing 2 worker threads..." main.dismissWorkers(2) i += 1 except KeyboardInterrupt: print "**** Interrupted!" break except NoResultsPending: print "**** No pending results." break if main.dismissedWorkers: print "Joining all dismissed worker threads..." main.joinAllDismissedWorkers()
def poller(once=False, process=0, total_processes=1, thread=0, total_threads=1, activities=None, sleep_time=60, fts_bulk=100, db_bulk=1000, older_than=60, activity_shares=None): """ Main loop to check the status of a transfer primitive with a transfertool. """ try: timeout = config_get('conveyor', 'poll_timeout') timeout = float(timeout) except NoOptionError: timeout = None logging.info( 'poller starting - process (%i/%i) thread (%i/%i) bulk (%i) timeout (%s)' % (process, total_processes, thread, total_threads, db_bulk, timeout)) executable = ' '.join(sys.argv) hostname = socket.getfqdn() pid = os.getpid() hb_thread = threading.current_thread() heartbeat.sanity_check(executable=executable, hostname=hostname) hb = heartbeat.live(executable, hostname, pid, hb_thread) logging.info('poller started - process (%i/%i) thread (%i/%i) bulk (%i)' % (process, total_processes, hb['assign_thread'], hb['nr_threads'], db_bulk)) activity_next_exe_time = defaultdict(time.time) threadPool = ThreadPool(total_threads) while not graceful_stop.is_set(): try: hb = heartbeat.live(executable, hostname, pid, hb_thread, older_than=3600) logging.debug('poller - thread (%i/%i)' % (hb['assign_thread'], hb['nr_threads'])) if activities is None: activities = [None] for activity in activities: if activity_next_exe_time[activity] > time.time(): graceful_stop.wait(1) continue ts = time.time() logging.debug( '%i:%i - start to poll transfers older than %i seconds for activity %s' % (process, hb['assign_thread'], older_than, activity)) transfs = transfer_core.get_next_transfers( request_type=[ RequestType.TRANSFER, RequestType.STAGEIN, RequestType.STAGEOUT ], state=[RequestState.SUBMITTED], limit=db_bulk, older_than=datetime.datetime.utcnow() - datetime.timedelta(seconds=older_than), process=process, total_processes=total_processes, thread=hb['assign_thread'], total_threads=hb['nr_threads'], activity=activity, activity_shares=activity_shares) record_timer('daemons.conveyor.poller.000-get_next_transfers', (time.time() - ts) * 1000) if transfs: logging.debug( '%i:%i - polling %i transfers for activity %s' % (process, hb['assign_thread'], len(transfs), activity)) xfers_ids = {} for transf in transfs: if not transf['external_host'] in xfers_ids: xfers_ids[transf['external_host']] = [] xfers_ids[transf['external_host']].append( transf['external_id']) for external_host in xfers_ids: for xfers in chunks(xfers_ids[external_host], fts_bulk): # poll transfers # xfer_requests = makeRequests(common.poll_transfers, args_list=[((external_host, xfers, process, thread), {})]) xfer_requests = makeRequests(poll_transfers, args_list=[((), { 'external_host': external_host, 'xfers': xfers, 'process': process, 'thread': hb['assign_thread'], 'timeout': timeout })]) [ threadPool.putRequest(xfer_req) for xfer_req in xfer_requests ] threadPool.wait() if len(transfs) < db_bulk / 2: logging.info( "%i:%i - only %s transfers for activity %s, which is less than half of the bulk %s, will sleep %s seconds" % (process, hb['assign_thread'], len(transfs), activity, db_bulk, sleep_time)) if activity_next_exe_time[activity] < time.time(): activity_next_exe_time[activity] = time.time( ) + sleep_time except: logging.critical( "%i:%i - %s" % (process, hb['assign_thread'], traceback.format_exc())) if once: break logging.info('%i:%i - graceful stop requests' % (process, hb['assign_thread'])) threadPool.dismissWorkers(total_threads, do_join=True) heartbeat.die(executable, hostname, pid, hb_thread) logging.info('%i:%i - graceful stop done' % (process, hb['assign_thread']))