class TaskPool(object):
def __init__(self, limit, logger=None, **kwargs):
self.limit = limit
self.logger = logger or log.get_default_logger()
self._pool = None
def start(self):
self._pool = ThreadPool(self.limit)
def stop(self):
self._pool.dismissWorkers(self.limit, do_join=True)
def apply_async(self,
target,
args=None,
kwargs=None,
callbacks=None,
errbacks=None,
accept_callback=None,
**compat):
args = args or []
kwargs = kwargs or {}
callbacks = callbacks or []
errbacks = errbacks or []
on_ready = curry(self.on_ready, callbacks, errbacks)
self.logger.debug("ThreadPool: Apply %s (args:%s kwargs:%s)" %
(target, args, kwargs))
req = WorkRequest(do_work,
(target, args, kwargs, on_ready, accept_callback))
self._pool.putRequest(req)
# threadpool also has callback support,
# but for some reason the callback is not triggered
# before you've collected the results.
# Clear the results (if any), so it doesn't grow too large.
self._pool._results_queue.queue.clear()
return req
def on_ready(self, callbacks, errbacks, ret_value):
"""What to do when a worker task is ready and its return value has
been collected."""
if isinstance(ret_value, ExceptionInfo):
if isinstance(ret_value.exception,
(SystemExit, KeyboardInterrupt)): # pragma: no cover
raise ret_value.exception
[errback(ret_value) for errback in errbacks]
else:
[callback(ret_value) for callback in callbacks]
def gits_download(self, url, output="/tmp", threads=20):
if not self.output:
self.output = output
results = self.index(url, output=output)
if not results:
return
args = [((i[0], i[1]), {}) for i in self.giturls]
# ... and build a WorkRequest object for each item in data
requests = makeRequests(self.callback,
args,
self.print_result,
self.handle_exception)
main = ThreadPool(threads)
for req in requests:
main.putRequest(req)
print "Work request #%s added." % req.requestID
i = 0
while True:
try:
main.poll()
print "Main thread working...",
print "(active worker threads: %i)" % (
threading.activeCount()-1, )
if i == 10:
print "**** Adding 3 more worker threads..."
main.createWorkers(3)
if i == 20:
print "**** Dismissing 2 worker threads..."
main.dismissWorkers(2)
i += 1
except KeyboardInterrupt:
print "**** Interrupted!"
break
except NoResultsPending:
print "**** No pending results."
break
if main.dismissedWorkers:
print "Joining all dismissed worker threads..."
main.joinAllDismissedWorkers()
class TaskPool(object):
def __init__(self, limit, logger=None, **kwargs):
self.limit = limit
self.logger = logger or log.get_default_logger()
self._pool = None
def start(self):
self._pool = ThreadPool(self.limit)
def stop(self):
self._pool.dismissWorkers(self.limit, do_join=True)
def apply_async(self, target, args=None, kwargs=None, callbacks=None,
errbacks=None, accept_callback=None, **compat):
args = args or []
kwargs = kwargs or {}
callbacks = callbacks or []
errbacks = errbacks or []
on_ready = partial(self.on_ready, callbacks, errbacks)
self.logger.debug("ThreadPool: Apply %s (args:%s kwargs:%s)" % (
target, args, kwargs))
req = WorkRequest(do_work, (target, args, kwargs, on_ready,
accept_callback))
self._pool.putRequest(req)
# threadpool also has callback support,
# but for some reason the callback is not triggered
# before you've collected the results.
# Clear the results (if any), so it doesn't grow too large.
self._pool._results_queue.queue.clear()
return req
def on_ready(self, callbacks, errbacks, ret_value):
"""What to do when a worker task is ready and its return value has
been collected."""
if isinstance(ret_value, ExceptionInfo):
if isinstance(ret_value.exception, (
SystemExit, KeyboardInterrupt)): # pragma: no cover
raise ret_value.exception
[errback(ret_value) for errback in errbacks]
else:
[callback(ret_value) for callback in callbacks]
def run_example():
num_workers = 3
pool = ThreadPool(num_workers)
# This variable will tell us whether all threads worked or not. Stored in
# an object (list) otherwise the inner definition cannot modify it.
success = [True]
# The exception handler is not required, but if it's not used the error
# will be silent.
def exc_handler(work_request, exc_info):
mldb.log(traceback.format_tb(exc_info[2]))
exception_type = exc_info[0]
exception_message = exc_info[1]
mldb.log(str(exception_type) + ': ' + str(exception_message))
success[0] = False
# If there is an error, stop all threads as soon as possible
pool.dismissWorkers(num_workers, do_join=True)
# makeRequests takes, as a second argument, a list of tuples where the
# first element is *args and the second one is **kwargs and where each
# tuple represents a job to run.
#
# Here we schedule two jobs.
requests = makeRequests(some_func, [(['patate'], {}), (['orange'], {})],
exc_callback=exc_handler)
[pool.putRequest(req) for req in requests]
# pool.wait will raise an exception if an error occurs in an early jobs and
# more jobs need to be run. It's ok, if there is an error we want to stop
# anyway.
pool.wait()
# If the error occurred in one of the last jobs pool.wait will have worked
# so we need to check it anyway.
if not success[0]:
mldb.log("An error occured")
return
# It is important (MLDBFB-470) to properly dismiss the workers
pool.dismissWorkers(num_workers, do_join=True)
mldb.log("Out of main thread")
def run_example():
num_workers = 3
pool = ThreadPool(num_workers)
# This variable will tell us whether all threads worked or not. Stored in
# an object (list) otherwise the inner definition cannot modify it.
success = [True]
# The exception handler is not required, but if it's not used the error
# will be silent.
def exc_handler(work_request, exc_info):
mldb.log(traceback.format_tb(exc_info[2]))
exception_type = exc_info[0]
exception_message = exc_info[1]
mldb.log(str(exception_type) + ': ' + str(exception_message))
success[0] = False
# If there is an error, stop all threads as soon as possible
pool.dismissWorkers(num_workers, do_join=True)
# makeRequests takes, as a second argument, a list of tuples where the
# first element is *args and the second one is **kwargs and where each
# tuple represents a job to run.
#
# Here we schedule two jobs.
requests = makeRequests(some_func, [(['patate'], {}), (['orange'], {})],
exc_callback=exc_handler)
[pool.putRequest(req) for req in requests]
# pool.wait will raise an exception if an error occurs in an early jobs and
# more jobs need to be run. It's ok, if there is an error we want to stop
# anyway.
pool.wait()
# If the error occurred in one of the last jobs pool.wait will have worked
# so we need to check it anyway.
if not success[0]:
mldb.log("An error occured")
return
# It is important (MLDBFB-470) to properly dismiss the workers
pool.dismissWorkers(num_workers, do_join=True)
mldb.log("Out of main thread")
def _run(self, urls, url, threads, num):
pool = ThreadPool(threads)
params = []
if url:
for i in range(num):
params.append(url)
else:
l = len(urls)
for i in range(num):
params.append(urls[i % l])
begin = time.time()
requests = threadpool.makeRequests(self._get_url, params)
[pool.putRequest(req) for req in requests]
pool.wait()
pool.dismissWorkers(threads)
end = time.time()
logging.info("all cost %f s" % (end - begin))
self._deal_results()
pass
class TaskPool(BasePool):
def on_start(self):
self._pool = ThreadPool(self.limit)
def on_stop(self):
self._pool.dismissWorkers(self.limit, do_join=True)
def on_apply(self, target, args=None, kwargs=None, callback=None,
accept_callback=None, **_):
req = WorkRequest(apply_target, (target, args, kwargs, callback,
accept_callback))
self._pool.putRequest(req)
# threadpool also has callback support,
# but for some reason the callback is not triggered
# before you've collected the results.
# Clear the results (if any), so it doesn't grow too large.
self._pool._results_queue.queue.clear()
return req
def submitter(once=False,
rses=[],
mock=False,
process=0,
total_processes=1,
total_threads=1,
bulk=100,
group_bulk=1,
group_policy='rule',
fts_source_strategy='auto',
activities=None,
sleep_time=600,
max_sources=4,
retry_other_fts=False):
"""
Main loop to submit a new transfer primitive to a transfertool.
"""
logging.info('Transfer submitter starting - process (%i/%i) threads (%i)' %
(process, total_processes, total_threads))
try:
scheme = config_get('conveyor', 'scheme')
except NoOptionError:
scheme = None
try:
failover_scheme = config_get('conveyor', 'failover_scheme')
except NoOptionError:
failover_scheme = None
try:
timeout = config_get('conveyor', 'submit_timeout')
timeout = float(timeout)
except NoOptionError:
timeout = None
try:
bring_online = config_get('conveyor', 'bring_online')
except NoOptionError:
bring_online = 43200
try:
max_time_in_queue = {}
timelife_conf = config_get('conveyor', 'max_time_in_queue')
timelife_confs = timelife_conf.split(",")
for conf in timelife_confs:
act, timelife = conf.split(":")
max_time_in_queue[act.strip()] = int(timelife.strip())
except NoOptionError:
max_time_in_queue = {}
if 'default' not in max_time_in_queue:
max_time_in_queue['default'] = 168
logging.debug("Maximum time in queue for different activities: %s" %
max_time_in_queue)
executable = ' '.join(sys.argv)
hostname = socket.getfqdn()
pid = os.getpid()
hb_thread = threading.current_thread()
heartbeat.sanity_check(executable=executable, hostname=hostname)
hb = heartbeat.live(executable, hostname, pid, hb_thread)
logging.info(
'Transfer submitter started - process (%i/%i) threads (%i/%i) timeout (%s)'
% (process, total_processes, hb['assign_thread'], hb['nr_threads'],
timeout))
threadPool = ThreadPool(total_threads)
activity_next_exe_time = defaultdict(time.time)
while not graceful_stop.is_set():
try:
hb = heartbeat.live(executable,
hostname,
pid,
hb_thread,
older_than=3600)
if activities is None:
activities = [None]
if rses:
rse_ids = [rse['id'] for rse in rses]
else:
rse_ids = None
for activity in activities:
if activity_next_exe_time[activity] > time.time():
graceful_stop.wait(1)
continue
user_transfer = False
if activity in USER_ACTIVITY and USER_TRANSFERS in ['cms']:
logging.info("CMS user transfer activity")
user_transfer = True
logging.info(
"%s:%s Starting to get transfer transfers for %s" %
(process, hb['assign_thread'], activity))
ts = time.time()
transfers = __get_transfers(process=process,
total_processes=total_processes,
thread=hb['assign_thread'],
total_threads=hb['nr_threads'],
failover_schemes=failover_scheme,
limit=bulk,
activity=activity,
rses=rse_ids,
schemes=scheme,
mock=mock,
max_sources=max_sources,
bring_online=bring_online,
retry_other_fts=retry_other_fts)
record_timer(
'daemons.conveyor.transfer_submitter.get_transfers.per_transfer',
(time.time() - ts) * 1000 /
(len(transfers) if len(transfers) else 1))
record_counter(
'daemons.conveyor.transfer_submitter.get_transfers',
len(transfers))
record_timer(
'daemons.conveyor.transfer_submitter.get_transfers.transfers',
len(transfers))
logging.info(
"%s:%s Got %s transfers for %s" %
(process, hb['assign_thread'], len(transfers), activity))
# group transfers
logging.info("%s:%s Starting to group transfers for %s" %
(process, hb['assign_thread'], activity))
ts = time.time()
grouped_jobs = bulk_group_transfer(transfers, group_policy,
group_bulk,
fts_source_strategy,
max_time_in_queue)
record_timer(
'daemons.conveyor.transfer_submitter.bulk_group_transfer',
(time.time() - ts) * 1000 /
(len(transfers) if len(transfers) else 1))
logging.info("%s:%s Starting to submit transfers for %s" %
(process, hb['assign_thread'], activity))
for external_host in grouped_jobs:
if not user_transfer:
for job in grouped_jobs[external_host]:
# submit transfers
job_requests = makeRequests(
submit_transfer,
args_list=[((), {
'external_host': external_host,
'job': job,
'submitter': 'transfer_submitter',
'process': process,
'thread': hb['assign_thread'],
'timeout': timeout
})])
[
threadPool.putRequest(job_req)
for job_req in job_requests
]
else:
for user, jobs in grouped_jobs[
external_host].iteritems():
# submit transfers
for job in jobs:
job_requests = makeRequests(
submit_transfer,
args_list=[((), {
'external_host':
external_host,
'job':
job,
'submitter':
'transfer_submitter',
'process':
process,
'thread':
hb['assign_thread'],
'timeout':
timeout,
'user_transfer_job':
user_transfer
})])
[
threadPool.putRequest(job_req)
for job_req in job_requests
]
threadPool.wait()
if len(transfers) < group_bulk:
logging.info(
'%i:%i - only %s transfers for %s which is less than group bulk %s, sleep %s seconds'
% (process, hb['assign_thread'], len(transfers),
activity, group_bulk, sleep_time))
if activity_next_exe_time[activity] < time.time():
activity_next_exe_time[activity] = time.time(
) + sleep_time
except:
logging.critical(
'%s:%s %s' %
(process, hb['assign_thread'], traceback.format_exc()))
if once:
break
logging.info('%s:%s graceful stop requested' %
(process, hb['assign_thread']))
threadPool.dismissWorkers(total_threads, do_join=True)
heartbeat.die(executable, hostname, pid, hb_thread)
logging.info('%s:%s graceful stop done' % (process, hb['assign_thread']))
return
# [main.putRequest(req) for req in requests]
# ...and wait for the results to arrive in the result queue
# by using ThreadPool.wait(). This would block until results for
# all work requests have arrived:
# main.wait()
# instead we can poll for results while doing something else:
i = 0
while True:
try:
time.sleep(0.5)
main.poll()
print "Main thread working...",
print "(active worker threads: %i)" % (threading.activeCount()-1, )
if i == 10:
print "**** Adding 3 more worker threads..."
main.createWorkers(3)
if i == 20:
print "**** Dismissing 2 worker threads..."
main.dismissWorkers(2)
i += 1
except KeyboardInterrupt:
print "**** Interrupted!"
break
except NoResultsPending:
print "**** No pending results."
break
if main.dismissedWorkers:
print "Joining all dismissed worker threads..."
main.joinAllDismissedWorkers()
def poller(once=False,
process=0,
total_processes=1,
thread=0,
total_threads=1,
activities=None,
sleep_time=60,
fts_bulk=100,
db_bulk=1000,
older_than=60,
activity_shares=None):
"""
Main loop to check the status of a transfer primitive with a transfertool.
"""
try:
timeout = config_get('conveyor', 'poll_timeout')
timeout = float(timeout)
except NoOptionError:
timeout = None
logging.info(
'poller starting - process (%i/%i) thread (%i/%i) bulk (%i) timeout (%s)'
% (process, total_processes, thread, total_threads, db_bulk, timeout))
executable = ' '.join(sys.argv)
hostname = socket.getfqdn()
pid = os.getpid()
hb_thread = threading.current_thread()
heartbeat.sanity_check(executable=executable, hostname=hostname)
hb = heartbeat.live(executable, hostname, pid, hb_thread)
logging.info('poller started - process (%i/%i) thread (%i/%i) bulk (%i)' %
(process, total_processes, hb['assign_thread'],
hb['nr_threads'], db_bulk))
activity_next_exe_time = defaultdict(time.time)
threadPool = ThreadPool(total_threads)
while not graceful_stop.is_set():
try:
hb = heartbeat.live(executable,
hostname,
pid,
hb_thread,
older_than=3600)
logging.debug('poller - thread (%i/%i)' %
(hb['assign_thread'], hb['nr_threads']))
if activities is None:
activities = [None]
for activity in activities:
if activity_next_exe_time[activity] > time.time():
graceful_stop.wait(1)
continue
ts = time.time()
logging.debug(
'%i:%i - start to poll transfers older than %i seconds for activity %s'
% (process, hb['assign_thread'], older_than, activity))
transfs = transfer_core.get_next_transfers(
request_type=[
RequestType.TRANSFER, RequestType.STAGEIN,
RequestType.STAGEOUT
],
state=[RequestState.SUBMITTED],
limit=db_bulk,
older_than=datetime.datetime.utcnow() -
datetime.timedelta(seconds=older_than),
process=process,
total_processes=total_processes,
thread=hb['assign_thread'],
total_threads=hb['nr_threads'],
activity=activity,
activity_shares=activity_shares)
record_timer('daemons.conveyor.poller.000-get_next_transfers',
(time.time() - ts) * 1000)
if transfs:
logging.debug(
'%i:%i - polling %i transfers for activity %s' %
(process, hb['assign_thread'], len(transfs), activity))
xfers_ids = {}
for transf in transfs:
if not transf['external_host'] in xfers_ids:
xfers_ids[transf['external_host']] = []
xfers_ids[transf['external_host']].append(
transf['external_id'])
for external_host in xfers_ids:
for xfers in chunks(xfers_ids[external_host], fts_bulk):
# poll transfers
# xfer_requests = makeRequests(common.poll_transfers, args_list=[((external_host, xfers, process, thread), {})])
xfer_requests = makeRequests(poll_transfers,
args_list=[((), {
'external_host':
external_host,
'xfers':
xfers,
'process':
process,
'thread':
hb['assign_thread'],
'timeout':
timeout
})])
[
threadPool.putRequest(xfer_req)
for xfer_req in xfer_requests
]
threadPool.wait()
if len(transfs) < db_bulk / 2:
logging.info(
"%i:%i - only %s transfers for activity %s, which is less than half of the bulk %s, will sleep %s seconds"
% (process, hb['assign_thread'], len(transfs),
activity, db_bulk, sleep_time))
if activity_next_exe_time[activity] < time.time():
activity_next_exe_time[activity] = time.time(
) + sleep_time
except:
logging.critical(
"%i:%i - %s" %
(process, hb['assign_thread'], traceback.format_exc()))
if once:
break
logging.info('%i:%i - graceful stop requests' %
(process, hb['assign_thread']))
threadPool.dismissWorkers(total_threads, do_join=True)
heartbeat.die(executable, hostname, pid, hb_thread)
logging.info('%i:%i - graceful stop done' % (process, hb['assign_thread']))
