class Invalidator(object):
def __init__(self, interval):
self.interval = interval
self._timers = []
self.queue = PriorityQueue()
self.repeater = Repeater(self.interval, self._worker)
atexit.register(self.stop)
self.repeater.start()
def stop(self):
self.repeater.stop()
self.queue.join()
def request(self, qst, instances):
"""Record a lookup request."""
self.queue.put((time.time(), (qst, set(i.id for i in instances))), False)
def _worker(self):
try:
_, item = self.queue.get(False)
except Queue.Empty:
return
qst, old_instances = item
instances = lookup_instance_by_name(qst.qinfo.qname_str)
if set(i.id for i in instances) != old_instances:
invalidateQueryInCache(qst, qst.qinfo)
else:
self.queue.put((time.time(), (qst, old_instances)), False)
self.queue.task_done()
def main(min=3, max=6, num_worker_threads=40):
t0 = now()
names_queue = PriorityQueue()
available_names_queue = Queue()
try:
# Start worker threads to check username availability.
for i in xrange(num_worker_threads):
t = Thread(target=partial(check_availability, names_queue, available_names_queue))
t.daemon = True
t.start()
# Start thread to print available usernames synchronously
t = Thread(target=partial(print_available_names, available_names_queue))
t.daemon = True
t.start()
for name in gen_names(min, max):
names_queue.put(name)
# Wait without blocking to let user cancel with ctrl-c
while not names_queue.empty():
sleep(0.1)
# Block waiting threads processing the last names.
names_queue.join()
except KeyboardInterrupt:
# Will work only before call to join.
sys.exit(1)
finally:
# Block waiting to print all known available names up to now.
available_names_queue.join()
print "\nFinished in %.2f seconds." % (now() - t0)
save_cache()
def update(self): #Queues to manage downloading dateQ = Queue(0) xmlTrialQ = PriorityQueue(100) # Create the single SQLAlchemy thread to update DB records sqlaThread = threading.Thread(target=self.updateTrials, args=(dateQ, xmlTrialQ)) sqlaThread.daemon = True sqlaThread.start() startDate = self.fetchStartDate() delta = dt.date.today() - startDate dateList = [startDate + dt.timedelta(days=x) for x in range(0, delta.days)] print dateList # Add relevant dates to the list to be fetched for currentDate in dateList: dateQ.put(currentDate) print dateQ.queue numWorkerThreads = min(10, dateQ.qsize()) # Create n worker threads to fetch the zip files of updated content and produce xmlTrial objects for i in range(numWorkerThreads): t = threading.Thread(target=self.fetchUpdatedTrials, args=(dateQ, xmlTrialQ)) t.daemon = True t.start() dateQ.join() # block until all dates have been fetched xmlTrialQ.join() # block until all xmlTrials have been created
class ThreadPool:
"""Pool of threads consuming tasks from a queue"""
def __init__(self, num_threads, name="ThreadPool"):
self.name = name
self.tasks = PriorityQueue(num_threads)
self.workers = [self._add_worker(i) for i in range(num_threads)]
def _add_worker(self, index):
name = "{0}-Worker-{1}".format(self.name, index)
return Worker(self.tasks, name)
def add_task(self, priority, func, *args, **kwds):
"""Add a task to the queue. Blocks if all threads are busy."""
self.tasks.put(Task(priority, func, args, kwds))
def _join_with_timeout(self, timeout):
"""Workaround for the fact Queue.join() doesn't support timeouts"""
tasks = self.tasks
tasks.all_tasks_done.acquire()
try:
endtime = time() + timeout
while tasks.unfinished_tasks:
remaining = endtime - time()
if remaining <= 0.0:
raise PendingTasks
tasks.all_tasks_done.wait(remaining)
finally:
tasks.all_tasks_done.release()
def wait_for_tasks(self, timeout=None):
"""Wait for completion of all the tasks in the queue"""
if timeout is None:
self.tasks.join()
elif timeout < 0:
raise ValueError("'timeout' must be a positive number")
else:
self._join_with_timeout(timeout)
class ThreadPool:
###
# __init__
#
# num_threads: total number of threads
# capacity: the max capacity of queue, if -1 then default capacity = num_threads * 3
# log: log file path
# name: ThreadPool ID in log file
#
def __init__(self, num_threads, capacity=-1, log=None):
self.workers = []
if capacity == -1:
capacity = num_threads * 3
self.tasks = PriorityQueue(capacity)
for n in range(num_threads):
self.workers.append(Worker(self.tasks, log=log))
###
# add_task
#
# return: task handle
#
def add_task(self, priority, func, callback, *args, **kargs):
task = lowLevelTask(func, callback, *args, **kargs)
self.tasks.put((priority, task))
return task
###
# wait_completion
#
# wait for all tasks done
#
def wait_completion(self):
self.tasks.join()
def main(min=3, max=6, num_worker_threads=40):
t0 = now()
names_queue = PriorityQueue()
available_names_queue = Queue()
try:
# Start worker threads to check username availability.
for i in xrange(num_worker_threads):
t = Thread(target=partial(check_availability, names_queue,
available_names_queue))
t.daemon = True
t.start()
# Start thread to print available usernames synchronously
t = Thread(
target=partial(print_available_names, available_names_queue))
t.daemon = True
t.start()
for name in gen_names(min, max):
names_queue.put(name)
# Wait without blocking to let user cancel with ctrl-c
while not names_queue.empty():
sleep(0.1)
# Block waiting threads processing the last names.
names_queue.join()
except KeyboardInterrupt:
# Will work only before call to join.
sys.exit(1)
finally:
# Block waiting to print all known available names up to now.
available_names_queue.join()
print "\nFinished in %.2f seconds." % (now() - t0)
save_cache()
raise Exception
print "Thread <%2s> processed item <%2s> in %.2f seconds. Active threads: %d." % (id, item, t, active_count())
except:
# I didn't do my job, so I let someone else do it
q.put(item + 0.1)
finally:
q.task_done()
return workfunc
q = PriorityQueue()
num_worker_threads = 5
for i in range(num_worker_threads):
t = Thread(target=worker(i))
t.daemon = True
t.start()
source = range(20)
for item in source:
q.put(item)
try:
while not q.empty():
sleep(0.1)
q.join() # block until all tasks are done
except KeyboardInterrupt:
# Okay, so you want to stop? Will work only before call to join.
raise SystemExit(1)
finally:
print "All done in %.2f seconds." % (now() - t0)
class DownloadManager(object):
"""Manages multiple donloadThreads.
This class itself is intentionally 'not' multithreading conform.
So don't use it in multithreading environments unless you know
what you are doing"""
def __init__(self):
self.priorityQueue = PriorityQueue()
self.threads = []
self.started = False
self.lock = Lock()
self.currentThreads = 0
self.retry = False #retries until the file is succesfully download
self.results = utils.Struct(done=0,failed=0, failed_data=[],doneB=0,doneDL=0,time=time.time())
log.info("Initialized download manager at %s"%(time.ctime()))
#init (this can be configurable)
self.startThreads = 2
self.maxThreads = 4
def _callback(self, status, **data):
"""This MUST be multithread secured"""
self.lock.acquire()
if status == DownloadThread.EXIT_STOP:
log.debug('Thread %s stoped was stopped.', data['thread'].name)
elif status == DownloadThread.EXIT_DONE:
log.debug('%s done.', data['file'])
self.results.done += 1
if data['url'].find('file://')!=0 and 'size' in data.keys():
self.results.doneB += (data['size'])
self.results.doneDL += 1 # doneDL only counts files downloaded over the Internet,
# unlike done which counts all files including local copies
elif status == DownloadThread.EXIT_DOWNLOAD_ERR:
log.warn('The download from %s failed.', data['url'])
if self.retry:
#reschedule this download
log.warn('Error downloading %s, retrying..', data['url'])
self.download(data['file'], data['url'])
else:
self.results.failed += 1
self.results.failed_data.append(data)
log.error('Could not download %s', data['url'])
self.lock.release()
def _addThread(self):
#we have a limited number of threads don't exceed that
if self.currentThreads >= self.maxThreads: return
self.currentThreads += 1
t = DownloadThread('DownloadThread-{0}'.format(len(self.threads)), self.priorityQueue, callback=self._callback)
t.start()
self.threads.append(t)
def _removeThread(self):
#we might want to wait a little but this guaranties we'll have at most 1 dead thread in the list
self._cleanList()
#nothing to remove!
if self.currentThreads <= 0: return
self.currentThreads -= 1
#next token will cause a thread to be killed
self.priorityQueue.put((DownloadThread.PRIO_HIGH, DownloadThread.STOP))
def _cleanList(self):
"""Remove al dead threads. Return number of removed threads."""
#
#if not self.threads: return 0
old = self.threads
self.threads = filter(lambda t: t.isAlive(), old)
return len(old) - len(self.threads)
def start(self):
if self.started: return
for i in range(self.startThreads):
self._addThread()
self.started = True
def stop(self, *force):
threads = self.threads
for i in range(len(self.threads)):
self._removeThread()
self.started = False
if force:
for t in threads: t.killProcess()
def download(self, url, file, size=None, flags=0, **args):
"""Schedule the download of this file. It will be queued in the current working queue.
This Methods returns immediately and does not guarantee the time at which the
download starts. It guarantees it will be sceduled"""
#log.debug('Adding %s..%s(%s)',url[:30], url[-30:], size)
try:
self.priorityQueue.put((DownloadThread.PRIO_NORMAL, (file, url, size, flags)))
except:
print "jfp exception thrown from priorityQueue.put() for",url
print sys.exc_info()[:3]
raise
def manage(self, verbose=False, benchmark_callback=None):
"""Tells the DownloadManager to manage the download. This call blocks until every file is
tried to be downloaded at least once or, if retry is activated, until every file is succesfully
downloaded (this implies it might never return if, e.g. the server is taken out of producion)
The current thread will be used for managing which involves, creating Threads and removing them as required.
bechmark_callback:= call back function(speed=byte/s, files_done=#file_finished, threads=#active_threads)"""
last_speed = 0.0001
old_speeds = []
checking = True
try_new_thread = True
last_time = time.time()
improvement_index = 0
threshold = 1<<17 #128k
speeds = self._getSpeeds()
while self.priorityQueue.qsize() > 0:
try:
#sleep a little
time.sleep(2)
speeds = self._getSpeeds()
if not speeds and self.currentThreads: continue #No active threads. Nothing to do.
total_speed = sum(speeds)
old_speeds.append(total_speed)
if verbose: self.showStatus(speeds)
#pass benchmarks if a call back was defined
if benchmark_callback: benchmark_callback(speed=total_speed, files_done=self.results.done, threads=len(speeds))
log.debug("%s Active Threads at %.2f MB/s (%.2f Mbps). files done:%s, failed:%s, still:%s on %s",len(speeds), total_speed/1024/1024, total_speed*8/1000000, self.results.done, self.results.failed, self.priorityQueue.qsize(),time.ctime())
if total_speed < threshold: continue #Something's wrong here, more threads will not help, that's for sure
now = time.time()
elapsed_time = now - last_time
if self.currentThreads < self.maxThreads:
#in this case we count the average from the first one
total_speed = sum(old_speeds)/len(old_speeds)
#until we reach the threads limit we'll dynamically add threads
if checking:
improvement_index = total_speed/last_speed
log.debug('index=%.2f, threads=%s', improvement_index, len(speeds))
if improvement_index > 1.2:
#this worked! reset everything
try_new_thread = True
elif elapsed_time > 30:
#stop checking this is it, don't add any more threads
checking = False
if self.currentThreads > 1 and improvement_index < 0.8:
#This is worse (definitely not better), so let's take down a thread
log.debug('Performance dropped, removing latest thread...')
self._removeThread()
else:
log.debug('No performance increase. Let\'s leave it here...')
if try_new_thread or elapsed_time > 300:
#reset time and start checking again
log.debug("Let's see if a new thread helps...")
last_speed = total_speed
self._addThread()
#start procedure
old_speeds = []
last_time = now
checking = True
try_new_thread = False
elif elapsed_time > 600 and self.currentThreads > 1:
#all threads are created, let's end one and see what happens...
log.debug('Everythings fine... we need some chaos, let\'s drop a thread...')
last_time = now
self._removeThread()
checking = True
except (KeyboardInterrupt, SystemExit):
#let us end the program if desired!
#we are going down, take all threads with us!!
self.stop(True)
raise
except:
# code from http://code.activestate.com/recipes/52215/
import traceback
tb = sys.exc_info()[2]
while 1:
if not tb.tb_next:
break
tb = tb.tb_next
stack = []
f = tb.tb_frame
while f:
stack.append(f)
f = f.f_back
stack.reverse()
traceback.print_exc()
log.error("Locals by frame, innermost last\n")
for frame in stack:
log.error("Frame %s in %s at line %s", frame.f_code.co_name, frame.f_code.co_filename, frame.f_lineno)
log.debug('The queue is empty, waiting for last threads to finish')
#The queue is empty, now we only have to wait for the threads to finish
self.priorityQueue.join()
#We should check again that the queue is empty, in case the last threads connection broke....
if verbose: # added by jfp
self.showStatus(speeds) #jfp
timeelapsed = time.time() - self.results.time
log.info("Completed download at %s"%(time.ctime()))
log.info('downloaded %d files, %.1f MB in %d seconds'%\
( self.results.doneDL, float(self.results.doneB)/1024/1024, timeelapsed ))
if timeelapsed>0:
avspeed = float(self.results.doneB)/timeelapsed
log.info('Average speed %.1f MB/s, %d Mb/s'%( avspeed/1024/1024, avspeed*8/1000/1000 ))
log.debug('Last thread reported back. Good bye.')
def _getSpeeds(self):
"""Returns an array of the mean speeds (byte/s) of all active threads, no particular order though"""
return [ t.getThroughput() for t in filter(lambda t: t.isActive(), self.threads)]
def showStatus(self, *speed ):
if not speed: speed = self._getSpeeds()
else: speed = speed[0]
total = sum(speed)
if speed: avg = total/len(speed)
else: avg = 0
#jfp added failed statisitic
print ("{0} Active Threads at {1:.2f} MB/s ({2:.2f} Mbps)."+\
" files done:{3}, failed:{4}, still:{5} on {6}").\
format(len(speed), total/1024/1024, total*8/1000000,\
self.results.done, self.results.failed, self.priorityQueue.qsize(),time.ctime())
q.put( calc )
for dimer in itertools.combinations(monomers, 2):
description = "{m1}-{m2}".format(m1=dimer[0].description, m2=dimer[1].description)
td = 0
td += distanceBetween(dimer[0], dimer[1])
priority = int(td)
calc = GamessCalculation( priority, description, nodes=4, ppn=8, qm=dimer )
q.put( calc )
for trimer in itertools.combinations(monomers, 3):
description = "{m1}-{m2}-{m3}".format(m1=trimer[0].description, m2=trimer[1].description, m3=trimer[2].description)
td = 0
td += distanceBetween(trimer[0], trimer[1])
td += distanceBetween(trimer[0], trimer[2])
td += distanceBetween(trimer[1], trimer[2])
priority = int(td)
calc = GamessCalculation( priority, description, nodes=4, ppn=8, qm=trimer )
q.put( calc )
print "{0} total calculations".format(q.qsize())
# initialize threads & launch calculations
initializeThreads(thread_count)
q.join()
finish = time.time()
print "walltime = {time}".format(time=(finish-start))
print "nodeq.qsize() = {0}".format(nodeq.qsize())
class PGoApi:
def __init__(self, signature_lib_path, hash_lib_path, hash_key):
self.set_logger()
self._signature_lib_path = signature_lib_path
self._hash_lib_path = hash_lib_path
self._work_queue = PriorityQueue()
self._auth_queue = PriorityQueue()
self._workers = []
self._api_endpoint = 'https://pgorelease.nianticlabs.com/plfe/rpc'
self._hashKey = hash_key
self.log.info('%s v%s - %s', __title__, __version__, __copyright__)
self.log.info('%s', __patchedBy__)
def create_workers(self, num_workers):
for i in xrange(num_workers):
worker = PGoApiWorker(self._signature_lib_path,
self._hash_lib_path, self._work_queue,
self._auth_queue, self._hashKey)
worker.daemon = True
worker.start()
self._workers.append(worker)
def resize_workers(self, num_workers):
workers_now = len(self._workers)
if workers_now < num_workers:
self.create_workers(num_workers - workers_now)
elif workers_now > num_workers:
for i in xrange(workers_now - num_workers):
worker = self._workers.pop()
worker.stop()
def set_accounts(self, accounts):
old_accounts = []
new_accounts = []
accounts_todo = {}
for account in accounts:
accounts_todo[account['username']] = accounts['password']
while not self._auth_queue.empty():
# Go through accounts in auth queue and only add those back
# that we still want to use
next_call, auth_provider = self._auth_queue.get()
if auth_provider.username in accounts_todo:
old_accounts.append((next_call, auth_provider))
del accounts_todo[auth_provider.username]
while old_accounts:
self._auth_queue.put(old_accounts.pop())
# Add new accounts
for username, password in accounts_todo.iteritems():
new_accounts.append({'username': username, 'password': password})
add_accounts(new_accounts)
def add_accounts(self, accounts):
for account in accounts:
username, password = account['username'], account['password']
if not isinstance(username, six.string_types) or not isinstance(
password, six.string_types):
raise AuthException(
"Username/password not correctly specified")
provider = account.get('provider', 'ptc')
if provider == 'ptc':
auth_provider = AuthPtc(username, password)
elif provider == 'google':
auth_provider = AuthGoogle(username, password)
else:
raise AuthException(
"Invalid authentication provider - only ptc/google available."
)
self._auth_queue.put((time.time(), auth_provider))
def set_logger(self, logger=None):
self.log = logger or logging.getLogger(__name__)
def get_api_endpoint(self):
return self._api_endpoint
def __getattr__(self, func):
def function(**kwargs):
name = func.upper()
position = kwargs.pop('position')
callback = kwargs.pop('callback')
priority = kwargs.pop('priority') if 'priority' in kwargs else 10.0
if kwargs:
method = {RequestType.Value(name): kwargs}
self.log.debug(
"Adding '%s' to RPC request including arguments", name)
self.log.debug("Arguments of '%s': \n\r%s", name, kwargs)
else:
method = RequestType.Value(name)
self.log.debug("Adding '%s' to RPC request", name)
self.call_method(method, position, callback, priority)
if func.upper() in RequestType.keys():
return function
else:
raise AttributeError
def call_method(self, method, position, callback, priority=10.0):
self._work_queue.put((priority, method, position, callback))
def empty_work_queue(self):
while not self._work_queue.empty():
try:
self._work_queue.get(False)
self._work_queue.task_done()
except Queue.Empty:
return
def is_work_queue_empty(self):
return self._work_queue.empty()
def wait_until_done(self):
self._work_queue.join()
class WorkerService(ObjectSingleton):
def __init__(self):
self.serviceQ = PriorityQueue()
self.networkQ = PriorityQueue()
self.aclcheckQ = PriorityQueue()
self.queues = {
'service' : self.serviceQ,
'network' : self.networkQ,
'aclcheck': self.aclcheckQ
}
def stop(self):
for (qname, queue) in self.queues.iteritems():
size = queue.qsize()
if size > 0:
assert ltrace(TRACE_DAEMON, 'queue %s has %d items left: %s' % (
qname, size, [
str(item) for item in queue.get_nowait() ]))
for q in self.queues.itervalues():
q.put((-1, None))
def background_service(self, prio=None):
if prio is None:
prio = priorities.NORMAL
def wrap1(func):
def wrap2(*a, **kw):
self.serviceQ.put((prio, func, a, kw))
return wrap2
return wrap1
def service_enqueue(self, prio, func, *args, **kwargs):
self.serviceQ.put((prio, func, args, kwargs))
def service_wait(self):
if isinstance(current_thread(), ServiceWorkerThread):
raise RuntimeError(_(u'Cannot join the serviceQ from '
u'a ServiceWorkerThread instance, this would deadblock!'))
self.serviceQ.join()
def background_network(self, prio=None):
if prio is None:
prio = priorities.LOW
def wrap1(func):
def wrap2(*args, **kwargs):
self.networkQ.put((prio, func, args, kwargs))
return wrap2
return wrap1
def network_enqueue(self, prio, func, *args, **kwargs):
self.networkQ.put((prio, func, args, kwargs))
def network_wait(self):
if isinstance(current_thread(), NetworkWorkerThread):
raise RuntimeError(_(u'Cannot join the networkQ from '
u'a NetworkWorkerThread instance, this would deadblock!'))
self.__queues.networkQ.join()
def background_aclcheck(self, prio=None):
if prio is None:
prio = priorities.HIGH
def wrap1(func):
def wrap2(*args, **kwargs):
self.aclcheckQ.put((prio, func, args, kwargs))
return wrap2
return wrap1
def aclcheck_enqueue(self, prio, func, *args, **kwargs):
self.aclcheckQ.put((prio, func, args, kwargs))
def aclcheck_wait(self):
if isinstance(current_thread(), ACLCkeckerThread):
raise RuntimeError(_(u'Cannot join the ackcheckerQ from '
u'a ACLCkeckerThread instance, this would deadblock!'))
self.aclcheckQ.join()
nodes=4,
ppn=8,
qm=dimer)
q.put(calc)
for trimer in itertools.combinations(monomers, 3):
description = "{m1}-{m2}-{m3}".format(m1=trimer[0].description,
m2=trimer[1].description,
m3=trimer[2].description)
td = 0
td += distanceBetween(trimer[0], trimer[1])
td += distanceBetween(trimer[0], trimer[2])
td += distanceBetween(trimer[1], trimer[2])
priority = int(td)
calc = GamessCalculation(priority,
description,
nodes=4,
ppn=8,
qm=trimer)
q.put(calc)
print "{0} total calculations".format(q.qsize())
# initialize threads & launch calculations
initializeThreads(thread_count)
q.join()
finish = time.time()
print "walltime = {time}".format(time=(finish - start))
print "nodeq.qsize() = {0}".format(nodeq.qsize())