Exemple #1
0
class Invalidator(object):

    def __init__(self, interval):
        self.interval = interval
        self._timers = []
        self.queue = PriorityQueue()
        self.repeater = Repeater(self.interval, self._worker)
        atexit.register(self.stop)
        self.repeater.start()

    def stop(self):
        self.repeater.stop()
        self.queue.join()

    def request(self, qst, instances):
        """Record a lookup request."""
        self.queue.put((time.time(), (qst, set(i.id for i in instances))), False)

    def _worker(self):
        try:
            _, item = self.queue.get(False)
        except Queue.Empty:
            return
        qst, old_instances = item
        instances = lookup_instance_by_name(qst.qinfo.qname_str)
        if set(i.id for i in instances) != old_instances:
            invalidateQueryInCache(qst, qst.qinfo)
        else:
            self.queue.put((time.time(), (qst, old_instances)), False)
        self.queue.task_done()
def main(min=3, max=6, num_worker_threads=40):
    t0 = now()
    
    names_queue = PriorityQueue()
    available_names_queue = Queue()
    
    try:
        # Start worker threads to check username availability.
        for i in xrange(num_worker_threads):
             t = Thread(target=partial(check_availability, names_queue, available_names_queue))
             t.daemon = True
             t.start()
        # Start thread to print available usernames synchronously
        t = Thread(target=partial(print_available_names, available_names_queue))
        t.daemon = True
        t.start()
        
        for name in gen_names(min, max):
            names_queue.put(name)
        
        # Wait without blocking to let user cancel with ctrl-c
        while not names_queue.empty():
            sleep(0.1)
        
        # Block waiting threads processing the last names.
        names_queue.join()
    except KeyboardInterrupt:
        # Will work only before call to join.
        sys.exit(1)
    finally:
        # Block waiting to print all known available names up to now.
        available_names_queue.join()
        print "\nFinished in %.2f seconds." % (now() - t0)
        save_cache()
Exemple #3
0
	def update(self):
		#Queues to manage downloading
		dateQ = Queue(0)
		xmlTrialQ = PriorityQueue(100)

		# Create the single SQLAlchemy thread to update DB records
		sqlaThread = threading.Thread(target=self.updateTrials, args=(dateQ, xmlTrialQ))
		sqlaThread.daemon = True
		sqlaThread.start()

		startDate = self.fetchStartDate()
		delta = dt.date.today() - startDate
		dateList = [startDate + dt.timedelta(days=x) for x in range(0, delta.days)]

		print dateList

		# Add relevant dates to the list to be fetched
		for currentDate in dateList:
			dateQ.put(currentDate)

		print dateQ.queue

		numWorkerThreads = min(10, dateQ.qsize())

		# Create n worker threads to fetch the zip files of updated content and produce xmlTrial objects
		for i in range(numWorkerThreads):
			t = threading.Thread(target=self.fetchUpdatedTrials, args=(dateQ, xmlTrialQ))
			t.daemon = True
			t.start()

		dateQ.join() 	 # block until all dates have been fetched
		xmlTrialQ.join() # block until all xmlTrials have been created
Exemple #4
0
class ThreadPool:
    """Pool of threads consuming tasks from a queue"""
    def __init__(self, num_threads, name="ThreadPool"):
        self.name = name
        self.tasks = PriorityQueue(num_threads)
        self.workers = [self._add_worker(i) for i in range(num_threads)]

    def _add_worker(self, index):
        name = "{0}-Worker-{1}".format(self.name, index)
        return Worker(self.tasks, name)

    def add_task(self, priority, func, *args, **kwds):
        """Add a task to the queue. Blocks if all threads are busy."""
        self.tasks.put(Task(priority, func, args, kwds))

    def _join_with_timeout(self, timeout):
        """Workaround for the fact Queue.join() doesn't support timeouts"""
        tasks = self.tasks
        tasks.all_tasks_done.acquire()
        try:
            endtime = time() + timeout
            while tasks.unfinished_tasks:
                remaining = endtime - time()
                if remaining <= 0.0:
                    raise PendingTasks
                tasks.all_tasks_done.wait(remaining)
        finally:
            tasks.all_tasks_done.release()

    def wait_for_tasks(self, timeout=None):
        """Wait for completion of all the tasks in the queue"""
        if timeout is None:
            self.tasks.join()
        elif timeout < 0:
            raise ValueError("'timeout' must be a positive number")
        else:
            self._join_with_timeout(timeout)
Exemple #5
0
class ThreadPool:

    ###
    #   __init__
    #
    #   num_threads: total number of threads
    #   capacity: the max capacity of queue, if -1 then default capacity = num_threads * 3
    #   log: log file path
    #   name: ThreadPool ID in log file
    #
    def __init__(self, num_threads, capacity=-1, log=None):
        self.workers = []
        if capacity == -1:
            capacity = num_threads * 3
        self.tasks = PriorityQueue(capacity)

        for n in range(num_threads):

            self.workers.append(Worker(self.tasks, log=log))

    ###
    #   add_task
    #
    #   return: task handle
    #
    def add_task(self, priority, func, callback, *args, **kargs):
        task = lowLevelTask(func, callback, *args, **kargs)
        self.tasks.put((priority, task))
        return task

    ###
    #   wait_completion
    #
    #   wait for all tasks done
    #
    def wait_completion(self):
        self.tasks.join()
Exemple #6
0
def main(min=3, max=6, num_worker_threads=40):
    t0 = now()

    names_queue = PriorityQueue()
    available_names_queue = Queue()

    try:
        # Start worker threads to check username availability.
        for i in xrange(num_worker_threads):
            t = Thread(target=partial(check_availability, names_queue,
                                      available_names_queue))
            t.daemon = True
            t.start()
        # Start thread to print available usernames synchronously
        t = Thread(
            target=partial(print_available_names, available_names_queue))
        t.daemon = True
        t.start()

        for name in gen_names(min, max):
            names_queue.put(name)

        # Wait without blocking to let user cancel with ctrl-c
        while not names_queue.empty():
            sleep(0.1)

        # Block waiting threads processing the last names.
        names_queue.join()
    except KeyboardInterrupt:
        # Will work only before call to join.
        sys.exit(1)
    finally:
        # Block waiting to print all known available names up to now.
        available_names_queue.join()
        print "\nFinished in %.2f seconds." % (now() - t0)
        save_cache()
Exemple #7
0
                    raise Exception
                print "Thread <%2s> processed item <%2s> in %.2f seconds. Active threads: %d." % (id, item, t, active_count())
            except:
                # I didn't do my job, so I let someone else do it
                q.put(item + 0.1)
            finally:
                q.task_done()
    return workfunc

q = PriorityQueue()
num_worker_threads = 5
for i in range(num_worker_threads):
     t = Thread(target=worker(i))
     t.daemon = True
     t.start()

source = range(20)
 
for item in source:
    q.put(item)

try:
    while not q.empty():
        sleep(0.1)
    q.join()       # block until all tasks are done
except KeyboardInterrupt:
    # Okay, so you want to stop? Will work only before call to join.
    raise SystemExit(1)
finally:
    print "All done in %.2f seconds." % (now() - t0)
Exemple #8
0
class DownloadManager(object):
    """Manages multiple donloadThreads.

        This class itself is intentionally 'not' multithreading conform. 
        So don't use it in multithreading environments unless you know 
        what you are doing""" 

    def __init__(self):
        self.priorityQueue = PriorityQueue()
        self.threads = []
        self.started = False
        self.lock = Lock()
        self.currentThreads = 0
        self.retry = False   #retries until the file is succesfully download
        self.results = utils.Struct(done=0,failed=0, failed_data=[],doneB=0,doneDL=0,time=time.time())
        log.info("Initialized download manager at %s"%(time.ctime()))

        #init (this can be configurable)
        self.startThreads = 2
        self.maxThreads = 4
        
    def _callback(self, status, **data):
        """This MUST be multithread secured"""
        self.lock.acquire()
        
        if status == DownloadThread.EXIT_STOP: 
            log.debug('Thread %s stoped was stopped.', data['thread'].name)
        elif status == DownloadThread.EXIT_DONE: 
            log.debug('%s done.', data['file'])
            self.results.done += 1
            if data['url'].find('file://')!=0 and 'size' in data.keys():
                self.results.doneB += (data['size'])
                self.results.doneDL += 1  # doneDL only counts files downloaded over the Internet,
                # unlike done which counts all files including local copies
        elif status == DownloadThread.EXIT_DOWNLOAD_ERR:
            log.warn('The download from %s failed.', data['url'])
            if self.retry:
                #reschedule this download
                log.warn('Error downloading %s, retrying..', data['url'])
                self.download(data['file'], data['url'])
            else:
                self.results.failed += 1
                self.results.failed_data.append(data)
                log.error('Could not download %s', data['url'])

        self.lock.release()

    def _addThread(self):
        #we have a limited number of threads don't exceed that
        if self.currentThreads >= self.maxThreads: return

        self.currentThreads += 1
        t = DownloadThread('DownloadThread-{0}'.format(len(self.threads)), self.priorityQueue, callback=self._callback)
        t.start()

        self.threads.append(t)


    def _removeThread(self):
        #we might want to wait a little but this guaranties we'll have at most 1 dead thread in the list
        self._cleanList()

        #nothing to remove!
        if self.currentThreads <= 0: return
        self.currentThreads -= 1

        #next token will cause a thread to be killed
        self.priorityQueue.put((DownloadThread.PRIO_HIGH, DownloadThread.STOP))
        
    def _cleanList(self):
        """Remove al dead threads. Return number of removed threads."""
        #
        #if not self.threads: return 0

        old = self.threads
        self.threads = filter(lambda t: t.isAlive(), old)
        return len(old) - len(self.threads)

    def start(self):
        if self.started: return

        for i in range(self.startThreads):
            self._addThread()

        self.started = True
    

    def stop(self, *force):
        threads = self.threads

        for i in range(len(self.threads)):
            self._removeThread()

        self.started = False
        if force:
            for t in threads: t.killProcess()

    def download(self, url, file, size=None, flags=0, **args):
        """Schedule the download of this file. It will be queued in the current working queue.
            This Methods returns immediately and does not guarantee the time at which the
            download starts. It guarantees it will be sceduled"""
        #log.debug('Adding %s..%s(%s)',url[:30], url[-30:], size)
        
        try:
            self.priorityQueue.put((DownloadThread.PRIO_NORMAL, (file, url, size, flags)))
        except:
            print "jfp exception thrown from priorityQueue.put() for",url
            print sys.exc_info()[:3]
            raise

    def manage(self, verbose=False, benchmark_callback=None):
        """Tells the DownloadManager to manage the download. This call blocks until every file is 
            tried to be downloaded at least once or, if retry is activated, until every file is succesfully
            downloaded (this implies it might never return if, e.g. the server is taken out of producion)
            The current thread will be used for managing which involves, creating Threads and removing them as required.
            bechmark_callback:= call back function(speed=byte/s, files_done=#file_finished, threads=#active_threads)"""

        last_speed = 0.0001
        old_speeds = []
        checking = True
        try_new_thread = True
        last_time = time.time()
        improvement_index = 0
        threshold = 1<<17   #128k
        speeds = self._getSpeeds()

        while self.priorityQueue.qsize() > 0:
            try:
                #sleep a little
                time.sleep(2)

                speeds = self._getSpeeds()

                if not speeds and self.currentThreads: continue #No active threads. Nothing to do.
                
                total_speed = sum(speeds)
                old_speeds.append(total_speed)

                if verbose: self.showStatus(speeds)
                
                #pass benchmarks if a call back was defined
                if benchmark_callback: benchmark_callback(speed=total_speed, files_done=self.results.done, threads=len(speeds))
                log.debug("%s Active Threads at %.2f MB/s (%.2f Mbps). files done:%s, failed:%s, still:%s on %s",len(speeds), total_speed/1024/1024, total_speed*8/1000000, self.results.done, self.results.failed, self.priorityQueue.qsize(),time.ctime())

                if total_speed < threshold: continue #Something's wrong here, more threads will not help, that's for sure

                now = time.time()
                elapsed_time = now - last_time
                if self.currentThreads < self.maxThreads:
                    
                    #in this case we count the average from the first one 
                    total_speed = sum(old_speeds)/len(old_speeds)
                    #until we reach the threads limit we'll dynamically add threads 

                    if checking:
                        improvement_index = total_speed/last_speed
                        log.debug('index=%.2f, threads=%s', improvement_index, len(speeds))
                        if improvement_index > 1.2:
                            #this worked! reset everything 
                            try_new_thread = True
                        elif elapsed_time > 30:
                            #stop checking this is it, don't add any more threads
                            checking = False
                            if self.currentThreads > 1 and improvement_index < 0.8: 
                                #This is worse (definitely not better), so let's take down a thread
                                log.debug('Performance dropped, removing latest thread...')
                                self._removeThread()
                            else:
                                log.debug('No performance increase. Let\'s leave it here...')
                

                    if try_new_thread or elapsed_time > 300:
                        #reset time and start checking again
                        log.debug("Let's see if a new thread helps...")
                        last_speed = total_speed
                        self._addThread()
                        #start procedure
                        old_speeds = []
                        last_time = now
                        checking = True
                        try_new_thread = False
                elif elapsed_time > 600 and self.currentThreads > 1:
                    #all threads are created, let's end one and see what happens...
                    log.debug('Everythings fine... we need some chaos, let\'s drop a thread...')
                    last_time = now
                    self._removeThread()
                    checking = True
                    

    
            except (KeyboardInterrupt, SystemExit):
                #let us end the program if desired!
                #we are going down, take all threads with us!!
                self.stop(True)
                raise
            except:
                # code from http://code.activestate.com/recipes/52215/
                import traceback
                tb = sys.exc_info()[2]
                while 1:
                    if not tb.tb_next:
                        break
                    tb = tb.tb_next
                stack = []
                f = tb.tb_frame
                while f:
                    stack.append(f)
                    f = f.f_back
                stack.reverse()
                traceback.print_exc()
                log.error("Locals by frame, innermost last\n")
                for frame in stack:
                    log.error("Frame %s in %s at line %s", frame.f_code.co_name, frame.f_code.co_filename, frame.f_lineno)
                
        log.debug('The queue is empty, waiting for last threads to finish')
        #The queue is empty, now we only have to wait for the threads to finish
        self.priorityQueue.join()
        #We should check again that the queue is empty, in case the last threads connection broke....
        if verbose:  # added by jfp
            self.showStatus(speeds) #jfp
            timeelapsed = time.time() - self.results.time
            log.info("Completed download at %s"%(time.ctime()))
            log.info('downloaded %d files, %.1f MB in %d seconds'%\
                     ( self.results.doneDL, float(self.results.doneB)/1024/1024, timeelapsed ))
            if timeelapsed>0:
                avspeed = float(self.results.doneB)/timeelapsed
                log.info('Average speed %.1f MB/s, %d Mb/s'%( avspeed/1024/1024, avspeed*8/1000/1000 ))
        log.debug('Last thread reported back. Good bye.')

            
    def _getSpeeds(self):
        """Returns an array of the mean speeds (byte/s) of all active threads, no particular order though"""
        return [ t.getThroughput() for t in filter(lambda t: t.isActive(), self.threads)]
            
        
    def showStatus(self, *speed ):
        if not speed: speed = self._getSpeeds()
        else: speed = speed[0]

        total = sum(speed)
        if speed: avg = total/len(speed)
        else: avg = 0
            
        #jfp added failed statisitic
        print ("{0} Active Threads at {1:.2f} MB/s ({2:.2f} Mbps)."+\
               " files done:{3}, failed:{4}, still:{5} on {6}").\
              format(len(speed), total/1024/1024, total*8/1000000,\
                     self.results.done, self.results.failed, self.priorityQueue.qsize(),time.ctime())
Exemple #9
0
        q.put( calc )

    for dimer in itertools.combinations(monomers, 2):
        description = "{m1}-{m2}".format(m1=dimer[0].description, m2=dimer[1].description)
        td = 0
        td += distanceBetween(dimer[0], dimer[1])
        priority = int(td)
        calc = GamessCalculation( priority, description, nodes=4, ppn=8, qm=dimer )
        q.put( calc )

    for trimer in itertools.combinations(monomers, 3):
        description = "{m1}-{m2}-{m3}".format(m1=trimer[0].description, m2=trimer[1].description, m3=trimer[2].description)
        td = 0
        td += distanceBetween(trimer[0], trimer[1])
        td += distanceBetween(trimer[0], trimer[2])
        td += distanceBetween(trimer[1], trimer[2])
        priority = int(td)
        calc = GamessCalculation( priority, description, nodes=4, ppn=8, qm=trimer )
        q.put( calc )

    print "{0} total calculations".format(q.qsize())

    # initialize threads & launch calculations
    initializeThreads(thread_count)
    q.join()

    finish = time.time()
    print "walltime = {time}".format(time=(finish-start))
    print "nodeq.qsize() = {0}".format(nodeq.qsize())

Exemple #10
0
class PGoApi:
    def __init__(self, signature_lib_path, hash_lib_path, hash_key):
        self.set_logger()

        self._signature_lib_path = signature_lib_path
        self._hash_lib_path = hash_lib_path
        self._work_queue = PriorityQueue()
        self._auth_queue = PriorityQueue()
        self._workers = []
        self._api_endpoint = 'https://pgorelease.nianticlabs.com/plfe/rpc'

        self._hashKey = hash_key

        self.log.info('%s v%s - %s', __title__, __version__, __copyright__)
        self.log.info('%s', __patchedBy__)

    def create_workers(self, num_workers):
        for i in xrange(num_workers):
            worker = PGoApiWorker(self._signature_lib_path,
                                  self._hash_lib_path, self._work_queue,
                                  self._auth_queue, self._hashKey)
            worker.daemon = True
            worker.start()
            self._workers.append(worker)

    def resize_workers(self, num_workers):
        workers_now = len(self._workers)
        if workers_now < num_workers:
            self.create_workers(num_workers - workers_now)
        elif workers_now > num_workers:
            for i in xrange(workers_now - num_workers):
                worker = self._workers.pop()
                worker.stop()

    def set_accounts(self, accounts):
        old_accounts = []
        new_accounts = []

        accounts_todo = {}
        for account in accounts:
            accounts_todo[account['username']] = accounts['password']

        while not self._auth_queue.empty():
            # Go through accounts in auth queue and only add those back
            # that we still want to use
            next_call, auth_provider = self._auth_queue.get()
            if auth_provider.username in accounts_todo:
                old_accounts.append((next_call, auth_provider))
                del accounts_todo[auth_provider.username]

        while old_accounts:
            self._auth_queue.put(old_accounts.pop())

        # Add new accounts
        for username, password in accounts_todo.iteritems():
            new_accounts.append({'username': username, 'password': password})
        add_accounts(new_accounts)

    def add_accounts(self, accounts):
        for account in accounts:
            username, password = account['username'], account['password']
            if not isinstance(username, six.string_types) or not isinstance(
                    password, six.string_types):
                raise AuthException(
                    "Username/password not correctly specified")

            provider = account.get('provider', 'ptc')
            if provider == 'ptc':
                auth_provider = AuthPtc(username, password)
            elif provider == 'google':
                auth_provider = AuthGoogle(username, password)
            else:
                raise AuthException(
                    "Invalid authentication provider - only ptc/google available."
                )

            self._auth_queue.put((time.time(), auth_provider))

    def set_logger(self, logger=None):
        self.log = logger or logging.getLogger(__name__)

    def get_api_endpoint(self):
        return self._api_endpoint

    def __getattr__(self, func):
        def function(**kwargs):
            name = func.upper()

            position = kwargs.pop('position')
            callback = kwargs.pop('callback')

            priority = kwargs.pop('priority') if 'priority' in kwargs else 10.0

            if kwargs:
                method = {RequestType.Value(name): kwargs}
                self.log.debug(
                    "Adding '%s' to RPC request including arguments", name)
                self.log.debug("Arguments of '%s': \n\r%s", name, kwargs)
            else:
                method = RequestType.Value(name)
                self.log.debug("Adding '%s' to RPC request", name)

            self.call_method(method, position, callback, priority)

        if func.upper() in RequestType.keys():
            return function
        else:
            raise AttributeError

    def call_method(self, method, position, callback, priority=10.0):
        self._work_queue.put((priority, method, position, callback))

    def empty_work_queue(self):
        while not self._work_queue.empty():
            try:
                self._work_queue.get(False)
                self._work_queue.task_done()
            except Queue.Empty:
                return

    def is_work_queue_empty(self):
        return self._work_queue.empty()

    def wait_until_done(self):
        self._work_queue.join()
Exemple #11
0
class WorkerService(ObjectSingleton):
	def __init__(self):
		self.serviceQ  = PriorityQueue()
		self.networkQ  = PriorityQueue()
		self.aclcheckQ = PriorityQueue()

		self.queues = {
				'service' : self.serviceQ,
				'network' : self.networkQ,
				'aclcheck': self.aclcheckQ
			}

	def stop(self):

		for (qname, queue) in self.queues.iteritems():
			size = queue.qsize()
			if size > 0:
				assert ltrace(TRACE_DAEMON, 'queue %s has %d items left: %s' % (
						qname, size, [
							str(item) for item in queue.get_nowait() ]))

		for q in self.queues.itervalues():
			q.put((-1, None))
	def background_service(self, prio=None):

		if prio is None:
			prio = priorities.NORMAL

		def wrap1(func):
			def wrap2(*a, **kw):
				self.serviceQ.put((prio, func, a, kw))
			return wrap2
		return wrap1
	def service_enqueue(self, prio, func, *args, **kwargs):
		self.serviceQ.put((prio, func, args, kwargs))
	def service_wait(self):
		if isinstance(current_thread(), ServiceWorkerThread):
			raise RuntimeError(_(u'Cannot join the serviceQ from '
				u'a ServiceWorkerThread instance, this would deadblock!'))
		self.serviceQ.join()
	def background_network(self, prio=None):

		if prio is None:
			prio = priorities.LOW

		def wrap1(func):
			def wrap2(*args, **kwargs):
				self.networkQ.put((prio, func, args, kwargs))
			return wrap2
		return wrap1
	def network_enqueue(self, prio, func, *args, **kwargs):
		self.networkQ.put((prio, func, args, kwargs))
	def network_wait(self):
		if isinstance(current_thread(), NetworkWorkerThread):
			raise RuntimeError(_(u'Cannot join the networkQ from '
				u'a NetworkWorkerThread instance, this would deadblock!'))
		self.__queues.networkQ.join()
	def background_aclcheck(self, prio=None):

		if prio is None:
			prio = priorities.HIGH

		def wrap1(func):
			def wrap2(*args, **kwargs):
				self.aclcheckQ.put((prio, func, args, kwargs))
			return wrap2
		return wrap1
	def aclcheck_enqueue(self, prio, func, *args, **kwargs):
		self.aclcheckQ.put((prio, func, args, kwargs))
	def aclcheck_wait(self):
		if isinstance(current_thread(), ACLCkeckerThread):
			raise RuntimeError(_(u'Cannot join the ackcheckerQ from '
				u'a ACLCkeckerThread instance, this would deadblock!'))
		self.aclcheckQ.join()
Exemple #12
0
                                 nodes=4,
                                 ppn=8,
                                 qm=dimer)
        q.put(calc)

    for trimer in itertools.combinations(monomers, 3):
        description = "{m1}-{m2}-{m3}".format(m1=trimer[0].description,
                                              m2=trimer[1].description,
                                              m3=trimer[2].description)
        td = 0
        td += distanceBetween(trimer[0], trimer[1])
        td += distanceBetween(trimer[0], trimer[2])
        td += distanceBetween(trimer[1], trimer[2])
        priority = int(td)
        calc = GamessCalculation(priority,
                                 description,
                                 nodes=4,
                                 ppn=8,
                                 qm=trimer)
        q.put(calc)

    print "{0} total calculations".format(q.qsize())

    # initialize threads & launch calculations
    initializeThreads(thread_count)
    q.join()

    finish = time.time()
    print "walltime = {time}".format(time=(finish - start))
    print "nodeq.qsize() = {0}".format(nodeq.qsize())