class RunThread(Thread): '''Very basic thread in which to run the sleep loop while the callbacks are running''' def __init__(self): super(RunThread,self).__init__() self._stop = Event() def stop(self): self._stop.set() def stopped(self): return self._stop.isSet() def run(self): second_count = 0 minute_count = 0 while True: time.sleep(1) # The callback will be firing all during this point if second_count % 60 == 0: print "{} minutes".format(minute_count) minute_count += 1 print ".", second_count += 1 if self._stop.isSet(): return
class MarkerThread(Thread): def __init__(self, da, zl, marker, coord, conf, pixDim): Thread.__init__(self) self.da = da self.update = Event() self.update.set() self.__stop = Event() self.zl = zl self.marker = marker self.coord = coord self.conf = conf self.pixDim = pixDim self.img = self.marker.get_marker_pixbuf(zl) def run(self): while not self.__stop.isSet(): self.update.wait() self.update.clear() self.draw_markers() def stop(self): self.__stop.set() self.update.set() def draw_markers(self): for string in self.marker.positions.keys(): if self.update.isSet() or self.__stop.isSet(): break mpos = self.marker.positions[string] if (self.zl <= mpos[2]) and (mpos[0], mpos[1]) != (self.coord[0], self.coord[1]): gtk.threads_enter() try: self.da.draw_marker(self.conf, mpos, self.zl, self.img, self.pixDim, string) finally: gtk.threads_leave()
class StreamWrapperHelper(Thread): def __init__(self, queue): Thread.__init__(self) self.__stop = Event() self.__queue = queue self.__full = Event() self.start() def run(self): while True: if self.__stop.isSet(): return stream = tweetstream.SampleStream(u'soldierkam', os.environ["PASSWORD"]) for s in stream: try: if self.__stop.isSet(): stream.close() return self.__queue.put(s, block=False) self.__full.clear() except Full: if not self.__full.isSet(): logger.warn("Queue is full!!") self.__full.set() def close(self): self.__stop.set()
class ChatReciever(Thread): def __init__(self, host, timeout=10): Thread.__init__(self) self.onMsg = [] self.host = host self.timeout = timeout self._stop = Event() self.start() def regHandler(self, type, func): if type == 'msg': self.onMsg.append(func) else: raise UnknownHandlerException() def run(self): lastmsg = 0 while 1: if self._stop.isSet(): break try: output = urllib2.urlopen(urllib2.Request(self.host+"/taigachat/list.json", urllib.urlencode({"lastrefresh":lastmsg})), timeout=self.timeout).read() new = bool(lastmsg) lastmsg = json.loads(output).get("lastrefresh") msglist = re.findall('<li.*?data-time="(.*?)".*?class="username" itemprop="name">(.*?)<\/a>.*?messagetext ugc\'>(.*?)</div> <\/li>', json.loads(output.replace("\\n","")).get("templateHtml")) for msg in msglist: for f in self.onMsg: f(msg[0], HTMLParser().unescape(re.sub('<[^>]*>', '', msg[1])), HTMLParser().unescape(re.sub('<[^>]*>', '', re.sub('<img.*?alt="(.*?)".*?>',r'\1', msg[2]))), new) except Exception as e: #print traceback.format_exc() sys.stderr.write('Error while getting messages: ' + str(e) + '\n') if self._stop.isSet(): break sleep(2) def stop(self): self._stop.set()
class PrinterThread(Thread): def __init__(self, stdout_queue, total_jobs): super(PrinterThread, self).__init__() self._stop = Event() self.stdout_queue = stdout_queue self.total_jobs = total_jobs def stop(self): self._stop.set() def stopped(self): return self._stop.isSet() def run(self): hash_to_counter = {} job_ids_done = set() while not self._stop.isSet(): (hash_, message) = self.stdout_queue.get() job_ids_done.add(hash_) if message is None: # job is done break else: print( "%s %s" % (colored( "[{jobs_done}/{total_jobs}]".format( jobs_done=len(job_ids_done), total_jobs=self.total_jobs), "magenta", attrs=["bold"]), message)) self.stdout_queue.task_done()
class CreateTurtleString(Thread): def __init__(self, string, rules, iterations, *args, **kwargs): super(CreateTurtleString, self).__init__(*args, **kwargs) self.string = string self.rules = rules self.iterations = iterations self._job_done = Event() self._stop_creating = Event() def job_done(self): return self._job_done.isSet() def stop_creating(self): self._stop_creating.set() def run(self): string = self.string iterations = self.iterations rules = self.rules for _ in range(iterations): if self._stop_creating.isSet(): break string = cached_expand_string(string, rules) self.string = string self._job_done.set()
def test_threadpool(): pool = ThreadPool(core_threads=2, keepalive=0) event1 = Event() event2 = Event() event3 = Event() pool.submit(event1.set) pool.submit(event2.set) pool.submit(event3.set) event1.wait(1) event2.wait(1) event3.wait(1) assert event1.isSet() assert event2.isSet() assert event3.isSet() sleep(0.3) eq_(repr(pool), '<ThreadPool at %x; threads=2/20>' % id(pool)) pool.shutdown() eq_(repr(pool), '<ThreadPool at %x; threads=0/20>' % id(pool)) # Make sure double shutdown is ok pool.shutdown() # Make sure one can't submit tasks to a thread pool that has been shut down assert_raises(RuntimeError, pool.submit, event1.set)
class FileSearchServer(Thread): """ Thread which answers to file/tag queries sent through unix socket. """ def __init__(self, pname='SET_ME_PLEASE'): Thread.__init__(self) self.name = "%s/%s" % ( pname, str(self.__class__).rsplit('.', 1)[1].split("'")[0]) # old socket from a crashed daemon ? # remove it, the ThreadingUnixStreamServer will create it. #if os.path.exists(socket_path): os.unlink(socket_path) self._stop_event = Event() self.server = ThreadingTCPServer(('127.0.0.1', searcher_port), FileSearchRequestHandler) self.server.allow_reuse_address = True # TODO: the socket is set to non-blocking to be able to gracefully terminate the thread, # but this could lead to CPU hogging problems. VERIFY THIS !! self.server.socket.setblocking(False) def run(self): logging.progress("%s: thread running." % (self.getName())) #os.chmod(socket_path, stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IWGRP|stat.S_IROTH|stat.S_IWOTH) while not self._stop_event.isSet(): self.server.handle_request() time.sleep(0.01) logging.progress("%s: thread ended." % (self.getName())) def stop(self): if not self._stop_event.isSet(): logging.progress("%s: stopping thread." % (self.getName())) self._stop_event.set() self.server.socket.close() self.server.server_close() if os.path.exists(socket_path): os.unlink(socket_path)
class AsyncTBWriter(): "Callback for GANLearners that writes to Tensorboard. Extends LearnerTensorboardWriter and adds output image writes." def __init__(self): super().__init__() self.stop_request = Event() self.queue = Queue() self.thread = Thread(target=self._queue_processor, daemon=True) self.thread.start() def request_write(self, request: TBWriteRequest)->None: "Queues up an asynchronous write request to Tensorboard." if self.stop_request.isSet(): return self.queue.put(request) def _queue_processor(self)->None: "Processes queued up write requests asynchronously to Tensorboard." while not self.stop_request.isSet(): while not self.queue.empty(): if self.stop_request.isSet(): return request = self.queue.get() request.write() sleep(0.2) #Provided this to stop thread explicitly or by context management (with statement) but thread should end on its own # upon program exit, due to being a daemon. So using this is probably unecessary. def close(self)->None: "Stops asynchronous request queue processing thread." self.stop_request.set() self.thread.join() # Nothing to do, thread already started. Could start thread here to enforce use of context manager # (but that sounds like a pain and a bit unweildy and unecessary for actual usage) def __enter__(self): pass def __exit__(self, exc_type, exc_value, traceback): self.close()
class Worker(Thread): """ 工作线程,取出线程池中的任务,执行任务中指定的函数,完成任务功能""" def __init__(self, task_queue, res_queue, poll_timeout=2, **kwds): Thread.__init__(self, **kwds) logger.debug('Worker Thread id :: %s', id(self)) self.setDaemon(True) self.task_queue = task_queue self.result_queue = res_queue self._poll_timeout = poll_timeout self._dismissed = Event() self.start() def run(self): """重复取出 task_queue 里的任务并执行,直到通知其退出。""" while True: if self._dismissed.isSet(): break try: task = self.task_queue.get( block=True, timeout=self._poll_timeout) except Queue.Empty: continue else: if self._dismissed.isSet(): # 线程退出,把请求放回 task_queue self.task_queue.put(task) break result = task.func(*task.args, **task.kwds) logger.debug('Worker Thread id :: %s' % id(self)) self.result_queue.put((task, result)) def dismiss(self): """当前任务结束后,退出线程""" self._dismissed.set()
def testBlocking(self): event = Event() getter = GetMessageProcess(self._queue, event) getter.start() time.sleep(1) self.assertFalse(event.isSet()) self._queue.append(MESSAGE) event.wait(5) self.assertTrue(event.isSet())
def testBlocking(self): event = Event() getter = GetItemProcess(self._queue, event) getter.start() time.sleep(1) self.assertFalse(event.isSet()) self._queue.append(TEST_ITEM) event.wait(5) self.assertTrue(event.isSet())
class StatusMonitor(Loggable): # valve_manager = None _stop_evt = None _clients = 0 state_freq = Int(3) lock_freq = Int(5) owner_freq = Int(5) update_period = Int(2) def start(self, vm): if not self._clients: if self._stop_evt: self._stop_evt.set() self._stop_evt.wait(0.25) self._stop_evt = Event() self._iter(1, vm) else: self.debug('Monitor already running') self._clients += 1 def isAlive(self): if self._stop_evt: return not self._stop_evt.isSet() def stop(self): self._clients -= 1 if not self._clients: self._stop_evt.set() self.debug('Status monitor stopped') else: self.debug('Alive clients {}'.format(self._clients)) def _iter(self, i, vm): if vm is None: self.debug('No valve manager') return if not i % self.state_freq: vm.load_valve_states() if not i % self.lock_freq: vm.load_valve_lock_states() if not i % self.owner_freq: vm.load_valve_owners() if i > 100: i = 0 if not self._stop_evt.isSet(): do_after(self.update_period * 1000, self._iter, i + 1, vm) #============= EOF =============================================
class AgentLogicBase: def __init__(self): self.wait_stop = Event() if (platform.system() == 'Windows') or (platform.system() == 'Microsoft'): vport_name = '\\\\.\\Global\\com.eayun.eayunstack.0' else: vport_name = '/dev/virtio-ports/com.eayun.eayunstack.0' self.vio = VirtIoChannel(vport_name) self.commandHandler = None def _send(self, name, arguments=None): self.vio.write(name, arguments or {}) def run(self): thread.start_new_thread(self.doListen, ()) while not self.wait_stop.isSet(): self.wait_stop.wait(1) def stop(self): self.wait_stop.set() def doListen(self): if self.commandHandler is None: return while not self.wait_stop.isSet(): try: cmd, args = self.vio.read() if cmd: self.parseCommand(cmd, args) except: pass def parseCommand(self, command, args): if command == 'get_infomation': name = args.get('name') result = self.commandHandler.get_infomation(name) self._send('get_infomation', {'result': result}) elif command == 'execute_script': path = args.get('path') type = args.get('type') result = self.commandHandler.execute_script(path, type) self._send('execute_script', {'result': result}) elif command == 'execute_command': cmd = args.get('cmd') try: result = self.commandHandler.execute_command(cmd) self._send('execute_command', {'result': result}) except: self._send('execute_command', {'result': '0e0r0r0o0r1'}) elif command == 'echo': self._send('echo', args) else: self._send(command, {'result': '0e0r0r0o0r0'})
class DrawTurtle(Thread): def __init__(self, canvas, turtle, colors={}, **kwargs): super(DrawTurtle, self).__init__(**kwargs) self._stop_drawing = Event() self.canvas = canvas self.turtle = turtle self.colors = colors def stop_drawing(self): self._stop_drawing.set() def run(self): self.canvas.begin_new() color = self.colors["1"] # Calculate size, and scale to fill the frame t_width = self.turtle.rightmost[0] - self.turtle.leftmost[0] t_height = self.turtle.bottommost[1] - self.turtle.topmost[1] c_width = int(self.canvas['width']) c_height = int(self.canvas['height']) if t_width / t_height > 1: # fat image scale according to width scale_factor = c_width / t_width else: scale_factor = c_height / t_height left_margin = (c_width - scale_factor*t_width) / 2 top_margin = (c_height - scale_factor*t_height) / 2 x_shift = left_margin - scale_factor*self.turtle.leftmost[0] y_shift = top_margin - scale_factor*self.turtle.topmost[1] coordinates = [] for item in self.turtle.lines: if self._stop_drawing.isSet(): return if isinstance(item, PlaceHolder): coordinates.append(item) else: coordinates.append((item[0]*scale_factor+x_shift, item[1]*scale_factor+y_shift, item[2]*scale_factor+x_shift, item[3]*scale_factor+y_shift)) for item in coordinates: if self._stop_drawing.isSet(): return if isinstance(item, PlaceHolder): # not a list of coordinates if item.value in self.colors: color = self.colors[item.value] else: self.canvas.queue_line(item, color)
class ChatUserList(Thread): def __init__(self, host, timeout=10): Thread.__init__(self) self.users = [] self.host = host self.timeout = timeout self.onJoin = [] self.onLeave = [] self.lock = False self._stop = Event() self.start() def regHandler(self, type, func): if type == 'join': self.onJoin.append(func) elif type == 'leave': self.onLeave.append(func) else: raise UnknownHandlerException() def run(self): while 1: if self._stop.isSet(): break try: usersload = json.loads(urllib2.urlopen(urllib2.Request(self.host+"/shoutbox/", urllib.urlencode({"_xfResponseType":"json"})), timeout=self.timeout).read()).get("sidebarHtml") usersload = usersload[usersload.rfind('<!-- end block: sidebar_online_users -->'):] users = re.findall('class="username">(.*?)</a>', usersload) users = map(lambda x: HTMLParser().unescape(re.sub('<[^>]*>', '', x)), users) if users != self.users and users != [] and self.users != []: self.lock = True for a in users: try: self.users.remove(a) except ValueError: for f in self.onJoin: f(a) for a in self.users: for f in self.onLeave: f(a) if users != []: self.users = users self.lock = False except Exception as e: #print traceback.format_exc() sys.stderr.write('Error while getting users list: ' + str(e) + '\n') if self._stop.isSet(): break sleep(5) def list(self): while self.lock or self.users == []: sleep(0.1) return self.users def stop(self): self._stop.set()
class Idler(object): def __init__(self, conn): self.thread = Thread(target=self.idle) self.M = conn self.event = Event() def work(self): try: for name, rec in RECIPES.items(): self.process(name, rec) except IMAP4_SSL.abort: pass def start(self): self.thread.start() def stop(self): self.event.set() def join(self): self.thread.join() def idle(self): while True: if self.event.isSet(): return self.needsync = False def callback(args): if not self.event.isSet(): self.needsync = True self.event.set() self.M.idle(callback=callback) self.event.wait() if self.needsync: self.event.clear() self.work() def process(self, label, recipe): self.M.select(FOLDER_LABEL+'/' + label) status, data = self.M.uid('search', None, 'UnSeen') if data[0]: for num in data[0].split(): status, data = self.M.uid('fetch', num, '(RFC822)') msg = parse(data[0][1]) skip_spf = any([True for s in SKIP_SPF if s in msg['From']]) if 'pass' in msg.get('Received-SPF', '') or skip_spf: print(recipe(msg)) else: from_fallback = msg.get('From', 'No sender') print("Didn't pass sender checks: %s" % from_fallback) self.M.uid('store', num, '+FLAGS', 'Seen') self.M.expunge()
class PasteGraber(Thread): def __init__(self, grabber_name, paste_url, query_url, links_xpath, target_fn, links_postprocess=None): self.grabber_name = grabber_name self.paste_url = paste_url self.query_url = query_url self.links_xpath = links_xpath self.target_fn = target_fn self.links_postprocess = links_postprocess self._stopevent = Event() self.cfg = Config() super(PasteGraber, self).__init__(name=grabber_name) def stop(self): self._stopevent.set() def stopped(self): return self._stopevent.isSet() def run(self): while not self._stopevent.isSet(): i = 0 listturples = list() try: data = lh.parse(self.paste_url) html = data.getroot() links = html.xpath(self.links_xpath) if self.links_postprocess is not None: links = map(self.links_postprocess, links) for l in links: if l in PASTESSEEN: i += 1 else: # paste/nopaste-%s-%s.txt" fn = self.target_fn % (datetime.today().strftime("%Y-%m-%d-%H:%M:%S"), l) que_url = self.query_url % l worker_turple = (que_url, fn) listturples.append(worker_turple) PASTESSEEN.add(l) except Exception, e: msg = "%s: Exception running scraper [%s]: %s %s" % ( self.grabber_name, str(e), e.message, e.args ) logger.error(msg) if i > 0: logger.warn ( "%s: Nber of paste already seen : %s" % (self.grabber_name, i) ) if self.cfg.pastegraber.debug: logger.debug("%s: list turples: %s " % (self.grabber_name, listturples)) PASTES.put(listturples) time.sleep(self.cfg.pastegraber.delay)
class BaseTimerClass(Thread): ''' This is the base class for timer (delay) based threads. Timer-based threads are ones that do not need to be looking at keyboard-mouse events to do their job. ''' def __init__(self, dir_lock, loggername, *args, **kwargs): Thread.__init__(self) # print "dir_lock:", dir_lock # print "loggername:", loggername self.finished = Event() self.dir_lock = dir_lock self.loggername = loggername self.args = args # arguments, if any, to pass to task_function self.kwargs = kwargs # keyword args, if any, to pass to task_function self.settings = _settings['settings'] self.cmdoptions = _cmdoptions['cmdoptions'] self.mainapp = _mainapp['mainapp'] # set this up for clarity self.subsettings = self.settings[loggername] # set these up here because we will usually need them. self.logger = logging.getLogger(self.loggername) # print "self.logger:", self.logger self.logfile_path = self.logger.handlers[0].stream.name self.log_full_dir = os.path.dirname(self.logfile_path) self.log_rel_dir = os.path.basename(self.log_full_dir) self.logfile_name = os.path.basename(self.logfile_path) self.interval = None # set this in derived class def cancel(self): ''' Stop the iteration ''' self.finished.set() def task_function(self): ''' to be overridden by derived classes ''' pass def run(self): while not self.finished.isSet(): self.finished.wait(self.interval) if not self.finished.isSet(): self.task_function(*self.args, **self.kwargs)
class StatusMonitor(Loggable): valve_manager = None _stop_evt = None _clients = 0 state_freq = 3 lock_freq = 5 owner_freq = 5 period = 2 def start(self): if not self._clients: if self._stop_evt: self._stop_evt.set() self._stop_evt.wait(0.25) self._stop_evt = Event() self._iter(1) else: self.debug('Monitor already running') self._clients += 1 def isAlive(self): if self._stop_evt: return not self._stop_evt.isSet() def stop(self): self._clients -= 1 if not self._clients: self._stop_evt.set() self.debug('Status monitor stopped') else: self.debug('Alive clients {}'.format(self._clients)) def _iter(self, i): vm = self.valve_manager if not i % self.state_freq: vm.load_valve_states() if not i % self.lock_freq: vm.load_valve_lock_states() if not i % self.owner_freq: vm.load_valve_owners() if i > 100: i = 0 if not self._stop_evt.isSet(): do_after(self.period * 1000, self._iter, i + 1)
class cardChecker(Thread): """ This class searches for a card with a given ATR and displays wether or not using a label and an image """ def __init__(self, lbl, img, atr, intervall=1): Thread.__init__(self) self._finished = Event() self._paused = Event() self.lbl = lbl self.img = img self.target_atr = atr self.intervall = intervall def _check_card(self): """ Actually this method should make use of pyscard, but I couldn't make it run on the OpenMoko yet. Therefor we call opensc-tool instead, which leads to higher delays """ proc = subprocess.Popen(["opensc-tool", "--atr"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) line = proc.stdout.readline().rstrip() if line == self.target_atr: gobject.idle_add(self.lbl.set_label, STR_CARD_FOUND) gobject.idle_add(self.img.set_from_file, IMAGES["apply"]) else: gobject.idle_add(self.lbl.set_label, STR_NO_CARD) gobject.idle_add(self.img.set_from_file, IMAGES["error"]) def run(self): """Main loop: Poll the card if the thread is not paused or finished""" while (True): if self._finished.isSet(): return if self._paused.isSet(): continue self._check_card() self._finished.wait(self.intervall) def stop(self): self._finished.set() def pause(self): self._paused.set() def resume(self): self._paused.clear()
class DeferredResponse( object ): """ A deferred that resolves to a response from TSServer. """ def __init__( self, timeout = RESPONSE_TIMEOUT_SECONDS ): self._event = Event() self._message = None self._timeout = timeout def resolve( self, message ): self._message = message self._event.set() def result( self ): self._event.wait( timeout = self._timeout ) if not self._event.isSet(): raise RuntimeError( 'Response Timeout' ) message = self._message if not message[ 'success' ]: raise RuntimeError( message[ 'message' ] ) if 'body' in message: return self._message[ 'body' ]
def TriggerEventWait( self, suffix, payload=None, prefix="Main", source=eg ): event = EventGhostEvent(suffix, payload, prefix, source) if event.source in self.filters: for filterFunc in self.filters[event.source]: if filterFunc(event) is True: return event executed = Event() def Execute(): try: event.Execute() finally: executed.set() def Transfer(): ActionThreadCall(Execute) event.SetShouldEnd() self.AppendAction(Transfer) executed.wait(5.0) if not executed.isSet(): eg.PrintWarningNotice( "timeout TriggerEventWait\n", traceback.format_stack() ) return event
class Queue(list): def __init__(self): super(Queue, self).__init__() self._lock = Lock() self._fill = Event() def put(self, obj): with self._lock: self.append(obj) self._fill.set() def get(self, block=True): with self._lock: if len(self) == 0: self._fill.clear() if not self._fill.isSet(): if block: self._fill.wait() else: return None with self._lock: return self.pop(0) def delete(self, index): if 0 <= index < len(self): with self._lock: del self[index] def remove(self, element): if element in self: with self._lock: del self[self.index(element)]
class Task(object): def __init__(self, name, start_time, calc_next_time, func): """ Initialize a Task. Arguments: name - Name of task. start_time - First time for task to run calc_next_time - Function to calculate the time of next run, gets one argument, the last run time as a datetime. Returns None when task should no longer be run func - A function to run """ self.name = name self.start_time = start_time self.scheduled_time = start_time self.calc_next_time = calc_next_time self.func = func self.halt_flag = Event() def run(self): logging.debug("Running %s task, scheduled at: %s" % (self.name, self.scheduled_time,)) if not self.halt_flag.isSet(): try: try: self.func() except: raise finally: self.scheduled_time = self.calc_next_time(self.scheduled_time) logging.debug("Scheduled next run of %s for: %s" % (self.name, self.scheduled_time,)) def halt(self): self.halt_flag.set()
class ResponseEvent: """Event which is fired when the response is returned for a request. For each request sent this event is created. An application can wait for the event to create a blocking request. """ def __init__(self): self.__evt = Event() def waiting(self): return not self.__evt.isSet() def waitForResponse(self, timeOut=None): """blocks until the response arrived or timeout is reached.""" self.__evt.wait(timeOut) if self.waiting(): raise Timeout() else: if self.response["error"]: raise Exception(self.response["error"]) else: return self.response["result"] def handleResponse(self, resp): self.response = resp self.__evt.set()
class ThreadedRunner(Runnable): def __init__(self, runnable): self._runnable = runnable self._notifier = Event() self._result = None self._error = None self._traceback = None self._thread = None def run(self): try: self._result = self._runnable() except: self._error, self._traceback = sys.exc_info()[1:] self._notifier.set() __call__ = run def run_in_thread(self, timeout): self._thread = Thread(self, name=TIMEOUT_THREAD_NAME) self._thread.setDaemon(True) self._thread.start() self._notifier.wait(timeout) return self._notifier.isSet() def get_result(self): if self._error: raise self._error, None, self._traceback return self._result def stop_thread(self): self._thread.stop()
class StoppableThreadWithResult(Thread): """Thread class with a stop() method. The thread itself has to check regularly for the stopped() condition.""" def __init__(self, group=None, target=None, name=None, args=(), kwargs=None, verbose=None): super(StoppableThreadWithResult, self).__init__(group=group, target=target, name=name, args=args, kwargs=kwargs, verbose=verbose) self._stop = Event() def stop(self): self._stop.set() self._Thread__stop() def stopped(self): return self._stop.isSet() def run(self): if self._Thread__target is not None: self._return = self._Thread__target(*self._Thread__args, **self._Thread__kwargs) def join(self, timeout=None): Thread.join(self, timeout=None) return self._return
class TimerWithResume(object): def __init__(self, status_subject, refresh_interval): self.status_subject = status_subject self.abort = Event() self.refresh_interval = refresh_interval def perform(self): while not self.abort.isSet(): self.status_subject.build_status() self.abort.wait(self.refresh_interval) def stop(self): self.abort.set() def start(self): self.thread = Thread(target=self.perform) self.thread.daemon = True self.thread.start() def resume(self): self.thread.join() self.abort.clear() self.start() def set_refresh_interval(self, new_interval): self.refresh_interval = new_interval
class ASyncDelayedResult(): def __init__(self, jobID=None): self.__result = None self.__exception = None self.__jobID = jobID self.isFinished = Event() def setResult(self, result): self.__result = result self.isFinished.set() def setException(self, exception, original_traceback): self.__original_traceback = original_traceback self.__exception = exception self.isFinished.set() def get(self, timeout = 100): if self.wait(timeout): if self.__exception: # exception was raised! self.__exception.originalTraceback = self.__original_traceback print >> sys.stderr, self.__original_traceback raise self.__exception return self.__result else: print_stack() print >> sys.stderr, "TIMEOUT on get", self.__jobID, timeout def wait(self, timeout = None): return self.isFinished.wait(timeout) or self.isFinished.isSet()
class MegaDownloader(Thread): def __init__(self, url, dest_path, callback): Thread.__init__(self) self.callback = callback self.progress = 0 self.path = dest_path if not self.path.endswith("/") and not self.path.endswith("\\"): self.path += SEP self.url = url self.schema = 'https' self.domain = 'mega.co.nz' self.timeout = 160 # max time (secs) to wait for resp from api requests self.sid = None self.sequence_num = random.randint(0, 0xFFFFFFFF) self.request_id = make_id(10) self.paused = False self.pause_cond = Condition(Lock()) self.stop_event = Event() pass def _api_request(self, data): params = {'id': self.sequence_num} self.sequence_num += 1 if self.sid: params.update({'sid': self.sid}) #ensure input data is a list if not isinstance(data, list): data = [data] req = requests.post('{0}://g.api.{1}/cs'.format( self.schema, self.domain), params=params, data=json.dumps(data), timeout=self.timeout) json_resp = json.loads(req.text) #if numeric error code response if isinstance(json_resp, int): raise RequestError(json_resp) return json_resp[0] def _parse_url(self, url): #parse file id and key from url if '!' in url: match = re.findall(r'/#!(.*)', url) path = match[0] return path else: raise RequestError('Url key missing') def run(self): while not self.stop_event.isSet(): path = self._parse_url(self.url).split('!') file_id = path[0] file_key = path[1] file_key = base64_to_a32(file_key) file_data = self._api_request({'a': 'g', 'g': 1, 'p': file_id}) k = (file_key[0] ^ file_key[4], file_key[1] ^ file_key[5], file_key[2] ^ file_key[6], file_key[3] ^ file_key[7]) iv = file_key[4:6] + (0, 0) meta_mac = file_key[6:8] # Seems to happens sometime... When this occurs, files are # inaccessible also in the official also in the official web app. # Strangely, files can come back later. if 'g' not in file_data: raise RequestError('File not accessible anymore') file_url = file_data['g'] file_size = file_data['s'] attribs = base64_url_decode(file_data['at']) attribs = decrypt_attr(attribs, k) file_name = attribs['n'] input_file = requests.get(file_url, stream=True).raw dest_path = self.path temp_output_file = tempfile.NamedTemporaryFile(mode='w+b', prefix='megapy_', delete=False) k_str = a32_to_str(k) counter = Counter.new(128, initial_value=((iv[0] << 32) + iv[1]) << 64) aes = AES.new(k_str, AES.MODE_CTR, counter=counter) mac_str = '\0' * 16 mac_encryptor = AES.new(k_str, AES.MODE_CBC, mac_str) iv_str = a32_to_str([iv[0], iv[1], iv[0], iv[1]]) for chunk_start, chunk_size in get_chunks(file_size): with lock: with self.pause_cond: while self.paused: self.pause_cond.wait() chunk = input_file.read(chunk_size) chunk = aes.decrypt(chunk) temp_output_file.write(chunk) encryptor = AES.new(k_str, AES.MODE_CBC, iv_str) for i in range(0, len(chunk) - 16, 16): block = chunk[i:i + 16] encryptor.encrypt(block) #fix for files under 16 bytes failing if file_size > 16: i += 16 else: i = 0 block = chunk[i:i + 16] if len(block) % 16: block += b'\0' * (16 - (len(block) % 16)) mac_str = mac_encryptor.encrypt( encryptor.encrypt(block)) file_info = os.stat(temp_output_file.name) self.progress = int( int(file_info.st_size) / int(file_size) * 100) file_mac = str_to_a32(mac_str) temp_output_file.close() # check mac integrity if (file_mac[0] ^ file_mac[1], file_mac[2] ^ file_mac[3]) != meta_mac: raise ValueError('Mismatched mac') shutil.move(temp_output_file.name, dest_path + file_name) self.callback(self, file_name, self.path) def pause(self): self.paused = True # If in sleep, we acquire immediately, otherwise we wait for thread # to release condition. In race, worker will still see self.paused # and begin waiting until it's set back to False self.pause_cond.acquire() #should just resume the thread def resume(self): self.paused = False # Notify so thread will wake after lock released self.pause_cond.notify() # Now release the lock self.pause_cond.release() def stop(self): self.stop_event.set() self.progress = 0
proba = model.predict_proba(X_test, batch_size=32, verbose=0) # pred_attr = mapAttributes((proba > 0.6)[0]) # print( proba) # print(pred_attr) probStream = getProbaStream(probStream, proba) if saidNothing == 0 and probStream.shape[0] < 10: saidNothing += 1 cv2.imshow("Webcam Preview", frame) rval, frame = vc.read() key = cv2.waitKey(20) if key == 27: # exit on ESC if process != None: os.kill(process.pid, signal.SIGTERM) say("I'm sorry Dave. I'm afraid I can't do that.", flag) while flag.isSet(): time.Clock().tick(10) break elif probStream.shape[0] > 10 and len(probStream.shape) >= 2: if process != None: os.kill(process.pid, signal.SIGTERM) process = None meanProbs = np.mean(probStream, axis=0) pred_attr = mapAttributes(meanProbs > 0.6) best = [] if meanProbs[0] > meanProbs[1] and meanProbs[0] > meanProbs[ 4] and meanProbs[0] > meanProbs[2]: best.append('Black_Hair') elif meanProbs[1] > meanProbs[0] and meanProbs[1] > meanProbs[ 4] and meanProbs[1] > meanProbs[2]:
class WebhookSender(Thread): """This handles sending embeds to one specific webhook. Obviously being a thread sub-class you should not add too many webhooks (in the order of the hundreds) for the same monitor.\n You should not use this directly, but `WebhookManager` instead""" def __init__(self, webhook: str, logconfig: LogConfig): self.config = logconfig self.webhook = webhook self.logger = get_logger(logconfig) self.queue = Queue() # type: Queue[Tuple[List[Any], List[Any], datetime]] # contains the webhook config, embeds and time at which they were added self.add_event = Event() self.has_to_quit = False super().__init__() def add_to_queue(self, webhook_values: Any, embed: Embed): now = datetime.now() self.queue.put((webhook_values, embed, now)) self.add_event.set() def is_done(self) -> bool: return not self.add_event.isSet() def quit(self): self.has_to_quit = True self.add_event.set() def run(self): while True: self.add_event.wait() if self.has_to_quit: self.add_event.clear() break while not self.queue.empty(): webhook_values, embed, now = self.queue.get() if "custom" in webhook_values: provider = webhook_values["custom"].get( "provider", self.config["WebhookConfig"]["provider"] ) timestamp_format = webhook_values["custom"].get( "timestamp_format", self.config["WebhookConfig"]["timestamp_format"], ) ts = now.strftime(timestamp_format) icon_url = webhook_values["custom"].get( "icon_url", self.config["WebhookConfig"]["provider_icon"] ) color = webhook_values["custom"].get( "color", int(self.config["WebhookConfig"]["embed_color"]) ) embed.set_footer(text=" | ".join([provider, ts]), icon_url=icon_url) embed.color = color else: ts = now.strftime(self.config["WebhookConfig"]["timestamp_format"]) embed.set_footer( text=f"{self.config['WebhookConfig']['provider']} | {ts}", icon_url=self.config["WebhookConfig"]["provider_icon"], ) embed.color = int(self.config["WebhookConfig"]["embed_color"]) embed.timestamp = Embed.Empty data = {"embeds": [embed.to_dict()]} if "custom" in webhook_values: if "avatar_image" in webhook_values["custom"]: data["avatar_url"] = webhook_values["custom"]["avatar_image"] data = json.dumps(data) while True: r = requests.post( self.webhook, data=data, headers={"Content-Type": "application/json"}, timeout=3, ) self.logger.debug( f"Posted to {self.webhook} with code {r.status_code}" ) if "x-rateLimit-remaining" in r.headers: remaining_requests = r.headers["x-rateLimit-remaining"] if remaining_requests == "0": delay = int(r.headers["x-rateLimit-reset-after"]) self.logger.debug( f"No available requests reminaing for {self.webhook}, waiting {str(delay)} secs" ) time.sleep(delay) continue if r.status_code == 429: delay = int(r.headers["x-rateLimit-reset-after"]) self.logger.debug( f"Got 429 for {self.webhook}, waiting {str(delay)} secs" ) time.sleep(delay) continue else: self.logger.warning( f"Attention: {self.webhook} posted with {r.status_code} but it doesnt contain the rateLimit header; are you sure the webhook is correct???" ) break self.add_event.clear()
class BackendsCall(object): def __init__(self, backends, function, *args, **kwargs): """ :param backends: List of backends to call :type backends: list[:class:`BaseBackend`] :param function: backends' method name, or callable object. :type function: :class:`str` or :class:`callable` """ self.logger = getLogger('bcall') # Store if a backend is finished self.backends = {} for backend in backends: self.backends[backend.name] = False # Global mutex on object self.mutex = RLock() # Event set when every backends have give their data self.finish_event = Event() # Event set when there are new responses self.response_event = Event() # Waiting responses self.responses = [] # Errors self.errors = [] # Threads self.threads = [] # Create jobs for each backend with self.mutex: for backend in backends: self.threads.append( Timer(0, self._caller, (backend, function, args, kwargs)).start()) if not backends: self.finish_event.set() def _store_error(self, backend, error): with self.mutex: backtrace = get_backtrace(error) self.errors.append((backend, error, backtrace)) def _store_result(self, backend, result): with self.mutex: if isinstance(result, CapBaseObject): result.backend = backend.name self.responses.append((backend, result)) self.response_event.set() def _caller(self, backend, function, args, kwargs): with backend: try: # Call method on backend try: self.logger.debug('%s: Calling function %s' % (backend, function)) if callable(function): result = function(backend, *args, **kwargs) else: result = getattr(backend, function)(*args, **kwargs) except Exception as error: self.logger.debug( '%s: Called function %s raised an error: %r' % (backend, function, error)) self._store_error(backend, error) else: self.logger.debug('%s: Called function %s returned: %r' % (backend, function, result)) if hasattr(result, '__iter__') and not isinstance( result, basestring): # Loop on iterator try: for subresult in result: # Lock mutex only in loop in case the iterator is slow # (for example if backend do some parsing operations) self._store_result(backend, subresult) except Exception as error: self._store_error(backend, error) else: self._store_result(backend, result) finally: with self.mutex: # This backend is now finished self.backends[backend.name] = True for finished in self.backends.itervalues(): if not finished: return self.response_event.set() self.finish_event.set() def _callback_thread_run(self, callback, errback): responses = [] while not self.finish_event.isSet() or self.response_event.isSet(): self.response_event.wait() with self.mutex: responses = self.responses self.responses = [] # Reset event self.response_event.clear() # Consume responses while responses: callback(*responses.pop(0)) if errback: with self.mutex: while self.errors: errback(*self.errors.pop(0)) callback(None, None) def callback_thread(self, callback, errback=None): """ Call this method to create a thread which will callback a specified function everytimes a new result comes. When the process is over, the function will be called with both arguments set to None. The functions prototypes: def callback(backend, result) def errback(backend, error) """ thread = Thread(target=self._callback_thread_run, args=(callback, errback)) thread.start() return thread def wait(self): self.finish_event.wait() with self.mutex: if self.errors: raise CallErrors(self.errors) def __iter__(self): # Don't know how to factorize with _callback_thread_run responses = [] while not self.finish_event.isSet() or self.response_event.isSet(): self.response_event.wait() with self.mutex: responses = self.responses self.responses = [] # Reset event self.response_event.clear() # Consume responses while responses: yield responses.pop(0) # Raise errors with self.mutex: if self.errors: raise CallErrors(self.errors)
class WorkerBase(ABC): def __init__(self, args, id, last_known_state, websocket_handler, walker_routemanager, devicesettings, db_wrapper, pogoWindowManager, NoOcr=True, walker=None): # self.thread_pool = ThreadPool(processes=2) self._walker_routemanager = walker_routemanager self._route_manager_last_time = None self._websocket_handler = websocket_handler self._communicator = Communicator(websocket_handler, id, self, args.websocket_command_timeout) self._id = id self._applicationArgs = args self._last_known_state = last_known_state self._work_mutex = Lock() self.loop = None self.loop_started = Event() self.loop_tid = None self._async_io_looper_thread = None self._location_count = 0 self._init = self._walker_routemanager.init self._walker = walker self._lastScreenshotTaken = 0 self._stop_worker_event = Event() self._db_wrapper = db_wrapper self._redErrorCount = 0 self._lastScreenHash = None self._lastScreenHashCount = 0 self._devicesettings = devicesettings self._resocalc = Resocalculator self._screen_x = 0 self._screen_y = 0 self._lastStart = "" self._geofix_sleeptime = 0 self._pogoWindowManager = pogoWindowManager self.current_location = Location(0.0, 0.0) self.last_location = self._devicesettings.get("last_location", None) if self.last_location is None: self.last_location = Location(0.0, 0.0) if self._devicesettings.get('last_mode', None) is not None and \ self._devicesettings['last_mode'] in ("raids_mitm", "mon_mitm", "iv_mitm", "raids_ocr"): # Reset last_location - no useless waiting delays (otherwise stop mode) logger.info('{}: last Mode not pokestop - reset saved location', str(self._id)) self.last_location = Location(0.0, 0.0) self._devicesettings['last_mode'] = self._walker_routemanager.mode self.last_processed_location = Location(0.0, 0.0) self.workerstart = None self.workerstarttime = datetime.datetime.now() def get_communicator(self): return self._communicator @abstractmethod def _pre_work_loop(self): """ Work to be done before the main while true work-loop Start off asyncio loops etc in here :return: """ pass @abstractmethod def _health_check(self): """ Health check before a location is grabbed. Internally, a self._start_pogo call is already executed since that usually includes a topmost check :return: """ pass @abstractmethod def _pre_location_update(self): """ Override to run stuff like update injections settings in MITM worker Runs before walk/teleport to the location previously grabbed :return: """ pass @abstractmethod def _move_to_location(self): """ Location has previously been grabbed, the overriden function will be called. You may teleport or walk by your choosing Any post walk/teleport delays/sleeps have to be run in the derived, override method :return: """ pass @abstractmethod def _post_move_location_routine(self, timestamp): """ Routine called after having moved to a new location. MITM worker e.g. has to wait_for_data :param timestamp: :return: """ @abstractmethod def _start_pogo(self): """ Routine to start pogo. Return the state as a boolean do indicate a successful start :return: """ pass @abstractmethod def _cleanup(self): """ Cleanup any threads you started in derived classes etc self.stop_worker() and self.loop.stop() will be called afterwards :return: """ @abstractmethod def _valid_modes(self): """ Return a list of valid modes for the health checks :return: """ def _start_asyncio_loop(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) self.loop_tid = current_thread() self.loop.call_soon(self.loop_started.set) self.loop.run_forever() def _add_task_to_loop(self, coro): f = functools.partial(self.loop.create_task, coro) if current_thread() == self.loop_tid: # We can call directly if we're not going between threads. return f() else: # We're in a non-event loop thread so we use a Future # to get the task from the event loop thread once # it's ready. return self.loop.call_soon_threadsafe(f) def start_worker(self): # async_result = self.thread_pool.apply_async(self._main_work_thread, ()) t_main_work = Thread(target=self._main_work_thread) t_main_work.daemon = False t_main_work.start() # do some other stuff in the main process while not self._stop_worker_event.isSet(): time.sleep(1) t_main_work.join() logger.info("Worker {} stopping gracefully", str(self._id)) # async_result.get() return self._last_known_state def stop_worker(self): self._stop_worker_event.set() logger.warning("Worker {} stop called", str(self._id)) def _internal_pre_work(self): current_thread().name = self._id self._work_mutex.acquire() try: self._turn_screen_on_and_start_pogo() except WebsocketWorkerRemovedException: logger.error("Timeout during init of worker {}", str(self._id)) # no cleanup required here? TODO: signal websocket server somehow self._stop_worker_event.set() return # register worker in routemanager logger.info("Try to register {} in Routemanager {}", str(self._id), str(self._walker_routemanager.name)) self._walker_routemanager.register_worker(self._id) self._work_mutex.release() self._async_io_looper_thread = Thread(name=str(self._id) + '_asyncio_' + self._id, target=self._start_asyncio_loop) self._async_io_looper_thread.daemon = False self._async_io_looper_thread.start() self.loop_started.wait() self._pre_work_loop() def _internal_health_check(self): # check if pogo is topmost and start if necessary logger.debug( "_internal_health_check: Calling _start_pogo routine to check if pogo is topmost" ) self._work_mutex.acquire() logger.debug("_internal_health_check: worker lock acquired") logger.debug("Checking if we need to restart pogo") # Restart pogo every now and then... if self._devicesettings.get("restart_pogo", 80) > 0: # logger.debug("main: Current time - lastPogoRestart: {}", str(curTime - lastPogoRestart)) # if curTime - lastPogoRestart >= (args.restart_pogo * 60): if self._location_count > self._devicesettings.get( "restart_pogo", 80): logger.error( "scanned " + str(self._devicesettings.get("restart_pogo", 80)) + " locations, restarting pogo") pogo_started = self._restart_pogo() self._location_count = 0 else: pogo_started = self._start_pogo() else: pogo_started = self._start_pogo() self._work_mutex.release() logger.debug("_internal_health_check: worker lock released") return pogo_started def _internal_cleanup(self): # set the event just to make sure - in case of exceptions for example self._stop_worker_event.set() logger.info("Internal cleanup of {} started", str(self._id)) self._cleanup() logger.info("Internal cleanup of {} signalling end to websocketserver", str(self._id)) self._walker_routemanager.unregister_worker(self._id) logger.info("Stopping Route") # self.stop_worker() if self._async_io_looper_thread is not None: logger.info("Stopping worker's asyncio loop") self.loop.call_soon_threadsafe(self.loop.stop) self._async_io_looper_thread.join() self._communicator.cleanup_websocket() logger.info("Internal cleanup of {} finished", str(self._id)) def _main_work_thread(self): # TODO: signal websocketserver the removal try: self._internal_pre_work() except (InternalStopWorkerException, WebsocketWorkerRemovedException, WebsocketWorkerTimeoutException): logger.error( "Failed initializing worker {}, connection terminated exceptionally", str(self._id)) self._internal_cleanup() return if not check_max_walkers_reached(self._walker, self._walker_routemanager): logger.warning('Max. Walkers in Area {} - closing connections', str(self._walker_routemanager.name)) self._devicesettings['finished'] = True self._internal_cleanup() return while not self._stop_worker_event.isSet(): try: # TODO: consider getting results of health checks and aborting the entire worker? walkercheck = self.check_walker() if not walkercheck: self._devicesettings['finished'] = True break except (InternalStopWorkerException, WebsocketWorkerRemovedException, WebsocketWorkerTimeoutException): logger.warning("Worker {} killed by walker settings", str(self._id)) break try: # TODO: consider getting results of health checks and aborting the entire worker? self._internal_health_check() self._health_check() except (InternalStopWorkerException, WebsocketWorkerRemovedException, WebsocketWorkerTimeoutException): logger.error( "Websocket connection to {} lost while running healthchecks, connection terminated exceptionally", str(self._id)) break try: settings = self._internal_grab_next_location() if settings is None: continue except (InternalStopWorkerException, WebsocketWorkerRemovedException, WebsocketWorkerTimeoutException): logger.warning( "Worker of {} does not support mode that's to be run, connection terminated exceptionally", str(self._id)) break try: logger.debug('Checking if new location is valid') valid = self._check_location_is_valid() if not valid: break except (InternalStopWorkerException, WebsocketWorkerRemovedException, WebsocketWorkerTimeoutException): logger.warning("Worker {} get non valid coords!", str(self._id)) break try: self._pre_location_update() except (InternalStopWorkerException, WebsocketWorkerRemovedException, WebsocketWorkerTimeoutException): logger.warning( "Worker of {} stopping because of stop signal in pre_location_update, connection terminated exceptionally", str(self._id)) break try: logger.debug( 'main worker {}: LastLat: {}, LastLng: {}, CurLat: {}, CurLng: {}', str(self._id), self._devicesettings["last_location"].lat, self._devicesettings["last_location"].lng, self.current_location.lat, self.current_location.lng) time_snapshot, process_location = self._move_to_location() except (InternalStopWorkerException, WebsocketWorkerRemovedException, WebsocketWorkerTimeoutException): logger.warning( "Worker {} failed moving to new location, stopping worker, connection terminated exceptionally", str(self._id)) break if process_location: self._add_task_to_loop(self._update_position_file()) self._location_count += 1 if self._applicationArgs.last_scanned: logger.debug("Seting new 'scannedlocation' in Database") # self.update_scanned_location(currentLocation.lat, currentLocation.lng, curTime) self._add_task_to_loop( self.update_scanned_location(self.current_location.lat, self.current_location.lng, time_snapshot)) try: self._post_move_location_routine(time_snapshot) except (InternalStopWorkerException, WebsocketWorkerRemovedException, WebsocketWorkerTimeoutException): logger.warning( "Worker {} failed running post_move_location_routine, stopping worker", str(self._id)) break logger.info("Worker {} finished iteration, continuing work", str(self._id)) self._internal_cleanup() async def _update_position_file(self): logger.debug("Updating .position file") if self.current_location is not None: with open( os.path.join(self._applicationArgs.file_path, self._id + '.position'), 'w') as outfile: outfile.write( str(self.current_location.lat) + ", " + str(self.current_location.lng)) async def update_scanned_location(self, latitude, longitude, timestamp): try: self._db_wrapper.set_scanned_location(str(latitude), str(longitude), str(timestamp)) except Exception as e: logger.error("Failed updating scanned location: {}", str(e)) return def check_walker(self): mode = self._walker['walkertype'] if mode == "countdown": logger.info("Checking walker mode 'countdown'") countdown = self._walker['walkervalue'] if not countdown: logger.error( "No Value for Mode - check your settings! Killing worker") return False if self.workerstart is None: self.workerstart = math.floor(time.time()) else: if math.floor( time.time()) >= int(self.workerstart) + int(countdown): return False return True elif mode == "timer": logger.debug("Checking walker mode 'timer'") exittime = self._walker['walkervalue'] if not exittime or ':' not in exittime: logger.error( "No or wrong Value for Mode - check your settings! Killing worker" ) return False return check_walker_value_type(exittime, self.workerstarttime) elif mode == "round": logger.debug("Checking walker mode 'round'") rounds = self._walker['walkervalue'] if len(rounds) == 0: logger.error( "No Value for Mode - check your settings! Killing worker") return False processed_rounds = self._walker_routemanager.get_rounds(self._id) if int(processed_rounds) >= int(rounds): return False return True elif mode == "period": logger.debug("Checking walker mode 'period'") period = self._walker['walkervalue'] if len(period) == 0: logger.error( "No Value for Mode - check your settings! Killing worker") return False return check_walker_value_type(period, self.workerstarttime) elif mode == "coords": exittime = self._walker['walkervalue'] if len(exittime) > 0: return check_walker_value_type(exittime, self.workerstarttime) return True elif mode == "idle": logger.debug("Checking walker mode 'idle'") if len(self._walker['walkervalue']) == 0: logger.error( "Wrong Value for mode - check your settings! Killing worker" ) return False sleeptime = self._walker['walkervalue'] logger.info('{} going to sleep', str(self._id)) killpogo = False if check_walker_value_type(sleeptime, self.workerstarttime): self._stop_pogo() killpogo = True while not self._stop_worker_event.isSet( ) and check_walker_value_type(sleeptime, self.workerstarttime): time.sleep(1) logger.info('{} just woke up', str(self._id)) if killpogo: self._start_pogo() return False else: logger.error("Unknown walker mode! Killing worker") return False return True def set_geofix_sleeptime(self, sleeptime): self._geofix_sleeptime = sleeptime return True def _internal_grab_next_location(self): # TODO: consider adding runWarningThreadEvent.set() self._last_known_state["last_location"] = self.last_location logger.debug("Requesting next location from routemanager") # requesting a location is blocking (iv_mitm will wait for a prioQ item), we really need to clean # the workers up... if int(self._geofix_sleeptime) > 0: logger.info( 'Getting a geofix position from MADMin - sleeping for {} seconds', str(self._geofix_sleeptime)) time.sleep(int(self._geofix_sleeptime)) self._geofix_sleeptime = 0 routemanager = self._walker_routemanager self.current_location = routemanager.get_next_location() return routemanager.settings def _init_routine(self): if self._applicationArgs.initial_restart is False: self._turn_screen_on_and_start_pogo() else: if not self._start_pogo(): while not self._restart_pogo(): logger.warning("failed starting pogo") # TODO: stop after X attempts def _check_location_is_valid(self): if self.current_location is None: # there are no more coords - so worker is finished successfully self._devicesettings['finished'] = True return None elif self.current_location is not None: logger.debug('Coords are valid') return True def _turn_screen_on_and_start_pogo(self): if not self._communicator.isScreenOn(): self._communicator.startApp("de.grennith.rgc.remotegpscontroller") logger.warning("Turning screen on") self._communicator.turnScreenOn() time.sleep(self._devicesettings.get("post_turn_screen_on_delay", 2)) # check if pogo is running and start it if necessary logger.info("turnScreenOnAndStartPogo: (Re-)Starting Pogo") self._start_pogo() def _check_screen_on(self): if not self._communicator.isScreenOn(): self._communicator.startApp("de.grennith.rgc.remotegpscontroller") logger.warning("Turning screen on") self._communicator.turnScreenOn() time.sleep(self._devicesettings.get("post_turn_screen_on_delay", 2)) def _stop_pogo(self): attempts = 0 stop_result = self._communicator.stopApp("com.nianticlabs.pokemongo") pogoTopmost = self._communicator.isPogoTopmost() while pogoTopmost: attempts += 1 if attempts > 10: return False stop_result = self._communicator.stopApp( "com.nianticlabs.pokemongo") time.sleep(1) pogoTopmost = self._communicator.isPogoTopmost() return stop_result def _reboot(self): try: start_result = self._communicator.reboot() except WebsocketWorkerRemovedException: logger.error( "Could not reboot due to client already having disconnected") start_result = False time.sleep(5) self._db_wrapper.save_last_reboot(self._id) self.stop_worker() return start_result def _start_pogodroid(self): start_result = self._communicator.startApp("com.mad.pogodroid") time.sleep(5) return start_result def _stopPogoDroid(self): stopResult = self._communicator.stopApp("com.mad.pogodroid") return stopResult def _restart_pogo(self, clear_cache=True): successful_stop = self._stop_pogo() self._db_wrapper.save_last_restart(self._id) logger.debug("restartPogo: stop pogo resulted in {}", str(successful_stop)) if successful_stop: if clear_cache: self._communicator.clearAppCache("com.nianticlabs.pokemongo") time.sleep(1) return self._start_pogo() else: return False def _restartPogoDroid(self): successfulStop = self._stopPogoDroid() time.sleep(1) logger.debug("restartPogoDroid: stop pogodriud resulted in {}", str(successfulStop)) if successfulStop: return self._start_pogodroid() else: return False def _reopenRaidTab(self): logger.debug("_reopenRaidTab: Taking screenshot...") logger.info( "reopenRaidTab: Attempting to retrieve screenshot before checking raidtab" ) if not self._takeScreenshot(): logger.debug("_reopenRaidTab: Failed getting screenshot...") logger.error( "reopenRaidTab: Failed retrieving screenshot before checking for closebutton" ) return logger.debug("_reopenRaidTab: Checking close except nearby...") pathToPass = os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id)) logger.debug("Path: {}", str(pathToPass)) self._pogoWindowManager.checkCloseExceptNearbyButton( pathToPass, self._id, self._communicator, 'True') logger.debug("_reopenRaidTab: Getting to raidscreen...") self._getToRaidscreen(3) time.sleep(1) def _takeScreenshot(self, delayAfter=0.0, delayBefore=0.0): logger.debug("Taking screenshot...") time.sleep(delayBefore) compareToTime = time.time() - self._lastScreenshotTaken logger.debug("Last screenshot taken: {}", str(self._lastScreenshotTaken)) if self._applicationArgs.use_media_projection: take_screenshot = self._communicator.getScreenshot( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id))) else: take_screenshot = self._communicator.get_screenshot_single( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id))) if self._lastScreenshotTaken and compareToTime < 0.5: logger.debug( "takeScreenshot: screenshot taken recently, returning immediately" ) logger.debug("Screenshot taken recently, skipping") return True # TODO: screenshot.png needs identifier in name elif not take_screenshot: logger.error("takeScreenshot: Failed retrieving screenshot") logger.debug("Failed retrieving screenshot") return False else: logger.debug("Success retrieving screenshot") self._lastScreenshotTaken = time.time() time.sleep(delayAfter) return True def _checkPogoFreeze(self): logger.debug("Checking if pogo froze") if not self._takeScreenshot(): logger.debug("_checkPogoFreeze: failed retrieving screenshot") return from utils.image_utils import getImageHash screenHash = getImageHash( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id))) logger.debug("checkPogoFreeze: Old Hash: {}", str(self._lastScreenHash)) logger.debug("checkPogoFreeze: New Hash: {}", str(screenHash)) if hamming_dist(str(self._lastScreenHash), str(screenHash)) < 4 and str( self._lastScreenHash) != '0': logger.debug( "checkPogoFreeze: New und old Screenshoot are the same - no processing" ) self._lastScreenHashCount += 1 logger.debug("checkPogoFreeze: Same Screen Count: " + str(self._lastScreenHashCount)) if self._lastScreenHashCount >= 100: self._lastScreenHashCount = 0 self._restart_pogo() else: self._lastScreenHash = screenHash self._lastScreenHashCount = 0 logger.debug("_checkPogoFreeze: done") def _check_pogo_main_screen(self, maxAttempts, again=False): logger.debug( "_check_pogo_main_screen: Trying to get to the Mainscreen with {} max attempts...", str(maxAttempts)) pogoTopmost = self._communicator.isPogoTopmost() if not pogoTopmost: return False if not self._takeScreenshot(delayBefore=self._devicesettings.get( "post_screenshot_delay", 1)): if again: logger.error( "_check_pogo_main_screen: failed getting a screenshot again" ) return False attempts = 0 if os.path.isdir( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id))): logger.error( "_check_pogo_main_screen: screenshot.png is not a file/corrupted" ) return False logger.info("_check_pogo_main_screen: checking mainscreen") buttoncheck = self._pogoWindowManager.lookForButton( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id)), 2.20, 3.01, self._communicator) if buttoncheck: logger.info('Found button on screen') self._takeScreenshot(delayBefore=self._devicesettings.get( "post_screenshot_delay", 1)) while not self._pogoWindowManager.checkpogomainscreen( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id)), self._id): logger.error("_check_pogo_main_screen: not on Mainscreen...") if attempts > maxAttempts: # could not reach raidtab in given maxAttempts logger.error( "_check_pogo_main_screen: Could not get to Mainscreen within {} attempts", str(maxAttempts)) return False # not using continue since we need to get a screen before the next round... found = self._pogoWindowManager.lookForButton( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id)), 2.20, 3.01, self._communicator) if found: logger.info("_check_pogo_main_screen: Found button (small)") if not found and self._pogoWindowManager.checkCloseExceptNearbyButton( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id)), self._id, self._communicator, closeraid=True): logger.info( "_check_pogo_main_screen: Found (X) button (except nearby)" ) found = True if not found and self._pogoWindowManager.lookForButton( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id)), 1.05, 2.20, self._communicator): logger.info("_check_pogo_main_screen: Found button (big)") found = True logger.info( "_check_pogo_main_screen: Previous checks found popups: {}", str(found)) self._takeScreenshot(delayBefore=self._devicesettings.get( "post_screenshot_delay", 1)) attempts += 1 logger.info("_check_pogo_main_screen: done") return True def _checkPogoButton(self): logger.debug("checkPogoButton: Trying to find buttons") pogoTopmost = self._communicator.isPogoTopmost() if not pogoTopmost: return False if not self._takeScreenshot(delayBefore=self._devicesettings.get( "post_screenshot_delay", 1)): # TODO: again? # if again: # logger.error("checkPogoButton: failed getting a screenshot again") # return False # TODO: throw? logger.debug("checkPogoButton: Failed getting screenshot") return False attempts = 0 if os.path.isdir( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id))): logger.error( "checkPogoButton: screenshot.png is not a file/corrupted") return False logger.info("checkPogoButton: checking for buttons") found = self._pogoWindowManager.lookForButton( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id)), 2.20, 3.01, self._communicator) if found: time.sleep(1) logger.info("checkPogoButton: Found button (small)") logger.info("checkPogoButton: done") return True logger.info("checkPogoButton: done") return False def _checkPogoClose(self): logger.debug("checkPogoClose: Trying to find closeX") pogoTopmost = self._communicator.isPogoTopmost() if not pogoTopmost: return False if not self._takeScreenshot(delayBefore=self._devicesettings.get( "post_screenshot_delay", 1)): # TODO: go again? # if again: # logger.error("checkPogoClose: failed getting a screenshot again") # return False # TODO: consider throwing? logger.debug("checkPogoClose: Could not get screenshot") return False attempts = 0 if os.path.isdir( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id))): logger.error( "checkPogoClose: screenshot.png is not a file/corrupted") return False logger.info("checkPogoClose: checking for CloseX") found = self._pogoWindowManager.checkCloseExceptNearbyButton( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id)), self._id, self._communicator) if found: time.sleep(1) logger.info("checkPogoClose: Found (X) button (except nearby)") logger.info("checkPogoClose: done") return True logger.info("checkPogoClose: done") return False def _getToRaidscreen(self, maxAttempts, again=False): # check for any popups (including post login OK) logger.debug( "getToRaidscreen: Trying to get to the raidscreen with {} max attempts...", str(maxAttempts)) pogoTopmost = self._communicator.isPogoTopmost() if not pogoTopmost: return False self._checkPogoFreeze() if not self._takeScreenshot(delayBefore=self._devicesettings.get( "post_screenshot_delay", 1)): if again: logger.error( "getToRaidscreen: failed getting a screenshot again") return False self._getToRaidscreen(maxAttempts, True) logger.debug("getToRaidscreen: Got screenshot, checking GPS") attempts = 0 if os.path.isdir( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id))): logger.error( "getToRaidscreen: screenshot.png is not a file/corrupted") return False # TODO: replace self._id with device ID while self._pogoWindowManager.isGpsSignalLost( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id)), self._id): logger.debug("getToRaidscreen: GPS signal lost") time.sleep(1) self._takeScreenshot() logger.warning("getToRaidscreen: GPS signal error") self._redErrorCount += 1 if self._redErrorCount > 3: logger.error( "getToRaidscreen: Red error multiple times in a row, restarting" ) self._redErrorCount = 0 self._restart_pogo() return False self._redErrorCount = 0 logger.debug("getToRaidscreen: checking raidscreen") while not self._pogoWindowManager.checkRaidscreen( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id)), self._id, self._communicator): logger.debug("getToRaidscreen: not on raidscreen...") if attempts > maxAttempts: # could not reach raidtab in given maxAttempts logger.error( "getToRaidscreen: Could not get to raidtab within {} attempts", str(maxAttempts)) return False self._checkPogoFreeze() # not using continue since we need to get a screen before the next round... found = self._pogoWindowManager.lookForButton( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id)), 2.20, 3.01, self._communicator) if found: logger.info("getToRaidscreen: Found button (small)") if not found and self._pogoWindowManager.checkCloseExceptNearbyButton( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id)), self._id, self._communicator): logger.info( "getToRaidscreen: Found (X) button (except nearby)") found = True if not found and self._pogoWindowManager.lookForButton( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id)), 1.05, 2.20, self._communicator): logger.info("getToRaidscreen: Found button (big)") found = True logger.info("getToRaidscreen: Previous checks found popups: {}", str(found)) if not found: logger.info( "getToRaidscreen: Previous checks found nothing. Checking nearby open" ) if self._pogoWindowManager.checkNearby( os.path.join(self._applicationArgs.temp_path, 'screenshot%s.png' % str(self._id)), self._id, self._communicator): return self._takeScreenshot( delayBefore=self._devicesettings.get( "post_screenshot_delay", 1)) if not self._takeScreenshot(delayBefore=self._devicesettings.get( "post_screenshot_delay", 1)): return False attempts += 1 logger.debug("getToRaidscreen: done") return True def _get_screen_size(self): screen = self._communicator.getscreensize().split(' ') self._screen_x = screen[0] self._screen_y = screen[1] x_offset = self._devicesettings.get("screenshot_x_offset", 0) y_offset = self._devicesettings.get("screenshot_y_offset", 0) logger.debug( 'Get Screensize of {}: X: {}, Y: {}, X-Offset: {}, Y-Offset: {}', str(self._id), str(self._screen_x), str(self._screen_y), str(x_offset), str(y_offset)) self._resocalc.get_x_y_ratio(self, self._screen_x, self._screen_y, x_offset, y_offset)
class EventGhostEvent(object): """ .. attribute:: string This is the full qualified event string as you see it inside the logger, with the exception that if the payload field (that is explained below) is not None the logger will also show it behind the event string, but this is not a part of the event string we are talking about here. .. attribute:: payload A plugin might publish additional data related to this event. Through payload you can access this data. For example the 'Network Event Receiver' plugin returns also the IP of the client that has generated the event. If there is no data, this field is ``None``. .. attribute:: prefix This is the first part of the event string till the first dot. This normally identifies the source of the event as a short string. .. attribute:: suffix This is the part of the event string behind the first dot. So you could say: event.string = event.prefix + '.' + event.suffix .. attribute:: time The time the event was generated as a floating point number in seconds (as returned by the clock() function of Python's time module). Since most events are processed very quickly, this is most likely nearly the current time. But in some situations it might be more clever to use this time, instead of the current time, since even small differences might matter (for example if you want to determine a double-press). .. attribute:: isEnded This boolean value indicates if the event is an enduring event and is still active. Some plugins (e.g. most of the remote receiver plugins) indicate if a button is pressed longer. As long as the button is pressed, this flag is ``False`` and in the moment the user releases the button the flag turns to ``True``. So you can poll this flag to see, if the button is still pressed. """ skipEvent = False def __init__(self, suffix="", payload=None, prefix="Main", source=eg): self.string = prefix + "." + suffix self.prefix = prefix self.suffix = suffix self.payload = payload self.source = source self.time = clock() self.isEnded = False self.shouldEnd = Event() self.upFuncList = [] def AddUpFunc(self, func, *args, **kwargs): if self.isEnded: func(*args, **kwargs) else: self.upFuncList.append((func, args, kwargs)) def DoUpFuncs(self): for func, args, kwargs in self.upFuncList: func(*args, **kwargs) del self.upFuncList[:] self.isEnded = True def Execute(self): #start = clock() eventString = self.string if eventString in eg.notificationHandlers: for listener in eg.notificationHandlers[eventString].listeners: if listener(self) is True: return eg.event = self eg.eventString = eventString eventHandlerList = [] for key, val in eg.eventTable.iteritems(): if ( eventString == key or (("*" in key or "?" in key) and fnmatchcase(eventString, key)) ): eventHandlerList += val activeHandlers = set() for eventHandler in eventHandlerList: obj = eventHandler while obj: if not obj.isEnabled: break obj = obj.parent else: activeHandlers.add(eventHandler) for listener in eg.log.eventListeners: listener.LogEvent(self) if config.onlyLogAssigned and len(activeHandlers) == 0: self.SetStarted() return # show the event in the logger LogEvent(self) activeHandlers = sorted(activeHandlers, key=GetItemPath) eg.SetProcessingState(2, self) for eventHandler in activeHandlers: try: eg.programCounter = (eventHandler.parent, None) eg.indent = 1 RunProgram() except: eg.PrintTraceback() if self.skipEvent: break self.SetStarted() eg.SetProcessingState(1, self) def SetShouldEnd(self): if not self.shouldEnd.isSet(): self.shouldEnd.set() eg.SetProcessingState(0, self) actionThread.Call(self.DoUpFuncs) def SetStarted(self): if self.shouldEnd.isSet(): self.DoUpFuncs()
class Scheduler(Queue): def __init__(self, nworkers, maxlarge, logger, email, branch): Queue.__init__(self) info('Initializing scheduler with %d jobs.', nworkers) self.nworkers = nworkers self.logger = logger self.maxlarge = maxlarge self.nlarge = 0 # not thread safe, don't really care right now self.passed = 0 self.failed = 0 self.workers = [] self.stopping = Event() self.timer = None self.error = None self.email = email self.branch = branch def run(self, timeout): info('Starting workers.') self.stopping.clear() for i in range(self.nworkers): w = Worker(self) self.workers.append(w) w.start() if timeout != 0: self.timer = Timer(timeout, self.stop) self.timer.start() while not self.stopping.isSet(): try: for w in self.workers: if self.stopping.isSet(): break w.join(timeout=1.0) except (KeyboardInterrupt, SystemExit): debug('Scheduler interrupted. Stopping and joining threads.') self.stop() self.join() sys.exit(0) else: debug('Scheduler stopped by someone else. Joining threads.') self.join() if self.error: send_mail( self.email, 'Stress tests scheduler stopped by something, on %s' % gethostname(), self.error) sys.exit(77) def join(self): if self.timer is not None: self.timer.cancel() while len(self.workers) > 0: self.workers.pop().join() def stop(self): info('Stopping workers.') self.stopping.set() def __getitem__(self, k): return self.__dict__[k] def reportstr(self): return '[PASS=%(passed)d FAIL=%(failed)d]' % self def report_success(self, runner): self.passed += 1 self.logger.info('PASSED %s', runner.infostr()) info('%s PASSED %s', self.reportstr(), runner.infostr()) def report_failure(self, runner): self.failed += 1 self.logger.warning('FAILED %s', runner.infostr()) warning('%s FAILED %s', self.reportstr(), runner.infostr()) def email_failure(self, runner, savedtarfile, commands, output): if self.email is None: return h = gethostname() if isinstance(runner, UpgradeTestRunnerMixin): upgradestr = ''' The test was upgrading from %s.''' % runner.oldversionstr else: upgradestr = '' send_mail( self.email, 'Stress test failure on %(hostname)s running %(branch)s.' % { 'hostname': h, 'branch': self.branch }, ('''A stress test failed on %(hostname)s running %(branch)s at revision %(rev)s after %(test_duration)d seconds.%(upgradestr)s Its environment is saved to %(tarfile)s on that machine. The test configuration was: testname: %(execf)s num_elements: %(tsize)d cachetable_size: %(csize)d num_ptquery_threads: %(num_ptquery)d num_update_threads: %(num_update)d Commands run: %(commands)s Test output: %(output)s ''' % { 'hostname': h, 'rev': runner.rev, 'test_duration': runner.time, 'upgradestr': upgradestr, 'tarfile': savedtarfile, 'execf': runner.execf, 'tsize': runner.tsize, 'csize': runner.csize, 'num_ptquery': runner.num_ptquery, 'num_update': runner.num_update, 'branch': self.branch, 'commands': commands, 'output': output, }))
class AbstractDCS(object): _INITIALIZE = 'initialize' _CONFIG = 'config' _LEADER = 'leader' _FAILOVER = 'failover' _HISTORY = 'history' _MEMBERS = 'members/' _OPTIME = 'optime' _LEADER_OPTIME = _OPTIME + '/' + _LEADER _SYNC = 'sync' def __init__(self, config): """ :param config: dict, reference to config section of selected DCS. i.e.: `zookeeper` for zookeeper, `etcd` for etcd, etc... """ self._name = config['name'] self._base_path = re.sub('/+', '/', '/'.join(['', config.get('namespace', 'service'), config['scope']])) self._set_loop_wait(config.get('loop_wait', 10)) self._ctl = bool(config.get('patronictl', False)) self._cluster = None self._cluster_valid_till = 0 self._cluster_thread_lock = Lock() self._last_leader_operation = '' self.event = Event() def client_path(self, path): return '/'.join([self._base_path, path.lstrip('/')]) @property def initialize_path(self): return self.client_path(self._INITIALIZE) @property def config_path(self): return self.client_path(self._CONFIG) @property def members_path(self): return self.client_path(self._MEMBERS) @property def member_path(self): return self.client_path(self._MEMBERS + self._name) @property def leader_path(self): return self.client_path(self._LEADER) @property def failover_path(self): return self.client_path(self._FAILOVER) @property def history_path(self): return self.client_path(self._HISTORY) @property def leader_optime_path(self): return self.client_path(self._LEADER_OPTIME) @property def sync_path(self): return self.client_path(self._SYNC) @abc.abstractmethod def set_ttl(self, ttl): """Set the new ttl value for leader key""" @abc.abstractmethod def ttl(self): """Get new ttl value""" @abc.abstractmethod def set_retry_timeout(self, retry_timeout): """Set the new value for retry_timeout""" def _set_loop_wait(self, loop_wait): self._loop_wait = loop_wait def reload_config(self, config): self._set_loop_wait(config['loop_wait']) self.set_ttl(config['ttl']) self.set_retry_timeout(config['retry_timeout']) @property def loop_wait(self): return self._loop_wait @abc.abstractmethod def _load_cluster(self): """Internally this method should build `Cluster` object which represents current state and topology of the cluster in DCS. this method supposed to be called only by `get_cluster` method. raise `~DCSError` in case of communication or other problems with DCS. If the current node was running as a master and exception raised, instance would be demoted.""" def get_cluster(self): try: cluster = self._load_cluster() except Exception: self.reset_cluster() raise with self._cluster_thread_lock: self._cluster = cluster self._cluster_valid_till = time.time() + self.ttl return cluster @property def cluster(self): with self._cluster_thread_lock: return self._cluster if self._cluster_valid_till > time.time() else None def reset_cluster(self): with self._cluster_thread_lock: self._cluster = None self._cluster_valid_till = 0 @abc.abstractmethod def _write_leader_optime(self, last_operation): """write current xlog location into `/optime/leader` key in DCS :param last_operation: absolute xlog location in bytes :returns: `!True` on success.""" def write_leader_optime(self, last_operation): if self._last_leader_operation != last_operation and self._write_leader_optime(last_operation): self._last_leader_operation = last_operation @abc.abstractmethod def _update_leader(self): """Update leader key (or session) ttl :returns: `!True` if leader key (or session) has been updated successfully. If not, `!False` must be returned and current instance would be demoted. You have to use CAS (Compare And Swap) operation in order to update leader key, for example for etcd `prevValue` parameter must be used.""" def update_leader(self, last_operation, access_is_restricted=False): """Update leader key (or session) ttl and optime/leader :param last_operation: absolute xlog location in bytes :returns: `!True` if leader key (or session) has been updated successfully. If not, `!False` must be returned and current instance would be demoted.""" ret = self._update_leader() if ret and last_operation: self.write_leader_optime(last_operation) return ret @abc.abstractmethod def attempt_to_acquire_leader(self, permanent=False): """Attempt to acquire leader lock This method should create `/leader` key with value=`~self._name` :param permanent: if set to `!True`, the leader key will never expire. Used in patronictl for the external master :returns: `!True` if key has been created successfully. Key must be created atomically. In case if key already exists it should not be overwritten and `!False` must be returned""" @abc.abstractmethod def set_failover_value(self, value, index=None): """Create or update `/failover` key""" def manual_failover(self, leader, candidate, scheduled_at=None, index=None): failover_value = {} if leader: failover_value['leader'] = leader if candidate: failover_value['member'] = candidate if scheduled_at: failover_value['scheduled_at'] = scheduled_at.isoformat() return self.set_failover_value(json.dumps(failover_value, separators=(',', ':')), index) @abc.abstractmethod def set_config_value(self, value, index=None): """Create or update `/config` key""" @abc.abstractmethod def touch_member(self, data, permanent=False): """Update member key in DCS. This method should create or update key with the name = '/members/' + `~self._name` and value = data in a given DCS. :param data: information about instance (including connection strings) :param ttl: ttl for member key, optional parameter. If it is None `~self.member_ttl will be used` :param permanent: if set to `!True`, the member key will never expire. Used in patronictl for the external master. :returns: `!True` on success otherwise `!False` """ @abc.abstractmethod def take_leader(self): """This method should create leader key with value = `~self._name` and ttl=`~self.ttl` Since it could be called only on initial cluster bootstrap it could create this key regardless, overwriting the key if necessary.""" @abc.abstractmethod def initialize(self, create_new=True, sysid=""): """Race for cluster initialization. :param create_new: False if the key should already exist (in the case we are setting the system_id) :param sysid: PostgreSQL cluster system identifier, if specified, is written to the key :returns: `!True` if key has been created successfully. this method should create atomically initialize key and return `!True` otherwise it should return `!False`""" @abc.abstractmethod def delete_leader(self): """Voluntarily remove leader key from DCS This method should remove leader key if current instance is the leader""" @abc.abstractmethod def cancel_initialization(self): """ Removes the initialize key for a cluster """ @abc.abstractmethod def delete_cluster(self): """Delete cluster from DCS""" @staticmethod def sync_state(leader, sync_standby): """Build sync_state dict""" return {'leader': leader, 'sync_standby': sync_standby} def write_sync_state(self, leader, sync_standby, index=None): sync_value = self.sync_state(leader, sync_standby) return self.set_sync_state_value(json.dumps(sync_value, separators=(',', ':')), index) @abc.abstractmethod def set_history_value(self, value): """""" @abc.abstractmethod def set_sync_state_value(self, value, index=None): """""" @abc.abstractmethod def delete_sync_state(self, index=None): """""" def watch(self, leader_index, timeout): """If the current node is a master it should just sleep. Any other node should watch for changes of leader key with a given timeout :param leader_index: index of a leader key :param timeout: timeout in seconds :returns: `!True` if you would like to reschedule the next run of ha cycle""" self.event.wait(timeout) return self.event.isSet()
class __CardMonitoringThreadSingleton(Thread): """The real card monitoring thread class. A single instance of this class is created by the public CardMonitoringThread class. """ def __init__(self, observable): Thread.__init__(self) self.observable = observable self.stopEvent = Event() self.stopEvent.clear() self.cards = [] self.setDaemon(True) # the actual monitoring thread def run(self): """Runs until stopEvent is notified, and notify observers of all card insertion/removal. """ self.cardrequest = CardRequest(timeout=0.1) while self.stopEvent.isSet() != 1: try: currentcards = self.cardrequest.waitforcardevent() addedcards = [] for card in currentcards: if not self.cards.__contains__(card): addedcards.append(card) removedcards = [] for card in self.cards: if not currentcards.__contains__(card): removedcards.append(card) if addedcards != [] or removedcards != []: self.cards = currentcards self.observable.setChanged() self.observable.notifyObservers( (addedcards, removedcards)) # when CardMonitoringThread.__del__() is invoked in # response to shutdown, e.g., when execution of the # program is done, other globals referenced by the # __del__() method may already have been deleted. # this causes ReaderMonitoringThread.run() to except # with a TypeError or AttributeError except TypeError: pass except AttributeError: pass except SmartcardException as exc: # FIXME Tighten the exceptions caught by this block traceback.print_exc() # Most likely raised during interpreter shutdown due # to unclean exit which failed to remove all observers. # To solve this, we set the stop event and pass the # exception to let the thread finish gracefully. if exc.hresult == SCARD_E_NO_SERVICE: self.stopEvent.set() # stop the thread by signaling stopEvent def stop(self): self.stopEvent.set()
class PlotModeBase(PlotMode): """ Intended parent class for plotting modes. Provides base functionality in conjunction with its parent, PlotMode. """ ## ## Class-Level Attributes ## """ The following attributes are meant to be set at the class level, and serve as parameters to the plot mode registry (in PlotMode). See plot_modes.py for concrete examples. """ """ i_vars 'x' for Cartesian2D 'xy' for Cartesian3D etc. d_vars 'y' for Cartesian2D 'r' for Polar etc. """ i_vars, d_vars = '', '' """ intervals Default intervals for each i_var, and in the same order. Specified [min, max, steps]. No variable can be given (it is bound later). """ intervals = [] """ aliases A list of strings which can be used to access this mode. 'cartesian' for Cartesian2D and Cartesian3D 'polar' for Polar 'cylindrical', 'polar' for Cylindrical Note that _init_mode chooses the first alias in the list as the mode's primary_alias, which will be displayed to the end user in certain contexts. """ aliases = [] """ is_default Whether to set this mode as the default for arguments passed to PlotMode() containing the same number of d_vars as this mode and at most the same number of i_vars. """ is_default = False """ All of the above attributes are defined in PlotMode. The following ones are specific to PlotModeBase. """ """ A list of the render styles. Do not modify. """ styles = {'wireframe': 1, 'solid': 2, 'both': 3} """ style_override Always use this style if not blank. """ style_override = '' """ default_wireframe_color default_solid_color Can be used when color is None or being calculated. Used by PlotCurve and PlotSurface, but not anywhere in PlotModeBase. """ default_wireframe_color = (0.85, 0.85, 0.85) default_solid_color = (0.6, 0.6, 0.9) default_rot_preset = 'xy' ## ## Instance-Level Attributes ## ## 'Abstract' member functions def _get_evaluator(self): if self.use_lambda_eval: try: e = self._get_lambda_evaluator() return e except Exception: warnings.warn("\nWarning: creating lambda evaluator failed. " "Falling back on sympy subs evaluator.") return self._get_sympy_evaluator() def _get_sympy_evaluator(self): raise NotImplementedError() def _get_lambda_evaluator(self): raise NotImplementedError() def _on_calculate_verts(self): raise NotImplementedError() def _on_calculate_cverts(self): raise NotImplementedError() ## Base member functions def __init__(self, *args, **kwargs): self.verts = [] self.cverts = [] self.bounds = [[S.Infinity, -S.Infinity, 0], [S.Infinity, -S.Infinity, 0], [S.Infinity, -S.Infinity, 0]] self.cbounds = [[S.Infinity, -S.Infinity, 0], [S.Infinity, -S.Infinity, 0], [S.Infinity, -S.Infinity, 0]] self._draw_lock = RLock() self._calculating_verts = Event() self._calculating_cverts = Event() self._calculating_verts_pos = 0.0 self._calculating_verts_len = 0.0 self._calculating_cverts_pos = 0.0 self._calculating_cverts_len = 0.0 self._max_render_stack_size = 3 self._draw_wireframe = [-1] self._draw_solid = [-1] self._style = None self._color = None self.predraw = [] self.postdraw = [] self.use_lambda_eval = self.options.pop('use_sympy_eval', None) is None self.style = self.options.pop('style', '') self.color = self.options.pop('color', 'rainbow') self.bounds_callback = kwargs.pop('bounds_callback', None) self._on_calculate() def synchronized(f): def w(self, *args, **kwargs): self._draw_lock.acquire() try: r = f(self, *args, **kwargs) return r finally: self._draw_lock.release() return w @synchronized def push_wireframe(self, function): """ Push a function which performs gl commands used to build a display list. (The list is built outside of the function) """ assert callable(function) self._draw_wireframe.append(function) if len(self._draw_wireframe) > self._max_render_stack_size: del self._draw_wireframe[1] # leave marker element @synchronized def push_solid(self, function): """ Push a function which performs gl commands used to build a display list. (The list is built outside of the function) """ assert callable(function) self._draw_solid.append(function) if len(self._draw_solid) > self._max_render_stack_size: del self._draw_solid[1] # leave marker element def _create_display_list(self, function): dl = glGenLists(1) glNewList(dl, GL_COMPILE) function() glEndList() return dl def _render_stack_top(self, render_stack): top = render_stack[-1] if top == -1: return -1 # nothing to display elif callable(top): dl = self._create_display_list(top) render_stack[-1] = (dl, top) return dl # display newly added list elif len(top) == 2: if GL_TRUE == glIsList(top[0]): return top[0] # display stored list dl = self._create_display_list(top[1]) render_stack[-1] = (dl, top[1]) return dl # display regenerated list def _draw_solid_display_list(self, dl): glPushAttrib(GL_ENABLE_BIT | GL_POLYGON_BIT) glPolygonMode(GL_FRONT_AND_BACK, GL_FILL) glCallList(dl) glPopAttrib() def _draw_wireframe_display_list(self, dl): glPushAttrib(GL_ENABLE_BIT | GL_POLYGON_BIT) glPolygonMode(GL_FRONT_AND_BACK, GL_LINE) glEnable(GL_POLYGON_OFFSET_LINE) glPolygonOffset(-0.005, -50.0) glCallList(dl) glPopAttrib() @synchronized def draw(self): for f in self.predraw: if callable(f): f() if self.style_override: style = self.styles[self.style_override] else: style = self.styles[self._style] # Draw solid component if style includes solid if style & 2: dl = self._render_stack_top(self._draw_solid) if dl > 0 and GL_TRUE == glIsList(dl): self._draw_solid_display_list(dl) # Draw wireframe component if style includes wireframe if style & 1: dl = self._render_stack_top(self._draw_wireframe) if dl > 0 and GL_TRUE == glIsList(dl): self._draw_wireframe_display_list(dl) for f in self.postdraw: if callable(f): f() def _on_change_color(self, color): Thread(target=self._calculate_cverts).start() def _on_calculate(self): Thread(target=self._calculate_all).start() def _calculate_all(self): self._calculate_verts() self._calculate_cverts() def _calculate_verts(self): if self._calculating_verts.isSet(): return self._calculating_verts.set() try: self._on_calculate_verts() finally: self._calculating_verts.clear() if callable(self.bounds_callback): self.bounds_callback() def _calculate_cverts(self): if self._calculating_verts.isSet(): return while self._calculating_cverts.isSet(): sleep(0) # wait for previous calculation self._calculating_cverts.set() try: self._on_calculate_cverts() finally: self._calculating_cverts.clear() def _get_calculating_verts(self): return self._calculating_verts.isSet() def _get_calculating_verts_pos(self): return self._calculating_verts_pos def _get_calculating_verts_len(self): return self._calculating_verts_len def _get_calculating_cverts(self): return self._calculating_cverts.isSet() def _get_calculating_cverts_pos(self): return self._calculating_cverts_pos def _get_calculating_cverts_len(self): return self._calculating_cverts_len ## Property handlers def _get_style(self): return self._style @synchronized def _set_style(self, v): if v is None: return if v == '': step_max = 0 for i in self.intervals: if i.v_steps is None: continue step_max = max([step_max, int(i.v_steps)]) v = ['both', 'solid'][step_max > 40] #try: if v not in self.styles: raise ValueError("v should be there in self.styles") if v == self._style: return self._style = v #except Exception as e: #raise RuntimeError(("Style change failed. " # "Reason: %s is not a valid " # "style. Use one of %s.") % # (str(v), ', '.join(self.styles.iterkeys()))) def _get_color(self): return self._color @synchronized def _set_color(self, v): try: if v is not None: if is_sequence(v): v = ColorScheme(*v) else: v = ColorScheme(v) if repr(v) == repr(self._color): return self._on_change_color(v) self._color = v except Exception as e: raise RuntimeError(("Color change failed. " "Reason: %s" % (str(e)))) style = property(_get_style, _set_style) color = property(_get_color, _set_color) calculating_verts = property(_get_calculating_verts) calculating_verts_pos = property(_get_calculating_verts_pos) calculating_verts_len = property(_get_calculating_verts_len) calculating_cverts = property(_get_calculating_cverts) calculating_cverts_pos = property(_get_calculating_cverts_pos) calculating_cverts_len = property(_get_calculating_cverts_len) ## String representations def __str__(self): f = ", ".join(str(d) for d in self.d_vars) o = "'mode=%s'" % (self.primary_alias) return ", ".join([f, o]) def __repr__(self): f = ", ".join(str(d) for d in self.d_vars) i = ", ".join(str(i) for i in self.intervals) d = [('mode', self.primary_alias), ('color', str(self.color)), ('style', str(self.style))] o = "'%s'" % (("; ".join("%s=%s" % (k, v) for k, v in d if v != 'None'))) return ", ".join([f, i, o])
class Client(object): def __init__(self, sock=None): """ Initialize the client. The socket parameter can be an already initialized socket. Should be used to pass specialized socket objects like SSL sockets. """ if sock is None: sock = self._sock = socket.socket() else: self._sock = sock self._map = {} # Because we don't know how large the frames are # we have to use non bolocking IO sock.setblocking(False) # We have our own timeout for operations we # pretend to be synchronous (like connect). self._timeout = None self._connected = Event() self._subscriptions = {} self._aclient = None self._adisp = None self._inbox = deque() @property def outgoing(self): return self._adisp.outgoing def _registerSubscription(self, sub): self._subscriptions[sub.id] = sub def _unregisterSubscription(self, sub): del self._subscriptions[sub.id] @property def connected(self): return self._connected.isSet() def handle_connect(self, aclient, frame): self._connected.set() def handle_message(self, aclient, frame): self._inbox.append(frame) def process(self, timeout=None): if timeout is None: timeout = self._timeout asyncore.loop(use_poll=True, timeout=timeout, map=self._map, count=1) def connect(self, address, hostname): sock = self._sock self._aclient = AsyncClient(self, hostname) adisp = self._adisp = AsyncDispatcher(self._aclient) disp = self._disp = Dispatcher(adisp, sock, self._map) sock.setblocking(True) disp.connect(address) sock.setblocking(False) self.recv() # wait for CONNECTED msg if not self._connected.isSet(): sock.close() raise socket.error() def recv(self): timeout = self._timeout s = monotonic_time() duration = 0 while timeout is None or duration < timeout: try: return self._inbox.popleft() except IndexError: td = timeout - duration if timeout is not None else None self.process(td) duration = monotonic_time() - s return None def put_subscribe(self, destination, ack=None): subid = self._aclient.subscribe(self._adisp, destination, ack) sub = _Subscription(self, subid, ack) self._registerSubscription(sub) return sub def put_send(self, destination, data="", headers=None): self._aclient.send(self._adisp, destination, data, headers) def put(self, frame): self._adisp.send_raw(frame) def send(self): disp = self._disp timeout = self._timeout duration = 0 s = monotonic_time() while ((timeout is None or duration < timeout) and (disp.writable() or not self._connected.isSet())): td = timeout - duration if timeout is not None else None self.process(td) duration = monotonic_time() - s def gettimout(self): return self._timeout def settimeout(self, value): self._timeout = value
class IdleThread(object): def __init__(self, parent, folder=None): """If invoked without 'folder', perform a NOOP and wait for self.stop() to be called. If invoked with folder, switch to IDLE mode and synchronize once we have a new message""" self.parent = parent self.folder = folder self.stop_sig = Event() self.ui = getglobalui() if folder is None: self.thread = Thread(target=self.noop) else: self.thread = Thread(target=self.__idle) self.thread.setDaemon(1) def start(self): self.thread.start() def stop(self): self.stop_sig.set() def join(self): self.thread.join() def noop(self): # TODO: AFAIK this is not optimal, we will send a NOOP on one # random connection (ie not enough to keep all connections # open). In case we do the noop multiple times, we can well use # the same connection every time, as we get a random one. This # function should IMHO send a noop on ALL available connections # to the server. imapobj = self.parent.acquireconnection() try: imapobj.noop() except imapobj.abort: self.ui.warn('Attempting NOOP on dropped connection %s' % imapobj.identifier) self.parent.releaseconnection(imapobj, True) imapobj = None finally: if imapobj: self.parent.releaseconnection(imapobj) self.stop_sig.wait() # wait until we are supposed to quit def __dosync(self): remoterepos = self.parent.repos account = remoterepos.account localrepos = account.localrepos remoterepos = account.remoterepos statusrepos = account.statusrepos remotefolder = remoterepos.getfolder(self.folder) hook = account.getconf('presynchook', '') account.callhook(hook) offlineimap.accounts.syncfolder(account, remotefolder, quick=False) hook = account.getconf('postsynchook', '') account.callhook(hook) ui = getglobalui() ui.unregisterthread(currentThread()) #syncfolder registered the thread def __idle(self): """Invoke IDLE mode until timeout or self.stop() is invoked.""" def callback(args): """IDLE callback function invoked by imaplib2. This is invoked when a) The IMAP server tells us something while in IDLE mode, b) we get an Exception (e.g. on dropped connections, or c) the standard imaplib IDLE timeout of 29 minutes kicks in.""" result, cb_arg, exc_data = args if exc_data is None and not self.stop_sig.isSet(): # No Exception, and we are not supposed to stop: self.needsync = True self.stop_sig.set() # Continue to sync. def noop(imapobj): """Factorize the noop code.""" try: # End IDLE mode with noop, imapobj can point to a dropped conn. imapobj.noop() except imapobj.abort: self.ui.warn('Attempting NOOP on dropped connection %s' % imapobj.identifier) self.parent.releaseconnection(imapobj, True) else: self.parent.releaseconnection(imapobj) while not self.stop_sig.isSet(): self.needsync = False success = False # Successfully selected FOLDER? while not success: imapobj = self.parent.acquireconnection() try: imapobj.select(self.folder) except OfflineImapError as e: if e.severity == OfflineImapError.ERROR.FOLDER_RETRY: # Connection closed, release connection and retry. self.ui.error(e, exc_info()[2]) self.parent.releaseconnection(imapobj, True) elif e.severity == OfflineImapError.ERROR.FOLDER: # Just continue the process on such error for now. self.ui.error(e, exc_info()[2]) else: # Stops future attempts to sync this account. raise else: success = True if "IDLE" in imapobj.capabilities: imapobj.idle(callback=callback) else: self.ui.warn("IMAP IDLE not supported on server '%s'." "Sleep until next refresh cycle." % imapobj.identifier) noop(imapobj) #XXX: why? self.stop_sig.wait() # self.stop() or IDLE callback are invoked. noop(imapobj) if self.needsync: # Here not via self.stop, but because IDLE responded. Do # another round and invoke actual syncing. self.stop_sig.clear() self.__dosync()
class Engine(object): """ A Engine starts working by starting a number of downloader threads and a number of parser threads. It keeps crawling internet from a given set of seeds until being stopped. """ def __init__(self, nDownloader = DEFAULT_DOWNLOADERS, manager = DEFAULT_MANAGER, \ regPort = DEFAULT_REG_PORT, dbPort = DEFAULT_DB_PORT, urlPort = None, pagePort = None): """ Initialize a crawler object. --------- Param -------- nDownloader (int): the nubmer of downloader threads. manager: the host on which manager is started. regPort: the port on which manager expects connection requests. urlPort: the port on which this worker sends url to manager. pagePort: the port on which this worker sends page to manager. --------- Return -------- None. """ ## prepare the url frontier and page queue self._pageQ = Queue(MAX_PAGE_QSIZE) self._urlFrontier = Frontier(3*nDownloader, MAX_URL_QSIZE, \ keyFunc=lambda url: urllib2.Request(url).get_host(), \ priorityFunc=self.getLastVisitTime) self._visitSite = {} self._lock = RLock() self._stopEvent = Event() ## prepare filters filetypeFilter = urlFilter.FileTypeFilter(True, ['text/html']) robotFilter = urlFilter.RobotFilter(Downloader.DEFAULT_USER_AGENT) self._urlDupEliminator = urlFilter.DupEliminator() self._urlFrontier.addFilter(filetypeFilter.disallow) self._urlFrontier.addFilter(self._urlDupEliminator.seenBefore) # self._urlFrontier.addFilter(robotFilter.disallow) ## initialize sockets. self._manager = manager self._regPort = regPort self._urlPort = urlPort self._thisHost = socket.gethostbyname(socket.gethostname()) self._dbclient = MongoClient(manager, dbPort) context = zmq.Context() self._regSocket = context.socket(zmq.REQ) self._regSocket.connect("tcp://%s:%d" % (manager, self._regPort)) self._urlPushSocket = context.socket(zmq.PUSH) self._urlPullSocket = context.socket(zmq.PULL) if (self._urlPort is None): self._urlPort = self._urlPullSocket.bind_to_random_port( "tcp://%s" % self._thisHost) else: self._urlPullSocket.bind("tcp://*:%d" % (self._urlPort)) ## prepare log files if (not os.path.exists("log")): os.makedirs("log") self._logger = logging.getLogger("engine") self._logger.addHandler( logging.FileHandler( os.path.abspath("log/engine%d.log" % self._urlPort))) self._logger.setLevel(logging.WARNING) parseLogger = logging.getLogger("parser") parseLogger.addHandler( logging.FileHandler(os.path.abspath("log/parser.log"))) parseLogger.setLevel(logging.WARNING) ## create threads for downloading and parsing tasks self._downloaders = [] for i in range(nDownloader): downloader = Downloader(self._urlFrontier, self._pageQ, self._logger, self.updateLastVisitTime) downloader.daemon = True self._downloaders.append(downloader) self._parser = Parser(self._pageQ, self._urlPushSocket, self._dbclient, parseLogger) def log(self, level, msg): """ Log info/warning/error message in log file. """ if (self._logger is not None): if level == logging.INFO: self._logger.info("[%s] INFO: %s" % (time.ctime(), msg)) elif level == logging.WARNING: self._logger.warn("[%s] WARNING: %s" % (time.ctime(), msg)) else: self._logger.error("[%s] ERROR: %s" % (tme.ctime(), msg)) def _register(self): """ Request connection to master. """ self._regSocket.send("REG %s %d" % (self._thisHost, self._urlPort)) response = self._regSocket.recv() managerurlPort = response.split()[1] self.log(logging.INFO, "success to connect to manager:%s" % managerurlPort) self._urlPushSocket.connect("tcp://%s:%s" % (self._manager, managerurlPort)) def start(self): """ Start crawling. """ print "[%s] : Crawler is started!" % time.ctime() self._register() self._dataAcceptor = Thread(target=self._recvData) self._dataAcceptor.daemon = True self._dataAcceptor.start() for downloader in self._downloaders: downloader.start() self._parser.start() def _recvData(self): """ Keeps listening to urlPort. Process each arrival data. """ while (not self._stopEvent.isSet()): urlSet = self._urlPullSocket.recv_pyobj() self.log(logging.INFO, "received %s" % urlSet) for url in urlSet: try: self._urlFrontier.put(url) except Full: time.sleep(5) def stop(self): """ Stop crawling. """ print "[%s] : Crawler is stopped!" % time.ctime() self._stopEvent.set() self._parser.stop() self._parser.join() self._dump() self._regSocket.send("UNREG %s %d" % (self._thisHost, self._urlPort)) response = self._regSocket.recv() self.log(logging.INFO, "received %s" % response) def _dump(self): """ Dump the ready_to_be_crawled urls to log file. """ total = self._urlDupEliminator.size() left = self._urlFrontier.size() print "%d url discovered, but only %d downloaded and %d ready for downloading." % ( total, total - left, left) unvisited = set() while (self._urlFrontier.size() > 0): unvisited.add(self._urlFrontier.get()) self._urlPushSocket.send_pyobj(unvisited) unvisited.clear() def getLastVisitTime(self, site): """ Return the last time a site was visited. """ self._lock.acquire() lastVT = 0 site = hashlib.sha1(site).hexdigest() if (self._visitSite.has_key(site)): lastVT = self._visitSite[site] self._lock.release() return lastVT def updateLastVisitTime(self, site): """ Return the last time a site was visited. """ self._lock.acquire() site = hashlib.sha1(site).hexdigest() self._visitSite[site] = time.time() self._lock.release()
class StatusMonitor(Loggable): # valve_manager = None _stop_evt = None _clients = List state_freq = Int(3) checksum_freq = Int(3) lock_freq = Int(5) owner_freq = Int(5) update_period = Int(1) # def __init__(self, *args, **kw): # super(StatusMonitor, self).__init__(*args, **kw) # self._clients = [] def start(self, oid, vm): if not self._clients: p = self.update_period s, c, l, o = self.state_freq, self.checksum_freq, self.lock_freq, self.owner_freq self.debug('StatusMonitor period={}. ' 'Frequencies(state={}, checksum={}, lock={}, owner={})'.format(p, s, c, l, o)) if self._stop_evt: self._stop_evt.set() self._stop_evt.wait(self.update_period) self._stop_evt = Event() t = Thread(target=self._run, args=(vm,)) t.setDaemon(True) t.start() # self._iter(1, vm) else: self.debug('Monitor already running') if oid not in self._clients: self._clients.append(oid) def isAlive(self): if self._stop_evt: return not self._stop_evt.isSet() def stop(self, oid): try: self._clients.remove(oid) except ValueError: pass if not self._clients: self._stop_evt.set() self.debug('Status monitor stopped') else: self.debug('Alive clients {}'.format(self._clients)) def _run(self, vm): i = 0 while 1: if self._stop_evt.is_set(): break if not self._iter(i, vm): break time.sleep(self.update_period) if i > 100: i = 0 i += 1 self.debug('Status monitor finished') def _iter(self, i, vm): self.debug('status monitor iteration i={}'.format(i)) if self._stop_evt.is_set(): return if vm is None: self.debug('No valve manager') return if self.state_freq and not i % self.state_freq: self.debug('load valve states') vm.load_valve_states() if self.lock_freq and not i % self.lock_freq: self.debug('load lock states') vm.load_valve_lock_states() if self.owner_freq and not i % self.owner_freq: self.debug('load owners') vm.load_valve_owners() if self.checksum_freq and not i % self.checksum_freq: if not vm.state_checksum: self.debug('State checksum failed') return not self._stop_evt.is_set()
class CookExecutor(pm.Executor): """This class is responsible for launching the task sent by the scheduler. It implements the Executor methods.""" def __init__(self, stop_signal, config): self.completed_signal = Event() self.config = config self.disconnect_signal = Event() self.stop_signal = stop_signal self.reregister_signal = None def registered(self, driver, executor_info, framework_info, agent_info): logging.info( 'Executor registered executor={}, framework={}, agent={}'.format( executor_info['executor_id']['value'], framework_info['id'], agent_info['id']['value'])) env = os.environ if 'EXECUTOR_TEST_EXIT' in env: # When running in docker, if the container exits too quickly it's logged in mesos as container launch failed # instead of mesos executor terminated. This sleep ensures that we have the correct reason code for our # integration tests. time.sleep(5) exit_code = int(env['EXECUTOR_TEST_EXIT']) logging.warn( 'Exiting with code {} from EXECUTOR_TEST_EXIT environment variable' .format(exit_code)) os._exit(exit_code) def reregistered(self, driver, agent_info): logging.info('Executor re-registered agent={}'.format(agent_info)) if self.config.checkpoint: if self.reregister_signal is not None: logging.info( 'Executor checkpointing is enabled. Notifying on reregister_signal' ) self.reregister_signal.set() self.reregister_signal = None else: logging.error( 'Checkpointing is enabled but reregister_signal is None. Unable to notify!' ) def disconnected(self, driver): logging.info('Mesos requested executor to disconnect') if self.config.checkpoint: if self.reregister_signal is None: logging.info( 'Executor checkpointing is enabled. Waiting for agent recovery.' ) new_event = Event() self.reregister_signal = new_event await_reregister(new_event, self.config.recovery_timeout_ms / 1000, self.stop_signal, self.disconnect_signal) else: logging.info( 'Checkpointing is enabled. Already launched await_reregister thread.' ) else: logging.info( 'Executor checkpointing is not enabled. Terminating task.') self.disconnect_signal.set() self.stop_signal.set() def launchTask(self, driver, task): logging.info('Driver {} launching task {}'.format(driver, task)) stop_signal = self.stop_signal completed_signal = self.completed_signal config = self.config task_thread = Thread(target=manage_task, args=(driver, task, stop_signal, completed_signal, config)) task_thread.daemon = True task_thread.start() def killTask(self, driver, task_id): logging.info( 'Mesos requested executor to kill task {}'.format(task_id)) task_id_str = task_id['value'] if 'value' in task_id else task_id grace_period = os.environ.get('MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD', '') cio.print_and_log( 'Received kill for task {} with grace period of {}'.format( task_id_str, grace_period)) self.stop_signal.set() def shutdown(self, driver): logging.info('Mesos requested executor to shutdown') self.stop_signal.set() def error(self, driver, message): logging.error(message) super().error(driver, message) def await_completion(self): """ Blocks until the internal flag completed_signal is set. The completed_signal Event is expected to be set by manage_task. """ logging.info('Waiting for CookExecutor to complete...') self.completed_signal.wait() logging.info('CookExecutor has completed') def await_disconnect(self): """ Blocks until the internal flag disconnect_signal is set or the disconnect grace period expires. The disconnect grace period is computed based on whether stop_signal is set. """ disconnect_grace_secs = cook.TERMINATE_GRACE_SECS if self.stop_signal.isSet( ) else cook.DAEMON_GRACE_SECS if not self.disconnect_signal.isSet(): logging.info( 'Waiting up to {} second(s) for CookExecutor to disconnect'. format(disconnect_grace_secs)) self.disconnect_signal.wait(disconnect_grace_secs) if not self.disconnect_signal.isSet(): logging.info( 'CookExecutor did not disconnect in {} seconds'.format( disconnect_grace_secs))
class Listener(Thread): def __init__(self, host=HOST): super(Listener, self).__init__() self.host = host self._stop_event = Event() self.clear_time = None self.s = socket.socket() self.queue = collections.deque() self.port = get_ephemeral_port() self.s.bind((self.host, self.port)) def run(self): if DEBUG: logger.info('listening on %s:%s' % (self.host, self.port)) self.s.listen(5) while True: if self.stopped: return c, addr = self.s.accept() if DEBUG: logger.info('got connection %s' % str(addr)) data = c.recv(1024) if DEBUG: logger.info('got data: %s' % str(data)) if data == TERMINATOR: self.stop() return elif data == CLEAR: if DEBUG: logger.info('clearing') self.clear_time = time.time() else: self.queue.appendleft(data) c.close() def put_data(self, data): s = socket.socket() s.connect((self.host, self.port)) s.send(data) s.close() def get_data(self): """ pops the latest off the queue, or None is there is none """ try: data = self.queue.pop() except IndexError: return None, None try: data = cPickle.loads(data) except: try: data = data.decode('utf-8') except: pass if DEBUG: logger.info('got %s' % str(data)) t = None if isinstance(data, TimedMsg): d = data.value t = data.time elif isinstance(data, string_types): try: d = json.loads(data) except: d = data else: d = data return d, t def _process_chunk(self, d, t): if t is not None: if DEBUG: logger.info('diff %s' % (t - self.clear_time)) if t < self.clear_time: if DEBUG: logger.info('%s < %s' % (t, self.clear_time)) logger.info('discarding cleared %s' % d) return True else: if DEBUG: logger.info('removed clearing') self.clear_time = None # unset as we've got one after last clear else: if DEBUG: logger.info('removed clearing (nmsg with no time)') self.clear_time = None return False def receive(self, timeout=TIMEOUT_DEFAULT): if timeout is not None: MAX_COUNT = int(timeout) * 10 d = None count = 0 while d is None and count < MAX_COUNT: d, t = self.get_data() if d is None: sleep(.1) if timeout is not None: count += 1 elif self.clear_time is not None and self._process_chunk(d, t): count = 0 d = None return d def send(self, data, timeout=TIMEOUT_DEFAULT): # @UnusedVariable payload = TimedMsg(data).pickled() if DEBUG: logger.info('sending %s' % str(data)) self.put_data(payload) def clear_queue(self): self.put_data(CLEAR) time.sleep(.05) def stop(self): self.s.shutdown(socket.SHUT_WR) self.s.close() self._stop_event.set() @property def stopped(self): return self._stop_event.isSet()
class InfluxClient(): def __init__(self, host, username, password, database, ssl=False, timeout=60, port=8086): self.connectionCredentials = { 'host': host, 'username': username, 'password': password, 'database': database, 'ssl': ssl, 'timeout': timeout, 'port': port, 'verify_ssl': True } self.connect() self.dataBucket = defaultdict(queue.Queue) self.dataPackedQueue = queue.Queue() self.endEvent = Event() self.bucketThread = Thread(target=self.prepareSendBatch) self.bucketThread.daemon = True self.bucketThread.start() self.dataSendNumThreads = 2 self.dataSendThreads = [] for numThread in range(self.dataSendNumThreads): t = Thread(target=self.sendData) t.daemon = True t.start() self.dataSendThreads.append(t) def connect(self): self.client = InfluxDBClient(**self.connectionCredentials) def writeData(self, measurement, tags, fields, measureTime=None, retentionPolicy=None): if not measurement or not type(tags) is dict or not type( fields) is dict: logger.warning("no data to insert. tags: %s. fields: %s" % (tags, fields)) return if not measurement: logger.warning("no measurement to insert") return body = {"measurement": measurement, "fields": fields, "tags": tags} if not measureTime: measureTime = int(time.time() * 1000000) body["time"] = measureTime self.dataBucket[retentionPolicy].put(body) def getBucketLen(self): dataBucketLen = 0 for bucket in self.dataBucket.values(): dataBucketLen += bucket.qsize() return dataBucketLen def prepareSendBatch(self): dataChunk = 5000 sleepTime = 5 while not self.endEvent.isSet() or self.getBucketLen(): # Sleep for some time and listen for the end event for _ in range(sleepTime): if self.endEvent.isSet(): break time.sleep(1) for retentionPolicy, dataQueue in self.dataBucket.items(): bucketEmpty = False while not bucketEmpty: dataToSend = [] for numOfItemsRetrieved in range(dataChunk): try: dataToSend.append(dataQueue.get(False)) except queue.Empty: bucketEmpty = True break if not dataToSend: break self.dataPackedQueue.put((dataToSend, retentionPolicy)) logger.debug( f"Added new package of {len(dataToSend)} values") for _ in range(self.dataSendNumThreads): self.dataPackedQueue.put(None) def sendData(self): client = InfluxDBClient(**self.connectionCredentials) while True: d = self.dataPackedQueue.get() if d is None: break dataToSend, rp = d logger.debug(f"Sending {len(dataToSend)} values") try: assert (client.write_points(dataToSend, time_precision='u', retention_policy=rp, batch_size=10000)) except: logger.error("Exception when saving the data", exc_info=True) # If the batch fails try to send the values one by one for value in dataToSend: try: assert (client.write_points([value], time_precision='u', retention_policy=rp)) except: logger.error(f"Ignoring value: {value}") client.close() def query(self, query, database=None): try: result = self.client.query(query, database=database, epoch="s") return result except KeyboardInterrupt: raise KeyboardInterrupt() except: logger.error("Exception when query the data.", exc_info=True) self.connect() def close(self, *args): self.endEvent.set() self.bucketThread.join() for t in self.dataSendThreads: t.join() self.client.close() logger.info("Influx closed")
class LocalDisplay(Thread): """ Class for facilitating the local display of inference results (as images). The class is designed to run on its own thread. In particular the class dumps the inference results into a FIFO located in the tmp directory (which lambda has access to). The results can be rendered using mplayer by typing: mplayer -demuxer lavf -lavfdopts format=mjpeg:probesize=32 /tmp/results.mjpeg """ def __init__(self, resolution): """ resolution - Desired resolution of the project stream """ # Initialize the base class, so that the object can run on its own # thread. super(LocalDisplay, self).__init__() # List of valid resolutions RESOLUTION = { '1080p': (1920, 1080), '720p': (1280, 720), '480p': (858, 480) } if resolution not in RESOLUTION: raise Exception("Invalid resolution") self.resolution = RESOLUTION[resolution] # Initialize the default image to be a white canvas. Clients # will update the image when ready. self.frame = cv2.imencode('.jpg', 255 * np.ones([640, 480, 3]))[1] self.stop_request = Event() def run(self): """ Overridden method that continually dumps images to the desired FIFO file. """ # Path to the FIFO file. The lambda only has permissions to the tmp # directory. Pointing to a FIFO file in another directory # will cause the lambda to crash. result_path = '/tmp/results.mjpeg' # Create the FIFO file if it doesn't exist. if not os.path.exists(result_path): os.mkfifo(result_path) # This call will block until a consumer is available with open(result_path, 'w') as fifo_file: while not self.stop_request.isSet(): try: # Write the data to the FIFO file. This call will block # meaning the code will come to a halt here until a consumer # is available. fifo_file.write(self.frame.tobytes()) except IOError: continue def set_frame_data(self, frame): """ Method updates the image data. This currently encodes the numpy array to jpg but can be modified to support other encodings. frame - Numpy array containing the image data of the next frame in the project stream. """ ret, jpeg = cv2.imencode('.jpg', cv2.resize(frame, self.resolution)) if not ret: raise Exception('Failed to set frame data') self.frame = jpeg def join(self): self.stop_request.set()
class FakePacketSniffer(Thread): """ A fake packet sniffing thread. This thread returns fake Wi-Fi ESSIDs for development and test purposes. """ def __init__(self, new_packets, **kwargs): super().__init__() self.new_packets = new_packets self.stop_sniffer = Event() self.exception = None self.debug = kwargs.get("debug", False) from faker import Faker from faker_wifi_essid import WifiESSID self.fake = Faker() self.fake.add_provider(WifiESSID) def run(self): from time import sleep try: while not self.stop_sniffer.isSet(): sleep(1) self.new_packet() except Exception as e: self.exception = e if self.debug: print("[!] Exception: " + str(e)) def join(self, timeout=None): """ Stops the fake packet sniffer. """ self.stop_sniffer.set() super().join(timeout) def new_packet(self): """ Adds a new fake packet to the queue to be processed. """ fake_probe_req = RadioTap() \ / Dot11( addr1="ff:ff:ff:ff:ff:ff", addr2=self.fake.mac_address(), addr3=self.fake.mac_address() ) \ / Dot11ProbeReq() \ / Dot11Elt( info=self.fake.wifi_essid() ) self.new_packets.put(fake_probe_req) def should_stop_sniffer(self, packet): """ Returns true if the fake sniffer should be stopped and false otherwise. """ return self.stop_sniffer.isSet() def get_exception(self): """ Returns the raised exception if any, otherwise returns none. """ return self.exception
class Task(Thread): """ Base class for all tasks """ DEFAULT_TIMEOUT = 60 * 60 * 24 # 1 Day TEST_TIMEOUT = 30 # test task timeout DNS_TIMEOUT = 10 # DNS request timeout SOCKET_TIMEOUT = 2 # socket timeout HTTP_TIMEOUT = 30 # HTTP timeout SMTP_TIMEOUT = 10 # SMTP timeout PARSE_FILES = True # read & parse all input files by default EXPAND_TARGETS = True # expand ip networks and ip ranges SYSTEM_LIBRARY_PATH = "/opt/gtta/scripts/system/lib" USER_LIBRARY_PATH = "/opt/gtta/scripts/lib" MULTITHREADED = False THREAD_COUNT = 10 TEST_TARGETS = ["google.com"] def __init__(self, worker=False): """ Constructor """ super(Task, self).__init__() self.arguments = None self.targets = None self.target = None self.host = None self.ip = None self.proto = None self.port = None self.timeout = None self.lang = None self.test_mode = False self.error = False self.produced_output = False self._stop = Event() self._result = None self._worker = worker def _check_stop(self): """ Check if thread should be stopped """ if self._stop.isSet(): raise TaskTimeout def _write_result(self, str): """ Write result to result file or print it to output """ if not str: return self.produced_output = True if self._result: self._result.write(str.encode('utf-8')) self._result.write('\n') self._result.flush() else: print str.encode('utf-8') stdout.flush() def _expand_targets(self): targets = [] for target in self.targets: # IP network if re.match('^\d+\.\d+\.\d+\.\d+\/(3[0-2]|2[0-9]{1}|[01]?[0-9])$', target): for ip in IPNetwork(target): targets.append('%s' % ip) # IP range elif re.match('^\d+\.\d+\.\d+\.\d+\s*\-\s*\d+\.\d+\.\d+\.\d+$', target): target = target.replace(" ", "") scope = target.split('-') for ip in IPRange(scope[0], scope[1]): targets.append('%s' % ip) else: targets.append(target) self.targets = targets def stop(self): """ Thread stop function """ self._stop.set() def parse_input(self): """ Parses input arguments """ self.arguments = [] if len(argv) < 3: raise NotEnoughArguments( 'At least 2 command line arguments should be specified.') # parse the first file with hostname or IP lines = open(argv[1], 'r').read().split('\n') lines = map(lambda x: x.replace('\r', ''), lines) if len(lines) < 5: raise InvalidTargetFile( 'Target file should contain at least 5 lines.') if not lines[0]: raise InvalidTargetFile( 'Target file should contain either host name or IP address of the target host on the 1st line.' ) targets = lines[0].split(",") self.targets = [] for target in targets: target = target.strip() if target: self.targets.append(target) if self.EXPAND_TARGETS: self._expand_targets() self.proto = lines[1] try: if lines[2]: self.port = int(lines[2]) else: self.port = 0 except ValueError: self.port = 0 if not lines[3]: raise InvalidTargetFile( 'Target file should contain language name on the 4th line.') self.lang = lines[3] try: if len(lines[4]) > 0: self.timeout = int(lines[4]) else: self.timeout = self.DEFAULT_TIMEOUT except ValueError: self.timeout = self.DEFAULT_TIMEOUT # open output file self._result = open(argv[2], 'w') # parse the remaining arguments for arg in argv[3:]: if self.PARSE_FILES: lines = open(arg, 'r').read().split('\n') lines = map(lambda x: x.replace('\r', ''), lines) self.arguments.append(lines) else: self.arguments.append(arg) def _get_library_path(self, library): """ Get library path """ path = None if isdir("%s/%s" % (self.SYSTEM_LIBRARY_PATH, library)): path = self.SYSTEM_LIBRARY_PATH elif isdir("%s/%s" % (self.USER_LIBRARY_PATH, library)): path = self.USER_LIBRARY_PATH if not path: raise Exception("Library not exists: %s" % library) return "%s/%s" % (path, library) def test(self): """ Test the task """ raise Exception("Test not implemented.") def main(self, *args): """ Main function """ raise Exception("Main function not implemented.") def _run_target(self, target): """Run task for a single target""" self.target = target try: inet_aton(target) self.ip = target except: self.host = target if self.test_mode: self.test() self.produced_output = True else: self.main(*self.arguments) def _run_singlethreaded(self): """Run single-threaded task""" map(self._run_target, self.targets) def _run_multithreaded(self): """Run multi-threaded task""" global target_queue map(target_queue.put, self.targets) thread_count = min(self.THREAD_COUNT, len(self.targets)) thread_pool = [] for i in xrange(thread_count): thread = type(self)(worker=True) thread.arguments = self.arguments thread.proto = self.proto thread.lang = self.lang thread.test_mode = self.test_mode thread.start() thread_pool.append(thread) for thread in thread_pool: thread.join(self.timeout) if thread.isAlive(): thread.stop() result = thread.worker_result() if result.endswith("\n"): result = result[:-1] self._write_result(result) def _run_worker(self): """Worker thread main function""" global target_queue self._result = StringIO() while True: try: task = target_queue.get(block=False) except: task = None if not task: break self._run_target(task) def worker_result(self): """Get worker result""" if type(self._result) == file: return "" return self._result.getvalue() def run(self): """ Run a task """ try: # run worker for multithreaded if self._worker: self._run_worker() return if self.test_mode: self.targets = self.TEST_TARGETS if self.MULTITHREADED: self._run_multithreaded() else: self._run_singlethreaded() except TaskTimeout: pass except Exception, e: error_str = e.__class__.__name__ if str(e): error_str += ': %s' % str(e) self._write_result(error_str) self.error = True
class ChipStack(object): def __init__(self, installDefaultLogHandler=True, bluetoothAdapter=0): builtins.enableDebugMode = False self.networkLock = Lock() self.completeEvent = Event() self.commissioningCompleteEvent = Event() self._ChipStackLib = None self._chipDLLPath = None self.devMgr = None self.callbackRes = None self.commissioningEventRes = None self._activeLogFunct = None self.addModulePrefixToLogMessage = True # Locate and load the chip shared library. self._loadLib() # Arrange to log output from the chip library to a python logger object with the # name 'chip.ChipStack'. If desired, applications can override this behavior by # setting self.logger to a different python logger object, or by calling setLogFunct() # with their own logging function. self.logger = logging.getLogger(__name__) self.setLogFunct(self.defaultLogFunct) # Determine if there are already handlers installed for the logger. Python 3.5+ # has a method for this; on older versions the check has to be done manually. if hasattr(self.logger, "hasHandlers"): hasHandlers = self.logger.hasHandlers() else: hasHandlers = False logger = self.logger while logger is not None: if len(logger.handlers) > 0: hasHandlers = True break if not logger.propagate: break logger = logger.parent # If a logging handler has not already been initialized for 'chip.ChipStack', # or any one of its parent loggers, automatically configure a handler to log to # stdout. This maintains compatibility with a number of applications which expect # chip log output to go to stdout by default. # # This behavior can be overridden in a variety of ways: # - Initialize a different log handler before ChipStack is initialized. # - Pass installDefaultLogHandler=False when initializing ChipStack. # - Replace the StreamHandler on self.logger with a different handler object. # - Set a different Formatter object on the existing StreamHandler object. # - Reconfigure the existing ChipLogFormatter object. # - Configure chip to call an application-specific logging function by # calling self.setLogFunct(). # - Call self.setLogFunct(None), which will configure the chip library # to log directly to stdout, bypassing python altogether. # if installDefaultLogHandler and not hasHandlers: logHandler = logging.StreamHandler(stream=sys.stdout) logHandler.setFormatter(ChipLogFormatter()) self.logger.addHandler(logHandler) self.logger.setLevel(logging.DEBUG) def HandleComplete(appState, reqState): self.callbackRes = True self.completeEvent.set() def HandleError(appState, reqState, err, devStatusPtr): self.callbackRes = self.ErrorToException(err, devStatusPtr) self.completeEvent.set() @_ChipThreadTaskRunnerFunct def HandleChipThreadRun(callback): callback() self.cbHandleChipThreadRun = HandleChipThreadRun self.cbHandleComplete = _CompleteFunct(HandleComplete) self.cbHandleError = _ErrorFunct(HandleError) # set by other modules(BLE) that require service by thread while thread blocks. self.blockingCB = None # Initialize the chip library res = self._ChipStackLib.pychip_Stack_Init() if res != 0: raise self.ErrorToException(res) res = self._ChipStackLib.pychip_BLEMgrImpl_ConfigureBle( bluetoothAdapter) if res != 0: raise self.ErrorToException(res) @property def defaultLogFunct(self): """Returns a python callable which, when called, logs a message to the python logger object currently associated with the ChipStack object. The returned function is suitable for passing to the setLogFunct() method.""" def logFunct(timestamp, timestampUSec, moduleName, logCat, message): moduleName = ChipUtility.CStringToString(moduleName) message = ChipUtility.CStringToString(message) if self.addModulePrefixToLogMessage: message = "CHIP:%s: %s" % (moduleName, message) logLevel = LogCategory.categoryToLogLevel(logCat) msgAttrs = { "chip-module": moduleName, "timestamp": timestamp, "timestamp-usec": timestampUSec, } self.logger.log(logLevel, message, extra=msgAttrs) return logFunct def setLogFunct(self, logFunct): """Set the function used by the chip library to log messages. The supplied object must be a python callable that accepts the following arguments: timestamp (integer) timestampUS (integer) module name (encoded UTF-8 string) log category (integer) message (encoded UTF-8 string) Specifying None configures the chip library to log directly to stdout.""" if logFunct is None: logFunct = 0 if not isinstance(logFunct, _LogMessageFunct): logFunct = _LogMessageFunct(logFunct) with self.networkLock: # NOTE: ChipStack must hold a reference to the CFUNCTYPE object while it is # set. Otherwise it may get garbage collected, and logging calls from the # chip library will fail. self._activeLogFunct = logFunct self._ChipStackLib.pychip_Stack_SetLogFunct(logFunct) def Shutdown(self): self._ChipStack.Call(lambda: self._dmLib.pychip_Stack_Shutdown()) self.networkLock = None self.completeEvent = None self._ChipStackLib = None self._chipDLLPath = None self.devMgr = None self.callbackRes = None def Call(self, callFunct): '''Run a Python function on CHIP stack, and wait for the response. This function is a wrapper of PostTaskOnChipThread, which includes some handling of application specific logics. Calling this function on CHIP on CHIP mainloop thread will cause deadlock. ''' # throw error if op in progress self.callbackRes = None self.completeEvent.clear() with self.networkLock: res = self.PostTaskOnChipThread(callFunct).Wait() self.completeEvent.set() if res == 0 and self.callbackRes != None: return self.callbackRes return res def CallAsync(self, callFunct): '''Run a Python function on CHIP stack, and wait for the application specific response. This function is a wrapper of PostTaskOnChipThread, which includes some handling of application specific logics. Calling this function on CHIP on CHIP mainloop thread will cause deadlock. ''' # throw error if op in progress self.callbackRes = None self.completeEvent.clear() with self.networkLock: res = self.PostTaskOnChipThread(callFunct).Wait() if res != 0: self.completeEvent.set() raise self.ErrorToException(res) while not self.completeEvent.isSet(): if self.blockingCB: self.blockingCB() self.completeEvent.wait(0.05) if isinstance(self.callbackRes, ChipStackException): raise self.callbackRes return self.callbackRes def PostTaskOnChipThread(self, callFunct) -> AsyncCallableHandle: '''Run a Python function on CHIP stack, and wait for the response. This function will post a task on CHIP mainloop, and return an object with Wait() method for getting the result. Calling Wait inside main loop will cause deadlock. ''' callObj = AsyncCallableHandle(callFunct) pythonapi.Py_IncRef(py_object(callObj)) res = self._ChipStackLib.pychip_DeviceController_PostTaskOnChipThread( self.cbHandleChipThreadRun, py_object(callObj)) if res != 0: pythonapi.Py_DecRef(py_object(callObj)) raise self.ErrorToException(res) return callObj def ErrorToException(self, err, devStatusPtr=None): if err == 0x2C and devStatusPtr: devStatus = devStatusPtr.contents msg = ChipUtility.CStringToString( ( self._ChipStackLib.pychip_Stack_StatusReportToString( devStatus.ProfileId, devStatus.StatusCode ) ) ) sysErrorCode = ( devStatus.SysErrorCode if ( devStatus.SysErrorCode != 0) else None ) if sysErrorCode != None: msg = msg + " (system err %d)" % (sysErrorCode) return DeviceError( devStatus.ProfileId, devStatus.StatusCode, sysErrorCode, msg ) else: return ChipStackError( err, ChipUtility.CStringToString( (self._ChipStackLib.pychip_Stack_ErrorToString(err)) ), ) def LocateChipDLL(self): if self._chipDLLPath: return self._chipDLLPath scriptDir = os.path.dirname(os.path.abspath(__file__)) # When properly installed in the chip package, the Chip Device Manager DLL will # be located in the package root directory, along side the package's # modules. dmDLLPath = os.path.join(scriptDir, ChipStackDLLBaseName) if os.path.exists(dmDLLPath): self._chipDLLPath = dmDLLPath return self._chipDLLPath # For the convenience of developers, search the list of parent paths relative to the # running script looking for an CHIP build directory containing the Chip Device # Manager DLL. This makes it possible to import and use the ChipDeviceMgr module # directly from a built copy of the CHIP source tree. buildMachineGlob = "%s-*-%s*" % (platform.machine(), platform.system().lower()) relDMDLLPathGlob = os.path.join( "build", buildMachineGlob, "src/controller/python/.libs", ChipStackDLLBaseName, ) for dir in self._AllDirsToRoot(scriptDir): dmDLLPathGlob = os.path.join(dir, relDMDLLPathGlob) for dmDLLPath in glob.glob(dmDLLPathGlob): if os.path.exists(dmDLLPath): self._chipDLLPath = dmDLLPath return self._chipDLLPath raise Exception( "Unable to locate Chip Device Manager DLL (%s); expected location: %s" % (ChipStackDLLBaseName, scriptDir) ) # ----- Private Members ----- def _AllDirsToRoot(self, dir): dir = os.path.abspath(dir) while True: yield dir parent = os.path.dirname(dir) if parent == "" or parent == dir: break dir = parent def _loadLib(self): if self._ChipStackLib is None: self._ChipStackLib = CDLL(self.LocateChipDLL()) self._ChipStackLib.pychip_Stack_Init.argtypes = [] self._ChipStackLib.pychip_Stack_Init.restype = c_uint32 self._ChipStackLib.pychip_Stack_Shutdown.argtypes = [] self._ChipStackLib.pychip_Stack_Shutdown.restype = c_uint32 self._ChipStackLib.pychip_Stack_StatusReportToString.argtypes = [ c_uint32, c_uint16, ] self._ChipStackLib.pychip_Stack_StatusReportToString.restype = c_char_p self._ChipStackLib.pychip_Stack_ErrorToString.argtypes = [c_uint32] self._ChipStackLib.pychip_Stack_ErrorToString.restype = c_char_p self._ChipStackLib.pychip_Stack_SetLogFunct.argtypes = [ _LogMessageFunct] self._ChipStackLib.pychip_Stack_SetLogFunct.restype = c_uint32 self._ChipStackLib.pychip_BLEMgrImpl_ConfigureBle.argtypes = [ c_uint32] self._ChipStackLib.pychip_BLEMgrImpl_ConfigureBle.restype = c_uint32 self._ChipStackLib.pychip_DeviceController_PostTaskOnChipThread.argtypes = [ _ChipThreadTaskRunnerFunct, py_object] self._ChipStackLib.pychip_DeviceController_PostTaskOnChipThread.restype = c_uint32
class ProbeRequestParser(Thread): """ A Wi-Fi probe request parsing thread. """ def __init__(self, new_packets, **kwargs): super().__init__() self.new_packets = new_packets self.essid_filters = kwargs.get("essid_filters", None) self.essid_regex = kwargs.get("essid_regex", None) self.ignore_case = kwargs.get("ignore_case", False) self.display_func = kwargs.get("display_func", lambda p: None) self.storage_func = kwargs.get("storage_func", lambda p: None) self.debug = kwargs.get("debug", False) self.stop_parser = Event() if self.debug: print("[!] ESSID filters: " + str(self.essid_filters)) print("[!] ESSID regex: " + str(self.essid_regex)) print("[!] Ignore case: " + str(self.ignore_case)) if self.essid_regex is not None: if self.ignore_case: self.essid_regex = rcompile(self.essid_regex, IGNORECASE) else: self.essid_regex = rcompile(self.essid_regex) else: self.essid_regex = None def run(self): # The parser continues to do its job even after the call of the # join method if the queue is not empty. while not self.stop_parser.isSet() or not self.new_packets.empty(): try: packet = self.new_packets.get(timeout=1) probe_request = self.parse(packet) if probe_request is None: continue if not probe_request.essid: continue if self.essid_filters is not None and not probe_request.essid in self.essid_filters: continue if self.essid_regex is not None and not match(self.essid_regex, probe_request.essid): continue self.display_func(probe_request) self.storage_func(probe_request) self.new_packets.task_done() except Empty: pass def join(self, timeout=None): """ Stops the probe request parsing thread. """ self.stop_parser.set() super().join(timeout) @staticmethod def parse(packet): """ Parses the raw packet and returns a probe request object. """ if packet.haslayer(Dot11ProbeReq): timestamp = packet.getlayer(RadioTap).time s_mac = packet.getlayer(RadioTap).addr2 essid = packet.getlayer(Dot11ProbeReq).info.decode("utf-8") return ProbeRequest(timestamp, s_mac, essid) else: return None
class Policy(ConfigurableNode): implements(IPolicy) security = SecurityInformation.from_default() secured_by(security) security.protect('context', 'Manage Users') security.protect('acquires', 'Manage Users') security.protect('rolemap', 'Manage Users') security.make_private('readonly') def __init__(self, *args): self.readonly = [] self.__lock = RLock() self.__active = Event() self.acquires = True self.context = "" self.filter = "" self.is_default = False self.uses_expression = False self.configured_expression = False self.rolemap = _RoleMap() super(Policy, self).__init__(*args) def get_readOnly(self, attr): return attr in self.readonly security.protect('configure', 'Manage Users') def configure(self, config): """ Configure node with from configuration dictionary. The policy may be configured to use regular expressions instead of simple node-URL contexts in two different ways. The configuration dictionary may include a True/False value "uses_expression", in which case the context regular expression is taken verbatim from the "context" parameter, and an optional filter expression from the "filter" parameter. Or the expression(s) may be encoded directly into the "context" value using the following format: REGEX: '<context regex>', FILTER: '<filter regex>' Note that the ", FILTER: [...]" part is completely optional and will default to no filter. """ for attrname in self.readonly: current = getattr(self, attrname, None) incoming = config.get(attrname) if None not in (current, incoming) and (current != incoming): message = 'Attribute "%s" is readonly for Policy "%s". ' message += 'Overriding new value %s with current value %s.' message = message % (attrname, self.name, incoming, current) msglog.log('broadway', msglog.types.WARN, message) config[attrname] = current self.acquires = as_boolean(config.get('acquires', self.acquires)) if not self.is_default: self.is_default = as_boolean(config.get('is_default', False)) self.configure_context(config) super(Policy, self).configure(config) # Getting set of mappings whose value is a callback function. # The assumption is being made here that such mappings are # those considered 'read-only.' self.__lock.acquire() try: inherent = self.rolemap.callable_subset() self.rolemap = _RoleMap(config.get('rolemap', self.rolemap)) self.rolemap.update(inherent) finally: self.__lock.release() self.__verify_setup() def configure_context(self, config): context = config.get('context', self.context) if not context: raise TypeError("Policy must have non-empty context") if config.has_key("uses_expression"): self.uses_expression = as_boolean(config["uses_expression"]) else: self.uses_expression = False if self.uses_expression: self.context = context if config.has_key("filter"): self.filter = config["filter"] self.configured_expression = True else: self.configured_expression = False if isinstance(context, str) and context.startswith("REGEX:"): self.uses_expression = True # Regular expression encoded into context # as "REGEX: '<expr>', FILTER: '<expr>'. REGEX, FILTER = "REGEX", "FILTER" expressions = eval("{%s}" % context) self.context = expressions.get(REGEX, "") self.filter = expressions.get(FILTER, "") elif context != self.context: if not isinstance(context, str): if ISecurityContext.providedBy(context): context = context.url else: raise TypeError("Context must be string or object" " providing ISecurityContext.") self.filter = "" self.context = context return self.context, self.filter def context_configuration(self, config): if self.configured_expression: config["context"] = self.context config["filter"] = self.filter config["uses_expression"] = "1" elif self.uses_expression: context = "REGEX: %r, FILTER: %r" % (self.context, self.filter) config["context"] = context else: config["context"] = self.context return config def configuration(self): config = super(Policy, self).configuration() config['acquires'] = str(self.acquires) config['is_default'] = as_boolean(self.is_default) self.context_configuration(config) self.__lock.acquire() try: config['rolemap'] = self.rolemap.copy() finally: self.__lock.release() return config def start(self): if not self.context: raise TypeError("Policy must have non-empty context") self.__lock.acquire() try: self.__verify_setup() self.__active.set() finally: self.__lock.release() super(Policy, self).start() def stop(self): if not self.parent.default is self: self.__active.clear() else: message = 'Policy "%s" is system default and has ' % self.name message += 'not been deactivated, although it has been stopped.' msglog.log('broadway', msglog.types.WARN, message) super(Policy, self).stop() def is_running(self): return self.__active.isSet() def is_removable(self): return not len(self.readonly) def is_configurable(self): return not self.is_default security.protect('prune', 'Manage Users') def prune(self): if not self.is_removable(): error = '%s "%s" is not removable.' raise TypeError(error % (type(self).__name__, self.name)) return super(Policy, self).prune() def __verify_setup(self): try: node = as_node(self.context) except: raise EInvalidValue("Invalid Context", "%s" % self.context) policies = list(self.parent.get_policies()) policies.remove(self) for policy in policies: if policy.uses_expression != self.uses_expression: continue if policy.context != self.context: continue # Policy contexts are same. message = ("Conflicting policy configurations detected." "Policies %s and %s have same context: '%s'.") msglog.log("broadway", msglog.types.WARN, message % (self, policy, self.context)) if policy.is_running(): message = "Running policy '%s' already has context: %s" raise TypeError(message % (policy.name, self.context)) else: message = ("%s ignoring context conflict: " "conflicting policy is not running.") msglog.log("broadway", msglog.types.WARN, message % self) permissions = self.parent.get_permissions() rolemap = self.rolemap.copy() for role, granted in rolemap.items(): if not self.parent.parent.role_manager.has_role(role): message = 'Policy "%s" ' % self.url message += 'removing role "%s". ' % role message += 'It does not exist.' msglog.log('broadway', msglog.types.WARN, message) del (self.rolemap[role]) elif isinstance(granted, (list, tuple)): for permission in granted: if permission not in permissions: message = 'Policy "%s" ' % self.url message += 'removing permission "%s" ' % permission message += 'from role "%s". ' % role message += 'Permission does not exist.' msglog.log('broadway', msglog.types.WARN, message) self.rolemap[role].remove(permission) return def get_permissions(self, role): if not isinstance(role, str): role = role.name return self.rolemap.get(role) security.protect('set_permissions', 'Manage Users') def set_permissions(self, role, *permissions): ## # Permissions parameter can be one of three things: # - A single list or tuple object, whose items will # replace the permissions tuple. # - A variable number of permission strings. # - A single callable object which will be called # and whose return value will be returned anytime # the permissions for this role are queried. if not isinstance(role, str): role = role.name if len(permissions) == 1: if type(permissions[0]) in (list, tuple): permissions = permissions[0][:] elif callable(permissions[0]): permissions = permissions[0] if not self.parent.parent.role_manager.has_role(role): raise ValueError('Role "%s" does not exist.' % role) if isinstance(permissions, (list, tuple)): defined = self.parent.get_permissions() for permission in permissions: if permission not in defined: raise ValueError('Permission "%s" not defined.' % permission) self.__lock.acquire() try: inherent = self.rolemap.callable_subset() self.rolemap[role] = permissions self.rolemap.update(inherent) finally: self.__lock.release() if inherent.has_key(role): message = 'Permissions for role "%s" in policy "%s" ' message += 'cannot be changed. An attempt has been ignored.' message = message % (role, self.name) msglog.log('broadway', msglog.types.WARN, message) return def rank_match(self, context): if not self.is_running(): raise ENotRunning('Not started. This may mean start failed.') if not isinstance(context, str): context = ISecurityContext(context).url if self.uses_expression: rank = self.rank_expression(context) else: rank = self.rank_overlap(context) return rank def rank_expression(self, context): context = urllib.unquote(context) match = re.match(self.context, context) if not match: count = 0 elif self.filter and re.match(self.filter, context): count = 0 else: matching = match.group() count = matching.count("/") if len(matching) > 1: count = count + 1 return count def rank_overlap(self, context): context = urllib.unquote(context) if not context.startswith(self.context): count = 0 else: count = self.context.count('/') if len(self.context) > 1: count = count + 1 return count
class PacketSniffer(Thread): """ A packet sniffing thread. """ def __init__(self, interface, new_packets, **kwargs): super().__init__() self.daemon = True self.interface = interface self.new_packets = new_packets self.mac_exclusions = kwargs.get("mac_exclusions", None) self.mac_filters = kwargs.get("mac_filters", None) self.debug = kwargs.get("debug", False) self.frame_filters = "type mgt subtype probe-req" self.socket = None self.stop_sniffer = Event() self.exception = None if self.mac_exclusions is not None: self.frame_filters += " and not (" for i, station in enumerate(self.mac_exclusions): if i == 0: self.frame_filters += "ether src host {s_mac}".format(s_mac=station) else: self.frame_filters += " || ether src host {s_mac}".format(s_mac=station) self.frame_filters += ")" if self.mac_filters is not None: self.frame_filters += " and (" for i, station in enumerate(self.mac_filters): if i == 0: self.frame_filters += "ether src host {s_mac}".format(s_mac=station) else: self.frame_filters += " || ether src host {s_mac}".format(s_mac=station) self.frame_filters += ")" if self.debug: print("[!] Frame filters: " + self.frame_filters) def run(self): try: self.socket = conf.L2listen( type=ETH_P_ALL, iface=self.interface, filter=self.frame_filters ) sniff( opened_socket=self.socket, store=False, prn=self.new_packet, stop_filter=self.should_stop_sniffer ) except Exception as e: self.exception = e if self.debug: print("[!] Exception: " + str(e)) def join(self, timeout=None): """ Stops the packet sniffer. """ self.stop_sniffer.set() super().join(timeout) def new_packet(self, packet): """ Adds a new packet to the queue to be processed. """ self.new_packets.put(packet) def should_stop_sniffer(self, packet): """ Returns true if the sniffer should be stopped and false otherwise. """ return self.stop_sniffer.isSet() def get_exception(self): """ Returns the raised exception if any, otherwise returns none. """ return self.exception
class HTTPServer(Thread): """ A simple, minimal HTTP server """ CONFIG_KEY = 'http' THREAD_NAME = 'simple-http-server' def __init__(self, config): self.__log.info("Instantiating Simple HTTP Server") Thread.__init__(self) self.setName(self.THREAD_NAME) self.setDaemon(True) self._stop = Event() self.config = config def stop(self): self._stop.set() def stopped(self): return self._stop.isSet() def _request_handler(self, *args): RequestHandler(self.config, *args) def run(self): try: self.__log.info("Staring Simple HTTP Server") host_key = 'http_host' port_key = 'http_port' ssl_port_key = 'http_ssl_port' cert_file_key = 'http_cert_file' host = self.config.get( self.CONFIG_KEY, host_key ) # Utils.get_env_var(host_key, self.config.get(self.CONFIG_KEY, host_key), self.__log) ssl_enabled = bool( self.config.get(self.CONFIG_KEY, 'http_ssl_enabled')) port = self.config.get( self.CONFIG_KEY, port_key) if ssl_enabled else self.config.get( self.CONFIG_KEY, ssl_port_key) cert_file = self.config.get(self.CONFIG_KEY, cert_file_key) self.__log.debug("host: {}".format(host)) self.__log.debug("port: {}".format(port)) self.__log.debug("cert file: {}".format(cert_file)) self.httpd = BaseHTTPServer.HTTPServer( (host, int(port)), self._request_handler ) # SimpleHTTPServer.SimpleHTTPRequestHandler if ssl_enabled == 'True': self.__log.info("Wrapping SSL") self.httpd.socket = ssl.wrap_socket(self.httpd.socket, server_side=True, certfile=cert_file) self.httpd.serve_forever() # self.http_thread = Thread(target=self.httpd.serve_forever(), ) # self.http_thread.daemon = True # Starting the HTTP server # try: # self.http_thread.start() # except Exception as e: # self.__log.error("Error while starting HTTP Server") # self.__log.error(str(e)) # self.httpd.shutdown() # Wait until HTTP server is ready time.sleep(1) except Exception as e: self.__log.error( "Error while staring Simple HTTP Server. Error: {}".format( str(e))) return False def stop(self): self.stop_httpd() self._stop.set() def stop_httpd(self): self.__log.info("Shtting down HTTP Server...") self.httpd.shutdown() #self.http_thread.stop() self.__log.info("HTTP Server Shutdown completed")
stream.stop_stream() stream.close() audio.terminate() if __name__ == '__main__': try: # event to synchronise threads to ensure they start at the same time e = Event() t1 = Thread(target=other_sensors, args=(e,)) t2 = Thread(target=noise_sensor, args=(e,)) # starting thread 1 t1.start() # starting thread 2 t2.start() # Keep the main thread running, otherwise signals are ignored. while True: # <- swap with "for frame in camera.capture_continuous()" loop" # image-processing can go here... time.sleep(1) except KeyboardInterrupt: print('Closing threads...') stop_threads = True # if program closed before thread 1 flags calibration is done, ensure thread 2 is unblocked and hence able to run to completion if not e.isSet(): e.set() # Wait for the threads to close t1.join() # starting thread 2 t2.join()
class Monitor(Thread): """ Initiates a connection to the database to store telemetry data at regular time intervals :param running: an event controlling the process operation :param appconfig: the application configuration object :param q: the telemetry data queue :param id: the recorder thread identifier :param enabled: a flag indicating if the monitor is enabled """ def __init__(self, q, appconfig, name=""): """ Initializes the recorder object :param q: the telemetry data queue :param appconfig: the application configuration object :param name: a name that can be attributed to the monitor """ Thread.__init__(self) self.running = Event() if name != "": self.id = name self.q = q self.appconfig = appconfig self.enabled = False self.lastdrivingtime = datetime.now() self.gpslogging = 0 self.gpsconnected = False def init_connection(self): """Initializes the connection to the GPSD service""" try: # Attempts to create a connection to the GPSD server logger.info( f"connect to gpsd server: {self.appconfig.gpsd_ip_address}:{self.appconfig.gpsd_port}" ) gpsd.connect(self.appconfig.gpsd_ip_address, self.appconfig.gpsd_port) self.gpsconnected = True return 0 except Exception as error: logger.error(f"Exception: {str(error)}") self.gpsconnected = False return -1 def start(self): """Starts the monitor thread""" self.running.set() self.enabled = True super(Monitor, self).start() def run(self): """ Runs the monitor infinite loop """ # Opens gpsd connection # insert data in database while (self.running.isSet()): retval = self.report_current_location() if retval < 0: logger.error("error report location, try reconnection!") self.init_connection() time.sleep(self.appconfig.monitor_delay) def report_current_location(self): """ Gets the current location data from the GPSD and reports it to the shared queue as a Location object :return: 0 if success or -1 if failure or an exception arises """ try: # Get current GPS position packet = gpsd.get_current() if self.lastdrivingtime is not None: delta = datetime.now() - self.lastdrivingtime # Unpack location parameters mode = packet.mode latitude = packet.lat longitude = packet.lon utc_time = packet.time track = packet.track hspeed = packet.hspeed if abs(hspeed) * 3.6 > SPEEDTHRESH_KMH: self.lastdrivingtime = datetime.now() self.gpslogging += 1 else: if self.lastdrivingtime is not None: delta = datetime.now() - self.lastdrivingtime if delta.seconds > WAIT_STOPLOGGING: logger.info( f"stop logging after {WAIT_STOPLOGGING} seconds not driving" ) self.gpslogging = 0 else: logger.info( f"wait for stop logging after driving {WAIT_STOPLOGGING - delta.seconds}" ) altitude = None climb = None if packet.mode == 3: altitude = packet.alt climb = packet.climb loc = location.Location(latitude=latitude, longitude=longitude, altitude=altitude, heading=track, \ climb=climb, horizontal_speed=hspeed, mode=mode, utc_time=utc_time) logger.info(str(loc)) # TODO: remove after DEBUG logger.info(packet.map_url()) # Put the location instance in the shared queue if self.gpslogging > 3 and packet.mode == 3: self.q.put(loc) logger.info("put gpsdata to queue!") else: logger.info("device is not driving!") return 0 except Exception as inst: logger.error( f'Type: {type(inst)} -- Args: {inst.args} -- Instance: {inst}') self.gpslogging = 0 return -1 def stop(self): """Stops the monitor thread""" self.running.clear() # disable the monitor self.enabled = False
class MCMolShapeDatabase: """ Maintains a database of MCMols that can be queried by shape similarity.""" def __init__(self, itf): """ Create a MCMolShapeDatabase from the parameters specified by the OEInterface. """ self.rwlock = ReadWriteLock() self.loadedEvent = Event() self.dbname = itf.GetString("-dbase") self.moldb = oechem.OEMolDatabase() self.dbtype = GetDatabaseType(itf.GetBool("-shapeOnly")) self.shapedb = oefastrocs.OEShapeDatabase(*GetShapeDatabaseArgs(itf)) # this thread is daemonic so a KeyboardInterupt # during the load will cancel the process self.loaderThread = DatabaseLoaderThread(self.shapedb, self.moldb, self.dbname, self.loadedEvent) self.loaderThread.setDaemon(True) self.loaderThread.start() def IsLoaded(self, blocking=False): """ Return whether the server has finished loading. """ if blocking: self.loadedEvent.wait() # clean up the load waiter thread if it's still there if self.loadedEvent.isSet() and self.loaderThread is not None: self.rwlock.AcquireWriteLock() try: # typical double checked locking if self.loaderThread is not None: self.loaderThread.join() self.loaderThread = None finally: self.rwlock.ReleaseWriteLock() return self.loadedEvent.isSet() def GetBestOverlays(self, querymolstr, options, iformat, oformat): """ Return a string of the format specified by 'oformat' containing nhits overlaid confomers using querymolstr as the query interpretted as iformat. querymolstr - a string containing a molecule to use as the query options - an instance of OEShapeDatabaseOptions iformat - a string representing the file extension to parse the querymolstr as. Note: old clients could be passing .sq files, so iformat == '.oeb' will try to interpret the file as a .sq file. oformat - file format to write the results as """ timer = oechem.OEWallTimer() # make sure to wait for the load to finish blocking = True loaded = self.IsLoaded(blocking) assert loaded if iformat.startswith(".sq"): query = ReadShapeQuery(querymolstr) else: # read in query qfs = oechem.oemolistream() qfs = SetupStream(qfs, iformat) if not qfs.openstring(querymolstr): raise ValueError("Unable to open input molecule string") query = oechem.OEGraphMol() if not oechem.OEReadMolecule(qfs, query): if iformat == ".oeb": # could be an old client trying to send a .sq file. query = ReadShapeQuery(querymolstr) else: raise ValueError( "Unable to read a molecule from the string of format '%s'" % iformat) ofs = oechem.oemolostream() ofs = SetupStream(ofs, oformat) if not ofs.openstring(): raise ValueError("Unable to openstring for output") # do we only want shape based results? # this is a "Write" lock to be paranoid and not overload the GPU self.rwlock.AcquireWriteLock() try: # do search scores = self.shapedb.GetSortedScores(query, options) sys.stderr.write("%f seconds to do search\n" % timer.Elapsed()) finally: self.rwlock.ReleaseWriteLock() timer.Start() # write results for score in scores: mcmol = oechem.OEMol() if not self.moldb.GetMolecule(mcmol, score.GetMolIdx()): oechem.OEThrow.Warning( "Can't retrieve molecule %i from the OEMolDatabase, " "skipping..." % score.GetMolIdx()) continue # remove hydrogens to make output smaller, this also # ensures OEPrepareFastROCSMol will have the same output oechem.OESuppressHydrogens(mcmol) mol = oechem.OEGraphMol( mcmol.GetConf(oechem.OEHasConfIdx(score.GetConfIdx()))) oechem.OECopySDData(mol, mcmol) if options.GetSimFunc() == oefastrocs.OEShapeSimFuncType_Tanimoto: oechem.OESetSDData(mol, "ShapeTanimoto", "%.4f" % score.GetShapeTanimoto()) oechem.OESetSDData(mol, "ColorTanimoto", "%.4f" % score.GetColorTanimoto()) oechem.OESetSDData(mol, "TanimotoCombo", "%.4f" % score.GetTanimotoCombo()) else: oechem.OESetSDData(mol, "ShapeTversky", "%.4f" % score.GetShapeTversky()) oechem.OESetSDData(mol, "ColorTversky", "%.4f" % score.GetColorTversky()) oechem.OESetSDData(mol, "TverskyCombo", "%.4f" % score.GetTverskyCombo()) if options.GetInitialOrientation( ) != oefastrocs.OEFastROCSOrientation_Inertial: oechem.OEAddSDData( mol, "Opt. Starting Pos.", GetAltStartsString(options.GetInitialOrientation())) score.Transform(mol) oechem.OEWriteMolecule(ofs, mol) output = ofs.GetString() sys.stderr.write("%f seconds to write hitlist\n" % timer.Elapsed()) sys.stderr.flush() ofs.close() return output def GetName(self): self.rwlock.AcquireReadLock() try: return self.dbname finally: self.rwlock.ReleaseReadLock() def SetName(self, name): self.rwlock.AcquireWriteLock() try: self.dbname = name finally: self.rwlock.ReleaseWriteLock()
class ResultHandler(SocketServer.BaseRequestHandler): """Result handler. This handler speaks our analysis log network protocol. """ def setup(self): self.rawlogfd = None self.protocol = None self.startbuf = "" self.end_request = Event() self.done_event = Event() self.pid, self.ppid, self.procname = None, None, None self.server.register_handler(self) def finish(self): self.done_event.set() if self.protocol: self.protocol.close() if self.rawlogfd: self.rawlogfd.close() def wait_sock_or_end(self): while True: if self.end_request.isSet(): return False rs, _, _ = select.select([self.request], [], [], 1) if rs: return True def seek(self, pos): pass def read(self, length): buf = "" while len(buf) < length: if not self.wait_sock_or_end(): raise Disconnect() tmp = self.request.recv(length - len(buf)) if not tmp: raise Disconnect() buf += tmp if isinstance(self.protocol, BsonParser): if self.rawlogfd: self.rawlogfd.write(buf) else: self.startbuf += buf return buf def read_any(self): if not self.wait_sock_or_end(): raise Disconnect() tmp = self.request.recv(BUFSIZE) if not tmp: raise Disconnect() return tmp def read_newline(self): buf = "" while "\n" not in buf: buf += self.read(1) return buf def negotiate_protocol(self): # Read until newline. buf = self.read_newline() if "BSON" in buf: self.protocol = BsonParser(self) elif "FILE" in buf: self.protocol = FileUpload(self) elif "LOG" in buf: self.protocol = LogHandler(self) else: raise CuckooOperationalError("Netlog failure, unknown " "protocol requested.") def handle(self): ip, port = self.client_address self.connect_time = datetime.datetime.now() self.storagepath = self.server.build_storage_path(ip) if not self.storagepath: return # Create all missing folders for this analysis. self.create_folders() try: # Initialize the protocol handler class for this connection. self.negotiate_protocol() for event in self.protocol: if isinstance(self.protocol, BsonParser) and event["type"] == "process": self.open_process_log(event) except CuckooResultError as e: log.warning( "ResultServer connection stopping because of " "CuckooResultError: %s.", str(e)) except (Disconnect, socket.error): pass except: log.exception("FIXME - exception in resultserver connection %s", str(self.client_address)) log.debug("Connection closed: {0}:{1}".format(ip, port)) def open_process_log(self, event): pid = event["pid"] ppid = event["ppid"] procname = event["process_name"] if self.pid is not None: log.debug( "ResultServer got a new process message but already " "has pid %d ppid %s procname %s.", pid, str(ppid), procname) raise CuckooResultError("ResultServer connection state " "inconsistent.") # Only report this process when we're tracking it. if event["track"]: log.debug("New process (pid=%s, ppid=%s, name=%s)", pid, ppid, procname) path = os.path.join(self.storagepath, "logs", str(pid) + ".bson") self.rawlogfd = open(path, "wb") self.rawlogfd.write(self.startbuf) self.pid, self.ppid, self.procname = pid, ppid, procname def create_folders(self): folders = "shots", "files", "logs", "buffer" for folder in folders: try: create_folder(self.storagepath, folder=folder) except CuckooOperationalError: log.error("Unable to create folder %s" % folder) return False
class GPU(object): """GPU is a representation of a physical gpu based on its index in the list of available gpus. Arguments --------- device_id int : Index of the GPU prevent_exceptions boolean : Silent exceptions display str : X display identifier """ def __init__(self, device_id, prevent_exceptions, display=":0"): # noqa: D107 self.id = device_id self.display = display self.check_exceptions = not prevent_exceptions self._stop = Event() self._thread = None def stop(self): """Signal the thread to stop execution.""" self._stop.set() def stopped(self): """Check if stopping execution is requested.""" return self._stop.isSet() def __setSpeed(self, speed): cmd = [ "nvidia-settings", "-c {0}".format(self.display), "-a [gpu:{0}]/GPUFanControlState=1".format(self.id), "-a [fan:{0}]/GPUTargetFanSpeed={1}".format(self.id, speed) ] # using sb.run(cmd, ...) did not work! I guess there is a conflict in -c switch. sb.run(" ".join(cmd), shell=True, stdout=sb.DEVNULL, stderr=sb.DEVNULL, check=self.check_exceptions) def __customCurveSpeed(self): nvmlInit() self._handle = nvmlDeviceGetHandleByIndex(self.id) curve = Curve() while (not self.stopped()): current_temp = self.__getTemp() new_fan_speed = curve.evaluate(current_temp) self.__setSpeed(new_fan_speed) time.sleep(1.0) nvmlShutdown() def __thread_alive(self): if self._thread and self._thread.is_alive(): return True return False def __getTemp(self): """Get temperature of the GPU.""" return nvmlDeviceGetTemperature(self._handle, NVML_TEMPERATURE_GPU) def constant(self, percentage): """Set a constant fan speed. Arguments --------- percentage int : An integer from 0 to 100 """ if self.__thread_alive(): self.stop() self._thread.join() self.__setSpeed(percentage) def aggressive(self): """Control GPU fan based on an aggressive regime. It tries to set GPU fan to a high speed as soon as temperature rises. """ if self.__thread_alive(): return self._thread = Thread(target=self.__customCurveSpeed) self._thread.daemon = True self._thread.start() def driver(self): """Return control of fan speed to the driver.""" if self.__thread_alive(): self.stop() self._thread.join() cmd = [ "nvidia-settings", "-c {0}".format(self.display), "-a [gpu:{0}]/GPUFanControlState=0".format(self.id) ] sb.run(" ".join(cmd), shell=True, stdout=sb.DEVNULL, stderr=sb.DEVNULL, check=self.check_exceptions) def __del__(self): """Make sure thread is stopped before destruction.""" if self.__thread_alive(): self._thread.stop() self._thread.join() self.driver()