class XBeeTransparentListener(Thread): def __init__(self, xbee_serial): super().__init__() self.xbser = xbee_serial self.daemon = True self.stopped = False self.pause = RLock() def run(self): while not self.stopped and self.xbser.is_open: with self.pause: try: line = self.xbser.readline() if line: print('>', line.strip()) except Exception as ex: print(str(ex)) def stop(self): self.stopped = True def pause(self): self.pause.acquire() def unpause(self): self.pause.release()
class PrintingThread(Thread): def __init__(self, printer, cmds): log.info("New printing thread, printer %s, len %d" % (printer, len(cmds))) self.cmds=cmds self.printer=printer self.lock=RLock() self.state=0 Thread.__init__(self) def run(self): printer=self.printer printer.zero() from time import time, sleep from os import path #wait=time() #log.info("Waiting for load page button.") #while path.getmtime(job_dir+"load_new_page")<wait: # sleep(1.0) #log.info("Load page button press detected. Starting to print.") for i, step in enumerate(self.cmds): printer.do(step) self.lock.acquire() self.state=i self.lock.release() printer.eject_page() def progress(self): self.lock.acquire() res=self.state self.lock.release() return res/float(len(self.cmds)-1)
class SingleIndexCache(object): def __init__(self): self.lock = RLock() self.cached_vals = [] def __enter__(self): self.lock.acquire() return self def fill(self, values): self.cached_vals = values def wipe(self): self.cached_vals = [] def get_index(self, real_funct, ind_name, **kwargs): kwargs.setdefault("max_results", 999999) if not self.cached_vals: recvs = real_funct(ind_name, **kwargs).results self.fill(recvs) def filter(self, startkey, endkey, max_results=1): c = self.cached_vals for (curr_val, obj_key) in c: if max_results == 0: break if curr_val >= startkey: if curr_val <= endkey: max_results -= 1 yield (curr_val, obj_key) else: break def __exit__(self, *args, **kwargs): self.lock.release()
class Database(object): UNICODE_TRANSLATE = {ord(u'ö'): u'o', ord(u'ä'): u'a', ord(u'ü'): u'u'} def __init__(self, filename): self.database_file = filename self.channel_map = {} self._db = None self.lock = RLock() def close(self): self.commit() cur = self.cursor(lock=True) log('Optimizing database for faster startup.', sender=self) cur.execute("VACUUM") cur.close() self.lock.release() self._db.close() self._db = None def log(self, message, *args, **kwargs): if False: try: message = message % args log('%s', message, sender=self) except TypeError, e: log('Exception in log(): %s: %s', e, message, sender=self)
class VM_Pool(object): def __init__(self, vm_map): self.proc_mgr = proc_mgmt.ProcMgr() self.vm_map = vm_map self.vm_rdy = {} self.init_map() self.pool_gate = RLock() def acquire(self, *names): self.pool_gate.acquire() for name in names: if self.vm_rdy.get(name): self.vm_rdy[name] = False return self.vm_map.get(name) self.pool_gate.release() return None def release(self, name): self.vm_rdy[name] = True def init_map(self): for name, vm_obj in self.vm_map.items(): self.vm_rdy[name] = True def __str__(self): string = 'Pool:' for vm in self.vm_map.keys(): string += vm + ": " + str(self.vm_rdy.get(vm)) + ", " return string
class ZenitherClient(): def __init__(self, robot): try: rospy.init_node('ZenitherClient') rospy.logout('ZenitherServer: Initialized Node') except rospy.ROSException: pass if robot not in zc.calib: raise RuntimeError('unknown robot') self.calib = zc.calib[robot] srv = '/zenither/move_position' rospy.wait_for_service(srv) self.move_position = rospy.ServiceProxy(srv, Float_Int) srv = '/zenither/stop' rospy.wait_for_service(srv) self.stop = rospy.ServiceProxy(srv, Float_Int) srv = '/zenither/apply_torque' rospy.wait_for_service(srv) self.apply_torque = rospy.ServiceProxy(srv, Float_Int) srv = '/zenither/torque_move_position' rospy.wait_for_service(srv) self.torque_move_position = rospy.ServiceProxy(srv, Float_Int) zenither_pose_topic = 'zenither_pose' self.h = None self.lock = RLock() rospy.Subscriber(zenither_pose_topic, FloatArray, self.pose_cb) #---------- functions to send zenither commands. ------------- def estop(self): self.stop(0) def zenith(self, torque=None): if torque == None: torque=self.calib['zenith_torque'] self.apply_torque(torque) def nadir(self, torque=None): if torque == None: torque=self.calib['nadir_torque'] self.apply_torque(torque) #--------- zenither height functions -------------- def pose_cb(self, fa): self.lock.acquire() self.h = fa.data[0] self.lock.release() ## return the current height of the zenither. def height(self): self.lock.acquire() h = self.h self.lock.release() return h
class SimpleCSVWriter: def __init__(self, filename, fields=None): self.filename = filename self.lock = RLock() self.isFirstRow = True self.fields = fields def writerow(self, d): self.lock.acquire() fields = self.fields if self.fields is not None else d.keys() if self.isFirstRow: # dump fields f = open(self.filename , "w") writer = csv.writer(f, lineterminator="\n", quoting=csv.QUOTE_ALL) row = [k for k in fields] writer.writerow(row) f.close() self.isFirstRow = False # dump object row = [d.get(k,'') for k in fields] f = open(self.filename , "a") writer = csv.writer(f, lineterminator="\n", quoting=csv.QUOTE_ALL) writer.writerow(row) f.close() self.lock.release()
def close_dynamic_queue(self, dynamic_queue_name): if self._disconnecting: self.logger.info("Connection factory disconnecting, aborting close_dynamic_queue") return else: self.logger.log(TRACE1, "close_dynamic_queue -> not disconnecting") if not self._is_connected: # If we're not connected then all dynamic queues had been already closed. self.logger.log(TRACE1, "close_dynamic_queue -> _is_connected1 %s" % self._is_connected) return else: self.logger.log(TRACE1, "close_dynamic_queue -> _is_connected2 %s" % self._is_connected) lock = RLock() lock.acquire() try: dynamic_queue = self._open_dynamic_queues_cache[dynamic_queue_name] dynamic_queue.close() self._open_dynamic_queues_cache.pop(dynamic_queue_name, None) self._open_send_queues_cache.pop(dynamic_queue_name, None) self._open_receive_queues_cache.pop(dynamic_queue_name, None) self.logger.log(TRACE1, "Successfully closed a dynamic queue [%s]" % ( dynamic_queue_name)) finally: lock.release()
class DataSignaler(object): def __init__(self, name, pruneFunc, data): super(DataSignaler, self).__init__() assert isinstance(data,TreeFunctioned) self.data = data self.event_signaler = EventSignaler(key=name) if pruneFunc is not None: assert callable(pruneFunc) self.prune_func = pruneFunc self._lock = RLock() def add(self, value): self._lock.acquire() try: self.data.addToTreeByFunction(value) finally: self._lock.release() data = {self.event_signaler.key : {'data': self.data}} self.event_signaler.signalEvent(data) def prune(self): if self.prune_func is not None: return criticalSection(self._lock, lambda: self.prune_func(dataStructure=self.data)) def inByFunction(self, value, hashFuncList=None, depth=0): return criticalSection(self._lock, lambda: self.data.inByFunction(value, hashFuncList, depth)) def getOriginalByFunction(self, value, hashFuncList=None, depth=0): return criticalSection(self._lock, lambda: self.data.getOriginalByFunction(value, hashFuncList, depth))
class XBeeTransparentListener(Thread): def __init__(self, on_received=None): super().__init__() self.xbser = None self.on_received = on_received self.daemon = True self.stopped = False self.pause = RLock() def run(self): while not self.stopped and self.xbser.is_open: with self.pause: try: line = self.xbser.readline() if line: self.received(line) except Exception as ex: print(str(ex)) def received(self, line): """Subclasses may override this method, or provide a callback function when instance is created""" if self.on_received: self.on_received(line) else: print('[XBee]', line.strip()) def stop(self): self.stopped = True def pause(self): self.pause.acquire() def unpause(self): self.pause.release()
def open_dynamic_queue(self): if self._disconnecting: self.logger.info("Connection factory disconnecting, aborting open_dynamic_queue") return else: self.logger.log(TRACE1, "open_dynamic_queue -> not disconnecting") if not self._is_connected: self.logger.log(TRACE1, "open_dynamic_queue -> _is_connected1 %s" % self._is_connected) self._connect() self.logger.log(TRACE1, "open_dynamic_queue -> _is_connected2 %s" % self._is_connected) dynamic_queue = self.mq.Queue(self.mgr, self.dynamic_queue_template, self.CMQC.MQOO_INPUT_SHARED) # A bit hackish, but there's no other way to get its name. dynamic_queue_name = dynamic_queue._Queue__qDesc.ObjectName.strip() lock = RLock() lock.acquire() try: self._open_dynamic_queues_cache[dynamic_queue_name] = dynamic_queue finally: lock.release() self.logger.log(TRACE1, "Successfully created a dynamic queue, descriptor [%s]" % ( dynamic_queue._Queue__qDesc)) return dynamic_queue_name
class TriggerDevice(NeuroDevice): def __init__(self, freq = 256, channels = 1): self.freq = freq self.channels = channels self.header = Header(TRIGGER_HEADER) self.header.channelCount = channels for i in range(channels): self.header.channels[i].samplingFrequency = freq self.header.channels[i].label = 'TRIGGER%d' % i self.values = [0,]*self.channels self.valLock = RLock() self.queue = Queue(15) self.thread = TriggerDeviceThread(self) self.thread.start() def getValues(self): self.valLock.acquire() try: return tuple(self.values) finally: self.valLock.release() def setValues(self, val): self.valLock.acquire() try: self.values[:] = val finally: self.valLock.release() def getHeader(self): return self.header.text() def getData(self): return self.queue.get(10.0)
class ThreadData(object): def __init__(self): self._data = {} self._lock = RLock() def __setitem__(self, key, value): self._lock.acquire() thread = current_thread() if thread not in self._data: self._data[thread] = {} self._data[thread][key] = value self._lock.release() def __getitem__(self, key): thread = current_thread() return self._data[thread][key] def __delitem__(self, key): del(self[key]) def __contains__(self, key): thread = current_thread() return key in self._data[thread] def update(self, data): self._lock.acquire() thread = current_thread() self._data[thread].update(data) self._lock.release() def clean(self): thread = current_thread() if thread in self._data: del(self._data[thread])
class CacheDict(dict): _cacheTimeout = 0 _accesslock = None def __init__(self, timeout): self._cacheTimeout = timeout self._accesslock = RLock() def get(self, key): return self[key]['value'] def cache(self, key, value): self[key] = { 'time' : time.time(), 'value' : value } def isObsolete(self, key): return (not self.has_key(key) or time.time() - self[key]['time'] > self._cacheTimeout) def invalidate(self, key): if self.has_key(key): return self.pop(key)['value'] def acquire(self): self._accesslock.acquire() def release(self): self._accesslock.release()
def resetMysqlMACAndSN(self,mac): mysql = MySQLCommand(host=self.sysXMLDict['mysqlhost'], port=int(self.sysXMLDict['mysqlport']), user=self.sysXMLDict['mysqluser'], passwd=self.sysXMLDict['mysqlpassword'], db=self.sysXMLDict['mysqldatabase'], table=self.sysXMLDict['mysqltable']) mysqlConFlag = mysql.connectMysql() if not mysqlConFlag: logging.info('reset status connect failed.') return False locker = RLock() locker.acquire() resetFlag = mysql.resetMysqlMACStatusAndSN(mac=mac,stbType=self.sysXMLDict['mysqlstbtype'], poNumber=self.poNumber) if resetFlag: logging.info('reset mysql status success.') else: logging.info('reset mysql status failed.') mysql.closeMysql() locker.release() return False mysql.closeMysql() locker.release() return True
class ThreadSafeFSM(InstrumentFSM): """ A FSM class that provides thread locking in on_event to prevent simultaneous thread reentry. """ def __init__(self, states, events, enter_event, exit_event): """ """ super(ThreadSafeFSM, self).__init__(states, events, enter_event, exit_event) self._lock = RLock() def on_event(self, event, *args, **kwargs): """ """ self._lock.acquire(True) ex = None try: result = super(ThreadSafeFSM, self).on_event(event, *args, **kwargs) except Exception as ex: result = None log.error("Unhandled Exception") log.exception(ex) finally: self._lock.release() if ex: raise ex return result
class AndroidSocket(asynchat.async_chat): def __init__(self): self.logger = logging.getLogger("AndroidSocket") # Connect to the Android (when we start the async loop - this is asynchronous) asynchat.async_chat.__init__(self) self.create_socket(socket.AF_INET, socket.SOCK_STREAM) self.connect( ANDROID_HOST ) # Set up input buffer and define message terminator self.input_buffer = [] self.set_terminator("\n") self.socket_lock = RLock() # Making async_chat thread-safe def push(self, data): try: self.socket_lock.acquire() asynchat.async_chat.push(self, data) finally: self.socket_lock.release() # Making async_chat thread-safe def initiate_send(self): try: self.socket_lock.acquire() asynchat.async_chat.initiate_send(self) finally: self.socket_lock.release() def handle_error(self): self.logger.error("================ ERROR! Failed to send something! ================ ") etype, value, tb = sys.exc_info() traceback.print_exception(etype, value, tb) def handle_connect(self): # Init all the devices we need to control try: self.commandDispatcher = CommandDispatcher(self) except Exception as e: print e raise e self.logger.info("Sending CREEPER_READY status") self.push("CREEPER_READY:\n") def collect_incoming_data(self, data): self.input_buffer.append(data) def found_terminator(self): self.handle_android_command(self.input_buffer[0]) self.input_buffer = [] def handle_android_command(self, command_data): self.logger.info("Received command: %s" % command_data) self.commandDispatcher.process_command(command_data)
class _Spinner(Thread): def __init__(self, message): Thread.__init__(self) self.rlock = RLock() self.cv = Condition() self.__chars = u"⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏" if len(message) > 0: self.__message = " " + message else: self.__message = "" self.__message_length = len(self.__message) def __clear(self): stdout.write( '\b' * (self.__message_length + 2) + ' ' * (self.__message_length + 2) + '\b' * (self.__message_length + 2)) stdout.flush() def __call__(self): self.start() def start(self): self.stopFlag = 0 Thread.start(self) def stop(self): """To be called by the 'main' thread: Will block and wait for the thread to stop before returning control to 'main'.""" self.stopFlag = 1 # Wake up ahead of time if needed self.cv.acquire() self.cv.notify() self.cv.release() # Block and wait here untill thread fully exits its run method. self.rlock.acquire() def run(self): self.rlock.acquire() self.cv.acquire() stdout.write(' ' + self.__message) stdout.write('\b' * self.__message_length) stdout.flush() while 1: for char in self.__chars: self.cv.wait(0.1) if self.stopFlag: self.__clear() try : return finally : # release lock immediatley after returning self.rlock.release() stdout.write('\b') stdout.write(char) stdout.flush()
class MysqlDb(BaseDb): def __init__(self, db_uri): parts = urlsplit(db_uri) self.host = parts.hostname self.port = parts.port or 3306 self.user = parts.username self.pwd = parts.password or "" self.dbname = parts.path[1:] self.db_con_lock = RLock() self.db_con = None self.db_cur = None def acquire_cursor(self): try: self.db_con_lock.acquire() self.__get_connection() self.db_cur = self.db_con.cursor() except StandardError: if self.db_con is not None: self.db_con.close() if self.db_cur is not None: self.db_cur.close() self.db_con_lock.release() return self.db_cur def release_cursor(self): self.db_cur.close() self.db_con.commit() self.db_con.close() self.db_con = None self.db_con_lock.release() def __get_connection(self): if self.db_con is None: con = MySQLdb.connect(self.host, self.user, self.pwd, self.dbname, self.port) else: self.db_con.ping(True) con = self.db_con self.db_con = con return con def cursor(self): c = Cursor(self) return c
class Main(HTTPServlet): registerShutdown = 1 def __init__(self): HTTPServlet.__init__(self) self.blogDirectory = WeblogDirectory("../blogs.xml") self.mutex = RLock() def awake(self, transaction): # Register our shutdown handler if it hasn't already been done. This is to # make sure the databases are properly closed when the system is shutdown. self.mutex.acquire() try: if (Main.registerShutdown == 1): transaction.application().addShutDownHandler(self.blogDirectory.shutdown) Main.registerShutdown = 0 finally: self.mutex.release() def respondToGet(self, transaction): request = transaction.request() response = transaction.response() pathInfo = request.extraURLPath() try: (blog, pathInfo) = self._parsePathInfo(pathInfo) weblog = self.blogDirectory.getBlog(blog) try: stylesheet = request.field('t', "") # Extra optional argument that can be passed to the stylesheet arg = request.field('a', "") # Content query that can be applied as a final step to extract # something from the rendered content contentQuery = request.field('c', "") result = weblog.handleRequest(pathInfo, stylesheet, arg, contentQuery) # Determine the content-type for the result if (result.startswith("<?xml")): contentType = "text/xml" elif (result.startswith("<html")): contentType = "text/html" else: contentType = "text/plain" #print result response.setStatus(200, 'OK') response.setHeader('Content-type', contentType) response.setHeader('Content-length', str(len(result))) response.write(result) except NotFoundError: response.setStatus(404, 'Not Found') except KeyError, IndexError: response.setStatus(404, 'Weblog Not Found')
class ScrollableHandler(StreamHandlerNoLF): """ ScrollableHandler is a StreamHandler that specially handles scrolling (log messages at the SCROLL level). It allows you to temporarily interrupt the constant scroll with other log messages of different levels (printed at the top of the scroll area) """ # the SCROLL level (a class var) LOGFILE = 11 SCROLL = 12 SHUTDOWN = 13 NOLOGFILE = 14 def __init__(self, *args, **kwargs): self.scrollLock = RLock() self.scrollFlag = False StreamHandlerNoLF.__init__(self, *args, **kwargs) def handle(self, record): """ The 'scroll' level is a constant scroll that can be interrupted. This interruption is done via prepending text to the scroll area """ rv = self.filter(record) if rv: if record.levelno == ScrollableHandler.SCROLL: self.emitSynchronized(record) elif record.levelno == ScrollableHandler.SHUTDOWN: record.msg = '\n\n\n%s\n' % record.msg self.emitSynchronized(record) else: self.scrollLock.acquire() # If scroll is on, interrupt scroll if ScrollableHandler.scrollFlag: self.scrollHeader(record) else: # otherwise if scroll isn't on, just log the message normally self.emitSynchronized(record) self.scrollLock.release() return rv def emitSynchronized(self, record): """ Write a log message atomically. Normal python logging Handler behavior """ self.acquire() try: self.emit(record) finally: self.release() def scrollHeader(self, record): """ Print a log message so that the user can see it during a SCROLL """ msg = self.format(record).rstrip() # Scroller appends newline for us from twisted.internet import reactor if inMainThread(): # FIXME: scrollBegin() should really be creating the scroller instance # FIXME: no unicode crap from normal python log emit Hellanzb.scroller.scrollHeader(msg) else: reactor.callFromThread(Hellanzb.scroller.scrollHeader, msg)
def exit(self): global command_output_thread print("Exiting thread {0}...".format(self)) lock = RLock() lock.acquire() self.EXIT_FLAG = True pgid = self.get_pgid(True) if pgid is not None: call(["ionice", "-c", "0", "-P", str(self.get_pgid(True))]) lock.release()
def _threadCallback(self,thread): """ A callback function which gets called when a code thread terminates. """ lock = RLock() lock.acquire() if thread.failed(): self._exceptions[thread._id] = thread.exceptionInfo() self._tracebacks[thread._id] = thread.tracebackInfo() lock.release()
class Chopstick: def __init__(self, name): self.lock = RLock() self.name = name def pickUp(self): return self.lock.acquire(blocking=False) def putDown(self): self.lock.release()
class MonitorLock(object): def __init__(self): self.lock = RLock() def acquire(self, flag=True): self.lock.acquire() def release(self): self.lock.release() def MONITOR_ENTER(self): self.acquire() def MONITOR_EXIT(self): self.release()
class ThreadSafeDict(dict) : def __init__(self, * p_arg, ** n_arg) : dict.__init__(self, * p_arg, ** n_arg) self._lock = RLock() def __enter__(self) : self._lock.acquire() return self def __exit__(self, type, value, traceback) : self._lock.release()
class SerialWriteHandler: def __init__(self, ser, incomingDataHandler, input): self._incomingDataHandler = incomingDataHandler self._input = input self._serial = ser self._lock = RLock() def write(self, data): self._lock.acquire() self._serial.write(chr(HEADER_START)) self._serial.write(str(data)) self._lock.release() def writeAndWaitForAck(self, data, idToAck): self._lock.acquire() resend = True while resend: self.write(data) ack = self.waitForACK() if ack != None and ack.getIdToAck() == idToAck and ack.getReqLen() == len(data): resend = False self._lock.release() def waitForACK(self): gotHeaderStart = False incomingLength = 0 headerId = 0 data = [] timeoutCount = 3 try: while timeoutCount > 0: if gotHeaderStart: if len(data) < 1: data.append(self._input.read()) incomingLength, headerId = self._incomingDataHandler.getIncomingHeaderSizeAndId(data) elif incomingLength >= 1 and headerId == ACK_RES: for i in range(1, incomingLength): data.append(self._input.read()) ack = ACKResponse() ack.buildRequest(data) if ack.checkPackage(): return ack data = [] timeoutCount -= 1 gotHeaderStart = False else: data = [] timeoutCount -= 1 gotHeaderStart = False elif ord(self._input.read()) == HEADER_START: gotHeaderStart = True except TypeError: rospy.logerr('ACK have not been send ,retransmitting.......') return None
class Client(KRPC): def __init__(self, table): self.table = table self.lock = RLock() timer(KRPC_TIMEOUT, self.timeout) KRPC.__init__(self) def find_node(self, address, nid=None): nid = self.get_neighbor(nid) if nid else self.table.nid tid = entropy(TID_LENGTH) msg = { "t": tid, "y": "q", "q": "find_node", "a": {"id": nid, "target": random_id()} } self.send_krpc(msg, address) def bootstrap(self): for address in BOOTSTRAP_NODES: self.find_node(address) def timeout(self): if not self.join_successed: self.bootstrap() timer(KRPC_TIMEOUT, self.timeout) def run(self): self.bootstrap() while 1: time.sleep(.001) try: data, address = self.ufd.recvfrom(65536) msg = bdecode(data) self.types[msg["y"]](msg, address) except Exception: pass def foreverloop(self): self.start() while 1: time.sleep(.001) if not self.table.nodes: self.join_successed = False time.sleep(1) continue for node in self.table.nodes: self.find_node((node.ip, node.port), node.nid) self.lock.acquire() self.table.nodes = [] self.lock.release()
class RateManager: def __init__(self): self.lock = RLock() self.statusmap = {} self.currenttotal = {} self.dset = Set() self.clear_downloadstates() def add_downloadstate(self, ds): if DEBUG: print >> sys.stderr, 'RateManager: add_downloadstate', `(ds.get_download().get_def().get_infohash())` self.lock.acquire() try: d = ds.get_download() if d not in self.dset: self.statusmap[ds.get_status()].append(ds) for dir in [UPLOAD, DOWNLOAD]: self.currenttotal[dir] += ds.get_current_speed(dir) self.dset.add(d) return len(self.dset) finally: self.lock.release() def add_downloadstatelist(self, dslist): for ds in dslist: self.add_downloadstate(ds) def adjust_speeds(self): self.lock.acquire() try: self.calc_and_set_speed_limits(DOWNLOAD) self.calc_and_set_speed_limits(UPLOAD) self.clear_downloadstates() finally: self.lock.release() def clear_downloadstates(self): self.statusmap[DLSTATUS_ALLOCATING_DISKSPACE] = [] self.statusmap[DLSTATUS_WAITING4HASHCHECK] = [] self.statusmap[DLSTATUS_HASHCHECKING] = [] self.statusmap[DLSTATUS_DOWNLOADING] = [] self.statusmap[DLSTATUS_SEEDING] = [] self.statusmap[DLSTATUS_STOPPED] = [] self.statusmap[DLSTATUS_STOPPED_ON_ERROR] = [] self.statusmap[DLSTATUS_REPEXING] = [] for dir in [UPLOAD, DOWNLOAD]: self.currenttotal[dir] = 0 self.dset.clear() def calc_and_set_speed_limits(self, direct): pass
def wrapper(self, *args, **kwargs): retval = getattr(self, attr, None) if retval is None: retval = fn(self, *args, **kwargs) _lock = RLock() try: _lock.acquire() setattr(self, attr, retval) finally: _lock.release() return retval
class SerialTalks: def __init__(self, port): # Serial things self.port = port self.is_connected = False # Threading things self.queues_dict = dict() self.queues_lock = RLock() self.history = list() self.history_lock = RLock() self.alias_retcode = dict() self.last_retcode = -1 # Instructions self.instructions = dict() self.instructions[RESEND_OPCODE] = self.resend def __enter__(self): self.connect() return self def __exit__(self, exc_type, exc_value, traceback): self.disconnect() def connect(self, timeout=5): if self.is_connected: raise AlreadyConnectedError('{} is already connected'.format( self.port)) # Connect to the serial port try: self.stream = serial.Serial(self.port, baudrate=BAUDRATE, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE, stopbits=serial.STOPBITS_ONE) self.stream.timeout = 1 except SerialException as e: raise ConnectionFailedError(str(e)) from None self.serial_buffer = SerialBuffer(self.stream.write, inf) # Try to bind FREE_BUFFER funct try: self.bind(FREE_BUFFER, self.serial_buffer.reset) except KeyError: # If it's not the first connect executed pass # Create a listening thread that will wait for inputs self.listener = SerialListener(self) self.listener.start() # Wait until the Arduino is operational startingtime = time.monotonic() while not self.is_connected: try: self.execute(PING_OPCODE, timeout=0.1) self.reset_queues() self.serial_buffer.buffer_size = self.execute( GETBUFFERSIZE_OPCODE, timeout=0.5).read(INT) self.is_connected = True except TimeoutError: if time.monotonic() - startingtime > timeout: self.disconnect() raise MuteError( '\'{}\' is mute. It may not be an Arduino or it\'s sketch may not be correctly loaded.' .format(self.stream.port)) from None else: continue except NotConnectedError: self.is_connected = False def disconnect(self): try: self.send(DISCONNECT_OPCODE) except NotConnectedError: pass # Stop the listening thread if hasattr(self, 'listener') and self.listener.is_alive(): self.listener.stop.set() if self.listener is not current_thread(): self.listener.join() # Close the serial port if hasattr(self, 'stream') and self.stream.is_open: self.stream.close() # Unset the connected flag self.is_connected = False def bind(self, opcode, instruction): if not opcode in self.instructions: self.instructions[opcode] = instruction else: raise KeyError( 'opcode {} is already bound to another instruction'.format( opcode)) def rawsend(self, rawbytes): try: if hasattr(self, 'stream') and self.stream.is_open: sentbytes = self.serial_buffer.send(rawbytes) return sentbytes except SerialException: pass raise NotConnectedError('\'{}\' is not connected.'.format( self.port)) from None def send(self, opcode, *args): retcode = random.randint(0, 0xFFFFFFFF) content = BYTE(opcode) + ULONG(retcode) + bytes().join(args) # crc calculation crc = CRCprocessBuffer(content) prefix = MASTER_BYTE + BYTE(len(content)) + USHORT(crc) self.history_lock.acquire() self.history.append((self.last_retcode, retcode, [opcode, args])) self.last_retcode = retcode if len(self.history) > 20: _ = self.history.pop(0) self.history_lock.release() self.rawsend(prefix + content + BYTE(0)) return retcode def get_queue(self, retcode): self.queues_lock.acquire() if retcode in self.alias_retcode.keys(): retcode = self.alias_retcode[retcode] try: queue = self.queues_dict[retcode] except KeyError: queue = self.queues_dict[retcode] = Queue() finally: self.queues_lock.release() return queue def delete_queue(self, retcode): self.queues_lock.acquire() try: del self.queues_dict[retcode] finally: self.queues_lock.release() def reset_queues(self): self.queues_lock.acquire() self.queues_dict = dict() self.queues_lock.release() def process(self, message): retcode = message.read(ULONG) queue = self.get_queue(retcode) queue.put(message) def poll(self, retcode, timeout=0): queue = self.get_queue(retcode) block = (timeout is None or timeout > 0) try: output = queue.get(block, timeout) except Empty: if timeout is not None: raise TimeoutError('timeout exceeded') from None else: return None if queue.qsize() == 0: self.delete_queue(retcode) return output def flush(self, retcode): while self.poll(retcode) is not None: pass def execute(self, opcode, *args, timeout=5): retcode = self.send(opcode, *args) output = self.poll(retcode, timeout) return output def receive(self, input): opcode = input.read(BYTE) retcode = input.read(ULONG) try: output = self.instructions[opcode](input) if output is None: return content = ULONG(retcode) + output prefix = SLAVE_BYTE + BYTE(len(content)) self.rawsend(prefix + content) except KeyError: pass def getuuid(self, timeout=5): output = self.execute(GETUUID_OPCODE, timeout=timeout) return output.read(STRING) def setuuid(self, uuid): return self.send(SETUUID_OPCODE, STRING(uuid)) def getlog(self, retcode, timeout=0): log = str() while True: try: output = self.poll(retcode, 0) log += output.read(STRING) except TimeoutError: break if timeout > 0: try: output = self.poll(retcode, timeout) log += output.read(STRING) except TimeoutError: pass return log def save_eeprom(self, file=None, size=1024): binary_file = open(file, mode='w+b') for i in range(size): output = self.execute(GETEEPROM_OPCODE, INT(i)) byte = output.read(BYTE) binary_file.write(bytes([byte])) binary_file.close() def load_eeprom(self, file=None): binary_file = open(file, mode='r+b') k = 0 for byte in binary_file.read(): self.send(SETEEPROM_OPCODE, INT(k), BYTE(byte)) k += 1 binary_file.close() def resend(self, message): warnings.warn("Message send corrupted !", SerialTalksWarning) prev_retcode = message.read(ULONG) to_send, old_retcode = None, None self.history_lock.acquire() for i in range(len(self.history)): if self.history[i][0] == prev_retcode: to_send = self.history[i][2] main_retcode = self.history[i][1] print("Message resend !") break self.history_lock.release() if not to_send is None: self.queues_lock.acquire() new_retcode = self.send(to_send[0], *to_send[1]) while main_retcode in self.alias_retcode.keys(): main_retcode = self.alias_retcode[main_retcode] self.alias_retcode[new_retcode] = main_retcode self.queues_lock.release() def re_receive(self, main_retcode): to_send = None self.history_lock.acquire() for i in range(len(self.history)): if self.history[i][1] == main_retcode: to_send = self.history[i][2] break self.history_lock.release() if not to_send is None: self.queues_lock.acquire() new_retcode = self.send(to_send[0], *to_send[1]) while main_retcode in self.alias_retcode.keys(): main_retcode = self.alias_retcode[main_retcode] self.alias_retcode[new_retcode] = main_retcode self.queues_lock.release() def getout(self, timeout=0): return self.getlog(STDOUT_RETCODE, timeout) def geterr(self, timeout=0): return self.getlog(STDERR_RETCODE, timeout)
class Monitor(Thread): def __init__(self, req, proxy, logger, task, exit_check=None, ignored_errors=[]): Thread.__init__(self, name="monitor%s" % task.guid) Thread.setDaemon(self, True) # the count of votes per error code self.vote_result = {} # the error code to be ignored self.vote_cleared = set().union(ignored_errors) self.thread_last_seen = {} self.dctlock = RLock() self.votelock = RLock() self.thread_ref = {} self.thread_zombie = set() # HttpReq instance self.req = req # proxy.Pool instance self.proxy = proxy self.logger = logger self.task = task self._exit = exit_check if exit_check else lambda x: False self._cleaning_up = False if os.name == "nt": self.set_title = lambda s: os.system("TITLE %s" % (s if PY3K else s.encode( CODEPAGE, 'replace'))) elif os.name == 'posix': import sys self.set_title = lambda s: sys.stdout.write("\033]2;%s\007" % ( s if PY3K else s.encode(CODEPAGE, 'replace'))) def set_vote_ns(self, tnames): t = time.time() self.thread_last_seen = {k: t for k in tnames} def vote(self, tname, code): # thread_id, result_code self.votelock.acquire() if code != ERR_NO_ERROR: self.logger.verbose("t-%s vote:%s" % (tname, code)) if code not in self.vote_result: self.vote_result[code] = 1 else: self.vote_result[code] += 1 self.votelock.release() def wrk_keepalive(self, wrk_thread, _exit=False): tname = wrk_thread.name if tname in self.thread_zombie: self.thread_zombie.remove(tname) # all image downloaded # task is finished or failed # monitor is exiting or worker notify its exit _ = self.task.meta['finished'] == self.task.meta['total'] or \ self.task.state in (TASK_STATE_FINISHED, TASK_STATE_FAILED) or \ self._exit("mon") or _exit # self.logger.verbose("mon#%s %s ask, %s, %s" % (self.task.guid, tname, _, # self.thread_last_seen)) if _ or not wrk_thread.is_alive(): self.dctlock.acquire() if tname in self.thread_last_seen: del self.thread_last_seen[tname] if tname in self.thread_ref: del self.thread_ref[tname] self.dctlock.release() else: self.thread_last_seen[tname] = time.time() if tname not in self.thread_ref: self.thread_ref[tname] = wrk_thread return _ # def _rescan_pages(self): # # not using # # throw away existing page urls # while True: # try: # self.task.page_q.get(False) # except Empty: # break # # put page into task.list_q # [self.task.list_q.put("%s/?p=%d" % (self.task.url, x) # for x in range(1, 1 + int(math.ceil(self.task.meta['total']/20.0)))) # ] # print(self.task.list_q.qsize()) def _check_vote(self): if False and ERR_IMAGE_RESAMPLED in self.vote_result and ERR_IMAGE_RESAMPLED not in self.vote_cleared: self.logger.warning(i18n.TASK_START_PAGE_RESCAN % self.task.guid) self._rescan_pages() self.task.meta['has_ori'] = True self.vote_cleared.add(ERR_IMAGE_RESAMPLED) elif ERR_QUOTA_EXCEEDED in self.vote_result and \ ERR_QUOTA_EXCEEDED not in self.vote_cleared and \ self.vote_result[ERR_QUOTA_EXCEEDED] >= len(self.thread_last_seen): self.logger.error(i18n.TASK_STOP_QUOTA_EXCEEDED % self.task.guid) self.task.state = TASK_STATE_FAILED def run(self): CHECK_INTERVAL = 10 STUCK_INTERVAL = 90 intv = 0 self.set_title(i18n.TASK_START % self.task.guid) last_change = time.time() last_finished = -1 while len(self.thread_last_seen) > 0: intv += 1 self._check_vote() for k in list(self.thread_last_seen.keys()): if time.time() - self.thread_last_seen[k] > 30: if k in self.thread_ref and self.thread_ref[k].is_alive(): self.logger.warning(i18n.THREAD_MAY_BECOME_ZOMBIE % k) self.thread_zombie.add(k) else: self.logger.warning(i18n.THREAD_SWEEP_OUT % k) del self.thread_last_seen[k] if intv == CHECK_INTERVAL: _ = "%s %dR/%dZ, %s %dR/%dD" % ( i18n.THREAD, len(self.thread_last_seen), len(self.thread_zombie), i18n.QUEUE, self.task.img_q.qsize(), self.task.meta['finished']) self.logger.info(_) self.set_title(_) intv = 0 # if not downloading any new images in 1.5 min, exit if last_finished != self.task.meta['finished']: last_change = time.time() last_finished = self.task.meta['finished'] else: if time.time() - last_change > STUCK_INTERVAL: self.logger.warning(i18n.TASK_STUCK % self.task.guid) break time.sleep(0.5) if self.task.meta['finished'] == self.task.meta['total']: _err = self.task.rename_fname() if _err: self.logger.warning( i18n.XEH_RENAME_HAS_ERRORS % ("\n".join(map(lambda x: "%s => %s : %s" % x, _err)))) self.set_title(i18n.TASK_FINISHED % self.task.guid) self.logger.info(i18n.TASK_FINISHED % self.task.guid) self.task.state = TASK_STATE_FINISHED self.task.cleanup()
class OnionOOFactory(object): # onionoo holds a dict of key:data pairs, with # key = 'details' or 'bandwidth' or 'weights' + ':' + fingerprint # data = onionoo.Document object holding the onionoo network response or None onionoo = {} executor = None query_lock = None nodes_lock = None # proxy = Proxy('127.0.0.1', 'default') futures = list() def __init__(self, proxy): self.onionoo = {} self.proxy = proxy self.executor = ThreadPoolExecutor(max_workers=30) # enough to query > 100 Tors at once... self.query_lock = RLock() self.nodes_lock = RLock() self.hidden_index = 1 self.is_refreshing = False self.new_nodes = {} def add(self, fingerprint): # there are currently three different documents we query from the onionoo db: # Details, Bandwidth & Weights # the key identifies the fingerprint as well as the document to allow storage in a flat dict. check_key = ['details:' + fingerprint, 'bandwidth:' + fingerprint, 'weights:' + fingerprint] retval = False # if the key in question isn't in the dict for key in check_key: if key not in self.onionoo: lgr = logging.getLogger('theonionbox') lgr.debug('Adding fingerprint {} to onionoo query queue.'.format((fingerprint))) # ... add it (yet without document! This indicates that we have no data so far.) self.nodes_lock.acquire() self.new_nodes[key] = Document() self.nodes_lock.release() retval = True return retval def remove(self, fingerprint): # to remove keys if demanded (which probably will happen rarely!) check_key = [ 'details:' + fingerprint, 'bandwidth:' + fingerprint, 'weights:' + fingerprint] for key in check_key: if key in self.onionoo: del self.onionoo[key] if key in self.new_nodes: self.nodes_lock.acquire() del self.new_nodes[key] self.nodes_lock.release() def refresh(self, only_keys_with_none_data=False, async_mode=True): self.is_refreshing = True lgr = logging.getLogger('theonionbox') lgr.info('Refreshing onionoo data => Only New: {} | Async: {}'.format(only_keys_with_none_data, async_mode)) self.nodes_lock.acquire() self.onionoo.update(self.new_nodes) self.new_nodes = {} self.nodes_lock.release() # run through the dict of keys and query onionoo for updated documents for key in self.onionoo: item = self.onionoo[key] if only_keys_with_none_data is True: if item.has_document() is True: continue try: data_type, fp = key.split(':') except ValueError: # This definitely is weird! continue # async_mode = True query_launched = False if async_mode is True: try: self.executor.submit(self.query, item, fp, data_type) query_launched = True except: # In case of error, we try to continue in sync mode! lgr.warning("Onionoo: Failed to launch thread to query for Tor network data.") pass if query_launched is False: try: self.query(item, fp, data_type) except: # Ok. We silently swallow this... pass # restart if meanwhile the landscape changed if len(self.new_nodes) > 0: self.refresh(True) self.is_refreshing = False def query(self, for_document, fingerprint, data_type): lgr = logging.getLogger('theonionbox') # https://trac.torproject.org/projects/tor/ticket/6320 hash = sha1(a2b_hex(fingerprint)).hexdigest() payload = {'lookup': hash} headers = {'accept-encoding': 'gzip'} if len(for_document.ifModSince) > 0: headers['if-modified-since'] = for_document.ifModSince proxy_address = self.proxy.address() if proxy_address is None: proxies = {} query_base = ONIONOO_OPEN else: proxies = { 'http': 'socks5h://' + proxy_address, 'https': 'socks5h://' + proxy_address } query_base = ONIONOO_HIDDEN[self.hidden_index] query_address = query_base + '/' + data_type r = None # even when querying async, there's just one query performed at a time # self.query_lock.acquire() lgr.debug("Onionoo: Launching query of '{}' for ${}.".format(query_address, fingerprint)) try: r = requests.get(query_address, params=payload, headers=headers, proxies=proxies, timeout=10) except requests.exceptions.ConnectTimeout: lgr.info("Onionoo: Failed querying '{}' due to connection timeout. Switching to alternative service." .format(query_base)) # this is quite manual ... but asserts the right result base_index = ONIONOO_HIDDEN.index(query_base) self.hidden_index = (base_index + 1) % len(ONIONOO_HIDDEN) # TODO: shall we restart the failed query here? except Exception as exc: lgr.warning("Onionoo: Failed querying '{}' -> {}".format(query_address, exc)) else: lgr.debug("Onionoo: Finished querying '{}' for ${} with status code {}: {} chars received." .format(query_address, fingerprint, r.status_code, len(r.text))) # if len(r.text) > 0: # lgr.debug(("Onionoo: Received {} chars for ${}".format(len(r.text), fingerprint))) # Ok! Now the next query may be launched... # self.query_lock.release() if r is None: return if r.status_code != requests.codes.ok: return for_document.ifModSince = r.headers['last-modified'] if r.status_code == requests.codes.not_modified: return try: data = r.json() except Exception as exc: lgr.debug("Onionoo: Failed to un-json network data; error code says '{}'.".format(exc)) return for_document.update(data) # ToDo: Where's the benefit doing it this way? # # if data_type == 'details': # node_details = Details(for_document) # # do_refresh = False # # #try: # fams = ['effective_family', 'alleged_family', 'indirect_family'] # for fam in fams: # fam_data = node_details(fam) # if fam_data is not None: # for fp in fam_data: # if fp[0] is '$': # do_refresh = self.add(fp[1:]) or do_refresh # #except: # # This probably wasn't a 'detail' document! # # pass # # if do_refresh and not self.is_refreshing: # self.refresh(True) return # except Exception as exc: # lgr.info("Onionoo: Failed to query '{}': {}".format(address, exc)) # pass #return def details(self, fingerprint): if len(fingerprint)> 0 and fingerprint[0] == '$': fingerprint = fingerprint[1:] key = 'details:' + fingerprint if key in self.onionoo: return Details(self.onionoo[key]) return Details(Document()) def bandwidth(self, fingerprint): if len(fingerprint)> 0 and fingerprint[0] == '$': fingerprint = fingerprint[1:] key = 'bandwidth:' + fingerprint if key in self.onionoo: return Bandwidth(self.onionoo[key]) return Bandwidth(Document()) def weights(self, fingerprint): if len(fingerprint)> 0 and fingerprint[0] == '$': fingerprint = fingerprint[1:] key = 'weights:' + fingerprint if key in self.onionoo: return Weights(self.onionoo[key]) return Weights(Document()) def shutdown(self): self.executor.shutdown(True) def nickname2fingerprint(self, nickname): if nickname[0] == '#': nickname = nickname[1:] data = self.search(nickname) or {} if 'relays' in data: for relay in data['relays']: if 'n' in relay and 'f' in relay: if nickname == relay['n']: return relay['f'] if 'bridges' in data: for bridge in data['bridges']: if 'n' in bridge and 'h' in bridge: if nickname == bridge['n']: return bridge['h'] return None def search(self, search_string, limit=None, offset=None): lgr = logging.getLogger('theonionbox') payload = {'search': search_string} if limit and limit > 0: payload['limit'] = limit if offset and offset > 0: payload['offset'] = offset headers = {'accept-encoding': 'gzip'} proxy_address = self.proxy.address() if proxy_address is None: proxies = {} query_base = ONIONOO_OPEN else: proxies = { 'http': 'socks5h://' + proxy_address, 'https': 'socks5h://' + proxy_address } query_base = ONIONOO_HIDDEN[self.hidden_index] query_address = query_base + '/summary' r = None try: r = requests.get(query_address, params=payload, headers=headers, proxies=proxies, timeout=10) except Exception as exc: lgr.debug("Onionoo: Failed querying '{}' -> {}".format(query_address, exc)) if r is None: return if r.status_code != requests.codes.ok: return try: data = r.json() except Exception as exc: lgr.debug("Onionoo: Failed to un-json network data; error code says '{}'.".format(exc)) return return data
class TrackingJobMaster(JobMaster): """ This class extends JobMaster with the following extras: - reporting of the average time each slave spends on a job - automatic adding of slave computers to PVM - different ways to be notified of a completed calculation - restarting of interrupted calculations The calculation is performed non-blocking in a thread after a call to master.start(). The end of calculation is signalled on master.lock / master.lockMsg. The result can then be obtained with getResult(). Alternatively, a callback method can be registered that is called after the calculation finished (master.setCallback()). The perhaps easiest (but also least flexible) way is to instead use the calculateResult() method. This starts the calculation and blocks execution until the result is returned. Consider overriding cleanup(), done() and getResult(). An interrupted calculation can be restarted from a restart file: - during calculation, pickle the result of getRst() to a file - call the script Biskit/restartPVM -i |file_name| Manual restart is possible as follows: 1. pickle master.data, master.result, master.status.objects 2. master.exit() / Exception / kill, etc. 3. initialize master with same parameters as before 4. unpickle and re-assign master.data, master.result, master.status.objects 5. master.start() @note: The master sends out an exit signal to all slaves but doesn't wait for a response (there isn't any) and continues in the finish() method. Since, at the end, the same job is distributed to several slaves, some of them might still be running when cleanup() or done() are executed. The slave script must tolerate errors that, e.g., happen if cleanup() is called while it is running. @todo: try finding some solution to the problem where the master sends out an exit signal to all slaves but doesn't wait for a response (see note) @todo: test restart function @todo: restart data are not automatically saved (e.g. in intervals) """ def __init__(self, data={}, chunk_size=5, hosts=[], niceness={'default': 20}, slave_script='', verbose=1, show_output=0, add_hosts=1, redistribute=1): """ @param data: dict of items to be processed @type data: {str_id:any} @param chunk_size: number of items that are processed per job @type chunk_size: int @param hosts: list of host-names @type hosts: [str] @param niceness: host niceness dictionary {str_host-name: int_niceness} @type niceness: {str:int} @param slave_script: absolute path to slave-script @type slave_script: str @param verbose: verbosity level (default: 1) @type verbose: 1|0 @param show_output: display one xterm per slave (default: 0) @type show_output: 1|0 @param add_hosts: add hosts to PVM before starting (default: 1) @type add_hosts: 1|0 @param redistribute: at the end, send same job out several times (default: 1) @type redistribute: 1|0 """ if add_hosts: if verbose: T.errWrite('adding %i hosts to pvm...' % len(hosts)) pvm.addHosts(hosts=hosts) if verbose: T.errWriteln('done') JobMaster.__init__(self, data, chunk_size, hosts, niceness, slave_script, show_output=show_output, redistribute=redistribute, verbose=verbose) self.progress = {} self.disabled_hosts = [] self.slow_hosts = {} self.verbose = verbose ## end of calculation is signalled on lockMsg self.lock = RLock() self.lockMsg = Condition(self.lock) ## this method is called when everything is calculated self.call_done = None def hostnameFromTID(self, slave_tid): """ Get nickname of host from TaskID. @param slave_tid: slave task tid @type slave_tid: int """ nickname = self.nicknameFromTID(slave_tid) return nickname.split('_')[0] def is_valid_slave(self, slave_tid): """ Override JobMaster method to disable slow nodes on the fly @param slave_tid: slave task tid @type slave_tid: int """ return self.hostnameFromTID(slave_tid) not in self.disabled_hosts def mark_slow_slaves(self, host_list, slow_factor): """ @param host_list: list of hosts @type host_list: [str] @param slow_factor: factor describing the calculation speed of a node @type slow_factor: float """ for h in host_list: self.slow_hosts[h] = slow_factor def start_job(self, slave_tid): """ Overriding JobMaster method @param slave_tid: slave task tid @type slave_tid: int """ host = self.nicknameFromTID(slave_tid) d = {'given': 0, 'done': 0, 'time': 0} if self.progress.has_key(host): d = self.progress[host] d['given'] += 1 d['timeStart'] = time.time() self.progress[host] = d def job_done(self, slave_tid, result): """ Overriding JobMaster method @param slave_tid: slave task tid @type slave_tid: int @param result: slave result dictionary @type result: dict """ host = self.nicknameFromTID(slave_tid) self.progress[host]['done'] += 1 self.progress[host]['time'] = time.time() \ - self.progress[host]['timeStart'] def reportProgress(self): """ Report how many jobs were processed in what time per host. """ if self.verbose: print 'host \tgiven\tdone\t time' for host in self.progress: d = self.progress[host] print '%-25s\t%i\t%i\t%6.2f s' %\ (host, d['given'], d['done'], d['time']) def setCallback(self, funct): """ Register function to be called after calculation is finished. @param funct: will be called with an instance of the master as single argument @type funct: function """ self.call_done = funct def cleanup(self): """ Called after exit. Override. """ pass def done(self): """ Called by finish() after exit(), cleanup(), and reportProgress(), but before thread notification (notifyAll() ) and before executing the callBack method. Override. """ pass def notifyAll(self): """ Notify thread waiting on self.lockMsg that master has finished. """ self.lock.acquire() self.lockMsg.notifyAll() self.lock.release() def finish(self): """ Called one time, after last job result has been received. It should not be necessary to override this further. Override done() instead. """ self.exit() self.cleanup() self.reportProgress() self.done() self.notifyAll() if self.call_done: self.call_done(self) def getResult(self, **arg): """ Return result dict, if it is available. Override to return something else - which will also be the return value of calculateResult(). @param arg: keyword-value pairs, for subclass implementations @type arg: {key:value} @return: {any:any} @rtype: {any:any} """ return self.result def calculateResult(self, **arg): """ Convenience function that is starting the parallel calculation and blocks execution until it is finished. @param arg: keyword-value pairs, for subclass implementations @type arg: {key:value} @return: array( (n_frames, n_frames), 'f'), matrix of pairwise rms @rtype: array """ self.start() self.lock.acquire() self.lockMsg.wait() self.lock.release() return self.getResult(**arg) def getRst(self): """ Get data necessary for a restart of the running calculation. Locks, file handles and private data are *NOT* saved. Override if necessary but call this method in child method. @return: {..}, dict with 'pickleable' fields of master @rtype: dict """ self.status.lock.acquire() ## collect master parameters that can be pickled rst = {} for k, v in self.__dict__.items(): skip = 0 for t in [Thread, _RLock, _Condition, Status, file]: if isinstance(v, t): skip = 1 if str(k)[0] == '_': skip = 1 if not skip: rst[k] = copy.copy(v) rst['status_objects'] = copy.deepcopy(self.status.objects) rst['master_class'] = self.__class__ self.status.lock.release() return rst def saveRst(self, fname): """ Pickle data necessary for a restart of the running calculation. @param fname: file name @type fname: str """ T.dump(self.getRst(), fname) def setRst(self, rst_data): """ Prepare this master for restart, called by restart(). Override if necessary but call in child. @param rst_data: {..}, parameters for master.__dict__ + some special fields @type rst_data: dict @return: {..}, parameters for master.__dict__ without special fields @rtype: dict """ self.__class__ = rst_data['master_class'] self.status.objects = rst_data['status_objects'] del rst_data['master_class'] del rst_data['status_objects'] return rst_data
class RecentlyUsedContainer(dict): """ Provides a dict-like that maintains up to ``maxsize`` keys while throwing away the least-recently-used keys beyond ``maxsize``. """ # If len(self.access_log) exceeds self._maxsize * CLEANUP_FACTOR, then we # will attempt to cleanup the invalidated entries in the access_log # datastructure during the next 'get' operation. CLEANUP_FACTOR = 10 def __init__(self, maxsize=10): self._maxsize = maxsize # We use a deque to to store our keys ordered by the last access. self.access_log = deque() self.access_log_lock = RLock() # We look up the access log entry by the key to invalidate it so we can # insert a new authorative entry at the head without having to dig and # find the old entry for removal immediately. self.access_lookup = {} # Trigger a heap cleanup when we get past this size self.access_log_limit = maxsize * self.CLEANUP_FACTOR def _invalidate_entry(self, key): "If exists: Invalidate old entry and return it." old_entry = self.access_lookup.get(key) if old_entry: old_entry.is_valid = False return old_entry def _push_entry(self, key): "Push entry onto our access log, invalidate the old entry if exists." self._invalidate_entry(key) new_entry = AccessEntry(key) self.access_lookup[key] = new_entry self.access_log_lock.acquire() self.access_log.appendleft(new_entry) self.access_log_lock.release() def _prune_entries(self, num): "Pop entries from our access log until we popped ``num`` valid ones." while num > 0: self.access_log_lock.acquire() p = self.access_log.pop() self.access_log_lock.release() if not p.is_valid: continue # Invalidated entry, skip dict.pop(self, p.key, None) self.access_lookup.pop(p.key, None) num -= 1 def _prune_invalidated_entries(self): "Rebuild our access_log without the invalidated entries." self.access_log_lock.acquire() self.access_log = deque(e for e in self.access_log if e.is_valid) self.access_log_lock.release() def _get_ordered_access_keys(self): "Return ordered access keys for inspection. Used for testing." self.access_log_lock.acquire() r = [e.key for e in self.access_log if e.is_valid] self.access_log_lock.release() return r def __getitem__(self, key): item = dict.get(self, key) if not item: raise KeyError(key) # Insert new entry with new high priority, also implicitly invalidates # the old entry. self._push_entry(key) if len(self.access_log) > self.access_log_limit: # Heap is getting too big, try to clean up any tailing invalidated # entries. self._prune_invalidated_entries() return item def __setitem__(self, key, item): # Add item to our container and access log dict.__setitem__(self, key, item) self._push_entry(key) # Discard invalid and excess entries self._prune_entries(len(self) - self._maxsize) def __delitem__(self, key): self._invalidate_entry(key) self.access_lookup.pop(key, None) dict.__delitem__(self, key) def get(self, key, default=None): try: return self[key] except KeyError: return default
class MCU: def __init__(self, port): self.port = port self.Serial = serial.Serial() self.lock = RLock() self.__open_serial() self.active = Event() self.error = Event() # Start #self.heartbeat() def heartbeat(self): def run(self): i = 0 while i < 5: i = i + 1 msg = MSG(msgType=MSG.TYPE.HEARTBEAT) resp = self.__send(msg) if resp != None and resp.type() == MSG.TYPE.HEARTBEAT: self.active.set() i = 0 else: self.active.clear() if resp.id != MSG.INFO.SUCCESS: self.error.set() else: self.error.clear() sleep(5) print('5 heartbeats missed: resetting') self.reset() self.heartbeat() t = Thread(target=run, args=(self, )) t.daemon = True t.start() def __open_serial(self, baudrate=115200, timeout=0, write_timeout=0): self.lock.acquire() if not self.Serial.is_open: self.Serial.port = self.port self.Serial.baudrate = baudrate self.Serial.timeout = timeout self.Serial.write_timeout = write_timeout self.Serial.open() sleep(2) self.lock.release() def __close_serial(self): self.lock.acquire() if self.Serial.is_open: self.Serial.close() self.lock.release() def reset(self): self.__close_serial() sleep(0.1) self.__open_serial() def __send(self, msg, timeout=2): assert type(msg) == MSG with self.lock: #print("Sending: {}".format(msg)) self.Serial.write(msg) start_time = time() while self.Serial.in_waiting < MSG.SIZE: if time() - start_time > timeout: print("Response timed out") print("Buffer: {}".format( self.Serial.read(size=self.Serial.in_waiting))) return None #timeout condition sleep(0.1) resp = self.Serial.read(size=self.Serial.in_waiting) resp = MSG(resp) print("Received: {}".format(resp)) return resp def set(self, id, value): msg = MSG(msgType=MSG.TYPE.SET, id=id, data=value) #print(msg) resp = self.__send(msg) assert resp.type() == MSG.TYPE.SET return resp def get(self, id, type): msg = MSG(msgType=MSG.TYPE.GET, id=id, data=type) #print(msg) resp = self.__send(msg) assert resp.type() == MSG.TYPE.GET return resp def status(self, id): msg = MSG(msgType=MSG.TYPE.GET, id=id, data=MSG.INFO.STATUS) resp = self.__send(msg) if resp != None: latched = int.from_bytes(resp.data()[0:2], 'big') nonLatched = int.from_bytes(resp.data()[2:4], 'big') latched = MSG.latchedStatusFlag[latched] nonLatched = MSG.nonLatchedStatusFlag[nonLatched] errorFlag = resp.type() return errorFlag, latched, nonLatched
class ConfigParser(object): """ Configuration info provider """ _list_parms = { "single_build_flags": BuildFlagScope.single, "global_build_flags": BuildFlagScope.all, "dependencies_build_flags": BuildFlagScope.dependencies, } _single_value_parms = ("builder", ) _deprecated_parameters = ("target_dir", ) _logger = logging.getLogger(__name__ + ".ConfigParser") def __init__(self, filename): # type: (Path) -> None self._logger.debug("Creating config parser for filename '%s'", filename) self._parms = {"builder": None} # type: Dict[str, Union[str, None]] self._flags = { FileType.vhdl: { BuildFlagScope.single: (), BuildFlagScope.all: (), BuildFlagScope.dependencies: (), }, FileType.verilog: { BuildFlagScope.single: (), BuildFlagScope.all: (), BuildFlagScope.dependencies: (), }, FileType.systemverilog: { BuildFlagScope.single: (), BuildFlagScope.all: (), BuildFlagScope.dependencies: (), }, } # type: Dict[FileType, Dict[BuildFlagScope, BuildFlags] ] self._sources = [] # type: List[Tuple[str, str, BuildFlags]] self.filename = filename self._timestamp = 0.0 self._parse_lock = RLock() def _shouldParse(self): # type: () -> bool """ Checks if we should parse the configuration file """ return self.filename.mtime > self._timestamp def _updateTimestamp(self): # type: (...) -> Any """ Updates our timestamp with the configuration file """ self._timestamp = self.filename.mtime def isParsing(self): # type: () -> bool "Checks if parsing is ongoing in another thread" locked = not self._parse_lock.acquire(False) if not locked: self._parse_lock.release() return locked def _parseIfNeeded(self): # type: () -> None """ Locks accesses to parsed attributes and parses the configuration file """ with self._parse_lock: if self._shouldParse(): self._parse() def _parse(self): # type: () -> None """ Parse the configuration file without any previous checking """ self._logger.info("Parsing '%s'", self.filename) self._updateTimestamp() self._sources = [] for _line in open(self.filename.name, mode="rb").readlines(): line = _replaceCfgComments("", _line.decode(errors="ignore")) self._parseLine(line) def _parseLine(self, line): # type: (str) -> None """ Parses a line a calls the appropriate extraction methods """ for match in _configFileScan(line): groupdict = match.groupdict() self._logger.debug("match: '%s'", groupdict) if groupdict["parameter"] is not None: self._handleParsedParameter(groupdict["parameter"], groupdict["parm_lang"], groupdict["value"]) else: for source_path in self._getSourcePaths(groupdict["path"]): self._sources.append(( source_path, { "library": groupdict["library"], "flags": _extractSet(groupdict["flags"]), }, )) def _handleParsedParameter(self, parameter, lang, value): # type: (str, str, str) -> None """ Handles a parsed line that sets a parameter """ self._logger.debug("Found parameter '%s' for '%s' with value '%s'", parameter, lang, value) if parameter in self._deprecated_parameters: self._logger.debug("Ignoring deprecated parameter '%s'", parameter) elif parameter in self._single_value_parms: self._logger.debug("Handling '%s' as a single value", parameter) self._parms[parameter] = value elif parameter in self._list_parms: self._logger.debug("Handling '%s' as a list of values", parameter) self._flags[FileType(lang)][ self._list_parms[parameter]] = _extractSet(value) else: raise exceptions.UnknownParameterError(parameter) def _getSourcePaths(self, path): # type: (str) -> Iterable[str] """ Normalizes and handles absolute/relative paths """ source_path = p.normpath(p.expanduser(path)) # If the path to the source file was not absolute, we assume # it was relative to the config file base path if not p.isabs(source_path): fname_base_dir = p.dirname(self.filename.abspath) source_path = p.join(fname_base_dir, source_path) return glob(source_path) or [source_path] def parse(self): # type: (...) -> Dict[Any, Any] """ Parses the file if it hasn't been parsed before or if the config file has been changed """ self._parseIfNeeded() data = {"sources": self._sources} # type: Dict[Any, Any] builder_name = self._parms.get("builder", None) if builder_name is not None: data["builder"] = builder_name for filetype, flags in self._flags.items(): flags_dict = {} for scope in BuildFlagScope: flags_dict[scope.value] = flags[scope] data.update({filetype.name: {"flags": flags_dict}}) return data
class SwarmTracker(Nodeable): ''' Object that maintains the status of all known swarm UAVs Currently, this object maintains the position, velocity, subswarm ID, swarm state, and waypoint intent for each swarm UAV as it is received from the network. Data for each element is maintained in a dictionary object with 1 record for each swarm member. Class member variables: ownID: ID (integer) of this particular aircraft subSwarmID: ID (integer) of the subswarm this vehicle is a part of swarmState: current swarm state of this vehicle (int) swarmBehavior: currently active swarm behavior for this vehicle (int) _possible_crash: set of swarm UAVs suspected of crashing (no reports) _crash_timeout: max non-reporting time before a UAV is considered crashed _baseAlt: Altitude from which rel_alt values are calculated for all AC _swarm: Dictionary of records for individual aircraft in the swarm _swarmPublisher: Object for publishing swarm state _swarmMessage: Container for swarm states to be published _lock: Prevents the callback thread modifications at a bad time Inherited from Nodeable: nodeName: name of the node to start or node in which the object is timer: ROS rate object that controls the timing loop DBUG_PRINT: set true to force screen debug messages (default FALSE) WARN_PRINT: set false to force screen warning messages (default FALSE) Class methods callbackSetup: sets up subscriptions publisherSetup: sets up publishers executeTimedLoop: executes one loop iteration _updateOwnPose: callback for updated own pose receipt _updateSwarmPose: callback for updated swarm UAV pose receipt _updateSwarmControlState: callback for updated swarm control state receipt _setSubSwarm: updates this UAV's subswarm assignment _setSwarmState: updates this UAV's swarm state _setSwarmBehavior: updates this UAV's current swarm behavior ''' def __init__(self, ownID, subswarm=0, nodeName=NODE_BASENAME): ''' Initializes variables, subscribes to required ROS topics, and creates required ros publishers. Initializer assumes that the object is already running within an initialized ROS node (i.e., the object does not initialize itself as a node). This enables multiple objects to run within a single node if rqd. @param ownID: ID (integer) of this particular aircraft @param nodeName: name of the ROS node for this object @param subswarm: ID (int) of the "subswarm" to which this vehicle belongs ''' Nodeable.__init__(self, nodeName) self.ownID = ownID self.subSwarmID = subswarm self.swarmState = 0 self.swarmBehavior = 0 self._baseAlt = 0.0 self._swarm = dict() self._possible_crash = set() self._crash_timeout = DEFAULT_CRASH_TIME self._swarmPublisher = None self._swarmMessage = SwarmStateStamped() self._swarmMessage.header.seq = 0 self._swarmMessage.header.frame_id = "base_footprint" self._lock = RLock() rospy.set_param('subswarm_id', self.subSwarmID) # self.DBUG_PRINT = True # self.INFO_PRINT = True # self.WARN_PRINT = True #------------------------------------------------- # Implementation of parent class virtual functions #------------------------------------------------- def callbackSetup(self): ''' Establishes the callbacks for the SwarmTracker object. @param params: list as follows: [] (no required parameters) ''' self.createSubscriber("acs_pose", apbrg.Geodometry, \ self._updateOwnPose) self.createSubscriber("recv_pose", SwarmVehicleState, \ self._updateSwarmPose) self.createSubscriber("recv_swarm_ctl_state", SwarmControlState, \ self._updateSwarmControlState) self.createSubscriber("subswarm_id", std_msgs.msg.UInt8, \ self._setSubSwarm) self.createSubscriber("swarm_state", std_msgs.msg.UInt8, \ self._setSwarmState) self.createSubscriber("swarm_behavior", std_msgs.msg.UInt8, \ self._setSwarmBehavior) def publisherSetup(self): ''' Sets up publishers for the SwarmTracker object. @param params: list as follows: [] (no required parameters) ''' self._swarmPublisher = \ self.createPublisher("swarm_uav_states", SwarmStateStamped, 1) def executeTimedLoop(self): ''' Executes one timed-loop iteration for the SwarmTracker object The loop computes a DR position for the current time for each UAV in the swarm and publishes a SwarmState message to the ROS topic. ''' self._lock.acquire() self._swarmMessage.header.stamp = rospy.Time.now() self._swarmMessage.crashed_list = list(self._possible_crash) del self._swarmMessage.swarm[:] # Clear current message contents vKeys = self._swarm.keys() for vID in vKeys: vehicle = self._swarm[vID] timeDiff = self._swarmMessage.header.stamp - \ vehicle.state.header.stamp # Half of an Eventually Reliable Crash Detector (ERCD) # (i.e., if a UAV dies, we'll eventually realize it reliably) if (vID not in self._possible_crash) and \ (timeDiff > self._crash_timeout): self._possible_crash.add(vID) self.log_warn("UAV %d possible crash: no updates for %f secs" \ %(vID, (float(str(self._crash_timeout))/1e9))) vehicle.computeDRPose(self._swarmMessage.header.stamp) vehicle._stateMsg.state.header.stamp = vehicle.state.header.stamp vehicle._stateMsg.state.pose = vehicle.drPose.pose vehicle._stateMsg.state.twist = vehicle.state.twist self._swarmMessage.swarm.append(vehicle._stateMsg) self._swarmPublisher.publish(self._swarmMessage) self._lock.release() #----------------------------------------- # ROS Subscriber callbacks for this object #----------------------------------------- def _updateOwnPose(self, stateMsg): ''' Updates swarm info for this UAV when a new pose is published @param poseMsg: Geodometry object with the new pose TODO: add covariances when they are added to the msg ''' try: if not self.ownID in self._swarm: try: self._lock.acquire() self._swarm[self.ownID] = \ SwarmElement(self.ownID, self.subSwarmID, apbrg.Geodometry()) except Exception as ex: self.log_warn("Self update callback error: " + str(ex)) finally: self._lock.release() element = self._swarm[self.ownID] element.updateState(stateMsg, self.subSwarmID, self.swarmState, \ self.swarmBehavior) element.subSwarmID = self.subSwarmID newBaseAlt = element.state.pose.pose.position.alt - \ element.state.pose.pose.position.rel_alt if abs(newBaseAlt - self._baseAlt) > 0.001: self._baseAlt = newBaseAlt except Exception as ex: self.log_warn("Self update callback error: " + str(ex)) def _updateSwarmPose(self, poseMsg): ''' Updates swarm info for swarm UAVs when new poses are received @param poseMsg: Geodometry object with the new pose TODO: add covariances when they are added to the msg ''' try: self._lock.acquire() poseTime = poseMsg.state.header.stamp # Update an existing element if it's already in the dictionary if poseMsg.vehicle_id in self._swarm: updateElement = self._swarm[poseMsg.vehicle_id] elTime = updateElement.state.header.stamp if poseTime < elTime: return # older than latest data # Half of an Eventually Reliable Crash Detector (ERCD) # (i.e., if a UAV dies, we'll eventually realize it reliably) if poseMsg.vehicle_id in self._possible_crash: t = rospy.Time.now() if (t - elTime) > self._crash_timeout: self._crash_timeout = t - elTime self._possible_crash.remove(poseMsg.vehicle_id) self.log_info("Received update to possibly crashed UAV %d"\ %poseMsg.vehicle_id) updateElement.updateState(poseMsg.state, poseMsg.subswarm_id, \ updateElement.swarmState, \ updateElement.swarmBehavior) updateElement.state.pose.pose.position.rel_alt = \ updateElement.state.pose.pose.position.alt - self._baseAlt else: # Create & initialize new element if this is the first report newElement = \ SwarmElement(poseMsg.vehicle_id, poseMsg.subswarm_id, \ poseMsg.state, self._baseAlt) self._swarm[poseMsg.vehicle_id] = newElement self.log_dbug("new aircraft id=" + str(poseMsg.vehicle_id) + \ " added to swarm") element = self._swarm[poseMsg.vehicle_id] element.subSwarmID = poseMsg.subswarm_id except Exception as ex: self.log_warn("Swarm update callback error: " + str(ex)) finally: self._lock.release() def _updateSwarmControlState(self, ctlMsg): ''' Updates swarm control state of a swarming aircraft (not this one) @param ctlMsg: SwarmControlState message with the new control info ''' try: self._lock.acquire() if ctlMsg.vehicle_id in self._swarm: updateElement = self._swarm[ctlMsg.vehicle_id] updateElement.swarmState = ctlMsg.swarm_state updateElement.swarmBehavior = ctlMsg.swarm_behavior except Exception as ex: self.log_warn("Swarm control update callback error: " + str(ex)) finally: self._lock.release() def _setSubSwarm(self, swarmMsg): ''' Updates the "subswarm" in which this vehicle is participating @param swarmMsg: message (UInt8) containing the updated swarm ID ''' self.subSwarmID = swarmMsg.data rospy.set_param('subswarm_id', self.subSwarmID) self.log_dbug("subswarm set to %d" % swarmMsg.data) def _setSwarmState(self, swarmMsg): ''' Updates the "swarm state" for this vehicle @param swarmMsg: message (UInt8) containing the updated swarm behavior ''' self.swarmState = swarmMsg.data self.log_dbug("swarm behavior set to %d" % swarmMsg.data) def _setSwarmBehavior(self, swarmMsg): ''' Updates the "swarm behavior" currently active for this vehicle @param swarmMsg: message (UInt8) containing the updated swarm state ''' self.swarmBehavior = swarmMsg.data self.log_dbug("swarm state set to %d" % swarmMsg.data)
from threading import RLock from threading import Thread sharedData = 22 mylock = RLock() def thread_read(): mylock.acquire() print(sharedData) mylock.release() mylock.acquire() mylock.acquire() mylock.release() mylock.release() thread = Thread(target=thread_read) thread.start() thread.join()
class Persistent(object): def __init__(self, name, **kw): super(Persistent, self).__init__() self.name = name self.storage = None self.synclock = RLock() self.path = kw.get('path', PDODIR) self.encode = kw.get('encode', repr) self.decode = kw.get('decode', eval) self.extension = kw.get('extension', 'dat') self.autopersist = kw.get('autopersist', True) if self.autopersist: self.load() def open(self): self.synclock.acquire() try: self.storage = FileStorage(self.path, self.name, self.extension) self.storage.open() finally: self.synclock.release() def close(self): self.synclock.acquire() try: self.storage.close() self.storage = None finally: self.synclock.release() def closed(self): storage = self.storage if storage is None: return True elif storage.closed(): return True return False def update_storage(self): """ Serialize and data associated with object and update storage record to match serialization. """ self.synclock.acquire() try: data = self.getencoded() self.storage.set(data) finally: self.synclock.release() def update_data(self): self.synclock.acquire() try: data = self.storage.getdata() self.setencoded(data) finally: self.synclock.release() def commit(self): """ Update storage with most recent data, then commit changes. """ self.synclock.acquire() try: self.update_storage() self.storage.commit() self.notify_committed() finally: self.synclock.release() def load(self): """ Load most recently stored data, then update current data with loaded content. """ self.synclock.acquire() try: if self.storage is None: self.open() self.storage.load() self.update_data() self.notify_loaded() finally: self.synclock.release() def serialize(self, data): if self.encode is not None: data = self.encode(data) return data def unserialize(self, data): if self.decode is not None: data = self.decode(data) return data def getencoded(self): """ Return encoded representation of current data object. This method must be overridden in type-specific subclasses. """ raise TypeError("Method must be overridden") def setencoded(self, data): """ Use encoded representation of persisted data object to update current data object. This method must be overridden in type-specific subclasses. """ raise TypeError("Method must be overridden") def notify_committed(self): pass def notify_loaded(self): pass
class VideoHTTPServer(ThreadingMixIn, BaseHTTPServer.HTTPServer): __single = None def __init__(self, port): if VideoHTTPServer.__single: raise RuntimeError, 'HTTPServer is Singleton' VideoHTTPServer.__single = self self.port = port if globalConfig.get_value('allow-non-local-client-connection'): bind_address = '' else: bind_address = '127.0.0.1' BaseHTTPServer.HTTPServer.__init__(self, (bind_address, self.port), SimpleServer) self.daemon_threads = True self.allow_reuse_address = True self.lock = RLock() self.urlpath2streaminfo = {} self.mappers = [] self.errorcallback = None self.statuscallback = None def getInstance(*args, **kw): if VideoHTTPServer.__single is None: VideoHTTPServer(*args, **kw) return VideoHTTPServer.__single getInstance = staticmethod(getInstance) def background_serve(self): name = 'VideoHTTPServerThread-1' self.thread2 = Thread(target=self.serve_forever, name=name) self.thread2.setDaemon(True) self.thread2.start() def register(self, errorcallback, statuscallback): self.errorcallback = errorcallback self.statuscallback = statuscallback def set_inputstream(self, streaminfo, urlpath): self.lock.acquire() if DEBUGLOCK: log('videoserver::set_inputstream: urlpath', urlpath, 'streaminfo', streaminfo, 'thread', currentThread().getName()) if self.urlpath2streaminfo.has_key(urlpath): if DEBUGLOCK: log( 'videoserver::set_inputstream: path exists, delete old: urlpath', urlpath, 'thread', currentThread().getName()) self.del_inputstream(urlpath) streaminfo['lock'] = RLock() self.urlpath2streaminfo[urlpath] = streaminfo self.lock.release() def acquire_inputstream(self, urlpath): global DEBUG if urlpath is None: return streaminfo = None for mapper in self.mappers: streaminfo = mapper.get(urlpath) if streaminfo is not None and (streaminfo['statuscode'] == 200 or streaminfo['statuscode'] == 301): return streaminfo self.lock.acquire() if DEBUGLOCK: log('VideoServer::acquire_inputstream: lock done', urlpath, currentThread().getName()) try: streaminfo = self.urlpath2streaminfo.get(urlpath, None) if DEBUG: log( 'videoserver::acquire_inputstream: got streaminfo: urlpath', urlpath, 'streaminfo', streaminfo) finally: if DEBUGLOCK: log('VideoServer::acquire_inputstream: unlock', urlpath, currentThread().getName()) self.lock.release() if streaminfo is not None and 'lock' in streaminfo: if DEBUGLOCK: log('VideoServer::acquire_inputstream: lock stream: urlpath', urlpath, 'streaminfo', streaminfo, 'thread', currentThread().getName()) streaminfo['lock'].acquire() if DEBUGLOCK: log( 'VideoServer::acquire_inputstream: lock stream done: urlpath', urlpath, 'thread', currentThread().getName()) return streaminfo def release_inputstream(self, urlpath): if DEBUGLOCK: log('VideoServer::release_inputstream: lock', urlpath, currentThread().getName()) self.lock.acquire() try: streaminfo = self.urlpath2streaminfo.get(urlpath, None) finally: if DEBUGLOCK: log('VideoServer::release_inputstream: unlock', urlpath, currentThread().getName()) self.lock.release() if streaminfo is not None and 'lock' in streaminfo: if DEBUGLOCK: log('VideoServer::release_inputstream: unlock stream: urlpath', urlpath, 'streaminfo', streaminfo, 'thread', currentThread().getName()) streaminfo['lock'].release() def del_inputstream(self, urlpath): if DEBUGLOCK: log('VideoServer::del_inputstream: enter', urlpath) streaminfo = self.acquire_inputstream(urlpath) self.lock.acquire() if DEBUGLOCK: log('VideoServer::del_inputstream: lock', urlpath, currentThread().getName()) try: del self.urlpath2streaminfo[urlpath] except KeyError: if DEBUGLOCK: log('videoserver::del_inputstream: path not found: urlpath', urlpath) finally: if DEBUGLOCK: log('VideoServer::del_inputstream: unlock', urlpath, currentThread().getName()) self.lock.release() if streaminfo is not None and 'lock' in streaminfo: if DEBUGLOCK: log('VideoServer::del_inputstream: stream: unlock', urlpath, currentThread().getName()) streaminfo['lock'].release() def get_port(self): return self.port def add_path_mapper(self, mapper): self.mappers.append(mapper) def shutdown(self): if DEBUG: print >> sys.stderr, 'videoserv: Shutting down HTTP' self.socket.close() def handle_error(self, request, client_address): if DEBUGBASESERV: print >> sys.stderr, 'VideoHTTPServer: handle_error', request, client_address log_exc()
class RequestQueue(object): """ Queue and rate limit HTTP requests as to not overload the socket count. """ def __init__(self, max_outstanding=50, timeout=15): """ Create a RequestQueue object for rate-limiting HTTP requests. :param max_outstanding: the maximum number of requests which can be unanswered at any given time. :param timeout: the time after which a request is assumed to never receive a response. """ self.max_outstanding = max_outstanding self.timeout = timeout self.critical_queue = [] self.high_queue = [] self.medium_queue = [] self.low_queue = [] self.lock = RLock() # Don't allow asynchronous access to the queue def parse_queue(self): """ Parse the queues and dispatch the request. """ self.lock.acquire() current_time = time() self.critical_queue = [ (request_manager, endpoint, read_callback, data, method, capture_errors, insertion_time) for (request_manager, endpoint, read_callback, data, method, capture_errors, insertion_time) in self.critical_queue if current_time - insertion_time < self.timeout or request_manager.cancel_request() ] self.high_queue = [ (request_manager, endpoint, read_callback, data, method, capture_errors, insertion_time) for (request_manager, endpoint, read_callback, data, method, capture_errors, insertion_time) in self.high_queue if current_time - insertion_time < self.timeout or request_manager.cancel_request() ] self.medium_queue = [ (request_manager, endpoint, read_callback, data, method, capture_errors, insertion_time) for (request_manager, endpoint, read_callback, data, method, capture_errors, insertion_time) in self.medium_queue if current_time - insertion_time < self.timeout or request_manager.cancel_request() ] self.low_queue = [ (request_manager, endpoint, read_callback, data, method, capture_errors, insertion_time) for (request_manager, endpoint, read_callback, data, method, capture_errors, insertion_time) in self.low_queue if current_time - insertion_time < self.timeout or request_manager.cancel_request() ] queue_item = None if self.critical_queue: queue_item = self.critical_queue.pop(0) elif self.high_queue: queue_item = self.high_queue.pop(0) elif self.medium_queue: queue_item = self.medium_queue.pop(0) elif self.low_queue: queue_item = self.low_queue.pop(0) if queue_item: dispatcher.perform_request(*queue_item[:-2]) self.lock.release() def enqueue(self, request_manager, method, endpoint, data, read_callback, capture_errors, priority=QueuePriorityEnum.HIGH): """ Add a new request to the queue based on priority Priority order - CRITICAL - HIGH - MEDIUM - LOW :param request_manager: the TriblerRequestManager wishing to perform a request. :param method: request method. :param endpoint: request endpoint. :param data: request data. :param read_callback: callback to call if the request is processed. :param capture_errors: whether to display the errors or not. :param priority: the priority for this request. """ self.lock.acquire() queue_item = (request_manager, endpoint, read_callback, data, method, capture_errors, time()) if priority == QueuePriorityEnum.CRITICAL: self.critical_queue.append(queue_item) if priority == QueuePriorityEnum.HIGH: if len(self.high_queue) < self.max_outstanding: self.high_queue.append(queue_item) else: # Get the last item of the queue last_item = self.high_queue.pop(self.max_outstanding - 1) # Add the original queue_item to the front of the queue self.high_queue.insert(0, queue_item) # reduce the priority of last_item and try to put in medium queue priority = QueuePriorityEnum.MEDIUM queue_item = last_item if priority == QueuePriorityEnum.MEDIUM: if len(self.medium_queue) < self.max_outstanding: self.medium_queue.append(queue_item) else: # Get the last item of the queue last_item = self.medium_queue.pop(self.max_outstanding - 1) # Add the original queue_item to the front of the queue self.medium_queue.insert(0, queue_item) # reduce the priority of last_item and try to put in low queue priority = QueuePriorityEnum.LOW queue_item = last_item if priority == QueuePriorityEnum.LOW: if len(self.low_queue) < self.max_outstanding: self.low_queue.append(queue_item) else: # Remove the last item of the queue which will be dropped self.low_queue.pop(self.max_outstanding - 1) # Add the original queue_item to the front of the queue self.low_queue.insert(0, queue_item) self.lock.release() self.parse_queue() def clear(self): """ Clear the queue. """ self.lock.acquire() for request_manager, _, _, _, _, _, _ in self.critical_queue: request_manager.cancel_request() for request_manager, _, _, _, _, _, _ in self.high_queue: request_manager.cancel_request() for request_manager, _, _, _, _, _, _ in self.medium_queue: request_manager.cancel_request() for request_manager, _, _, _, _, _, _ in self.low_queue: request_manager.cancel_request() self.critical_queue = [] self.high_queue = [] self.medium_queue = [] self.low_queue = [] self.lock.release()
class CursesUtil: def __init__(self): self.pairlock = Lock() # iolock protects access to the self.iolock = RLock() self.start() def initpairs(self): self.pairlock.acquire() try: self.pairs = { self._getpairindex(curses.COLOR_WHITE, curses.COLOR_BLACK): 0 } self.nextpair = 1 finally: self.pairlock.release() def lock(self): """Locks the Curses ui thread Can be invoked multiple times from the owning thread. Invoking from a non-owning thread blocks and waits until it has been unlocked by the owning thread.""" self.iolock.acquire() def unlock(self): """Unlocks the Curses ui thread Decrease the lock counter by one and unlock the ui thread if the counter reaches 0. Only call this method when the calling thread owns the lock. A RuntimeError is raised if this method is called when the lock is unlocked.""" self.iolock.release() def locked(self, target, *args, **kwargs): """Perform an operation with full locking.""" self.lock() try: apply(target, args, kwargs) finally: self.unlock() def refresh(self): def lockedstuff(): curses.panel.update_panels() curses.doupdate() self.locked(lockedstuff) def isactive(self): return hasattr(self, 'stdscr') def _getpairindex(self, fg, bg): return '%d/%d' % (fg, bg) def getpair(self, fg, bg): if not self.has_color: return 0 pindex = self._getpairindex(fg, bg) self.pairlock.acquire() try: if self.pairs.has_key(pindex): return curses.color_pair(self.pairs[pindex]) else: self.pairs[pindex] = self.nextpair curses.init_pair(self.nextpair, fg, bg) self.nextpair += 1 return curses.color_pair(self.nextpair - 1) finally: self.pairlock.release() def start(self): self.stdscr = curses.initscr() curses.noecho() curses.cbreak() self.stdscr.keypad(1) try: curses.start_color() self.has_color = curses.has_colors() except: self.has_color = 0 self.oldcursor = None try: self.oldcursor = curses.curs_set(0) except: pass self.stdscr.clear() self.stdscr.refresh() (self.height, self.width) = self.stdscr.getmaxyx() self.initpairs() def stop(self): if not hasattr(self, 'stdscr'): return #self.stdscr.addstr(self.height - 1, 0, "\n", # self.getpair(curses.COLOR_WHITE, # curses.COLOR_BLACK)) if self.oldcursor != None: curses.curs_set(self.oldcursor) self.stdscr.refresh() self.stdscr.keypad(0) curses.nocbreak() curses.echo() curses.endwin() del self.stdscr def reset(self): # dirty walkaround for bug http://bugs.python.org/issue7567 in python 2.6 to 2.6.5 (fixed since #83743) if (sys.version_info[0:3] >= (2, 6) and sys.version_info[0:3] <= (2, 6, 5)): return self.stop() self.start()
class MsgAutoSender(object): headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', "Accept-Language": "en-US,en;q=0.5", #"Cookie": "; ".join(map(lambda x: "=".join(x), data.items())) } messages = ["在不在呢[疑问]"] data = { 'txtLoginEMail': "", 'txtLoginPwd': "", 'chkRememberMe': "", 'codeId': "", 'codeValue': '', 'event':'3', 'spmp':'4.20.53.225.685', '_': "%d"%(time.time()*1000) } # 预登陆url url1 = 'http://my.baihe.com/Getinterlogin/gotoLogin?jsonCallBack=jQuery18308807729283968166_%d&'%(time.time()*1000) # 登陆成功后跳转到主页 url2 = "http://u.baihe.com/?userid=&time=%d"%(time.time()*1000) # 用来获取一些默认的搜索条件(百合会根据你个人信息筛选出一些基本符合你要求的人群) url3 = "http://search.baihe.com/mystruts/nextSolrSearch.action?jsoncallback=jQuery183042376943520885857_1479472584212&callType=next&pageId=%%s&ord=1&_=%d"%(time.time()*1000) # 用来搜索默认条件下的妹纸 url4 = "http://search.baihe.com/solrAdvanceSearch.action" # 向妹纸发送消息 url5 = "http://msg.baihe.com/owner/api/sendMessage?jsonCallBack=jQuery18304671662130587029_1479300393335&toUserID=%%s&content=%%s&type=1&pathID=01.00.10402&_=%d&"%(time.time()*1000) # 登陆过频繁的话,会要求输验证,此url用来获取验证码图片 url6 = "http://my.baihe.com/Getinterlogin/getVerifyPic?jsonCallBack=?&tmpId=%s" # 用来检查登陆次数,来判定是否需要输验证码了 url7 = "http://my.baihe.com/Getinterlogin/getAccountTimes?jsonCallBack=jQuery183013238800936369732_1479556200186&userAccount=18353130797&_=1479556585384" # 用来验证验证码是否正确 url8 = "http://my.baihe.com/Getinterlogin/checkVerifyPic?jsonCallBack=jQuery183010981413646438898_1480223919788&tmpId=%%s&checkcode=%%s&_=%d"%(time.time()*1000) # access_token生成 acc_token = Cookie(version=0, name="accessToken", value='BH%d%d'%(time.time()*1000,math.floor(random.random()*1000000)), domain=".baihe.com", path="/", port=None, port_specified=False, domain_specified=True, domain_initial_dot=False, path_specified=True, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest={}, rfc2109=False) def __init__(self): self.page = 1 self.order = 1 self.product_ids = set() self.error_count = 0 self.lock = RLock() self.alive = True self.cookie = LWPCookieJar() try: self.have_send_list = set(open("have_send_list.txt").read().strip(",").split(",")) except IOError: self.have_send_list = set() self.logger = logging.getLogger("send_msg") self.logger.setLevel(logging.DEBUG) self.logger.addHandler(logging.StreamHandler(sys.stdout)) def get_account_times(self, opener): resp = opener.open(self.url7) buf = resp.read() data = json.loads(re.search(r"\((\{.*\})\)", buf).group(1), encoding="gbk") self.logger.debug("Check whether need input captcha or not. ") if data["data"]["showCode"]: return self.get_captcha(opener) else: return "", "" def get_captcha(self, opener): tmpId = "%d.%s"%(time.time()*1000, str(round(random.random(), 4))[2:]) resp = opener.open(self.url6%tmpId) img = Image.open( StringIO(resp.read())) img.show() return raw_input("Please input captcha recognization: "), tmpId def send_captcha(self, opener, captcha, tmpId): self.logger.debug("Send captcha. ") url = self.url8 % (tmpId, captcha) req = Request(url=url, headers=self.headers) resp = opener.open(req) data = json.loads(re.search(r"\((\{.*\})\)", resp.read()).group(1), encoding="gbk") if data["data"] == 1: return tmpId, captcha def login(self, opener): url = self.url1 + urlencode(self.data) req = Request(url=url, headers=self.headers) resp = opener.open(req) data = json.loads(re.search(r"\((\{.*\})\)", resp.read()).group(1), encoding="gbk") self.logger.debug("Login jsonp response state:%s"%data["state"]) if data["state"] == 0: return "Wrong account or password. " req = Request(url=self.url2, headers=self.headers) resp = opener.open(req) self.logger.debug("Login redirect response code:%s"%resp.code) def get_auth_cookies(self, opener): while True: self.enter_password() captcha, tmpId = self.get_account_times(opener) if tmpId: while not self.send_captcha(opener, captcha, tmpId): captcha, tmpId = self.get_account_times(opener) self.data["codeValue"] = captcha self.data["codeId"] = tmpId result = self.login(opener) if result: self.logger.info(result) else: break def get_search_cookies(self, opener): req = Request(url=self.url4, headers=self.headers) resp = opener.open(req) self.logger.debug("Finish get default search cookies, response code:%s" % resp.code) def search(self, opener): conti = True while conti: while True: try: id = self.product_ids.pop() except KeyError: break self.send_msg(opener, id) self.order += 1 req = Request(url=self.url3 % self.page, headers=self.headers) self.logger.debug("Start to find girls in page NO.%s. "%self.page) resp = opener.open(req) self.logger.debug("Search response code:%s" % resp.code) buf = resp.read() data = json.loads(re.search(r"\((\{.*\})\)", buf).group(1), encoding="gbk") if data["result"]: self.product_ids = set([i.split(":")[0] for i in data["result"].split(",")]) self.page += 1 elif self.page > 100: return "finished" else: raise SendMessageError("You need relogin. ") def send_msg(self, opener, id): if id not in self.have_send_list: msg = random.choice(self.messages) d = quote(msg) url = self.url5 % (id, d) req = Request(url=url, headers=self.headers) resp = opener.open(req) buf = resp.read() recv = json.loads(re.search(r"\((\{.*\})\)", buf).group(1), encoding="gbk") code = recv["code"] self.logger.info("Send %s to No.%s girl whose id is %s, status code is %s" % (msg.decode("utf-8"), self.order, id, code)) if code == 200: if self.error_count > 0: self.error_count -= 1 self.have_send_list.add(id) else: self.error_count += 1 if code == u"-701": self.alive = False self.logger.error(u"坑爹的百合每天每个账号只允许给100个人发消息。。") sys.exit(0) if self.error_count > 3: raise SendMessageError("code: %s error: %s" % (code.encode("gbk"), (recv.get("msg") or u"empty").encode("gbk"))) time.sleep(1) else: self.logger.info("The No.%s girl whose id is %s has been sent, don't molesting her any more. "%(self.order, id)) def pwd_input(self, msg=''): if msg != '': sys.stdout.write(msg) chars = [] while True: newChar = getch() if newChar in '\3\r\n': # 如果是换行,Ctrl+C,则输入结束 print '' if newChar in '\3': # 如果是Ctrl+C,则将输入清空,返回空字符串 chars = [] break elif newChar == '\b' or ord(newChar) == 127: # 如果是退格,则删除末尾一位 if chars: del chars[-1] sys.stdout.write('\b \b') # 左移一位,用空格抹掉星号,再退格 else: chars.append(newChar) sys.stdout.write('*') # 显示为星号 return ''.join(chars) def enter_password(self): account = raw_input("Please input your baihe account number: ") self.data["txtLoginEMail"] = account self.data["txtLoginPwd"] = self.pwd_input("Please input your baihe account password: "******"Please input what you want to send, input empty to break. ") if not msg: break else: try: msg = msg.decode("gbk").encode("utf-8") except UnicodeDecodeError: pass self.messages.append(msg) def start(self): self.enter_msg() self.cookie.set_cookie(self.acc_token) have_load = False try: if os.path.exists(("baihe.cookie")): self.cookie.load("baihe.cookie", True, True) have_load = True opener = build_opener(HTTPCookieProcessor(self.cookie)) if not have_load: self.get_auth_cookies(opener) self.get_search_cookies(opener) # 有时意外不正常关闭cookie和send_list无法保存,所以启动一个进程来做这件事。 Thread(target=self.saveing).start() while True: try: if self.search(opener) == "finished": self.logger.info("No more girls to send. ") break except Exception, e: time.sleep(1) self.logger.error(e) self.get_auth_cookies(opener) self.get_search_cookies(opener) except KeyboardInterrupt: self.logger.info("Closing...") self.alive = False finally: self.save() self.alive = False def saveing(self): while self.alive: self.save() time.sleep(2) def save(self): self.lock.acquire() open("have_send_list.txt", "w").write(",".join(self.have_send_list)) self.cookie.save("baihe.cookie", True, True) self.lock.release()
class Task(object): def __init__(self, url, cfgdict): self.url = url if url: _ = index_re.findall(url) if _: self.gid, self.sethash = _[0] self.failcode = 0 self.state = TASK_STATE_WAITING self.guid = str(uuid.uuid4())[:8] self.config = cfgdict self.meta = {} self.has_ori = False self.reload_map = {} # {url:reload_url} self.filehash_map = { } # map same hash to different ids, {url:((id, fname), )} self.img_q = None self.page_q = None self.list_q = None self._flist_done = set( ) # store id, don't save, will generate when scan self._monitor = None self._cnt_lock = RLock() self._f_lock = RLock() def cleanup(self): if self.state in (TASK_STATE_FINISHED, TASK_STATE_FAILED): self.img_q = None self.page_q = None self.list_q = None self.reload_map = {} # if 'filelist' in self.meta: # del self.meta['filelist'] # if 'resampled' in self.meta: # del self.meta['resampled'] def set_fail(self, code): self.state = TASK_STATE_FAILED self.failcode = code # cleanup all we cached self.meta = {} def migrate_exhentai(self): _ = re.findall("(?:https*://g\.e\-hentai\.org)(.+)", self.url) if not _: return False self.url = "https://exhentai.org%s" % _[0] self.state = TASK_STATE_WAITING if self.state == TASK_STATE_FAILED else self.state self.failcode = 0 return True # def guess_ori(self): # # guess if this gallery has resampled files depending on some sample hashes # # return True if it's ori # if 'sample_hash' not in self.meta: # return # all_keys = map(lambda x:x[:10], self.meta['filelist'].keys()) # for h in self.meta['sample_hash']: # if h not in all_keys: # self.has_ori = True # break # del self.meta['sample_hash'] def base_url(self): return re.findall("(https*://(?:g\.e\-|ex)hentai\.org)", self.url)[0] # def get_picpage_url(self, pichash): # # if file resized, this url not works # # http://%s.org/s/hash_s/gid-picid' # return "%s/s/%s/%s-%s" % ( # self.base_url(), pichash[:10], self.gid, self.meta['filelist'][pichash][0] # ) def set_reload_url(self, imgurl, reload_url, fname): # if same file occurs severl times in a gallery if imgurl in self.reload_map: fpath = self.get_fpath() old_fid = self.get_fname(imgurl)[0] old_f = os.path.join(fpath, self.get_fidpad(old_fid)) this_fid = int(gallery_re.findall(reload_url)[0][1]) this_f = os.path.join(fpath, self.get_fidpad(this_fid)) self._f_lock.acquire() if os.path.exists(old_f): # we can just copy old file if already downloaded try: with open(old_f, 'rb') as _of: with open(this_f, 'wb') as _nf: _nf.write(_of.read()) except Exception as ex: self._f_lock.release() raise ex else: self._f_lock.release() self._cnt_lock.acquire() self.meta['finished'] += 1 self._cnt_lock.release() else: # if not downloaded, we will copy them in save_file if imgurl not in self.filehash_map: self.filehash_map[imgurl] = [] self.filehash_map[imgurl].append((this_fid, old_fid)) self._f_lock.release() else: self.reload_map[imgurl] = [reload_url, fname] def get_reload_url(self, imgurl): if not imgurl: return return self.reload_map[imgurl][0] def scan_downloaded(self, scaled=True): fpath = self.get_fpath() donefile = False if os.path.exists(os.path.join(fpath, ".xehdone")) or os.path.exists( "%s.zip" % fpath): donefile = True # can only check un-renamed files for fid in range(1, self.meta['total'] + 1): fname = os.path.join(fpath, self.get_fidpad(fid)) # id if donefile or (os.path.exists(fname) and os.stat(fname).st_size > 0): self._flist_done.add(int(fid)) self.meta['finished'] = len(self._flist_done) if self.meta['finished'] == self.meta['total']: self.state == TASK_STATE_FINISHED def queue_wrapper(self, callback, pichash=None, url=None): # if url is not finished, call callback to put into queue # type 1: normal file; type 2: resampled url # if pichash: # fid = int(self.meta['filelist'][pichash][0]) # if fid not in self._flist_done: # callback(self.get_picpage_url(pichash)) # elif url: fhash, fid = gallery_re.findall(url)[0] # if fhash not in self.meta['filelist']: # self.meta['resampled'][fhash] = int(fid) # self.has_ori = True] if int(fid) not in self._flist_done: callback(url) def save_file(self, imgurl, redirect_url, binary): # TODO: Rlock for finished += 1 fpath = self.get_fpath() self._f_lock.acquire() if not os.path.exists(fpath): os.mkdir(fpath) self._f_lock.release() pageurl, fname = self.reload_map[imgurl] _ = re.findall("/([^/\?]+)(?:\?|$)", redirect_url) if _: # change it if it's a full image fname = _[0] self.reload_map[imgurl][1] = fname _, fid = gallery_re.findall(pageurl)[0] fn = os.path.join(fpath, self.get_fidpad(int(fid))) if os.path.exists(fn) and os.stat(fn).st_size > 0: return fn self._cnt_lock.acquire() self.meta['finished'] += 1 self._cnt_lock.release() self._f_lock.acquire() with open(fn, "wb") as f: f.write(binary) if imgurl in self.filehash_map: for fid, _ in self.filehash_map[imgurl]: fn_rep = os.path.join(fpath, self.get_fidpad(fid)) with open(fn_rep, "wb") as f: f.write(binary) self.meta['finished'] += 1 del self.filehash_map[imgurl] self._f_lock.release() def get_fname(self, imgurl): pageurl, fname = self.reload_map[imgurl] _, fid = gallery_re.findall(pageurl)[0] return int(fid), fname def get_fpath(self): return os.path.join(self.config['dir'], util.legalpath(self.meta['title'])) def get_fidpad(self, fid, ext='jpg'): fid = int(fid) _ = "%%0%dd.%%s" % (len(str(self.meta['total']))) return _ % (fid, ext) def rename_fname(self): fpath = self.get_fpath() cnt = 0 error_list = [] for h in self.reload_map: fid, fname = self.get_fname(h) # if we don't need to rename to original name and file type matches if not self.config['rename_ori'] and os.path.splitext( fname)[1].lower() == '.jpg': continue fname_ori = os.path.join(fpath, self.get_fidpad(fid)) # id if self.config['rename_ori']: fname_to = os.path.join(fpath, util.legalpath(fname)) else: # Q: Why we don't just use id.ext when saving files instead of using # id.jpg? # A: If former task doesn't download all files, a new task with same gallery # will have zero knowledge about file type before scanning all per page, # thus can't determine if this id is downloaded, because file type is not # necessarily .jpg fname_to = os.path.join( fpath, self.get_fidpad(fid, os.path.splitext(fname)[1][1:])) if fname_ori != fname_to: if os.path.exists(fname_ori): while os.path.exists(fname_to): _base, _ext = os.path.splitext(fname_to) _ = re.findall("\((\d+)\)$", _base) if _: # if ...(1) exists, use ...(2) _base = re.sub( "\((\d+)\)$", _base, lambda x: "(%d)" % (int(x.group(1)) + 1)) else: _base = "%s(1)" % _base fname_to = "".join((_base, _ext)) try: os.rename(fname_ori, fname_to) except Exception as ex: error_list.append( os.path.split(fname_ori)[1], os.path.split(fname_to)[1], str(ex)) cnt += 1 if cnt == self.meta['total']: with open(os.path.join(fpath, ".xehdone"), "w"): pass return error_list def make_archive(self): dpath = self.get_fpath() arc = "%s.zip" % dpath if os.path.exists(arc): return arc with zipfile.ZipFile(arc, 'w') as zipFile: zipFile.comment = ( "xeHentai Archiver v%s\nTitle:%s\nOriginal URL:%s" % (__version__, self.meta['title'], self.url)).encode('utf-8') for f in sorted(os.listdir(dpath)): fullpath = os.path.join(dpath, f) zipFile.write(fullpath, f, zipfile.ZIP_STORED) shutil.rmtree(dpath) return arc def from_dict(self, j): for k in self.__dict__: if k not in j: continue if k.endswith('_q') and j[k]: setattr(self, k, Queue()) [getattr(self, k).put(e, False) for e in j[k]] else: setattr(self, k, j[k]) _ = index_re.findall(self.url) if _: self.gid, self.sethash = _[0] return self def to_dict(self): d = dict({ k: v for k, v in self.__dict__.items() if not k.endswith('_q') and not k.startswith("_") }) for k in ['img_q', 'page_q', 'list_q']: if getattr(self, k): d[k] = [e for e in getattr(self, k).queue] return d
class DebugClientThreads(DebugClientBase.DebugClientBase, AsyncIO): """ Class implementing the client side of the debugger. This variant of the debugger implements a threaded debugger client by subclassing all relevant base classes. """ def __init__(self): """ Constructor """ AsyncIO.__init__(self) DebugClientBase.DebugClientBase.__init__(self) # protection lock for synchronization self.clientLock = RLock() # the "current" thread, basically the thread we are at a breakpoint # for. self.currentThread = None # special objects representing the main scripts thread and frame self.mainThread = None self.mainFrame = None self.variant = 'Threaded' def attachThread(self, target=None, args=None, kwargs=None, mainThread=0): """ Public method to setup a thread for DebugClient to debug. If mainThread is non-zero, then we are attaching to the already started mainthread of the app and the rest of the args are ignored. @param target the start function of the target thread (i.e. the user code) @param args arguments to pass to target @param kwargs keyword arguments to pass to target @param mainThread non-zero, if we are attaching to the already started mainthread of the app @return The identifier of the created thread """ try: self.lockClient() newThread = DebugThread(self, target, args, kwargs, mainThread) ident = -1 if mainThread: ident = thread.get_ident() self.mainThread = newThread if self.debugging: sys.setprofile(newThread.profile) else: ident = _original_start_thread(newThread.bootstrap, ()) newThread.set_ident(ident) self.threads[newThread.get_ident()] = newThread finally: self.unlockClient() return ident def threadTerminated(self, dbgThread): """ Public method called when a DebugThread has exited. @param dbgThread the DebugThread that has exited """ try: self.lockClient() try: del self.threads[dbgThread.get_ident()] except KeyError: pass finally: self.unlockClient() def lockClient(self, blocking=1): """ Public method to acquire the lock for this client. @param blocking flag to indicating a blocking lock @return flag indicating successful locking """ if blocking: self.clientLock.acquire() else: return self.clientLock.acquire(blocking) def unlockClient(self): """ Public method to release the lock for this client. """ try: self.clientLock.release() except AssertionError: pass def setCurrentThread(self, id): """ Public method to set the current thread. @param id the id the current thread should be set to. """ try: self.lockClient() if id is None: self.currentThread = None else: self.currentThread = self.threads[id] finally: self.unlockClient() def eventLoop(self, disablePolling=False): """ Public method implementing our event loop. @param disablePolling flag indicating to enter an event loop with polling disabled (boolean) """ # make sure we set the current thread appropriately threadid = thread.get_ident() self.setCurrentThread(threadid) DebugClientBase.DebugClientBase.eventLoop(self, disablePolling) self.setCurrentThread(None) def set_quit(self): """ Public method to do a 'set quit' on all threads. """ try: locked = self.lockClient(0) try: for key in self.threads.keys(): self.threads[key].set_quit() except: pass finally: if locked: self.unlockClient()
class ProxyObject(object): """A proxy to the remote Object. A ProxyObject is provided by the Bus. ProxyObjects have member functions, and can be called like normal Python objects. """ ProxyMethodClass = _ProxyMethod DeferredMethodClass = _DeferredMethod INTROSPECT_STATE_DONT_INTROSPECT = 0 INTROSPECT_STATE_INTROSPECT_IN_PROGRESS = 1 INTROSPECT_STATE_INTROSPECT_DONE = 2 def __init__(self, conn=None, bus_name=None, object_path=None, introspect=True, follow_name_owner_changes=False, **kwargs): """Initialize the proxy object. :Parameters: `conn` : `dbus.connection.Connection` The bus or connection on which to find this object. The keyword argument `bus` is a deprecated alias for this. `bus_name` : str A bus name for the application owning the object, to be used as the destination for method calls and the sender for signal matches. The keyword argument ``named_service`` is a deprecated alias for this. `object_path` : str The object path at which the application exports the object `introspect` : bool If true (default), attempt to introspect the remote object to find out supported methods and their signatures `follow_name_owner_changes` : bool If true (default is false) and the `bus_name` is a well-known name, follow ownership changes for that name """ bus = kwargs.pop('bus', None) if bus is not None: if conn is not None: raise TypeError('conn and bus cannot both be specified') conn = bus from warnings import warn warn( 'Passing the bus parameter to ProxyObject by name is ' 'deprecated: please use positional parameters', DeprecationWarning, stacklevel=2) named_service = kwargs.pop('named_service', None) if named_service is not None: if bus_name is not None: raise TypeError('bus_name and named_service cannot both be ' 'specified') bus_name = named_service from warnings import warn warn( 'Passing the named_service parameter to ProxyObject by name ' 'is deprecated: please use positional parameters', DeprecationWarning, stacklevel=2) if kwargs: raise TypeError('ProxyObject.__init__ does not take these ' 'keyword arguments: %s' % ', '.join(kwargs.keys())) if follow_name_owner_changes: # we don't get the signals unless the Bus has a main loop # XXX: using Bus internals conn._require_main_loop() self._bus = conn if bus_name is not None: _dbus_bindings.validate_bus_name(bus_name) # the attribute is still called _named_service for the moment, # for the benefit of telepathy-python self._named_service = self._requested_bus_name = bus_name _dbus_bindings.validate_object_path(object_path) self.__dbus_object_path__ = object_path if not follow_name_owner_changes: self._named_service = conn.activate_name_owner(bus_name) #PendingCall object for Introspect call self._pending_introspect = None #queue of async calls waiting on the Introspect to return self._pending_introspect_queue = [] #dictionary mapping method names to their input signatures self._introspect_method_map = {} # must be a recursive lock because block() is called while locked, # and calls the callback which re-takes the lock self._introspect_lock = RLock() if not introspect or self.__dbus_object_path__ == LOCAL_PATH: self._introspect_state = self.INTROSPECT_STATE_DONT_INTROSPECT else: self._introspect_state = self.INTROSPECT_STATE_INTROSPECT_IN_PROGRESS self._pending_introspect = self._Introspect() bus_name = property( lambda self: self._named_service, None, None, """The bus name to which this proxy is bound. (Read-only, may change.) If the proxy was instantiated using a unique name, this property is that unique name. If the proxy was instantiated with a well-known name and with ``follow_name_owner_changes`` set false (the default), this property is the unique name of the connection that owned that well-known name when the proxy was instantiated, which might not actually own the requested well-known name any more. If the proxy was instantiated with a well-known name and with ``follow_name_owner_changes`` set true, this property is that well-known name. """) requested_bus_name = property( lambda self: self._requested_bus_name, None, None, """The bus name which was requested when this proxy was instantiated. """) object_path = property(lambda self: self.__dbus_object_path__, None, None, """The object-path of this proxy.""") # XXX: We don't currently support this because it's the signal receiver # that's responsible for tracking name owner changes, but it # seems a natural thing to add in future. #unique_bus_name = property(lambda self: something, None, None, # """The unique name of the connection to which this proxy is # currently bound. (Read-only, may change.) # """) def connect_to_signal(self, signal_name, handler_function, dbus_interface=None, **keywords): """Arrange for the given function to be called when the given signal is received. :Parameters: `signal_name` : str The name of the signal `handler_function` : callable A function to be called when the signal is emitted by the remote object. Its positional arguments will be the arguments of the signal; optionally, it may be given keyword arguments as described below. `dbus_interface` : str Optional interface with which to qualify the signal name. If None (the default) the handler will be called whenever a signal of the given member name is received, whatever its interface. :Keywords: `utf8_strings` : bool If True, the handler function will receive any string arguments as dbus.UTF8String objects (a subclass of str guaranteed to be UTF-8). If False (default) it will receive any string arguments as dbus.String objects (a subclass of unicode). `byte_arrays` : bool If True, the handler function will receive any byte-array arguments as dbus.ByteArray objects (a subclass of str). If False (default) it will receive any byte-array arguments as a dbus.Array of dbus.Byte (subclasses of: a list of ints). `sender_keyword` : str If not None (the default), the handler function will receive the unique name of the sending endpoint as a keyword argument with this name `destination_keyword` : str If not None (the default), the handler function will receive the bus name of the destination (or None if the signal is a broadcast, as is usual) as a keyword argument with this name. `interface_keyword` : str If not None (the default), the handler function will receive the signal interface as a keyword argument with this name. `member_keyword` : str If not None (the default), the handler function will receive the signal name as a keyword argument with this name. `path_keyword` : str If not None (the default), the handler function will receive the object-path of the sending object as a keyword argument with this name `message_keyword` : str If not None (the default), the handler function will receive the `dbus.lowlevel.SignalMessage` as a keyword argument with this name. `arg...` : unicode or UTF-8 str If there are additional keyword parameters of the form ``arg``\ *n*, match only signals where the *n*\ th argument is the value given for that keyword parameter. As of this time only string arguments can be matched (in particular, object paths and signatures can't). """ return \ self._bus.add_signal_receiver(handler_function, signal_name=signal_name, dbus_interface=dbus_interface, bus_name=self._named_service, path=self.__dbus_object_path__, **keywords) def _Introspect(self): kwargs = {} if is_py2: kwargs['utf8_strings'] = True return self._bus.call_async(self._named_service, self.__dbus_object_path__, INTROSPECTABLE_IFACE, 'Introspect', '', (), self._introspect_reply_handler, self._introspect_error_handler, require_main_loop=False, **kwargs) def _introspect_execute_queue(self): # FIXME: potential to flood the bus # We should make sure mainloops all have idle handlers # and do one message per idle for (proxy_method, args, keywords) in self._pending_introspect_queue: proxy_method(*args, **keywords) self._pending_introspect_queue = [] def _introspect_reply_handler(self, data): self._introspect_lock.acquire() try: try: self._introspect_method_map = process_introspection_data(data) except IntrospectionParserException as e: self._introspect_error_handler(e) return self._introspect_state = self.INTROSPECT_STATE_INTROSPECT_DONE self._pending_introspect = None self._introspect_execute_queue() finally: self._introspect_lock.release() def _introspect_error_handler(self, error): logging.basicConfig() _logger.error("Introspect error on %s:%s: %s.%s: %s", self._named_service, self.__dbus_object_path__, error.__class__.__module__, error.__class__.__name__, error) self._introspect_lock.acquire() try: _logger.debug('Executing introspect queue due to error') self._introspect_state = self.INTROSPECT_STATE_DONT_INTROSPECT self._pending_introspect = None self._introspect_execute_queue() finally: self._introspect_lock.release() def _introspect_block(self): self._introspect_lock.acquire() try: if self._pending_introspect is not None: self._pending_introspect.block() # else someone still has a _DeferredMethod from before we # finished introspection: no need to do anything special any more finally: self._introspect_lock.release() def _introspect_add_to_queue(self, callback, args, kwargs): self._introspect_lock.acquire() try: if self._introspect_state == self.INTROSPECT_STATE_INTROSPECT_IN_PROGRESS: self._pending_introspect_queue.append((callback, args, kwargs)) else: # someone still has a _DeferredMethod from before we # finished introspection callback(*args, **kwargs) finally: self._introspect_lock.release() def __getattr__(self, member): if member.startswith('__') and member.endswith('__'): raise AttributeError(member) else: return self.get_dbus_method(member) def get_dbus_method(self, member, dbus_interface=None): """Return a proxy method representing the given D-Bus method. The returned proxy method can be called in the usual way. For instance, :: proxy.get_dbus_method("Foo", dbus_interface='com.example.Bar')(123) is equivalent to:: proxy.Foo(123, dbus_interface='com.example.Bar') or even:: getattr(proxy, "Foo")(123, dbus_interface='com.example.Bar') However, using `get_dbus_method` is the only way to call D-Bus methods with certain awkward names - if the author of a service implements a method called ``connect_to_signal`` or even ``__getattr__``, you'll need to use `get_dbus_method` to call them. For services which follow the D-Bus convention of CamelCaseMethodNames this won't be a problem. """ ret = self.ProxyMethodClass(self, self._bus, self._named_service, self.__dbus_object_path__, member, dbus_interface) # this can be done without taking the lock - the worst that can # happen is that we accidentally return a _DeferredMethod just after # finishing introspection, in which case _introspect_add_to_queue and # _introspect_block will do the right thing anyway if self._introspect_state == self.INTROSPECT_STATE_INTROSPECT_IN_PROGRESS: ret = self.DeferredMethodClass(ret, self._introspect_add_to_queue, self._introspect_block) return ret def __repr__(self): return '<ProxyObject wrapping %s %s %s at %#x>' % ( self._bus, self._named_service, self.__dbus_object_path__, id(self)) __str__ = __repr__
class LookAtBehavior: def __init__(self, camera_root_topic): #self.wait = False #self.point3d = None self.state = 'ready' self.lock = RLock() self.lock.acquire() self.message = None self.STATES = { 'ready': 'ready', # none 'head_turn': 'head_turn', # something #'head_turn_drive': 'head_turn_drive', # something 'driving': 'driving' } # something rospy.init_node('look_at_point_behavior', anonymous=True) rospy.Subscriber('cursor3d', PointStamped, self.laser_point_handler) self.point_pub = rospy.Publisher('cursor3dcentered', PointStamped) self.double_click = rospy.Subscriber('mouse_left_double_click', String, self.move_base_double_click) self.double_click2 = rospy.Subscriber( 'mouse_left_double_click', String, self.cancel_move_base_double_click) self.camera_model = cam.ROSStereoCalibration( '/' + camera_root_topic + '/left/camera_info', '/' + camera_root_topic + '/right/camera_info') self.head_client = actionlib.SimpleActionClient( 'head_traj_controller/point_head_action', PointHeadAction) #self.head_client.wait_for_server() self.base_client = actionlib.SimpleActionClient( 'move_base', MoveBaseAction) #self.base_client.wait_for_server() #self.move_pub = rospy.Publisher('move_base_simple/goal', PoseStamped) self.move_pub = rospy.Publisher('look_at_point_goal', PoseStamped) #self.move_pub2 = rospy.Publisher('hai_constant', PoseStamped) self.tflistener = tf.TransformListener() self.lock.release() print 'running' def move_base_double_click(self, a_str): if self.message == None: rospy.logwarn('Unable to go, no message heard.') return else: self.lock.acquire() self.state = self.STATES['driving'] #Looking at the point last clicked on... (maybe keep doing this as we drive?) #self.look_at(self.message) #Move base self.move_base(self.message) self.message = None self.state = self.STATES['ready'] self.lock.release() def transform_point(self, point_stamped): point_head = point_stamped.point #Tranform into base link target_link = '/base_link' base_T_head = tfu.transform(target_link, point_stamped.header.frame_id, self.tflistener) point_mat_head = tfu.translation_matrix( [point_head.x, point_head.y, point_head.z]) point_mat_base = base_T_head * point_mat_head t_base, o_base = tfu.matrix_as_tf(point_mat_base) #Calculate angle robot should face angle = math.atan2(t_base[1], t_base[0]) q = tf.transformations.quaternion_from_euler(0, 0, angle) return (t_base, q, target_link) def move_base(self, point, wait=False): t_base, q, target_link = point ps = PoseStamped() ps.header.frame_id = target_link ps.pose.position = geometry_msgs.msg.Point(t_base[0], t_base[1], 0) ps.pose.orientation = geometry_msgs.msg.Quaternion(*q) self.move_pub.publish(ps) #Uncomment to actually move goal = MoveBaseGoal() goal.target_pose.header.frame_id = target_link goal.target_pose.pose.position = geometry_msgs.msg.Point( t_base[0], t_base[1], 0) goal.target_pose.pose.orientation = geometry_msgs.msg.Quaternion(*q) self.base_client.send_goal(goal) print 'Sent GOAL' if wait: self.base_client.wait_for_result() if self.base_client.get_state() == GoalStatus.SUCCEEDED: return True else: return False def laser_point_handler(self, point_stamped): p = np.matrix([ point_stamped.point.x, point_stamped.point.y, point_stamped.point.z, 1. ]).T p2d = self.camera_model.left.P * p p2d = p2d / p2d[2, 0] bx = ((self.camera_model.left.w / 2.) * .9) by = ((self.camera_model.left.h / 2.) * .9) xlim = [bx, self.camera_model.left.w - bx] ylim = [by, self.camera_model.left.h - by] if (self.state == self.STATES['driving']): return self.message = self.transform_point(point_stamped) if not in_bounds(p2d, xlim, ylim): if self.state != self.STATES['head_turn']: self.lock.acquire() self.state = self.STATES['head_turn'] self.lock.release() #else if we are in bounds, we do nothing #always update laser's location def run(self): r = rospy.Rate(50) while not rospy.is_shutdown(): r.sleep() if self.state == self.STATES['head_turn']: self.lock.acquire() result = self.look_at(self.message, False) self.state = self.STATES['ready'] self.lock.release() def look_at(self, message, wait=True): g = PointHeadGoal() g.target.header.frame_id = message[2] g.target.point = geometry_msgs.msg.Point(*message[0]) g.min_duration = rospy.Duration(1.0) g.max_velocity = 10. self.head_client.send_goal(g) #rospy.loginfo('Sent look at goal ' + str(g)) if wait: self.head_client.wait_for_result() if self.head_client.get_state() == GoalStatus.SUCCEEDED: return True else: return False
class PygletPlot(object): """ Plot Examples ============= See examples/advaned/pyglet_plotting.py for many more examples. >>> from sympy.plotting.pygletplot import PygletPlot as Plot >>> from sympy.abc import x, y, z >>> Plot(x*y**3-y*x**3) [0]: -x**3*y + x*y**3, 'mode=cartesian' >>> p = Plot() >>> p[1] = x*y >>> p[1].color = z, (0.4,0.4,0.9), (0.9,0.4,0.4) >>> p = Plot() >>> p[1] = x**2+y**2 >>> p[2] = -x**2-y**2 Variable Intervals ================== The basic format is [var, min, max, steps], but the syntax is flexible and arguments left out are taken from the defaults for the current coordinate mode: >>> Plot(x**2) # implies [x,-5,5,100] [0]: x**2, 'mode=cartesian' >>> Plot(x**2, [], []) # [x,-1,1,40], [y,-1,1,40] [0]: x**2, 'mode=cartesian' >>> Plot(x**2-y**2, [100], [100]) # [x,-1,1,100], [y,-1,1,100] [0]: x**2 - y**2, 'mode=cartesian' >>> Plot(x**2, [x,-13,13,100]) [0]: x**2, 'mode=cartesian' >>> Plot(x**2, [-13,13]) # [x,-13,13,100] [0]: x**2, 'mode=cartesian' >>> Plot(x**2, [x,-13,13]) # [x,-13,13,10] [0]: x**2, 'mode=cartesian' >>> Plot(1*x, [], [x], mode='cylindrical') ... # [unbound_theta,0,2*Pi,40], [x,-1,1,20] [0]: x, 'mode=cartesian' Coordinate Modes ================ Plot supports several curvilinear coordinate modes, and they independent for each plotted function. You can specify a coordinate mode explicitly with the 'mode' named argument, but it can be automatically determined for Cartesian or parametric plots, and therefore must only be specified for polar, cylindrical, and spherical modes. Specifically, Plot(function arguments) and Plot[n] = (function arguments) will interpret your arguments as a Cartesian plot if you provide one function and a parametric plot if you provide two or three functions. Similarly, the arguments will be interpreted as a curve if one variable is used, and a surface if two are used. Supported mode names by number of variables: 1: parametric, cartesian, polar 2: parametric, cartesian, cylindrical = polar, spherical >>> Plot(1, mode='spherical') # doctest: +SKIP Calculator-like Interface ========================= >>> p = Plot(visible=False) >>> f = x**2 >>> p[1] = f >>> p[2] = f.diff(x) >>> p[3] = f.diff(x).diff(x) # doctest: +SKIP >>> p # doctest: +SKIP [1]: x**2, 'mode=cartesian' [2]: 2*x, 'mode=cartesian' [3]: 2, 'mode=cartesian' >>> p.show() >>> p.clear() >>> p <blank plot> >>> p[1] = x**2+y**2 >>> p[1].style = 'solid' >>> p[2] = -x**2-y**2 >>> p[2].style = 'wireframe' >>> p[1].color = z, (0.4,0.4,0.9), (0.9,0.4,0.4) >>> p[1].style = 'both' >>> p[2].style = 'both' >>> p.close() Plot Window Keyboard Controls ============================= Screen Rotation: X,Y axis Arrow Keys, A,S,D,W, Numpad 4,6,8,2 Z axis Q,E, Numpad 7,9 Model Rotation: Z axis Z,C, Numpad 1,3 Zoom: R,F, PgUp,PgDn, Numpad +,- Reset Camera: X, Numpad 5 Camera Presets: XY F1 XZ F2 YZ F3 Perspective F4 Sensitivity Modifier: SHIFT Axes Toggle: Visible F5 Colors F6 Close Window: ESCAPE ============================= """ #python 2.5 does not support class decorators so use this workaround _doctest_depends_on = {'modules': ('pyglet', )} @doctest_depends_on(modules=('pyglet', )) def __init__(self, *fargs, **win_args): """ Positional Arguments ==================== Any given positional arguments are used to initialize a plot function at index 1. In other words... >>> from sympy.plotting.pygletplot import PygletPlot as Plot >>> from sympy.core import Symbol >>> from sympy.abc import x >>> p = Plot(x**2, visible=False) ...is equivalent to... >>> p = Plot(visible=False) >>> p[1] = x**2 Note that in earlier versions of the plotting module, you were able to specify multiple functions in the initializer. This functionality has been dropped in favor of better automatic plot plot_mode detection. Named Arguments =============== axes An option string of the form "key1=value1; key2 = value2" which can use the following options: style = ordinate none OR frame OR box OR ordinate stride = 0.25 val OR (val_x, val_y, val_z) overlay = True (draw on top of plot) True OR False colored = False (False uses Black, True uses colors R,G,B = X,Y,Z) True OR False label_axes = False (display axis names at endpoints) True OR False visible = True (show immediately True OR False The following named arguments are passed as arguments to window initialization: antialiasing = True True OR False ortho = False True OR False invert_mouse_zoom = False True OR False """ self._win_args = win_args self._window = None self._render_lock = RLock() self._functions = {} self._pobjects = [] self._screenshot = ScreenShot(self) axe_options = parse_option_string(win_args.pop('axes', '')) self.axes = PlotAxes(**axe_options) self._pobjects.append(self.axes) self[0] = fargs if win_args.get('visible', True): self.show() ## Window Interfaces def show(self): """ Creates and displays a plot window, or activates it (gives it focus) if it has already been created. """ if self._window and not self._window.has_exit: self._window.activate() else: self._win_args['visible'] = True self.axes.reset_resources() if hasattr(self, '_doctest_depends_on'): self._win_args['runfromdoctester'] = True self._window = PlotWindow(self, **self._win_args) def close(self): """ Closes the plot window. """ if self._window: self._window.close() def saveimage(self, outfile=None, format='', size=(600, 500)): """ Saves a screen capture of the plot window to an image file. If outfile is given, it can either be a path or a file object. Otherwise a png image will be saved to the current working directory. If the format is omitted, it is determined from the filename extension. """ self._screenshot.save(outfile, format, size) ## Function List Interfaces def clear(self): """ Clears the function list of this plot. """ self._render_lock.acquire() self._functions = {} self.adjust_all_bounds() self._render_lock.release() def __getitem__(self, i): """ Returns the function at position i in the function list. """ return self._functions[i] def __setitem__(self, i, args): """ Parses and adds a PlotMode to the function list. """ if not (isinstance(i, (int, Integer)) and i >= 0): raise ValueError("Function index must " "be an integer >= 0.") if isinstance(args, PlotObject): f = args else: if (not is_sequence(args)) or isinstance(args, GeometryEntity): args = [args] if len(args) == 0: return # no arguments given kwargs = dict(bounds_callback=self.adjust_all_bounds) f = PlotMode(*args, **kwargs) if f: self._render_lock.acquire() self._functions[i] = f self._render_lock.release() else: raise ValueError("Failed to parse '%s'." % ', '.join(str(a) for a in args)) def __delitem__(self, i): """ Removes the function in the function list at position i. """ self._render_lock.acquire() del self._functions[i] self.adjust_all_bounds() self._render_lock.release() def firstavailableindex(self): """ Returns the first unused index in the function list. """ i = 0 self._render_lock.acquire() while i in self._functions: i += 1 self._render_lock.release() return i def append(self, *args): """ Parses and adds a PlotMode to the function list at the first available index. """ self.__setitem__(self.firstavailableindex(), args) def __len__(self): """ Returns the number of functions in the function list. """ return len(self._functions) def __iter__(self): """ Allows iteration of the function list. """ return self._functions.itervalues() def __repr__(self): return str(self) def __str__(self): """ Returns a string containing a new-line separated list of the functions in the function list. """ s = "" if len(self._functions) == 0: s += "<blank plot>" else: self._render_lock.acquire() s += "\n".join([ "%s[%i]: %s" % ("", i, str(self._functions[i])) for i in self._functions ]) self._render_lock.release() return s def adjust_all_bounds(self): self._render_lock.acquire() self.axes.reset_bounding_box() for f in self._functions: self.axes.adjust_bounds(self._functions[f].bounds) self._render_lock.release() def wait_for_calculations(self): sleep(0) self._render_lock.acquire() for f in self._functions: a = self._functions[f]._get_calculating_verts b = self._functions[f]._get_calculating_cverts while a() or b(): sleep(0) self._render_lock.release()
class Connection(Framer): def __init__(self, sock, delegate=client, **args): Framer.__init__(self, sock) self.lock = RLock() self.attached = {} self.sessions = {} self.condition = Condition() # XXX: we should combine this into a single comprehensive state # model (whatever that means) self.opened = False self.failed = False self.closed = False self.close_code = (None, "connection aborted") self.thread = Thread(target=self.run) self.thread.setDaemon(True) self.channel_max = 65535 self.user_id = None self.op_enc = OpEncoder() self.seg_enc = SegmentEncoder() self.frame_enc = FrameEncoder() self.delegate = delegate(self, **args) def attach(self, name, ch, delegate, force=False): self.lock.acquire() try: ssn = self.attached.get(ch.id) if ssn is not None: if ssn.name != name: raise ChannelBusy(ch, ssn) else: ssn = self.sessions.get(name) if ssn is None: ssn = Session(name, delegate=delegate) self.sessions[name] = ssn elif ssn.channel is not None: if force: del self.attached[ssn.channel.id] ssn.channel = None else: raise SessionBusy(ssn) self.attached[ch.id] = ssn ssn.channel = ch ch.session = ssn return ssn finally: self.lock.release() def detach(self, name, ch): self.lock.acquire() try: self.attached.pop(ch.id, None) ssn = self.sessions.pop(name, None) if ssn is not None: ssn.channel = None ssn.closed() return ssn finally: self.lock.release() def __channel(self): for i in xrange(1, self.channel_max): if not self.attached.has_key(i): return i else: raise ChannelsBusy() def session(self, name, timeout=None, delegate=session.client): self.lock.acquire() try: ch = Channel(self, self.__channel()) ssn = self.attach(name, ch, delegate) ssn.channel.session_attach(name) if wait(ssn.condition, lambda: ssn.channel is not None, timeout): return ssn else: self.detach(name, ch) raise Timeout() finally: self.lock.release() def detach_all(self): self.lock.acquire() self.failed = True try: for ssn in self.attached.values(): if self.close_code[0] != 200: ssn.exceptions.append(self.close_code) self.detach(ssn.name, ssn.channel) finally: self.lock.release() def start(self, timeout=None): self.delegate.start() self.thread.start() if not wait(self.condition, lambda: self.opened or self.failed, timeout): self.thread.join() raise Timeout() if self.failed: self.thread.join() raise ConnectionFailed(*self.close_code) def run(self): frame_dec = FrameDecoder() seg_dec = SegmentDecoder() op_dec = OpDecoder() while not self.closed: try: data = self.sock.recv(64 * 1024) if not data: self.detach_all() break # If we have a security layer and it sends us no decoded data, # that's OK as long as its return code is happy. if self.security_layer_rx: try: data = self.security_layer_rx.decode(data) except: self.detach_all() break # When we do not use SSL transport, we get periodic # spurious timeout events on the socket. When using SSL, # these events show up as timeout *errors*. Both should be # ignored unless we have aborted. except socket.timeout: if self.aborted(): self.close_code = (None, "connection timed out") self.detach_all() break else: continue except socket.error, e: if self.aborted() or str(e) != "The read operation timed out": self.close_code = (None, str(e)) self.detach_all() break else: continue frame_dec.write(data) seg_dec.write(*frame_dec.read()) op_dec.write(*seg_dec.read()) for op in op_dec.read(): try: self.delegate.received(op) except Closed, e: self.close_code = (None, str(e)) if not self.opened: self.failed = True self.closed = True notify(self.condition)
class gui_queue: """wakes up the gui thread which then clears our queue""" def __init__(self, gui, listenport=0): """If listenport is 0, we create a random port to listen on""" self.mylock = RLock() self.myqueue = [] if listenport == 0: self.listenport = random.randint(1025, 10000) else: self.listenport = listenport print "Local GUI Queue listening on port %s" % self.listenport s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind(("127.0.0.1", self.listenport)) self.listensocket = s self.listensocket.listen(300) #listen for activity. s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) #time.sleep(15) self.gui = gui self.useconnect = 0 if not self.useconnect: s.connect(("127.0.0.1", self.listenport)) self.s = s self.readconn, addr = self.listensocket.accept() return def get_event_socket(self): if self.useconnect: return self.listensocket else: return self.readconn def append(self, command, args): """ Append can be called by any thread """ #print "about to aquire..." self.mylock.acquire() self.myqueue.append((command, args)) if self.useconnect: #this won't work on a host with a ZoneAlarm firewall or no internet connectivity... s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) #small timeout will wake up the gui thread, but not #cause painful pauses if we are already in the gui thread. #important to note that we use timeoutsocket and it #is already loaded. s.set_timeout(0.01) #wakey wakey! #print "Connecting to port %d"%self.listenport try: s.connect(("localhost", self.listenport)) except: #ignore timeouts pass else: self.s.send("A") #just send a byte #print "About to release" self.mylock.release() return def clearqueue(self): """ Clearqueue is only called by the main GUI thread Don't forget to return 1 """ #print "Clearing queue" #clear this...TODO: add select call here. if self.useconnect: newconn, addr = self.listensocket.accept() else: self.readconn.recv(1) for i in self.myqueue: (command, args) = i try: #any error in this thread is deadly, so we #catch and print them all! self.gui.handle_gui_queue(command, args) except: import traceback traceback.print_exc(file=sys.stdout) sys.stdout.flush() self.myqueue = [] return 1 def handle_gui_queue(self, command, args): """ Callback the gui_queue uses whenever it recieves a command for us. command is a string args is a list of arguments for the command """ gtk.threads_enter() #print "handle_gui_queue" if command == "addLine": #print "addLine called in canvasguigtk2.py" obj = args[0] self.addLine(obj) else: print "Did not recognize action to take %s: %s" % (command, args) #print "Done handling gui queue" gtk.threads_leave() return 1 def gui_queue_append(self, command, args): "Called by other classes to add to our list of things to do" self.gui_queue.append(command, args) return 1
class priority_queue: def __init__(self): self.heap_list = [0] self.size = 0 self.rlock = RLock() def insert(self, item): self.rlock.acquire() self.heap_list.append(item) self.size += 1 self.__perc_up(self.size) self.rlock.release() def __perc_up(self, index): while index // 2 > 0: if self.heap_list[index] < self.heap_list[index // 2]: self.heap_list[index // 2], self.heap_list[ index] = self.heap_list[index], self.heap_list[index // 2] index = index // 2 def get_min(self): if self.size > 0: return self.heap_list[1] def del_min(self): self.rlock.acquire() if self.size > 0: ret_val = self.heap_list[1] self.heap_list[1] = self.heap_list[self.size] self.size -= 1 self.heap_list.pop() self.__perc_down(1) return ret_val self.rlock.release() def __perc_down(self, index): def get_min_child(index): if index * 2 + 1 > self.size: return index * 2 else: if self.heap_list[index * 2] < self.heap_list[index * 2 + 1]: return index * 2 else: return index * 2 + 1 while index * 2 <= self.size: min_child = get_min_child(index) if self.heap_list[index] > self.heap_list[min_child]: self.heap_list[index], self.heap_list[ min_child] = self.heap_list[min_child], self.heap_list[ index] index = min_child def delete(self, item): self.heap_list.remove(item) self.size -= 1 i = self.size // 2 while i > 0: self.__perc_down(i) i -= 1 def __len__(self): return self.size def __contains__(self, item): return item in self.heap_list def __iter__(self): #TODO returns 0 element too return iter(self.heap_list)
class LibraryBroker(object): def __init__(self, libraries): self.lock = Lock() self.lmap = OrderedDict() self.library_name_map = {} self.original_path_map = {} seen = set() for original_path in libraries: path = canonicalize_path(original_path) if path in seen: continue is_samefile = False for s in seen: if samefile(s, path): is_samefile = True break seen.add(path) if is_samefile or not LibraryDatabase.exists_at(path): continue library_id = library_id_from_path(original_path, self.lmap) self.lmap[library_id] = path self.library_name_map[library_id] = basename(original_path) self.original_path_map[path] = original_path self.loaded_dbs = {} self.category_caches, self.search_caches, self.tag_browser_caches = ( defaultdict(OrderedDict), defaultdict(OrderedDict), defaultdict(OrderedDict)) def get(self, library_id=None): with self: library_id = library_id or self.default_library if library_id in self.loaded_dbs: return self.loaded_dbs[library_id] path = self.lmap.get(library_id) if path is None: return try: self.loaded_dbs[library_id] = ans = self.init_library( path, library_id == self.default_library) ans.new_api.server_library_id = library_id except Exception: self.loaded_dbs[library_id] = None raise return ans def init_library(self, library_path, is_default_library): library_path = self.original_path_map.get(library_path, library_path) return init_library(library_path, is_default_library) def close(self): with self: for db in itervalues(self.loaded_dbs): getattr(db, 'close', lambda: None)() self.lmap, self.loaded_dbs = OrderedDict(), {} @property def default_library(self): return next(iter(self.lmap)) @property def library_map(self): with self: return self.library_name_map.copy() def allowed_libraries(self, filter_func): with self: allowed_names = filter_func( basename(l) for l in itervalues(self.lmap)) return OrderedDict(((lid, self.library_map[lid]) for lid, path in iteritems(self.lmap) if basename(path) in allowed_names)) def path_for_library_id(self, library_id): with self: return self.original_path_map.get(self.lmap.get(library_id)) def __enter__(self): self.lock.acquire() def __exit__(self, *a): self.lock.release()
class TwitterInstances(object): def __init__(self, dataCollection, tweetProvider): super(TwitterInstances, self).__init__() assert isinstance(dataCollection, DataCollection) self._by_oauth = dict() self._by_instance_key = dict() self._lock = RLock() self.data_collection = dataCollection self.tweet_provider = tweetProvider def add(self, twitterInstance): assert isinstance(twitterInstance, TwitterInstance) self._lock.acquire() try: self._by_instance_key[ twitterInstance.instance_key] = twitterInstance self._by_oauth[twitterInstance.oauth] = twitterInstance finally: self._lock.release() def getUniqueInstanceKey(self): def func(): instanceKey = unicode(getUniqueId()) while instanceKey in self._by_instance_key: instanceKey = unicode(getUniqueId()) return instanceKey return criticalSection(self._lock, func) def createInstance(self, twitterAuthentication, geographic_setup_string, keywords, instance_setup_code): def func(): twitterInstance = TwitterInstance(self.getUniqueInstanceKey(), self, twitterAuthentication, geographic_setup_string, keywords, instance_setup_code) return twitterInstance return criticalSection(self._lock, func) def getInstanceList(self): return criticalSection(self._lock, lambda: list(self._by_instance_key.values())) def isInstanceKeyInUse(self, instanceKey): return criticalSection(self._lock, lambda: instanceKey in self._by_instance_key) def isAuthInUse(self, oauth): return criticalSection(self._lock, lambda: oauth in self._by_oauth) def getInstanceByInstanceKey(self, instanceKey): result = criticalSection( self._lock, lambda: self._by_instance_key.get(instanceKey, None)) return result def getInstanceByAuth(self, oauth): result = criticalSection(self._lock, lambda: self._by_oauth.get(oauth, None)) return result def removeTwitterInstanceByInstanceKey(self, instanceKey): self._lock.acquire() try: instance = self._by_instance_key.get(instanceKey) if instance is None: return None assert isinstance(instance, TwitterInstance) # Remove from dictionaries first so that it is no # longer accessible from the rest of the application. del self._by_instance_key[instanceKey] del self._by_oauth[instance.oauth] finally: self._lock.release() # Cleanup instance. instance.shutdownInstance(False) self.data_collection.removeInstanceData(instanceKey) return instance def removeTwitterInstanceByAuth(self, oauth): self._lock.acquire() try: instance = self._by_oauth.get(oauth) if instance is None: return None assert isinstance(instance, TwitterInstance) # Remove from dictionaries first so that it is no # longer accessible from the rest of the application. del self._by_oauth[oauth] del self._by_instance_key[instance.instance_key] print unicode(self._by_instance_key) finally: self._lock.release() # Cleanup instance. instance.shutdownInstance(False) self.data_collection.removeInstanceData(unicode(instance.instance_key)) return instance
class FakeHttpServer(Thread): TIMEOUT = 0.5 DEFAULT_RESPONSE = "text of the response" def __init__(self, port): Thread.__init__(self) class FakeHttpHandler(BaseHTTPRequestHandler): responses = { '/path' : dict( code = 200, contentType = 'text', body = """Response body""", ) } def __init__(self, *args, **kargs): BaseHTTPRequestHandler.__init__(self, *args, **kargs) def do_GET(self): response = FakeHttpHandler.responses[self.path] self.send_response(response['code']) if response.has_key('contentType'): self.send_header('Content-Type', response['contentType']) else: self.send_header('Content-Type', 'text') if response.has_key('locationAddr'): self.send_header('Location', response['locationAddr']) else: self.send_header('Location', 'locationAddr') if not response.has_key('dontProvideLength'): self.send_header('Content-Length', len(response['body'])) self.end_headers() self.wfile.write(response['body']) self.wfile.close() def log_message(self, *args, **kargs): pass self.running = True self.requestHandler = FakeHttpHandler self.server = _AvoidTimeoutHTTPServer( ('', port), FakeHttpHandler ) self.server.socket.settimeout(self.TIMEOUT) self.handlingLock = RLock() self.handling = False def waitUntilHandling(self): # Improve this with threading.Event / threading.Condition n = 5 while n > 0: self.handlingLock.acquire() try: if self.handling: return finally: self.handlingLock.release() n -= 1 time.sleep(self.TIMEOUT) raise Exception("Still waiting for the http server to handle requests...") def run(self): self.handlingLock.acquire() try: self.handling = True finally: self.handlingLock.release() while self.running: try: self.server.handle_request() except socket.timeout: pass def stop(self): self.running = False self.server.socket.close() def setResponses(self, responses): self.requestHandler.responses = responses
class AzureCloud(Cloud): def __init__(self, *args, **kwargs): super(AzureCloud, self).__init__(*args, **kwargs) self.vnet_lock = RLock() self.pace_lock = RLock() self.pace_timer = 300 # Is only allowed to be called once every 5 seconds # TODO: Why is there a limitation like this on Azure ... 5 seconds # wait per request is too much ... #@rate_limit(0.2) def execute(self, command, obj={}): ret = super(AzureCloud, self).execute(command, obj) # If we are too fast, backoff for 5 minutes before continuing again if 'Too many requests received' in obj['stderr']: print "Sleeping for 5 minutes:\n > %s, %s, %s" % ( command, obj['stderr'], obj['stdout']) time.sleep(self.pace_timer) return self.execute(command, obj) return ret def start_virtual_machine(self, vm): """ Start a virtual machine """ cmd = ['azure', 'vm', 'start', self.unique(vm.name)] vm._started = True return self.execute(cmd) def stop_virtual_machine(self, vm): """ Stop a virtual machine """ cmd = ['azure', 'vm', 'shutdown', self.unique(vm.name)] vm._started = False return self.execute(cmd) def status_virtual_machine(self, vm): return vm._started # cmd = ['azure', 'vm', 'show', self.unique(vm.name)] # # output = {} # self.execute(cmd, output) # if 'ReadyRole' in output['stdout']: # return True # if 'Stopped' in output['stdout']: # return False # return None def exists_virtual_machine(self, vm): cmd = ['azure', 'vm', 'show', self.unique(vm.name)] output = {} self.execute(cmd, output) return 'No VMs found' in output['stdout'] def address_virtual_machine(self, vm): """ Returns the address of a vm """ # TODO: Change the name to address_virtual_machine return self.unique(vm.name) + ".cloudapp.net" def hashify_22(self, name): import hashlib return str(hashlib.md5(name).hexdigest())[0:22] def create_location(self, group): """ Create an affinity group in microsoft terms """ # cmd = ['azure', 'storage', 'account', 'create'] # cmd += ['-a', self.unique(group.name)] # cmd += ['--type', group.storage_type] # cmd += [self.hashify_22(self.unique(group.name))] # return self.execute(cmd) cmd = ['azure', 'account', 'affinity-group', 'create'] cmd += self.if_available('-l', group.location) cmd += ['-e', base64.b64encode(self.unique(group.name))] cmd += [self.unique(group.name)] self.execute(cmd) # TODO: creating a location tends to fail because we can't # cleanly delete it ... for now return true on creating a # location return True def create_security_group(self, ep): """ Create endpoints in the microsoft terms """ ret = True # TODO: Can parallelize here def create_endpoint(vm): cmd = ['azure', 'vm', 'endpoint', 'create'] cmd += [self.unique(vm), ep.public_port, ep.private_port] cmd += ['--name', self.unique(ep.name)[-15:] ] # Endpoint name should be at most 15 characters cmd += ['--protocol', ep.protocol] self.execute(cmd) parallel(create_endpoint, ep.virtual_machines()) return ret def create_virtual_machine(self, vm): """ Create a virtual machine """ cmd = ['azure', 'vm', 'create', '-z', vm.type] cmd += self.if_available('-a', self.unique(vm.location())) cmd += self.if_available('-w', self.unique(vm.virtual_network())) cmd += [ '-e', '22', self.unique(vm.name), vm.image, 'cloudbench', '-P', '-t', constants.DEFAULT_VM_PUBLIC_KEY ] ret = self.execute(cmd) return True def create_virtual_network(self, vnet): """ Create a virtual network """ # Azure cannot create multiple VNets together, lock on creation # of each VNet self.vnet_lock.acquire() ret = False try: cmd = ['azure', 'network', 'vnet', 'create'] cmd += self.if_available('-e', vnet.address_range) cmd += self.if_available('-a', self.unique(vnet.location())) cmd += [self.unique(vnet.name)] ret = self.execute(cmd) finally: self.vnet_lock.release() return True def delete_security_group(self, _): """ Delete an azure 'security-group' a.k.a. an endpoint. We do not need to delete anything here, Azure takes care of it when we wipe out the machine""" return True def delete_virtual_machine(self, virtual_machine): """ Delete a virtual machine and the associated storage """ cmd = [ 'azure', 'vm', 'delete', '-b', '-q', self.unique(virtual_machine.name) ] return self.execute(cmd) def delete_virtual_network(self, vnet): """ Delete a virtual network """ # Serialize network creation self.vnet_lock.acquire() ret = False try: cmd = [ 'azure', 'network', 'vnet', 'delete', '-q', self.unique(vnet.name) ] ret = self.execute(cmd) finally: self.vnet_lock.release() return ret def delete_location(self, group): cmd = ['azure', 'account', 'affinity-group', 'delete', '-q'] cmd += [self.unique(group.name)] self.execute(cmd) # TODO: creating a location tends to fail because we can't # cleanly delete it ... for now return true on creating a # location return True
class testGoogle(): ips=[] tlist=[] succIp=[] def getIp(self): url="http://www.legendsec.org/google.html" hds={"User-agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36"} import urllib2 req=urllib2.Request(url,None,hds) result=urllib2.urlopen(req).read() from bs4 import BeautifulSoup as bs import re root=bs(result) ip_list=root.findAll("td",text=re.compile(r"\d+\.\d+\.\d+\.\d+")) for ip in ip_list: self.ips.append(ip.a.text) def endThread(self): for t in self.tlist: t.join() self.saveto(OUTFILE) def saveto(self,path): try: fd=open(path,'w') if self.succIp: fd.write('|'.join(self.succIp)) print "已将结果输出到%s"%(OUTFILE) except IOError: print '|'.join(self.succIp) print "保存到文件出错~" finally: fd.close() def connIp(self,options): import random atexit.register(self.endThread) self.r=RLock() while self.ips: now_ip=random.choice(self.ips) now_t=Thread(target=self.__connection, args=(now_ip,options.count)) now_t.setDaemon(True) self.tlist.append(now_t) self.ips.remove(now_ip) if len(self.tlist) == 5: for t in self.tlist: t.start() for t in self.tlist: t.join() self.tlist=[] def __connection(self,host,count): import socket import time socket.setdefaulttimeout(1) cost=0 for port in (80,443): for i in range(count): start=time.time() s=socket.socket() try: s.connect((host,port)) end=time.time() cost+=(end-start) except: break finally: s.close() self.r.acquire() if cost: cost=cost*1000.0/count print u'connect to %s 平均用时%.2f ms'%(host, cost) if cost<MAXTIMEOUT: self.succIp.append(host) else: print "connect to %s time out"%(host) self.r.release()
class Logger(object): # paste from goagent CRITICAL = 5 FATAL = CRITICAL ERROR = 4 WARNING = 3 WARN = WARNING INFO = 2 DEBUG = 1 VERBOSE = 0 def __init__(self, *args, **kwargs): # self.level = self.__class__.INFO self.logf = None self.__write = __write = lambda x: sys.stdout.write(safestr(x)) self.isatty = getattr(sys.stdout, 'isatty', lambda: False)() self.__set_error_color = lambda: None self.__set_warning_color = lambda: None self.__set_debug_color = lambda: None self.__set_verbose_color = lambda: None self.__reset_color = lambda: None if self.isatty: if os.name == 'nt': self._nt_color_lock = RLock() import ctypes SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute GetStdHandle = ctypes.windll.kernel32.GetStdHandle self.__set_error_color = lambda: (self._nt_color_lock.acquire( ), SetConsoleTextAttribute(GetStdHandle(-11), 0x0C)) self.__set_warning_color = lambda: ( self._nt_color_lock.acquire(), SetConsoleTextAttribute(GetStdHandle(-11), 0x06)) self.__set_debug_color = lambda: (self._nt_color_lock.acquire( ), SetConsoleTextAttribute(GetStdHandle(-11), 0x02)) self.__set_verbose_color = lambda: ( self._nt_color_lock.acquire(), SetConsoleTextAttribute(GetStdHandle(-11), 0x08)) self.__set_bright_color = lambda: (self._nt_color_lock.acquire( ), SetConsoleTextAttribute(GetStdHandle(-11), 0x0F)) self.__reset_color = lambda: (SetConsoleTextAttribute( GetStdHandle(-11), 0x07), self._nt_color_lock.release()) elif os.name == 'posix': self.__set_error_color = lambda: __write('\033[31m') self.__set_warning_color = lambda: __write('\033[33m') self.__set_debug_color = lambda: __write('\033[32m') self.__set_verbose_color = lambda: __write('\033[36m') self.__set_bright_color = lambda: __write('\033[32m') self.__reset_color = lambda: __write('\033[0m') @classmethod def getLogger(cls, *args, **kwargs): return cls(*args, **kwargs) def cleanup(self): if self.logf: _ = self.logf self.logf = None _.close() def set_logfile(self, fpath): if self.logf: self.logf.close() self.logf = open(fpath, "ab") def set_level(self, level): f = ('verbose', 'debug', 'info') lv = min(max(level, 0), 3) for p in range(lv): setattr(self, f[p], self.dummy) def log(self, level, fmt, *args, **kwargs): # fmt=du8(fmt) try: self.__write('%-4s - [%s] %s\n' % (level, datetime.datetime.now( tz_GMT8()).strftime('%X'), fmt % args)) except (ValueError, TypeError): fmt = fmt.replace('%', '%%') self.__write('%-4s - [%s] %s\n' % (level, datetime.datetime.now( tz_GMT8()).strftime('%X'), fmt % args)) sys.stdout.flush() if self.logf: _ = ('[%s] %s%s' % (datetime.datetime.now( tz_GMT8()).strftime('%b %d %X'), fmt % args, endl)) self.logf.write(_.encode("utf-8", 'replace')) def dummy(self, *args, **kwargs): pass def debug(self, fmt, *args, **kwargs): self.__set_debug_color() self.log('DEBG', fmt, *args, **kwargs) self.__reset_color() def info(self, fmt, *args, **kwargs): puretext = self.log('INFO', fmt, *args) # if self.logfile: # self.logfile.write(puretext) def verbose(self, fmt, *args, **kwargs): self.__set_verbose_color() self.log('VERB', fmt, *args, **kwargs) self.__reset_color() def warning(self, fmt, *args, **kwargs): self.__set_warning_color() self.log('WARN', fmt, *args, **kwargs) self.__reset_color() def warn(self, fmt, *args, **kwargs): self.warning(fmt, *args, **kwargs) def error(self, fmt, *args, **kwargs): self.__set_error_color() self.log('ERROR', fmt, *args, **kwargs) self.__reset_color() def exception(self, fmt, *args, **kwargs): self.error(fmt, *args, **kwargs) traceback.print_exc(file=sys.stderr) def critical(self, fmt, *args, **kwargs): self.__set_error_color() self.log('CRITICAL', fmt, *args, **kwargs) self.__reset_color()
class VideoRawVLCServer(): __single = None def __init__(self): if VideoRawVLCServer.__single: raise RuntimeError, 'VideoRawVLCServer is Singleton' VideoRawVLCServer.__single = self self.lock = RLock() self.oldsid = None self.sid2streaminfo = {} def getInstance(*args, **kw): if VideoRawVLCServer.__single is None: VideoRawVLCServer(*args, **kw) return VideoRawVLCServer.__single getInstance = staticmethod(getInstance) def set_inputstream(self, streaminfo, sid): self.lock.acquire() try: print >> sys.stderr, 'VLCRawServer: setting sid', sid self.sid2streaminfo[sid] = streaminfo finally: self.lock.release() def get_inputstream(self, sid): self.lock.acquire() try: return self.sid2streaminfo[sid] finally: self.lock.release() def shutdown(self): pass def ReadDataCallback(self, bufc, buflen, sid): try: if self.oldsid is not None and self.oldsid != sid: oldstream = self.sid2streaminfo[self.oldsid]['stream'] del self.sid2streaminfo[self.oldsid] try: oldstream.close() except: log_exc() self.oldsid = sid streaminfo = self.get_inputstream(sid) data = streaminfo['stream'].read(buflen) size = len(data) if size == 0: return 0 bufc[0:size] = data return size except: log_exc() return -1 def SeekDataCallback(self, pos, sid): try: if True: streaminfo = self.get_inputstream(sid) streaminfo['stream'].seek(pos, os.SEEK_SET) return 0 return -1 except: log_exc() return -1
if resType is None: GLOBAL["mainWnd"].emit(SIGNAL("CRITICAL_MESSAGE(QString)"), u"设备初始化失败") return None rs = resType(param) resources[resName] = rs try: rs.initResource() except TestResourceInitException, e: GLOBAL["mainWnd"].emit(SIGNAL("CRITICAL_MESSAGE(QString)"), e.message) del resources[resName] return None return rs finally: resourceLock.release() class TestResource: '''测试资源从此处派生,此为接口定义''' def initResource(self): '''初始化资源''' def retrive(self): '''回收资源''' class TestResourceInitException(Exception): '''测试资源初始化失败时,抛出此异常''' def __init__(self, message): self.message = message