class XBeeTransparentListener(Thread):
def __init__(self, xbee_serial):
super().__init__()
self.xbser = xbee_serial
self.daemon = True
self.stopped = False
self.pause = RLock()
def run(self):
while not self.stopped and self.xbser.is_open:
with self.pause:
try:
line = self.xbser.readline()
if line:
print('>', line.strip())
except Exception as ex:
print(str(ex))
def stop(self):
self.stopped = True
def pause(self):
self.pause.acquire()
def unpause(self):
self.pause.release()
class PrintingThread(Thread):
def __init__(self, printer, cmds):
log.info("New printing thread, printer %s, len %d" %
(printer, len(cmds)))
self.cmds=cmds
self.printer=printer
self.lock=RLock()
self.state=0
Thread.__init__(self)
def run(self):
printer=self.printer
printer.zero()
from time import time, sleep
from os import path
#wait=time()
#log.info("Waiting for load page button.")
#while path.getmtime(job_dir+"load_new_page")<wait:
# sleep(1.0)
#log.info("Load page button press detected. Starting to print.")
for i, step in enumerate(self.cmds):
printer.do(step)
self.lock.acquire()
self.state=i
self.lock.release()
printer.eject_page()
def progress(self):
self.lock.acquire()
res=self.state
self.lock.release()
return res/float(len(self.cmds)-1)
class SingleIndexCache(object):
def __init__(self):
self.lock = RLock()
self.cached_vals = []
def __enter__(self):
self.lock.acquire()
return self
def fill(self, values):
self.cached_vals = values
def wipe(self):
self.cached_vals = []
def get_index(self, real_funct, ind_name, **kwargs):
kwargs.setdefault("max_results", 999999)
if not self.cached_vals:
recvs = real_funct(ind_name, **kwargs).results
self.fill(recvs)
def filter(self, startkey, endkey, max_results=1):
c = self.cached_vals
for (curr_val, obj_key) in c:
if max_results == 0:
break
if curr_val >= startkey:
if curr_val <= endkey:
max_results -= 1
yield (curr_val, obj_key)
else:
break
def __exit__(self, *args, **kwargs):
self.lock.release()
class Database(object):
UNICODE_TRANSLATE = {ord(u'ö'): u'o', ord(u'ä'): u'a', ord(u'ü'): u'u'}
def __init__(self, filename):
self.database_file = filename
self.channel_map = {}
self._db = None
self.lock = RLock()
def close(self):
self.commit()
cur = self.cursor(lock=True)
log('Optimizing database for faster startup.', sender=self)
cur.execute("VACUUM")
cur.close()
self.lock.release()
self._db.close()
self._db = None
def log(self, message, *args, **kwargs):
if False:
try:
message = message % args
log('%s', message, sender=self)
except TypeError, e:
log('Exception in log(): %s: %s', e, message, sender=self)
class VM_Pool(object):
def __init__(self, vm_map):
self.proc_mgr = proc_mgmt.ProcMgr()
self.vm_map = vm_map
self.vm_rdy = {}
self.init_map()
self.pool_gate = RLock()
def acquire(self, *names):
self.pool_gate.acquire()
for name in names:
if self.vm_rdy.get(name):
self.vm_rdy[name] = False
return self.vm_map.get(name)
self.pool_gate.release()
return None
def release(self, name):
self.vm_rdy[name] = True
def init_map(self):
for name, vm_obj in self.vm_map.items():
self.vm_rdy[name] = True
def __str__(self):
string = 'Pool:'
for vm in self.vm_map.keys():
string += vm + ": " + str(self.vm_rdy.get(vm)) + ", "
return string
class ZenitherClient():
def __init__(self, robot):
try:
rospy.init_node('ZenitherClient')
rospy.logout('ZenitherServer: Initialized Node')
except rospy.ROSException:
pass
if robot not in zc.calib:
raise RuntimeError('unknown robot')
self.calib = zc.calib[robot]
srv = '/zenither/move_position'
rospy.wait_for_service(srv)
self.move_position = rospy.ServiceProxy(srv, Float_Int)
srv = '/zenither/stop'
rospy.wait_for_service(srv)
self.stop = rospy.ServiceProxy(srv, Float_Int)
srv = '/zenither/apply_torque'
rospy.wait_for_service(srv)
self.apply_torque = rospy.ServiceProxy(srv, Float_Int)
srv = '/zenither/torque_move_position'
rospy.wait_for_service(srv)
self.torque_move_position = rospy.ServiceProxy(srv, Float_Int)
zenither_pose_topic = 'zenither_pose'
self.h = None
self.lock = RLock()
rospy.Subscriber(zenither_pose_topic, FloatArray, self.pose_cb)
#---------- functions to send zenither commands. -------------
def estop(self):
self.stop(0)
def zenith(self, torque=None):
if torque == None:
torque=self.calib['zenith_torque']
self.apply_torque(torque)
def nadir(self, torque=None):
if torque == None:
torque=self.calib['nadir_torque']
self.apply_torque(torque)
#--------- zenither height functions --------------
def pose_cb(self, fa):
self.lock.acquire()
self.h = fa.data[0]
self.lock.release()
## return the current height of the zenither.
def height(self):
self.lock.acquire()
h = self.h
self.lock.release()
return h
class SimpleCSVWriter:
def __init__(self, filename, fields=None):
self.filename = filename
self.lock = RLock()
self.isFirstRow = True
self.fields = fields
def writerow(self, d):
self.lock.acquire()
fields = self.fields if self.fields is not None else d.keys()
if self.isFirstRow:
# dump fields
f = open(self.filename , "w")
writer = csv.writer(f, lineterminator="\n", quoting=csv.QUOTE_ALL)
row = [k for k in fields]
writer.writerow(row)
f.close()
self.isFirstRow = False
# dump object
row = [d.get(k,'') for k in fields]
f = open(self.filename , "a")
writer = csv.writer(f, lineterminator="\n", quoting=csv.QUOTE_ALL)
writer.writerow(row)
f.close()
self.lock.release()
def close_dynamic_queue(self, dynamic_queue_name):
if self._disconnecting:
self.logger.info("Connection factory disconnecting, aborting close_dynamic_queue")
return
else:
self.logger.log(TRACE1, "close_dynamic_queue -> not disconnecting")
if not self._is_connected:
# If we're not connected then all dynamic queues had been already closed.
self.logger.log(TRACE1, "close_dynamic_queue -> _is_connected1 %s" % self._is_connected)
return
else:
self.logger.log(TRACE1, "close_dynamic_queue -> _is_connected2 %s" % self._is_connected)
lock = RLock()
lock.acquire()
try:
dynamic_queue = self._open_dynamic_queues_cache[dynamic_queue_name]
dynamic_queue.close()
self._open_dynamic_queues_cache.pop(dynamic_queue_name, None)
self._open_send_queues_cache.pop(dynamic_queue_name, None)
self._open_receive_queues_cache.pop(dynamic_queue_name, None)
self.logger.log(TRACE1, "Successfully closed a dynamic queue [%s]" % (
dynamic_queue_name))
finally:
lock.release()
class DataSignaler(object):
def __init__(self, name, pruneFunc, data):
super(DataSignaler, self).__init__()
assert isinstance(data,TreeFunctioned)
self.data = data
self.event_signaler = EventSignaler(key=name)
if pruneFunc is not None:
assert callable(pruneFunc)
self.prune_func = pruneFunc
self._lock = RLock()
def add(self, value):
self._lock.acquire()
try:
self.data.addToTreeByFunction(value)
finally:
self._lock.release()
data = {self.event_signaler.key : {'data': self.data}}
self.event_signaler.signalEvent(data)
def prune(self):
if self.prune_func is not None:
return criticalSection(self._lock, lambda: self.prune_func(dataStructure=self.data))
def inByFunction(self, value, hashFuncList=None, depth=0):
return criticalSection(self._lock, lambda: self.data.inByFunction(value, hashFuncList, depth))
def getOriginalByFunction(self, value, hashFuncList=None, depth=0):
return criticalSection(self._lock, lambda: self.data.getOriginalByFunction(value, hashFuncList, depth))
class XBeeTransparentListener(Thread):
def __init__(self, on_received=None):
super().__init__()
self.xbser = None
self.on_received = on_received
self.daemon = True
self.stopped = False
self.pause = RLock()
def run(self):
while not self.stopped and self.xbser.is_open:
with self.pause:
try:
line = self.xbser.readline()
if line:
self.received(line)
except Exception as ex:
print(str(ex))
def received(self, line):
"""Subclasses may override this method, or provide a callback function when instance is created"""
if self.on_received:
self.on_received(line)
else:
print('[XBee]', line.strip())
def stop(self):
self.stopped = True
def pause(self):
self.pause.acquire()
def unpause(self):
self.pause.release()
def open_dynamic_queue(self):
if self._disconnecting:
self.logger.info("Connection factory disconnecting, aborting open_dynamic_queue")
return
else:
self.logger.log(TRACE1, "open_dynamic_queue -> not disconnecting")
if not self._is_connected:
self.logger.log(TRACE1, "open_dynamic_queue -> _is_connected1 %s" % self._is_connected)
self._connect()
self.logger.log(TRACE1, "open_dynamic_queue -> _is_connected2 %s" % self._is_connected)
dynamic_queue = self.mq.Queue(self.mgr, self.dynamic_queue_template,
self.CMQC.MQOO_INPUT_SHARED)
# A bit hackish, but there's no other way to get its name.
dynamic_queue_name = dynamic_queue._Queue__qDesc.ObjectName.strip()
lock = RLock()
lock.acquire()
try:
self._open_dynamic_queues_cache[dynamic_queue_name] = dynamic_queue
finally:
lock.release()
self.logger.log(TRACE1, "Successfully created a dynamic queue, descriptor [%s]" % (
dynamic_queue._Queue__qDesc))
return dynamic_queue_name
class TriggerDevice(NeuroDevice):
def __init__(self, freq = 256, channels = 1):
self.freq = freq
self.channels = channels
self.header = Header(TRIGGER_HEADER)
self.header.channelCount = channels
for i in range(channels):
self.header.channels[i].samplingFrequency = freq
self.header.channels[i].label = 'TRIGGER%d' % i
self.values = [0,]*self.channels
self.valLock = RLock()
self.queue = Queue(15)
self.thread = TriggerDeviceThread(self)
self.thread.start()
def getValues(self):
self.valLock.acquire()
try:
return tuple(self.values)
finally:
self.valLock.release()
def setValues(self, val):
self.valLock.acquire()
try:
self.values[:] = val
finally:
self.valLock.release()
def getHeader(self):
return self.header.text()
def getData(self):
return self.queue.get(10.0)
class ThreadData(object):
def __init__(self):
self._data = {}
self._lock = RLock()
def __setitem__(self, key, value):
self._lock.acquire()
thread = current_thread()
if thread not in self._data:
self._data[thread] = {}
self._data[thread][key] = value
self._lock.release()
def __getitem__(self, key):
thread = current_thread()
return self._data[thread][key]
def __delitem__(self, key):
del(self[key])
def __contains__(self, key):
thread = current_thread()
return key in self._data[thread]
def update(self, data):
self._lock.acquire()
thread = current_thread()
self._data[thread].update(data)
self._lock.release()
def clean(self):
thread = current_thread()
if thread in self._data:
del(self._data[thread])
class CacheDict(dict):
_cacheTimeout = 0
_accesslock = None
def __init__(self, timeout):
self._cacheTimeout = timeout
self._accesslock = RLock()
def get(self, key):
return self[key]['value']
def cache(self, key, value):
self[key] = { 'time' : time.time(), 'value' : value }
def isObsolete(self, key):
return (not self.has_key(key) or
time.time() - self[key]['time'] > self._cacheTimeout)
def invalidate(self, key):
if self.has_key(key):
return self.pop(key)['value']
def acquire(self):
self._accesslock.acquire()
def release(self):
self._accesslock.release()
def resetMysqlMACAndSN(self,mac):
mysql = MySQLCommand(host=self.sysXMLDict['mysqlhost'], port=int(self.sysXMLDict['mysqlport']),
user=self.sysXMLDict['mysqluser'], passwd=self.sysXMLDict['mysqlpassword'],
db=self.sysXMLDict['mysqldatabase'], table=self.sysXMLDict['mysqltable'])
mysqlConFlag = mysql.connectMysql()
if not mysqlConFlag:
logging.info('reset status connect failed.')
return False
locker = RLock()
locker.acquire()
resetFlag = mysql.resetMysqlMACStatusAndSN(mac=mac,stbType=self.sysXMLDict['mysqlstbtype'], poNumber=self.poNumber)
if resetFlag:
logging.info('reset mysql status success.')
else:
logging.info('reset mysql status failed.')
mysql.closeMysql()
locker.release()
return False
mysql.closeMysql()
locker.release()
return True
class ThreadSafeFSM(InstrumentFSM):
"""
A FSM class that provides thread locking in on_event to
prevent simultaneous thread reentry.
"""
def __init__(self, states, events, enter_event, exit_event):
"""
"""
super(ThreadSafeFSM, self).__init__(states, events, enter_event, exit_event)
self._lock = RLock()
def on_event(self, event, *args, **kwargs):
"""
"""
self._lock.acquire(True)
ex = None
try:
result = super(ThreadSafeFSM, self).on_event(event, *args, **kwargs)
except Exception as ex:
result = None
log.error("Unhandled Exception")
log.exception(ex)
finally:
self._lock.release()
if ex:
raise ex
return result
class AndroidSocket(asynchat.async_chat):
def __init__(self):
self.logger = logging.getLogger("AndroidSocket")
# Connect to the Android (when we start the async loop - this is asynchronous)
asynchat.async_chat.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.connect( ANDROID_HOST )
# Set up input buffer and define message terminator
self.input_buffer = []
self.set_terminator("\n")
self.socket_lock = RLock()
# Making async_chat thread-safe
def push(self, data):
try:
self.socket_lock.acquire()
asynchat.async_chat.push(self, data)
finally:
self.socket_lock.release()
# Making async_chat thread-safe
def initiate_send(self):
try:
self.socket_lock.acquire()
asynchat.async_chat.initiate_send(self)
finally:
self.socket_lock.release()
def handle_error(self):
self.logger.error("================ ERROR! Failed to send something! ================ ")
etype, value, tb = sys.exc_info()
traceback.print_exception(etype, value, tb)
def handle_connect(self):
# Init all the devices we need to control
try:
self.commandDispatcher = CommandDispatcher(self)
except Exception as e:
print e
raise e
self.logger.info("Sending CREEPER_READY status")
self.push("CREEPER_READY:\n")
def collect_incoming_data(self, data):
self.input_buffer.append(data)
def found_terminator(self):
self.handle_android_command(self.input_buffer[0])
self.input_buffer = []
def handle_android_command(self, command_data):
self.logger.info("Received command: %s" % command_data)
self.commandDispatcher.process_command(command_data)
class _Spinner(Thread):
def __init__(self, message):
Thread.__init__(self)
self.rlock = RLock()
self.cv = Condition()
self.__chars = u"⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"
if len(message) > 0:
self.__message = " " + message
else:
self.__message = ""
self.__message_length = len(self.__message)
def __clear(self):
stdout.write(
'\b' * (self.__message_length + 2) +
' ' * (self.__message_length + 2) +
'\b' * (self.__message_length + 2))
stdout.flush()
def __call__(self):
self.start()
def start(self):
self.stopFlag = 0
Thread.start(self)
def stop(self):
"""To be called by the 'main' thread: Will block and wait for the
thread to stop before returning control to 'main'."""
self.stopFlag = 1
# Wake up ahead of time if needed
self.cv.acquire()
self.cv.notify()
self.cv.release()
# Block and wait here untill thread fully exits its run method.
self.rlock.acquire()
def run(self):
self.rlock.acquire()
self.cv.acquire()
stdout.write(' ' + self.__message)
stdout.write('\b' * self.__message_length)
stdout.flush()
while 1:
for char in self.__chars:
self.cv.wait(0.1)
if self.stopFlag:
self.__clear()
try :
return
finally :
# release lock immediatley after returning
self.rlock.release()
stdout.write('\b')
stdout.write(char)
stdout.flush()
class MysqlDb(BaseDb):
def __init__(self, db_uri):
parts = urlsplit(db_uri)
self.host = parts.hostname
self.port = parts.port or 3306
self.user = parts.username
self.pwd = parts.password or ""
self.dbname = parts.path[1:]
self.db_con_lock = RLock()
self.db_con = None
self.db_cur = None
def acquire_cursor(self):
try:
self.db_con_lock.acquire()
self.__get_connection()
self.db_cur = self.db_con.cursor()
except StandardError:
if self.db_con is not None:
self.db_con.close()
if self.db_cur is not None:
self.db_cur.close()
self.db_con_lock.release()
return self.db_cur
def release_cursor(self):
self.db_cur.close()
self.db_con.commit()
self.db_con.close()
self.db_con = None
self.db_con_lock.release()
def __get_connection(self):
if self.db_con is None:
con = MySQLdb.connect(self.host, self.user, self.pwd, self.dbname, self.port)
else:
self.db_con.ping(True)
con = self.db_con
self.db_con = con
return con
def cursor(self):
c = Cursor(self)
return c
class Main(HTTPServlet):
registerShutdown = 1
def __init__(self):
HTTPServlet.__init__(self)
self.blogDirectory = WeblogDirectory("../blogs.xml")
self.mutex = RLock()
def awake(self, transaction):
# Register our shutdown handler if it hasn't already been done. This is to
# make sure the databases are properly closed when the system is shutdown.
self.mutex.acquire()
try:
if (Main.registerShutdown == 1):
transaction.application().addShutDownHandler(self.blogDirectory.shutdown)
Main.registerShutdown = 0
finally:
self.mutex.release()
def respondToGet(self, transaction):
request = transaction.request()
response = transaction.response()
pathInfo = request.extraURLPath()
try:
(blog, pathInfo) = self._parsePathInfo(pathInfo)
weblog = self.blogDirectory.getBlog(blog)
try:
stylesheet = request.field('t', "")
# Extra optional argument that can be passed to the stylesheet
arg = request.field('a', "")
# Content query that can be applied as a final step to extract
# something from the rendered content
contentQuery = request.field('c', "")
result = weblog.handleRequest(pathInfo, stylesheet, arg, contentQuery)
# Determine the content-type for the result
if (result.startswith("<?xml")):
contentType = "text/xml"
elif (result.startswith("<html")):
contentType = "text/html"
else:
contentType = "text/plain"
#print result
response.setStatus(200, 'OK')
response.setHeader('Content-type', contentType)
response.setHeader('Content-length', str(len(result)))
response.write(result)
except NotFoundError:
response.setStatus(404, 'Not Found')
except KeyError, IndexError:
response.setStatus(404, 'Weblog Not Found')
class ScrollableHandler(StreamHandlerNoLF):
""" ScrollableHandler is a StreamHandler that specially handles scrolling (log
messages at the SCROLL level). It allows you to temporarily interrupt the constant
scroll with other log messages of different levels (printed at the top of the scroll
area) """
# the SCROLL level (a class var)
LOGFILE = 11
SCROLL = 12
SHUTDOWN = 13
NOLOGFILE = 14
def __init__(self, *args, **kwargs):
self.scrollLock = RLock()
self.scrollFlag = False
StreamHandlerNoLF.__init__(self, *args, **kwargs)
def handle(self, record):
""" The 'scroll' level is a constant scroll that can be interrupted. This interruption is
done via prepending text to the scroll area """
rv = self.filter(record)
if rv:
if record.levelno == ScrollableHandler.SCROLL:
self.emitSynchronized(record)
elif record.levelno == ScrollableHandler.SHUTDOWN:
record.msg = '\n\n\n%s\n' % record.msg
self.emitSynchronized(record)
else:
self.scrollLock.acquire()
# If scroll is on, interrupt scroll
if ScrollableHandler.scrollFlag:
self.scrollHeader(record)
else:
# otherwise if scroll isn't on, just log the message normally
self.emitSynchronized(record)
self.scrollLock.release()
return rv
def emitSynchronized(self, record):
""" Write a log message atomically. Normal python logging Handler behavior """
self.acquire()
try:
self.emit(record)
finally:
self.release()
def scrollHeader(self, record):
""" Print a log message so that the user can see it during a SCROLL """
msg = self.format(record).rstrip() # Scroller appends newline for us
from twisted.internet import reactor
if inMainThread():
# FIXME: scrollBegin() should really be creating the scroller instance
# FIXME: no unicode crap from normal python log emit
Hellanzb.scroller.scrollHeader(msg)
else:
reactor.callFromThread(Hellanzb.scroller.scrollHeader, msg)
def exit(self):
global command_output_thread
print("Exiting thread {0}...".format(self))
lock = RLock()
lock.acquire()
self.EXIT_FLAG = True
pgid = self.get_pgid(True)
if pgid is not None:
call(["ionice", "-c", "0", "-P", str(self.get_pgid(True))])
lock.release()
def _threadCallback(self,thread):
"""
A callback function which gets called when a code thread terminates.
"""
lock = RLock()
lock.acquire()
if thread.failed():
self._exceptions[thread._id] = thread.exceptionInfo()
self._tracebacks[thread._id] = thread.tracebackInfo()
lock.release()
class Chopstick:
def __init__(self, name):
self.lock = RLock()
self.name = name
def pickUp(self):
return self.lock.acquire(blocking=False)
def putDown(self):
self.lock.release()
class MonitorLock(object): def __init__(self): self.lock = RLock() def acquire(self, flag=True): self.lock.acquire() def release(self): self.lock.release() def MONITOR_ENTER(self): self.acquire() def MONITOR_EXIT(self): self.release()
class ThreadSafeDict(dict) :
def __init__(self, * p_arg, ** n_arg) :
dict.__init__(self, * p_arg, ** n_arg)
self._lock = RLock()
def __enter__(self) :
self._lock.acquire()
return self
def __exit__(self, type, value, traceback) :
self._lock.release()
class SerialWriteHandler:
def __init__(self, ser, incomingDataHandler, input):
self._incomingDataHandler = incomingDataHandler
self._input = input
self._serial = ser
self._lock = RLock()
def write(self, data):
self._lock.acquire()
self._serial.write(chr(HEADER_START))
self._serial.write(str(data))
self._lock.release()
def writeAndWaitForAck(self, data, idToAck):
self._lock.acquire()
resend = True
while resend:
self.write(data)
ack = self.waitForACK()
if ack != None and ack.getIdToAck() == idToAck and ack.getReqLen() == len(data):
resend = False
self._lock.release()
def waitForACK(self):
gotHeaderStart = False
incomingLength = 0
headerId = 0
data = []
timeoutCount = 3
try:
while timeoutCount > 0:
if gotHeaderStart:
if len(data) < 1:
data.append(self._input.read())
incomingLength, headerId = self._incomingDataHandler.getIncomingHeaderSizeAndId(data)
elif incomingLength >= 1 and headerId == ACK_RES:
for i in range(1, incomingLength):
data.append(self._input.read())
ack = ACKResponse()
ack.buildRequest(data)
if ack.checkPackage():
return ack
data = []
timeoutCount -= 1
gotHeaderStart = False
else:
data = []
timeoutCount -= 1
gotHeaderStart = False
elif ord(self._input.read()) == HEADER_START:
gotHeaderStart = True
except TypeError:
rospy.logerr('ACK have not been send ,retransmitting.......')
return None
class Client(KRPC):
def __init__(self, table):
self.table = table
self.lock = RLock()
timer(KRPC_TIMEOUT, self.timeout)
KRPC.__init__(self)
def find_node(self, address, nid=None):
nid = self.get_neighbor(nid) if nid else self.table.nid
tid = entropy(TID_LENGTH)
msg = {
"t": tid,
"y": "q",
"q": "find_node",
"a": {"id": nid, "target": random_id()}
}
self.send_krpc(msg, address)
def bootstrap(self):
for address in BOOTSTRAP_NODES:
self.find_node(address)
def timeout(self):
if not self.join_successed:
self.bootstrap()
timer(KRPC_TIMEOUT, self.timeout)
def run(self):
self.bootstrap()
while 1:
time.sleep(.001)
try:
data, address = self.ufd.recvfrom(65536)
msg = bdecode(data)
self.types[msg["y"]](msg, address)
except Exception:
pass
def foreverloop(self):
self.start()
while 1:
time.sleep(.001)
if not self.table.nodes:
self.join_successed = False
time.sleep(1)
continue
for node in self.table.nodes:
self.find_node((node.ip, node.port), node.nid)
self.lock.acquire()
self.table.nodes = []
self.lock.release()
class RateManager:
def __init__(self):
self.lock = RLock()
self.statusmap = {}
self.currenttotal = {}
self.dset = Set()
self.clear_downloadstates()
def add_downloadstate(self, ds):
if DEBUG:
print >> sys.stderr, 'RateManager: add_downloadstate', `(ds.get_download().get_def().get_infohash())`
self.lock.acquire()
try:
d = ds.get_download()
if d not in self.dset:
self.statusmap[ds.get_status()].append(ds)
for dir in [UPLOAD, DOWNLOAD]:
self.currenttotal[dir] += ds.get_current_speed(dir)
self.dset.add(d)
return len(self.dset)
finally:
self.lock.release()
def add_downloadstatelist(self, dslist):
for ds in dslist:
self.add_downloadstate(ds)
def adjust_speeds(self):
self.lock.acquire()
try:
self.calc_and_set_speed_limits(DOWNLOAD)
self.calc_and_set_speed_limits(UPLOAD)
self.clear_downloadstates()
finally:
self.lock.release()
def clear_downloadstates(self):
self.statusmap[DLSTATUS_ALLOCATING_DISKSPACE] = []
self.statusmap[DLSTATUS_WAITING4HASHCHECK] = []
self.statusmap[DLSTATUS_HASHCHECKING] = []
self.statusmap[DLSTATUS_DOWNLOADING] = []
self.statusmap[DLSTATUS_SEEDING] = []
self.statusmap[DLSTATUS_STOPPED] = []
self.statusmap[DLSTATUS_STOPPED_ON_ERROR] = []
self.statusmap[DLSTATUS_REPEXING] = []
for dir in [UPLOAD, DOWNLOAD]:
self.currenttotal[dir] = 0
self.dset.clear()
def calc_and_set_speed_limits(self, direct):
pass
def wrapper(self, *args, **kwargs):
retval = getattr(self, attr, None)
if retval is None:
retval = fn(self, *args, **kwargs)
_lock = RLock()
try:
_lock.acquire()
setattr(self, attr, retval)
finally:
_lock.release()
return retval
class SerialTalks:
def __init__(self, port):
# Serial things
self.port = port
self.is_connected = False
# Threading things
self.queues_dict = dict()
self.queues_lock = RLock()
self.history = list()
self.history_lock = RLock()
self.alias_retcode = dict()
self.last_retcode = -1
# Instructions
self.instructions = dict()
self.instructions[RESEND_OPCODE] = self.resend
def __enter__(self):
self.connect()
return self
def __exit__(self, exc_type, exc_value, traceback):
self.disconnect()
def connect(self, timeout=5):
if self.is_connected:
raise AlreadyConnectedError('{} is already connected'.format(
self.port))
# Connect to the serial port
try:
self.stream = serial.Serial(self.port,
baudrate=BAUDRATE,
bytesize=serial.EIGHTBITS,
parity=serial.PARITY_NONE,
stopbits=serial.STOPBITS_ONE)
self.stream.timeout = 1
except SerialException as e:
raise ConnectionFailedError(str(e)) from None
self.serial_buffer = SerialBuffer(self.stream.write, inf)
# Try to bind FREE_BUFFER funct
try:
self.bind(FREE_BUFFER, self.serial_buffer.reset)
except KeyError: # If it's not the first connect executed
pass
# Create a listening thread that will wait for inputs
self.listener = SerialListener(self)
self.listener.start()
# Wait until the Arduino is operational
startingtime = time.monotonic()
while not self.is_connected:
try:
self.execute(PING_OPCODE, timeout=0.1)
self.reset_queues()
self.serial_buffer.buffer_size = self.execute(
GETBUFFERSIZE_OPCODE, timeout=0.5).read(INT)
self.is_connected = True
except TimeoutError:
if time.monotonic() - startingtime > timeout:
self.disconnect()
raise MuteError(
'\'{}\' is mute. It may not be an Arduino or it\'s sketch may not be correctly loaded.'
.format(self.stream.port)) from None
else:
continue
except NotConnectedError:
self.is_connected = False
def disconnect(self):
try:
self.send(DISCONNECT_OPCODE)
except NotConnectedError:
pass
# Stop the listening thread
if hasattr(self, 'listener') and self.listener.is_alive():
self.listener.stop.set()
if self.listener is not current_thread():
self.listener.join()
# Close the serial port
if hasattr(self, 'stream') and self.stream.is_open:
self.stream.close()
# Unset the connected flag
self.is_connected = False
def bind(self, opcode, instruction):
if not opcode in self.instructions:
self.instructions[opcode] = instruction
else:
raise KeyError(
'opcode {} is already bound to another instruction'.format(
opcode))
def rawsend(self, rawbytes):
try:
if hasattr(self, 'stream') and self.stream.is_open:
sentbytes = self.serial_buffer.send(rawbytes)
return sentbytes
except SerialException:
pass
raise NotConnectedError('\'{}\' is not connected.'.format(
self.port)) from None
def send(self, opcode, *args):
retcode = random.randint(0, 0xFFFFFFFF)
content = BYTE(opcode) + ULONG(retcode) + bytes().join(args)
# crc calculation
crc = CRCprocessBuffer(content)
prefix = MASTER_BYTE + BYTE(len(content)) + USHORT(crc)
self.history_lock.acquire()
self.history.append((self.last_retcode, retcode, [opcode, args]))
self.last_retcode = retcode
if len(self.history) > 20:
_ = self.history.pop(0)
self.history_lock.release()
self.rawsend(prefix + content + BYTE(0))
return retcode
def get_queue(self, retcode):
self.queues_lock.acquire()
if retcode in self.alias_retcode.keys():
retcode = self.alias_retcode[retcode]
try:
queue = self.queues_dict[retcode]
except KeyError:
queue = self.queues_dict[retcode] = Queue()
finally:
self.queues_lock.release()
return queue
def delete_queue(self, retcode):
self.queues_lock.acquire()
try:
del self.queues_dict[retcode]
finally:
self.queues_lock.release()
def reset_queues(self):
self.queues_lock.acquire()
self.queues_dict = dict()
self.queues_lock.release()
def process(self, message):
retcode = message.read(ULONG)
queue = self.get_queue(retcode)
queue.put(message)
def poll(self, retcode, timeout=0):
queue = self.get_queue(retcode)
block = (timeout is None or timeout > 0)
try:
output = queue.get(block, timeout)
except Empty:
if timeout is not None:
raise TimeoutError('timeout exceeded') from None
else:
return None
if queue.qsize() == 0:
self.delete_queue(retcode)
return output
def flush(self, retcode):
while self.poll(retcode) is not None:
pass
def execute(self, opcode, *args, timeout=5):
retcode = self.send(opcode, *args)
output = self.poll(retcode, timeout)
return output
def receive(self, input):
opcode = input.read(BYTE)
retcode = input.read(ULONG)
try:
output = self.instructions[opcode](input)
if output is None: return
content = ULONG(retcode) + output
prefix = SLAVE_BYTE + BYTE(len(content))
self.rawsend(prefix + content)
except KeyError:
pass
def getuuid(self, timeout=5):
output = self.execute(GETUUID_OPCODE, timeout=timeout)
return output.read(STRING)
def setuuid(self, uuid):
return self.send(SETUUID_OPCODE, STRING(uuid))
def getlog(self, retcode, timeout=0):
log = str()
while True:
try:
output = self.poll(retcode, 0)
log += output.read(STRING)
except TimeoutError:
break
if timeout > 0:
try:
output = self.poll(retcode, timeout)
log += output.read(STRING)
except TimeoutError:
pass
return log
def save_eeprom(self, file=None, size=1024):
binary_file = open(file, mode='w+b')
for i in range(size):
output = self.execute(GETEEPROM_OPCODE, INT(i))
byte = output.read(BYTE)
binary_file.write(bytes([byte]))
binary_file.close()
def load_eeprom(self, file=None):
binary_file = open(file, mode='r+b')
k = 0
for byte in binary_file.read():
self.send(SETEEPROM_OPCODE, INT(k), BYTE(byte))
k += 1
binary_file.close()
def resend(self, message):
warnings.warn("Message send corrupted !", SerialTalksWarning)
prev_retcode = message.read(ULONG)
to_send, old_retcode = None, None
self.history_lock.acquire()
for i in range(len(self.history)):
if self.history[i][0] == prev_retcode:
to_send = self.history[i][2]
main_retcode = self.history[i][1]
print("Message resend !")
break
self.history_lock.release()
if not to_send is None:
self.queues_lock.acquire()
new_retcode = self.send(to_send[0], *to_send[1])
while main_retcode in self.alias_retcode.keys():
main_retcode = self.alias_retcode[main_retcode]
self.alias_retcode[new_retcode] = main_retcode
self.queues_lock.release()
def re_receive(self, main_retcode):
to_send = None
self.history_lock.acquire()
for i in range(len(self.history)):
if self.history[i][1] == main_retcode:
to_send = self.history[i][2]
break
self.history_lock.release()
if not to_send is None:
self.queues_lock.acquire()
new_retcode = self.send(to_send[0], *to_send[1])
while main_retcode in self.alias_retcode.keys():
main_retcode = self.alias_retcode[main_retcode]
self.alias_retcode[new_retcode] = main_retcode
self.queues_lock.release()
def getout(self, timeout=0):
return self.getlog(STDOUT_RETCODE, timeout)
def geterr(self, timeout=0):
return self.getlog(STDERR_RETCODE, timeout)
class Monitor(Thread):
def __init__(self,
req,
proxy,
logger,
task,
exit_check=None,
ignored_errors=[]):
Thread.__init__(self, name="monitor%s" % task.guid)
Thread.setDaemon(self, True)
# the count of votes per error code
self.vote_result = {}
# the error code to be ignored
self.vote_cleared = set().union(ignored_errors)
self.thread_last_seen = {}
self.dctlock = RLock()
self.votelock = RLock()
self.thread_ref = {}
self.thread_zombie = set()
# HttpReq instance
self.req = req
# proxy.Pool instance
self.proxy = proxy
self.logger = logger
self.task = task
self._exit = exit_check if exit_check else lambda x: False
self._cleaning_up = False
if os.name == "nt":
self.set_title = lambda s: os.system("TITLE %s" %
(s if PY3K else s.encode(
CODEPAGE, 'replace')))
elif os.name == 'posix':
import sys
self.set_title = lambda s: sys.stdout.write("\033]2;%s\007" % (
s if PY3K else s.encode(CODEPAGE, 'replace')))
def set_vote_ns(self, tnames):
t = time.time()
self.thread_last_seen = {k: t for k in tnames}
def vote(self, tname, code):
# thread_id, result_code
self.votelock.acquire()
if code != ERR_NO_ERROR:
self.logger.verbose("t-%s vote:%s" % (tname, code))
if code not in self.vote_result:
self.vote_result[code] = 1
else:
self.vote_result[code] += 1
self.votelock.release()
def wrk_keepalive(self, wrk_thread, _exit=False):
tname = wrk_thread.name
if tname in self.thread_zombie:
self.thread_zombie.remove(tname)
# all image downloaded
# task is finished or failed
# monitor is exiting or worker notify its exit
_ = self.task.meta['finished'] == self.task.meta['total'] or \
self.task.state in (TASK_STATE_FINISHED, TASK_STATE_FAILED) or \
self._exit("mon") or _exit
# self.logger.verbose("mon#%s %s ask, %s, %s" % (self.task.guid, tname, _,
# self.thread_last_seen))
if _ or not wrk_thread.is_alive():
self.dctlock.acquire()
if tname in self.thread_last_seen:
del self.thread_last_seen[tname]
if tname in self.thread_ref:
del self.thread_ref[tname]
self.dctlock.release()
else:
self.thread_last_seen[tname] = time.time()
if tname not in self.thread_ref:
self.thread_ref[tname] = wrk_thread
return _
# def _rescan_pages(self):
# # not using
# # throw away existing page urls
# while True:
# try:
# self.task.page_q.get(False)
# except Empty:
# break
# # put page into task.list_q
# [self.task.list_q.put("%s/?p=%d" % (self.task.url, x)
# for x in range(1, 1 + int(math.ceil(self.task.meta['total']/20.0))))
# ]
# print(self.task.list_q.qsize())
def _check_vote(self):
if False and ERR_IMAGE_RESAMPLED in self.vote_result and ERR_IMAGE_RESAMPLED not in self.vote_cleared:
self.logger.warning(i18n.TASK_START_PAGE_RESCAN % self.task.guid)
self._rescan_pages()
self.task.meta['has_ori'] = True
self.vote_cleared.add(ERR_IMAGE_RESAMPLED)
elif ERR_QUOTA_EXCEEDED in self.vote_result and \
ERR_QUOTA_EXCEEDED not in self.vote_cleared and \
self.vote_result[ERR_QUOTA_EXCEEDED] >= len(self.thread_last_seen):
self.logger.error(i18n.TASK_STOP_QUOTA_EXCEEDED % self.task.guid)
self.task.state = TASK_STATE_FAILED
def run(self):
CHECK_INTERVAL = 10
STUCK_INTERVAL = 90
intv = 0
self.set_title(i18n.TASK_START % self.task.guid)
last_change = time.time()
last_finished = -1
while len(self.thread_last_seen) > 0:
intv += 1
self._check_vote()
for k in list(self.thread_last_seen.keys()):
if time.time() - self.thread_last_seen[k] > 30:
if k in self.thread_ref and self.thread_ref[k].is_alive():
self.logger.warning(i18n.THREAD_MAY_BECOME_ZOMBIE % k)
self.thread_zombie.add(k)
else:
self.logger.warning(i18n.THREAD_SWEEP_OUT % k)
del self.thread_last_seen[k]
if intv == CHECK_INTERVAL:
_ = "%s %dR/%dZ, %s %dR/%dD" % (
i18n.THREAD, len(self.thread_last_seen),
len(self.thread_zombie), i18n.QUEUE,
self.task.img_q.qsize(), self.task.meta['finished'])
self.logger.info(_)
self.set_title(_)
intv = 0
# if not downloading any new images in 1.5 min, exit
if last_finished != self.task.meta['finished']:
last_change = time.time()
last_finished = self.task.meta['finished']
else:
if time.time() - last_change > STUCK_INTERVAL:
self.logger.warning(i18n.TASK_STUCK % self.task.guid)
break
time.sleep(0.5)
if self.task.meta['finished'] == self.task.meta['total']:
_err = self.task.rename_fname()
if _err:
self.logger.warning(
i18n.XEH_RENAME_HAS_ERRORS %
("\n".join(map(lambda x: "%s => %s : %s" % x, _err))))
self.set_title(i18n.TASK_FINISHED % self.task.guid)
self.logger.info(i18n.TASK_FINISHED % self.task.guid)
self.task.state = TASK_STATE_FINISHED
self.task.cleanup()
class OnionOOFactory(object):
# onionoo holds a dict of key:data pairs, with
# key = 'details' or 'bandwidth' or 'weights' + ':' + fingerprint
# data = onionoo.Document object holding the onionoo network response or None
onionoo = {}
executor = None
query_lock = None
nodes_lock = None
# proxy = Proxy('127.0.0.1', 'default')
futures = list()
def __init__(self, proxy):
self.onionoo = {}
self.proxy = proxy
self.executor = ThreadPoolExecutor(max_workers=30) # enough to query > 100 Tors at once...
self.query_lock = RLock()
self.nodes_lock = RLock()
self.hidden_index = 1
self.is_refreshing = False
self.new_nodes = {}
def add(self, fingerprint):
# there are currently three different documents we query from the onionoo db:
# Details, Bandwidth & Weights
# the key identifies the fingerprint as well as the document to allow storage in a flat dict.
check_key = ['details:' + fingerprint, 'bandwidth:' + fingerprint, 'weights:' + fingerprint]
retval = False
# if the key in question isn't in the dict
for key in check_key:
if key not in self.onionoo:
lgr = logging.getLogger('theonionbox')
lgr.debug('Adding fingerprint {} to onionoo query queue.'.format((fingerprint)))
# ... add it (yet without document! This indicates that we have no data so far.)
self.nodes_lock.acquire()
self.new_nodes[key] = Document()
self.nodes_lock.release()
retval = True
return retval
def remove(self, fingerprint):
# to remove keys if demanded (which probably will happen rarely!)
check_key = [ 'details:' + fingerprint, 'bandwidth:' + fingerprint, 'weights:' + fingerprint]
for key in check_key:
if key in self.onionoo:
del self.onionoo[key]
if key in self.new_nodes:
self.nodes_lock.acquire()
del self.new_nodes[key]
self.nodes_lock.release()
def refresh(self, only_keys_with_none_data=False, async_mode=True):
self.is_refreshing = True
lgr = logging.getLogger('theonionbox')
lgr.info('Refreshing onionoo data => Only New: {} | Async: {}'.format(only_keys_with_none_data, async_mode))
self.nodes_lock.acquire()
self.onionoo.update(self.new_nodes)
self.new_nodes = {}
self.nodes_lock.release()
# run through the dict of keys and query onionoo for updated documents
for key in self.onionoo:
item = self.onionoo[key]
if only_keys_with_none_data is True:
if item.has_document() is True:
continue
try:
data_type, fp = key.split(':')
except ValueError:
# This definitely is weird!
continue
# async_mode = True
query_launched = False
if async_mode is True:
try:
self.executor.submit(self.query, item, fp, data_type)
query_launched = True
except:
# In case of error, we try to continue in sync mode!
lgr.warning("Onionoo: Failed to launch thread to query for Tor network data.")
pass
if query_launched is False:
try:
self.query(item, fp, data_type)
except:
# Ok. We silently swallow this...
pass
# restart if meanwhile the landscape changed
if len(self.new_nodes) > 0:
self.refresh(True)
self.is_refreshing = False
def query(self, for_document, fingerprint, data_type):
lgr = logging.getLogger('theonionbox')
# https://trac.torproject.org/projects/tor/ticket/6320
hash = sha1(a2b_hex(fingerprint)).hexdigest()
payload = {'lookup': hash}
headers = {'accept-encoding': 'gzip'}
if len(for_document.ifModSince) > 0:
headers['if-modified-since'] = for_document.ifModSince
proxy_address = self.proxy.address()
if proxy_address is None:
proxies = {}
query_base = ONIONOO_OPEN
else:
proxies = {
'http': 'socks5h://' + proxy_address,
'https': 'socks5h://' + proxy_address
}
query_base = ONIONOO_HIDDEN[self.hidden_index]
query_address = query_base + '/' + data_type
r = None
# even when querying async, there's just one query performed at a time
# self.query_lock.acquire()
lgr.debug("Onionoo: Launching query of '{}' for ${}.".format(query_address, fingerprint))
try:
r = requests.get(query_address, params=payload, headers=headers, proxies=proxies, timeout=10)
except requests.exceptions.ConnectTimeout:
lgr.info("Onionoo: Failed querying '{}' due to connection timeout. Switching to alternative service."
.format(query_base))
# this is quite manual ... but asserts the right result
base_index = ONIONOO_HIDDEN.index(query_base)
self.hidden_index = (base_index + 1) % len(ONIONOO_HIDDEN)
# TODO: shall we restart the failed query here?
except Exception as exc:
lgr.warning("Onionoo: Failed querying '{}' -> {}".format(query_address, exc))
else:
lgr.debug("Onionoo: Finished querying '{}' for ${} with status code {}: {} chars received."
.format(query_address, fingerprint, r.status_code, len(r.text)))
# if len(r.text) > 0:
# lgr.debug(("Onionoo: Received {} chars for ${}".format(len(r.text), fingerprint)))
# Ok! Now the next query may be launched...
# self.query_lock.release()
if r is None:
return
if r.status_code != requests.codes.ok:
return
for_document.ifModSince = r.headers['last-modified']
if r.status_code == requests.codes.not_modified:
return
try:
data = r.json()
except Exception as exc:
lgr.debug("Onionoo: Failed to un-json network data; error code says '{}'.".format(exc))
return
for_document.update(data)
# ToDo: Where's the benefit doing it this way?
#
# if data_type == 'details':
# node_details = Details(for_document)
#
# do_refresh = False
#
# #try:
# fams = ['effective_family', 'alleged_family', 'indirect_family']
# for fam in fams:
# fam_data = node_details(fam)
# if fam_data is not None:
# for fp in fam_data:
# if fp[0] is '$':
# do_refresh = self.add(fp[1:]) or do_refresh
# #except:
# # This probably wasn't a 'detail' document!
# # pass
#
# if do_refresh and not self.is_refreshing:
# self.refresh(True)
return
# except Exception as exc:
# lgr.info("Onionoo: Failed to query '{}': {}".format(address, exc))
# pass
#return
def details(self, fingerprint):
if len(fingerprint)> 0 and fingerprint[0] == '$':
fingerprint = fingerprint[1:]
key = 'details:' + fingerprint
if key in self.onionoo:
return Details(self.onionoo[key])
return Details(Document())
def bandwidth(self, fingerprint):
if len(fingerprint)> 0 and fingerprint[0] == '$':
fingerprint = fingerprint[1:]
key = 'bandwidth:' + fingerprint
if key in self.onionoo:
return Bandwidth(self.onionoo[key])
return Bandwidth(Document())
def weights(self, fingerprint):
if len(fingerprint)> 0 and fingerprint[0] == '$':
fingerprint = fingerprint[1:]
key = 'weights:' + fingerprint
if key in self.onionoo:
return Weights(self.onionoo[key])
return Weights(Document())
def shutdown(self):
self.executor.shutdown(True)
def nickname2fingerprint(self, nickname):
if nickname[0] == '#':
nickname = nickname[1:]
data = self.search(nickname) or {}
if 'relays' in data:
for relay in data['relays']:
if 'n' in relay and 'f' in relay:
if nickname == relay['n']:
return relay['f']
if 'bridges' in data:
for bridge in data['bridges']:
if 'n' in bridge and 'h' in bridge:
if nickname == bridge['n']:
return bridge['h']
return None
def search(self, search_string, limit=None, offset=None):
lgr = logging.getLogger('theonionbox')
payload = {'search': search_string}
if limit and limit > 0:
payload['limit'] = limit
if offset and offset > 0:
payload['offset'] = offset
headers = {'accept-encoding': 'gzip'}
proxy_address = self.proxy.address()
if proxy_address is None:
proxies = {}
query_base = ONIONOO_OPEN
else:
proxies = {
'http': 'socks5h://' + proxy_address,
'https': 'socks5h://' + proxy_address
}
query_base = ONIONOO_HIDDEN[self.hidden_index]
query_address = query_base + '/summary'
r = None
try:
r = requests.get(query_address, params=payload, headers=headers, proxies=proxies, timeout=10)
except Exception as exc:
lgr.debug("Onionoo: Failed querying '{}' -> {}".format(query_address, exc))
if r is None:
return
if r.status_code != requests.codes.ok:
return
try:
data = r.json()
except Exception as exc:
lgr.debug("Onionoo: Failed to un-json network data; error code says '{}'.".format(exc))
return
return data
class TrackingJobMaster(JobMaster):
"""
This class extends JobMaster with the following extras:
- reporting of the average time each slave spends on a job
- automatic adding of slave computers to PVM
- different ways to be notified of a completed calculation
- restarting of interrupted calculations
The calculation is performed non-blocking in a thread after a call
to master.start().
The end of calculation is signalled on master.lock / master.lockMsg.
The result can then be obtained with getResult().
Alternatively, a callback method can be registered that is called
after the calculation finished (master.setCallback()).
The perhaps easiest (but also least flexible) way is to instead use the
calculateResult() method. This starts the calculation and blocks execution
until the result is returned.
Consider overriding cleanup(), done() and getResult().
An interrupted calculation can be restarted from a restart file:
- during calculation, pickle the result of getRst() to a file
- call the script Biskit/restartPVM -i |file_name|
Manual restart is possible as follows:
1. pickle master.data, master.result, master.status.objects
2. master.exit() / Exception / kill, etc.
3. initialize master with same parameters as before
4. unpickle and re-assign master.data, master.result,
master.status.objects
5. master.start()
@note: The master sends out an exit signal to all slaves but doesn't
wait for a response (there isn't any) and continues in the finish()
method. Since, at the end, the same job is distributed to several slaves,
some of them might still be running when cleanup() or done() are
executed. The slave script must tolerate errors that, e.g., happen
if cleanup() is called while it is running.
@todo: try finding some solution to the problem where the master
sends out an exit signal to all slaves but doesn't wait for
a response (see note)
@todo: test restart function
@todo: restart data are not automatically saved (e.g. in intervals)
"""
def __init__(self,
data={},
chunk_size=5,
hosts=[],
niceness={'default': 20},
slave_script='',
verbose=1,
show_output=0,
add_hosts=1,
redistribute=1):
"""
@param data: dict of items to be processed
@type data: {str_id:any}
@param chunk_size: number of items that are processed per job
@type chunk_size: int
@param hosts: list of host-names
@type hosts: [str]
@param niceness: host niceness dictionary {str_host-name: int_niceness}
@type niceness: {str:int}
@param slave_script: absolute path to slave-script
@type slave_script: str
@param verbose: verbosity level (default: 1)
@type verbose: 1|0
@param show_output: display one xterm per slave (default: 0)
@type show_output: 1|0
@param add_hosts: add hosts to PVM before starting (default: 1)
@type add_hosts: 1|0
@param redistribute: at the end, send same job out several times
(default: 1)
@type redistribute: 1|0
"""
if add_hosts:
if verbose: T.errWrite('adding %i hosts to pvm...' % len(hosts))
pvm.addHosts(hosts=hosts)
if verbose: T.errWriteln('done')
JobMaster.__init__(self,
data,
chunk_size,
hosts,
niceness,
slave_script,
show_output=show_output,
redistribute=redistribute,
verbose=verbose)
self.progress = {}
self.disabled_hosts = []
self.slow_hosts = {}
self.verbose = verbose
## end of calculation is signalled on lockMsg
self.lock = RLock()
self.lockMsg = Condition(self.lock)
## this method is called when everything is calculated
self.call_done = None
def hostnameFromTID(self, slave_tid):
"""
Get nickname of host from TaskID.
@param slave_tid: slave task tid
@type slave_tid: int
"""
nickname = self.nicknameFromTID(slave_tid)
return nickname.split('_')[0]
def is_valid_slave(self, slave_tid):
"""
Override JobMaster method to disable slow nodes on the fly
@param slave_tid: slave task tid
@type slave_tid: int
"""
return self.hostnameFromTID(slave_tid) not in self.disabled_hosts
def mark_slow_slaves(self, host_list, slow_factor):
"""
@param host_list: list of hosts
@type host_list: [str]
@param slow_factor: factor describing the calculation speed of a node
@type slow_factor: float
"""
for h in host_list:
self.slow_hosts[h] = slow_factor
def start_job(self, slave_tid):
"""
Overriding JobMaster method
@param slave_tid: slave task tid
@type slave_tid: int
"""
host = self.nicknameFromTID(slave_tid)
d = {'given': 0, 'done': 0, 'time': 0}
if self.progress.has_key(host):
d = self.progress[host]
d['given'] += 1
d['timeStart'] = time.time()
self.progress[host] = d
def job_done(self, slave_tid, result):
"""
Overriding JobMaster method
@param slave_tid: slave task tid
@type slave_tid: int
@param result: slave result dictionary
@type result: dict
"""
host = self.nicknameFromTID(slave_tid)
self.progress[host]['done'] += 1
self.progress[host]['time'] = time.time() \
- self.progress[host]['timeStart']
def reportProgress(self):
"""
Report how many jobs were processed in what time per host.
"""
if self.verbose:
print 'host \tgiven\tdone\t time'
for host in self.progress:
d = self.progress[host]
print '%-25s\t%i\t%i\t%6.2f s' %\
(host, d['given'], d['done'], d['time'])
def setCallback(self, funct):
"""
Register function to be called after calculation is finished.
@param funct: will be called with an instance of the master
as single argument
@type funct: function
"""
self.call_done = funct
def cleanup(self):
"""
Called after exit. Override.
"""
pass
def done(self):
"""
Called by finish() after exit(), cleanup(), and reportProgress(), but
before thread notification (notifyAll() ) and before executing
the callBack method. Override.
"""
pass
def notifyAll(self):
"""
Notify thread waiting on self.lockMsg that master has finished.
"""
self.lock.acquire()
self.lockMsg.notifyAll()
self.lock.release()
def finish(self):
"""
Called one time, after last job result has been received. It should
not be necessary to override this further. Override done() instead.
"""
self.exit()
self.cleanup()
self.reportProgress()
self.done()
self.notifyAll()
if self.call_done:
self.call_done(self)
def getResult(self, **arg):
"""
Return result dict, if it is available.
Override to return something else - which will also be the return value
of calculateResult().
@param arg: keyword-value pairs, for subclass implementations
@type arg: {key:value}
@return: {any:any}
@rtype: {any:any}
"""
return self.result
def calculateResult(self, **arg):
"""
Convenience function that is starting the parallel calculation and
blocks execution until it is finished.
@param arg: keyword-value pairs, for subclass implementations
@type arg: {key:value}
@return: array( (n_frames, n_frames), 'f'), matrix of pairwise rms
@rtype: array
"""
self.start()
self.lock.acquire()
self.lockMsg.wait()
self.lock.release()
return self.getResult(**arg)
def getRst(self):
"""
Get data necessary for a restart of the running calculation.
Locks, file handles and private data are *NOT* saved.
Override if necessary but call this method in child method.
@return: {..}, dict with 'pickleable' fields of master
@rtype: dict
"""
self.status.lock.acquire()
## collect master parameters that can be pickled
rst = {}
for k, v in self.__dict__.items():
skip = 0
for t in [Thread, _RLock, _Condition, Status, file]:
if isinstance(v, t):
skip = 1
if str(k)[0] == '_':
skip = 1
if not skip:
rst[k] = copy.copy(v)
rst['status_objects'] = copy.deepcopy(self.status.objects)
rst['master_class'] = self.__class__
self.status.lock.release()
return rst
def saveRst(self, fname):
"""
Pickle data necessary for a restart of the running calculation.
@param fname: file name
@type fname: str
"""
T.dump(self.getRst(), fname)
def setRst(self, rst_data):
"""
Prepare this master for restart, called by restart().
Override if necessary but call in child.
@param rst_data: {..}, parameters for master.__dict__ + some
special fields
@type rst_data: dict
@return: {..}, parameters for master.__dict__ without special fields
@rtype: dict
"""
self.__class__ = rst_data['master_class']
self.status.objects = rst_data['status_objects']
del rst_data['master_class']
del rst_data['status_objects']
return rst_data
class RecentlyUsedContainer(dict):
"""
Provides a dict-like that maintains up to ``maxsize`` keys while throwing
away the least-recently-used keys beyond ``maxsize``.
"""
# If len(self.access_log) exceeds self._maxsize * CLEANUP_FACTOR, then we
# will attempt to cleanup the invalidated entries in the access_log
# datastructure during the next 'get' operation.
CLEANUP_FACTOR = 10
def __init__(self, maxsize=10):
self._maxsize = maxsize
# We use a deque to to store our keys ordered by the last access.
self.access_log = deque()
self.access_log_lock = RLock()
# We look up the access log entry by the key to invalidate it so we can
# insert a new authorative entry at the head without having to dig and
# find the old entry for removal immediately.
self.access_lookup = {}
# Trigger a heap cleanup when we get past this size
self.access_log_limit = maxsize * self.CLEANUP_FACTOR
def _invalidate_entry(self, key):
"If exists: Invalidate old entry and return it."
old_entry = self.access_lookup.get(key)
if old_entry:
old_entry.is_valid = False
return old_entry
def _push_entry(self, key):
"Push entry onto our access log, invalidate the old entry if exists."
self._invalidate_entry(key)
new_entry = AccessEntry(key)
self.access_lookup[key] = new_entry
self.access_log_lock.acquire()
self.access_log.appendleft(new_entry)
self.access_log_lock.release()
def _prune_entries(self, num):
"Pop entries from our access log until we popped ``num`` valid ones."
while num > 0:
self.access_log_lock.acquire()
p = self.access_log.pop()
self.access_log_lock.release()
if not p.is_valid:
continue # Invalidated entry, skip
dict.pop(self, p.key, None)
self.access_lookup.pop(p.key, None)
num -= 1
def _prune_invalidated_entries(self):
"Rebuild our access_log without the invalidated entries."
self.access_log_lock.acquire()
self.access_log = deque(e for e in self.access_log if e.is_valid)
self.access_log_lock.release()
def _get_ordered_access_keys(self):
"Return ordered access keys for inspection. Used for testing."
self.access_log_lock.acquire()
r = [e.key for e in self.access_log if e.is_valid]
self.access_log_lock.release()
return r
def __getitem__(self, key):
item = dict.get(self, key)
if not item:
raise KeyError(key)
# Insert new entry with new high priority, also implicitly invalidates
# the old entry.
self._push_entry(key)
if len(self.access_log) > self.access_log_limit:
# Heap is getting too big, try to clean up any tailing invalidated
# entries.
self._prune_invalidated_entries()
return item
def __setitem__(self, key, item):
# Add item to our container and access log
dict.__setitem__(self, key, item)
self._push_entry(key)
# Discard invalid and excess entries
self._prune_entries(len(self) - self._maxsize)
def __delitem__(self, key):
self._invalidate_entry(key)
self.access_lookup.pop(key, None)
dict.__delitem__(self, key)
def get(self, key, default=None):
try:
return self[key]
except KeyError:
return default
class MCU:
def __init__(self, port):
self.port = port
self.Serial = serial.Serial()
self.lock = RLock()
self.__open_serial()
self.active = Event()
self.error = Event()
# Start
#self.heartbeat()
def heartbeat(self):
def run(self):
i = 0
while i < 5:
i = i + 1
msg = MSG(msgType=MSG.TYPE.HEARTBEAT)
resp = self.__send(msg)
if resp != None and resp.type() == MSG.TYPE.HEARTBEAT:
self.active.set()
i = 0
else:
self.active.clear()
if resp.id != MSG.INFO.SUCCESS:
self.error.set()
else:
self.error.clear()
sleep(5)
print('5 heartbeats missed: resetting')
self.reset()
self.heartbeat()
t = Thread(target=run, args=(self, ))
t.daemon = True
t.start()
def __open_serial(self, baudrate=115200, timeout=0, write_timeout=0):
self.lock.acquire()
if not self.Serial.is_open:
self.Serial.port = self.port
self.Serial.baudrate = baudrate
self.Serial.timeout = timeout
self.Serial.write_timeout = write_timeout
self.Serial.open()
sleep(2)
self.lock.release()
def __close_serial(self):
self.lock.acquire()
if self.Serial.is_open:
self.Serial.close()
self.lock.release()
def reset(self):
self.__close_serial()
sleep(0.1)
self.__open_serial()
def __send(self, msg, timeout=2):
assert type(msg) == MSG
with self.lock:
#print("Sending: {}".format(msg))
self.Serial.write(msg)
start_time = time()
while self.Serial.in_waiting < MSG.SIZE:
if time() - start_time > timeout:
print("Response timed out")
print("Buffer: {}".format(
self.Serial.read(size=self.Serial.in_waiting)))
return None #timeout condition
sleep(0.1)
resp = self.Serial.read(size=self.Serial.in_waiting)
resp = MSG(resp)
print("Received: {}".format(resp))
return resp
def set(self, id, value):
msg = MSG(msgType=MSG.TYPE.SET, id=id, data=value)
#print(msg)
resp = self.__send(msg)
assert resp.type() == MSG.TYPE.SET
return resp
def get(self, id, type):
msg = MSG(msgType=MSG.TYPE.GET, id=id, data=type)
#print(msg)
resp = self.__send(msg)
assert resp.type() == MSG.TYPE.GET
return resp
def status(self, id):
msg = MSG(msgType=MSG.TYPE.GET, id=id, data=MSG.INFO.STATUS)
resp = self.__send(msg)
if resp != None:
latched = int.from_bytes(resp.data()[0:2], 'big')
nonLatched = int.from_bytes(resp.data()[2:4], 'big')
latched = MSG.latchedStatusFlag[latched]
nonLatched = MSG.nonLatchedStatusFlag[nonLatched]
errorFlag = resp.type()
return errorFlag, latched, nonLatched
class ConfigParser(object):
"""
Configuration info provider
"""
_list_parms = {
"single_build_flags": BuildFlagScope.single,
"global_build_flags": BuildFlagScope.all,
"dependencies_build_flags": BuildFlagScope.dependencies,
}
_single_value_parms = ("builder", )
_deprecated_parameters = ("target_dir", )
_logger = logging.getLogger(__name__ + ".ConfigParser")
def __init__(self, filename): # type: (Path) -> None
self._logger.debug("Creating config parser for filename '%s'",
filename)
self._parms = {"builder": None} # type: Dict[str, Union[str, None]]
self._flags = {
FileType.vhdl: {
BuildFlagScope.single: (),
BuildFlagScope.all: (),
BuildFlagScope.dependencies: (),
},
FileType.verilog: {
BuildFlagScope.single: (),
BuildFlagScope.all: (),
BuildFlagScope.dependencies: (),
},
FileType.systemverilog: {
BuildFlagScope.single: (),
BuildFlagScope.all: (),
BuildFlagScope.dependencies: (),
},
} # type: Dict[FileType, Dict[BuildFlagScope, BuildFlags] ]
self._sources = [] # type: List[Tuple[str, str, BuildFlags]]
self.filename = filename
self._timestamp = 0.0
self._parse_lock = RLock()
def _shouldParse(self): # type: () -> bool
"""
Checks if we should parse the configuration file
"""
return self.filename.mtime > self._timestamp
def _updateTimestamp(self):
# type: (...) -> Any
"""
Updates our timestamp with the configuration file
"""
self._timestamp = self.filename.mtime
def isParsing(self): # type: () -> bool
"Checks if parsing is ongoing in another thread"
locked = not self._parse_lock.acquire(False)
if not locked:
self._parse_lock.release()
return locked
def _parseIfNeeded(self):
# type: () -> None
"""
Locks accesses to parsed attributes and parses the configuration file
"""
with self._parse_lock:
if self._shouldParse():
self._parse()
def _parse(self): # type: () -> None
"""
Parse the configuration file without any previous checking
"""
self._logger.info("Parsing '%s'", self.filename)
self._updateTimestamp()
self._sources = []
for _line in open(self.filename.name, mode="rb").readlines():
line = _replaceCfgComments("", _line.decode(errors="ignore"))
self._parseLine(line)
def _parseLine(self, line): # type: (str) -> None
"""
Parses a line a calls the appropriate extraction methods
"""
for match in _configFileScan(line):
groupdict = match.groupdict()
self._logger.debug("match: '%s'", groupdict)
if groupdict["parameter"] is not None:
self._handleParsedParameter(groupdict["parameter"],
groupdict["parm_lang"],
groupdict["value"])
else:
for source_path in self._getSourcePaths(groupdict["path"]):
self._sources.append((
source_path,
{
"library": groupdict["library"],
"flags": _extractSet(groupdict["flags"]),
},
))
def _handleParsedParameter(self, parameter, lang, value):
# type: (str, str, str) -> None
"""
Handles a parsed line that sets a parameter
"""
self._logger.debug("Found parameter '%s' for '%s' with value '%s'",
parameter, lang, value)
if parameter in self._deprecated_parameters:
self._logger.debug("Ignoring deprecated parameter '%s'", parameter)
elif parameter in self._single_value_parms:
self._logger.debug("Handling '%s' as a single value", parameter)
self._parms[parameter] = value
elif parameter in self._list_parms:
self._logger.debug("Handling '%s' as a list of values", parameter)
self._flags[FileType(lang)][
self._list_parms[parameter]] = _extractSet(value)
else:
raise exceptions.UnknownParameterError(parameter)
def _getSourcePaths(self, path): # type: (str) -> Iterable[str]
"""
Normalizes and handles absolute/relative paths
"""
source_path = p.normpath(p.expanduser(path))
# If the path to the source file was not absolute, we assume
# it was relative to the config file base path
if not p.isabs(source_path):
fname_base_dir = p.dirname(self.filename.abspath)
source_path = p.join(fname_base_dir, source_path)
return glob(source_path) or [source_path]
def parse(self):
# type: (...) -> Dict[Any, Any]
"""
Parses the file if it hasn't been parsed before or if the config file
has been changed
"""
self._parseIfNeeded()
data = {"sources": self._sources} # type: Dict[Any, Any]
builder_name = self._parms.get("builder", None)
if builder_name is not None:
data["builder"] = builder_name
for filetype, flags in self._flags.items():
flags_dict = {}
for scope in BuildFlagScope:
flags_dict[scope.value] = flags[scope]
data.update({filetype.name: {"flags": flags_dict}})
return data
class SwarmTracker(Nodeable):
''' Object that maintains the status of all known swarm UAVs
Currently, this object maintains the position, velocity, subswarm ID,
swarm state, and waypoint intent for each swarm UAV as it is received
from the network. Data for each element is maintained in a dictionary
object with 1 record for each swarm member.
Class member variables:
ownID: ID (integer) of this particular aircraft
subSwarmID: ID (integer) of the subswarm this vehicle is a part of
swarmState: current swarm state of this vehicle (int)
swarmBehavior: currently active swarm behavior for this vehicle (int)
_possible_crash: set of swarm UAVs suspected of crashing (no reports)
_crash_timeout: max non-reporting time before a UAV is considered crashed
_baseAlt: Altitude from which rel_alt values are calculated for all AC
_swarm: Dictionary of records for individual aircraft in the swarm
_swarmPublisher: Object for publishing swarm state
_swarmMessage: Container for swarm states to be published
_lock: Prevents the callback thread modifications at a bad time
Inherited from Nodeable:
nodeName: name of the node to start or node in which the object is
timer: ROS rate object that controls the timing loop
DBUG_PRINT: set true to force screen debug messages (default FALSE)
WARN_PRINT: set false to force screen warning messages (default FALSE)
Class methods
callbackSetup: sets up subscriptions
publisherSetup: sets up publishers
executeTimedLoop: executes one loop iteration
_updateOwnPose: callback for updated own pose receipt
_updateSwarmPose: callback for updated swarm UAV pose receipt
_updateSwarmControlState: callback for updated swarm control state receipt
_setSubSwarm: updates this UAV's subswarm assignment
_setSwarmState: updates this UAV's swarm state
_setSwarmBehavior: updates this UAV's current swarm behavior
'''
def __init__(self, ownID, subswarm=0, nodeName=NODE_BASENAME):
''' Initializes variables, subscribes to required ROS topics, and
creates required ros publishers. Initializer assumes that the object
is already running within an initialized ROS node (i.e., the object
does not initialize itself as a node). This enables multiple objects
to run within a single node if rqd.
@param ownID: ID (integer) of this particular aircraft
@param nodeName: name of the ROS node for this object
@param subswarm: ID (int) of the "subswarm" to which this vehicle belongs
'''
Nodeable.__init__(self, nodeName)
self.ownID = ownID
self.subSwarmID = subswarm
self.swarmState = 0
self.swarmBehavior = 0
self._baseAlt = 0.0
self._swarm = dict()
self._possible_crash = set()
self._crash_timeout = DEFAULT_CRASH_TIME
self._swarmPublisher = None
self._swarmMessage = SwarmStateStamped()
self._swarmMessage.header.seq = 0
self._swarmMessage.header.frame_id = "base_footprint"
self._lock = RLock()
rospy.set_param('subswarm_id', self.subSwarmID)
# self.DBUG_PRINT = True
# self.INFO_PRINT = True
# self.WARN_PRINT = True
#-------------------------------------------------
# Implementation of parent class virtual functions
#-------------------------------------------------
def callbackSetup(self):
''' Establishes the callbacks for the SwarmTracker object.
@param params: list as follows: [] (no required parameters)
'''
self.createSubscriber("acs_pose", apbrg.Geodometry, \
self._updateOwnPose)
self.createSubscriber("recv_pose", SwarmVehicleState, \
self._updateSwarmPose)
self.createSubscriber("recv_swarm_ctl_state", SwarmControlState, \
self._updateSwarmControlState)
self.createSubscriber("subswarm_id", std_msgs.msg.UInt8, \
self._setSubSwarm)
self.createSubscriber("swarm_state", std_msgs.msg.UInt8, \
self._setSwarmState)
self.createSubscriber("swarm_behavior", std_msgs.msg.UInt8, \
self._setSwarmBehavior)
def publisherSetup(self):
''' Sets up publishers for the SwarmTracker object.
@param params: list as follows: [] (no required parameters)
'''
self._swarmPublisher = \
self.createPublisher("swarm_uav_states", SwarmStateStamped, 1)
def executeTimedLoop(self):
''' Executes one timed-loop iteration for the SwarmTracker object
The loop computes a DR position for the current time for each UAV
in the swarm and publishes a SwarmState message to the ROS topic.
'''
self._lock.acquire()
self._swarmMessage.header.stamp = rospy.Time.now()
self._swarmMessage.crashed_list = list(self._possible_crash)
del self._swarmMessage.swarm[:] # Clear current message contents
vKeys = self._swarm.keys()
for vID in vKeys:
vehicle = self._swarm[vID]
timeDiff = self._swarmMessage.header.stamp - \
vehicle.state.header.stamp
# Half of an Eventually Reliable Crash Detector (ERCD)
# (i.e., if a UAV dies, we'll eventually realize it reliably)
if (vID not in self._possible_crash) and \
(timeDiff > self._crash_timeout):
self._possible_crash.add(vID)
self.log_warn("UAV %d possible crash: no updates for %f secs" \
%(vID, (float(str(self._crash_timeout))/1e9)))
vehicle.computeDRPose(self._swarmMessage.header.stamp)
vehicle._stateMsg.state.header.stamp = vehicle.state.header.stamp
vehicle._stateMsg.state.pose = vehicle.drPose.pose
vehicle._stateMsg.state.twist = vehicle.state.twist
self._swarmMessage.swarm.append(vehicle._stateMsg)
self._swarmPublisher.publish(self._swarmMessage)
self._lock.release()
#-----------------------------------------
# ROS Subscriber callbacks for this object
#-----------------------------------------
def _updateOwnPose(self, stateMsg):
''' Updates swarm info for this UAV when a new pose is published
@param poseMsg: Geodometry object with the new pose
TODO: add covariances when they are added to the msg
'''
try:
if not self.ownID in self._swarm:
try:
self._lock.acquire()
self._swarm[self.ownID] = \
SwarmElement(self.ownID, self.subSwarmID, apbrg.Geodometry())
except Exception as ex:
self.log_warn("Self update callback error: " + str(ex))
finally:
self._lock.release()
element = self._swarm[self.ownID]
element.updateState(stateMsg, self.subSwarmID, self.swarmState, \
self.swarmBehavior)
element.subSwarmID = self.subSwarmID
newBaseAlt = element.state.pose.pose.position.alt - \
element.state.pose.pose.position.rel_alt
if abs(newBaseAlt - self._baseAlt) > 0.001:
self._baseAlt = newBaseAlt
except Exception as ex:
self.log_warn("Self update callback error: " + str(ex))
def _updateSwarmPose(self, poseMsg):
''' Updates swarm info for swarm UAVs when new poses are received
@param poseMsg: Geodometry object with the new pose
TODO: add covariances when they are added to the msg
'''
try:
self._lock.acquire()
poseTime = poseMsg.state.header.stamp
# Update an existing element if it's already in the dictionary
if poseMsg.vehicle_id in self._swarm:
updateElement = self._swarm[poseMsg.vehicle_id]
elTime = updateElement.state.header.stamp
if poseTime < elTime: return # older than latest data
# Half of an Eventually Reliable Crash Detector (ERCD)
# (i.e., if a UAV dies, we'll eventually realize it reliably)
if poseMsg.vehicle_id in self._possible_crash:
t = rospy.Time.now()
if (t - elTime) > self._crash_timeout:
self._crash_timeout = t - elTime
self._possible_crash.remove(poseMsg.vehicle_id)
self.log_info("Received update to possibly crashed UAV %d"\
%poseMsg.vehicle_id)
updateElement.updateState(poseMsg.state, poseMsg.subswarm_id, \
updateElement.swarmState, \
updateElement.swarmBehavior)
updateElement.state.pose.pose.position.rel_alt = \
updateElement.state.pose.pose.position.alt - self._baseAlt
else: # Create & initialize new element if this is the first report
newElement = \
SwarmElement(poseMsg.vehicle_id, poseMsg.subswarm_id, \
poseMsg.state, self._baseAlt)
self._swarm[poseMsg.vehicle_id] = newElement
self.log_dbug("new aircraft id=" + str(poseMsg.vehicle_id) + \
" added to swarm")
element = self._swarm[poseMsg.vehicle_id]
element.subSwarmID = poseMsg.subswarm_id
except Exception as ex:
self.log_warn("Swarm update callback error: " + str(ex))
finally:
self._lock.release()
def _updateSwarmControlState(self, ctlMsg):
''' Updates swarm control state of a swarming aircraft (not this one)
@param ctlMsg: SwarmControlState message with the new control info
'''
try:
self._lock.acquire()
if ctlMsg.vehicle_id in self._swarm:
updateElement = self._swarm[ctlMsg.vehicle_id]
updateElement.swarmState = ctlMsg.swarm_state
updateElement.swarmBehavior = ctlMsg.swarm_behavior
except Exception as ex:
self.log_warn("Swarm control update callback error: " + str(ex))
finally:
self._lock.release()
def _setSubSwarm(self, swarmMsg):
''' Updates the "subswarm" in which this vehicle is participating
@param swarmMsg: message (UInt8) containing the updated swarm ID
'''
self.subSwarmID = swarmMsg.data
rospy.set_param('subswarm_id', self.subSwarmID)
self.log_dbug("subswarm set to %d" % swarmMsg.data)
def _setSwarmState(self, swarmMsg):
''' Updates the "swarm state" for this vehicle
@param swarmMsg: message (UInt8) containing the updated swarm behavior
'''
self.swarmState = swarmMsg.data
self.log_dbug("swarm behavior set to %d" % swarmMsg.data)
def _setSwarmBehavior(self, swarmMsg):
''' Updates the "swarm behavior" currently active for this vehicle
@param swarmMsg: message (UInt8) containing the updated swarm state
'''
self.swarmBehavior = swarmMsg.data
self.log_dbug("swarm state set to %d" % swarmMsg.data)
from threading import RLock
from threading import Thread
sharedData = 22
mylock = RLock()
def thread_read():
mylock.acquire()
print(sharedData)
mylock.release()
mylock.acquire()
mylock.acquire()
mylock.release()
mylock.release()
thread = Thread(target=thread_read)
thread.start()
thread.join()
class Persistent(object):
def __init__(self, name, **kw):
super(Persistent, self).__init__()
self.name = name
self.storage = None
self.synclock = RLock()
self.path = kw.get('path', PDODIR)
self.encode = kw.get('encode', repr)
self.decode = kw.get('decode', eval)
self.extension = kw.get('extension', 'dat')
self.autopersist = kw.get('autopersist', True)
if self.autopersist:
self.load()
def open(self):
self.synclock.acquire()
try:
self.storage = FileStorage(self.path, self.name, self.extension)
self.storage.open()
finally:
self.synclock.release()
def close(self):
self.synclock.acquire()
try:
self.storage.close()
self.storage = None
finally:
self.synclock.release()
def closed(self):
storage = self.storage
if storage is None:
return True
elif storage.closed():
return True
return False
def update_storage(self):
"""
Serialize and data associated with object and
update storage record to match serialization.
"""
self.synclock.acquire()
try:
data = self.getencoded()
self.storage.set(data)
finally:
self.synclock.release()
def update_data(self):
self.synclock.acquire()
try:
data = self.storage.getdata()
self.setencoded(data)
finally:
self.synclock.release()
def commit(self):
"""
Update storage with most recent data, then
commit changes.
"""
self.synclock.acquire()
try:
self.update_storage()
self.storage.commit()
self.notify_committed()
finally:
self.synclock.release()
def load(self):
"""
Load most recently stored data, then update
current data with loaded content.
"""
self.synclock.acquire()
try:
if self.storage is None:
self.open()
self.storage.load()
self.update_data()
self.notify_loaded()
finally:
self.synclock.release()
def serialize(self, data):
if self.encode is not None:
data = self.encode(data)
return data
def unserialize(self, data):
if self.decode is not None:
data = self.decode(data)
return data
def getencoded(self):
"""
Return encoded representation of current data object.
This method must be overridden in type-specific
subclasses.
"""
raise TypeError("Method must be overridden")
def setencoded(self, data):
"""
Use encoded representation of persisted data object
to update current data object.
This method must be overridden in type-specific
subclasses.
"""
raise TypeError("Method must be overridden")
def notify_committed(self):
pass
def notify_loaded(self):
pass
class VideoHTTPServer(ThreadingMixIn, BaseHTTPServer.HTTPServer):
__single = None
def __init__(self, port):
if VideoHTTPServer.__single:
raise RuntimeError, 'HTTPServer is Singleton'
VideoHTTPServer.__single = self
self.port = port
if globalConfig.get_value('allow-non-local-client-connection'):
bind_address = ''
else:
bind_address = '127.0.0.1'
BaseHTTPServer.HTTPServer.__init__(self, (bind_address, self.port),
SimpleServer)
self.daemon_threads = True
self.allow_reuse_address = True
self.lock = RLock()
self.urlpath2streaminfo = {}
self.mappers = []
self.errorcallback = None
self.statuscallback = None
def getInstance(*args, **kw):
if VideoHTTPServer.__single is None:
VideoHTTPServer(*args, **kw)
return VideoHTTPServer.__single
getInstance = staticmethod(getInstance)
def background_serve(self):
name = 'VideoHTTPServerThread-1'
self.thread2 = Thread(target=self.serve_forever, name=name)
self.thread2.setDaemon(True)
self.thread2.start()
def register(self, errorcallback, statuscallback):
self.errorcallback = errorcallback
self.statuscallback = statuscallback
def set_inputstream(self, streaminfo, urlpath):
self.lock.acquire()
if DEBUGLOCK:
log('videoserver::set_inputstream: urlpath', urlpath, 'streaminfo',
streaminfo, 'thread',
currentThread().getName())
if self.urlpath2streaminfo.has_key(urlpath):
if DEBUGLOCK:
log(
'videoserver::set_inputstream: path exists, delete old: urlpath',
urlpath, 'thread',
currentThread().getName())
self.del_inputstream(urlpath)
streaminfo['lock'] = RLock()
self.urlpath2streaminfo[urlpath] = streaminfo
self.lock.release()
def acquire_inputstream(self, urlpath):
global DEBUG
if urlpath is None:
return
streaminfo = None
for mapper in self.mappers:
streaminfo = mapper.get(urlpath)
if streaminfo is not None and (streaminfo['statuscode'] == 200
or streaminfo['statuscode'] == 301):
return streaminfo
self.lock.acquire()
if DEBUGLOCK:
log('VideoServer::acquire_inputstream: lock done', urlpath,
currentThread().getName())
try:
streaminfo = self.urlpath2streaminfo.get(urlpath, None)
if DEBUG:
log(
'videoserver::acquire_inputstream: got streaminfo: urlpath',
urlpath, 'streaminfo', streaminfo)
finally:
if DEBUGLOCK:
log('VideoServer::acquire_inputstream: unlock', urlpath,
currentThread().getName())
self.lock.release()
if streaminfo is not None and 'lock' in streaminfo:
if DEBUGLOCK:
log('VideoServer::acquire_inputstream: lock stream: urlpath',
urlpath, 'streaminfo', streaminfo, 'thread',
currentThread().getName())
streaminfo['lock'].acquire()
if DEBUGLOCK:
log(
'VideoServer::acquire_inputstream: lock stream done: urlpath',
urlpath, 'thread',
currentThread().getName())
return streaminfo
def release_inputstream(self, urlpath):
if DEBUGLOCK:
log('VideoServer::release_inputstream: lock', urlpath,
currentThread().getName())
self.lock.acquire()
try:
streaminfo = self.urlpath2streaminfo.get(urlpath, None)
finally:
if DEBUGLOCK:
log('VideoServer::release_inputstream: unlock', urlpath,
currentThread().getName())
self.lock.release()
if streaminfo is not None and 'lock' in streaminfo:
if DEBUGLOCK:
log('VideoServer::release_inputstream: unlock stream: urlpath',
urlpath, 'streaminfo', streaminfo, 'thread',
currentThread().getName())
streaminfo['lock'].release()
def del_inputstream(self, urlpath):
if DEBUGLOCK:
log('VideoServer::del_inputstream: enter', urlpath)
streaminfo = self.acquire_inputstream(urlpath)
self.lock.acquire()
if DEBUGLOCK:
log('VideoServer::del_inputstream: lock', urlpath,
currentThread().getName())
try:
del self.urlpath2streaminfo[urlpath]
except KeyError:
if DEBUGLOCK:
log('videoserver::del_inputstream: path not found: urlpath',
urlpath)
finally:
if DEBUGLOCK:
log('VideoServer::del_inputstream: unlock', urlpath,
currentThread().getName())
self.lock.release()
if streaminfo is not None and 'lock' in streaminfo:
if DEBUGLOCK:
log('VideoServer::del_inputstream: stream: unlock', urlpath,
currentThread().getName())
streaminfo['lock'].release()
def get_port(self):
return self.port
def add_path_mapper(self, mapper):
self.mappers.append(mapper)
def shutdown(self):
if DEBUG:
print >> sys.stderr, 'videoserv: Shutting down HTTP'
self.socket.close()
def handle_error(self, request, client_address):
if DEBUGBASESERV:
print >> sys.stderr, 'VideoHTTPServer: handle_error', request, client_address
log_exc()
class RequestQueue(object):
"""
Queue and rate limit HTTP requests as to not overload the socket count.
"""
def __init__(self, max_outstanding=50, timeout=15):
"""
Create a RequestQueue object for rate-limiting HTTP requests.
:param max_outstanding: the maximum number of requests which can be unanswered at any given time.
:param timeout: the time after which a request is assumed to never receive a response.
"""
self.max_outstanding = max_outstanding
self.timeout = timeout
self.critical_queue = []
self.high_queue = []
self.medium_queue = []
self.low_queue = []
self.lock = RLock() # Don't allow asynchronous access to the queue
def parse_queue(self):
"""
Parse the queues and dispatch the request.
"""
self.lock.acquire()
current_time = time()
self.critical_queue = [
(request_manager, endpoint, read_callback, data, method,
capture_errors, insertion_time)
for (request_manager, endpoint, read_callback, data, method,
capture_errors, insertion_time) in self.critical_queue
if current_time -
insertion_time < self.timeout or request_manager.cancel_request()
]
self.high_queue = [
(request_manager, endpoint, read_callback, data, method,
capture_errors, insertion_time)
for (request_manager, endpoint, read_callback, data, method,
capture_errors, insertion_time) in self.high_queue
if current_time -
insertion_time < self.timeout or request_manager.cancel_request()
]
self.medium_queue = [
(request_manager, endpoint, read_callback, data, method,
capture_errors, insertion_time)
for (request_manager, endpoint, read_callback, data, method,
capture_errors, insertion_time) in self.medium_queue
if current_time -
insertion_time < self.timeout or request_manager.cancel_request()
]
self.low_queue = [
(request_manager, endpoint, read_callback, data, method,
capture_errors, insertion_time)
for (request_manager, endpoint, read_callback, data, method,
capture_errors, insertion_time) in self.low_queue
if current_time -
insertion_time < self.timeout or request_manager.cancel_request()
]
queue_item = None
if self.critical_queue:
queue_item = self.critical_queue.pop(0)
elif self.high_queue:
queue_item = self.high_queue.pop(0)
elif self.medium_queue:
queue_item = self.medium_queue.pop(0)
elif self.low_queue:
queue_item = self.low_queue.pop(0)
if queue_item:
dispatcher.perform_request(*queue_item[:-2])
self.lock.release()
def enqueue(self,
request_manager,
method,
endpoint,
data,
read_callback,
capture_errors,
priority=QueuePriorityEnum.HIGH):
"""
Add a new request to the queue based on priority
Priority order
- CRITICAL
- HIGH
- MEDIUM
- LOW
:param request_manager: the TriblerRequestManager wishing to perform a request.
:param method: request method.
:param endpoint: request endpoint.
:param data: request data.
:param read_callback: callback to call if the request is processed.
:param capture_errors: whether to display the errors or not.
:param priority: the priority for this request.
"""
self.lock.acquire()
queue_item = (request_manager, endpoint, read_callback, data, method,
capture_errors, time())
if priority == QueuePriorityEnum.CRITICAL:
self.critical_queue.append(queue_item)
if priority == QueuePriorityEnum.HIGH:
if len(self.high_queue) < self.max_outstanding:
self.high_queue.append(queue_item)
else:
# Get the last item of the queue
last_item = self.high_queue.pop(self.max_outstanding - 1)
# Add the original queue_item to the front of the queue
self.high_queue.insert(0, queue_item)
# reduce the priority of last_item and try to put in medium queue
priority = QueuePriorityEnum.MEDIUM
queue_item = last_item
if priority == QueuePriorityEnum.MEDIUM:
if len(self.medium_queue) < self.max_outstanding:
self.medium_queue.append(queue_item)
else:
# Get the last item of the queue
last_item = self.medium_queue.pop(self.max_outstanding - 1)
# Add the original queue_item to the front of the queue
self.medium_queue.insert(0, queue_item)
# reduce the priority of last_item and try to put in low queue
priority = QueuePriorityEnum.LOW
queue_item = last_item
if priority == QueuePriorityEnum.LOW:
if len(self.low_queue) < self.max_outstanding:
self.low_queue.append(queue_item)
else:
# Remove the last item of the queue which will be dropped
self.low_queue.pop(self.max_outstanding - 1)
# Add the original queue_item to the front of the queue
self.low_queue.insert(0, queue_item)
self.lock.release()
self.parse_queue()
def clear(self):
"""
Clear the queue.
"""
self.lock.acquire()
for request_manager, _, _, _, _, _, _ in self.critical_queue:
request_manager.cancel_request()
for request_manager, _, _, _, _, _, _ in self.high_queue:
request_manager.cancel_request()
for request_manager, _, _, _, _, _, _ in self.medium_queue:
request_manager.cancel_request()
for request_manager, _, _, _, _, _, _ in self.low_queue:
request_manager.cancel_request()
self.critical_queue = []
self.high_queue = []
self.medium_queue = []
self.low_queue = []
self.lock.release()
class CursesUtil:
def __init__(self):
self.pairlock = Lock()
# iolock protects access to the
self.iolock = RLock()
self.start()
def initpairs(self):
self.pairlock.acquire()
try:
self.pairs = {
self._getpairindex(curses.COLOR_WHITE, curses.COLOR_BLACK): 0
}
self.nextpair = 1
finally:
self.pairlock.release()
def lock(self):
"""Locks the Curses ui thread
Can be invoked multiple times from the owning thread. Invoking
from a non-owning thread blocks and waits until it has been
unlocked by the owning thread."""
self.iolock.acquire()
def unlock(self):
"""Unlocks the Curses ui thread
Decrease the lock counter by one and unlock the ui thread if the
counter reaches 0. Only call this method when the calling
thread owns the lock. A RuntimeError is raised if this method is
called when the lock is unlocked."""
self.iolock.release()
def locked(self, target, *args, **kwargs):
"""Perform an operation with full locking."""
self.lock()
try:
apply(target, args, kwargs)
finally:
self.unlock()
def refresh(self):
def lockedstuff():
curses.panel.update_panels()
curses.doupdate()
self.locked(lockedstuff)
def isactive(self):
return hasattr(self, 'stdscr')
def _getpairindex(self, fg, bg):
return '%d/%d' % (fg, bg)
def getpair(self, fg, bg):
if not self.has_color:
return 0
pindex = self._getpairindex(fg, bg)
self.pairlock.acquire()
try:
if self.pairs.has_key(pindex):
return curses.color_pair(self.pairs[pindex])
else:
self.pairs[pindex] = self.nextpair
curses.init_pair(self.nextpair, fg, bg)
self.nextpair += 1
return curses.color_pair(self.nextpair - 1)
finally:
self.pairlock.release()
def start(self):
self.stdscr = curses.initscr()
curses.noecho()
curses.cbreak()
self.stdscr.keypad(1)
try:
curses.start_color()
self.has_color = curses.has_colors()
except:
self.has_color = 0
self.oldcursor = None
try:
self.oldcursor = curses.curs_set(0)
except:
pass
self.stdscr.clear()
self.stdscr.refresh()
(self.height, self.width) = self.stdscr.getmaxyx()
self.initpairs()
def stop(self):
if not hasattr(self, 'stdscr'):
return
#self.stdscr.addstr(self.height - 1, 0, "\n",
# self.getpair(curses.COLOR_WHITE,
# curses.COLOR_BLACK))
if self.oldcursor != None:
curses.curs_set(self.oldcursor)
self.stdscr.refresh()
self.stdscr.keypad(0)
curses.nocbreak()
curses.echo()
curses.endwin()
del self.stdscr
def reset(self):
# dirty walkaround for bug http://bugs.python.org/issue7567 in python 2.6 to 2.6.5 (fixed since #83743)
if (sys.version_info[0:3] >= (2, 6) and sys.version_info[0:3] <=
(2, 6, 5)):
return
self.stop()
self.start()
class MsgAutoSender(object):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
"Accept-Language": "en-US,en;q=0.5",
#"Cookie": "; ".join(map(lambda x: "=".join(x), data.items()))
}
messages = ["在不在呢[疑问]"]
data = {
'txtLoginEMail': "",
'txtLoginPwd': "",
'chkRememberMe': "",
'codeId': "",
'codeValue': '',
'event':'3',
'spmp':'4.20.53.225.685',
'_': "%d"%(time.time()*1000)
}
# 预登陆url
url1 = 'http://my.baihe.com/Getinterlogin/gotoLogin?jsonCallBack=jQuery18308807729283968166_%d&'%(time.time()*1000)
# 登陆成功后跳转到主页
url2 = "http://u.baihe.com/?userid=&time=%d"%(time.time()*1000)
# 用来获取一些默认的搜索条件（百合会根据你个人信息筛选出一些基本符合你要求的人群）
url3 = "http://search.baihe.com/mystruts/nextSolrSearch.action?jsoncallback=jQuery183042376943520885857_1479472584212&callType=next&pageId=%%s&ord=1&_=%d"%(time.time()*1000)
# 用来搜索默认条件下的妹纸
url4 = "http://search.baihe.com/solrAdvanceSearch.action"
# 向妹纸发送消息
url5 = "http://msg.baihe.com/owner/api/sendMessage?jsonCallBack=jQuery18304671662130587029_1479300393335&toUserID=%%s&content=%%s&type=1&pathID=01.00.10402&_=%d&"%(time.time()*1000)
# 登陆过频繁的话，会要求输验证，此url用来获取验证码图片
url6 = "http://my.baihe.com/Getinterlogin/getVerifyPic?jsonCallBack=?&tmpId=%s"
# 用来检查登陆次数，来判定是否需要输验证码了
url7 = "http://my.baihe.com/Getinterlogin/getAccountTimes?jsonCallBack=jQuery183013238800936369732_1479556200186&userAccount=18353130797&_=1479556585384"
# 用来验证验证码是否正确
url8 = "http://my.baihe.com/Getinterlogin/checkVerifyPic?jsonCallBack=jQuery183010981413646438898_1480223919788&tmpId=%%s&checkcode=%%s&_=%d"%(time.time()*1000)
# access_token生成
acc_token = Cookie(version=0,
name="accessToken",
value='BH%d%d'%(time.time()*1000,math.floor(random.random()*1000000)),
domain=".baihe.com",
path="/",
port=None,
port_specified=False,
domain_specified=True,
domain_initial_dot=False,
path_specified=True,
secure=False,
expires=None,
discard=False,
comment=None,
comment_url=None,
rest={},
rfc2109=False)
def __init__(self):
self.page = 1
self.order = 1
self.product_ids = set()
self.error_count = 0
self.lock = RLock()
self.alive = True
self.cookie = LWPCookieJar()
try:
self.have_send_list = set(open("have_send_list.txt").read().strip(",").split(","))
except IOError:
self.have_send_list = set()
self.logger = logging.getLogger("send_msg")
self.logger.setLevel(logging.DEBUG)
self.logger.addHandler(logging.StreamHandler(sys.stdout))
def get_account_times(self, opener):
resp = opener.open(self.url7)
buf = resp.read()
data = json.loads(re.search(r"\((\{.*\})\)", buf).group(1), encoding="gbk")
self.logger.debug("Check whether need input captcha or not. ")
if data["data"]["showCode"]:
return self.get_captcha(opener)
else:
return "", ""
def get_captcha(self, opener):
tmpId = "%d.%s"%(time.time()*1000, str(round(random.random(), 4))[2:])
resp = opener.open(self.url6%tmpId)
img = Image.open( StringIO(resp.read()))
img.show()
return raw_input("Please input captcha recognization: "), tmpId
def send_captcha(self, opener, captcha, tmpId):
self.logger.debug("Send captcha. ")
url = self.url8 % (tmpId, captcha)
req = Request(url=url, headers=self.headers)
resp = opener.open(req)
data = json.loads(re.search(r"\((\{.*\})\)", resp.read()).group(1), encoding="gbk")
if data["data"] == 1:
return tmpId, captcha
def login(self, opener):
url = self.url1 + urlencode(self.data)
req = Request(url=url, headers=self.headers)
resp = opener.open(req)
data = json.loads(re.search(r"\((\{.*\})\)", resp.read()).group(1), encoding="gbk")
self.logger.debug("Login jsonp response state:%s"%data["state"])
if data["state"] == 0:
return "Wrong account or password. "
req = Request(url=self.url2, headers=self.headers)
resp = opener.open(req)
self.logger.debug("Login redirect response code:%s"%resp.code)
def get_auth_cookies(self, opener):
while True:
self.enter_password()
captcha, tmpId = self.get_account_times(opener)
if tmpId:
while not self.send_captcha(opener, captcha, tmpId):
captcha, tmpId = self.get_account_times(opener)
self.data["codeValue"] = captcha
self.data["codeId"] = tmpId
result = self.login(opener)
if result:
self.logger.info(result)
else:
break
def get_search_cookies(self, opener):
req = Request(url=self.url4, headers=self.headers)
resp = opener.open(req)
self.logger.debug("Finish get default search cookies, response code:%s" % resp.code)
def search(self, opener):
conti = True
while conti:
while True:
try:
id = self.product_ids.pop()
except KeyError:
break
self.send_msg(opener, id)
self.order += 1
req = Request(url=self.url3 % self.page, headers=self.headers)
self.logger.debug("Start to find girls in page NO.%s. "%self.page)
resp = opener.open(req)
self.logger.debug("Search response code:%s" % resp.code)
buf = resp.read()
data = json.loads(re.search(r"\((\{.*\})\)", buf).group(1), encoding="gbk")
if data["result"]:
self.product_ids = set([i.split(":")[0] for i in data["result"].split(",")])
self.page += 1
elif self.page > 100:
return "finished"
else:
raise SendMessageError("You need relogin. ")
def send_msg(self, opener, id):
if id not in self.have_send_list:
msg = random.choice(self.messages)
d = quote(msg)
url = self.url5 % (id, d)
req = Request(url=url, headers=self.headers)
resp = opener.open(req)
buf = resp.read()
recv = json.loads(re.search(r"\((\{.*\})\)", buf).group(1), encoding="gbk")
code = recv["code"]
self.logger.info("Send %s to No.%s girl whose id is %s, status code is %s" % (msg.decode("utf-8"), self.order, id, code))
if code == 200:
if self.error_count > 0:
self.error_count -= 1
self.have_send_list.add(id)
else:
self.error_count += 1
if code == u"-701":
self.alive = False
self.logger.error(u"坑爹的百合每天每个账号只允许给100个人发消息。。")
sys.exit(0)
if self.error_count > 3:
raise SendMessageError("code: %s error: %s" % (code.encode("gbk"), (recv.get("msg") or u"empty").encode("gbk")))
time.sleep(1)
else:
self.logger.info("The No.%s girl whose id is %s has been sent, don't molesting her any more. "%(self.order, id))
def pwd_input(self, msg=''):
if msg != '':
sys.stdout.write(msg)
chars = []
while True:
newChar = getch()
if newChar in '\3\r\n': # 如果是换行，Ctrl+C，则输入结束
print ''
if newChar in '\3': # 如果是Ctrl+C，则将输入清空，返回空字符串
chars = []
break
elif newChar == '\b' or ord(newChar) == 127: # 如果是退格，则删除末尾一位
if chars:
del chars[-1]
sys.stdout.write('\b \b') # 左移一位，用空格抹掉星号，再退格
else:
chars.append(newChar)
sys.stdout.write('*') # 显示为星号
return ''.join(chars)
def enter_password(self):
account = raw_input("Please input your baihe account number: ")
self.data["txtLoginEMail"] = account
self.data["txtLoginPwd"] = self.pwd_input("Please input your baihe account password: "******"Please input what you want to send, input empty to break. ")
if not msg:
break
else:
try:
msg = msg.decode("gbk").encode("utf-8")
except UnicodeDecodeError:
pass
self.messages.append(msg)
def start(self):
self.enter_msg()
self.cookie.set_cookie(self.acc_token)
have_load = False
try:
if os.path.exists(("baihe.cookie")):
self.cookie.load("baihe.cookie", True, True)
have_load = True
opener = build_opener(HTTPCookieProcessor(self.cookie))
if not have_load:
self.get_auth_cookies(opener)
self.get_search_cookies(opener)
# 有时意外不正常关闭cookie和send_list无法保存，所以启动一个进程来做这件事。
Thread(target=self.saveing).start()
while True:
try:
if self.search(opener) == "finished":
self.logger.info("No more girls to send. ")
break
except Exception, e:
time.sleep(1)
self.logger.error(e)
self.get_auth_cookies(opener)
self.get_search_cookies(opener)
except KeyboardInterrupt:
self.logger.info("Closing...")
self.alive = False
finally:
self.save()
self.alive = False
def saveing(self):
while self.alive:
self.save()
time.sleep(2)
def save(self):
self.lock.acquire()
open("have_send_list.txt", "w").write(",".join(self.have_send_list))
self.cookie.save("baihe.cookie", True, True)
self.lock.release()
class Task(object):
def __init__(self, url, cfgdict):
self.url = url
if url:
_ = index_re.findall(url)
if _:
self.gid, self.sethash = _[0]
self.failcode = 0
self.state = TASK_STATE_WAITING
self.guid = str(uuid.uuid4())[:8]
self.config = cfgdict
self.meta = {}
self.has_ori = False
self.reload_map = {} # {url:reload_url}
self.filehash_map = {
} # map same hash to different ids, {url:((id, fname), )}
self.img_q = None
self.page_q = None
self.list_q = None
self._flist_done = set(
) # store id, don't save, will generate when scan
self._monitor = None
self._cnt_lock = RLock()
self._f_lock = RLock()
def cleanup(self):
if self.state in (TASK_STATE_FINISHED, TASK_STATE_FAILED):
self.img_q = None
self.page_q = None
self.list_q = None
self.reload_map = {}
# if 'filelist' in self.meta:
# del self.meta['filelist']
# if 'resampled' in self.meta:
# del self.meta['resampled']
def set_fail(self, code):
self.state = TASK_STATE_FAILED
self.failcode = code
# cleanup all we cached
self.meta = {}
def migrate_exhentai(self):
_ = re.findall("(?:https*://g\.e\-hentai\.org)(.+)", self.url)
if not _:
return False
self.url = "https://exhentai.org%s" % _[0]
self.state = TASK_STATE_WAITING if self.state == TASK_STATE_FAILED else self.state
self.failcode = 0
return True
# def guess_ori(self):
# # guess if this gallery has resampled files depending on some sample hashes
# # return True if it's ori
# if 'sample_hash' not in self.meta:
# return
# all_keys = map(lambda x:x[:10], self.meta['filelist'].keys())
# for h in self.meta['sample_hash']:
# if h not in all_keys:
# self.has_ori = True
# break
# del self.meta['sample_hash']
def base_url(self):
return re.findall("(https*://(?:g\.e\-|ex)hentai\.org)", self.url)[0]
# def get_picpage_url(self, pichash):
# # if file resized, this url not works
# # http://%s.org/s/hash_s/gid-picid'
# return "%s/s/%s/%s-%s" % (
# self.base_url(), pichash[:10], self.gid, self.meta['filelist'][pichash][0]
# )
def set_reload_url(self, imgurl, reload_url, fname):
# if same file occurs severl times in a gallery
if imgurl in self.reload_map:
fpath = self.get_fpath()
old_fid = self.get_fname(imgurl)[0]
old_f = os.path.join(fpath, self.get_fidpad(old_fid))
this_fid = int(gallery_re.findall(reload_url)[0][1])
this_f = os.path.join(fpath, self.get_fidpad(this_fid))
self._f_lock.acquire()
if os.path.exists(old_f):
# we can just copy old file if already downloaded
try:
with open(old_f, 'rb') as _of:
with open(this_f, 'wb') as _nf:
_nf.write(_of.read())
except Exception as ex:
self._f_lock.release()
raise ex
else:
self._f_lock.release()
self._cnt_lock.acquire()
self.meta['finished'] += 1
self._cnt_lock.release()
else:
# if not downloaded, we will copy them in save_file
if imgurl not in self.filehash_map:
self.filehash_map[imgurl] = []
self.filehash_map[imgurl].append((this_fid, old_fid))
self._f_lock.release()
else:
self.reload_map[imgurl] = [reload_url, fname]
def get_reload_url(self, imgurl):
if not imgurl:
return
return self.reload_map[imgurl][0]
def scan_downloaded(self, scaled=True):
fpath = self.get_fpath()
donefile = False
if os.path.exists(os.path.join(fpath, ".xehdone")) or os.path.exists(
"%s.zip" % fpath):
donefile = True
# can only check un-renamed files
for fid in range(1, self.meta['total'] + 1):
fname = os.path.join(fpath, self.get_fidpad(fid)) # id
if donefile or (os.path.exists(fname)
and os.stat(fname).st_size > 0):
self._flist_done.add(int(fid))
self.meta['finished'] = len(self._flist_done)
if self.meta['finished'] == self.meta['total']:
self.state == TASK_STATE_FINISHED
def queue_wrapper(self, callback, pichash=None, url=None):
# if url is not finished, call callback to put into queue
# type 1: normal file; type 2: resampled url
# if pichash:
# fid = int(self.meta['filelist'][pichash][0])
# if fid not in self._flist_done:
# callback(self.get_picpage_url(pichash))
# elif url:
fhash, fid = gallery_re.findall(url)[0]
# if fhash not in self.meta['filelist']:
# self.meta['resampled'][fhash] = int(fid)
# self.has_ori = True]
if int(fid) not in self._flist_done:
callback(url)
def save_file(self, imgurl, redirect_url, binary):
# TODO: Rlock for finished += 1
fpath = self.get_fpath()
self._f_lock.acquire()
if not os.path.exists(fpath):
os.mkdir(fpath)
self._f_lock.release()
pageurl, fname = self.reload_map[imgurl]
_ = re.findall("/([^/\?]+)(?:\?|$)", redirect_url)
if _: # change it if it's a full image
fname = _[0]
self.reload_map[imgurl][1] = fname
_, fid = gallery_re.findall(pageurl)[0]
fn = os.path.join(fpath, self.get_fidpad(int(fid)))
if os.path.exists(fn) and os.stat(fn).st_size > 0:
return fn
self._cnt_lock.acquire()
self.meta['finished'] += 1
self._cnt_lock.release()
self._f_lock.acquire()
with open(fn, "wb") as f:
f.write(binary)
if imgurl in self.filehash_map:
for fid, _ in self.filehash_map[imgurl]:
fn_rep = os.path.join(fpath, self.get_fidpad(fid))
with open(fn_rep, "wb") as f:
f.write(binary)
self.meta['finished'] += 1
del self.filehash_map[imgurl]
self._f_lock.release()
def get_fname(self, imgurl):
pageurl, fname = self.reload_map[imgurl]
_, fid = gallery_re.findall(pageurl)[0]
return int(fid), fname
def get_fpath(self):
return os.path.join(self.config['dir'],
util.legalpath(self.meta['title']))
def get_fidpad(self, fid, ext='jpg'):
fid = int(fid)
_ = "%%0%dd.%%s" % (len(str(self.meta['total'])))
return _ % (fid, ext)
def rename_fname(self):
fpath = self.get_fpath()
cnt = 0
error_list = []
for h in self.reload_map:
fid, fname = self.get_fname(h)
# if we don't need to rename to original name and file type matches
if not self.config['rename_ori'] and os.path.splitext(
fname)[1].lower() == '.jpg':
continue
fname_ori = os.path.join(fpath, self.get_fidpad(fid)) # id
if self.config['rename_ori']:
fname_to = os.path.join(fpath, util.legalpath(fname))
else:
# Q: Why we don't just use id.ext when saving files instead of using
# id.jpg?
# A: If former task doesn't download all files, a new task with same gallery
# will have zero knowledge about file type before scanning all per page,
# thus can't determine if this id is downloaded, because file type is not
# necessarily .jpg
fname_to = os.path.join(
fpath, self.get_fidpad(fid,
os.path.splitext(fname)[1][1:]))
if fname_ori != fname_to:
if os.path.exists(fname_ori):
while os.path.exists(fname_to):
_base, _ext = os.path.splitext(fname_to)
_ = re.findall("\((\d+)\)$", _base)
if _: # if ...(1) exists, use ...(2)
_base = re.sub(
"\((\d+)\)$", _base, lambda x: "(%d)" %
(int(x.group(1)) + 1))
else:
_base = "%s(1)" % _base
fname_to = "".join((_base, _ext))
try:
os.rename(fname_ori, fname_to)
except Exception as ex:
error_list.append(
os.path.split(fname_ori)[1],
os.path.split(fname_to)[1], str(ex))
cnt += 1
if cnt == self.meta['total']:
with open(os.path.join(fpath, ".xehdone"), "w"):
pass
return error_list
def make_archive(self):
dpath = self.get_fpath()
arc = "%s.zip" % dpath
if os.path.exists(arc):
return arc
with zipfile.ZipFile(arc, 'w') as zipFile:
zipFile.comment = (
"xeHentai Archiver v%s\nTitle:%s\nOriginal URL:%s" %
(__version__, self.meta['title'], self.url)).encode('utf-8')
for f in sorted(os.listdir(dpath)):
fullpath = os.path.join(dpath, f)
zipFile.write(fullpath, f, zipfile.ZIP_STORED)
shutil.rmtree(dpath)
return arc
def from_dict(self, j):
for k in self.__dict__:
if k not in j:
continue
if k.endswith('_q') and j[k]:
setattr(self, k, Queue())
[getattr(self, k).put(e, False) for e in j[k]]
else:
setattr(self, k, j[k])
_ = index_re.findall(self.url)
if _:
self.gid, self.sethash = _[0]
return self
def to_dict(self):
d = dict({
k: v
for k, v in self.__dict__.items()
if not k.endswith('_q') and not k.startswith("_")
})
for k in ['img_q', 'page_q', 'list_q']:
if getattr(self, k):
d[k] = [e for e in getattr(self, k).queue]
return d
class DebugClientThreads(DebugClientBase.DebugClientBase, AsyncIO):
"""
Class implementing the client side of the debugger.
This variant of the debugger implements a threaded debugger client
by subclassing all relevant base classes.
"""
def __init__(self):
"""
Constructor
"""
AsyncIO.__init__(self)
DebugClientBase.DebugClientBase.__init__(self)
# protection lock for synchronization
self.clientLock = RLock()
# the "current" thread, basically the thread we are at a breakpoint
# for.
self.currentThread = None
# special objects representing the main scripts thread and frame
self.mainThread = None
self.mainFrame = None
self.variant = 'Threaded'
def attachThread(self, target=None, args=None, kwargs=None, mainThread=0):
"""
Public method to setup a thread for DebugClient to debug.
If mainThread is non-zero, then we are attaching to the already
started mainthread of the app and the rest of the args are ignored.
@param target the start function of the target thread (i.e. the
user code)
@param args arguments to pass to target
@param kwargs keyword arguments to pass to target
@param mainThread non-zero, if we are attaching to the already
started mainthread of the app
@return The identifier of the created thread
"""
try:
self.lockClient()
newThread = DebugThread(self, target, args, kwargs, mainThread)
ident = -1
if mainThread:
ident = thread.get_ident()
self.mainThread = newThread
if self.debugging:
sys.setprofile(newThread.profile)
else:
ident = _original_start_thread(newThread.bootstrap, ())
newThread.set_ident(ident)
self.threads[newThread.get_ident()] = newThread
finally:
self.unlockClient()
return ident
def threadTerminated(self, dbgThread):
"""
Public method called when a DebugThread has exited.
@param dbgThread the DebugThread that has exited
"""
try:
self.lockClient()
try:
del self.threads[dbgThread.get_ident()]
except KeyError:
pass
finally:
self.unlockClient()
def lockClient(self, blocking=1):
"""
Public method to acquire the lock for this client.
@param blocking flag to indicating a blocking lock
@return flag indicating successful locking
"""
if blocking:
self.clientLock.acquire()
else:
return self.clientLock.acquire(blocking)
def unlockClient(self):
"""
Public method to release the lock for this client.
"""
try:
self.clientLock.release()
except AssertionError:
pass
def setCurrentThread(self, id):
"""
Public method to set the current thread.
@param id the id the current thread should be set to.
"""
try:
self.lockClient()
if id is None:
self.currentThread = None
else:
self.currentThread = self.threads[id]
finally:
self.unlockClient()
def eventLoop(self, disablePolling=False):
"""
Public method implementing our event loop.
@param disablePolling flag indicating to enter an event loop with
polling disabled (boolean)
"""
# make sure we set the current thread appropriately
threadid = thread.get_ident()
self.setCurrentThread(threadid)
DebugClientBase.DebugClientBase.eventLoop(self, disablePolling)
self.setCurrentThread(None)
def set_quit(self):
"""
Public method to do a 'set quit' on all threads.
"""
try:
locked = self.lockClient(0)
try:
for key in self.threads.keys():
self.threads[key].set_quit()
except:
pass
finally:
if locked:
self.unlockClient()
class ProxyObject(object):
"""A proxy to the remote Object.
A ProxyObject is provided by the Bus. ProxyObjects
have member functions, and can be called like normal Python objects.
"""
ProxyMethodClass = _ProxyMethod
DeferredMethodClass = _DeferredMethod
INTROSPECT_STATE_DONT_INTROSPECT = 0
INTROSPECT_STATE_INTROSPECT_IN_PROGRESS = 1
INTROSPECT_STATE_INTROSPECT_DONE = 2
def __init__(self,
conn=None,
bus_name=None,
object_path=None,
introspect=True,
follow_name_owner_changes=False,
**kwargs):
"""Initialize the proxy object.
:Parameters:
`conn` : `dbus.connection.Connection`
The bus or connection on which to find this object.
The keyword argument `bus` is a deprecated alias for this.
`bus_name` : str
A bus name for the application owning the object, to be used
as the destination for method calls and the sender for
signal matches. The keyword argument ``named_service`` is a
deprecated alias for this.
`object_path` : str
The object path at which the application exports the object
`introspect` : bool
If true (default), attempt to introspect the remote
object to find out supported methods and their signatures
`follow_name_owner_changes` : bool
If true (default is false) and the `bus_name` is a
well-known name, follow ownership changes for that name
"""
bus = kwargs.pop('bus', None)
if bus is not None:
if conn is not None:
raise TypeError('conn and bus cannot both be specified')
conn = bus
from warnings import warn
warn(
'Passing the bus parameter to ProxyObject by name is '
'deprecated: please use positional parameters',
DeprecationWarning,
stacklevel=2)
named_service = kwargs.pop('named_service', None)
if named_service is not None:
if bus_name is not None:
raise TypeError('bus_name and named_service cannot both be '
'specified')
bus_name = named_service
from warnings import warn
warn(
'Passing the named_service parameter to ProxyObject by name '
'is deprecated: please use positional parameters',
DeprecationWarning,
stacklevel=2)
if kwargs:
raise TypeError('ProxyObject.__init__ does not take these '
'keyword arguments: %s' % ', '.join(kwargs.keys()))
if follow_name_owner_changes:
# we don't get the signals unless the Bus has a main loop
# XXX: using Bus internals
conn._require_main_loop()
self._bus = conn
if bus_name is not None:
_dbus_bindings.validate_bus_name(bus_name)
# the attribute is still called _named_service for the moment,
# for the benefit of telepathy-python
self._named_service = self._requested_bus_name = bus_name
_dbus_bindings.validate_object_path(object_path)
self.__dbus_object_path__ = object_path
if not follow_name_owner_changes:
self._named_service = conn.activate_name_owner(bus_name)
#PendingCall object for Introspect call
self._pending_introspect = None
#queue of async calls waiting on the Introspect to return
self._pending_introspect_queue = []
#dictionary mapping method names to their input signatures
self._introspect_method_map = {}
# must be a recursive lock because block() is called while locked,
# and calls the callback which re-takes the lock
self._introspect_lock = RLock()
if not introspect or self.__dbus_object_path__ == LOCAL_PATH:
self._introspect_state = self.INTROSPECT_STATE_DONT_INTROSPECT
else:
self._introspect_state = self.INTROSPECT_STATE_INTROSPECT_IN_PROGRESS
self._pending_introspect = self._Introspect()
bus_name = property(
lambda self: self._named_service, None, None,
"""The bus name to which this proxy is bound. (Read-only,
may change.)
If the proxy was instantiated using a unique name, this property
is that unique name.
If the proxy was instantiated with a well-known name and with
``follow_name_owner_changes`` set false (the default), this
property is the unique name of the connection that owned that
well-known name when the proxy was instantiated, which might
not actually own the requested well-known name any more.
If the proxy was instantiated with a well-known name and with
``follow_name_owner_changes`` set true, this property is that
well-known name.
""")
requested_bus_name = property(
lambda self: self._requested_bus_name, None, None,
"""The bus name which was requested when this proxy was
instantiated.
""")
object_path = property(lambda self: self.__dbus_object_path__, None, None,
"""The object-path of this proxy.""")
# XXX: We don't currently support this because it's the signal receiver
# that's responsible for tracking name owner changes, but it
# seems a natural thing to add in future.
#unique_bus_name = property(lambda self: something, None, None,
# """The unique name of the connection to which this proxy is
# currently bound. (Read-only, may change.)
# """)
def connect_to_signal(self,
signal_name,
handler_function,
dbus_interface=None,
**keywords):
"""Arrange for the given function to be called when the given signal
is received.
:Parameters:
`signal_name` : str
The name of the signal
`handler_function` : callable
A function to be called when the signal is emitted by
the remote object. Its positional arguments will be the
arguments of the signal; optionally, it may be given
keyword arguments as described below.
`dbus_interface` : str
Optional interface with which to qualify the signal name.
If None (the default) the handler will be called whenever a
signal of the given member name is received, whatever
its interface.
:Keywords:
`utf8_strings` : bool
If True, the handler function will receive any string
arguments as dbus.UTF8String objects (a subclass of str
guaranteed to be UTF-8). If False (default) it will receive
any string arguments as dbus.String objects (a subclass of
unicode).
`byte_arrays` : bool
If True, the handler function will receive any byte-array
arguments as dbus.ByteArray objects (a subclass of str).
If False (default) it will receive any byte-array
arguments as a dbus.Array of dbus.Byte (subclasses of:
a list of ints).
`sender_keyword` : str
If not None (the default), the handler function will receive
the unique name of the sending endpoint as a keyword
argument with this name
`destination_keyword` : str
If not None (the default), the handler function will receive
the bus name of the destination (or None if the signal is a
broadcast, as is usual) as a keyword argument with this name.
`interface_keyword` : str
If not None (the default), the handler function will receive
the signal interface as a keyword argument with this name.
`member_keyword` : str
If not None (the default), the handler function will receive
the signal name as a keyword argument with this name.
`path_keyword` : str
If not None (the default), the handler function will receive
the object-path of the sending object as a keyword argument
with this name
`message_keyword` : str
If not None (the default), the handler function will receive
the `dbus.lowlevel.SignalMessage` as a keyword argument with
this name.
`arg...` : unicode or UTF-8 str
If there are additional keyword parameters of the form
``arg``\ *n*, match only signals where the *n*\ th argument
is the value given for that keyword parameter. As of this time
only string arguments can be matched (in particular,
object paths and signatures can't).
"""
return \
self._bus.add_signal_receiver(handler_function,
signal_name=signal_name,
dbus_interface=dbus_interface,
bus_name=self._named_service,
path=self.__dbus_object_path__,
**keywords)
def _Introspect(self):
kwargs = {}
if is_py2:
kwargs['utf8_strings'] = True
return self._bus.call_async(self._named_service,
self.__dbus_object_path__,
INTROSPECTABLE_IFACE,
'Introspect',
'', (),
self._introspect_reply_handler,
self._introspect_error_handler,
require_main_loop=False,
**kwargs)
def _introspect_execute_queue(self):
# FIXME: potential to flood the bus
# We should make sure mainloops all have idle handlers
# and do one message per idle
for (proxy_method, args, keywords) in self._pending_introspect_queue:
proxy_method(*args, **keywords)
self._pending_introspect_queue = []
def _introspect_reply_handler(self, data):
self._introspect_lock.acquire()
try:
try:
self._introspect_method_map = process_introspection_data(data)
except IntrospectionParserException as e:
self._introspect_error_handler(e)
return
self._introspect_state = self.INTROSPECT_STATE_INTROSPECT_DONE
self._pending_introspect = None
self._introspect_execute_queue()
finally:
self._introspect_lock.release()
def _introspect_error_handler(self, error):
logging.basicConfig()
_logger.error("Introspect error on %s:%s: %s.%s: %s",
self._named_service, self.__dbus_object_path__,
error.__class__.__module__, error.__class__.__name__,
error)
self._introspect_lock.acquire()
try:
_logger.debug('Executing introspect queue due to error')
self._introspect_state = self.INTROSPECT_STATE_DONT_INTROSPECT
self._pending_introspect = None
self._introspect_execute_queue()
finally:
self._introspect_lock.release()
def _introspect_block(self):
self._introspect_lock.acquire()
try:
if self._pending_introspect is not None:
self._pending_introspect.block()
# else someone still has a _DeferredMethod from before we
# finished introspection: no need to do anything special any more
finally:
self._introspect_lock.release()
def _introspect_add_to_queue(self, callback, args, kwargs):
self._introspect_lock.acquire()
try:
if self._introspect_state == self.INTROSPECT_STATE_INTROSPECT_IN_PROGRESS:
self._pending_introspect_queue.append((callback, args, kwargs))
else:
# someone still has a _DeferredMethod from before we
# finished introspection
callback(*args, **kwargs)
finally:
self._introspect_lock.release()
def __getattr__(self, member):
if member.startswith('__') and member.endswith('__'):
raise AttributeError(member)
else:
return self.get_dbus_method(member)
def get_dbus_method(self, member, dbus_interface=None):
"""Return a proxy method representing the given D-Bus method. The
returned proxy method can be called in the usual way. For instance, ::
proxy.get_dbus_method("Foo", dbus_interface='com.example.Bar')(123)
is equivalent to::
proxy.Foo(123, dbus_interface='com.example.Bar')
or even::
getattr(proxy, "Foo")(123, dbus_interface='com.example.Bar')
However, using `get_dbus_method` is the only way to call D-Bus
methods with certain awkward names - if the author of a service
implements a method called ``connect_to_signal`` or even
``__getattr__``, you'll need to use `get_dbus_method` to call them.
For services which follow the D-Bus convention of CamelCaseMethodNames
this won't be a problem.
"""
ret = self.ProxyMethodClass(self, self._bus, self._named_service,
self.__dbus_object_path__, member,
dbus_interface)
# this can be done without taking the lock - the worst that can
# happen is that we accidentally return a _DeferredMethod just after
# finishing introspection, in which case _introspect_add_to_queue and
# _introspect_block will do the right thing anyway
if self._introspect_state == self.INTROSPECT_STATE_INTROSPECT_IN_PROGRESS:
ret = self.DeferredMethodClass(ret, self._introspect_add_to_queue,
self._introspect_block)
return ret
def __repr__(self):
return '<ProxyObject wrapping %s %s %s at %#x>' % (
self._bus, self._named_service, self.__dbus_object_path__,
id(self))
__str__ = __repr__
class LookAtBehavior:
def __init__(self, camera_root_topic):
#self.wait = False
#self.point3d = None
self.state = 'ready'
self.lock = RLock()
self.lock.acquire()
self.message = None
self.STATES = {
'ready': 'ready', # none
'head_turn': 'head_turn', # something
#'head_turn_drive': 'head_turn_drive', # something
'driving': 'driving'
} # something
rospy.init_node('look_at_point_behavior', anonymous=True)
rospy.Subscriber('cursor3d', PointStamped, self.laser_point_handler)
self.point_pub = rospy.Publisher('cursor3dcentered', PointStamped)
self.double_click = rospy.Subscriber('mouse_left_double_click', String,
self.move_base_double_click)
self.double_click2 = rospy.Subscriber(
'mouse_left_double_click', String,
self.cancel_move_base_double_click)
self.camera_model = cam.ROSStereoCalibration(
'/' + camera_root_topic + '/left/camera_info',
'/' + camera_root_topic + '/right/camera_info')
self.head_client = actionlib.SimpleActionClient(
'head_traj_controller/point_head_action', PointHeadAction)
#self.head_client.wait_for_server()
self.base_client = actionlib.SimpleActionClient(
'move_base', MoveBaseAction)
#self.base_client.wait_for_server()
#self.move_pub = rospy.Publisher('move_base_simple/goal', PoseStamped)
self.move_pub = rospy.Publisher('look_at_point_goal', PoseStamped)
#self.move_pub2 = rospy.Publisher('hai_constant', PoseStamped)
self.tflistener = tf.TransformListener()
self.lock.release()
print 'running'
def move_base_double_click(self, a_str):
if self.message == None:
rospy.logwarn('Unable to go, no message heard.')
return
else:
self.lock.acquire()
self.state = self.STATES['driving']
#Looking at the point last clicked on... (maybe keep doing this as we drive?)
#self.look_at(self.message)
#Move base
self.move_base(self.message)
self.message = None
self.state = self.STATES['ready']
self.lock.release()
def transform_point(self, point_stamped):
point_head = point_stamped.point
#Tranform into base link
target_link = '/base_link'
base_T_head = tfu.transform(target_link, point_stamped.header.frame_id,
self.tflistener)
point_mat_head = tfu.translation_matrix(
[point_head.x, point_head.y, point_head.z])
point_mat_base = base_T_head * point_mat_head
t_base, o_base = tfu.matrix_as_tf(point_mat_base)
#Calculate angle robot should face
angle = math.atan2(t_base[1], t_base[0])
q = tf.transformations.quaternion_from_euler(0, 0, angle)
return (t_base, q, target_link)
def move_base(self, point, wait=False):
t_base, q, target_link = point
ps = PoseStamped()
ps.header.frame_id = target_link
ps.pose.position = geometry_msgs.msg.Point(t_base[0], t_base[1], 0)
ps.pose.orientation = geometry_msgs.msg.Quaternion(*q)
self.move_pub.publish(ps)
#Uncomment to actually move
goal = MoveBaseGoal()
goal.target_pose.header.frame_id = target_link
goal.target_pose.pose.position = geometry_msgs.msg.Point(
t_base[0], t_base[1], 0)
goal.target_pose.pose.orientation = geometry_msgs.msg.Quaternion(*q)
self.base_client.send_goal(goal)
print 'Sent GOAL'
if wait:
self.base_client.wait_for_result()
if self.base_client.get_state() == GoalStatus.SUCCEEDED:
return True
else:
return False
def laser_point_handler(self, point_stamped):
p = np.matrix([
point_stamped.point.x, point_stamped.point.y,
point_stamped.point.z, 1.
]).T
p2d = self.camera_model.left.P * p
p2d = p2d / p2d[2, 0]
bx = ((self.camera_model.left.w / 2.) * .9)
by = ((self.camera_model.left.h / 2.) * .9)
xlim = [bx, self.camera_model.left.w - bx]
ylim = [by, self.camera_model.left.h - by]
if (self.state == self.STATES['driving']):
return
self.message = self.transform_point(point_stamped)
if not in_bounds(p2d, xlim, ylim):
if self.state != self.STATES['head_turn']:
self.lock.acquire()
self.state = self.STATES['head_turn']
self.lock.release()
#else if we are in bounds, we do nothing
#always update laser's location
def run(self):
r = rospy.Rate(50)
while not rospy.is_shutdown():
r.sleep()
if self.state == self.STATES['head_turn']:
self.lock.acquire()
result = self.look_at(self.message, False)
self.state = self.STATES['ready']
self.lock.release()
def look_at(self, message, wait=True):
g = PointHeadGoal()
g.target.header.frame_id = message[2]
g.target.point = geometry_msgs.msg.Point(*message[0])
g.min_duration = rospy.Duration(1.0)
g.max_velocity = 10.
self.head_client.send_goal(g)
#rospy.loginfo('Sent look at goal ' + str(g))
if wait:
self.head_client.wait_for_result()
if self.head_client.get_state() == GoalStatus.SUCCEEDED:
return True
else:
return False
class PygletPlot(object):
"""
Plot Examples
=============
See examples/advaned/pyglet_plotting.py for many more examples.
>>> from sympy.plotting.pygletplot import PygletPlot as Plot
>>> from sympy.abc import x, y, z
>>> Plot(x*y**3-y*x**3)
[0]: -x**3*y + x*y**3, 'mode=cartesian'
>>> p = Plot()
>>> p[1] = x*y
>>> p[1].color = z, (0.4,0.4,0.9), (0.9,0.4,0.4)
>>> p = Plot()
>>> p[1] = x**2+y**2
>>> p[2] = -x**2-y**2
Variable Intervals
==================
The basic format is [var, min, max, steps], but the
syntax is flexible and arguments left out are taken
from the defaults for the current coordinate mode:
>>> Plot(x**2) # implies [x,-5,5,100]
[0]: x**2, 'mode=cartesian'
>>> Plot(x**2, [], []) # [x,-1,1,40], [y,-1,1,40]
[0]: x**2, 'mode=cartesian'
>>> Plot(x**2-y**2, [100], [100]) # [x,-1,1,100], [y,-1,1,100]
[0]: x**2 - y**2, 'mode=cartesian'
>>> Plot(x**2, [x,-13,13,100])
[0]: x**2, 'mode=cartesian'
>>> Plot(x**2, [-13,13]) # [x,-13,13,100]
[0]: x**2, 'mode=cartesian'
>>> Plot(x**2, [x,-13,13]) # [x,-13,13,10]
[0]: x**2, 'mode=cartesian'
>>> Plot(1*x, [], [x], mode='cylindrical')
... # [unbound_theta,0,2*Pi,40], [x,-1,1,20]
[0]: x, 'mode=cartesian'
Coordinate Modes
================
Plot supports several curvilinear coordinate modes, and
they independent for each plotted function. You can specify
a coordinate mode explicitly with the 'mode' named argument,
but it can be automatically determined for Cartesian or
parametric plots, and therefore must only be specified for
polar, cylindrical, and spherical modes.
Specifically, Plot(function arguments) and Plot[n] =
(function arguments) will interpret your arguments as a
Cartesian plot if you provide one function and a parametric
plot if you provide two or three functions. Similarly, the
arguments will be interpreted as a curve if one variable is
used, and a surface if two are used.
Supported mode names by number of variables:
1: parametric, cartesian, polar
2: parametric, cartesian, cylindrical = polar, spherical
>>> Plot(1, mode='spherical') # doctest: +SKIP
Calculator-like Interface
=========================
>>> p = Plot(visible=False)
>>> f = x**2
>>> p[1] = f
>>> p[2] = f.diff(x)
>>> p[3] = f.diff(x).diff(x) # doctest: +SKIP
>>> p # doctest: +SKIP
[1]: x**2, 'mode=cartesian'
[2]: 2*x, 'mode=cartesian'
[3]: 2, 'mode=cartesian'
>>> p.show()
>>> p.clear()
>>> p
<blank plot>
>>> p[1] = x**2+y**2
>>> p[1].style = 'solid'
>>> p[2] = -x**2-y**2
>>> p[2].style = 'wireframe'
>>> p[1].color = z, (0.4,0.4,0.9), (0.9,0.4,0.4)
>>> p[1].style = 'both'
>>> p[2].style = 'both'
>>> p.close()
Plot Window Keyboard Controls
=============================
Screen Rotation:
X,Y axis Arrow Keys, A,S,D,W, Numpad 4,6,8,2
Z axis Q,E, Numpad 7,9
Model Rotation:
Z axis Z,C, Numpad 1,3
Zoom: R,F, PgUp,PgDn, Numpad +,-
Reset Camera: X, Numpad 5
Camera Presets:
XY F1
XZ F2
YZ F3
Perspective F4
Sensitivity Modifier: SHIFT
Axes Toggle:
Visible F5
Colors F6
Close Window: ESCAPE
=============================
"""
#python 2.5 does not support class decorators so use this workaround
_doctest_depends_on = {'modules': ('pyglet', )}
@doctest_depends_on(modules=('pyglet', ))
def __init__(self, *fargs, **win_args):
"""
Positional Arguments
====================
Any given positional arguments are used to
initialize a plot function at index 1. In
other words...
>>> from sympy.plotting.pygletplot import PygletPlot as Plot
>>> from sympy.core import Symbol
>>> from sympy.abc import x
>>> p = Plot(x**2, visible=False)
...is equivalent to...
>>> p = Plot(visible=False)
>>> p[1] = x**2
Note that in earlier versions of the plotting
module, you were able to specify multiple
functions in the initializer. This functionality
has been dropped in favor of better automatic
plot plot_mode detection.
Named Arguments
===============
axes
An option string of the form
"key1=value1; key2 = value2" which
can use the following options:
style = ordinate
none OR frame OR box OR ordinate
stride = 0.25
val OR (val_x, val_y, val_z)
overlay = True (draw on top of plot)
True OR False
colored = False (False uses Black,
True uses colors
R,G,B = X,Y,Z)
True OR False
label_axes = False (display axis names
at endpoints)
True OR False
visible = True (show immediately
True OR False
The following named arguments are passed as
arguments to window initialization:
antialiasing = True
True OR False
ortho = False
True OR False
invert_mouse_zoom = False
True OR False
"""
self._win_args = win_args
self._window = None
self._render_lock = RLock()
self._functions = {}
self._pobjects = []
self._screenshot = ScreenShot(self)
axe_options = parse_option_string(win_args.pop('axes', ''))
self.axes = PlotAxes(**axe_options)
self._pobjects.append(self.axes)
self[0] = fargs
if win_args.get('visible', True):
self.show()
## Window Interfaces
def show(self):
"""
Creates and displays a plot window, or activates it
(gives it focus) if it has already been created.
"""
if self._window and not self._window.has_exit:
self._window.activate()
else:
self._win_args['visible'] = True
self.axes.reset_resources()
if hasattr(self, '_doctest_depends_on'):
self._win_args['runfromdoctester'] = True
self._window = PlotWindow(self, **self._win_args)
def close(self):
"""
Closes the plot window.
"""
if self._window:
self._window.close()
def saveimage(self, outfile=None, format='', size=(600, 500)):
"""
Saves a screen capture of the plot window to an
image file.
If outfile is given, it can either be a path
or a file object. Otherwise a png image will
be saved to the current working directory.
If the format is omitted, it is determined from
the filename extension.
"""
self._screenshot.save(outfile, format, size)
## Function List Interfaces
def clear(self):
"""
Clears the function list of this plot.
"""
self._render_lock.acquire()
self._functions = {}
self.adjust_all_bounds()
self._render_lock.release()
def __getitem__(self, i):
"""
Returns the function at position i in the
function list.
"""
return self._functions[i]
def __setitem__(self, i, args):
"""
Parses and adds a PlotMode to the function
list.
"""
if not (isinstance(i, (int, Integer)) and i >= 0):
raise ValueError("Function index must " "be an integer >= 0.")
if isinstance(args, PlotObject):
f = args
else:
if (not is_sequence(args)) or isinstance(args, GeometryEntity):
args = [args]
if len(args) == 0:
return # no arguments given
kwargs = dict(bounds_callback=self.adjust_all_bounds)
f = PlotMode(*args, **kwargs)
if f:
self._render_lock.acquire()
self._functions[i] = f
self._render_lock.release()
else:
raise ValueError("Failed to parse '%s'." %
', '.join(str(a) for a in args))
def __delitem__(self, i):
"""
Removes the function in the function list at
position i.
"""
self._render_lock.acquire()
del self._functions[i]
self.adjust_all_bounds()
self._render_lock.release()
def firstavailableindex(self):
"""
Returns the first unused index in the function list.
"""
i = 0
self._render_lock.acquire()
while i in self._functions:
i += 1
self._render_lock.release()
return i
def append(self, *args):
"""
Parses and adds a PlotMode to the function
list at the first available index.
"""
self.__setitem__(self.firstavailableindex(), args)
def __len__(self):
"""
Returns the number of functions in the function list.
"""
return len(self._functions)
def __iter__(self):
"""
Allows iteration of the function list.
"""
return self._functions.itervalues()
def __repr__(self):
return str(self)
def __str__(self):
"""
Returns a string containing a new-line separated
list of the functions in the function list.
"""
s = ""
if len(self._functions) == 0:
s += "<blank plot>"
else:
self._render_lock.acquire()
s += "\n".join([
"%s[%i]: %s" % ("", i, str(self._functions[i]))
for i in self._functions
])
self._render_lock.release()
return s
def adjust_all_bounds(self):
self._render_lock.acquire()
self.axes.reset_bounding_box()
for f in self._functions:
self.axes.adjust_bounds(self._functions[f].bounds)
self._render_lock.release()
def wait_for_calculations(self):
sleep(0)
self._render_lock.acquire()
for f in self._functions:
a = self._functions[f]._get_calculating_verts
b = self._functions[f]._get_calculating_cverts
while a() or b():
sleep(0)
self._render_lock.release()
class Connection(Framer):
def __init__(self, sock, delegate=client, **args):
Framer.__init__(self, sock)
self.lock = RLock()
self.attached = {}
self.sessions = {}
self.condition = Condition()
# XXX: we should combine this into a single comprehensive state
# model (whatever that means)
self.opened = False
self.failed = False
self.closed = False
self.close_code = (None, "connection aborted")
self.thread = Thread(target=self.run)
self.thread.setDaemon(True)
self.channel_max = 65535
self.user_id = None
self.op_enc = OpEncoder()
self.seg_enc = SegmentEncoder()
self.frame_enc = FrameEncoder()
self.delegate = delegate(self, **args)
def attach(self, name, ch, delegate, force=False):
self.lock.acquire()
try:
ssn = self.attached.get(ch.id)
if ssn is not None:
if ssn.name != name:
raise ChannelBusy(ch, ssn)
else:
ssn = self.sessions.get(name)
if ssn is None:
ssn = Session(name, delegate=delegate)
self.sessions[name] = ssn
elif ssn.channel is not None:
if force:
del self.attached[ssn.channel.id]
ssn.channel = None
else:
raise SessionBusy(ssn)
self.attached[ch.id] = ssn
ssn.channel = ch
ch.session = ssn
return ssn
finally:
self.lock.release()
def detach(self, name, ch):
self.lock.acquire()
try:
self.attached.pop(ch.id, None)
ssn = self.sessions.pop(name, None)
if ssn is not None:
ssn.channel = None
ssn.closed()
return ssn
finally:
self.lock.release()
def __channel(self):
for i in xrange(1, self.channel_max):
if not self.attached.has_key(i):
return i
else:
raise ChannelsBusy()
def session(self, name, timeout=None, delegate=session.client):
self.lock.acquire()
try:
ch = Channel(self, self.__channel())
ssn = self.attach(name, ch, delegate)
ssn.channel.session_attach(name)
if wait(ssn.condition, lambda: ssn.channel is not None, timeout):
return ssn
else:
self.detach(name, ch)
raise Timeout()
finally:
self.lock.release()
def detach_all(self):
self.lock.acquire()
self.failed = True
try:
for ssn in self.attached.values():
if self.close_code[0] != 200:
ssn.exceptions.append(self.close_code)
self.detach(ssn.name, ssn.channel)
finally:
self.lock.release()
def start(self, timeout=None):
self.delegate.start()
self.thread.start()
if not wait(self.condition, lambda: self.opened or self.failed,
timeout):
self.thread.join()
raise Timeout()
if self.failed:
self.thread.join()
raise ConnectionFailed(*self.close_code)
def run(self):
frame_dec = FrameDecoder()
seg_dec = SegmentDecoder()
op_dec = OpDecoder()
while not self.closed:
try:
data = self.sock.recv(64 * 1024)
if not data:
self.detach_all()
break
# If we have a security layer and it sends us no decoded data,
# that's OK as long as its return code is happy.
if self.security_layer_rx:
try:
data = self.security_layer_rx.decode(data)
except:
self.detach_all()
break
# When we do not use SSL transport, we get periodic
# spurious timeout events on the socket. When using SSL,
# these events show up as timeout *errors*. Both should be
# ignored unless we have aborted.
except socket.timeout:
if self.aborted():
self.close_code = (None, "connection timed out")
self.detach_all()
break
else:
continue
except socket.error, e:
if self.aborted() or str(e) != "The read operation timed out":
self.close_code = (None, str(e))
self.detach_all()
break
else:
continue
frame_dec.write(data)
seg_dec.write(*frame_dec.read())
op_dec.write(*seg_dec.read())
for op in op_dec.read():
try:
self.delegate.received(op)
except Closed, e:
self.close_code = (None, str(e))
if not self.opened:
self.failed = True
self.closed = True
notify(self.condition)
class gui_queue:
"""wakes up the gui thread which then clears our queue"""
def __init__(self, gui, listenport=0):
"""If listenport is 0, we create a random port to listen on"""
self.mylock = RLock()
self.myqueue = []
if listenport == 0:
self.listenport = random.randint(1025, 10000)
else:
self.listenport = listenport
print "Local GUI Queue listening on port %s" % self.listenport
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(("127.0.0.1", self.listenport))
self.listensocket = s
self.listensocket.listen(300) #listen for activity.
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
#time.sleep(15)
self.gui = gui
self.useconnect = 0
if not self.useconnect:
s.connect(("127.0.0.1", self.listenport))
self.s = s
self.readconn, addr = self.listensocket.accept()
return
def get_event_socket(self):
if self.useconnect:
return self.listensocket
else:
return self.readconn
def append(self, command, args):
"""
Append can be called by any thread
"""
#print "about to aquire..."
self.mylock.acquire()
self.myqueue.append((command, args))
if self.useconnect:
#this won't work on a host with a ZoneAlarm firewall or no internet connectivity...
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#small timeout will wake up the gui thread, but not
#cause painful pauses if we are already in the gui thread.
#important to note that we use timeoutsocket and it
#is already loaded.
s.set_timeout(0.01)
#wakey wakey!
#print "Connecting to port %d"%self.listenport
try:
s.connect(("localhost", self.listenport))
except:
#ignore timeouts
pass
else:
self.s.send("A") #just send a byte
#print "About to release"
self.mylock.release()
return
def clearqueue(self):
"""
Clearqueue is only called by the main GUI thread
Don't forget to return 1
"""
#print "Clearing queue"
#clear this...TODO: add select call here.
if self.useconnect:
newconn, addr = self.listensocket.accept()
else:
self.readconn.recv(1)
for i in self.myqueue:
(command, args) = i
try:
#any error in this thread is deadly, so we
#catch and print them all!
self.gui.handle_gui_queue(command, args)
except:
import traceback
traceback.print_exc(file=sys.stdout)
sys.stdout.flush()
self.myqueue = []
return 1
def handle_gui_queue(self, command, args):
"""
Callback the gui_queue uses whenever it recieves a command for us.
command is a string
args is a list of arguments for the command
"""
gtk.threads_enter()
#print "handle_gui_queue"
if command == "addLine":
#print "addLine called in canvasguigtk2.py"
obj = args[0]
self.addLine(obj)
else:
print "Did not recognize action to take %s: %s" % (command, args)
#print "Done handling gui queue"
gtk.threads_leave()
return 1
def gui_queue_append(self, command, args):
"Called by other classes to add to our list of things to do"
self.gui_queue.append(command, args)
return 1
class priority_queue:
def __init__(self):
self.heap_list = [0]
self.size = 0
self.rlock = RLock()
def insert(self, item):
self.rlock.acquire()
self.heap_list.append(item)
self.size += 1
self.__perc_up(self.size)
self.rlock.release()
def __perc_up(self, index):
while index // 2 > 0:
if self.heap_list[index] < self.heap_list[index // 2]:
self.heap_list[index // 2], self.heap_list[
index] = self.heap_list[index], self.heap_list[index // 2]
index = index // 2
def get_min(self):
if self.size > 0:
return self.heap_list[1]
def del_min(self):
self.rlock.acquire()
if self.size > 0:
ret_val = self.heap_list[1]
self.heap_list[1] = self.heap_list[self.size]
self.size -= 1
self.heap_list.pop()
self.__perc_down(1)
return ret_val
self.rlock.release()
def __perc_down(self, index):
def get_min_child(index):
if index * 2 + 1 > self.size:
return index * 2
else:
if self.heap_list[index * 2] < self.heap_list[index * 2 + 1]:
return index * 2
else:
return index * 2 + 1
while index * 2 <= self.size:
min_child = get_min_child(index)
if self.heap_list[index] > self.heap_list[min_child]:
self.heap_list[index], self.heap_list[
min_child] = self.heap_list[min_child], self.heap_list[
index]
index = min_child
def delete(self, item):
self.heap_list.remove(item)
self.size -= 1
i = self.size // 2
while i > 0:
self.__perc_down(i)
i -= 1
def __len__(self):
return self.size
def __contains__(self, item):
return item in self.heap_list
def __iter__(self): #TODO returns 0 element too
return iter(self.heap_list)
class LibraryBroker(object):
def __init__(self, libraries):
self.lock = Lock()
self.lmap = OrderedDict()
self.library_name_map = {}
self.original_path_map = {}
seen = set()
for original_path in libraries:
path = canonicalize_path(original_path)
if path in seen:
continue
is_samefile = False
for s in seen:
if samefile(s, path):
is_samefile = True
break
seen.add(path)
if is_samefile or not LibraryDatabase.exists_at(path):
continue
library_id = library_id_from_path(original_path, self.lmap)
self.lmap[library_id] = path
self.library_name_map[library_id] = basename(original_path)
self.original_path_map[path] = original_path
self.loaded_dbs = {}
self.category_caches, self.search_caches, self.tag_browser_caches = (
defaultdict(OrderedDict), defaultdict(OrderedDict),
defaultdict(OrderedDict))
def get(self, library_id=None):
with self:
library_id = library_id or self.default_library
if library_id in self.loaded_dbs:
return self.loaded_dbs[library_id]
path = self.lmap.get(library_id)
if path is None:
return
try:
self.loaded_dbs[library_id] = ans = self.init_library(
path, library_id == self.default_library)
ans.new_api.server_library_id = library_id
except Exception:
self.loaded_dbs[library_id] = None
raise
return ans
def init_library(self, library_path, is_default_library):
library_path = self.original_path_map.get(library_path, library_path)
return init_library(library_path, is_default_library)
def close(self):
with self:
for db in itervalues(self.loaded_dbs):
getattr(db, 'close', lambda: None)()
self.lmap, self.loaded_dbs = OrderedDict(), {}
@property
def default_library(self):
return next(iter(self.lmap))
@property
def library_map(self):
with self:
return self.library_name_map.copy()
def allowed_libraries(self, filter_func):
with self:
allowed_names = filter_func(
basename(l) for l in itervalues(self.lmap))
return OrderedDict(((lid, self.library_map[lid])
for lid, path in iteritems(self.lmap)
if basename(path) in allowed_names))
def path_for_library_id(self, library_id):
with self:
return self.original_path_map.get(self.lmap.get(library_id))
def __enter__(self):
self.lock.acquire()
def __exit__(self, *a):
self.lock.release()
class TwitterInstances(object):
def __init__(self, dataCollection, tweetProvider):
super(TwitterInstances, self).__init__()
assert isinstance(dataCollection, DataCollection)
self._by_oauth = dict()
self._by_instance_key = dict()
self._lock = RLock()
self.data_collection = dataCollection
self.tweet_provider = tweetProvider
def add(self, twitterInstance):
assert isinstance(twitterInstance, TwitterInstance)
self._lock.acquire()
try:
self._by_instance_key[
twitterInstance.instance_key] = twitterInstance
self._by_oauth[twitterInstance.oauth] = twitterInstance
finally:
self._lock.release()
def getUniqueInstanceKey(self):
def func():
instanceKey = unicode(getUniqueId())
while instanceKey in self._by_instance_key:
instanceKey = unicode(getUniqueId())
return instanceKey
return criticalSection(self._lock, func)
def createInstance(self, twitterAuthentication, geographic_setup_string,
keywords, instance_setup_code):
def func():
twitterInstance = TwitterInstance(self.getUniqueInstanceKey(),
self, twitterAuthentication,
geographic_setup_string,
keywords, instance_setup_code)
return twitterInstance
return criticalSection(self._lock, func)
def getInstanceList(self):
return criticalSection(self._lock,
lambda: list(self._by_instance_key.values()))
def isInstanceKeyInUse(self, instanceKey):
return criticalSection(self._lock,
lambda: instanceKey in self._by_instance_key)
def isAuthInUse(self, oauth):
return criticalSection(self._lock, lambda: oauth in self._by_oauth)
def getInstanceByInstanceKey(self, instanceKey):
result = criticalSection(
self._lock, lambda: self._by_instance_key.get(instanceKey, None))
return result
def getInstanceByAuth(self, oauth):
result = criticalSection(self._lock,
lambda: self._by_oauth.get(oauth, None))
return result
def removeTwitterInstanceByInstanceKey(self, instanceKey):
self._lock.acquire()
try:
instance = self._by_instance_key.get(instanceKey)
if instance is None:
return None
assert isinstance(instance, TwitterInstance)
# Remove from dictionaries first so that it is no
# longer accessible from the rest of the application.
del self._by_instance_key[instanceKey]
del self._by_oauth[instance.oauth]
finally:
self._lock.release()
# Cleanup instance.
instance.shutdownInstance(False)
self.data_collection.removeInstanceData(instanceKey)
return instance
def removeTwitterInstanceByAuth(self, oauth):
self._lock.acquire()
try:
instance = self._by_oauth.get(oauth)
if instance is None:
return None
assert isinstance(instance, TwitterInstance)
# Remove from dictionaries first so that it is no
# longer accessible from the rest of the application.
del self._by_oauth[oauth]
del self._by_instance_key[instance.instance_key]
print unicode(self._by_instance_key)
finally:
self._lock.release()
# Cleanup instance.
instance.shutdownInstance(False)
self.data_collection.removeInstanceData(unicode(instance.instance_key))
return instance
class FakeHttpServer(Thread):
TIMEOUT = 0.5
DEFAULT_RESPONSE = "text of the response"
def __init__(self, port):
Thread.__init__(self)
class FakeHttpHandler(BaseHTTPRequestHandler):
responses = {
'/path' : dict(
code = 200,
contentType = 'text',
body = """Response body""",
)
}
def __init__(self, *args, **kargs):
BaseHTTPRequestHandler.__init__(self, *args, **kargs)
def do_GET(self):
response = FakeHttpHandler.responses[self.path]
self.send_response(response['code'])
if response.has_key('contentType'):
self.send_header('Content-Type', response['contentType'])
else:
self.send_header('Content-Type', 'text')
if response.has_key('locationAddr'):
self.send_header('Location', response['locationAddr'])
else:
self.send_header('Location', 'locationAddr')
if not response.has_key('dontProvideLength'):
self.send_header('Content-Length', len(response['body']))
self.end_headers()
self.wfile.write(response['body'])
self.wfile.close()
def log_message(self, *args, **kargs):
pass
self.running = True
self.requestHandler = FakeHttpHandler
self.server = _AvoidTimeoutHTTPServer( ('', port), FakeHttpHandler )
self.server.socket.settimeout(self.TIMEOUT)
self.handlingLock = RLock()
self.handling = False
def waitUntilHandling(self):
# Improve this with threading.Event / threading.Condition
n = 5
while n > 0:
self.handlingLock.acquire()
try:
if self.handling:
return
finally:
self.handlingLock.release()
n -= 1
time.sleep(self.TIMEOUT)
raise Exception("Still waiting for the http server to handle requests...")
def run(self):
self.handlingLock.acquire()
try:
self.handling = True
finally:
self.handlingLock.release()
while self.running:
try:
self.server.handle_request()
except socket.timeout:
pass
def stop(self):
self.running = False
self.server.socket.close()
def setResponses(self, responses):
self.requestHandler.responses = responses
class AzureCloud(Cloud):
def __init__(self, *args, **kwargs):
super(AzureCloud, self).__init__(*args, **kwargs)
self.vnet_lock = RLock()
self.pace_lock = RLock()
self.pace_timer = 300
# Is only allowed to be called once every 5 seconds
# TODO: Why is there a limitation like this on Azure ... 5 seconds
# wait per request is too much ...
#@rate_limit(0.2)
def execute(self, command, obj={}):
ret = super(AzureCloud, self).execute(command, obj)
# If we are too fast, backoff for 5 minutes before continuing again
if 'Too many requests received' in obj['stderr']:
print "Sleeping for 5 minutes:\n > %s, %s, %s" % (
command, obj['stderr'], obj['stdout'])
time.sleep(self.pace_timer)
return self.execute(command, obj)
return ret
def start_virtual_machine(self, vm):
""" Start a virtual machine """
cmd = ['azure', 'vm', 'start', self.unique(vm.name)]
vm._started = True
return self.execute(cmd)
def stop_virtual_machine(self, vm):
""" Stop a virtual machine """
cmd = ['azure', 'vm', 'shutdown', self.unique(vm.name)]
vm._started = False
return self.execute(cmd)
def status_virtual_machine(self, vm):
return vm._started
# cmd = ['azure', 'vm', 'show', self.unique(vm.name)]
#
# output = {}
# self.execute(cmd, output)
# if 'ReadyRole' in output['stdout']:
# return True
# if 'Stopped' in output['stdout']:
# return False
# return None
def exists_virtual_machine(self, vm):
cmd = ['azure', 'vm', 'show', self.unique(vm.name)]
output = {}
self.execute(cmd, output)
return 'No VMs found' in output['stdout']
def address_virtual_machine(self, vm):
""" Returns the address of a vm """
# TODO: Change the name to address_virtual_machine
return self.unique(vm.name) + ".cloudapp.net"
def hashify_22(self, name):
import hashlib
return str(hashlib.md5(name).hexdigest())[0:22]
def create_location(self, group):
""" Create an affinity group in microsoft terms """
# cmd = ['azure', 'storage', 'account', 'create']
# cmd += ['-a', self.unique(group.name)]
# cmd += ['--type', group.storage_type]
# cmd += [self.hashify_22(self.unique(group.name))]
# return self.execute(cmd)
cmd = ['azure', 'account', 'affinity-group', 'create']
cmd += self.if_available('-l', group.location)
cmd += ['-e', base64.b64encode(self.unique(group.name))]
cmd += [self.unique(group.name)]
self.execute(cmd)
# TODO: creating a location tends to fail because we can't
# cleanly delete it ... for now return true on creating a
# location
return True
def create_security_group(self, ep):
""" Create endpoints in the microsoft terms """
ret = True
# TODO: Can parallelize here
def create_endpoint(vm):
cmd = ['azure', 'vm', 'endpoint', 'create']
cmd += [self.unique(vm), ep.public_port, ep.private_port]
cmd += ['--name', self.unique(ep.name)[-15:]
] # Endpoint name should be at most 15 characters
cmd += ['--protocol', ep.protocol]
self.execute(cmd)
parallel(create_endpoint, ep.virtual_machines())
return ret
def create_virtual_machine(self, vm):
""" Create a virtual machine """
cmd = ['azure', 'vm', 'create', '-z', vm.type]
cmd += self.if_available('-a', self.unique(vm.location()))
cmd += self.if_available('-w', self.unique(vm.virtual_network()))
cmd += [
'-e', '22',
self.unique(vm.name), vm.image, 'cloudbench', '-P', '-t',
constants.DEFAULT_VM_PUBLIC_KEY
]
ret = self.execute(cmd)
return True
def create_virtual_network(self, vnet):
""" Create a virtual network """
# Azure cannot create multiple VNets together, lock on creation
# of each VNet
self.vnet_lock.acquire()
ret = False
try:
cmd = ['azure', 'network', 'vnet', 'create']
cmd += self.if_available('-e', vnet.address_range)
cmd += self.if_available('-a', self.unique(vnet.location()))
cmd += [self.unique(vnet.name)]
ret = self.execute(cmd)
finally:
self.vnet_lock.release()
return True
def delete_security_group(self, _):
""" Delete an azure 'security-group' a.k.a. an endpoint.
We do not need to delete anything here, Azure takes care of it when we wipe out the machine"""
return True
def delete_virtual_machine(self, virtual_machine):
""" Delete a virtual machine and the associated storage """
cmd = [
'azure', 'vm', 'delete', '-b', '-q',
self.unique(virtual_machine.name)
]
return self.execute(cmd)
def delete_virtual_network(self, vnet):
""" Delete a virtual network """
# Serialize network creation
self.vnet_lock.acquire()
ret = False
try:
cmd = [
'azure', 'network', 'vnet', 'delete', '-q',
self.unique(vnet.name)
]
ret = self.execute(cmd)
finally:
self.vnet_lock.release()
return ret
def delete_location(self, group):
cmd = ['azure', 'account', 'affinity-group', 'delete', '-q']
cmd += [self.unique(group.name)]
self.execute(cmd)
# TODO: creating a location tends to fail because we can't
# cleanly delete it ... for now return true on creating a
# location
return True
class testGoogle():
ips=[]
tlist=[]
succIp=[]
def getIp(self):
url="http://www.legendsec.org/google.html"
hds={"User-agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36"}
import urllib2
req=urllib2.Request(url,None,hds)
result=urllib2.urlopen(req).read()
from bs4 import BeautifulSoup as bs
import re
root=bs(result)
ip_list=root.findAll("td",text=re.compile(r"\d+\.\d+\.\d+\.\d+"))
for ip in ip_list:
self.ips.append(ip.a.text)
def endThread(self):
for t in self.tlist:
t.join()
self.saveto(OUTFILE)
def saveto(self,path):
try:
fd=open(path,'w')
if self.succIp:
fd.write('|'.join(self.succIp))
print "已将结果输出到%s"%(OUTFILE)
except IOError:
print '|'.join(self.succIp)
print "保存到文件出错~"
finally:
fd.close()
def connIp(self,options):
import random
atexit.register(self.endThread)
self.r=RLock()
while self.ips:
now_ip=random.choice(self.ips)
now_t=Thread(target=self.__connection,
args=(now_ip,options.count))
now_t.setDaemon(True)
self.tlist.append(now_t)
self.ips.remove(now_ip)
if len(self.tlist) == 5:
for t in self.tlist:
t.start()
for t in self.tlist:
t.join()
self.tlist=[]
def __connection(self,host,count):
import socket
import time
socket.setdefaulttimeout(1)
cost=0
for port in (80,443):
for i in range(count):
start=time.time()
s=socket.socket()
try:
s.connect((host,port))
end=time.time()
cost+=(end-start)
except:
break
finally:
s.close()
self.r.acquire()
if cost:
cost=cost*1000.0/count
print u'connect to %s 平均用时%.2f ms'%(host,
cost)
if cost<MAXTIMEOUT:
self.succIp.append(host)
else:
print "connect to %s time out"%(host)
self.r.release()
class Logger(object):
# paste from goagent
CRITICAL = 5
FATAL = CRITICAL
ERROR = 4
WARNING = 3
WARN = WARNING
INFO = 2
DEBUG = 1
VERBOSE = 0
def __init__(self, *args, **kwargs):
# self.level = self.__class__.INFO
self.logf = None
self.__write = __write = lambda x: sys.stdout.write(safestr(x))
self.isatty = getattr(sys.stdout, 'isatty', lambda: False)()
self.__set_error_color = lambda: None
self.__set_warning_color = lambda: None
self.__set_debug_color = lambda: None
self.__set_verbose_color = lambda: None
self.__reset_color = lambda: None
if self.isatty:
if os.name == 'nt':
self._nt_color_lock = RLock()
import ctypes
SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute
GetStdHandle = ctypes.windll.kernel32.GetStdHandle
self.__set_error_color = lambda: (self._nt_color_lock.acquire(
), SetConsoleTextAttribute(GetStdHandle(-11), 0x0C))
self.__set_warning_color = lambda: (
self._nt_color_lock.acquire(),
SetConsoleTextAttribute(GetStdHandle(-11), 0x06))
self.__set_debug_color = lambda: (self._nt_color_lock.acquire(
), SetConsoleTextAttribute(GetStdHandle(-11), 0x02))
self.__set_verbose_color = lambda: (
self._nt_color_lock.acquire(),
SetConsoleTextAttribute(GetStdHandle(-11), 0x08))
self.__set_bright_color = lambda: (self._nt_color_lock.acquire(
), SetConsoleTextAttribute(GetStdHandle(-11), 0x0F))
self.__reset_color = lambda: (SetConsoleTextAttribute(
GetStdHandle(-11), 0x07), self._nt_color_lock.release())
elif os.name == 'posix':
self.__set_error_color = lambda: __write('\033[31m')
self.__set_warning_color = lambda: __write('\033[33m')
self.__set_debug_color = lambda: __write('\033[32m')
self.__set_verbose_color = lambda: __write('\033[36m')
self.__set_bright_color = lambda: __write('\033[32m')
self.__reset_color = lambda: __write('\033[0m')
@classmethod
def getLogger(cls, *args, **kwargs):
return cls(*args, **kwargs)
def cleanup(self):
if self.logf:
_ = self.logf
self.logf = None
_.close()
def set_logfile(self, fpath):
if self.logf:
self.logf.close()
self.logf = open(fpath, "ab")
def set_level(self, level):
f = ('verbose', 'debug', 'info')
lv = min(max(level, 0), 3)
for p in range(lv):
setattr(self, f[p], self.dummy)
def log(self, level, fmt, *args, **kwargs):
# fmt=du8(fmt)
try:
self.__write('%-4s - [%s] %s\n' %
(level, datetime.datetime.now(
tz_GMT8()).strftime('%X'), fmt % args))
except (ValueError, TypeError):
fmt = fmt.replace('%', '%%')
self.__write('%-4s - [%s] %s\n' %
(level, datetime.datetime.now(
tz_GMT8()).strftime('%X'), fmt % args))
sys.stdout.flush()
if self.logf:
_ = ('[%s] %s%s' % (datetime.datetime.now(
tz_GMT8()).strftime('%b %d %X'), fmt % args, endl))
self.logf.write(_.encode("utf-8", 'replace'))
def dummy(self, *args, **kwargs):
pass
def debug(self, fmt, *args, **kwargs):
self.__set_debug_color()
self.log('DEBG', fmt, *args, **kwargs)
self.__reset_color()
def info(self, fmt, *args, **kwargs):
puretext = self.log('INFO', fmt, *args)
# if self.logfile:
# self.logfile.write(puretext)
def verbose(self, fmt, *args, **kwargs):
self.__set_verbose_color()
self.log('VERB', fmt, *args, **kwargs)
self.__reset_color()
def warning(self, fmt, *args, **kwargs):
self.__set_warning_color()
self.log('WARN', fmt, *args, **kwargs)
self.__reset_color()
def warn(self, fmt, *args, **kwargs):
self.warning(fmt, *args, **kwargs)
def error(self, fmt, *args, **kwargs):
self.__set_error_color()
self.log('ERROR', fmt, *args, **kwargs)
self.__reset_color()
def exception(self, fmt, *args, **kwargs):
self.error(fmt, *args, **kwargs)
traceback.print_exc(file=sys.stderr)
def critical(self, fmt, *args, **kwargs):
self.__set_error_color()
self.log('CRITICAL', fmt, *args, **kwargs)
self.__reset_color()
class VideoRawVLCServer():
__single = None
def __init__(self):
if VideoRawVLCServer.__single:
raise RuntimeError, 'VideoRawVLCServer is Singleton'
VideoRawVLCServer.__single = self
self.lock = RLock()
self.oldsid = None
self.sid2streaminfo = {}
def getInstance(*args, **kw):
if VideoRawVLCServer.__single is None:
VideoRawVLCServer(*args, **kw)
return VideoRawVLCServer.__single
getInstance = staticmethod(getInstance)
def set_inputstream(self, streaminfo, sid):
self.lock.acquire()
try:
print >> sys.stderr, 'VLCRawServer: setting sid', sid
self.sid2streaminfo[sid] = streaminfo
finally:
self.lock.release()
def get_inputstream(self, sid):
self.lock.acquire()
try:
return self.sid2streaminfo[sid]
finally:
self.lock.release()
def shutdown(self):
pass
def ReadDataCallback(self, bufc, buflen, sid):
try:
if self.oldsid is not None and self.oldsid != sid:
oldstream = self.sid2streaminfo[self.oldsid]['stream']
del self.sid2streaminfo[self.oldsid]
try:
oldstream.close()
except:
log_exc()
self.oldsid = sid
streaminfo = self.get_inputstream(sid)
data = streaminfo['stream'].read(buflen)
size = len(data)
if size == 0:
return 0
bufc[0:size] = data
return size
except:
log_exc()
return -1
def SeekDataCallback(self, pos, sid):
try:
if True:
streaminfo = self.get_inputstream(sid)
streaminfo['stream'].seek(pos, os.SEEK_SET)
return 0
return -1
except:
log_exc()
return -1
if resType is None:
GLOBAL["mainWnd"].emit(SIGNAL("CRITICAL_MESSAGE(QString)"),
u"设备初始化失败")
return None
rs = resType(param)
resources[resName] = rs
try:
rs.initResource()
except TestResourceInitException, e:
GLOBAL["mainWnd"].emit(SIGNAL("CRITICAL_MESSAGE(QString)"),
e.message)
del resources[resName]
return None
return rs
finally:
resourceLock.release()
class TestResource:
'''测试资源从此处派生，此为接口定义'''
def initResource(self):
'''初始化资源'''
def retrive(self):
'''回收资源'''
class TestResourceInitException(Exception):
'''测试资源初始化失败时，抛出此异常'''
def __init__(self, message):
self.message = message
