class RunThread(Thread):
'''Very basic thread in which to run the sleep loop while the callbacks are running'''
def __init__(self):
super(RunThread,self).__init__()
self._stop = Event()
def stop(self):
self._stop.set()
def stopped(self):
return self._stop.isSet()
def run(self):
second_count = 0
minute_count = 0
while True:
time.sleep(1) # The callback will be firing all during this point
if second_count % 60 == 0:
print "{} minutes".format(minute_count)
minute_count += 1
print ".",
second_count += 1
if self._stop.isSet():
return
class MarkerThread(Thread):
def __init__(self, da, zl, marker, coord, conf, pixDim):
Thread.__init__(self)
self.da = da
self.update = Event()
self.update.set()
self.__stop = Event()
self.zl = zl
self.marker = marker
self.coord = coord
self.conf = conf
self.pixDim = pixDim
self.img = self.marker.get_marker_pixbuf(zl)
def run(self):
while not self.__stop.isSet():
self.update.wait()
self.update.clear()
self.draw_markers()
def stop(self):
self.__stop.set()
self.update.set()
def draw_markers(self):
for string in self.marker.positions.keys():
if self.update.isSet() or self.__stop.isSet():
break
mpos = self.marker.positions[string]
if (self.zl <= mpos[2]) and (mpos[0], mpos[1]) != (self.coord[0], self.coord[1]):
gtk.threads_enter()
try:
self.da.draw_marker(self.conf, mpos, self.zl, self.img, self.pixDim, string)
finally:
gtk.threads_leave()
class StreamWrapperHelper(Thread):
def __init__(self, queue):
Thread.__init__(self)
self.__stop = Event()
self.__queue = queue
self.__full = Event()
self.start()
def run(self):
while True:
if self.__stop.isSet():
return
stream = tweetstream.SampleStream(u'soldierkam', os.environ["PASSWORD"])
for s in stream:
try:
if self.__stop.isSet():
stream.close()
return
self.__queue.put(s, block=False)
self.__full.clear()
except Full:
if not self.__full.isSet():
logger.warn("Queue is full!!")
self.__full.set()
def close(self):
self.__stop.set()
class ChatReciever(Thread):
def __init__(self, host, timeout=10):
Thread.__init__(self)
self.onMsg = []
self.host = host
self.timeout = timeout
self._stop = Event()
self.start()
def regHandler(self, type, func):
if type == 'msg':
self.onMsg.append(func)
else:
raise UnknownHandlerException()
def run(self):
lastmsg = 0
while 1:
if self._stop.isSet(): break
try:
output = urllib2.urlopen(urllib2.Request(self.host+"/taigachat/list.json", urllib.urlencode({"lastrefresh":lastmsg})), timeout=self.timeout).read()
new = bool(lastmsg)
lastmsg = json.loads(output).get("lastrefresh")
msglist = re.findall('<li.*?data-time="(.*?)".*?class="username" itemprop="name">(.*?)<\/a>.*?messagetext ugc\'>(.*?)</div> <\/li>', json.loads(output.replace("\\n","")).get("templateHtml"))
for msg in msglist:
for f in self.onMsg:
f(msg[0], HTMLParser().unescape(re.sub('<[^>]*>', '', msg[1])), HTMLParser().unescape(re.sub('<[^>]*>', '', re.sub('<img.*?alt="(.*?)".*?>',r'\1', msg[2]))), new)
except Exception as e:
#print traceback.format_exc()
sys.stderr.write('Error while getting messages: ' + str(e) + '\n')
if self._stop.isSet(): break
sleep(2)
def stop(self):
self._stop.set()
class PrinterThread(Thread):
def __init__(self, stdout_queue, total_jobs):
super(PrinterThread, self).__init__()
self._stop = Event()
self.stdout_queue = stdout_queue
self.total_jobs = total_jobs
def stop(self):
self._stop.set()
def stopped(self):
return self._stop.isSet()
def run(self):
hash_to_counter = {}
job_ids_done = set()
while not self._stop.isSet():
(hash_, message) = self.stdout_queue.get()
job_ids_done.add(hash_)
if message is None: # job is done
break
else:
print(
"%s %s" %
(colored(
"[{jobs_done}/{total_jobs}]".format(
jobs_done=len(job_ids_done),
total_jobs=self.total_jobs),
"magenta",
attrs=["bold"]),
message))
self.stdout_queue.task_done()
class CreateTurtleString(Thread):
def __init__(self, string, rules, iterations, *args, **kwargs):
super(CreateTurtleString, self).__init__(*args, **kwargs)
self.string = string
self.rules = rules
self.iterations = iterations
self._job_done = Event()
self._stop_creating = Event()
def job_done(self):
return self._job_done.isSet()
def stop_creating(self):
self._stop_creating.set()
def run(self):
string = self.string
iterations = self.iterations
rules = self.rules
for _ in range(iterations):
if self._stop_creating.isSet():
break
string = cached_expand_string(string, rules)
self.string = string
self._job_done.set()
def test_threadpool():
pool = ThreadPool(core_threads=2, keepalive=0)
event1 = Event()
event2 = Event()
event3 = Event()
pool.submit(event1.set)
pool.submit(event2.set)
pool.submit(event3.set)
event1.wait(1)
event2.wait(1)
event3.wait(1)
assert event1.isSet()
assert event2.isSet()
assert event3.isSet()
sleep(0.3)
eq_(repr(pool), '<ThreadPool at %x; threads=2/20>' % id(pool))
pool.shutdown()
eq_(repr(pool), '<ThreadPool at %x; threads=0/20>' % id(pool))
# Make sure double shutdown is ok
pool.shutdown()
# Make sure one can't submit tasks to a thread pool that has been shut down
assert_raises(RuntimeError, pool.submit, event1.set)
class FileSearchServer(Thread):
""" Thread which answers to file/tag queries sent through unix socket. """
def __init__(self, pname='SET_ME_PLEASE'):
Thread.__init__(self)
self.name = "%s/%s" % (
pname, str(self.__class__).rsplit('.', 1)[1].split("'")[0])
# old socket from a crashed daemon ?
# remove it, the ThreadingUnixStreamServer will create it.
#if os.path.exists(socket_path): os.unlink(socket_path)
self._stop_event = Event()
self.server = ThreadingTCPServer(('127.0.0.1', searcher_port), FileSearchRequestHandler)
self.server.allow_reuse_address = True
# TODO: the socket is set to non-blocking to be able to gracefully terminate the thread,
# but this could lead to CPU hogging problems. VERIFY THIS !!
self.server.socket.setblocking(False)
def run(self):
logging.progress("%s: thread running." % (self.getName()))
#os.chmod(socket_path, stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IWGRP|stat.S_IROTH|stat.S_IWOTH)
while not self._stop_event.isSet():
self.server.handle_request()
time.sleep(0.01)
logging.progress("%s: thread ended." % (self.getName()))
def stop(self):
if not self._stop_event.isSet():
logging.progress("%s: stopping thread." % (self.getName()))
self._stop_event.set()
self.server.socket.close()
self.server.server_close()
if os.path.exists(socket_path):
os.unlink(socket_path)
class AsyncTBWriter():
"Callback for GANLearners that writes to Tensorboard. Extends LearnerTensorboardWriter and adds output image writes."
def __init__(self):
super().__init__()
self.stop_request = Event()
self.queue = Queue()
self.thread = Thread(target=self._queue_processor, daemon=True)
self.thread.start()
def request_write(self, request: TBWriteRequest)->None:
"Queues up an asynchronous write request to Tensorboard."
if self.stop_request.isSet(): return
self.queue.put(request)
def _queue_processor(self)->None:
"Processes queued up write requests asynchronously to Tensorboard."
while not self.stop_request.isSet():
while not self.queue.empty():
if self.stop_request.isSet(): return
request = self.queue.get()
request.write()
sleep(0.2)
#Provided this to stop thread explicitly or by context management (with statement) but thread should end on its own
# upon program exit, due to being a daemon. So using this is probably unecessary.
def close(self)->None:
"Stops asynchronous request queue processing thread."
self.stop_request.set()
self.thread.join()
# Nothing to do, thread already started. Could start thread here to enforce use of context manager
# (but that sounds like a pain and a bit unweildy and unecessary for actual usage)
def __enter__(self): pass
def __exit__(self, exc_type, exc_value, traceback): self.close()
class Worker(Thread):
""" 工作线程，取出线程池中的任务，执行任务中指定的函数，完成任务功能"""
def __init__(self, task_queue, res_queue, poll_timeout=2, **kwds):
Thread.__init__(self, **kwds)
logger.debug('Worker Thread id :: %s', id(self))
self.setDaemon(True)
self.task_queue = task_queue
self.result_queue = res_queue
self._poll_timeout = poll_timeout
self._dismissed = Event()
self.start()
def run(self):
"""重复取出 task_queue 里的任务并执行，直到通知其退出。"""
while True:
if self._dismissed.isSet():
break
try:
task = self.task_queue.get(
block=True, timeout=self._poll_timeout)
except Queue.Empty:
continue
else:
if self._dismissed.isSet():
# 线程退出，把请求放回 task_queue
self.task_queue.put(task)
break
result = task.func(*task.args, **task.kwds)
logger.debug('Worker Thread id :: %s' % id(self))
self.result_queue.put((task, result))
def dismiss(self):
"""当前任务结束后，退出线程"""
self._dismissed.set()
def testBlocking(self):
event = Event()
getter = GetMessageProcess(self._queue, event)
getter.start()
time.sleep(1)
self.assertFalse(event.isSet())
self._queue.append(MESSAGE)
event.wait(5)
self.assertTrue(event.isSet())
def testBlocking(self):
event = Event()
getter = GetItemProcess(self._queue, event)
getter.start()
time.sleep(1)
self.assertFalse(event.isSet())
self._queue.append(TEST_ITEM)
event.wait(5)
self.assertTrue(event.isSet())
class StatusMonitor(Loggable):
# valve_manager = None
_stop_evt = None
_clients = 0
state_freq = Int(3)
lock_freq = Int(5)
owner_freq = Int(5)
update_period = Int(2)
def start(self, vm):
if not self._clients:
if self._stop_evt:
self._stop_evt.set()
self._stop_evt.wait(0.25)
self._stop_evt = Event()
self._iter(1, vm)
else:
self.debug('Monitor already running')
self._clients += 1
def isAlive(self):
if self._stop_evt:
return not self._stop_evt.isSet()
def stop(self):
self._clients -= 1
if not self._clients:
self._stop_evt.set()
self.debug('Status monitor stopped')
else:
self.debug('Alive clients {}'.format(self._clients))
def _iter(self, i, vm):
if vm is None:
self.debug('No valve manager')
return
if not i % self.state_freq:
vm.load_valve_states()
if not i % self.lock_freq:
vm.load_valve_lock_states()
if not i % self.owner_freq:
vm.load_valve_owners()
if i > 100:
i = 0
if not self._stop_evt.isSet():
do_after(self.update_period * 1000, self._iter, i + 1, vm)
#============= EOF =============================================
class AgentLogicBase:
def __init__(self):
self.wait_stop = Event()
if (platform.system() == 'Windows') or (platform.system() == 'Microsoft'):
vport_name = '\\\\.\\Global\\com.eayun.eayunstack.0'
else:
vport_name = '/dev/virtio-ports/com.eayun.eayunstack.0'
self.vio = VirtIoChannel(vport_name)
self.commandHandler = None
def _send(self, name, arguments=None):
self.vio.write(name, arguments or {})
def run(self):
thread.start_new_thread(self.doListen, ())
while not self.wait_stop.isSet():
self.wait_stop.wait(1)
def stop(self):
self.wait_stop.set()
def doListen(self):
if self.commandHandler is None:
return
while not self.wait_stop.isSet():
try:
cmd, args = self.vio.read()
if cmd:
self.parseCommand(cmd, args)
except:
pass
def parseCommand(self, command, args):
if command == 'get_infomation':
name = args.get('name')
result = self.commandHandler.get_infomation(name)
self._send('get_infomation', {'result': result})
elif command == 'execute_script':
path = args.get('path')
type = args.get('type')
result = self.commandHandler.execute_script(path, type)
self._send('execute_script', {'result': result})
elif command == 'execute_command':
cmd = args.get('cmd')
try:
result = self.commandHandler.execute_command(cmd)
self._send('execute_command', {'result': result})
except:
self._send('execute_command', {'result': '0e0r0r0o0r1'})
elif command == 'echo':
self._send('echo', args)
else:
self._send(command, {'result': '0e0r0r0o0r0'})
class DrawTurtle(Thread):
def __init__(self, canvas, turtle, colors={}, **kwargs):
super(DrawTurtle, self).__init__(**kwargs)
self._stop_drawing = Event()
self.canvas = canvas
self.turtle = turtle
self.colors = colors
def stop_drawing(self):
self._stop_drawing.set()
def run(self):
self.canvas.begin_new()
color = self.colors["1"]
# Calculate size, and scale to fill the frame
t_width = self.turtle.rightmost[0] - self.turtle.leftmost[0]
t_height = self.turtle.bottommost[1] - self.turtle.topmost[1]
c_width = int(self.canvas['width'])
c_height = int(self.canvas['height'])
if t_width / t_height > 1: # fat image scale according to width
scale_factor = c_width / t_width
else:
scale_factor = c_height / t_height
left_margin = (c_width - scale_factor*t_width) / 2
top_margin = (c_height - scale_factor*t_height) / 2
x_shift = left_margin - scale_factor*self.turtle.leftmost[0]
y_shift = top_margin - scale_factor*self.turtle.topmost[1]
coordinates = []
for item in self.turtle.lines:
if self._stop_drawing.isSet():
return
if isinstance(item, PlaceHolder):
coordinates.append(item)
else:
coordinates.append((item[0]*scale_factor+x_shift,
item[1]*scale_factor+y_shift,
item[2]*scale_factor+x_shift,
item[3]*scale_factor+y_shift))
for item in coordinates:
if self._stop_drawing.isSet():
return
if isinstance(item, PlaceHolder): # not a list of coordinates
if item.value in self.colors:
color = self.colors[item.value]
else:
self.canvas.queue_line(item, color)
class ChatUserList(Thread):
def __init__(self, host, timeout=10):
Thread.__init__(self)
self.users = []
self.host = host
self.timeout = timeout
self.onJoin = []
self.onLeave = []
self.lock = False
self._stop = Event()
self.start()
def regHandler(self, type, func):
if type == 'join':
self.onJoin.append(func)
elif type == 'leave':
self.onLeave.append(func)
else:
raise UnknownHandlerException()
def run(self):
while 1:
if self._stop.isSet(): break
try:
usersload = json.loads(urllib2.urlopen(urllib2.Request(self.host+"/shoutbox/", urllib.urlencode({"_xfResponseType":"json"})), timeout=self.timeout).read()).get("sidebarHtml")
usersload = usersload[usersload.rfind('<!-- end block: sidebar_online_users -->'):]
users = re.findall('class="username">(.*?)</a>', usersload)
users = map(lambda x: HTMLParser().unescape(re.sub('<[^>]*>', '', x)), users)
if users != self.users and users != [] and self.users != []:
self.lock = True
for a in users:
try:
self.users.remove(a)
except ValueError:
for f in self.onJoin:
f(a)
for a in self.users:
for f in self.onLeave:
f(a)
if users != []:
self.users = users
self.lock = False
except Exception as e:
#print traceback.format_exc()
sys.stderr.write('Error while getting users list: ' + str(e) + '\n')
if self._stop.isSet(): break
sleep(5)
def list(self):
while self.lock or self.users == []:
sleep(0.1)
return self.users
def stop(self):
self._stop.set()
class Idler(object):
def __init__(self, conn):
self.thread = Thread(target=self.idle)
self.M = conn
self.event = Event()
def work(self):
try:
for name, rec in RECIPES.items():
self.process(name, rec)
except IMAP4_SSL.abort:
pass
def start(self):
self.thread.start()
def stop(self):
self.event.set()
def join(self):
self.thread.join()
def idle(self):
while True:
if self.event.isSet():
return
self.needsync = False
def callback(args):
if not self.event.isSet():
self.needsync = True
self.event.set()
self.M.idle(callback=callback)
self.event.wait()
if self.needsync:
self.event.clear()
self.work()
def process(self, label, recipe):
self.M.select(FOLDER_LABEL+'/' + label)
status, data = self.M.uid('search', None, 'UnSeen')
if data[0]:
for num in data[0].split():
status, data = self.M.uid('fetch', num, '(RFC822)')
msg = parse(data[0][1])
skip_spf = any([True for s in SKIP_SPF if s in msg['From']])
if 'pass' in msg.get('Received-SPF', '') or skip_spf:
print(recipe(msg))
else:
from_fallback = msg.get('From', 'No sender')
print("Didn't pass sender checks: %s" % from_fallback)
self.M.uid('store', num, '+FLAGS', 'Seen')
self.M.expunge()
class PasteGraber(Thread):
def __init__(self, grabber_name, paste_url, query_url, links_xpath, target_fn, links_postprocess=None):
self.grabber_name = grabber_name
self.paste_url = paste_url
self.query_url = query_url
self.links_xpath = links_xpath
self.target_fn = target_fn
self.links_postprocess = links_postprocess
self._stopevent = Event()
self.cfg = Config()
super(PasteGraber, self).__init__(name=grabber_name)
def stop(self):
self._stopevent.set()
def stopped(self):
return self._stopevent.isSet()
def run(self):
while not self._stopevent.isSet():
i = 0
listturples = list()
try:
data = lh.parse(self.paste_url)
html = data.getroot()
links = html.xpath(self.links_xpath)
if self.links_postprocess is not None:
links = map(self.links_postprocess, links)
for l in links:
if l in PASTESSEEN:
i += 1
else:
# paste/nopaste-%s-%s.txt"
fn = self.target_fn % (datetime.today().strftime("%Y-%m-%d-%H:%M:%S"), l)
que_url = self.query_url % l
worker_turple = (que_url, fn)
listturples.append(worker_turple)
PASTESSEEN.add(l)
except Exception, e:
msg = "%s: Exception running scraper [%s]: %s %s" % (
self.grabber_name, str(e), e.message, e.args
)
logger.error(msg)
if i > 0:
logger.warn ( "%s: Nber of paste already seen : %s" % (self.grabber_name, i) )
if self.cfg.pastegraber.debug:
logger.debug("%s: list turples: %s " % (self.grabber_name, listturples))
PASTES.put(listturples)
time.sleep(self.cfg.pastegraber.delay)
class BaseTimerClass(Thread):
'''
This is the base class for timer (delay) based threads.
Timer-based threads are ones that do not need to be looking at
keyboard-mouse events to do their job.
'''
def __init__(self, dir_lock, loggername, *args, **kwargs):
Thread.__init__(self)
# print "dir_lock:", dir_lock
# print "loggername:", loggername
self.finished = Event()
self.dir_lock = dir_lock
self.loggername = loggername
self.args = args # arguments, if any, to pass to task_function
self.kwargs = kwargs # keyword args, if any, to pass to task_function
self.settings = _settings['settings']
self.cmdoptions = _cmdoptions['cmdoptions']
self.mainapp = _mainapp['mainapp']
# set this up for clarity
self.subsettings = self.settings[loggername]
# set these up here because we will usually need them.
self.logger = logging.getLogger(self.loggername)
# print "self.logger:", self.logger
self.logfile_path = self.logger.handlers[0].stream.name
self.log_full_dir = os.path.dirname(self.logfile_path)
self.log_rel_dir = os.path.basename(self.log_full_dir)
self.logfile_name = os.path.basename(self.logfile_path)
self.interval = None # set this in derived class
def cancel(self):
'''
Stop the iteration
'''
self.finished.set()
def task_function(self):
'''
to be overridden by derived classes
'''
pass
def run(self):
while not self.finished.isSet():
self.finished.wait(self.interval)
if not self.finished.isSet():
self.task_function(*self.args, **self.kwargs)
class StatusMonitor(Loggable):
valve_manager = None
_stop_evt = None
_clients = 0
state_freq = 3
lock_freq = 5
owner_freq = 5
period = 2
def start(self):
if not self._clients:
if self._stop_evt:
self._stop_evt.set()
self._stop_evt.wait(0.25)
self._stop_evt = Event()
self._iter(1)
else:
self.debug('Monitor already running')
self._clients += 1
def isAlive(self):
if self._stop_evt:
return not self._stop_evt.isSet()
def stop(self):
self._clients -= 1
if not self._clients:
self._stop_evt.set()
self.debug('Status monitor stopped')
else:
self.debug('Alive clients {}'.format(self._clients))
def _iter(self, i):
vm = self.valve_manager
if not i % self.state_freq:
vm.load_valve_states()
if not i % self.lock_freq:
vm.load_valve_lock_states()
if not i % self.owner_freq:
vm.load_valve_owners()
if i > 100:
i = 0
if not self._stop_evt.isSet():
do_after(self.period * 1000, self._iter, i + 1)
class cardChecker(Thread):
""" This class searches for a card with a given ATR and displays
wether or not using a label and an image """
def __init__(self, lbl, img, atr, intervall=1):
Thread.__init__(self)
self._finished = Event()
self._paused = Event()
self.lbl = lbl
self.img = img
self.target_atr = atr
self.intervall = intervall
def _check_card(self):
"""
Actually this method should make use of pyscard, but I couldn't make
it run on the OpenMoko yet. Therefor we call opensc-tool instead, which
leads to higher delays """
proc = subprocess.Popen(["opensc-tool", "--atr"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
line = proc.stdout.readline().rstrip()
if line == self.target_atr:
gobject.idle_add(self.lbl.set_label, STR_CARD_FOUND)
gobject.idle_add(self.img.set_from_file, IMAGES["apply"])
else:
gobject.idle_add(self.lbl.set_label, STR_NO_CARD)
gobject.idle_add(self.img.set_from_file, IMAGES["error"])
def run(self):
"""Main loop: Poll the card if the thread is not paused or finished"""
while (True):
if self._finished.isSet():
return
if self._paused.isSet():
continue
self._check_card()
self._finished.wait(self.intervall)
def stop(self):
self._finished.set()
def pause(self):
self._paused.set()
def resume(self):
self._paused.clear()
class DeferredResponse( object ):
"""
A deferred that resolves to a response from TSServer.
"""
def __init__( self, timeout = RESPONSE_TIMEOUT_SECONDS ):
self._event = Event()
self._message = None
self._timeout = timeout
def resolve( self, message ):
self._message = message
self._event.set()
def result( self ):
self._event.wait( timeout = self._timeout )
if not self._event.isSet():
raise RuntimeError( 'Response Timeout' )
message = self._message
if not message[ 'success' ]:
raise RuntimeError( message[ 'message' ] )
if 'body' in message:
return self._message[ 'body' ]
def TriggerEventWait(
self,
suffix,
payload=None,
prefix="Main",
source=eg
):
event = EventGhostEvent(suffix, payload, prefix, source)
if event.source in self.filters:
for filterFunc in self.filters[event.source]:
if filterFunc(event) is True:
return event
executed = Event()
def Execute():
try:
event.Execute()
finally:
executed.set()
def Transfer():
ActionThreadCall(Execute)
event.SetShouldEnd()
self.AppendAction(Transfer)
executed.wait(5.0)
if not executed.isSet():
eg.PrintWarningNotice(
"timeout TriggerEventWait\n", traceback.format_stack()
)
return event
class Queue(list):
def __init__(self):
super(Queue, self).__init__()
self._lock = Lock()
self._fill = Event()
def put(self, obj):
with self._lock:
self.append(obj)
self._fill.set()
def get(self, block=True):
with self._lock:
if len(self) == 0:
self._fill.clear()
if not self._fill.isSet():
if block:
self._fill.wait()
else:
return None
with self._lock:
return self.pop(0)
def delete(self, index):
if 0 <= index < len(self):
with self._lock:
del self[index]
def remove(self, element):
if element in self:
with self._lock:
del self[self.index(element)]
class Task(object):
def __init__(self, name, start_time, calc_next_time, func):
"""
Initialize a Task.
Arguments:
name - Name of task.
start_time - First time for task to run
calc_next_time - Function to calculate the time of next run,
gets one argument, the last run time as a datetime.
Returns None when task should no longer be run
func - A function to run
"""
self.name = name
self.start_time = start_time
self.scheduled_time = start_time
self.calc_next_time = calc_next_time
self.func = func
self.halt_flag = Event()
def run(self):
logging.debug("Running %s task, scheduled at: %s" % (self.name, self.scheduled_time,))
if not self.halt_flag.isSet():
try:
try:
self.func()
except:
raise
finally:
self.scheduled_time = self.calc_next_time(self.scheduled_time)
logging.debug("Scheduled next run of %s for: %s" % (self.name, self.scheduled_time,))
def halt(self):
self.halt_flag.set()
class ResponseEvent:
"""Event which is fired when the response is returned for a request.
For each request sent this event is created.
An application can wait for the event to create a blocking request.
"""
def __init__(self):
self.__evt = Event()
def waiting(self):
return not self.__evt.isSet()
def waitForResponse(self, timeOut=None):
"""blocks until the response arrived or timeout is reached."""
self.__evt.wait(timeOut)
if self.waiting():
raise Timeout()
else:
if self.response["error"]:
raise Exception(self.response["error"])
else:
return self.response["result"]
def handleResponse(self, resp):
self.response = resp
self.__evt.set()
class ThreadedRunner(Runnable):
def __init__(self, runnable):
self._runnable = runnable
self._notifier = Event()
self._result = None
self._error = None
self._traceback = None
self._thread = None
def run(self):
try:
self._result = self._runnable()
except:
self._error, self._traceback = sys.exc_info()[1:]
self._notifier.set()
__call__ = run
def run_in_thread(self, timeout):
self._thread = Thread(self, name=TIMEOUT_THREAD_NAME)
self._thread.setDaemon(True)
self._thread.start()
self._notifier.wait(timeout)
return self._notifier.isSet()
def get_result(self):
if self._error:
raise self._error, None, self._traceback
return self._result
def stop_thread(self):
self._thread.stop()
class StoppableThreadWithResult(Thread):
"""Thread class with a stop() method. The thread itself has to check
regularly for the stopped() condition."""
def __init__(self, group=None, target=None, name=None,
args=(), kwargs=None, verbose=None):
super(StoppableThreadWithResult, self).__init__(group=group, target=target,
name=name, args=args, kwargs=kwargs, verbose=verbose)
self._stop = Event()
def stop(self):
self._stop.set()
self._Thread__stop()
def stopped(self):
return self._stop.isSet()
def run(self):
if self._Thread__target is not None:
self._return = self._Thread__target(*self._Thread__args,
**self._Thread__kwargs)
def join(self, timeout=None):
Thread.join(self, timeout=None)
return self._return
class TimerWithResume(object):
def __init__(self, status_subject, refresh_interval):
self.status_subject = status_subject
self.abort = Event()
self.refresh_interval = refresh_interval
def perform(self):
while not self.abort.isSet():
self.status_subject.build_status()
self.abort.wait(self.refresh_interval)
def stop(self):
self.abort.set()
def start(self):
self.thread = Thread(target=self.perform)
self.thread.daemon = True
self.thread.start()
def resume(self):
self.thread.join()
self.abort.clear()
self.start()
def set_refresh_interval(self, new_interval):
self.refresh_interval = new_interval
class ASyncDelayedResult():
def __init__(self, jobID=None):
self.__result = None
self.__exception = None
self.__jobID = jobID
self.isFinished = Event()
def setResult(self, result):
self.__result = result
self.isFinished.set()
def setException(self, exception, original_traceback):
self.__original_traceback = original_traceback
self.__exception = exception
self.isFinished.set()
def get(self, timeout = 100):
if self.wait(timeout):
if self.__exception: # exception was raised!
self.__exception.originalTraceback = self.__original_traceback
print >> sys.stderr, self.__original_traceback
raise self.__exception
return self.__result
else:
print_stack()
print >> sys.stderr, "TIMEOUT on get", self.__jobID, timeout
def wait(self, timeout = None):
return self.isFinished.wait(timeout) or self.isFinished.isSet()
class MegaDownloader(Thread):
def __init__(self, url, dest_path, callback):
Thread.__init__(self)
self.callback = callback
self.progress = 0
self.path = dest_path
if not self.path.endswith("/") and not self.path.endswith("\\"):
self.path += SEP
self.url = url
self.schema = 'https'
self.domain = 'mega.co.nz'
self.timeout = 160 # max time (secs) to wait for resp from api requests
self.sid = None
self.sequence_num = random.randint(0, 0xFFFFFFFF)
self.request_id = make_id(10)
self.paused = False
self.pause_cond = Condition(Lock())
self.stop_event = Event()
pass
def _api_request(self, data):
params = {'id': self.sequence_num}
self.sequence_num += 1
if self.sid:
params.update({'sid': self.sid})
#ensure input data is a list
if not isinstance(data, list):
data = [data]
req = requests.post('{0}://g.api.{1}/cs'.format(
self.schema, self.domain),
params=params,
data=json.dumps(data),
timeout=self.timeout)
json_resp = json.loads(req.text)
#if numeric error code response
if isinstance(json_resp, int):
raise RequestError(json_resp)
return json_resp[0]
def _parse_url(self, url):
#parse file id and key from url
if '!' in url:
match = re.findall(r'/#!(.*)', url)
path = match[0]
return path
else:
raise RequestError('Url key missing')
def run(self):
while not self.stop_event.isSet():
path = self._parse_url(self.url).split('!')
file_id = path[0]
file_key = path[1]
file_key = base64_to_a32(file_key)
file_data = self._api_request({'a': 'g', 'g': 1, 'p': file_id})
k = (file_key[0] ^ file_key[4], file_key[1] ^ file_key[5],
file_key[2] ^ file_key[6], file_key[3] ^ file_key[7])
iv = file_key[4:6] + (0, 0)
meta_mac = file_key[6:8]
# Seems to happens sometime... When this occurs, files are
# inaccessible also in the official also in the official web app.
# Strangely, files can come back later.
if 'g' not in file_data:
raise RequestError('File not accessible anymore')
file_url = file_data['g']
file_size = file_data['s']
attribs = base64_url_decode(file_data['at'])
attribs = decrypt_attr(attribs, k)
file_name = attribs['n']
input_file = requests.get(file_url, stream=True).raw
dest_path = self.path
temp_output_file = tempfile.NamedTemporaryFile(mode='w+b',
prefix='megapy_',
delete=False)
k_str = a32_to_str(k)
counter = Counter.new(128,
initial_value=((iv[0] << 32) + iv[1]) << 64)
aes = AES.new(k_str, AES.MODE_CTR, counter=counter)
mac_str = '\0' * 16
mac_encryptor = AES.new(k_str, AES.MODE_CBC, mac_str)
iv_str = a32_to_str([iv[0], iv[1], iv[0], iv[1]])
for chunk_start, chunk_size in get_chunks(file_size):
with lock:
with self.pause_cond:
while self.paused:
self.pause_cond.wait()
chunk = input_file.read(chunk_size)
chunk = aes.decrypt(chunk)
temp_output_file.write(chunk)
encryptor = AES.new(k_str, AES.MODE_CBC, iv_str)
for i in range(0, len(chunk) - 16, 16):
block = chunk[i:i + 16]
encryptor.encrypt(block)
#fix for files under 16 bytes failing
if file_size > 16:
i += 16
else:
i = 0
block = chunk[i:i + 16]
if len(block) % 16:
block += b'\0' * (16 - (len(block) % 16))
mac_str = mac_encryptor.encrypt(
encryptor.encrypt(block))
file_info = os.stat(temp_output_file.name)
self.progress = int(
int(file_info.st_size) / int(file_size) * 100)
file_mac = str_to_a32(mac_str)
temp_output_file.close()
# check mac integrity
if (file_mac[0] ^ file_mac[1],
file_mac[2] ^ file_mac[3]) != meta_mac:
raise ValueError('Mismatched mac')
shutil.move(temp_output_file.name, dest_path + file_name)
self.callback(self, file_name, self.path)
def pause(self):
self.paused = True
# If in sleep, we acquire immediately, otherwise we wait for thread
# to release condition. In race, worker will still see self.paused
# and begin waiting until it's set back to False
self.pause_cond.acquire()
#should just resume the thread
def resume(self):
self.paused = False
# Notify so thread will wake after lock released
self.pause_cond.notify()
# Now release the lock
self.pause_cond.release()
def stop(self):
self.stop_event.set()
self.progress = 0
proba = model.predict_proba(X_test, batch_size=32, verbose=0)
# pred_attr = mapAttributes((proba > 0.6)[0])
# print( proba)
# print(pred_attr)
probStream = getProbaStream(probStream, proba)
if saidNothing == 0 and probStream.shape[0] < 10:
saidNothing += 1
cv2.imshow("Webcam Preview", frame)
rval, frame = vc.read()
key = cv2.waitKey(20)
if key == 27: # exit on ESC
if process != None:
os.kill(process.pid, signal.SIGTERM)
say("I'm sorry Dave. I'm afraid I can't do that.", flag)
while flag.isSet():
time.Clock().tick(10)
break
elif probStream.shape[0] > 10 and len(probStream.shape) >= 2:
if process != None:
os.kill(process.pid, signal.SIGTERM)
process = None
meanProbs = np.mean(probStream, axis=0)
pred_attr = mapAttributes(meanProbs > 0.6)
best = []
if meanProbs[0] > meanProbs[1] and meanProbs[0] > meanProbs[
4] and meanProbs[0] > meanProbs[2]:
best.append('Black_Hair')
elif meanProbs[1] > meanProbs[0] and meanProbs[1] > meanProbs[
4] and meanProbs[1] > meanProbs[2]:
class WebhookSender(Thread):
"""This handles sending embeds to one specific webhook. Obviously being a thread sub-class you should not add too many webhooks (in the order of the hundreds) for the same monitor.\n
You should not use this directly, but `WebhookManager` instead"""
def __init__(self, webhook: str, logconfig: LogConfig):
self.config = logconfig
self.webhook = webhook
self.logger = get_logger(logconfig)
self.queue = Queue() # type: Queue[Tuple[List[Any], List[Any], datetime]]
# contains the webhook config, embeds and time at which they were added
self.add_event = Event()
self.has_to_quit = False
super().__init__()
def add_to_queue(self, webhook_values: Any, embed: Embed):
now = datetime.now()
self.queue.put((webhook_values, embed, now))
self.add_event.set()
def is_done(self) -> bool:
return not self.add_event.isSet()
def quit(self):
self.has_to_quit = True
self.add_event.set()
def run(self):
while True:
self.add_event.wait()
if self.has_to_quit:
self.add_event.clear()
break
while not self.queue.empty():
webhook_values, embed, now = self.queue.get()
if "custom" in webhook_values:
provider = webhook_values["custom"].get(
"provider", self.config["WebhookConfig"]["provider"]
)
timestamp_format = webhook_values["custom"].get(
"timestamp_format",
self.config["WebhookConfig"]["timestamp_format"],
)
ts = now.strftime(timestamp_format)
icon_url = webhook_values["custom"].get(
"icon_url", self.config["WebhookConfig"]["provider_icon"]
)
color = webhook_values["custom"].get(
"color", int(self.config["WebhookConfig"]["embed_color"])
)
embed.set_footer(text=" | ".join([provider, ts]), icon_url=icon_url)
embed.color = color
else:
ts = now.strftime(self.config["WebhookConfig"]["timestamp_format"])
embed.set_footer(
text=f"{self.config['WebhookConfig']['provider']} | {ts}",
icon_url=self.config["WebhookConfig"]["provider_icon"],
)
embed.color = int(self.config["WebhookConfig"]["embed_color"])
embed.timestamp = Embed.Empty
data = {"embeds": [embed.to_dict()]}
if "custom" in webhook_values:
if "avatar_image" in webhook_values["custom"]:
data["avatar_url"] = webhook_values["custom"]["avatar_image"]
data = json.dumps(data)
while True:
r = requests.post(
self.webhook,
data=data,
headers={"Content-Type": "application/json"},
timeout=3,
)
self.logger.debug(
f"Posted to {self.webhook} with code {r.status_code}"
)
if "x-rateLimit-remaining" in r.headers:
remaining_requests = r.headers["x-rateLimit-remaining"]
if remaining_requests == "0":
delay = int(r.headers["x-rateLimit-reset-after"])
self.logger.debug(
f"No available requests reminaing for {self.webhook}, waiting {str(delay)} secs"
)
time.sleep(delay)
continue
if r.status_code == 429:
delay = int(r.headers["x-rateLimit-reset-after"])
self.logger.debug(
f"Got 429 for {self.webhook}, waiting {str(delay)} secs"
)
time.sleep(delay)
continue
else:
self.logger.warning(
f"Attention: {self.webhook} posted with {r.status_code} but it doesnt contain the rateLimit header; are you sure the webhook is correct???"
)
break
self.add_event.clear()
class BackendsCall(object):
def __init__(self, backends, function, *args, **kwargs):
"""
:param backends: List of backends to call
:type backends: list[:class:`BaseBackend`]
:param function: backends' method name, or callable object.
:type function: :class:`str` or :class:`callable`
"""
self.logger = getLogger('bcall')
# Store if a backend is finished
self.backends = {}
for backend in backends:
self.backends[backend.name] = False
# Global mutex on object
self.mutex = RLock()
# Event set when every backends have give their data
self.finish_event = Event()
# Event set when there are new responses
self.response_event = Event()
# Waiting responses
self.responses = []
# Errors
self.errors = []
# Threads
self.threads = []
# Create jobs for each backend
with self.mutex:
for backend in backends:
self.threads.append(
Timer(0, self._caller,
(backend, function, args, kwargs)).start())
if not backends:
self.finish_event.set()
def _store_error(self, backend, error):
with self.mutex:
backtrace = get_backtrace(error)
self.errors.append((backend, error, backtrace))
def _store_result(self, backend, result):
with self.mutex:
if isinstance(result, CapBaseObject):
result.backend = backend.name
self.responses.append((backend, result))
self.response_event.set()
def _caller(self, backend, function, args, kwargs):
with backend:
try:
# Call method on backend
try:
self.logger.debug('%s: Calling function %s' %
(backend, function))
if callable(function):
result = function(backend, *args, **kwargs)
else:
result = getattr(backend, function)(*args, **kwargs)
except Exception as error:
self.logger.debug(
'%s: Called function %s raised an error: %r' %
(backend, function, error))
self._store_error(backend, error)
else:
self.logger.debug('%s: Called function %s returned: %r' %
(backend, function, result))
if hasattr(result, '__iter__') and not isinstance(
result, basestring):
# Loop on iterator
try:
for subresult in result:
# Lock mutex only in loop in case the iterator is slow
# (for example if backend do some parsing operations)
self._store_result(backend, subresult)
except Exception as error:
self._store_error(backend, error)
else:
self._store_result(backend, result)
finally:
with self.mutex:
# This backend is now finished
self.backends[backend.name] = True
for finished in self.backends.itervalues():
if not finished:
return
self.response_event.set()
self.finish_event.set()
def _callback_thread_run(self, callback, errback):
responses = []
while not self.finish_event.isSet() or self.response_event.isSet():
self.response_event.wait()
with self.mutex:
responses = self.responses
self.responses = []
# Reset event
self.response_event.clear()
# Consume responses
while responses:
callback(*responses.pop(0))
if errback:
with self.mutex:
while self.errors:
errback(*self.errors.pop(0))
callback(None, None)
def callback_thread(self, callback, errback=None):
"""
Call this method to create a thread which will callback a
specified function everytimes a new result comes.
When the process is over, the function will be called with
both arguments set to None.
The functions prototypes:
def callback(backend, result)
def errback(backend, error)
"""
thread = Thread(target=self._callback_thread_run,
args=(callback, errback))
thread.start()
return thread
def wait(self):
self.finish_event.wait()
with self.mutex:
if self.errors:
raise CallErrors(self.errors)
def __iter__(self):
# Don't know how to factorize with _callback_thread_run
responses = []
while not self.finish_event.isSet() or self.response_event.isSet():
self.response_event.wait()
with self.mutex:
responses = self.responses
self.responses = []
# Reset event
self.response_event.clear()
# Consume responses
while responses:
yield responses.pop(0)
# Raise errors
with self.mutex:
if self.errors:
raise CallErrors(self.errors)
class WorkerBase(ABC):
def __init__(self,
args,
id,
last_known_state,
websocket_handler,
walker_routemanager,
devicesettings,
db_wrapper,
pogoWindowManager,
NoOcr=True,
walker=None):
# self.thread_pool = ThreadPool(processes=2)
self._walker_routemanager = walker_routemanager
self._route_manager_last_time = None
self._websocket_handler = websocket_handler
self._communicator = Communicator(websocket_handler, id, self,
args.websocket_command_timeout)
self._id = id
self._applicationArgs = args
self._last_known_state = last_known_state
self._work_mutex = Lock()
self.loop = None
self.loop_started = Event()
self.loop_tid = None
self._async_io_looper_thread = None
self._location_count = 0
self._init = self._walker_routemanager.init
self._walker = walker
self._lastScreenshotTaken = 0
self._stop_worker_event = Event()
self._db_wrapper = db_wrapper
self._redErrorCount = 0
self._lastScreenHash = None
self._lastScreenHashCount = 0
self._devicesettings = devicesettings
self._resocalc = Resocalculator
self._screen_x = 0
self._screen_y = 0
self._lastStart = ""
self._geofix_sleeptime = 0
self._pogoWindowManager = pogoWindowManager
self.current_location = Location(0.0, 0.0)
self.last_location = self._devicesettings.get("last_location", None)
if self.last_location is None:
self.last_location = Location(0.0, 0.0)
if self._devicesettings.get('last_mode', None) is not None and \
self._devicesettings['last_mode'] in ("raids_mitm", "mon_mitm", "iv_mitm", "raids_ocr"):
# Reset last_location - no useless waiting delays (otherwise stop mode)
logger.info('{}: last Mode not pokestop - reset saved location',
str(self._id))
self.last_location = Location(0.0, 0.0)
self._devicesettings['last_mode'] = self._walker_routemanager.mode
self.last_processed_location = Location(0.0, 0.0)
self.workerstart = None
self.workerstarttime = datetime.datetime.now()
def get_communicator(self):
return self._communicator
@abstractmethod
def _pre_work_loop(self):
"""
Work to be done before the main while true work-loop
Start off asyncio loops etc in here
:return:
"""
pass
@abstractmethod
def _health_check(self):
"""
Health check before a location is grabbed. Internally, a self._start_pogo call is already executed since
that usually includes a topmost check
:return:
"""
pass
@abstractmethod
def _pre_location_update(self):
"""
Override to run stuff like update injections settings in MITM worker
Runs before walk/teleport to the location previously grabbed
:return:
"""
pass
@abstractmethod
def _move_to_location(self):
"""
Location has previously been grabbed, the overriden function will be called.
You may teleport or walk by your choosing
Any post walk/teleport delays/sleeps have to be run in the derived, override method
:return:
"""
pass
@abstractmethod
def _post_move_location_routine(self, timestamp):
"""
Routine called after having moved to a new location. MITM worker e.g. has to wait_for_data
:param timestamp:
:return:
"""
@abstractmethod
def _start_pogo(self):
"""
Routine to start pogo.
Return the state as a boolean do indicate a successful start
:return:
"""
pass
@abstractmethod
def _cleanup(self):
"""
Cleanup any threads you started in derived classes etc
self.stop_worker() and self.loop.stop() will be called afterwards
:return:
"""
@abstractmethod
def _valid_modes(self):
"""
Return a list of valid modes for the health checks
:return:
"""
def _start_asyncio_loop(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
self.loop_tid = current_thread()
self.loop.call_soon(self.loop_started.set)
self.loop.run_forever()
def _add_task_to_loop(self, coro):
f = functools.partial(self.loop.create_task, coro)
if current_thread() == self.loop_tid:
# We can call directly if we're not going between threads.
return f()
else:
# We're in a non-event loop thread so we use a Future
# to get the task from the event loop thread once
# it's ready.
return self.loop.call_soon_threadsafe(f)
def start_worker(self):
# async_result = self.thread_pool.apply_async(self._main_work_thread, ())
t_main_work = Thread(target=self._main_work_thread)
t_main_work.daemon = False
t_main_work.start()
# do some other stuff in the main process
while not self._stop_worker_event.isSet():
time.sleep(1)
t_main_work.join()
logger.info("Worker {} stopping gracefully", str(self._id))
# async_result.get()
return self._last_known_state
def stop_worker(self):
self._stop_worker_event.set()
logger.warning("Worker {} stop called", str(self._id))
def _internal_pre_work(self):
current_thread().name = self._id
self._work_mutex.acquire()
try:
self._turn_screen_on_and_start_pogo()
except WebsocketWorkerRemovedException:
logger.error("Timeout during init of worker {}", str(self._id))
# no cleanup required here? TODO: signal websocket server somehow
self._stop_worker_event.set()
return
# register worker in routemanager
logger.info("Try to register {} in Routemanager {}", str(self._id),
str(self._walker_routemanager.name))
self._walker_routemanager.register_worker(self._id)
self._work_mutex.release()
self._async_io_looper_thread = Thread(name=str(self._id) +
'_asyncio_' + self._id,
target=self._start_asyncio_loop)
self._async_io_looper_thread.daemon = False
self._async_io_looper_thread.start()
self.loop_started.wait()
self._pre_work_loop()
def _internal_health_check(self):
# check if pogo is topmost and start if necessary
logger.debug(
"_internal_health_check: Calling _start_pogo routine to check if pogo is topmost"
)
self._work_mutex.acquire()
logger.debug("_internal_health_check: worker lock acquired")
logger.debug("Checking if we need to restart pogo")
# Restart pogo every now and then...
if self._devicesettings.get("restart_pogo", 80) > 0:
# logger.debug("main: Current time - lastPogoRestart: {}", str(curTime - lastPogoRestart))
# if curTime - lastPogoRestart >= (args.restart_pogo * 60):
if self._location_count > self._devicesettings.get(
"restart_pogo", 80):
logger.error(
"scanned " +
str(self._devicesettings.get("restart_pogo", 80)) +
" locations, restarting pogo")
pogo_started = self._restart_pogo()
self._location_count = 0
else:
pogo_started = self._start_pogo()
else:
pogo_started = self._start_pogo()
self._work_mutex.release()
logger.debug("_internal_health_check: worker lock released")
return pogo_started
def _internal_cleanup(self):
# set the event just to make sure - in case of exceptions for example
self._stop_worker_event.set()
logger.info("Internal cleanup of {} started", str(self._id))
self._cleanup()
logger.info("Internal cleanup of {} signalling end to websocketserver",
str(self._id))
self._walker_routemanager.unregister_worker(self._id)
logger.info("Stopping Route")
# self.stop_worker()
if self._async_io_looper_thread is not None:
logger.info("Stopping worker's asyncio loop")
self.loop.call_soon_threadsafe(self.loop.stop)
self._async_io_looper_thread.join()
self._communicator.cleanup_websocket()
logger.info("Internal cleanup of {} finished", str(self._id))
def _main_work_thread(self):
# TODO: signal websocketserver the removal
try:
self._internal_pre_work()
except (InternalStopWorkerException, WebsocketWorkerRemovedException,
WebsocketWorkerTimeoutException):
logger.error(
"Failed initializing worker {}, connection terminated exceptionally",
str(self._id))
self._internal_cleanup()
return
if not check_max_walkers_reached(self._walker,
self._walker_routemanager):
logger.warning('Max. Walkers in Area {} - closing connections',
str(self._walker_routemanager.name))
self._devicesettings['finished'] = True
self._internal_cleanup()
return
while not self._stop_worker_event.isSet():
try:
# TODO: consider getting results of health checks and aborting the entire worker?
walkercheck = self.check_walker()
if not walkercheck:
self._devicesettings['finished'] = True
break
except (InternalStopWorkerException,
WebsocketWorkerRemovedException,
WebsocketWorkerTimeoutException):
logger.warning("Worker {} killed by walker settings",
str(self._id))
break
try:
# TODO: consider getting results of health checks and aborting the entire worker?
self._internal_health_check()
self._health_check()
except (InternalStopWorkerException,
WebsocketWorkerRemovedException,
WebsocketWorkerTimeoutException):
logger.error(
"Websocket connection to {} lost while running healthchecks, connection terminated exceptionally",
str(self._id))
break
try:
settings = self._internal_grab_next_location()
if settings is None:
continue
except (InternalStopWorkerException,
WebsocketWorkerRemovedException,
WebsocketWorkerTimeoutException):
logger.warning(
"Worker of {} does not support mode that's to be run, connection terminated exceptionally",
str(self._id))
break
try:
logger.debug('Checking if new location is valid')
valid = self._check_location_is_valid()
if not valid:
break
except (InternalStopWorkerException,
WebsocketWorkerRemovedException,
WebsocketWorkerTimeoutException):
logger.warning("Worker {} get non valid coords!",
str(self._id))
break
try:
self._pre_location_update()
except (InternalStopWorkerException,
WebsocketWorkerRemovedException,
WebsocketWorkerTimeoutException):
logger.warning(
"Worker of {} stopping because of stop signal in pre_location_update, connection terminated exceptionally",
str(self._id))
break
try:
logger.debug(
'main worker {}: LastLat: {}, LastLng: {}, CurLat: {}, CurLng: {}',
str(self._id), self._devicesettings["last_location"].lat,
self._devicesettings["last_location"].lng,
self.current_location.lat, self.current_location.lng)
time_snapshot, process_location = self._move_to_location()
except (InternalStopWorkerException,
WebsocketWorkerRemovedException,
WebsocketWorkerTimeoutException):
logger.warning(
"Worker {} failed moving to new location, stopping worker, connection terminated exceptionally",
str(self._id))
break
if process_location:
self._add_task_to_loop(self._update_position_file())
self._location_count += 1
if self._applicationArgs.last_scanned:
logger.debug("Seting new 'scannedlocation' in Database")
# self.update_scanned_location(currentLocation.lat, currentLocation.lng, curTime)
self._add_task_to_loop(
self.update_scanned_location(self.current_location.lat,
self.current_location.lng,
time_snapshot))
try:
self._post_move_location_routine(time_snapshot)
except (InternalStopWorkerException,
WebsocketWorkerRemovedException,
WebsocketWorkerTimeoutException):
logger.warning(
"Worker {} failed running post_move_location_routine, stopping worker",
str(self._id))
break
logger.info("Worker {} finished iteration, continuing work",
str(self._id))
self._internal_cleanup()
async def _update_position_file(self):
logger.debug("Updating .position file")
if self.current_location is not None:
with open(
os.path.join(self._applicationArgs.file_path,
self._id + '.position'), 'w') as outfile:
outfile.write(
str(self.current_location.lat) + ", " +
str(self.current_location.lng))
async def update_scanned_location(self, latitude, longitude, timestamp):
try:
self._db_wrapper.set_scanned_location(str(latitude),
str(longitude),
str(timestamp))
except Exception as e:
logger.error("Failed updating scanned location: {}", str(e))
return
def check_walker(self):
mode = self._walker['walkertype']
if mode == "countdown":
logger.info("Checking walker mode 'countdown'")
countdown = self._walker['walkervalue']
if not countdown:
logger.error(
"No Value for Mode - check your settings! Killing worker")
return False
if self.workerstart is None:
self.workerstart = math.floor(time.time())
else:
if math.floor(
time.time()) >= int(self.workerstart) + int(countdown):
return False
return True
elif mode == "timer":
logger.debug("Checking walker mode 'timer'")
exittime = self._walker['walkervalue']
if not exittime or ':' not in exittime:
logger.error(
"No or wrong Value for Mode - check your settings! Killing worker"
)
return False
return check_walker_value_type(exittime, self.workerstarttime)
elif mode == "round":
logger.debug("Checking walker mode 'round'")
rounds = self._walker['walkervalue']
if len(rounds) == 0:
logger.error(
"No Value for Mode - check your settings! Killing worker")
return False
processed_rounds = self._walker_routemanager.get_rounds(self._id)
if int(processed_rounds) >= int(rounds):
return False
return True
elif mode == "period":
logger.debug("Checking walker mode 'period'")
period = self._walker['walkervalue']
if len(period) == 0:
logger.error(
"No Value for Mode - check your settings! Killing worker")
return False
return check_walker_value_type(period, self.workerstarttime)
elif mode == "coords":
exittime = self._walker['walkervalue']
if len(exittime) > 0:
return check_walker_value_type(exittime, self.workerstarttime)
return True
elif mode == "idle":
logger.debug("Checking walker mode 'idle'")
if len(self._walker['walkervalue']) == 0:
logger.error(
"Wrong Value for mode - check your settings! Killing worker"
)
return False
sleeptime = self._walker['walkervalue']
logger.info('{} going to sleep', str(self._id))
killpogo = False
if check_walker_value_type(sleeptime, self.workerstarttime):
self._stop_pogo()
killpogo = True
while not self._stop_worker_event.isSet(
) and check_walker_value_type(sleeptime, self.workerstarttime):
time.sleep(1)
logger.info('{} just woke up', str(self._id))
if killpogo:
self._start_pogo()
return False
else:
logger.error("Unknown walker mode! Killing worker")
return False
return True
def set_geofix_sleeptime(self, sleeptime):
self._geofix_sleeptime = sleeptime
return True
def _internal_grab_next_location(self):
# TODO: consider adding runWarningThreadEvent.set()
self._last_known_state["last_location"] = self.last_location
logger.debug("Requesting next location from routemanager")
# requesting a location is blocking (iv_mitm will wait for a prioQ item), we really need to clean
# the workers up...
if int(self._geofix_sleeptime) > 0:
logger.info(
'Getting a geofix position from MADMin - sleeping for {} seconds',
str(self._geofix_sleeptime))
time.sleep(int(self._geofix_sleeptime))
self._geofix_sleeptime = 0
routemanager = self._walker_routemanager
self.current_location = routemanager.get_next_location()
return routemanager.settings
def _init_routine(self):
if self._applicationArgs.initial_restart is False:
self._turn_screen_on_and_start_pogo()
else:
if not self._start_pogo():
while not self._restart_pogo():
logger.warning("failed starting pogo")
# TODO: stop after X attempts
def _check_location_is_valid(self):
if self.current_location is None:
# there are no more coords - so worker is finished successfully
self._devicesettings['finished'] = True
return None
elif self.current_location is not None:
logger.debug('Coords are valid')
return True
def _turn_screen_on_and_start_pogo(self):
if not self._communicator.isScreenOn():
self._communicator.startApp("de.grennith.rgc.remotegpscontroller")
logger.warning("Turning screen on")
self._communicator.turnScreenOn()
time.sleep(self._devicesettings.get("post_turn_screen_on_delay",
2))
# check if pogo is running and start it if necessary
logger.info("turnScreenOnAndStartPogo: (Re-)Starting Pogo")
self._start_pogo()
def _check_screen_on(self):
if not self._communicator.isScreenOn():
self._communicator.startApp("de.grennith.rgc.remotegpscontroller")
logger.warning("Turning screen on")
self._communicator.turnScreenOn()
time.sleep(self._devicesettings.get("post_turn_screen_on_delay",
2))
def _stop_pogo(self):
attempts = 0
stop_result = self._communicator.stopApp("com.nianticlabs.pokemongo")
pogoTopmost = self._communicator.isPogoTopmost()
while pogoTopmost:
attempts += 1
if attempts > 10:
return False
stop_result = self._communicator.stopApp(
"com.nianticlabs.pokemongo")
time.sleep(1)
pogoTopmost = self._communicator.isPogoTopmost()
return stop_result
def _reboot(self):
try:
start_result = self._communicator.reboot()
except WebsocketWorkerRemovedException:
logger.error(
"Could not reboot due to client already having disconnected")
start_result = False
time.sleep(5)
self._db_wrapper.save_last_reboot(self._id)
self.stop_worker()
return start_result
def _start_pogodroid(self):
start_result = self._communicator.startApp("com.mad.pogodroid")
time.sleep(5)
return start_result
def _stopPogoDroid(self):
stopResult = self._communicator.stopApp("com.mad.pogodroid")
return stopResult
def _restart_pogo(self, clear_cache=True):
successful_stop = self._stop_pogo()
self._db_wrapper.save_last_restart(self._id)
logger.debug("restartPogo: stop pogo resulted in {}",
str(successful_stop))
if successful_stop:
if clear_cache:
self._communicator.clearAppCache("com.nianticlabs.pokemongo")
time.sleep(1)
return self._start_pogo()
else:
return False
def _restartPogoDroid(self):
successfulStop = self._stopPogoDroid()
time.sleep(1)
logger.debug("restartPogoDroid: stop pogodriud resulted in {}",
str(successfulStop))
if successfulStop:
return self._start_pogodroid()
else:
return False
def _reopenRaidTab(self):
logger.debug("_reopenRaidTab: Taking screenshot...")
logger.info(
"reopenRaidTab: Attempting to retrieve screenshot before checking raidtab"
)
if not self._takeScreenshot():
logger.debug("_reopenRaidTab: Failed getting screenshot...")
logger.error(
"reopenRaidTab: Failed retrieving screenshot before checking for closebutton"
)
return
logger.debug("_reopenRaidTab: Checking close except nearby...")
pathToPass = os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id))
logger.debug("Path: {}", str(pathToPass))
self._pogoWindowManager.checkCloseExceptNearbyButton(
pathToPass, self._id, self._communicator, 'True')
logger.debug("_reopenRaidTab: Getting to raidscreen...")
self._getToRaidscreen(3)
time.sleep(1)
def _takeScreenshot(self, delayAfter=0.0, delayBefore=0.0):
logger.debug("Taking screenshot...")
time.sleep(delayBefore)
compareToTime = time.time() - self._lastScreenshotTaken
logger.debug("Last screenshot taken: {}",
str(self._lastScreenshotTaken))
if self._applicationArgs.use_media_projection:
take_screenshot = self._communicator.getScreenshot(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id)))
else:
take_screenshot = self._communicator.get_screenshot_single(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id)))
if self._lastScreenshotTaken and compareToTime < 0.5:
logger.debug(
"takeScreenshot: screenshot taken recently, returning immediately"
)
logger.debug("Screenshot taken recently, skipping")
return True
# TODO: screenshot.png needs identifier in name
elif not take_screenshot:
logger.error("takeScreenshot: Failed retrieving screenshot")
logger.debug("Failed retrieving screenshot")
return False
else:
logger.debug("Success retrieving screenshot")
self._lastScreenshotTaken = time.time()
time.sleep(delayAfter)
return True
def _checkPogoFreeze(self):
logger.debug("Checking if pogo froze")
if not self._takeScreenshot():
logger.debug("_checkPogoFreeze: failed retrieving screenshot")
return
from utils.image_utils import getImageHash
screenHash = getImageHash(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id)))
logger.debug("checkPogoFreeze: Old Hash: {}",
str(self._lastScreenHash))
logger.debug("checkPogoFreeze: New Hash: {}", str(screenHash))
if hamming_dist(str(self._lastScreenHash),
str(screenHash)) < 4 and str(
self._lastScreenHash) != '0':
logger.debug(
"checkPogoFreeze: New und old Screenshoot are the same - no processing"
)
self._lastScreenHashCount += 1
logger.debug("checkPogoFreeze: Same Screen Count: " +
str(self._lastScreenHashCount))
if self._lastScreenHashCount >= 100:
self._lastScreenHashCount = 0
self._restart_pogo()
else:
self._lastScreenHash = screenHash
self._lastScreenHashCount = 0
logger.debug("_checkPogoFreeze: done")
def _check_pogo_main_screen(self, maxAttempts, again=False):
logger.debug(
"_check_pogo_main_screen: Trying to get to the Mainscreen with {} max attempts...",
str(maxAttempts))
pogoTopmost = self._communicator.isPogoTopmost()
if not pogoTopmost:
return False
if not self._takeScreenshot(delayBefore=self._devicesettings.get(
"post_screenshot_delay", 1)):
if again:
logger.error(
"_check_pogo_main_screen: failed getting a screenshot again"
)
return False
attempts = 0
if os.path.isdir(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id))):
logger.error(
"_check_pogo_main_screen: screenshot.png is not a file/corrupted"
)
return False
logger.info("_check_pogo_main_screen: checking mainscreen")
buttoncheck = self._pogoWindowManager.lookForButton(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id)), 2.20, 3.01,
self._communicator)
if buttoncheck:
logger.info('Found button on screen')
self._takeScreenshot(delayBefore=self._devicesettings.get(
"post_screenshot_delay", 1))
while not self._pogoWindowManager.checkpogomainscreen(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id)), self._id):
logger.error("_check_pogo_main_screen: not on Mainscreen...")
if attempts > maxAttempts:
# could not reach raidtab in given maxAttempts
logger.error(
"_check_pogo_main_screen: Could not get to Mainscreen within {} attempts",
str(maxAttempts))
return False
# not using continue since we need to get a screen before the next round...
found = self._pogoWindowManager.lookForButton(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id)), 2.20, 3.01,
self._communicator)
if found:
logger.info("_check_pogo_main_screen: Found button (small)")
if not found and self._pogoWindowManager.checkCloseExceptNearbyButton(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id)),
self._id,
self._communicator,
closeraid=True):
logger.info(
"_check_pogo_main_screen: Found (X) button (except nearby)"
)
found = True
if not found and self._pogoWindowManager.lookForButton(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id)), 1.05,
2.20, self._communicator):
logger.info("_check_pogo_main_screen: Found button (big)")
found = True
logger.info(
"_check_pogo_main_screen: Previous checks found popups: {}",
str(found))
self._takeScreenshot(delayBefore=self._devicesettings.get(
"post_screenshot_delay", 1))
attempts += 1
logger.info("_check_pogo_main_screen: done")
return True
def _checkPogoButton(self):
logger.debug("checkPogoButton: Trying to find buttons")
pogoTopmost = self._communicator.isPogoTopmost()
if not pogoTopmost:
return False
if not self._takeScreenshot(delayBefore=self._devicesettings.get(
"post_screenshot_delay", 1)):
# TODO: again?
# if again:
# logger.error("checkPogoButton: failed getting a screenshot again")
# return False
# TODO: throw?
logger.debug("checkPogoButton: Failed getting screenshot")
return False
attempts = 0
if os.path.isdir(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id))):
logger.error(
"checkPogoButton: screenshot.png is not a file/corrupted")
return False
logger.info("checkPogoButton: checking for buttons")
found = self._pogoWindowManager.lookForButton(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id)), 2.20, 3.01,
self._communicator)
if found:
time.sleep(1)
logger.info("checkPogoButton: Found button (small)")
logger.info("checkPogoButton: done")
return True
logger.info("checkPogoButton: done")
return False
def _checkPogoClose(self):
logger.debug("checkPogoClose: Trying to find closeX")
pogoTopmost = self._communicator.isPogoTopmost()
if not pogoTopmost:
return False
if not self._takeScreenshot(delayBefore=self._devicesettings.get(
"post_screenshot_delay", 1)):
# TODO: go again?
# if again:
# logger.error("checkPogoClose: failed getting a screenshot again")
# return False
# TODO: consider throwing?
logger.debug("checkPogoClose: Could not get screenshot")
return False
attempts = 0
if os.path.isdir(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id))):
logger.error(
"checkPogoClose: screenshot.png is not a file/corrupted")
return False
logger.info("checkPogoClose: checking for CloseX")
found = self._pogoWindowManager.checkCloseExceptNearbyButton(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id)), self._id,
self._communicator)
if found:
time.sleep(1)
logger.info("checkPogoClose: Found (X) button (except nearby)")
logger.info("checkPogoClose: done")
return True
logger.info("checkPogoClose: done")
return False
def _getToRaidscreen(self, maxAttempts, again=False):
# check for any popups (including post login OK)
logger.debug(
"getToRaidscreen: Trying to get to the raidscreen with {} max attempts...",
str(maxAttempts))
pogoTopmost = self._communicator.isPogoTopmost()
if not pogoTopmost:
return False
self._checkPogoFreeze()
if not self._takeScreenshot(delayBefore=self._devicesettings.get(
"post_screenshot_delay", 1)):
if again:
logger.error(
"getToRaidscreen: failed getting a screenshot again")
return False
self._getToRaidscreen(maxAttempts, True)
logger.debug("getToRaidscreen: Got screenshot, checking GPS")
attempts = 0
if os.path.isdir(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id))):
logger.error(
"getToRaidscreen: screenshot.png is not a file/corrupted")
return False
# TODO: replace self._id with device ID
while self._pogoWindowManager.isGpsSignalLost(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id)), self._id):
logger.debug("getToRaidscreen: GPS signal lost")
time.sleep(1)
self._takeScreenshot()
logger.warning("getToRaidscreen: GPS signal error")
self._redErrorCount += 1
if self._redErrorCount > 3:
logger.error(
"getToRaidscreen: Red error multiple times in a row, restarting"
)
self._redErrorCount = 0
self._restart_pogo()
return False
self._redErrorCount = 0
logger.debug("getToRaidscreen: checking raidscreen")
while not self._pogoWindowManager.checkRaidscreen(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id)), self._id,
self._communicator):
logger.debug("getToRaidscreen: not on raidscreen...")
if attempts > maxAttempts:
# could not reach raidtab in given maxAttempts
logger.error(
"getToRaidscreen: Could not get to raidtab within {} attempts",
str(maxAttempts))
return False
self._checkPogoFreeze()
# not using continue since we need to get a screen before the next round...
found = self._pogoWindowManager.lookForButton(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id)), 2.20, 3.01,
self._communicator)
if found:
logger.info("getToRaidscreen: Found button (small)")
if not found and self._pogoWindowManager.checkCloseExceptNearbyButton(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id)), self._id,
self._communicator):
logger.info(
"getToRaidscreen: Found (X) button (except nearby)")
found = True
if not found and self._pogoWindowManager.lookForButton(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id)), 1.05,
2.20, self._communicator):
logger.info("getToRaidscreen: Found button (big)")
found = True
logger.info("getToRaidscreen: Previous checks found popups: {}",
str(found))
if not found:
logger.info(
"getToRaidscreen: Previous checks found nothing. Checking nearby open"
)
if self._pogoWindowManager.checkNearby(
os.path.join(self._applicationArgs.temp_path,
'screenshot%s.png' % str(self._id)),
self._id, self._communicator):
return self._takeScreenshot(
delayBefore=self._devicesettings.get(
"post_screenshot_delay", 1))
if not self._takeScreenshot(delayBefore=self._devicesettings.get(
"post_screenshot_delay", 1)):
return False
attempts += 1
logger.debug("getToRaidscreen: done")
return True
def _get_screen_size(self):
screen = self._communicator.getscreensize().split(' ')
self._screen_x = screen[0]
self._screen_y = screen[1]
x_offset = self._devicesettings.get("screenshot_x_offset", 0)
y_offset = self._devicesettings.get("screenshot_y_offset", 0)
logger.debug(
'Get Screensize of {}: X: {}, Y: {}, X-Offset: {}, Y-Offset: {}',
str(self._id), str(self._screen_x), str(self._screen_y),
str(x_offset), str(y_offset))
self._resocalc.get_x_y_ratio(self, self._screen_x, self._screen_y,
x_offset, y_offset)
class EventGhostEvent(object):
"""
.. attribute:: string
This is the full qualified event string as you see it inside the
logger, with the exception that if the payload field
(that is explained below) is not None the logger will also show it
behind the event string, but this is not a part of the event string
we are talking about here.
.. attribute:: payload
A plugin might publish additional data related to this event.
Through payload you can access this data. For example the 'Network
Event Receiver' plugin returns also the IP of the client that has
generated the event. If there is no data, this field is ``None``.
.. attribute:: prefix
This is the first part of the event string till the first dot. This
normally identifies the source of the event as a short string.
.. attribute:: suffix
This is the part of the event string behind the first dot. So you
could say:
event.string = event.prefix + '.' + event.suffix
.. attribute:: time
The time the event was generated as a floating point number in
seconds (as returned by the clock() function of Python's time module).
Since most events are processed very quickly, this is most likely
nearly the current time. But in some situations it might be more
clever to use this time, instead of the current time, since even
small differences might matter (for example if you want to determine
a double-press).
.. attribute:: isEnded
This boolean value indicates if the event is an enduring event and is
still active. Some plugins (e.g. most of the remote receiver plugins)
indicate if a button is pressed longer. As long as the button is
pressed, this flag is ``False`` and in the moment the user releases the
button the flag turns to ``True``. So you can poll this flag to see, if
the button is still pressed.
"""
skipEvent = False
def __init__(self, suffix="", payload=None, prefix="Main", source=eg):
self.string = prefix + "." + suffix
self.prefix = prefix
self.suffix = suffix
self.payload = payload
self.source = source
self.time = clock()
self.isEnded = False
self.shouldEnd = Event()
self.upFuncList = []
def AddUpFunc(self, func, *args, **kwargs):
if self.isEnded:
func(*args, **kwargs)
else:
self.upFuncList.append((func, args, kwargs))
def DoUpFuncs(self):
for func, args, kwargs in self.upFuncList:
func(*args, **kwargs)
del self.upFuncList[:]
self.isEnded = True
def Execute(self):
#start = clock()
eventString = self.string
if eventString in eg.notificationHandlers:
for listener in eg.notificationHandlers[eventString].listeners:
if listener(self) is True:
return
eg.event = self
eg.eventString = eventString
eventHandlerList = []
for key, val in eg.eventTable.iteritems():
if (
eventString == key or
(("*" in key or "?" in key) and fnmatchcase(eventString, key))
):
eventHandlerList += val
activeHandlers = set()
for eventHandler in eventHandlerList:
obj = eventHandler
while obj:
if not obj.isEnabled:
break
obj = obj.parent
else:
activeHandlers.add(eventHandler)
for listener in eg.log.eventListeners:
listener.LogEvent(self)
if config.onlyLogAssigned and len(activeHandlers) == 0:
self.SetStarted()
return
# show the event in the logger
LogEvent(self)
activeHandlers = sorted(activeHandlers, key=GetItemPath)
eg.SetProcessingState(2, self)
for eventHandler in activeHandlers:
try:
eg.programCounter = (eventHandler.parent, None)
eg.indent = 1
RunProgram()
except:
eg.PrintTraceback()
if self.skipEvent:
break
self.SetStarted()
eg.SetProcessingState(1, self)
def SetShouldEnd(self):
if not self.shouldEnd.isSet():
self.shouldEnd.set()
eg.SetProcessingState(0, self)
actionThread.Call(self.DoUpFuncs)
def SetStarted(self):
if self.shouldEnd.isSet():
self.DoUpFuncs()
class Scheduler(Queue):
def __init__(self, nworkers, maxlarge, logger, email, branch):
Queue.__init__(self)
info('Initializing scheduler with %d jobs.', nworkers)
self.nworkers = nworkers
self.logger = logger
self.maxlarge = maxlarge
self.nlarge = 0 # not thread safe, don't really care right now
self.passed = 0
self.failed = 0
self.workers = []
self.stopping = Event()
self.timer = None
self.error = None
self.email = email
self.branch = branch
def run(self, timeout):
info('Starting workers.')
self.stopping.clear()
for i in range(self.nworkers):
w = Worker(self)
self.workers.append(w)
w.start()
if timeout != 0:
self.timer = Timer(timeout, self.stop)
self.timer.start()
while not self.stopping.isSet():
try:
for w in self.workers:
if self.stopping.isSet():
break
w.join(timeout=1.0)
except (KeyboardInterrupt, SystemExit):
debug('Scheduler interrupted. Stopping and joining threads.')
self.stop()
self.join()
sys.exit(0)
else:
debug('Scheduler stopped by someone else. Joining threads.')
self.join()
if self.error:
send_mail(
self.email,
'Stress tests scheduler stopped by something, on %s' %
gethostname(), self.error)
sys.exit(77)
def join(self):
if self.timer is not None:
self.timer.cancel()
while len(self.workers) > 0:
self.workers.pop().join()
def stop(self):
info('Stopping workers.')
self.stopping.set()
def __getitem__(self, k):
return self.__dict__[k]
def reportstr(self):
return '[PASS=%(passed)d FAIL=%(failed)d]' % self
def report_success(self, runner):
self.passed += 1
self.logger.info('PASSED %s', runner.infostr())
info('%s PASSED %s', self.reportstr(), runner.infostr())
def report_failure(self, runner):
self.failed += 1
self.logger.warning('FAILED %s', runner.infostr())
warning('%s FAILED %s', self.reportstr(), runner.infostr())
def email_failure(self, runner, savedtarfile, commands, output):
if self.email is None:
return
h = gethostname()
if isinstance(runner, UpgradeTestRunnerMixin):
upgradestr = '''
The test was upgrading from %s.''' % runner.oldversionstr
else:
upgradestr = ''
send_mail(
self.email,
'Stress test failure on %(hostname)s running %(branch)s.' % {
'hostname': h,
'branch': self.branch
},
('''A stress test failed on %(hostname)s running %(branch)s at revision %(rev)s after %(test_duration)d seconds.%(upgradestr)s
Its environment is saved to %(tarfile)s on that machine.
The test configuration was:
testname: %(execf)s
num_elements: %(tsize)d
cachetable_size: %(csize)d
num_ptquery_threads: %(num_ptquery)d
num_update_threads: %(num_update)d
Commands run:
%(commands)s
Test output:
%(output)s
''' % {
'hostname': h,
'rev': runner.rev,
'test_duration': runner.time,
'upgradestr': upgradestr,
'tarfile': savedtarfile,
'execf': runner.execf,
'tsize': runner.tsize,
'csize': runner.csize,
'num_ptquery': runner.num_ptquery,
'num_update': runner.num_update,
'branch': self.branch,
'commands': commands,
'output': output,
}))
class AbstractDCS(object):
_INITIALIZE = 'initialize'
_CONFIG = 'config'
_LEADER = 'leader'
_FAILOVER = 'failover'
_HISTORY = 'history'
_MEMBERS = 'members/'
_OPTIME = 'optime'
_LEADER_OPTIME = _OPTIME + '/' + _LEADER
_SYNC = 'sync'
def __init__(self, config):
"""
:param config: dict, reference to config section of selected DCS.
i.e.: `zookeeper` for zookeeper, `etcd` for etcd, etc...
"""
self._name = config['name']
self._base_path = re.sub('/+', '/', '/'.join(['', config.get('namespace', 'service'), config['scope']]))
self._set_loop_wait(config.get('loop_wait', 10))
self._ctl = bool(config.get('patronictl', False))
self._cluster = None
self._cluster_valid_till = 0
self._cluster_thread_lock = Lock()
self._last_leader_operation = ''
self.event = Event()
def client_path(self, path):
return '/'.join([self._base_path, path.lstrip('/')])
@property
def initialize_path(self):
return self.client_path(self._INITIALIZE)
@property
def config_path(self):
return self.client_path(self._CONFIG)
@property
def members_path(self):
return self.client_path(self._MEMBERS)
@property
def member_path(self):
return self.client_path(self._MEMBERS + self._name)
@property
def leader_path(self):
return self.client_path(self._LEADER)
@property
def failover_path(self):
return self.client_path(self._FAILOVER)
@property
def history_path(self):
return self.client_path(self._HISTORY)
@property
def leader_optime_path(self):
return self.client_path(self._LEADER_OPTIME)
@property
def sync_path(self):
return self.client_path(self._SYNC)
@abc.abstractmethod
def set_ttl(self, ttl):
"""Set the new ttl value for leader key"""
@abc.abstractmethod
def ttl(self):
"""Get new ttl value"""
@abc.abstractmethod
def set_retry_timeout(self, retry_timeout):
"""Set the new value for retry_timeout"""
def _set_loop_wait(self, loop_wait):
self._loop_wait = loop_wait
def reload_config(self, config):
self._set_loop_wait(config['loop_wait'])
self.set_ttl(config['ttl'])
self.set_retry_timeout(config['retry_timeout'])
@property
def loop_wait(self):
return self._loop_wait
@abc.abstractmethod
def _load_cluster(self):
"""Internally this method should build `Cluster` object which
represents current state and topology of the cluster in DCS.
this method supposed to be called only by `get_cluster` method.
raise `~DCSError` in case of communication or other problems with DCS.
If the current node was running as a master and exception raised,
instance would be demoted."""
def get_cluster(self):
try:
cluster = self._load_cluster()
except Exception:
self.reset_cluster()
raise
with self._cluster_thread_lock:
self._cluster = cluster
self._cluster_valid_till = time.time() + self.ttl
return cluster
@property
def cluster(self):
with self._cluster_thread_lock:
return self._cluster if self._cluster_valid_till > time.time() else None
def reset_cluster(self):
with self._cluster_thread_lock:
self._cluster = None
self._cluster_valid_till = 0
@abc.abstractmethod
def _write_leader_optime(self, last_operation):
"""write current xlog location into `/optime/leader` key in DCS
:param last_operation: absolute xlog location in bytes
:returns: `!True` on success."""
def write_leader_optime(self, last_operation):
if self._last_leader_operation != last_operation and self._write_leader_optime(last_operation):
self._last_leader_operation = last_operation
@abc.abstractmethod
def _update_leader(self):
"""Update leader key (or session) ttl
:returns: `!True` if leader key (or session) has been updated successfully.
If not, `!False` must be returned and current instance would be demoted.
You have to use CAS (Compare And Swap) operation in order to update leader key,
for example for etcd `prevValue` parameter must be used."""
def update_leader(self, last_operation, access_is_restricted=False):
"""Update leader key (or session) ttl and optime/leader
:param last_operation: absolute xlog location in bytes
:returns: `!True` if leader key (or session) has been updated successfully.
If not, `!False` must be returned and current instance would be demoted."""
ret = self._update_leader()
if ret and last_operation:
self.write_leader_optime(last_operation)
return ret
@abc.abstractmethod
def attempt_to_acquire_leader(self, permanent=False):
"""Attempt to acquire leader lock
This method should create `/leader` key with value=`~self._name`
:param permanent: if set to `!True`, the leader key will never expire.
Used in patronictl for the external master
:returns: `!True` if key has been created successfully.
Key must be created atomically. In case if key already exists it should not be
overwritten and `!False` must be returned"""
@abc.abstractmethod
def set_failover_value(self, value, index=None):
"""Create or update `/failover` key"""
def manual_failover(self, leader, candidate, scheduled_at=None, index=None):
failover_value = {}
if leader:
failover_value['leader'] = leader
if candidate:
failover_value['member'] = candidate
if scheduled_at:
failover_value['scheduled_at'] = scheduled_at.isoformat()
return self.set_failover_value(json.dumps(failover_value, separators=(',', ':')), index)
@abc.abstractmethod
def set_config_value(self, value, index=None):
"""Create or update `/config` key"""
@abc.abstractmethod
def touch_member(self, data, permanent=False):
"""Update member key in DCS.
This method should create or update key with the name = '/members/' + `~self._name`
and value = data in a given DCS.
:param data: information about instance (including connection strings)
:param ttl: ttl for member key, optional parameter. If it is None `~self.member_ttl will be used`
:param permanent: if set to `!True`, the member key will never expire.
Used in patronictl for the external master.
:returns: `!True` on success otherwise `!False`
"""
@abc.abstractmethod
def take_leader(self):
"""This method should create leader key with value = `~self._name` and ttl=`~self.ttl`
Since it could be called only on initial cluster bootstrap it could create this key regardless,
overwriting the key if necessary."""
@abc.abstractmethod
def initialize(self, create_new=True, sysid=""):
"""Race for cluster initialization.
:param create_new: False if the key should already exist (in the case we are setting the system_id)
:param sysid: PostgreSQL cluster system identifier, if specified, is written to the key
:returns: `!True` if key has been created successfully.
this method should create atomically initialize key and return `!True`
otherwise it should return `!False`"""
@abc.abstractmethod
def delete_leader(self):
"""Voluntarily remove leader key from DCS
This method should remove leader key if current instance is the leader"""
@abc.abstractmethod
def cancel_initialization(self):
""" Removes the initialize key for a cluster """
@abc.abstractmethod
def delete_cluster(self):
"""Delete cluster from DCS"""
@staticmethod
def sync_state(leader, sync_standby):
"""Build sync_state dict"""
return {'leader': leader, 'sync_standby': sync_standby}
def write_sync_state(self, leader, sync_standby, index=None):
sync_value = self.sync_state(leader, sync_standby)
return self.set_sync_state_value(json.dumps(sync_value, separators=(',', ':')), index)
@abc.abstractmethod
def set_history_value(self, value):
""""""
@abc.abstractmethod
def set_sync_state_value(self, value, index=None):
""""""
@abc.abstractmethod
def delete_sync_state(self, index=None):
""""""
def watch(self, leader_index, timeout):
"""If the current node is a master it should just sleep.
Any other node should watch for changes of leader key with a given timeout
:param leader_index: index of a leader key
:param timeout: timeout in seconds
:returns: `!True` if you would like to reschedule the next run of ha cycle"""
self.event.wait(timeout)
return self.event.isSet()
class __CardMonitoringThreadSingleton(Thread):
"""The real card monitoring thread class.
A single instance of this class is created
by the public CardMonitoringThread class.
"""
def __init__(self, observable):
Thread.__init__(self)
self.observable = observable
self.stopEvent = Event()
self.stopEvent.clear()
self.cards = []
self.setDaemon(True)
# the actual monitoring thread
def run(self):
"""Runs until stopEvent is notified, and notify
observers of all card insertion/removal.
"""
self.cardrequest = CardRequest(timeout=0.1)
while self.stopEvent.isSet() != 1:
try:
currentcards = self.cardrequest.waitforcardevent()
addedcards = []
for card in currentcards:
if not self.cards.__contains__(card):
addedcards.append(card)
removedcards = []
for card in self.cards:
if not currentcards.__contains__(card):
removedcards.append(card)
if addedcards != [] or removedcards != []:
self.cards = currentcards
self.observable.setChanged()
self.observable.notifyObservers(
(addedcards, removedcards))
# when CardMonitoringThread.__del__() is invoked in
# response to shutdown, e.g., when execution of the
# program is done, other globals referenced by the
# __del__() method may already have been deleted.
# this causes ReaderMonitoringThread.run() to except
# with a TypeError or AttributeError
except TypeError:
pass
except AttributeError:
pass
except SmartcardException as exc:
# FIXME Tighten the exceptions caught by this block
traceback.print_exc()
# Most likely raised during interpreter shutdown due
# to unclean exit which failed to remove all observers.
# To solve this, we set the stop event and pass the
# exception to let the thread finish gracefully.
if exc.hresult == SCARD_E_NO_SERVICE:
self.stopEvent.set()
# stop the thread by signaling stopEvent
def stop(self):
self.stopEvent.set()
class PlotModeBase(PlotMode):
"""
Intended parent class for plotting
modes. Provides base functionality
in conjunction with its parent,
PlotMode.
"""
##
## Class-Level Attributes
##
"""
The following attributes are meant
to be set at the class level, and serve
as parameters to the plot mode registry
(in PlotMode). See plot_modes.py for
concrete examples.
"""
"""
i_vars
'x' for Cartesian2D
'xy' for Cartesian3D
etc.
d_vars
'y' for Cartesian2D
'r' for Polar
etc.
"""
i_vars, d_vars = '', ''
"""
intervals
Default intervals for each i_var, and in the
same order. Specified [min, max, steps].
No variable can be given (it is bound later).
"""
intervals = []
"""
aliases
A list of strings which can be used to
access this mode.
'cartesian' for Cartesian2D and Cartesian3D
'polar' for Polar
'cylindrical', 'polar' for Cylindrical
Note that _init_mode chooses the first alias
in the list as the mode's primary_alias, which
will be displayed to the end user in certain
contexts.
"""
aliases = []
"""
is_default
Whether to set this mode as the default
for arguments passed to PlotMode() containing
the same number of d_vars as this mode and
at most the same number of i_vars.
"""
is_default = False
"""
All of the above attributes are defined in PlotMode.
The following ones are specific to PlotModeBase.
"""
"""
A list of the render styles. Do not modify.
"""
styles = {'wireframe': 1, 'solid': 2, 'both': 3}
"""
style_override
Always use this style if not blank.
"""
style_override = ''
"""
default_wireframe_color
default_solid_color
Can be used when color is None or being calculated.
Used by PlotCurve and PlotSurface, but not anywhere
in PlotModeBase.
"""
default_wireframe_color = (0.85, 0.85, 0.85)
default_solid_color = (0.6, 0.6, 0.9)
default_rot_preset = 'xy'
##
## Instance-Level Attributes
##
## 'Abstract' member functions
def _get_evaluator(self):
if self.use_lambda_eval:
try:
e = self._get_lambda_evaluator()
return e
except Exception:
warnings.warn("\nWarning: creating lambda evaluator failed. "
"Falling back on sympy subs evaluator.")
return self._get_sympy_evaluator()
def _get_sympy_evaluator(self):
raise NotImplementedError()
def _get_lambda_evaluator(self):
raise NotImplementedError()
def _on_calculate_verts(self):
raise NotImplementedError()
def _on_calculate_cverts(self):
raise NotImplementedError()
## Base member functions
def __init__(self, *args, **kwargs):
self.verts = []
self.cverts = []
self.bounds = [[S.Infinity, -S.Infinity, 0],
[S.Infinity, -S.Infinity, 0],
[S.Infinity, -S.Infinity, 0]]
self.cbounds = [[S.Infinity, -S.Infinity, 0],
[S.Infinity, -S.Infinity, 0],
[S.Infinity, -S.Infinity, 0]]
self._draw_lock = RLock()
self._calculating_verts = Event()
self._calculating_cverts = Event()
self._calculating_verts_pos = 0.0
self._calculating_verts_len = 0.0
self._calculating_cverts_pos = 0.0
self._calculating_cverts_len = 0.0
self._max_render_stack_size = 3
self._draw_wireframe = [-1]
self._draw_solid = [-1]
self._style = None
self._color = None
self.predraw = []
self.postdraw = []
self.use_lambda_eval = self.options.pop('use_sympy_eval', None) is None
self.style = self.options.pop('style', '')
self.color = self.options.pop('color', 'rainbow')
self.bounds_callback = kwargs.pop('bounds_callback', None)
self._on_calculate()
def synchronized(f):
def w(self, *args, **kwargs):
self._draw_lock.acquire()
try:
r = f(self, *args, **kwargs)
return r
finally:
self._draw_lock.release()
return w
@synchronized
def push_wireframe(self, function):
"""
Push a function which performs gl commands
used to build a display list. (The list is
built outside of the function)
"""
assert callable(function)
self._draw_wireframe.append(function)
if len(self._draw_wireframe) > self._max_render_stack_size:
del self._draw_wireframe[1] # leave marker element
@synchronized
def push_solid(self, function):
"""
Push a function which performs gl commands
used to build a display list. (The list is
built outside of the function)
"""
assert callable(function)
self._draw_solid.append(function)
if len(self._draw_solid) > self._max_render_stack_size:
del self._draw_solid[1] # leave marker element
def _create_display_list(self, function):
dl = glGenLists(1)
glNewList(dl, GL_COMPILE)
function()
glEndList()
return dl
def _render_stack_top(self, render_stack):
top = render_stack[-1]
if top == -1:
return -1 # nothing to display
elif callable(top):
dl = self._create_display_list(top)
render_stack[-1] = (dl, top)
return dl # display newly added list
elif len(top) == 2:
if GL_TRUE == glIsList(top[0]):
return top[0] # display stored list
dl = self._create_display_list(top[1])
render_stack[-1] = (dl, top[1])
return dl # display regenerated list
def _draw_solid_display_list(self, dl):
glPushAttrib(GL_ENABLE_BIT | GL_POLYGON_BIT)
glPolygonMode(GL_FRONT_AND_BACK, GL_FILL)
glCallList(dl)
glPopAttrib()
def _draw_wireframe_display_list(self, dl):
glPushAttrib(GL_ENABLE_BIT | GL_POLYGON_BIT)
glPolygonMode(GL_FRONT_AND_BACK, GL_LINE)
glEnable(GL_POLYGON_OFFSET_LINE)
glPolygonOffset(-0.005, -50.0)
glCallList(dl)
glPopAttrib()
@synchronized
def draw(self):
for f in self.predraw:
if callable(f):
f()
if self.style_override:
style = self.styles[self.style_override]
else:
style = self.styles[self._style]
# Draw solid component if style includes solid
if style & 2:
dl = self._render_stack_top(self._draw_solid)
if dl > 0 and GL_TRUE == glIsList(dl):
self._draw_solid_display_list(dl)
# Draw wireframe component if style includes wireframe
if style & 1:
dl = self._render_stack_top(self._draw_wireframe)
if dl > 0 and GL_TRUE == glIsList(dl):
self._draw_wireframe_display_list(dl)
for f in self.postdraw:
if callable(f):
f()
def _on_change_color(self, color):
Thread(target=self._calculate_cverts).start()
def _on_calculate(self):
Thread(target=self._calculate_all).start()
def _calculate_all(self):
self._calculate_verts()
self._calculate_cverts()
def _calculate_verts(self):
if self._calculating_verts.isSet():
return
self._calculating_verts.set()
try:
self._on_calculate_verts()
finally:
self._calculating_verts.clear()
if callable(self.bounds_callback):
self.bounds_callback()
def _calculate_cverts(self):
if self._calculating_verts.isSet():
return
while self._calculating_cverts.isSet():
sleep(0) # wait for previous calculation
self._calculating_cverts.set()
try:
self._on_calculate_cverts()
finally:
self._calculating_cverts.clear()
def _get_calculating_verts(self):
return self._calculating_verts.isSet()
def _get_calculating_verts_pos(self):
return self._calculating_verts_pos
def _get_calculating_verts_len(self):
return self._calculating_verts_len
def _get_calculating_cverts(self):
return self._calculating_cverts.isSet()
def _get_calculating_cverts_pos(self):
return self._calculating_cverts_pos
def _get_calculating_cverts_len(self):
return self._calculating_cverts_len
## Property handlers
def _get_style(self):
return self._style
@synchronized
def _set_style(self, v):
if v is None:
return
if v == '':
step_max = 0
for i in self.intervals:
if i.v_steps is None:
continue
step_max = max([step_max, int(i.v_steps)])
v = ['both', 'solid'][step_max > 40]
#try:
if v not in self.styles:
raise ValueError("v should be there in self.styles")
if v == self._style:
return
self._style = v
#except Exception as e:
#raise RuntimeError(("Style change failed. "
# "Reason: %s is not a valid "
# "style. Use one of %s.") %
# (str(v), ', '.join(self.styles.iterkeys())))
def _get_color(self):
return self._color
@synchronized
def _set_color(self, v):
try:
if v is not None:
if is_sequence(v):
v = ColorScheme(*v)
else:
v = ColorScheme(v)
if repr(v) == repr(self._color):
return
self._on_change_color(v)
self._color = v
except Exception as e:
raise RuntimeError(("Color change failed. "
"Reason: %s" % (str(e))))
style = property(_get_style, _set_style)
color = property(_get_color, _set_color)
calculating_verts = property(_get_calculating_verts)
calculating_verts_pos = property(_get_calculating_verts_pos)
calculating_verts_len = property(_get_calculating_verts_len)
calculating_cverts = property(_get_calculating_cverts)
calculating_cverts_pos = property(_get_calculating_cverts_pos)
calculating_cverts_len = property(_get_calculating_cverts_len)
## String representations
def __str__(self):
f = ", ".join(str(d) for d in self.d_vars)
o = "'mode=%s'" % (self.primary_alias)
return ", ".join([f, o])
def __repr__(self):
f = ", ".join(str(d) for d in self.d_vars)
i = ", ".join(str(i) for i in self.intervals)
d = [('mode', self.primary_alias),
('color', str(self.color)),
('style', str(self.style))]
o = "'%s'" % (("; ".join("%s=%s" % (k, v)
for k, v in d if v != 'None')))
return ", ".join([f, i, o])
class Client(object):
def __init__(self, sock=None):
"""
Initialize the client.
The socket parameter can be an already initialized socket. Should be
used to pass specialized socket objects like SSL sockets.
"""
if sock is None:
sock = self._sock = socket.socket()
else:
self._sock = sock
self._map = {}
# Because we don't know how large the frames are
# we have to use non bolocking IO
sock.setblocking(False)
# We have our own timeout for operations we
# pretend to be synchronous (like connect).
self._timeout = None
self._connected = Event()
self._subscriptions = {}
self._aclient = None
self._adisp = None
self._inbox = deque()
@property
def outgoing(self):
return self._adisp.outgoing
def _registerSubscription(self, sub):
self._subscriptions[sub.id] = sub
def _unregisterSubscription(self, sub):
del self._subscriptions[sub.id]
@property
def connected(self):
return self._connected.isSet()
def handle_connect(self, aclient, frame):
self._connected.set()
def handle_message(self, aclient, frame):
self._inbox.append(frame)
def process(self, timeout=None):
if timeout is None:
timeout = self._timeout
asyncore.loop(use_poll=True, timeout=timeout, map=self._map, count=1)
def connect(self, address, hostname):
sock = self._sock
self._aclient = AsyncClient(self, hostname)
adisp = self._adisp = AsyncDispatcher(self._aclient)
disp = self._disp = Dispatcher(adisp, sock, self._map)
sock.setblocking(True)
disp.connect(address)
sock.setblocking(False)
self.recv() # wait for CONNECTED msg
if not self._connected.isSet():
sock.close()
raise socket.error()
def recv(self):
timeout = self._timeout
s = monotonic_time()
duration = 0
while timeout is None or duration < timeout:
try:
return self._inbox.popleft()
except IndexError:
td = timeout - duration if timeout is not None else None
self.process(td)
duration = monotonic_time() - s
return None
def put_subscribe(self, destination, ack=None):
subid = self._aclient.subscribe(self._adisp, destination, ack)
sub = _Subscription(self, subid, ack)
self._registerSubscription(sub)
return sub
def put_send(self, destination, data="", headers=None):
self._aclient.send(self._adisp, destination, data, headers)
def put(self, frame):
self._adisp.send_raw(frame)
def send(self):
disp = self._disp
timeout = self._timeout
duration = 0
s = monotonic_time()
while ((timeout is None or duration < timeout)
and (disp.writable() or not self._connected.isSet())):
td = timeout - duration if timeout is not None else None
self.process(td)
duration = monotonic_time() - s
def gettimout(self):
return self._timeout
def settimeout(self, value):
self._timeout = value
class IdleThread(object):
def __init__(self, parent, folder=None):
"""If invoked without 'folder', perform a NOOP and wait for
self.stop() to be called. If invoked with folder, switch to IDLE
mode and synchronize once we have a new message"""
self.parent = parent
self.folder = folder
self.stop_sig = Event()
self.ui = getglobalui()
if folder is None:
self.thread = Thread(target=self.noop)
else:
self.thread = Thread(target=self.__idle)
self.thread.setDaemon(1)
def start(self):
self.thread.start()
def stop(self):
self.stop_sig.set()
def join(self):
self.thread.join()
def noop(self):
# TODO: AFAIK this is not optimal, we will send a NOOP on one
# random connection (ie not enough to keep all connections
# open). In case we do the noop multiple times, we can well use
# the same connection every time, as we get a random one. This
# function should IMHO send a noop on ALL available connections
# to the server.
imapobj = self.parent.acquireconnection()
try:
imapobj.noop()
except imapobj.abort:
self.ui.warn('Attempting NOOP on dropped connection %s' %
imapobj.identifier)
self.parent.releaseconnection(imapobj, True)
imapobj = None
finally:
if imapobj:
self.parent.releaseconnection(imapobj)
self.stop_sig.wait() # wait until we are supposed to quit
def __dosync(self):
remoterepos = self.parent.repos
account = remoterepos.account
localrepos = account.localrepos
remoterepos = account.remoterepos
statusrepos = account.statusrepos
remotefolder = remoterepos.getfolder(self.folder)
hook = account.getconf('presynchook', '')
account.callhook(hook)
offlineimap.accounts.syncfolder(account, remotefolder, quick=False)
hook = account.getconf('postsynchook', '')
account.callhook(hook)
ui = getglobalui()
ui.unregisterthread(currentThread()) #syncfolder registered the thread
def __idle(self):
"""Invoke IDLE mode until timeout or self.stop() is invoked."""
def callback(args):
"""IDLE callback function invoked by imaplib2.
This is invoked when a) The IMAP server tells us something
while in IDLE mode, b) we get an Exception (e.g. on dropped
connections, or c) the standard imaplib IDLE timeout of 29
minutes kicks in."""
result, cb_arg, exc_data = args
if exc_data is None and not self.stop_sig.isSet():
# No Exception, and we are not supposed to stop:
self.needsync = True
self.stop_sig.set() # Continue to sync.
def noop(imapobj):
"""Factorize the noop code."""
try:
# End IDLE mode with noop, imapobj can point to a dropped conn.
imapobj.noop()
except imapobj.abort:
self.ui.warn('Attempting NOOP on dropped connection %s' %
imapobj.identifier)
self.parent.releaseconnection(imapobj, True)
else:
self.parent.releaseconnection(imapobj)
while not self.stop_sig.isSet():
self.needsync = False
success = False # Successfully selected FOLDER?
while not success:
imapobj = self.parent.acquireconnection()
try:
imapobj.select(self.folder)
except OfflineImapError as e:
if e.severity == OfflineImapError.ERROR.FOLDER_RETRY:
# Connection closed, release connection and retry.
self.ui.error(e, exc_info()[2])
self.parent.releaseconnection(imapobj, True)
elif e.severity == OfflineImapError.ERROR.FOLDER:
# Just continue the process on such error for now.
self.ui.error(e, exc_info()[2])
else:
# Stops future attempts to sync this account.
raise
else:
success = True
if "IDLE" in imapobj.capabilities:
imapobj.idle(callback=callback)
else:
self.ui.warn("IMAP IDLE not supported on server '%s'."
"Sleep until next refresh cycle." %
imapobj.identifier)
noop(imapobj) #XXX: why?
self.stop_sig.wait() # self.stop() or IDLE callback are invoked.
noop(imapobj)
if self.needsync:
# Here not via self.stop, but because IDLE responded. Do
# another round and invoke actual syncing.
self.stop_sig.clear()
self.__dosync()
class Engine(object):
"""
A Engine starts working by starting a number of downloader threads and a number of parser threads.
It keeps crawling internet from a given set of seeds until being stopped.
"""
def __init__(self, nDownloader = DEFAULT_DOWNLOADERS, manager = DEFAULT_MANAGER, \
regPort = DEFAULT_REG_PORT, dbPort = DEFAULT_DB_PORT, urlPort = None, pagePort = None):
"""
Initialize a crawler object.
--------- Param --------
nDownloader (int):
the nubmer of downloader threads.
manager:
the host on which manager is started.
regPort:
the port on which manager expects connection requests.
urlPort:
the port on which this worker sends url to manager.
pagePort:
the port on which this worker sends page to manager.
--------- Return --------
None.
"""
## prepare the url frontier and page queue
self._pageQ = Queue(MAX_PAGE_QSIZE)
self._urlFrontier = Frontier(3*nDownloader, MAX_URL_QSIZE, \
keyFunc=lambda url: urllib2.Request(url).get_host(), \
priorityFunc=self.getLastVisitTime)
self._visitSite = {}
self._lock = RLock()
self._stopEvent = Event()
## prepare filters
filetypeFilter = urlFilter.FileTypeFilter(True, ['text/html'])
robotFilter = urlFilter.RobotFilter(Downloader.DEFAULT_USER_AGENT)
self._urlDupEliminator = urlFilter.DupEliminator()
self._urlFrontier.addFilter(filetypeFilter.disallow)
self._urlFrontier.addFilter(self._urlDupEliminator.seenBefore)
# self._urlFrontier.addFilter(robotFilter.disallow)
## initialize sockets.
self._manager = manager
self._regPort = regPort
self._urlPort = urlPort
self._thisHost = socket.gethostbyname(socket.gethostname())
self._dbclient = MongoClient(manager, dbPort)
context = zmq.Context()
self._regSocket = context.socket(zmq.REQ)
self._regSocket.connect("tcp://%s:%d" % (manager, self._regPort))
self._urlPushSocket = context.socket(zmq.PUSH)
self._urlPullSocket = context.socket(zmq.PULL)
if (self._urlPort is None):
self._urlPort = self._urlPullSocket.bind_to_random_port(
"tcp://%s" % self._thisHost)
else:
self._urlPullSocket.bind("tcp://*:%d" % (self._urlPort))
## prepare log files
if (not os.path.exists("log")):
os.makedirs("log")
self._logger = logging.getLogger("engine")
self._logger.addHandler(
logging.FileHandler(
os.path.abspath("log/engine%d.log" % self._urlPort)))
self._logger.setLevel(logging.WARNING)
parseLogger = logging.getLogger("parser")
parseLogger.addHandler(
logging.FileHandler(os.path.abspath("log/parser.log")))
parseLogger.setLevel(logging.WARNING)
## create threads for downloading and parsing tasks
self._downloaders = []
for i in range(nDownloader):
downloader = Downloader(self._urlFrontier, self._pageQ,
self._logger, self.updateLastVisitTime)
downloader.daemon = True
self._downloaders.append(downloader)
self._parser = Parser(self._pageQ, self._urlPushSocket, self._dbclient,
parseLogger)
def log(self, level, msg):
"""
Log info/warning/error message in log file.
"""
if (self._logger is not None):
if level == logging.INFO:
self._logger.info("[%s] INFO: %s" % (time.ctime(), msg))
elif level == logging.WARNING:
self._logger.warn("[%s] WARNING: %s" % (time.ctime(), msg))
else:
self._logger.error("[%s] ERROR: %s" % (tme.ctime(), msg))
def _register(self):
"""
Request connection to master.
"""
self._regSocket.send("REG %s %d" % (self._thisHost, self._urlPort))
response = self._regSocket.recv()
managerurlPort = response.split()[1]
self.log(logging.INFO,
"success to connect to manager:%s" % managerurlPort)
self._urlPushSocket.connect("tcp://%s:%s" %
(self._manager, managerurlPort))
def start(self):
"""
Start crawling.
"""
print "[%s] : Crawler is started!" % time.ctime()
self._register()
self._dataAcceptor = Thread(target=self._recvData)
self._dataAcceptor.daemon = True
self._dataAcceptor.start()
for downloader in self._downloaders:
downloader.start()
self._parser.start()
def _recvData(self):
"""
Keeps listening to urlPort.
Process each arrival data.
"""
while (not self._stopEvent.isSet()):
urlSet = self._urlPullSocket.recv_pyobj()
self.log(logging.INFO, "received %s" % urlSet)
for url in urlSet:
try:
self._urlFrontier.put(url)
except Full:
time.sleep(5)
def stop(self):
"""
Stop crawling.
"""
print "[%s] : Crawler is stopped!" % time.ctime()
self._stopEvent.set()
self._parser.stop()
self._parser.join()
self._dump()
self._regSocket.send("UNREG %s %d" % (self._thisHost, self._urlPort))
response = self._regSocket.recv()
self.log(logging.INFO, "received %s" % response)
def _dump(self):
"""
Dump the ready_to_be_crawled urls to log file.
"""
total = self._urlDupEliminator.size()
left = self._urlFrontier.size()
print "%d url discovered, but only %d downloaded and %d ready for downloading." % (
total, total - left, left)
unvisited = set()
while (self._urlFrontier.size() > 0):
unvisited.add(self._urlFrontier.get())
self._urlPushSocket.send_pyobj(unvisited)
unvisited.clear()
def getLastVisitTime(self, site):
"""
Return the last time a site was visited.
"""
self._lock.acquire()
lastVT = 0
site = hashlib.sha1(site).hexdigest()
if (self._visitSite.has_key(site)):
lastVT = self._visitSite[site]
self._lock.release()
return lastVT
def updateLastVisitTime(self, site):
"""
Return the last time a site was visited.
"""
self._lock.acquire()
site = hashlib.sha1(site).hexdigest()
self._visitSite[site] = time.time()
self._lock.release()
class StatusMonitor(Loggable):
# valve_manager = None
_stop_evt = None
_clients = List
state_freq = Int(3)
checksum_freq = Int(3)
lock_freq = Int(5)
owner_freq = Int(5)
update_period = Int(1)
# def __init__(self, *args, **kw):
# super(StatusMonitor, self).__init__(*args, **kw)
# self._clients = []
def start(self, oid, vm):
if not self._clients:
p = self.update_period
s, c, l, o = self.state_freq, self.checksum_freq, self.lock_freq, self.owner_freq
self.debug('StatusMonitor period={}. '
'Frequencies(state={}, checksum={}, lock={}, owner={})'.format(p, s, c, l, o))
if self._stop_evt:
self._stop_evt.set()
self._stop_evt.wait(self.update_period)
self._stop_evt = Event()
t = Thread(target=self._run, args=(vm,))
t.setDaemon(True)
t.start()
# self._iter(1, vm)
else:
self.debug('Monitor already running')
if oid not in self._clients:
self._clients.append(oid)
def isAlive(self):
if self._stop_evt:
return not self._stop_evt.isSet()
def stop(self, oid):
try:
self._clients.remove(oid)
except ValueError:
pass
if not self._clients:
self._stop_evt.set()
self.debug('Status monitor stopped')
else:
self.debug('Alive clients {}'.format(self._clients))
def _run(self, vm):
i = 0
while 1:
if self._stop_evt.is_set():
break
if not self._iter(i, vm):
break
time.sleep(self.update_period)
if i > 100:
i = 0
i += 1
self.debug('Status monitor finished')
def _iter(self, i, vm):
self.debug('status monitor iteration i={}'.format(i))
if self._stop_evt.is_set():
return
if vm is None:
self.debug('No valve manager')
return
if self.state_freq and not i % self.state_freq:
self.debug('load valve states')
vm.load_valve_states()
if self.lock_freq and not i % self.lock_freq:
self.debug('load lock states')
vm.load_valve_lock_states()
if self.owner_freq and not i % self.owner_freq:
self.debug('load owners')
vm.load_valve_owners()
if self.checksum_freq and not i % self.checksum_freq:
if not vm.state_checksum:
self.debug('State checksum failed')
return not self._stop_evt.is_set()
class CookExecutor(pm.Executor):
"""This class is responsible for launching the task sent by the scheduler.
It implements the Executor methods."""
def __init__(self, stop_signal, config):
self.completed_signal = Event()
self.config = config
self.disconnect_signal = Event()
self.stop_signal = stop_signal
self.reregister_signal = None
def registered(self, driver, executor_info, framework_info, agent_info):
logging.info(
'Executor registered executor={}, framework={}, agent={}'.format(
executor_info['executor_id']['value'], framework_info['id'],
agent_info['id']['value']))
env = os.environ
if 'EXECUTOR_TEST_EXIT' in env:
# When running in docker, if the container exits too quickly it's logged in mesos as container launch failed
# instead of mesos executor terminated. This sleep ensures that we have the correct reason code for our
# integration tests.
time.sleep(5)
exit_code = int(env['EXECUTOR_TEST_EXIT'])
logging.warn(
'Exiting with code {} from EXECUTOR_TEST_EXIT environment variable'
.format(exit_code))
os._exit(exit_code)
def reregistered(self, driver, agent_info):
logging.info('Executor re-registered agent={}'.format(agent_info))
if self.config.checkpoint:
if self.reregister_signal is not None:
logging.info(
'Executor checkpointing is enabled. Notifying on reregister_signal'
)
self.reregister_signal.set()
self.reregister_signal = None
else:
logging.error(
'Checkpointing is enabled but reregister_signal is None. Unable to notify!'
)
def disconnected(self, driver):
logging.info('Mesos requested executor to disconnect')
if self.config.checkpoint:
if self.reregister_signal is None:
logging.info(
'Executor checkpointing is enabled. Waiting for agent recovery.'
)
new_event = Event()
self.reregister_signal = new_event
await_reregister(new_event,
self.config.recovery_timeout_ms / 1000,
self.stop_signal, self.disconnect_signal)
else:
logging.info(
'Checkpointing is enabled. Already launched await_reregister thread.'
)
else:
logging.info(
'Executor checkpointing is not enabled. Terminating task.')
self.disconnect_signal.set()
self.stop_signal.set()
def launchTask(self, driver, task):
logging.info('Driver {} launching task {}'.format(driver, task))
stop_signal = self.stop_signal
completed_signal = self.completed_signal
config = self.config
task_thread = Thread(target=manage_task,
args=(driver, task, stop_signal, completed_signal,
config))
task_thread.daemon = True
task_thread.start()
def killTask(self, driver, task_id):
logging.info(
'Mesos requested executor to kill task {}'.format(task_id))
task_id_str = task_id['value'] if 'value' in task_id else task_id
grace_period = os.environ.get('MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD',
'')
cio.print_and_log(
'Received kill for task {} with grace period of {}'.format(
task_id_str, grace_period))
self.stop_signal.set()
def shutdown(self, driver):
logging.info('Mesos requested executor to shutdown')
self.stop_signal.set()
def error(self, driver, message):
logging.error(message)
super().error(driver, message)
def await_completion(self):
"""
Blocks until the internal flag completed_signal is set.
The completed_signal Event is expected to be set by manage_task.
"""
logging.info('Waiting for CookExecutor to complete...')
self.completed_signal.wait()
logging.info('CookExecutor has completed')
def await_disconnect(self):
"""
Blocks until the internal flag disconnect_signal is set or the disconnect grace period expires.
The disconnect grace period is computed based on whether stop_signal is set.
"""
disconnect_grace_secs = cook.TERMINATE_GRACE_SECS if self.stop_signal.isSet(
) else cook.DAEMON_GRACE_SECS
if not self.disconnect_signal.isSet():
logging.info(
'Waiting up to {} second(s) for CookExecutor to disconnect'.
format(disconnect_grace_secs))
self.disconnect_signal.wait(disconnect_grace_secs)
if not self.disconnect_signal.isSet():
logging.info(
'CookExecutor did not disconnect in {} seconds'.format(
disconnect_grace_secs))
class Listener(Thread):
def __init__(self, host=HOST):
super(Listener, self).__init__()
self.host = host
self._stop_event = Event()
self.clear_time = None
self.s = socket.socket()
self.queue = collections.deque()
self.port = get_ephemeral_port()
self.s.bind((self.host, self.port))
def run(self):
if DEBUG:
logger.info('listening on %s:%s' % (self.host, self.port))
self.s.listen(5)
while True:
if self.stopped:
return
c, addr = self.s.accept()
if DEBUG:
logger.info('got connection %s' % str(addr))
data = c.recv(1024)
if DEBUG:
logger.info('got data: %s' % str(data))
if data == TERMINATOR:
self.stop()
return
elif data == CLEAR:
if DEBUG:
logger.info('clearing')
self.clear_time = time.time()
else:
self.queue.appendleft(data)
c.close()
def put_data(self, data):
s = socket.socket()
s.connect((self.host, self.port))
s.send(data)
s.close()
def get_data(self):
""" pops the latest off the queue, or None is there is none
"""
try:
data = self.queue.pop()
except IndexError:
return None, None
try:
data = cPickle.loads(data)
except:
try:
data = data.decode('utf-8')
except:
pass
if DEBUG:
logger.info('got %s' % str(data))
t = None
if isinstance(data, TimedMsg):
d = data.value
t = data.time
elif isinstance(data, string_types):
try:
d = json.loads(data)
except:
d = data
else:
d = data
return d, t
def _process_chunk(self, d, t):
if t is not None:
if DEBUG:
logger.info('diff %s' % (t - self.clear_time))
if t < self.clear_time:
if DEBUG:
logger.info('%s < %s' % (t, self.clear_time))
logger.info('discarding cleared %s' % d)
return True
else:
if DEBUG:
logger.info('removed clearing')
self.clear_time = None # unset as we've got one after last clear
else:
if DEBUG:
logger.info('removed clearing (nmsg with no time)')
self.clear_time = None
return False
def receive(self, timeout=TIMEOUT_DEFAULT):
if timeout is not None:
MAX_COUNT = int(timeout) * 10
d = None
count = 0
while d is None and count < MAX_COUNT:
d, t = self.get_data()
if d is None:
sleep(.1)
if timeout is not None:
count += 1
elif self.clear_time is not None and self._process_chunk(d, t):
count = 0
d = None
return d
def send(self, data, timeout=TIMEOUT_DEFAULT): # @UnusedVariable
payload = TimedMsg(data).pickled()
if DEBUG:
logger.info('sending %s' % str(data))
self.put_data(payload)
def clear_queue(self):
self.put_data(CLEAR)
time.sleep(.05)
def stop(self):
self.s.shutdown(socket.SHUT_WR)
self.s.close()
self._stop_event.set()
@property
def stopped(self):
return self._stop_event.isSet()
class InfluxClient():
def __init__(self,
host,
username,
password,
database,
ssl=False,
timeout=60,
port=8086):
self.connectionCredentials = {
'host': host,
'username': username,
'password': password,
'database': database,
'ssl': ssl,
'timeout': timeout,
'port': port,
'verify_ssl': True
}
self.connect()
self.dataBucket = defaultdict(queue.Queue)
self.dataPackedQueue = queue.Queue()
self.endEvent = Event()
self.bucketThread = Thread(target=self.prepareSendBatch)
self.bucketThread.daemon = True
self.bucketThread.start()
self.dataSendNumThreads = 2
self.dataSendThreads = []
for numThread in range(self.dataSendNumThreads):
t = Thread(target=self.sendData)
t.daemon = True
t.start()
self.dataSendThreads.append(t)
def connect(self):
self.client = InfluxDBClient(**self.connectionCredentials)
def writeData(self,
measurement,
tags,
fields,
measureTime=None,
retentionPolicy=None):
if not measurement or not type(tags) is dict or not type(
fields) is dict:
logger.warning("no data to insert. tags: %s. fields: %s" %
(tags, fields))
return
if not measurement:
logger.warning("no measurement to insert")
return
body = {"measurement": measurement, "fields": fields, "tags": tags}
if not measureTime:
measureTime = int(time.time() * 1000000)
body["time"] = measureTime
self.dataBucket[retentionPolicy].put(body)
def getBucketLen(self):
dataBucketLen = 0
for bucket in self.dataBucket.values():
dataBucketLen += bucket.qsize()
return dataBucketLen
def prepareSendBatch(self):
dataChunk = 5000
sleepTime = 5
while not self.endEvent.isSet() or self.getBucketLen():
# Sleep for some time and listen for the end event
for _ in range(sleepTime):
if self.endEvent.isSet():
break
time.sleep(1)
for retentionPolicy, dataQueue in self.dataBucket.items():
bucketEmpty = False
while not bucketEmpty:
dataToSend = []
for numOfItemsRetrieved in range(dataChunk):
try:
dataToSend.append(dataQueue.get(False))
except queue.Empty:
bucketEmpty = True
break
if not dataToSend:
break
self.dataPackedQueue.put((dataToSend, retentionPolicy))
logger.debug(
f"Added new package of {len(dataToSend)} values")
for _ in range(self.dataSendNumThreads):
self.dataPackedQueue.put(None)
def sendData(self):
client = InfluxDBClient(**self.connectionCredentials)
while True:
d = self.dataPackedQueue.get()
if d is None:
break
dataToSend, rp = d
logger.debug(f"Sending {len(dataToSend)} values")
try:
assert (client.write_points(dataToSend,
time_precision='u',
retention_policy=rp,
batch_size=10000))
except:
logger.error("Exception when saving the data", exc_info=True)
# If the batch fails try to send the values one by one
for value in dataToSend:
try:
assert (client.write_points([value],
time_precision='u',
retention_policy=rp))
except:
logger.error(f"Ignoring value: {value}")
client.close()
def query(self, query, database=None):
try:
result = self.client.query(query, database=database, epoch="s")
return result
except KeyboardInterrupt:
raise KeyboardInterrupt()
except:
logger.error("Exception when query the data.", exc_info=True)
self.connect()
def close(self, *args):
self.endEvent.set()
self.bucketThread.join()
for t in self.dataSendThreads:
t.join()
self.client.close()
logger.info("Influx closed")
class LocalDisplay(Thread):
""" Class for facilitating the local display of inference results
(as images). The class is designed to run on its own thread. In
particular the class dumps the inference results into a FIFO
located in the tmp directory (which lambda has access to). The
results can be rendered using mplayer by typing:
mplayer -demuxer lavf -lavfdopts format=mjpeg:probesize=32 /tmp/results.mjpeg
"""
def __init__(self, resolution):
""" resolution - Desired resolution of the project stream """
# Initialize the base class, so that the object can run on its own
# thread.
super(LocalDisplay, self).__init__()
# List of valid resolutions
RESOLUTION = {
'1080p': (1920, 1080),
'720p': (1280, 720),
'480p': (858, 480)
}
if resolution not in RESOLUTION:
raise Exception("Invalid resolution")
self.resolution = RESOLUTION[resolution]
# Initialize the default image to be a white canvas. Clients
# will update the image when ready.
self.frame = cv2.imencode('.jpg', 255 * np.ones([640, 480, 3]))[1]
self.stop_request = Event()
def run(self):
""" Overridden method that continually dumps images to the desired
FIFO file.
"""
# Path to the FIFO file. The lambda only has permissions to the tmp
# directory. Pointing to a FIFO file in another directory
# will cause the lambda to crash.
result_path = '/tmp/results.mjpeg'
# Create the FIFO file if it doesn't exist.
if not os.path.exists(result_path):
os.mkfifo(result_path)
# This call will block until a consumer is available
with open(result_path, 'w') as fifo_file:
while not self.stop_request.isSet():
try:
# Write the data to the FIFO file. This call will block
# meaning the code will come to a halt here until a consumer
# is available.
fifo_file.write(self.frame.tobytes())
except IOError:
continue
def set_frame_data(self, frame):
""" Method updates the image data. This currently encodes the
numpy array to jpg but can be modified to support other encodings.
frame - Numpy array containing the image data of the next frame
in the project stream.
"""
ret, jpeg = cv2.imencode('.jpg', cv2.resize(frame, self.resolution))
if not ret:
raise Exception('Failed to set frame data')
self.frame = jpeg
def join(self):
self.stop_request.set()
class FakePacketSniffer(Thread):
"""
A fake packet sniffing thread.
This thread returns fake Wi-Fi ESSIDs for development
and test purposes.
"""
def __init__(self, new_packets, **kwargs):
super().__init__()
self.new_packets = new_packets
self.stop_sniffer = Event()
self.exception = None
self.debug = kwargs.get("debug", False)
from faker import Faker
from faker_wifi_essid import WifiESSID
self.fake = Faker()
self.fake.add_provider(WifiESSID)
def run(self):
from time import sleep
try:
while not self.stop_sniffer.isSet():
sleep(1)
self.new_packet()
except Exception as e:
self.exception = e
if self.debug:
print("[!] Exception: " + str(e))
def join(self, timeout=None):
"""
Stops the fake packet sniffer.
"""
self.stop_sniffer.set()
super().join(timeout)
def new_packet(self):
"""
Adds a new fake packet to the queue to be processed.
"""
fake_probe_req = RadioTap() \
/ Dot11(
addr1="ff:ff:ff:ff:ff:ff",
addr2=self.fake.mac_address(),
addr3=self.fake.mac_address()
) \
/ Dot11ProbeReq() \
/ Dot11Elt(
info=self.fake.wifi_essid()
)
self.new_packets.put(fake_probe_req)
def should_stop_sniffer(self, packet):
"""
Returns true if the fake sniffer should be stopped and false otherwise.
"""
return self.stop_sniffer.isSet()
def get_exception(self):
"""
Returns the raised exception if any, otherwise returns none.
"""
return self.exception
class Task(Thread):
"""
Base class for all tasks
"""
DEFAULT_TIMEOUT = 60 * 60 * 24 # 1 Day
TEST_TIMEOUT = 30 # test task timeout
DNS_TIMEOUT = 10 # DNS request timeout
SOCKET_TIMEOUT = 2 # socket timeout
HTTP_TIMEOUT = 30 # HTTP timeout
SMTP_TIMEOUT = 10 # SMTP timeout
PARSE_FILES = True # read & parse all input files by default
EXPAND_TARGETS = True # expand ip networks and ip ranges
SYSTEM_LIBRARY_PATH = "/opt/gtta/scripts/system/lib"
USER_LIBRARY_PATH = "/opt/gtta/scripts/lib"
MULTITHREADED = False
THREAD_COUNT = 10
TEST_TARGETS = ["google.com"]
def __init__(self, worker=False):
"""
Constructor
"""
super(Task, self).__init__()
self.arguments = None
self.targets = None
self.target = None
self.host = None
self.ip = None
self.proto = None
self.port = None
self.timeout = None
self.lang = None
self.test_mode = False
self.error = False
self.produced_output = False
self._stop = Event()
self._result = None
self._worker = worker
def _check_stop(self):
"""
Check if thread should be stopped
"""
if self._stop.isSet():
raise TaskTimeout
def _write_result(self, str):
"""
Write result to result file or print it to output
"""
if not str:
return
self.produced_output = True
if self._result:
self._result.write(str.encode('utf-8'))
self._result.write('\n')
self._result.flush()
else:
print str.encode('utf-8')
stdout.flush()
def _expand_targets(self):
targets = []
for target in self.targets:
# IP network
if re.match('^\d+\.\d+\.\d+\.\d+\/(3[0-2]|2[0-9]{1}|[01]?[0-9])$',
target):
for ip in IPNetwork(target):
targets.append('%s' % ip)
# IP range
elif re.match('^\d+\.\d+\.\d+\.\d+\s*\-\s*\d+\.\d+\.\d+\.\d+$',
target):
target = target.replace(" ", "")
scope = target.split('-')
for ip in IPRange(scope[0], scope[1]):
targets.append('%s' % ip)
else:
targets.append(target)
self.targets = targets
def stop(self):
"""
Thread stop function
"""
self._stop.set()
def parse_input(self):
"""
Parses input arguments
"""
self.arguments = []
if len(argv) < 3:
raise NotEnoughArguments(
'At least 2 command line arguments should be specified.')
# parse the first file with hostname or IP
lines = open(argv[1], 'r').read().split('\n')
lines = map(lambda x: x.replace('\r', ''), lines)
if len(lines) < 5:
raise InvalidTargetFile(
'Target file should contain at least 5 lines.')
if not lines[0]:
raise InvalidTargetFile(
'Target file should contain either host name or IP address of the target host on the 1st line.'
)
targets = lines[0].split(",")
self.targets = []
for target in targets:
target = target.strip()
if target:
self.targets.append(target)
if self.EXPAND_TARGETS:
self._expand_targets()
self.proto = lines[1]
try:
if lines[2]:
self.port = int(lines[2])
else:
self.port = 0
except ValueError:
self.port = 0
if not lines[3]:
raise InvalidTargetFile(
'Target file should contain language name on the 4th line.')
self.lang = lines[3]
try:
if len(lines[4]) > 0:
self.timeout = int(lines[4])
else:
self.timeout = self.DEFAULT_TIMEOUT
except ValueError:
self.timeout = self.DEFAULT_TIMEOUT
# open output file
self._result = open(argv[2], 'w')
# parse the remaining arguments
for arg in argv[3:]:
if self.PARSE_FILES:
lines = open(arg, 'r').read().split('\n')
lines = map(lambda x: x.replace('\r', ''), lines)
self.arguments.append(lines)
else:
self.arguments.append(arg)
def _get_library_path(self, library):
"""
Get library path
"""
path = None
if isdir("%s/%s" % (self.SYSTEM_LIBRARY_PATH, library)):
path = self.SYSTEM_LIBRARY_PATH
elif isdir("%s/%s" % (self.USER_LIBRARY_PATH, library)):
path = self.USER_LIBRARY_PATH
if not path:
raise Exception("Library not exists: %s" % library)
return "%s/%s" % (path, library)
def test(self):
"""
Test the task
"""
raise Exception("Test not implemented.")
def main(self, *args):
"""
Main function
"""
raise Exception("Main function not implemented.")
def _run_target(self, target):
"""Run task for a single target"""
self.target = target
try:
inet_aton(target)
self.ip = target
except:
self.host = target
if self.test_mode:
self.test()
self.produced_output = True
else:
self.main(*self.arguments)
def _run_singlethreaded(self):
"""Run single-threaded task"""
map(self._run_target, self.targets)
def _run_multithreaded(self):
"""Run multi-threaded task"""
global target_queue
map(target_queue.put, self.targets)
thread_count = min(self.THREAD_COUNT, len(self.targets))
thread_pool = []
for i in xrange(thread_count):
thread = type(self)(worker=True)
thread.arguments = self.arguments
thread.proto = self.proto
thread.lang = self.lang
thread.test_mode = self.test_mode
thread.start()
thread_pool.append(thread)
for thread in thread_pool:
thread.join(self.timeout)
if thread.isAlive():
thread.stop()
result = thread.worker_result()
if result.endswith("\n"):
result = result[:-1]
self._write_result(result)
def _run_worker(self):
"""Worker thread main function"""
global target_queue
self._result = StringIO()
while True:
try:
task = target_queue.get(block=False)
except:
task = None
if not task:
break
self._run_target(task)
def worker_result(self):
"""Get worker result"""
if type(self._result) == file:
return ""
return self._result.getvalue()
def run(self):
"""
Run a task
"""
try:
# run worker for multithreaded
if self._worker:
self._run_worker()
return
if self.test_mode:
self.targets = self.TEST_TARGETS
if self.MULTITHREADED:
self._run_multithreaded()
else:
self._run_singlethreaded()
except TaskTimeout:
pass
except Exception, e:
error_str = e.__class__.__name__
if str(e):
error_str += ': %s' % str(e)
self._write_result(error_str)
self.error = True
class ChipStack(object):
def __init__(self, installDefaultLogHandler=True, bluetoothAdapter=0):
builtins.enableDebugMode = False
self.networkLock = Lock()
self.completeEvent = Event()
self.commissioningCompleteEvent = Event()
self._ChipStackLib = None
self._chipDLLPath = None
self.devMgr = None
self.callbackRes = None
self.commissioningEventRes = None
self._activeLogFunct = None
self.addModulePrefixToLogMessage = True
# Locate and load the chip shared library.
self._loadLib()
# Arrange to log output from the chip library to a python logger object with the
# name 'chip.ChipStack'. If desired, applications can override this behavior by
# setting self.logger to a different python logger object, or by calling setLogFunct()
# with their own logging function.
self.logger = logging.getLogger(__name__)
self.setLogFunct(self.defaultLogFunct)
# Determine if there are already handlers installed for the logger. Python 3.5+
# has a method for this; on older versions the check has to be done manually.
if hasattr(self.logger, "hasHandlers"):
hasHandlers = self.logger.hasHandlers()
else:
hasHandlers = False
logger = self.logger
while logger is not None:
if len(logger.handlers) > 0:
hasHandlers = True
break
if not logger.propagate:
break
logger = logger.parent
# If a logging handler has not already been initialized for 'chip.ChipStack',
# or any one of its parent loggers, automatically configure a handler to log to
# stdout. This maintains compatibility with a number of applications which expect
# chip log output to go to stdout by default.
#
# This behavior can be overridden in a variety of ways:
# - Initialize a different log handler before ChipStack is initialized.
# - Pass installDefaultLogHandler=False when initializing ChipStack.
# - Replace the StreamHandler on self.logger with a different handler object.
# - Set a different Formatter object on the existing StreamHandler object.
# - Reconfigure the existing ChipLogFormatter object.
# - Configure chip to call an application-specific logging function by
# calling self.setLogFunct().
# - Call self.setLogFunct(None), which will configure the chip library
# to log directly to stdout, bypassing python altogether.
#
if installDefaultLogHandler and not hasHandlers:
logHandler = logging.StreamHandler(stream=sys.stdout)
logHandler.setFormatter(ChipLogFormatter())
self.logger.addHandler(logHandler)
self.logger.setLevel(logging.DEBUG)
def HandleComplete(appState, reqState):
self.callbackRes = True
self.completeEvent.set()
def HandleError(appState, reqState, err, devStatusPtr):
self.callbackRes = self.ErrorToException(err, devStatusPtr)
self.completeEvent.set()
@_ChipThreadTaskRunnerFunct
def HandleChipThreadRun(callback):
callback()
self.cbHandleChipThreadRun = HandleChipThreadRun
self.cbHandleComplete = _CompleteFunct(HandleComplete)
self.cbHandleError = _ErrorFunct(HandleError)
# set by other modules(BLE) that require service by thread while thread blocks.
self.blockingCB = None
# Initialize the chip library
res = self._ChipStackLib.pychip_Stack_Init()
if res != 0:
raise self.ErrorToException(res)
res = self._ChipStackLib.pychip_BLEMgrImpl_ConfigureBle(
bluetoothAdapter)
if res != 0:
raise self.ErrorToException(res)
@property
def defaultLogFunct(self):
"""Returns a python callable which, when called, logs a message to the python logger object
currently associated with the ChipStack object.
The returned function is suitable for passing to the setLogFunct() method."""
def logFunct(timestamp, timestampUSec, moduleName, logCat, message):
moduleName = ChipUtility.CStringToString(moduleName)
message = ChipUtility.CStringToString(message)
if self.addModulePrefixToLogMessage:
message = "CHIP:%s: %s" % (moduleName, message)
logLevel = LogCategory.categoryToLogLevel(logCat)
msgAttrs = {
"chip-module": moduleName,
"timestamp": timestamp,
"timestamp-usec": timestampUSec,
}
self.logger.log(logLevel, message, extra=msgAttrs)
return logFunct
def setLogFunct(self, logFunct):
"""Set the function used by the chip library to log messages.
The supplied object must be a python callable that accepts the following
arguments:
timestamp (integer)
timestampUS (integer)
module name (encoded UTF-8 string)
log category (integer)
message (encoded UTF-8 string)
Specifying None configures the chip library to log directly to stdout."""
if logFunct is None:
logFunct = 0
if not isinstance(logFunct, _LogMessageFunct):
logFunct = _LogMessageFunct(logFunct)
with self.networkLock:
# NOTE: ChipStack must hold a reference to the CFUNCTYPE object while it is
# set. Otherwise it may get garbage collected, and logging calls from the
# chip library will fail.
self._activeLogFunct = logFunct
self._ChipStackLib.pychip_Stack_SetLogFunct(logFunct)
def Shutdown(self):
self._ChipStack.Call(lambda: self._dmLib.pychip_Stack_Shutdown())
self.networkLock = None
self.completeEvent = None
self._ChipStackLib = None
self._chipDLLPath = None
self.devMgr = None
self.callbackRes = None
def Call(self, callFunct):
'''Run a Python function on CHIP stack, and wait for the response.
This function is a wrapper of PostTaskOnChipThread, which includes some handling of application specific logics.
Calling this function on CHIP on CHIP mainloop thread will cause deadlock.
'''
# throw error if op in progress
self.callbackRes = None
self.completeEvent.clear()
with self.networkLock:
res = self.PostTaskOnChipThread(callFunct).Wait()
self.completeEvent.set()
if res == 0 and self.callbackRes != None:
return self.callbackRes
return res
def CallAsync(self, callFunct):
'''Run a Python function on CHIP stack, and wait for the application specific response.
This function is a wrapper of PostTaskOnChipThread, which includes some handling of application specific logics.
Calling this function on CHIP on CHIP mainloop thread will cause deadlock.
'''
# throw error if op in progress
self.callbackRes = None
self.completeEvent.clear()
with self.networkLock:
res = self.PostTaskOnChipThread(callFunct).Wait()
if res != 0:
self.completeEvent.set()
raise self.ErrorToException(res)
while not self.completeEvent.isSet():
if self.blockingCB:
self.blockingCB()
self.completeEvent.wait(0.05)
if isinstance(self.callbackRes, ChipStackException):
raise self.callbackRes
return self.callbackRes
def PostTaskOnChipThread(self, callFunct) -> AsyncCallableHandle:
'''Run a Python function on CHIP stack, and wait for the response.
This function will post a task on CHIP mainloop, and return an object with Wait() method for getting the result.
Calling Wait inside main loop will cause deadlock.
'''
callObj = AsyncCallableHandle(callFunct)
pythonapi.Py_IncRef(py_object(callObj))
res = self._ChipStackLib.pychip_DeviceController_PostTaskOnChipThread(
self.cbHandleChipThreadRun, py_object(callObj))
if res != 0:
pythonapi.Py_DecRef(py_object(callObj))
raise self.ErrorToException(res)
return callObj
def ErrorToException(self, err, devStatusPtr=None):
if err == 0x2C and devStatusPtr:
devStatus = devStatusPtr.contents
msg = ChipUtility.CStringToString(
(
self._ChipStackLib.pychip_Stack_StatusReportToString(
devStatus.ProfileId, devStatus.StatusCode
)
)
)
sysErrorCode = (
devStatus.SysErrorCode if (
devStatus.SysErrorCode != 0) else None
)
if sysErrorCode != None:
msg = msg + " (system err %d)" % (sysErrorCode)
return DeviceError(
devStatus.ProfileId, devStatus.StatusCode, sysErrorCode, msg
)
else:
return ChipStackError(
err,
ChipUtility.CStringToString(
(self._ChipStackLib.pychip_Stack_ErrorToString(err))
),
)
def LocateChipDLL(self):
if self._chipDLLPath:
return self._chipDLLPath
scriptDir = os.path.dirname(os.path.abspath(__file__))
# When properly installed in the chip package, the Chip Device Manager DLL will
# be located in the package root directory, along side the package's
# modules.
dmDLLPath = os.path.join(scriptDir, ChipStackDLLBaseName)
if os.path.exists(dmDLLPath):
self._chipDLLPath = dmDLLPath
return self._chipDLLPath
# For the convenience of developers, search the list of parent paths relative to the
# running script looking for an CHIP build directory containing the Chip Device
# Manager DLL. This makes it possible to import and use the ChipDeviceMgr module
# directly from a built copy of the CHIP source tree.
buildMachineGlob = "%s-*-%s*" % (platform.machine(),
platform.system().lower())
relDMDLLPathGlob = os.path.join(
"build",
buildMachineGlob,
"src/controller/python/.libs",
ChipStackDLLBaseName,
)
for dir in self._AllDirsToRoot(scriptDir):
dmDLLPathGlob = os.path.join(dir, relDMDLLPathGlob)
for dmDLLPath in glob.glob(dmDLLPathGlob):
if os.path.exists(dmDLLPath):
self._chipDLLPath = dmDLLPath
return self._chipDLLPath
raise Exception(
"Unable to locate Chip Device Manager DLL (%s); expected location: %s"
% (ChipStackDLLBaseName, scriptDir)
)
# ----- Private Members -----
def _AllDirsToRoot(self, dir):
dir = os.path.abspath(dir)
while True:
yield dir
parent = os.path.dirname(dir)
if parent == "" or parent == dir:
break
dir = parent
def _loadLib(self):
if self._ChipStackLib is None:
self._ChipStackLib = CDLL(self.LocateChipDLL())
self._ChipStackLib.pychip_Stack_Init.argtypes = []
self._ChipStackLib.pychip_Stack_Init.restype = c_uint32
self._ChipStackLib.pychip_Stack_Shutdown.argtypes = []
self._ChipStackLib.pychip_Stack_Shutdown.restype = c_uint32
self._ChipStackLib.pychip_Stack_StatusReportToString.argtypes = [
c_uint32,
c_uint16,
]
self._ChipStackLib.pychip_Stack_StatusReportToString.restype = c_char_p
self._ChipStackLib.pychip_Stack_ErrorToString.argtypes = [c_uint32]
self._ChipStackLib.pychip_Stack_ErrorToString.restype = c_char_p
self._ChipStackLib.pychip_Stack_SetLogFunct.argtypes = [
_LogMessageFunct]
self._ChipStackLib.pychip_Stack_SetLogFunct.restype = c_uint32
self._ChipStackLib.pychip_BLEMgrImpl_ConfigureBle.argtypes = [
c_uint32]
self._ChipStackLib.pychip_BLEMgrImpl_ConfigureBle.restype = c_uint32
self._ChipStackLib.pychip_DeviceController_PostTaskOnChipThread.argtypes = [
_ChipThreadTaskRunnerFunct, py_object]
self._ChipStackLib.pychip_DeviceController_PostTaskOnChipThread.restype = c_uint32
class ProbeRequestParser(Thread):
"""
A Wi-Fi probe request parsing thread.
"""
def __init__(self, new_packets, **kwargs):
super().__init__()
self.new_packets = new_packets
self.essid_filters = kwargs.get("essid_filters", None)
self.essid_regex = kwargs.get("essid_regex", None)
self.ignore_case = kwargs.get("ignore_case", False)
self.display_func = kwargs.get("display_func", lambda p: None)
self.storage_func = kwargs.get("storage_func", lambda p: None)
self.debug = kwargs.get("debug", False)
self.stop_parser = Event()
if self.debug:
print("[!] ESSID filters: " + str(self.essid_filters))
print("[!] ESSID regex: " + str(self.essid_regex))
print("[!] Ignore case: " + str(self.ignore_case))
if self.essid_regex is not None:
if self.ignore_case:
self.essid_regex = rcompile(self.essid_regex, IGNORECASE)
else:
self.essid_regex = rcompile(self.essid_regex)
else:
self.essid_regex = None
def run(self):
# The parser continues to do its job even after the call of the
# join method if the queue is not empty.
while not self.stop_parser.isSet() or not self.new_packets.empty():
try:
packet = self.new_packets.get(timeout=1)
probe_request = self.parse(packet)
if probe_request is None:
continue
if not probe_request.essid:
continue
if self.essid_filters is not None and not probe_request.essid in self.essid_filters:
continue
if self.essid_regex is not None and not match(self.essid_regex, probe_request.essid):
continue
self.display_func(probe_request)
self.storage_func(probe_request)
self.new_packets.task_done()
except Empty:
pass
def join(self, timeout=None):
"""
Stops the probe request parsing thread.
"""
self.stop_parser.set()
super().join(timeout)
@staticmethod
def parse(packet):
"""
Parses the raw packet and returns a probe request object.
"""
if packet.haslayer(Dot11ProbeReq):
timestamp = packet.getlayer(RadioTap).time
s_mac = packet.getlayer(RadioTap).addr2
essid = packet.getlayer(Dot11ProbeReq).info.decode("utf-8")
return ProbeRequest(timestamp, s_mac, essid)
else:
return None
class Policy(ConfigurableNode):
implements(IPolicy)
security = SecurityInformation.from_default()
secured_by(security)
security.protect('context', 'Manage Users')
security.protect('acquires', 'Manage Users')
security.protect('rolemap', 'Manage Users')
security.make_private('readonly')
def __init__(self, *args):
self.readonly = []
self.__lock = RLock()
self.__active = Event()
self.acquires = True
self.context = ""
self.filter = ""
self.is_default = False
self.uses_expression = False
self.configured_expression = False
self.rolemap = _RoleMap()
super(Policy, self).__init__(*args)
def get_readOnly(self, attr):
return attr in self.readonly
security.protect('configure', 'Manage Users')
def configure(self, config):
"""
Configure node with from configuration dictionary.
The policy may be configured to use regular expressions
instead of simple node-URL contexts in two different ways.
The configuration dictionary may include a True/False
value "uses_expression", in which case the context
regular expression is taken verbatim from the "context"
parameter, and an optional filter expression from the
"filter" parameter.
Or the expression(s) may be encoded directly into the
"context" value using the following format:
REGEX: '<context regex>', FILTER: '<filter regex>'
Note that the ", FILTER: [...]" part is completely
optional and will default to no filter.
"""
for attrname in self.readonly:
current = getattr(self, attrname, None)
incoming = config.get(attrname)
if None not in (current, incoming) and (current != incoming):
message = 'Attribute "%s" is readonly for Policy "%s". '
message += 'Overriding new value %s with current value %s.'
message = message % (attrname, self.name, incoming, current)
msglog.log('broadway', msglog.types.WARN, message)
config[attrname] = current
self.acquires = as_boolean(config.get('acquires', self.acquires))
if not self.is_default:
self.is_default = as_boolean(config.get('is_default', False))
self.configure_context(config)
super(Policy, self).configure(config)
# Getting set of mappings whose value is a callback function.
# The assumption is being made here that such mappings are
# those considered 'read-only.'
self.__lock.acquire()
try:
inherent = self.rolemap.callable_subset()
self.rolemap = _RoleMap(config.get('rolemap', self.rolemap))
self.rolemap.update(inherent)
finally:
self.__lock.release()
self.__verify_setup()
def configure_context(self, config):
context = config.get('context', self.context)
if not context:
raise TypeError("Policy must have non-empty context")
if config.has_key("uses_expression"):
self.uses_expression = as_boolean(config["uses_expression"])
else:
self.uses_expression = False
if self.uses_expression:
self.context = context
if config.has_key("filter"):
self.filter = config["filter"]
self.configured_expression = True
else:
self.configured_expression = False
if isinstance(context, str) and context.startswith("REGEX:"):
self.uses_expression = True
# Regular expression encoded into context
# as "REGEX: '<expr>', FILTER: '<expr>'.
REGEX, FILTER = "REGEX", "FILTER"
expressions = eval("{%s}" % context)
self.context = expressions.get(REGEX, "")
self.filter = expressions.get(FILTER, "")
elif context != self.context:
if not isinstance(context, str):
if ISecurityContext.providedBy(context):
context = context.url
else:
raise TypeError("Context must be string or object"
" providing ISecurityContext.")
self.filter = ""
self.context = context
return self.context, self.filter
def context_configuration(self, config):
if self.configured_expression:
config["context"] = self.context
config["filter"] = self.filter
config["uses_expression"] = "1"
elif self.uses_expression:
context = "REGEX: %r, FILTER: %r" % (self.context, self.filter)
config["context"] = context
else:
config["context"] = self.context
return config
def configuration(self):
config = super(Policy, self).configuration()
config['acquires'] = str(self.acquires)
config['is_default'] = as_boolean(self.is_default)
self.context_configuration(config)
self.__lock.acquire()
try:
config['rolemap'] = self.rolemap.copy()
finally:
self.__lock.release()
return config
def start(self):
if not self.context:
raise TypeError("Policy must have non-empty context")
self.__lock.acquire()
try:
self.__verify_setup()
self.__active.set()
finally:
self.__lock.release()
super(Policy, self).start()
def stop(self):
if not self.parent.default is self:
self.__active.clear()
else:
message = 'Policy "%s" is system default and has ' % self.name
message += 'not been deactivated, although it has been stopped.'
msglog.log('broadway', msglog.types.WARN, message)
super(Policy, self).stop()
def is_running(self):
return self.__active.isSet()
def is_removable(self):
return not len(self.readonly)
def is_configurable(self):
return not self.is_default
security.protect('prune', 'Manage Users')
def prune(self):
if not self.is_removable():
error = '%s "%s" is not removable.'
raise TypeError(error % (type(self).__name__, self.name))
return super(Policy, self).prune()
def __verify_setup(self):
try:
node = as_node(self.context)
except:
raise EInvalidValue("Invalid Context", "%s" % self.context)
policies = list(self.parent.get_policies())
policies.remove(self)
for policy in policies:
if policy.uses_expression != self.uses_expression:
continue
if policy.context != self.context:
continue
# Policy contexts are same.
message = ("Conflicting policy configurations detected."
"Policies %s and %s have same context: '%s'.")
msglog.log("broadway", msglog.types.WARN,
message % (self, policy, self.context))
if policy.is_running():
message = "Running policy '%s' already has context: %s"
raise TypeError(message % (policy.name, self.context))
else:
message = ("%s ignoring context conflict: "
"conflicting policy is not running.")
msglog.log("broadway", msglog.types.WARN, message % self)
permissions = self.parent.get_permissions()
rolemap = self.rolemap.copy()
for role, granted in rolemap.items():
if not self.parent.parent.role_manager.has_role(role):
message = 'Policy "%s" ' % self.url
message += 'removing role "%s". ' % role
message += 'It does not exist.'
msglog.log('broadway', msglog.types.WARN, message)
del (self.rolemap[role])
elif isinstance(granted, (list, tuple)):
for permission in granted:
if permission not in permissions:
message = 'Policy "%s" ' % self.url
message += 'removing permission "%s" ' % permission
message += 'from role "%s". ' % role
message += 'Permission does not exist.'
msglog.log('broadway', msglog.types.WARN, message)
self.rolemap[role].remove(permission)
return
def get_permissions(self, role):
if not isinstance(role, str):
role = role.name
return self.rolemap.get(role)
security.protect('set_permissions', 'Manage Users')
def set_permissions(self, role, *permissions):
##
# Permissions parameter can be one of three things:
# - A single list or tuple object, whose items will
# replace the permissions tuple.
# - A variable number of permission strings.
# - A single callable object which will be called
# and whose return value will be returned anytime
# the permissions for this role are queried.
if not isinstance(role, str):
role = role.name
if len(permissions) == 1:
if type(permissions[0]) in (list, tuple):
permissions = permissions[0][:]
elif callable(permissions[0]):
permissions = permissions[0]
if not self.parent.parent.role_manager.has_role(role):
raise ValueError('Role "%s" does not exist.' % role)
if isinstance(permissions, (list, tuple)):
defined = self.parent.get_permissions()
for permission in permissions:
if permission not in defined:
raise ValueError('Permission "%s" not defined.' %
permission)
self.__lock.acquire()
try:
inherent = self.rolemap.callable_subset()
self.rolemap[role] = permissions
self.rolemap.update(inherent)
finally:
self.__lock.release()
if inherent.has_key(role):
message = 'Permissions for role "%s" in policy "%s" '
message += 'cannot be changed. An attempt has been ignored.'
message = message % (role, self.name)
msglog.log('broadway', msglog.types.WARN, message)
return
def rank_match(self, context):
if not self.is_running():
raise ENotRunning('Not started. This may mean start failed.')
if not isinstance(context, str):
context = ISecurityContext(context).url
if self.uses_expression:
rank = self.rank_expression(context)
else:
rank = self.rank_overlap(context)
return rank
def rank_expression(self, context):
context = urllib.unquote(context)
match = re.match(self.context, context)
if not match:
count = 0
elif self.filter and re.match(self.filter, context):
count = 0
else:
matching = match.group()
count = matching.count("/")
if len(matching) > 1:
count = count + 1
return count
def rank_overlap(self, context):
context = urllib.unquote(context)
if not context.startswith(self.context):
count = 0
else:
count = self.context.count('/')
if len(self.context) > 1:
count = count + 1
return count
class PacketSniffer(Thread):
"""
A packet sniffing thread.
"""
def __init__(self, interface, new_packets, **kwargs):
super().__init__()
self.daemon = True
self.interface = interface
self.new_packets = new_packets
self.mac_exclusions = kwargs.get("mac_exclusions", None)
self.mac_filters = kwargs.get("mac_filters", None)
self.debug = kwargs.get("debug", False)
self.frame_filters = "type mgt subtype probe-req"
self.socket = None
self.stop_sniffer = Event()
self.exception = None
if self.mac_exclusions is not None:
self.frame_filters += " and not ("
for i, station in enumerate(self.mac_exclusions):
if i == 0:
self.frame_filters += "ether src host {s_mac}".format(s_mac=station)
else:
self.frame_filters += " || ether src host {s_mac}".format(s_mac=station)
self.frame_filters += ")"
if self.mac_filters is not None:
self.frame_filters += " and ("
for i, station in enumerate(self.mac_filters):
if i == 0:
self.frame_filters += "ether src host {s_mac}".format(s_mac=station)
else:
self.frame_filters += " || ether src host {s_mac}".format(s_mac=station)
self.frame_filters += ")"
if self.debug:
print("[!] Frame filters: " + self.frame_filters)
def run(self):
try:
self.socket = conf.L2listen(
type=ETH_P_ALL,
iface=self.interface,
filter=self.frame_filters
)
sniff(
opened_socket=self.socket,
store=False,
prn=self.new_packet,
stop_filter=self.should_stop_sniffer
)
except Exception as e:
self.exception = e
if self.debug:
print("[!] Exception: " + str(e))
def join(self, timeout=None):
"""
Stops the packet sniffer.
"""
self.stop_sniffer.set()
super().join(timeout)
def new_packet(self, packet):
"""
Adds a new packet to the queue to be processed.
"""
self.new_packets.put(packet)
def should_stop_sniffer(self, packet):
"""
Returns true if the sniffer should be stopped and false otherwise.
"""
return self.stop_sniffer.isSet()
def get_exception(self):
"""
Returns the raised exception if any, otherwise returns none.
"""
return self.exception
class HTTPServer(Thread):
"""
A simple, minimal HTTP server
"""
CONFIG_KEY = 'http'
THREAD_NAME = 'simple-http-server'
def __init__(self, config):
self.__log.info("Instantiating Simple HTTP Server")
Thread.__init__(self)
self.setName(self.THREAD_NAME)
self.setDaemon(True)
self._stop = Event()
self.config = config
def stop(self):
self._stop.set()
def stopped(self):
return self._stop.isSet()
def _request_handler(self, *args):
RequestHandler(self.config, *args)
def run(self):
try:
self.__log.info("Staring Simple HTTP Server")
host_key = 'http_host'
port_key = 'http_port'
ssl_port_key = 'http_ssl_port'
cert_file_key = 'http_cert_file'
host = self.config.get(
self.CONFIG_KEY, host_key
) # Utils.get_env_var(host_key, self.config.get(self.CONFIG_KEY, host_key), self.__log)
ssl_enabled = bool(
self.config.get(self.CONFIG_KEY, 'http_ssl_enabled'))
port = self.config.get(
self.CONFIG_KEY, port_key) if ssl_enabled else self.config.get(
self.CONFIG_KEY, ssl_port_key)
cert_file = self.config.get(self.CONFIG_KEY, cert_file_key)
self.__log.debug("host: {}".format(host))
self.__log.debug("port: {}".format(port))
self.__log.debug("cert file: {}".format(cert_file))
self.httpd = BaseHTTPServer.HTTPServer(
(host, int(port)), self._request_handler
) # SimpleHTTPServer.SimpleHTTPRequestHandler
if ssl_enabled == 'True':
self.__log.info("Wrapping SSL")
self.httpd.socket = ssl.wrap_socket(self.httpd.socket,
server_side=True,
certfile=cert_file)
self.httpd.serve_forever()
# self.http_thread = Thread(target=self.httpd.serve_forever(), )
# self.http_thread.daemon = True
# Starting the HTTP server
# try:
# self.http_thread.start()
# except Exception as e:
# self.__log.error("Error while starting HTTP Server")
# self.__log.error(str(e))
# self.httpd.shutdown()
# Wait until HTTP server is ready
time.sleep(1)
except Exception as e:
self.__log.error(
"Error while staring Simple HTTP Server. Error: {}".format(
str(e)))
return False
def stop(self):
self.stop_httpd()
self._stop.set()
def stop_httpd(self):
self.__log.info("Shtting down HTTP Server...")
self.httpd.shutdown()
#self.http_thread.stop()
self.__log.info("HTTP Server Shutdown completed")
stream.stop_stream()
stream.close()
audio.terminate()
if __name__ == '__main__':
try:
# event to synchronise threads to ensure they start at the same time
e = Event()
t1 = Thread(target=other_sensors, args=(e,))
t2 = Thread(target=noise_sensor, args=(e,))
# starting thread 1
t1.start()
# starting thread 2
t2.start()
# Keep the main thread running, otherwise signals are ignored.
while True: # <- swap with "for frame in camera.capture_continuous()" loop"
# image-processing can go here...
time.sleep(1)
except KeyboardInterrupt:
print('Closing threads...')
stop_threads = True
# if program closed before thread 1 flags calibration is done, ensure thread 2 is unblocked and hence able to run to completion
if not e.isSet():
e.set()
# Wait for the threads to close
t1.join()
# starting thread 2
t2.join()
class Monitor(Thread):
""" Initiates a connection to the database to store telemetry data
at regular time intervals
:param running: an event controlling the process operation
:param appconfig: the application configuration object
:param q: the telemetry data queue
:param id: the recorder thread identifier
:param enabled: a flag indicating if the monitor is enabled
"""
def __init__(self, q, appconfig, name=""):
""" Initializes the recorder object
:param q: the telemetry data queue
:param appconfig: the application configuration object
:param name: a name that can be attributed to the monitor
"""
Thread.__init__(self)
self.running = Event()
if name != "":
self.id = name
self.q = q
self.appconfig = appconfig
self.enabled = False
self.lastdrivingtime = datetime.now()
self.gpslogging = 0
self.gpsconnected = False
def init_connection(self):
"""Initializes the connection to the GPSD service"""
try:
# Attempts to create a connection to the GPSD server
logger.info(
f"connect to gpsd server: {self.appconfig.gpsd_ip_address}:{self.appconfig.gpsd_port}"
)
gpsd.connect(self.appconfig.gpsd_ip_address,
self.appconfig.gpsd_port)
self.gpsconnected = True
return 0
except Exception as error:
logger.error(f"Exception: {str(error)}")
self.gpsconnected = False
return -1
def start(self):
"""Starts the monitor thread"""
self.running.set()
self.enabled = True
super(Monitor, self).start()
def run(self):
""" Runs the monitor infinite loop """
# Opens gpsd connection
# insert data in database
while (self.running.isSet()):
retval = self.report_current_location()
if retval < 0:
logger.error("error report location, try reconnection!")
self.init_connection()
time.sleep(self.appconfig.monitor_delay)
def report_current_location(self):
""" Gets the current location data from the GPSD and reports it to
the shared queue as a Location object
:return: 0 if success or -1 if failure or an exception arises
"""
try:
# Get current GPS position
packet = gpsd.get_current()
if self.lastdrivingtime is not None:
delta = datetime.now() - self.lastdrivingtime
# Unpack location parameters
mode = packet.mode
latitude = packet.lat
longitude = packet.lon
utc_time = packet.time
track = packet.track
hspeed = packet.hspeed
if abs(hspeed) * 3.6 > SPEEDTHRESH_KMH:
self.lastdrivingtime = datetime.now()
self.gpslogging += 1
else:
if self.lastdrivingtime is not None:
delta = datetime.now() - self.lastdrivingtime
if delta.seconds > WAIT_STOPLOGGING:
logger.info(
f"stop logging after {WAIT_STOPLOGGING} seconds not driving"
)
self.gpslogging = 0
else:
logger.info(
f"wait for stop logging after driving {WAIT_STOPLOGGING - delta.seconds}"
)
altitude = None
climb = None
if packet.mode == 3:
altitude = packet.alt
climb = packet.climb
loc = location.Location(latitude=latitude, longitude=longitude, altitude=altitude, heading=track, \
climb=climb, horizontal_speed=hspeed, mode=mode, utc_time=utc_time)
logger.info(str(loc)) # TODO: remove after DEBUG
logger.info(packet.map_url())
# Put the location instance in the shared queue
if self.gpslogging > 3 and packet.mode == 3:
self.q.put(loc)
logger.info("put gpsdata to queue!")
else:
logger.info("device is not driving!")
return 0
except Exception as inst:
logger.error(
f'Type: {type(inst)} -- Args: {inst.args} -- Instance: {inst}')
self.gpslogging = 0
return -1
def stop(self):
"""Stops the monitor thread"""
self.running.clear()
# disable the monitor
self.enabled = False
class MCMolShapeDatabase:
""" Maintains a database of MCMols that can be queried by shape
similarity."""
def __init__(self, itf):
""" Create a MCMolShapeDatabase from the parameters specified by the OEInterface. """
self.rwlock = ReadWriteLock()
self.loadedEvent = Event()
self.dbname = itf.GetString("-dbase")
self.moldb = oechem.OEMolDatabase()
self.dbtype = GetDatabaseType(itf.GetBool("-shapeOnly"))
self.shapedb = oefastrocs.OEShapeDatabase(*GetShapeDatabaseArgs(itf))
# this thread is daemonic so a KeyboardInterupt
# during the load will cancel the process
self.loaderThread = DatabaseLoaderThread(self.shapedb, self.moldb,
self.dbname, self.loadedEvent)
self.loaderThread.setDaemon(True)
self.loaderThread.start()
def IsLoaded(self, blocking=False):
""" Return whether the server has finished loading. """
if blocking:
self.loadedEvent.wait()
# clean up the load waiter thread if it's still there
if self.loadedEvent.isSet() and self.loaderThread is not None:
self.rwlock.AcquireWriteLock()
try: # typical double checked locking
if self.loaderThread is not None:
self.loaderThread.join()
self.loaderThread = None
finally:
self.rwlock.ReleaseWriteLock()
return self.loadedEvent.isSet()
def GetBestOverlays(self, querymolstr, options, iformat, oformat):
""" Return a string of the format specified by 'oformat'
containing nhits overlaid confomers using querymolstr as the
query interpretted as iformat.
querymolstr - a string containing a molecule to use as the query
options - an instance of OEShapeDatabaseOptions
iformat - a string representing the file extension to parse the querymolstr as.
Note: old clients could be passing .sq files, so
iformat == '.oeb' will try to interpret the file as
a .sq file.
oformat - file format to write the results as
"""
timer = oechem.OEWallTimer()
# make sure to wait for the load to finish
blocking = True
loaded = self.IsLoaded(blocking)
assert loaded
if iformat.startswith(".sq"):
query = ReadShapeQuery(querymolstr)
else:
# read in query
qfs = oechem.oemolistream()
qfs = SetupStream(qfs, iformat)
if not qfs.openstring(querymolstr):
raise ValueError("Unable to open input molecule string")
query = oechem.OEGraphMol()
if not oechem.OEReadMolecule(qfs, query):
if iformat == ".oeb": # could be an old client trying to send a .sq file.
query = ReadShapeQuery(querymolstr)
else:
raise ValueError(
"Unable to read a molecule from the string of format '%s'"
% iformat)
ofs = oechem.oemolostream()
ofs = SetupStream(ofs, oformat)
if not ofs.openstring():
raise ValueError("Unable to openstring for output")
# do we only want shape based results?
# this is a "Write" lock to be paranoid and not overload the GPU
self.rwlock.AcquireWriteLock()
try:
# do search
scores = self.shapedb.GetSortedScores(query, options)
sys.stderr.write("%f seconds to do search\n" % timer.Elapsed())
finally:
self.rwlock.ReleaseWriteLock()
timer.Start()
# write results
for score in scores:
mcmol = oechem.OEMol()
if not self.moldb.GetMolecule(mcmol, score.GetMolIdx()):
oechem.OEThrow.Warning(
"Can't retrieve molecule %i from the OEMolDatabase, "
"skipping..." % score.GetMolIdx())
continue
# remove hydrogens to make output smaller, this also
# ensures OEPrepareFastROCSMol will have the same output
oechem.OESuppressHydrogens(mcmol)
mol = oechem.OEGraphMol(
mcmol.GetConf(oechem.OEHasConfIdx(score.GetConfIdx())))
oechem.OECopySDData(mol, mcmol)
if options.GetSimFunc() == oefastrocs.OEShapeSimFuncType_Tanimoto:
oechem.OESetSDData(mol, "ShapeTanimoto",
"%.4f" % score.GetShapeTanimoto())
oechem.OESetSDData(mol, "ColorTanimoto",
"%.4f" % score.GetColorTanimoto())
oechem.OESetSDData(mol, "TanimotoCombo",
"%.4f" % score.GetTanimotoCombo())
else:
oechem.OESetSDData(mol, "ShapeTversky",
"%.4f" % score.GetShapeTversky())
oechem.OESetSDData(mol, "ColorTversky",
"%.4f" % score.GetColorTversky())
oechem.OESetSDData(mol, "TverskyCombo",
"%.4f" % score.GetTverskyCombo())
if options.GetInitialOrientation(
) != oefastrocs.OEFastROCSOrientation_Inertial:
oechem.OEAddSDData(
mol, "Opt. Starting Pos.",
GetAltStartsString(options.GetInitialOrientation()))
score.Transform(mol)
oechem.OEWriteMolecule(ofs, mol)
output = ofs.GetString()
sys.stderr.write("%f seconds to write hitlist\n" % timer.Elapsed())
sys.stderr.flush()
ofs.close()
return output
def GetName(self):
self.rwlock.AcquireReadLock()
try:
return self.dbname
finally:
self.rwlock.ReleaseReadLock()
def SetName(self, name):
self.rwlock.AcquireWriteLock()
try:
self.dbname = name
finally:
self.rwlock.ReleaseWriteLock()
class ResultHandler(SocketServer.BaseRequestHandler):
"""Result handler.
This handler speaks our analysis log network protocol.
"""
def setup(self):
self.rawlogfd = None
self.protocol = None
self.startbuf = ""
self.end_request = Event()
self.done_event = Event()
self.pid, self.ppid, self.procname = None, None, None
self.server.register_handler(self)
def finish(self):
self.done_event.set()
if self.protocol:
self.protocol.close()
if self.rawlogfd:
self.rawlogfd.close()
def wait_sock_or_end(self):
while True:
if self.end_request.isSet():
return False
rs, _, _ = select.select([self.request], [], [], 1)
if rs:
return True
def seek(self, pos):
pass
def read(self, length):
buf = ""
while len(buf) < length:
if not self.wait_sock_or_end():
raise Disconnect()
tmp = self.request.recv(length - len(buf))
if not tmp:
raise Disconnect()
buf += tmp
if isinstance(self.protocol, BsonParser):
if self.rawlogfd:
self.rawlogfd.write(buf)
else:
self.startbuf += buf
return buf
def read_any(self):
if not self.wait_sock_or_end():
raise Disconnect()
tmp = self.request.recv(BUFSIZE)
if not tmp:
raise Disconnect()
return tmp
def read_newline(self):
buf = ""
while "\n" not in buf:
buf += self.read(1)
return buf
def negotiate_protocol(self):
# Read until newline.
buf = self.read_newline()
if "BSON" in buf:
self.protocol = BsonParser(self)
elif "FILE" in buf:
self.protocol = FileUpload(self)
elif "LOG" in buf:
self.protocol = LogHandler(self)
else:
raise CuckooOperationalError("Netlog failure, unknown "
"protocol requested.")
def handle(self):
ip, port = self.client_address
self.connect_time = datetime.datetime.now()
self.storagepath = self.server.build_storage_path(ip)
if not self.storagepath:
return
# Create all missing folders for this analysis.
self.create_folders()
try:
# Initialize the protocol handler class for this connection.
self.negotiate_protocol()
for event in self.protocol:
if isinstance(self.protocol,
BsonParser) and event["type"] == "process":
self.open_process_log(event)
except CuckooResultError as e:
log.warning(
"ResultServer connection stopping because of "
"CuckooResultError: %s.", str(e))
except (Disconnect, socket.error):
pass
except:
log.exception("FIXME - exception in resultserver connection %s",
str(self.client_address))
log.debug("Connection closed: {0}:{1}".format(ip, port))
def open_process_log(self, event):
pid = event["pid"]
ppid = event["ppid"]
procname = event["process_name"]
if self.pid is not None:
log.debug(
"ResultServer got a new process message but already "
"has pid %d ppid %s procname %s.", pid, str(ppid), procname)
raise CuckooResultError("ResultServer connection state "
"inconsistent.")
# Only report this process when we're tracking it.
if event["track"]:
log.debug("New process (pid=%s, ppid=%s, name=%s)", pid, ppid,
procname)
path = os.path.join(self.storagepath, "logs", str(pid) + ".bson")
self.rawlogfd = open(path, "wb")
self.rawlogfd.write(self.startbuf)
self.pid, self.ppid, self.procname = pid, ppid, procname
def create_folders(self):
folders = "shots", "files", "logs", "buffer"
for folder in folders:
try:
create_folder(self.storagepath, folder=folder)
except CuckooOperationalError:
log.error("Unable to create folder %s" % folder)
return False
class GPU(object):
"""GPU is a representation of a physical gpu based on its index in the list of available gpus.
Arguments
---------
device_id int : Index of the GPU
prevent_exceptions boolean : Silent exceptions
display str : X display identifier
"""
def __init__(self,
device_id,
prevent_exceptions,
display=":0"): # noqa: D107
self.id = device_id
self.display = display
self.check_exceptions = not prevent_exceptions
self._stop = Event()
self._thread = None
def stop(self):
"""Signal the thread to stop execution."""
self._stop.set()
def stopped(self):
"""Check if stopping execution is requested."""
return self._stop.isSet()
def __setSpeed(self, speed):
cmd = [
"nvidia-settings", "-c {0}".format(self.display),
"-a [gpu:{0}]/GPUFanControlState=1".format(self.id),
"-a [fan:{0}]/GPUTargetFanSpeed={1}".format(self.id, speed)
]
# using sb.run(cmd, ...) did not work! I guess there is a conflict in -c switch.
sb.run(" ".join(cmd),
shell=True,
stdout=sb.DEVNULL,
stderr=sb.DEVNULL,
check=self.check_exceptions)
def __customCurveSpeed(self):
nvmlInit()
self._handle = nvmlDeviceGetHandleByIndex(self.id)
curve = Curve()
while (not self.stopped()):
current_temp = self.__getTemp()
new_fan_speed = curve.evaluate(current_temp)
self.__setSpeed(new_fan_speed)
time.sleep(1.0)
nvmlShutdown()
def __thread_alive(self):
if self._thread and self._thread.is_alive():
return True
return False
def __getTemp(self):
"""Get temperature of the GPU."""
return nvmlDeviceGetTemperature(self._handle, NVML_TEMPERATURE_GPU)
def constant(self, percentage):
"""Set a constant fan speed.
Arguments
---------
percentage int : An integer from 0 to 100
"""
if self.__thread_alive():
self.stop()
self._thread.join()
self.__setSpeed(percentage)
def aggressive(self):
"""Control GPU fan based on an aggressive regime.
It tries to set GPU fan to a high speed as soon as temperature rises.
"""
if self.__thread_alive():
return
self._thread = Thread(target=self.__customCurveSpeed)
self._thread.daemon = True
self._thread.start()
def driver(self):
"""Return control of fan speed to the driver."""
if self.__thread_alive():
self.stop()
self._thread.join()
cmd = [
"nvidia-settings", "-c {0}".format(self.display),
"-a [gpu:{0}]/GPUFanControlState=0".format(self.id)
]
sb.run(" ".join(cmd),
shell=True,
stdout=sb.DEVNULL,
stderr=sb.DEVNULL,
check=self.check_exceptions)
def __del__(self):
"""Make sure thread is stopped before destruction."""
if self.__thread_alive():
self._thread.stop()
self._thread.join()
self.driver()
