class PromptService(object):
def __init__(self):
self.semaphore = Semaphore(0)
self.commandWindow = None
self.response = None
def setCommandWindow(self, window):
self.commandWindow = window
def requestInput(self, prompt):
if self.commandWindow is None:
raise RuntimeError("Command window hasn't registered itself")
if prompt is None:
prompt = ''
self.commandWindow.prompt(prompt, 'standard-output', self.respond, 'standard-input')
self.semaphore.acquire()
if self.response is None:
raise KeyboardInterrupt
else:
res = self.response
self.response = None
return str(res)
def respond(self, value):
self.response = value
self.semaphore.release()
class Wc:
def __init__(self):
self.flush()
def flush(self):
self.sem = Semaphore(1)
self.user = None
self.waiting = []
def used_by(self, who):
self.user = who
self.waiting.remove(who)
def being_used_by(self, who):
return self.user == who
def acquire(self, a):
return self.sem.acquire(a)
def release(self):
self.user = None
self.sem.release()
def enqueue(self, nick):
self.waiting.append(nick)
def is_waiting(self, nick):
return (nick in self.waiting)
def who(self):
return self.user
class Request_deque():
from collections import deque
def __init__(self, value=1):
self.sema = Semaphore(value)
self.time_stamp_q = deque()
self.sync_lock = Lock()
def acquire(self, blocking=True):
if self.sema.acquire(blocking):
# released under blocked mode or happened to have spare under
#non-blocking mode
return True, self.time_stamp_q.popleft()
else:
# non-blocking mode with unsuccessful acquiring
return False, None
def release(self, stop=False):
with self.sync_lock:
# need to guarantee the order matching between request and time
#stamp, the operation shall be atomic. This could be rare to have
#but unaffordable if any.
if stop:
self.time_stamp_q.append(None)
else:
self.time_stamp_q.append(dt.now())
self.sema.release()
class StatisticQueue:
def __init__(self, stats):
self._semaphore = Semaphore()
self.result = {}
self.stats = stats
def write_result(self, data):
self._semaphore.acquire()
self.result.update(data)
self._semaphore.release()
def start_parse(self):
self.stats.connect()
self.stats.init_message_stack()
func_to_start = [
self.stats.get_top3_speakers,
self.stats.get_most_frequent_youtube_video,
self.stats.get_time_activity,
self.stats.get_abusive_expressions,
]
threads = []
for func in func_to_start:
thread = Thread(target=func, args=(self, ))
threads.append(thread)
thread.start()
for t in threads:
t.join()
return self.result
class VDOM_mutex: """ This class should be used to lock objects """ def __init__(self): """ Constructor """ self.__mutex = Semaphore(1) # self.__mutex = RLock(); # self.__counter = 0 # self.__trylock = Lock(); def lock(self): """ Lock object """ self.__mutex.acquire() # self.__counter = self.__counter + 1 # return self.__counter def unlock(self): """ Unlock object """ # self.__trylock.acquire() # cntr = self.__counter # if cntr: # self.__mutex.release() # self.__counter = self.__counter - 1 # cntr = self.__counter self.__mutex.release()
class JobManager(LogMaster):
"""
keeps a queue of jobs, and runs them in a separate thread, while keeping the number of worker thread under
a specified treshold.
it is not a real thread pool as new thread are fired every time
"""
def __init__(self, maxthreads, loglevel=logging.INFO):
self.setLogger(self.__class__.__name__, loglevel)
self.maxthreads = maxthreads
self.semaph = Semaphore(value=self.maxthreads)
self.jobq = Queue()
self.running = True
self.dispatcher = Thread(target=self._jobdispatcher, daemon=True)
self.dispatcher.start()
def putjob(self, job):
self.jobq.put(job)
def harness(self, job):
job.execute()
self.semaph.release()
def _jobdispatcher(self):
self.logger.debug("Started job dispatcher thread")
while self.running:
self.semaph.acquire()
job = self.jobq.get()
if job is None:
self.semaph.release()
continue
t = Thread(target=self.harness, args=(job,), daemon=True)
t.start()
self.logger.debug("Stopped job dispatcher thread")
class Footman:
def __init__(self, num_philosophers, num_meals):
self.num_philosophers = num_philosophers
self.num_meals = num_meals
self.forks = [Semaphore(1) for i in range(self.num_philosophers)]
self.footman = Semaphore(self.num_philosophers - 1) # at most one philosopher cannot dine
def left(self, i):
return i
def right(self, i):
return (i + 1) % self.num_philosophers
def get_forks(self, i):
self.footman.acquire()
self.forks[self.right(i)].acquire()
self.forks[self.left(i)].acquire()
def put_forks(self, i):
self.forks[self.right(i)].release()
self.forks[self.left(i)].release()
self.footman.release()
def philosopher(self, id):
while self.num_meals > 0:
self.get_forks(id)
# eating
self.num_meals -= 1
sleep(rng.random() / 100)
self.put_forks(id)
# thinking
sleep(rng.random() / 100)
class FadeInTransition(XiboTransition):
"Abstract Class - Interface for Transitions"
def run(self):
self.lock = Semaphore()
self.lock.acquire()
if self.media1 != None:
if self.options1['transOutDuration'] > 0:
self.outDuration = int(self.options1['transOutDuration'])
else:
self.outDuration = 1000
self.p.enqueue('setOpacity',(self.media1.getName(),0.0))
self.p.enqueue('anim',('fadeIn',self.media1.getName(),self.outDuration,self.next))
self.lock.acquire()
if self.media2 != None:
if self.options2['transInDuration'] > 0:
self.inDuration = int(self.options2['transInDuration'])
else:
self.inDuration = 1000
self.p.enqueue('setOpacity',(self.media2.getName(),0.0))
self.p.enqueue('anim',('fadeIn',self.media2.getName(),self.inDuration,self.next))
self.lock.acquire()
self.callback()
def next(self):
self.lock.release()
class _Result(object):
'''
Class for a result composed of partials.
Partials are stored in a dict:
```
{
start_index: chunk,
...
}
```
To get the result, a call to `get` method must be done.
The call to `get` is blocking until the result is `completed`.
Everytime a partial is added using the method
`add_partial`, the count of chunks is incremented and
compared with the expected number of chunks (given on `__init__`).
If the count is equal, then a semaphore is released.
The same semaphore is acquired on `get` call.
That's why `get` call is blocking.
'''
def __init__(self, no_chunks):
super(_Result, self).__init__()
self._n = no_chunks
self._res = {}
self._lock = Lock()
self._no_collected = 0
self._completed_sem = Semaphore(0)
@property
def completed(self):
self._lock.acquire()
comp = self._no_collected == self._n
self._lock.release()
return comp
# no setter provided
@property
def partial(self):
return self._res
def add_partial(self, key, partial):
# thread-safe
self._lock.acquire()
self._res[key] = partial
self._no_collected += 1
self._lock.release()
if self.completed:
self._completed_sem.release()
def get(self):
self._completed_sem.acquire()
# release the semaphore for further
# invokations of get()
self._completed_sem.release()
return self._res
class ThreadSemaphore(object):
def __init__(self):
self._semaphore = Semaphore(1)
self._thread = None
def acquire(self, wait=True):
if self._thread is not currentThread():
#print currentThread(), 'acquiring'
result = self._semaphore.acquire(wait)
if result:
#print currentThread(), 'got it'
self._thread = currentThread()
return result
return False
def release(self):
if self._thread is not currentThread():
raise ValueError, 'current thread did not acquire semaphore'
else:
self._thread = None
self._semaphore.release()
class BarberShop:
def __init__(self, numchairs):
self.open_seats = numchairs
self.seats_mutex = Semaphore(1)
self.customers = Semaphore(0)
self.barbers = Semaphore(0)
# check for waiting customers
# if there are none, wait
# if there are waiting customers, signal one
def barber_readytocut(self):
self.customers.acquire()
self.seats_mutex.acquire()
self.open_seats += 1
self.barbers.release()
self.seats_mutex.release()
# enter the barbershop if all numchairs are not occupied
# returns true if the customer entered successfully, and
# false if he was turned away at the door
def customer_enter(self):
self.seats_mutex.acquire()
if self.open_seats > 0:
self.open_seats -= 1
return True
else:
self.seats_mutex.release()
return False
# take a seat and wait until the barber is ready to cut hair
def customer_takeaseat(self):
self.customers.release()
self.seats_mutex.release()
self.barbers.acquire()
class OneLaneBridge(object):
"""
A one-lane bridge allows multiple cars to pass in either direction, but at any
point in time, all cars on the bridge must be going in the same direction.
Cars wishing to cross should call the cross function, once they have crossed
they should call finished()
"""
def __init__(self):
self.direction = random.randrange(2)
self.cars_on = Semaphore(0)
self.num_cars_on = 0
self.waiting = 0
def cross(self,direction):
"""wait for permission to cross the bridge. direction should be either
north (0) or south (1)."""
if (direction != self.direction) and self.num_cars_on:
print "Waiting going direction %d" % direction
self.waiting += 1
self.cars_on.acquire()
# Bridge is empty or == direction so we can take the opening and cross
self.direction = direction
self.num_cars_on += 1
def finished(self, direction):
self.num_cars_on -= 1
if not self.num_cars_on:
for car in range(self.waiting):
self.cars_on.release()
self.waiting = 0
class Synchronized:
def __init__(self):
from threading import Semaphore
self.__lock = Semaphore()
self.__ownerThread = None
classdict = self.__class__.__dict__
for attr in classdict.get("__synchronized__", ()):
try:
method = classdict[attr]
if callable(method):
self.__dict__[attr] = CallHook(self, method)
else:
if VERBOSE: print "! Synchronized: Object is not callable: %s" % (attr,)
except KeyError:
if VERBOSE: print "! Synchronized: Method not found: %s" % (attr,)
def releaseInstance(self):
self.__ownerThread = None
self.__lock.release()
def acquireInstance(self):
self.__lock.acquire()
self.__ownerThread = currentThread()
def ownerThread(self):
return self.__ownerThread
class AsyncWorker(object):
def __init__(self, view):
self.view = view
self.plugin = get_plugin(view)
self.content = view.substr(sublime.Region(0, view.size()))
self.filename = view.file_name()
self.view_id = view.buffer_id()
self.errors = None
self.sem = Semaphore()
self.sem.acquire()
self.has_round_queued = False
def do_more_work(self):
self.content = self.view.substr(sublime.Region(0, self.view.size()))
if not self.has_round_queued:
self.sem.release()
self.has_round_queued = True
def final(self):
self.plugin.handle_errors(self.view, self.errors)
self.plugin.set_error_status(self.view)
self.has_round_queued = False
def work(self):
while True:
# Wait on semaphore
self.sem.acquire()
# Update the script
self.plugin.update_server_code(self.filename, self.content)
# Get errors
self.errors = self.plugin.serv_get_errors(self.filename)
sublime.set_timeout(self.final, 1)
self.content = self.plugin.views_text[self.filename]
sleep(1.3)
class LEDManager:
def __init__(self):
self.threadStopEvent = Event()
self.sem = Semaphore()
self.controlThread = None
self._cancelPowerOffEvent = None
self._colorSetter = ColorSetter(1.0)
def setBrightness(self, brightness):
self._colorSetter.setBrightness(brightness)
def getBrightness(self):
return self._colorSetter.getBrightness()
def startProgram(self, program):
self.sem.acquire()
program.setColorSetter(self._colorSetter)
if self.controlThread is not None:
self.controlThread.threadStopEvent.set()
lastValue = self.controlThread.program.getCurrentValue()
program.setLastValue(lastValue)
self.controlThread = LEDControlThread(program)
self.controlThread.start()
self.sem.release()
def getCurrentProgram(self):
if self.controlThread is not None:
if self.controlThread.program is not None:
return self.controlThread.program
return None
def getCurrentValue(self):
if self.controlThread is not None:
if self.controlThread.program is not None:
return self.controlThread.program.getCurrentValue()
return None
def powerOffWaiter(self, duration, cancelEvent):
cancelEvent.wait(duration)
if cancelEvent.is_set():
logging.getLogger("main").info("canceled power off")
return
logging.getLogger("main").info("wait finished starting SoftOffProgram")
self.startProgram(SmoothNextColorProgram(LEDState(0.0, 0.0, 0.0, 1.0), 1, 3))
self._cancelPowerOffEvent = None
def schedulePowerOff(self, duration):
if self._cancelPowerOffEvent is not None:
self._cancelPowerOffEvent.set()
self._cancelPowerOffEvent = Event()
t = Thread(target=self.powerOffWaiter, args=(duration, self._cancelPowerOffEvent))
t.start()
def cancelPowerOff(self):
if self._cancelPowerOffEvent is not None:
self._cancelPowerOffEvent.set()
self._cancelPowerOffEvent = None
def isPowerOffScheduled(self):
return self._cancelPowerOffEvent is not None
class ObjKeeper(object):
"""
每种资源
"""
def __init__(self, max_size):
self.lock = Semaphore(max_size)
self.objs = deque()
def pop(self):
# 获取锁
self.lock.acquire()
try:
return self.objs.popleft()
except:
# 代表外面要重新生成新的
return None
def push(self, obj):
if obj:
self.objs.append(obj)
# 无论如何都要释放
self.lock.release()
class FadeInTransition(XiboTransition):
"Abstract Class - Interface for Transitions"
def run(self):
self.lock = Semaphore()
self.lock.acquire()
if not self.media1 is None:
if self.options1["transOutDuration"] > 0:
self.outDuration = int(self.options1["transOutDuration"])
else:
self.outDuration = 1000
self.p.enqueue("setOpacity", (self.media1.getName(), 0.0))
self.p.enqueue("anim", ("fadeIn", self.media1.getName(), self.outDuration, self.next))
self.lock.acquire()
if not self.media2 is None:
self.media2.start()
if not self.media2 is None:
if self.options2["transInDuration"] > 0:
self.inDuration = int(self.options2["transInDuration"])
else:
self.inDuration = 1000
self.p.enqueue("setOpacity", (self.media2.getName(), 0.0))
self.p.enqueue("anim", ("fadeIn", self.media2.getName(), self.inDuration, self.next))
self.lock.acquire()
self.callback()
def next(self):
self.lock.release()
class BinarySemaphore:
def __init__(self, initial):
self.sem = Semaphore(initial)
def wait(self):
self.sem.acquire()
def signal(self):
self.sem.release()
class IterableThread(Thread):
def __init__(self, storage_object):
Thread.__init__(self)
self.data = ''
self.storage_object = storage_object
self.notstarted = True
self.semaphore = Semaphore()
self.closed = False
def read(self, size):
while len(self.data) < size and not self.closed:
self.semaphore.acquire(True)
ret = self.data[:size]
self.data = self.data[size:]
return ret
def write(self, data):
self.data += data
if not self.isAlive(): self.start()
self.semaphore.release()
def run(self):
self.storage_object.send(self)
def close(self):
self.closed = True
self.semaphore.release()
class ReusableBarrierSem():
def __init__(self, num_threads):
self.num_threads = num_threads
self.count_threads1 = self.num_threads
self.count_threads2 = self.num_threads
self.counter_lock = Lock() # protejam decrementarea numarului de threaduri
self.threads_sem1 = Semaphore(0) # contorizam numarul de threaduri pentru prima etapa
self.threads_sem2 = Semaphore(0) # contorizam numarul de threaduri pentru a doua etapa
def wait(self):
self.phase1()
self.phase2()
def phase1(self):
with self.counter_lock:
self.count_threads1 -= 1
if self.count_threads1 == 0:
for i in range(self.num_threads):
self.threads_sem1.release()
self.count_threads2 = self.num_threads
self.threads_sem1.acquire()
def phase2(self):
with self.counter_lock:
self.count_threads2 -= 1
if self.count_threads2 == 0:
for i in range(self.num_threads):
self.threads_sem2.release()
self.count_threads1 = self.num_threads
self.threads_sem2.acquire()
class CollapseTransition(XiboTransition):
def run(self):
self.lock = Semaphore()
self.lock.acquire()
# Only valid as an exit transition
if self.media1 != None:
if self.options1['transOutDuration'] > 0:
self.outDuration = int(self.options1['transOutDuration'])
else:
self.outDuration = 1000
self.__animate__(self.media1.getName(),self.media1.getX(), self.media1.getY(),self.media1.getWidth(),self.media1.getHeight(),self.outDuration,self.next)
self.lock.acquire()
self.callback()
def next(self):
self.lock.release()
def __animate__(self,name,currentX,currentY,w,h,duration,callback):
# ('ease', nodeName, animation duration, animation attribute, start position, finish position, callback on Stop, easeIn duration, easeOut duration)
self.log.log(5,"info","CollapseTransition: Collapsing " + name + " over " + str(duration) + "ms")
self.p.enqueue('anim',('linear',name,duration,'y',currentY,int(h/2),None))
self.p.enqueue('anim',('linear',name,duration,'height',int(h),0,callback))
self.p.enqueue('timer',(duration,self.next))
class Barrier:
def __init__(self, n):
self.n = n
self.count = 0
self.mutex = Semaphore(value=1)
self.turnstile1 = Semaphore(value=0)
self.turnstile2 = Semaphore(value=0)
def phase1(self):
with self.mutex:
self.count += 1
if self.count == self.n:
logging.debug("Releasing the entry barrier.")
for _ in range(self.n):
self.turnstile1.release()
self.turnstile1.acquire()
def phase2(self):
with self.mutex:
self.count -= 1
if self.count == 1:
logging.debug("Unlocking the exit barrier.")
for _ in range(self.n):
self.turnstile2.release()
self.turnstile2.acquire()
def wait(self):
logging.debug("At the barrier.")
self.phase1()
logging.debug("Past the barrier.")
self.phase2()
class ThreadMaster:
'''
classdocs
'''
def __init__(self, maxThreads):
self.maxThreads = maxThreads
self.freeThreads = Semaphore(value=self.maxThreads)
def waitUntilAllReady(self):
for c in range(0, self.maxThreads ):
self.freeThreads.acquire()
for c in range(0, self.maxThreads ):
self.freeThreads.release()
def startFunction(self, fun, ar ):
newThread = Thread(target=fun, args=ar)
self.freeThreads.acquire()
newThread.start()
def done(self):
self.freeThreads.release();
class FadeOutTransition(XiboTransition):
def run(self):
self.lock = Semaphore()
self.lock.acquire()
if self.media1 != None:
if self.options1['transOutDuration'] > 0:
self.outDuration = int(self.options1['transOutDuration'])
else:
self.outDuration = 1000
self.log.log(5,"info","FadeOutTransition: Fading " + self.media1.getName() + " over " + str(self.outDuration) + "ms")
self.p.enqueue('anim',('fadeOut',self.media1.getName(),self.outDuration,self.next))
self.p.enqueue('timer',(self.outDuration,self.next))
self.lock.acquire()
if self.media2 != None:
if self.options2['transInDuration'] > 0:
self.inDuration = int(self.options2['transInDuration'])
else:
self.inDuration = 1000
self.log.log(5,"info","FadeOutTransition: Fading " + self.media2.getName() + " over " + str(self.inDuration) + "ms")
self.p.enqueue('anim',('fadeOut',self.media2.getName(),self.inDuration,self.next))
self.p.enqueue('timer',(self.inDuration,self.next))
self.lock.acquire()
self.log.log(5,"info","FadeOutTransition: Complete. Callback to " + str(self.callback))
self.callback()
def next(self):
self.lock.release()
def call_on_main_thread(func, *args, **kwargs):
done = Semaphore(0)
# TODO use other name than "result"
result = []
def wrapped_call():
try:
res_cb = func(*args, **kwargs)
except Exception, e:
res_cb = Callback()
res_cb(e)
if not isinstance(res_cb, Callback):
raise ValueError("Expected a monocle Callback from %r, got %r" % (func, res_cb))
@_o
def wait_for_result():
try:
res = yield res_cb
except Exception, e:
# TODO print traceback to a StringIO?
res = e
result.append(res)
done.release()
class Race:
def __init__(self):
self.stageFinishCount = 0
self.stageStartCount = 0
self.mutex = Semaphore(1) #used for mutual exclusion while writing
self.stageFinishSema = Semaphore(0)
self.stageStartSema = Semaphore(0) #This is used so that people dont try to finish the next stage until everyone has left the prev stage
def teammate_start_stage(self):
count = 0
with self.mutex:
self.stageStartCount = self.stageStartCount + 1
count = self.stageStartCount
if count < NUM_TEAMMATES:
self.stageStartSema.acquire()
else:
self.stageStartCount = 0
for i in range(NUM_TEAMMATES-1):
self.stageStartSema.release() # only last person starting the stage would release all thread.
def teammate_finish_stage(self):
count = 0 #local variable separate to each thread
with self.mutex:
self.stageFinishCount = self.stageFinishCount + 1
count = self.stageFinishCount
if count < NUM_TEAMMATES:
self.stageFinishSema.acquire()
else:
self.stageFinishCount = 0
for i in range(NUM_TEAMMATES-1):
self.stageFinishSema.release() #last teammate only can do this
class BoundedHashSet(object):
def __init__(self, capacity):
"""
Lock is a mutex or a semaphore with count = 1
This is used to guard the critical section and ensure mutual exclusion so only 1 thread
has access at a time.
Semaphore is to enforce capacity. Everytime sem.acquire() is called, capacity decrements
by 1. When sem.release() is called, capacity increments by 1. If sem.acquire() is called
when capacity == 0, it blocks.
:param capacity:
:return:
"""
self.mutex = Lock()
self.st = set()
self.sem = Semaphore(capacity)
def add(self, item):
if item not in self.st:
self.sem.acquire()
self.mutex.acquire()
self.st.add(item)
self.mutex.release()
def erase(self, item):
self.mutex.acquire()
self.st.remove(item)
self.mutex.release()
self.sem.release()
class AuthenticationController(object):
"""
This class handles User authentication required
for privileged activies, like Repository updates
and Application management.
"""
def __init__(self, mainloop):
self._authenticated = False
self._authenticated_sem = Semaphore(1)
self._mainloop = mainloop
def authenticate(self, action_id, authentication_callback):
"""
Authenticate current User asking Administrator
passwords.
authentication_callback is the function that
is called after the authentication procedure,
providing one boolean argument describing the
process result: True for authenticated, False
for not authenticated.
This method must be called from the MainLoop.
If authentication has been already
"""
self._authenticated_sem.acquire()
if self._authenticated:
try:
authentication_callback(True)
finally:
self._authenticated_sem.release()
return
def _polkit_auth_callback(authority, res, loop):
authenticated = False
try:
result = authority.check_authorization_finish(res)
if result.get_is_authorized():
authenticated = True
elif result.get_is_challenge():
authenticated = True
except GObject.GError as err:
raise err
finally:
self._authenticated = authenticated
self._authenticated_sem.release()
authentication_callback(authenticated)
# authenticated_sem will be released in the callback
authority = Polkit.Authority.get()
subject = Polkit.UnixProcess.new(os.getppid())
authority.check_authorization(
subject,
action_id,
None,
Polkit.CheckAuthorizationFlags.ALLOW_USER_INTERACTION,
None, # Gio.Cancellable()
_polkit_auth_callback,
self._mainloop)
def test():
s = Semaphore(0)
future1 = yield batchy_futures.future(self.pool.submit(acq, s))
future2 = yield batchy_futures.future(self.pool.submit(acq, s))
s.release()
yield future1
s.release()
yield future2
class RPCClient(object):
def __init__(self, addr): # server addr
self.addr = addr
self._mutex = Semaphore()
self._connected = False
def connect(self):
if "unix://" in self.addr:
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
addr = self.addr[7:]
else:
host, port = self.addr.split(':')
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
addr = (host ,int(port))
self.sock = Transport(sock)
self.sock.connect(addr)
self._connected = True
def close(self):
if hasattr(self, 'sock'):
self.sock.close()
self._connected = False
def call(self, method, *args, **kwargs):
timeout = kwargs.pop('timeout', None)
data = build_request(method, args, kwargs)
self._mutex.acquire()
try:
if not self._connected:
self.connect()
sent = self.sock.send(data)
if sent == -1:
r = build_response(None, id=data['id'], error=ConnectionClosedError())
self.close()
else:
if timeout is None:
r = self.sock.recv()
else:
rest = select.select([self.sock, ], [], [], timeout) #TODO: check message id
if rest[0]:
r = self.sock.recv()
else:
r = build_response(None, id=data['id'], error=RPCTimeout())
if not r:
self.close()
r = build_response(None, id=data['id'], error=ConnectionClosedError())
except Exception as e:
raise e
finally:
self._mutex.release()
return r
def batch_call(self):
pass
class StreamLambdaProxy(AbstractStreamProxy):
"""Invoke a lambda for each connection"""
class Connection(AbstractStreamProxy.Connection):
def __init__(self, host, port):
self.host = host
self.port = port
def close(self):
pass
def __str__(self):
return self.host + ':' + self.port
def __init__(self,
functions,
maxParallelRequests,
pubKeyFile,
streamServer,
stats,
maxIdleTimeout=1):
self.__connIdleTimeout = maxIdleTimeout
self.__functions = functions
self.__functionToClient = {}
self.__regionToClient = {}
self.__lambdaRateSemaphore = Semaphore(maxParallelRequests)
self.__lambda = boto3.client('lambda')
if 'lambda' not in stats.models:
stats.register_model('lambda', LambdaStatsModel())
self.__lambdaStats = stats.get_model('lambda')
self.__streamServer = streamServer
# Enable encryption
self.__enableEncryption = False
if pubKeyFile is not None:
with open(pubKeyFile, 'rb') as ifs:
self.__rsaCipher = PKCS1_OAEP.new(RSA.importKey(ifs.read()))
self.__enableEncryption = True
def __get_lambda_client(self, function):
"""Get a lambda client from the right region"""
client = self.__functionToClient.get(function)
if client is not None:
return client
if 'arn:' not in function:
# using function name in the default region
client = self.__lambda
self.__functionToClient[function] = client
else:
region = _get_region_from_arn(function)
client = self.__regionToClient.get(region)
if client is None:
client = boto3.client('lambda', region_name=region)
self.__regionToClient[region] = client
self.__functionToClient[function] = client
return client
def connect(self, host, port):
return StreamLambdaProxy.Connection(host, port)
def stream(self, cliSock, servInfo):
assert isinstance(servInfo, StreamLambdaProxy.Connection)
socketId = '%016x' % random.getrandbits(128)
invokeArgs = {
'stream': True,
'socketId': socketId,
'streamServer': self.__streamServer.publicHostAndPort,
'host': servInfo.host,
'port': int(servInfo.port),
'idleTimeout': self.__connIdleTimeout
}
function = random.choice(self.__functions)
lambdaClient = self.__get_lambda_client(function)
self.__lambdaRateSemaphore.acquire()
try:
self.__streamServer.take_ownership_of_socket(
socketId, cliSock, self.__connIdleTimeout)
with self.__lambdaStats.record() as billingObject:
invokeResponse = lambdaClient.invoke(
FunctionName=function,
Payload=json.dumps(invokeArgs),
LogType='Tail')
billingObject.parse_log(invokeResponse['LogResult'])
finally:
self.__lambdaRateSemaphore.release()
if invokeResponse['StatusCode'] != 200:
logger.error('%s: status=%d', invokeResponse['FunctionError'],
invokeResponse['StatusCode'])
if 'FunctionError' in invokeResponse:
logger.error('%s error: %s', invokeResponse['FunctionError'],
invokeResponse['Payload'].read())
class page:
def __init__(self):
self.tmp_q = Queue()
self.ip_q = Queue()
self.sema = Semaphore(1)
self.__init_localhost_q()
def get_ip(self):
self.sema.acquire()
url = """http://ip.11jsq.com/index.php/api/entry?method=proxyServer.generate_api_url&packid=0&fa=0&fetch_key=&qty=1&time=101&pro=&city=&port=1&format=txt&ss=1&css=&dt=1&specialTxt=3&specialJson="""
r = requests.get(url)
time.sleep(1)
self.ip_q.put(r.text)
self.sema.release()
return r.text
def __init_localhost_q(self, num=2):
self.localhost_q = Queue()
for i in range(num):
self.localhost_q.put(i)
def __init_tmp_q(self, arr):
self.tmp_q.queue.clear()
for i in arr:
self.tmp_q.put(i)
def __read_thread(self, f):
conp = self.conp
if self.localhost_q.empty():
chrome_option = webdriver.ChromeOptions()
ip = self.get_ip()
#ip="1.28.0.90:20455"
print("本次ip %s" % ip)
if re.match(
"[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}:[0-9]{1,5}",
ip) is None:
print("ip不合法")
return False
chrome_option.add_argument("--proxy-server=http://%s" % (ip))
try:
driver = webdriver.Chrome(chrome_options=chrome_option)
driver.minimize_window()
driver.set_page_load_timeout(20)
except Exception as e:
traceback.print_exc()
driver.quit()
return False
else:
try:
print("使用本机ip")
self.localhost_q.get()
driver = webdriver.Chrome()
driver.minimize_window()
driver.set_page_load_timeout(20)
except Exception as e:
traceback.print_exc()
driver.quit()
return False
while not self.tmp_q.empty():
x = self.tmp_q.get()
if x is None: continue
try:
df = f(driver, x)
self.db_write(conp, x, df)
time.sleep(0.1)
size = self.tmp_q.qsize()
if size % 100 == 0: print("还剩 %d 页" % size)
except Exception as e:
traceback.print_exc()
print("第 %s 页面异常" % x)
self.tmp_q.put(x)
driver.quit()
return False
driver.quit()
print("线程正常退出")
return True
def read_thread(self, f):
num = 10
flag = self.__read_thread(f)
while not flag and num > 0:
num -= 1
print("切换ip,本线程第%d次" % (5 - num))
print("已经消耗ip %d 个" % self.ip_q.qsize())
flag = self.__read_thread(f)
def read_threads(self, f, arr, num=10):
bg = time.time()
ths = []
dfs = []
total = len(arr)
if total <= 5: num = 1
if total != 0:
if num / total > 1:
num = int(total / 5) + 1 if int(total / 5) + 1 < 4 else num
print("本次共 %d 个页面,共%d 个线程" % (total, num))
self.__init_tmp_q(arr)
for _ in range(num):
t = Thread(target=self.read_thread, args=(f, ))
ths.append(t)
for t in ths:
t.start()
for t in ths:
t.join()
self.__init_localhost_q()
left_page = self.tmp_q.qsize()
print("剩余 %d页" % (left_page))
if left_page > 0:
self.read_thread(f)
left_page = self.tmp_q.qsize()
print("剩余 %d页" % (left_page))
ed = time.time()
cost = ed - bg
if cost < 100:
print("耗时%d 秒" % cost)
else:
print("耗时%.4f 分" % (cost / 60))
def db_write(self, conp, href, page):
dbtype = "postgresql"
if dbtype == 'postgresql':
con = psycopg2.connect(user=conp[0],
password=conp[1],
host=conp[2],
port="5432",
database=conp[3])
elif dbtype == 'mssql':
con = pymssql.connect(user=conp[0],
password=conp[1],
host=conp[2],
database=conp[3])
elif dbtype == 'oracle':
con = cx_Oracle.connect("%s/%s@%s/%s" %
(conp[0], conp[1], conp[2], conp[3]))
else:
con = MySQLdb.connect(user=conp[0],
passwd=conp[1],
host=conp[2],
db=conp[3],
charset='utf8')
# con.set_character_set('utf8')
# cur.execute('SET NAMES utf8;')
# cur.execute('SET CHARACTER SET utf8;')
# cur.execute('SET character_set_connection=utf8;')
sql = """insert into %s.%s values($lmf$%s$lmf$,$lmf$%s$lmf$)""" % (
conp[4], conp[5], href, page)
cur = con.cursor()
cur.execute(sql)
con.commit()
cur.close()
con.close()
def db_write_many(self, conp, data):
dbtype = "postgresql"
if dbtype == 'postgresql':
con = psycopg2.connect(user=conp[0],
password=conp[1],
host=conp[2],
port="5432",
database=conp[3])
elif dbtype == 'mssql':
con = pymssql.connect(user=conp[0],
password=conp[1],
host=conp[2],
database=conp[3])
elif dbtype == 'oracle':
con = cx_Oracle.connect("%s/%s@%s/%s" %
(conp[0], conp[1], conp[2], conp[3]))
else:
con = MySQLdb.connect(user=conp[0],
passwd=conp[1],
host=conp[2],
db=conp[3],
charset='utf8')
# cur=con.cursor()
# con.set_character_set('utf8')
# cur.execute('SET NAMES utf8;')
# cur.execute('SET CHARACTER SET utf8;')
# cur.execute('SET character_set_connection=utf8;')
sql = """insert into %s.%s values(href,page)""" % (conp[4], conp[5])
cur = con.cursor()
cur.executemany(sql, data)
con.commit()
cur.close()
con.close()
def db_command(self, sql, conp):
"""db_command 仅仅到数据库"""
dbtype = "postgresql"
if dbtype == 'postgresql':
con = psycopg2.connect(user=conp[0],
password=conp[1],
host=conp[2],
port="5432",
database=conp[3])
elif dbtype == 'mssql':
con = pymssql.connect(user=conp[0],
password=conp[1],
host=conp[2],
database=conp[3])
elif dbtype == 'oracle':
con = cx_Oracle.connect("%s/%s@%s/%s" %
(conp[0], conp[1], conp[2], conp[3]))
else:
con = MySQLdb.connect(user=conp[0],
passwd=conp[1],
host=conp[2],
db=conp[3],
charset='utf8')
# con.set_character_set('utf8')
# cur.execute('SET NAMES utf8;')
# cur.execute('SET CHARACTER SET utf8;')
# cur.execute('SET character_set_connection=utf8;')
cur = con.cursor()
cur.execute(sql)
con.commit()
cur.close()
con.close()
def write(self, **arg):
tb = arg['tb']
conp = arg["conp"]
f = arg["f"]
num = arg["num"]
arr = arg["arr"]
sql = "create table if not exists %s.%s(href text,page text)" % (
conp[4], tb)
self.db_command(sql, conp)
print("创建表if不存在")
conp.append(tb)
print(conp)
self.conp = conp
self.read_threads(f=f, num=num, arr=arr)
return self.tmp_q.qsize()
class BrowserView(QMainWindow):
instances = {}
inspector_port = None # The localhost port at which the Remote debugger listens
create_window_trigger = QtCore.pyqtSignal(object)
set_title_trigger = QtCore.pyqtSignal(str)
load_url_trigger = QtCore.pyqtSignal(str)
html_trigger = QtCore.pyqtSignal(str, str)
dialog_trigger = QtCore.pyqtSignal(int, str, bool, str, str)
destroy_trigger = QtCore.pyqtSignal()
hide_trigger = QtCore.pyqtSignal()
show_trigger = QtCore.pyqtSignal()
fullscreen_trigger = QtCore.pyqtSignal()
window_size_trigger = QtCore.pyqtSignal(int, int)
window_move_trigger = QtCore.pyqtSignal(int, int)
window_minimize_trigger = QtCore.pyqtSignal()
window_restore_trigger = QtCore.pyqtSignal()
current_url_trigger = QtCore.pyqtSignal()
evaluate_js_trigger = QtCore.pyqtSignal(str, str)
on_top_trigger = QtCore.pyqtSignal(bool)
class JSBridge(QtCore.QObject):
qtype = QtCore.QJsonValue if is_webengine else str
def __init__(self):
super(BrowserView.JSBridge, self).__init__()
@QtCore.pyqtSlot(str, qtype, str, result=str)
def call(self, func_name, param, value_id):
func_name = BrowserView._convert_string(func_name)
param = BrowserView._convert_string(param)
return js_bridge_call(self.window, func_name, json.loads(param),
value_id)
class WebView(QWebView):
def __init__(self, parent=None):
super(BrowserView.WebView, self).__init__(parent)
if parent.frameless and parent.easy_drag:
QApplication.instance().installEventFilter(self)
self.setMouseTracking(True)
def contextMenuEvent(self, event):
menu = self.page().createStandardContextMenu()
# If 'Inspect Element' is present in the default context menu, it
# means the inspector is already up and running.
for i in menu.actions():
if i.text() == "Inspect Element":
break
else:
# Inspector is not up yet, so create a pseudo 'Inspect Element'
# menu that will fire it up.
inspect_element = QAction("Inspect Element", menu)
inspect_element.triggered.connect(self.show_inspector)
menu.addAction(inspect_element)
menu.exec_(event.globalPos())
# Create a new webviewb window pointing at the Remote debugger server
def show_inspector(self):
uid = self.parent().uid + "-inspector"
try:
# If inspector already exists, bring it to the front
BrowserView.instances[uid].raise_()
BrowserView.instances[uid].activateWindow()
except KeyError:
title = "Web Inspector - {}".format(self.parent().title)
url = "http://localhost:{}".format(BrowserView.inspector_port)
window = Window(
"web_inspector",
title,
url,
"",
700,
500,
None,
None,
True,
False,
(300, 200),
False,
False,
False,
False,
False,
"#fff",
None,
False,
False,
)
inspector = BrowserView(window)
inspector.show()
def mousePressEvent(self, event):
if event.button() == QtCore.Qt.LeftButton:
self.drag_pos = (event.globalPos() -
self.parent().frameGeometry().topLeft())
event.accept()
def mouseMoveEvent(self, event):
parent = self.parent()
if (parent.frameless and parent.easy_drag
and int(event.buttons()) == 1): # left button is pressed
parent.move(event.globalPos() - self.drag_pos)
def eventFilter(self, object, event):
if object.parent() == self:
if event.type() == QtCore.QEvent.MouseMove:
self.mouseMoveEvent(event)
elif event.type() == QtCore.QEvent.MouseButtonPress:
self.mousePressEvent(event)
return False
# New-window-requests handler for Qt 5.5+ only
class NavigationHandler(QWebPage):
def __init__(self, parent=None):
super(BrowserView.NavigationHandler, self).__init__(parent)
def acceptNavigationRequest(self, url, type, is_main_frame):
webbrowser.open(url.toString(), 2, True)
return False
class WebPage(QWebPage):
def __init__(self, parent=None):
super(BrowserView.WebPage, self).__init__(parent)
if is_webengine:
self.featurePermissionRequested.connect(
self.onFeaturePermissionRequested)
self.nav_handler = BrowserView.NavigationHandler(self)
else:
self.nav_handler = None
if is_webengine:
def onFeaturePermissionRequested(self, url, feature):
if feature in (
QWebPage.MediaAudioCapture,
QWebPage.MediaVideoCapture,
QWebPage.MediaAudioVideoCapture,
):
self.setFeaturePermission(url, feature,
QWebPage.PermissionGrantedByUser)
else:
self.setFeaturePermission(url, feature,
QWebPage.PermissionDeniedByUser)
else:
def acceptNavigationRequest(self, frame, request, type):
if frame is None:
webbrowser.open(request.url().toString(), 2, True)
return False
return True
def userAgentForUrl(self, url):
user_agent = settings.get("user_agent") or _user_agent
if user_agent:
return user_agent
else:
return super().userAgentForUrl(url)
def createWindow(self, type):
return self.nav_handler
def __init__(self, window):
super(BrowserView, self).__init__()
BrowserView.instances[window.uid] = self
self.uid = window.uid
self.pywebview_window = window
self.js_bridge = BrowserView.JSBridge()
self.js_bridge.window = window
self.is_fullscreen = False
self.confirm_close = window.confirm_close
self.text_select = window.text_select
self._file_name_semaphore = Semaphore(0)
self._current_url_semaphore = Semaphore(0)
self.loaded = window.loaded
self.shown = window.shown
self._js_results = {}
self._current_url = None
self._file_name = None
self.resize(window.initial_width, window.initial_height)
self.title = window.title
self.setWindowTitle(window.title)
# Set window background color
self.background_color = QColor()
self.background_color.setNamedColor(window.background_color)
palette = self.palette()
palette.setColor(self.backgroundRole(), self.background_color)
self.setPalette(palette)
if not window.resizable:
self.setFixedSize(window.initial_width, window.initial_height)
self.setMinimumSize(window.min_size[0], window.min_size[1])
self.frameless = window.frameless
self.easy_drag = window.easy_drag
flags = self.windowFlags()
if self.frameless:
flags = flags | QtCore.Qt.FramelessWindowHint
if window.on_top:
flags = flags | QtCore.Qt.WindowStaysOnTopHint
self.setWindowFlags(flags)
self.view = BrowserView.WebView(self)
if is_webengine:
os.environ[
"QTWEBENGINE_CHROMIUM_FLAGS"] = "--use-fake-ui-for-media-stream --enable-features=AutoplayIgnoreWebAudio"
if _debug and is_webengine:
# Initialise Remote debugging (need to be done only once)
if not BrowserView.inspector_port:
BrowserView.inspector_port = BrowserView._get_debug_port()
os.environ[
"QTWEBENGINE_REMOTE_DEBUGGING"] = BrowserView.inspector_port
else:
self.view.setContextMenuPolicy(
QtCore.Qt.NoContextMenu) # disable right click context menu
self.view.setPage(BrowserView.WebPage(self.view))
self.view.page().loadFinished.connect(self.on_load_finished)
self.setCentralWidget(self.view)
self.create_window_trigger.connect(BrowserView.on_create_window)
self.load_url_trigger.connect(self.on_load_url)
self.html_trigger.connect(self.on_load_html)
self.dialog_trigger.connect(self.on_file_dialog)
self.destroy_trigger.connect(self.on_destroy_window)
self.show_trigger.connect(self.on_show_window)
self.hide_trigger.connect(self.on_hide_window)
self.fullscreen_trigger.connect(self.on_fullscreen)
self.window_size_trigger.connect(self.on_window_size)
self.window_move_trigger.connect(self.on_window_move)
self.window_minimize_trigger.connect(self.on_window_minimize)
self.window_restore_trigger.connect(self.on_window_restore)
self.current_url_trigger.connect(self.on_current_url)
self.evaluate_js_trigger.connect(self.on_evaluate_js)
self.set_title_trigger.connect(self.on_set_title)
self.on_top_trigger.connect(self.on_set_on_top)
if is_webengine and platform.system() != "OpenBSD":
self.channel = QWebChannel(self.view.page())
self.view.page().setWebChannel(self.channel)
if window.fullscreen:
self.toggle_fullscreen()
if window.url is not None:
self.view.setUrl(QtCore.QUrl(window.url))
elif window.html:
self.view.setHtml(window.html, QtCore.QUrl(""))
else:
self.view.setHtml(default_html, QtCore.QUrl(""))
if window.initial_x is not None and window.initial_y is not None:
self.move(window.initial_x, window.initial_y)
else:
center = (QApplication.desktop().availableGeometry().center() -
self.rect().center())
self.move(center.x(), center.y())
if not window.minimized:
self.activateWindow()
self.raise_()
self.shown.set()
def on_set_title(self, title):
self.setWindowTitle(title)
def on_file_dialog(self, dialog_type, directory, allow_multiple,
save_filename, file_filter):
if dialog_type == FOLDER_DIALOG:
self._file_name = QFileDialog.getExistingDirectory(
self,
localization["linux.openFolder"],
options=QFileDialog.ShowDirsOnly)
elif dialog_type == OPEN_DIALOG:
if allow_multiple:
self._file_name = QFileDialog.getOpenFileNames(
self, localization["linux.openFiles"], directory,
file_filter)
else:
self._file_name = QFileDialog.getOpenFileName(
self, localization["linux.openFile"], directory,
file_filter)
elif dialog_type == SAVE_DIALOG:
if directory:
save_filename = os.path.join(str(directory),
str(save_filename))
self._file_name = QFileDialog.getSaveFileName(
self, localization["global.saveFile"], save_filename,
file_filter)
self._file_name_semaphore.release()
def on_current_url(self):
url = BrowserView._convert_string(self.view.url().toString())
self._current_url = (None if url == ""
or url.startswith("data:text/html") else url)
self._current_url_semaphore.release()
def on_load_url(self, url):
self.view.setUrl(QtCore.QUrl(url))
def on_load_html(self, content, base_uri):
self.view.setHtml(content, QtCore.QUrl(base_uri))
def on_set_on_top(self, top):
flags = self.windowFlags()
if top:
self.setWindowFlags(flags | QtCore.Qt.WindowStaysOnTopHint)
else:
self.setWindowFlags(flags & ~QtCore.Qt.WindowStaysOnTopHint)
self.show()
def closeEvent(self, event):
self.pywebview_window.closing.set()
if self.confirm_close:
reply = QMessageBox.question(
self,
self.title,
localization["global.quitConfirmation"],
QMessageBox.Yes,
QMessageBox.No,
)
if reply == QMessageBox.No:
event.ignore()
return
event.accept()
BrowserView.instances[self.uid].close()
del BrowserView.instances[self.uid]
if self.pywebview_window in windows:
windows.remove(self.pywebview_window)
self.pywebview_window.closed.set()
if len(BrowserView.instances) == 0:
self.hide()
_app.exit()
def on_show_window(self):
self.show()
def on_hide_window(self):
self.hide()
def on_destroy_window(self):
self.close()
def on_fullscreen(self):
if self.is_fullscreen:
self.showNormal()
else:
self.showFullScreen()
self.is_fullscreen = not self.is_fullscreen
def on_window_size(self, width, height):
self.setFixedSize(width, height)
def on_window_move(self, x, y):
self.move(x, y)
def on_window_minimize(self):
self.setWindowState(QtCore.Qt.WindowMinimized)
def on_window_restore(self):
self.setWindowState(QtCore.Qt.WindowNoState)
self.raise_()
self.activateWindow()
def on_evaluate_js(self, script, uuid):
def return_result(result):
result = BrowserView._convert_string(result)
uuid_ = BrowserView._convert_string(uuid)
js_result = self._js_results[uuid_]
js_result["result"] = (
None if result is None or result == "null" else
result if result == "" else json.loads(result))
js_result["semaphore"].release()
try: # < Qt5.6
self.view.page().runJavaScript(script, return_result)
except AttributeError:
result = self.view.page().mainFrame().evaluateJavaScript(script)
return_result(result)
except Exception as e:
logger.exception(e)
def on_load_finished(self):
self._set_js_api()
if not self.text_select:
script = disable_text_select.replace("\n", "")
try:
self.view.page().runJavaScript(script)
except: # QT < 5.6
self.view.page().mainFrame().evaluateJavaScript(script)
def set_title(self, title):
self.set_title_trigger.emit(title)
def get_current_url(self):
self.loaded.wait()
self.current_url_trigger.emit()
self._current_url_semaphore.acquire()
return self._current_url
def load_url(self, url):
self.loaded.clear()
self.load_url_trigger.emit(url)
def load_html(self, content, base_uri):
self.loaded.clear()
self.html_trigger.emit(content, base_uri)
def create_file_dialog(self, dialog_type, directory, allow_multiple,
save_filename, file_filter):
self.dialog_trigger.emit(dialog_type, directory, allow_multiple,
save_filename, file_filter)
self._file_name_semaphore.acquire()
if dialog_type == FOLDER_DIALOG:
file_names = (self._file_name, )
elif dialog_type == SAVE_DIALOG or not allow_multiple:
file_names = (self._file_name[0], )
else:
file_names = tuple(self._file_name[0])
# Check if we got an empty tuple, or a tuple with empty string
if len(file_names) == 0 or len(file_names[0]) == 0:
return None
else:
return file_names
def hide_(self):
self.hide_trigger.emit()
def show_(self):
self.show_trigger.emit()
def destroy_(self):
self.destroy_trigger.emit()
def toggle_fullscreen(self):
self.fullscreen_trigger.emit()
def resize_(self, width, height):
self.window_size_trigger.emit(width, height)
def move_window(self, x, y):
self.window_move_trigger.emit(x, y)
def minimize(self):
self.window_minimize_trigger.emit()
def restore(self):
self.window_restore_trigger.emit()
def set_on_top(self, top):
self.on_top_trigger.emit(top)
def evaluate_js(self, script):
self.loaded.wait()
result_semaphore = Semaphore(0)
unique_id = uuid1().hex
self._js_results[unique_id] = {
"semaphore": result_semaphore,
"result": ""
}
self.evaluate_js_trigger.emit(script, unique_id)
result_semaphore.acquire()
result = deepcopy(self._js_results[unique_id]["result"])
del self._js_results[unique_id]
return result
def _set_js_api(self):
def _register_window_object():
frame.addToJavaScriptWindowObject("external", self.js_bridge)
code = "qtwebengine" if is_webengine else "qtwebkit"
script = parse_api_js(self.js_bridge.window, code)
if is_webengine:
qwebchannel_js = QtCore.QFile("://qtwebchannel/qwebchannel.js")
if qwebchannel_js.open(QtCore.QFile.ReadOnly):
source = bytes(qwebchannel_js.readAll()).decode("utf-8")
self.view.page().runJavaScript(source)
self.channel.registerObject("external", self.js_bridge)
qwebchannel_js.close()
else:
frame = self.view.page().mainFrame()
_register_window_object()
try:
self.view.page().runJavaScript(script)
except AttributeError: # < QT 5.6
self.view.page().mainFrame().evaluateJavaScript(script)
self.loaded.set()
@staticmethod
def _convert_string(result):
try:
if result is None or result.isNull():
return None
result = result.toString() # QJsonValue conversion
except AttributeError:
pass
return convert_string(result)
@staticmethod
def _get_debug_port():
"""
Check if default debug port 8228 is available,
increment it by 1 until a port is available.
:return: port: str
"""
port_available = False
port = 8228
while not port_available:
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(("localhost", port))
port_available = True
except:
port_available = False
logger.warning("Port %s is in use" % port)
port += 1
finally:
sock.close()
return str(port)
@staticmethod
# Receive func from subthread and execute it on the main thread
def on_create_window(func):
func()
class BrowserView(QMainWindow):
instances = {}
inspector_port = None # The localhost port at which the Remote debugger listens
create_window_trigger = QtCore.pyqtSignal(object)
set_title_trigger = QtCore.pyqtSignal(str)
load_url_trigger = QtCore.pyqtSignal(str)
html_trigger = QtCore.pyqtSignal(str, str)
dialog_trigger = QtCore.pyqtSignal(int, str, bool, str, str)
destroy_trigger = QtCore.pyqtSignal()
fullscreen_trigger = QtCore.pyqtSignal()
window_size_trigger = QtCore.pyqtSignal(int, int)
window_move_trigger = QtCore.pyqtSignal(int, int)
current_url_trigger = QtCore.pyqtSignal()
evaluate_js_trigger = QtCore.pyqtSignal(str, str)
class JSBridge(QtCore.QObject):
qtype = QtCore.QJsonValue if is_webengine else str
def __init__(self):
super(BrowserView.JSBridge, self).__init__()
@QtCore.pyqtSlot(str, qtype, str, result=str)
def call(self, func_name, param, value_id):
func_name = BrowserView._convert_string(func_name)
param = BrowserView._convert_string(param)
return js_bridge_call(self.window, func_name, param, value_id)
class WebView(QWebView):
def __init__(self, parent=None):
super(BrowserView.WebView, self).__init__(parent)
if parent.frameless:
QApplication.instance().installEventFilter(self)
self.setMouseTracking(True)
def contextMenuEvent(self, event):
menu = self.page().createStandardContextMenu()
# If 'Inspect Element' is present in the default context menu, it
# means the inspector is already up and running.
for i in menu.actions():
if i.text() == 'Inspect Element':
break
else:
# Inspector is not up yet, so create a pseudo 'Inspect Element'
# menu that will fire it up.
inspect_element = QAction('Inspect Element', menu)
inspect_element.triggered.connect(self.show_inspector)
menu.addAction(inspect_element)
menu.exec_(event.globalPos())
# Create a new webview window pointing at the Remote debugger server
def show_inspector(self):
uid = self.parent().uid + '-inspector'
try:
# If inspector already exists, bring it to the front
BrowserView.instances[uid].raise_()
BrowserView.instances[uid].activateWindow()
except KeyError:
title = 'Web Inspector - {}'.format(self.parent().title)
url = 'http://localhost:{}'.format(BrowserView.inspector_port)
window = Window('web_inspector', title, url, '', 700, 500,
True, False, (300, 200), False, '#fff', None,
False, False)
inspector = BrowserView(window)
inspector.show()
def mousePressEvent(self, event):
if event.button() == QtCore.Qt.LeftButton:
self.drag_pos = event.globalPos() - self.parent(
).frameGeometry().topLeft()
event.accept()
def mouseMoveEvent(self, event):
if self.parent().frameless and int(
event.buttons()) == 1: # left button is pressed
self.parent().move(event.globalPos() - self.drag_pos)
def eventFilter(self, object, event):
if object.parent() == self:
if event.type() == QtCore.QEvent.MouseMove:
self.mouseMoveEvent(event)
elif event.type() == QtCore.QEvent.MouseButtonPress:
self.mousePressEvent(event)
return False
# New-window-requests handler for Qt 5.5+ only
class NavigationHandler(QWebPage):
def __init__(self, parent=None):
super(BrowserView.NavigationHandler, self).__init__(parent)
def acceptNavigationRequest(self, url, type, is_main_frame):
webbrowser.open(url.toString(), 2, True)
return False
class WebPage(QWebPage):
def __init__(self, parent=None):
super(BrowserView.WebPage, self).__init__(parent)
self.nav_handler = BrowserView.NavigationHandler(
self) if is_webengine else None
if not is_webengine:
def acceptNavigationRequest(self, frame, request, type):
if frame is None:
webbrowser.open(request.url().toString(), 2, True)
return False
return True
def createWindow(self, type):
return self.nav_handler
def __init__(self, window):
super(BrowserView, self).__init__()
BrowserView.instances[window.uid] = self
self.uid = window.uid
self.pywebview_window = window
self.js_bridge = BrowserView.JSBridge()
self.js_bridge.window = window
self.is_fullscreen = False
self.confirm_close = window.confirm_close
self.text_select = window.text_select
self._file_name_semaphore = Semaphore(0)
self._current_url_semaphore = Semaphore(0)
self.loaded = window.loaded
self.shown = window.shown
self._js_results = {}
self._current_url = None
self._file_name = None
self.resize(window.width, window.height)
self.title = window.title
self.setWindowTitle(window.title)
# Set window background color
self.background_color = QColor()
self.background_color.setNamedColor(window.background_color)
palette = self.palette()
palette.setColor(self.backgroundRole(), self.background_color)
self.setPalette(palette)
if not window.resizable:
self.setFixedSize(window.width, window.height)
self.setMinimumSize(window.min_size[0], window.min_size[1])
self.frameless = window.frameless
if self.frameless:
self.setWindowFlags(QtCore.Qt.Window
| QtCore.Qt.FramelessWindowHint)
self.view = BrowserView.WebView(self)
if _debug and is_webengine:
# Initialise Remote debugging (need to be done only once)
if not BrowserView.inspector_port:
BrowserView.inspector_port = BrowserView._get_free_port()
os.environ[
'QTWEBENGINE_REMOTE_DEBUGGING'] = BrowserView.inspector_port
else:
self.view.setContextMenuPolicy(
QtCore.Qt.NoContextMenu) # disable right click context menu
self.view.setPage(BrowserView.WebPage(self.view))
self.view.page().loadFinished.connect(self.on_load_finished)
self.setCentralWidget(self.view)
self.create_window_trigger.connect(BrowserView.on_create_window)
self.load_url_trigger.connect(self.on_load_url)
self.html_trigger.connect(self.on_load_html)
self.dialog_trigger.connect(self.on_file_dialog)
self.destroy_trigger.connect(self.on_destroy_window)
self.fullscreen_trigger.connect(self.on_fullscreen)
self.window_size_trigger.connect(self.on_window_size)
self.window_move_trigger.connect(self.on_window_move)
self.current_url_trigger.connect(self.on_current_url)
self.evaluate_js_trigger.connect(self.on_evaluate_js)
self.set_title_trigger.connect(self.on_set_title)
if is_webengine and platform.system() != 'OpenBSD':
self.channel = QWebChannel(self.view.page())
self.view.page().setWebChannel(self.channel)
if window.fullscreen:
self.toggle_fullscreen()
if window.url is not None:
self.view.setUrl(QtCore.QUrl(window.url))
elif window.html:
self.view.setHtml(window.html, QtCore.QUrl(''))
else:
self.view.setHtml(default_html, QtCore.QUrl(''))
if window.x is not None and window.y is not None:
self.move(window.x, window.y)
else:
center = QApplication.desktop().availableGeometry().center(
) - self.rect().center()
self.move(center.x(), center.y())
self.activateWindow()
self.raise_()
self.shown.set()
def on_set_title(self, title):
self.setWindowTitle(title)
def on_file_dialog(self, dialog_type, directory, allow_multiple,
save_filename, file_filter):
if dialog_type == FOLDER_DIALOG:
self._file_name = QFileDialog.getExistingDirectory(
self,
localization['linux.openFolder'],
options=QFileDialog.ShowDirsOnly)
elif dialog_type == OPEN_DIALOG:
if allow_multiple:
self._file_name = QFileDialog.getOpenFileNames(
self, localization['linux.openFiles'], directory,
file_filter)
else:
self._file_name = QFileDialog.getOpenFileName(
self, localization['linux.openFile'], directory,
file_filter)
elif dialog_type == SAVE_DIALOG:
if directory:
save_filename = os.path.join(str(directory),
str(save_filename))
self._file_name = QFileDialog.getSaveFileName(
self, localization['global.saveFile'], save_filename)
self._file_name_semaphore.release()
def on_current_url(self):
url = BrowserView._convert_string(self.view.url().toString())
self._current_url = None if url == '' or url.startswith(
'data:text/html') else url
self._current_url_semaphore.release()
def on_load_url(self, url):
self.view.setUrl(QtCore.QUrl(url))
def on_load_html(self, content, base_uri):
self.view.setHtml(content, QtCore.QUrl(base_uri))
def closeEvent(self, event):
if self.confirm_close:
reply = QMessageBox.question(
self, self.title, localization['global.quitConfirmation'],
QMessageBox.Yes, QMessageBox.No)
if reply == QMessageBox.No:
event.ignore()
return
event.accept()
del BrowserView.instances[self.uid]
if self.pywebview_window in windows:
windows.remove(self.pywebview_window)
try: # Close inspector if open
BrowserView.instances[self.uid + '-inspector'].close()
del BrowserView.instances[self.uid + '-inspector']
except KeyError:
pass
if len(BrowserView.instances) == 0:
self.hide()
_app.exit()
def on_destroy_window(self):
self.close()
def on_fullscreen(self):
if self.is_fullscreen:
self.showNormal()
else:
self.showFullScreen()
self.is_fullscreen = not self.is_fullscreen
def on_window_size(self, width, height):
self.setFixedSize(width, height)
def on_window_move(self, x, y):
self.move(x, y)
def on_evaluate_js(self, script, uuid):
def return_result(result):
result = BrowserView._convert_string(result)
uuid_ = BrowserView._convert_string(uuid)
js_result = self._js_results[uuid_]
js_result[
'result'] = None if result is None or result == 'null' else result if result == '' else json.loads(
result)
js_result['semaphore'].release()
try: # < Qt5.6
result = self.view.page().mainFrame().evaluateJavaScript(script)
return_result(result)
except AttributeError:
self.view.page().runJavaScript(script, return_result)
except Exception as e:
print(e)
def on_load_finished(self):
self._set_js_api()
if not self.text_select:
script = disable_text_select.replace('\n', '')
try: # QT < 5.6
self.view.page().mainFrame().evaluateJavaScript(script)
except AttributeError:
self.view.page().runJavaScript(script)
def set_title(self, title):
self.set_title_trigger.emit(title)
def get_current_url(self):
self.loaded.wait()
self.current_url_trigger.emit()
self._current_url_semaphore.acquire()
return self._current_url
def load_url(self, url):
self.loaded.clear()
self.load_url_trigger.emit(url)
def load_html(self, content, base_uri):
self.loaded.clear()
self.html_trigger.emit(content, base_uri)
def create_file_dialog(self, dialog_type, directory, allow_multiple,
save_filename, file_filter):
self.dialog_trigger.emit(dialog_type, directory, allow_multiple,
save_filename, file_filter)
self._file_name_semaphore.acquire()
if dialog_type == FOLDER_DIALOG:
file_names = (self._file_name, )
elif dialog_type == SAVE_DIALOG or not allow_multiple:
file_names = (self._file_name[0], )
else:
file_names = tuple(self._file_name[0])
# Check if we got an empty tuple, or a tuple with empty string
if len(file_names) == 0 or len(file_names[0]) == 0:
return None
else:
return file_names
def destroy_(self):
self.destroy_trigger.emit()
def toggle_fullscreen(self):
self.fullscreen_trigger.emit()
def set_window_size(self, width, height):
self.window_size_trigger.emit(width, height)
def move_window(self, x, y):
self.window_move_trigger.emit(x, y)
def evaluate_js(self, script):
self.loaded.wait()
result_semaphore = Semaphore(0)
unique_id = uuid1().hex
self._js_results[unique_id] = {
'semaphore': result_semaphore,
'result': ''
}
self.evaluate_js_trigger.emit(script, unique_id)
result_semaphore.acquire()
result = deepcopy(self._js_results[unique_id]['result'])
del self._js_results[unique_id]
return result
def _set_js_api(self):
def _register_window_object():
frame.addToJavaScriptWindowObject('external', self.js_bridge)
code = 'qtwebengine' if is_webengine else 'qtwebkit'
script = parse_api_js(self.js_bridge.window.js_api, code)
if is_webengine:
qwebchannel_js = QtCore.QFile('://qtwebchannel/qwebchannel.js')
if qwebchannel_js.open(QtCore.QFile.ReadOnly):
source = bytes(qwebchannel_js.readAll()).decode('utf-8')
self.view.page().runJavaScript(source)
self.channel.registerObject('external', self.js_bridge)
qwebchannel_js.close()
else:
frame = self.view.page().mainFrame()
_register_window_object()
try: # < QT 5.6
self.view.page().mainFrame().evaluateJavaScript(script)
except AttributeError:
self.view.page().runJavaScript(script)
self.loaded.set()
@staticmethod
def _convert_string(result):
try:
if result is None or result.isNull():
return None
result = result.toString() # QJsonValue conversion
except AttributeError:
pass
return convert_string(result)
@staticmethod
# A simple function to obtain an unused localhost port from the os return it
def _get_free_port():
s = socket()
s.bind(('localhost', 0))
port = str(s.getsockname()[1])
s.close()
return port
@staticmethod
# Receive func from subthread and execute it on the main thread
def on_create_window(func):
func()
class MSHTML:
alert = None
class JSBridge(IWebBrowserInterop):
__namespace__ = 'MSHTML.JSBridge'
window = None
def call(self, func_name, param, value_id):
return js_bridge_call(self.window, func_name, json.loads(param),
value_id)
def alert(self, message):
MSHTML.alert(message)
def console(self, message):
print(message)
def __init__(self, form, window, alert):
self.pywebview_window = window
self.web_browser = WebBrowserEx()
self.web_browser.Dock = WinForms.DockStyle.Fill
self.web_browser.ScriptErrorsSuppressed = not _debug['mode']
self.web_browser.IsWebBrowserContextMenuEnabled = _debug['mode']
self.web_browser.WebBrowserShortcutsEnabled = False
self.web_browser.DpiAware = True
MSHTML.alert = alert
user_agent = _user_agent or settings.get('user_agent')
if user_agent:
self.web_browser.ChangeUserAgent(user_agent)
self.web_browser.ScriptErrorsSuppressed = not _debug['mode']
self.web_browser.IsWebBrowserContextMenuEnabled = _debug['mode']
self.js_result_semaphore = Semaphore(0)
self.js_bridge = MSHTML.JSBridge()
self.js_bridge.window = window
self.web_browser.ObjectForScripting = self.js_bridge
# HACK. Hiding the WebBrowser is needed in order to show a non-default background color. Tweaking the Visible property
# results in showing a non-responsive control, until it is loaded fully. To avoid this, we need to disable this behaviour
# for the default background color.
if window.background_color != '#FFFFFF':
self.web_browser.Visible = False
self.first_load = True
else:
self.first_load = False
self.cancel_back = False
self.web_browser.PreviewKeyDown += self.on_preview_keydown
self.web_browser.Navigating += self.on_navigating
self.web_browser.NewWindow3 += self.on_new_window
self.web_browser.DownloadComplete += self.on_download_complete
self.web_browser.DocumentCompleted += self.on_document_completed
if window.real_url:
self.web_browser.Navigate(window.real_url)
elif window.html:
self.web_browser.DocumentText = window.html
else:
self.web_browser.DocumentText = default_html
self.form = form
form.Controls.Add(self.web_browser)
def evaluate_js(self, script):
result = self.web_browser.Document.InvokeScript('eval', (script, ))
#self.js_result = None if result is None or result is 'null' else json.loads(result)
self.js_result = None if result is None or result == 'null' else json.loads(
result) ##
self.js_result_semaphore.release()
def load_html(self, content, base_uri):
self.web_browser.DocumentText = inject_base_uri(content, base_uri)
self.pywebview_window.loaded.clear()
def load_url(self, url):
self.web_browser.Navigate(url)
def on_preview_keydown(self, sender, args):
if args.KeyCode == WinForms.Keys.Back:
self.cancel_back = True
elif args.KeyCode == WinForms.Keys.Delete:
self.web_browser.Document.ExecCommand('Delete', False, None)
elif args.Modifiers == WinForms.Keys.Control and args.KeyCode == WinForms.Keys.C:
self.web_browser.Document.ExecCommand('Copy', False, None)
elif args.Modifiers == WinForms.Keys.Control and args.KeyCode == WinForms.Keys.X:
self.web_browser.Document.ExecCommand('Cut', False, None)
elif args.Modifiers == WinForms.Keys.Control and args.KeyCode == WinForms.Keys.V:
self.web_browser.Document.ExecCommand('Paste', False, None)
elif args.Modifiers == WinForms.Keys.Control and args.KeyCode == WinForms.Keys.Z:
self.web_browser.Document.ExecCommand('Undo', False, None)
elif args.Modifiers == WinForms.Keys.Control and args.KeyCode == WinForms.Keys.A:
self.web_browser.Document.ExecCommand('selectAll', False, None)
def on_new_window(self, sender, args):
args.Cancel = True
webbrowser.open(args.Url)
def on_download_complete(self, sender, args):
pass
def on_navigating(self, sender, args):
if self.cancel_back:
args.Cancel = True
self.cancel_back = False
def on_document_completed(self, sender, args):
document = self.web_browser.Document
document.InvokeScript('eval', (alert.src, ))
if _debug['mode']:
document.InvokeScript('eval', (
'window.console = { log: function(msg) { window.external.console(JSON.stringify(msg)) }}',
))
if self.first_load:
self.web_browser.Visible = True
self.first_load = False
self.url = None if args.Url.AbsoluteUri == 'about:blank' else str(
args.Url.AbsoluteUri)
document.InvokeScript(
'eval', (parse_api_js(self.pywebview_window, 'mshtml'), ))
if not self.pywebview_window.text_select:
document.InvokeScript('eval', (disable_text_select, ))
self.pywebview_window.loaded.set()
if self.pywebview_window.easy_drag:
document.MouseMove += self.on_mouse_move
def on_mouse_move(self, sender, e):
if e.MouseButtonsPressed == WinForms.MouseButtons.Left:
WebBrowserEx.ReleaseCapture()
windll.user32.SendMessageW(self.form.Handle.ToInt32(),
WebBrowserEx.WM_NCLBUTTONDOWN,
WebBrowserEx.HT_CAPTION, 6)
class OneLaneBridge(object):
"""
A one-lane bridge allows multiple cars to pass in either direction, but at any
point in time, all cars on the bridge must be going in the same direction.
Cars wishing to cross should call the cross function, once they have crossed
they should call finished()
"""
def __init__(self):
# TODO
self.status = 1 # 0 for left and 1 for right. Cars' direction on bridge.
self.driving_left_num = 0
self.waiting_to_left_num = 0
self.sema_left = Semaphore(0)
self.driving_right_num = 0
self.waiting_to_right_num = 0
self.sema_right = Semaphore(0)
# variables for debug:
self.start_count = 1
self.fin_count = 1
self.start_count_mutex = Semaphore(1)
self.fin_count_mutex = Semaphore(1)
def cross(self, direction):
"""wait for permission to cross the bridge. direction should be either
north (0) or south (1)."""
# TODO
# with lock:
with self.start_count_mutex:
print 'enter_cross:' + str(
self.start_count), 'direction:', direction
self.start_count += 1
if direction == 1: # toward right
if self.status == 1:
self.driving_right_num += 1
elif self.status == 0:
self.waiting_to_right_num += 1
self.sema_right.acquire()
else:
print 'error 2'
elif direction == 0: # toward left
if self.status == 0:
self.driving_left_num += 1
elif self.status == 1:
self.waiting_to_left_num += 1
self.sema_left.acquire()
else:
print 'error 1'
else:
print 'At cross function: direction error!'
def finished(self):
# TODO
# with lock2:
if self.status == 1:
if self.driving_right_num > 0:
self.driving_right_num -= 1
# Next line: change else to if
if self.driving_right_num == 0:
# self.driving_right_num = 0
self.status = 0
for i in xrange(self.waiting_to_left_num):
self.sema_left.release()
self.waiting_to_left_num = 0
# elif self.driving_right_num == 0:
# pass
else:
print 'error 3'
if self.status == 0:
if self.driving_left_num > 0:
self.driving_left_num -= 1
# Next line: change else to if
if self.driving_left_num == 0:
# self.driving_left_num = 0
self.status = 1
for i in xrange(self.waiting_to_right_num):
self.sema_right.release()
self.waiting_to_right_num = 0
# elif self.driving_left_num == 0:
# pass
else:
print 'error 4'
with self.fin_count_mutex:
print 'finish ' + str(self.fin_count)
self.fin_count += 1
thread.join()
logging.info("All done.")
logging.info("Semaphore vs Bounded Semaphore")
# Usually, you create a Semaphore that will allow a certain number of threads
# into a section of code. This one starts at 5.
s1 = Semaphore(5)
# When you want to enter the section of code, you acquire it first.
# That lowers it to 4. (Four more threads could enter this section.)
s1.acquire()
# Then you do whatever sensitive thing needed to be restricted to five threads.
# When you're finished, you release the semaphore, and it goes back to 5.
s1.release()
# That's all fine, but you can also release it without acquiring it first.
s1.release()
# The counter is now 6! That might make sense in some situations, but not in most.
logging.info(s1._value) # => 6
# If that doesn't make sense in your situation, use a BoundedSemaphore.
s2 = BoundedSemaphore(5) # Start at 5.
s2.acquire() # Lower to 4.
s2.release() # Go back to 5.
try:
s2.release() # Try to raise to 6, above starting value.
except ValueError:
class SyncedDictionary:
"""
SyncedDictionary class, implements SyncedDictionary that can be used across multiple threads simultaneously
Attributes
----------
__SyncedDictionary : dict
a dictionary containing the SyncedDictionary
name : str
the name of the list (default "list")
max_readers : int
the maximum amount of simultaneous readers per instance (default 2)
semaphore_lock : Semaphore
a semaphore lock used to limit reading privileges
write_lock : Lock
a lock used to limit writing privileges
Methods
-------
__init__(name="dictionary")
initializes the list and locks
__getitem__(flag)
returns the value of SyncedDictionary[flag]
__setitem__(flag, value)
sets the flag to value
__str__()
returns the dictionary as a string
acquire_edit_permissions(acquired=0)
acquires the write lock and read locks
release_edit_permissions(released=0)
releases the write and read locks
"""
def __init__(self, max_readers=2):
"""
initializer for SyncedDictionary objects
:param max_readers: maximum amount of simultaneous readers (default 2)
:type max_readers: int
"""
if not isinstance(max_readers, int):
raise TypeError(
"SyncedDictionary.__init__: expected max_readers to be of type int"
)
self.__dict = {}
self.max_readers = max_readers
self.semaphore_lock = Semaphore(value=self.max_readers)
self.write_lock = Lock()
def __getitem__(self, key):
"""
returns the value of SyncedDictionary[flag]
:param key: flag to return item for
:type key: Any
:return: SyncedDictionary[flag]
:rtype: Any
"""
self.semaphore_lock.acquire()
item = self.__dict.get(key)
self.semaphore_lock.release()
return item
def __setitem__(self, key, value):
self.acquire_edit_permissions()
self.__dict[key] = value
self.release_edit_permissions()
def __str__(self):
"""
returns string version of the SyncedDictionary
:return: string representation of the SyncedDictionary
:rtype: str
"""
self.semaphore_lock.acquire()
string_representation = ""
for key, value in self.__dict.items():
string_representation += "{}: {}, ".format(key, value)
string_representation = "{" + string_representation[:-1] + "}"
self.semaphore_lock.release()
return string_representation
def acquire_edit_permissions(self, acquired=0):
if not isinstance(acquired, int):
raise TypeError(
"SyncedDictionary.acquire_edit_permissions: expected acquired to be of type int"
)
if acquired > self.max_readers:
raise ValueError(
"SyncedDictionary.acquire_edit_permission: expected acquired to be less than max_readers"
)
for x in range(self.max_readers - acquired):
self.semaphore_lock.acquire()
self.write_lock.acquire()
def release_edit_permissions(self, released=0):
if not isinstance(released, int):
raise TypeError(
"SyncedDictionary.release_edit_permissions: expected released to be of type int"
)
if released > self.max_readers:
raise ValueError(
"SyncedDictionary.release_edit_permission: expected released to be less than max_readers"
)
for x in range(self.max_readers - released):
self.semaphore_lock.release()
self.write_lock.release()
class GCSounds:
def __init__(self):
self.filename = None
self.process = None
self.processlock = Semaphore()
return
def playy(self):
self.processlock.acquire()
self.process = subprocess.Popen(
["aplay", "sounds/" + self.filename + ".wav"])
self.processlock.release()
self.process.wait()
self.processlock.acquire()
self.process = None
self.processlock.release()
return
def play(self, filename):
if self.isPlaying():
self.stopPlaying()
self.filename = filename
self.processlock.acquire()
self.process = True
self.processlock.release()
T = Thread(target=self.playy)
T.start()
return
def isPlaying(self):
self.processlock.acquire()
if self.process is None:
self.processlock.release()
return False
if self.process is True:
self.processlock.release()
return True
try:
self.process.poll()
except:
self.processlock.release()
return True
rv = self.process.returncode
self.processlock.release()
return rv is None
def stopPlaying(self):
self.processlock.acquire()
if self.process is None:
self.processlock.release()
return
try:
os.kill(self.process.pid, signal.SIGINT)
except:
self.processlock.release()
return
self.processlock.release()
return
def dtmf(self, key):
if key == "#":
key = "hash"
if key == "*":
key = "star"
self.play("dtmf-" + key)
return
def ringing(self):
a = random.randint(0, 2)
if a == 0:
self.play("ringing-3.5s")
return
if a == 1:
self.play("ringing-4.5s")
return
if a == 2:
self.play("ringing-6.5s")
return
self.play("ringing-3.5s")
return
def dialtone(self):
self.play("dialtone-30s")
return
def error(self):
self.play("error-1s")
return
def reorder(self):
self.play("reorder")
return
class ThreadManager:
def __init__(self,
alpha,
start_cond,
start_point,
args=(),
kwargs=None,
time_sleep=1):
self.semaphore = Semaphore(value=alpha)
self.start_cond = start_cond
self.args = args
self.kwargs = kwargs if kwargs is not None else {}
self._cont = 0
self._semcont = Semaphore()
self.time_sleep = time_sleep
def locked_start(*largs, **lkwargs):
self.semaphore.acquire()
start_point(*largs, **lkwargs)
self._semcont.acquire()
self._cont -= 1
self._semcont.release()
self.semaphore.release()
self.start_point = locked_start
def start(self):
while True:
value = self.start_cond()
debug(
f'ThreadManager.start - Result of start condition is: {value}')
if value:
debug(f'ThreadManager.start - Acquire General Semaphore')
self.semaphore.acquire()
debug(f'ThreadManager.start - Try to create Thread')
t = Thread(target=self.start_point,
args=self.args,
kwargs=self.kwargs)
debug(f'ThreadManager.start - Acquire Secondary Semaphore')
self._semcont.acquire()
self._cont += 1
debug(
f'ThreadManager.start - Add one to counter of threads. _cont = {self._cont}'
)
debug(f'ThreadManager.start - Release Secondary Semaphore')
self._semcont.release()
debug(f'ThreadManager.start - Start thread')
t.start()
debug(f'ThreadManager.start - Release General Semaphore')
self.semaphore.release()
else:
self._semcont.acquire()
debug(f'ThreadManager.start - Acquire Secondary Semaphore')
if self._cont == 0:
debug(f'ThreadManager.start - Release Secondary Semaphore')
self._semcont.release()
debug(f'ThreadManager.start - Finish manager')
return
debug(f'ThreadManager.start - Release Secondary Semaphore')
self._semcont.release()
debug(
f'ThreadManager.start - Sleeping {self.time_sleep} seconds and try again'
)
sleep(self.time_sleep)
class VacuumControl(BaseClass):
def initialize(self):
self._version = 1.1
self._lock = Semaphore(1)
# run over all covers an check if configurations are available
# then start the spcific handlers for each covers
statedict = self.get_state()
self._vacuumdict = dict()
changeduration = 10
self._log_info(f"Runnging version: {self._version}")
for entity in statedict:
if re.match('^vacuum.*', entity, re.IGNORECASE):
# detected vacuum
id_ = self._getid(statedict, entity)
handledict = dict()
# create listeners for config changes
for configvar in VacuumControlConfiguration.variables_boolean:
cvarname = "input_boolean.control_vacuum_%s_%s" % (
id_, configvar)
if self.entity_exists(cvarname):
self._log_debug(
f"Listen for config change on: {cvarname}")
handle = self.listen_state(self._config_change,
cvarname,
entityid=id_,
duration=changeduration)
handledict.update({cvarname: handle})
for configvar in VacuumControlConfiguration.variables_number:
cvarname = "input_number.control_vacuum_%s_%s" % (
id_, configvar)
if self.entity_exists(cvarname):
self._log_debug(
f"Listen for config change on: {cvarname}")
handle = self.listen_state(self._config_change,
cvarname,
entityid=id_,
duration=changeduration)
handledict.update({cvarname: handle})
for configvar in VacuumControlConfiguration.variables_datetime:
cvarname = "input_datetime.control_vacuum_%s_%s" % (
id_, configvar)
if self.entity_exists(cvarname):
self._log_debug(
f"Listen for config change on: {cvarname}")
handle = self.listen_state(self._config_change,
cvarname,
entityid=id_,
duration=changeduration)
handledict.update({cvarname: handle})
# create variables per vacuum
vardict = dict()
vardict.update({"vacuumID": entity})
# create vacuum control handle
if len(handledict) > 0:
vc_handle = None
self._log_debug(
"input_boolean.control_vacuum_%s_automatic_control: %s"
%
(id_,
self.get_state(
"input_boolean.control_vacuum_%s_automatic_control"
% id_)),
prefix=id_)
self._log_debug(
"input_boolean.control_vacuum_enable_global: %s" %
(self.get_state(
"input_boolean.control_vacuum_enable_global")),
prefix=id_)
if (self.get_state(
"input_boolean.control_vacuum_%s_automatic_control"
% id_) == "on" and self.get_state(
"input_boolean.control_vacuum_enable_global")
== "on"):
self._log_debug(f"Create handle entityid: {id_}")
vc_handle = self.run_at(self._control_vacuum,
datetime.now() +
timedelta(seconds=5),
entityid=id_)
handledict.update({"vc_handle": vc_handle})
d = dict()
d.update({"handledict": handledict})
d.update({"vardict": vardict})
self._vacuumdict.update({id_: d})
# add global config handlers
handledict = dict()
for configvar in VacuumControlConfiguration.variables_boolean_global:
cvarname = "input_boolean.control_vacuum_%s" % configvar
self._log_debug(f"cvarname: {cvarname}")
if self.entity_exists(cvarname):
self._log_debug(f"Listen for config change on: {cvarname}")
handle = self.listen_state(self._config_change_global,
cvarname,
duration=changeduration)
handledict.update({cvarname: handle})
d = dict()
d.update({"handledict": handledict})
self._vacuumdict.update({"global": d})
def _get_handle(self, entityid, handle):
edict = self._vacuumdict.get(entityid, dict())
handledict = edict.get('handledict', dict())
return handledict.get(handle, None)
def _set_handle(self, entityid, varname, handle):
edict = self._vacuumdict.get(entityid, dict())
handledict = edict.get('handledict', dict())
handledict.update({varname: handle})
edict.update({"handledict": handledict})
def _get_variable(self, entityid, varname):
edict = self._vacuumdict.get(entityid, dict())
vardict = edict.get('vardict', dict())
self._log_debug("entityid: %s, varname: %s, len(edict):%s,\
len(vardict):%s" %
(entityid, varname, len(edict), len(vardict)))
self._log_debug(f"vardict: varname: {vardict.get(varname, None)}")
return vardict.get(varname, None)
def _set_variable(self, entityid, varname, value):
edict = self._vacuumdict.get(entityid, dict())
vardict = edict.get('vardict', dict())
vardict.update({varname: value})
edict.update({"vardict": vardict})
def _cancel_restart_handle(self, entityid):
#config has changed for a specific entity
self._log_debug("cancel_restart_handle", prefix=entityid)
# cancel and create new vacuum control handle
vc_handle = self._get_handle(entityid, 'vc_handle')
if vc_handle is not None:
self._log_debug(f"Cancel handle for {entityid}")
self.cancel_timer(vc_handle)
vc_handle = None
if (self.get_state(
"input_boolean.control_vacuum_%s_automatic_control" % entityid)
== "on" and
self.get_state("input_boolean.control_vacuum_enable_global")
== "on"):
vc_handle = self.run_at(self._control_vacuum,
datetime.now() + timedelta(seconds=5),
entityid=entityid)
else:
self._log_info(
"Control vacuum global or per vacuum is disabled\
(Enable per vacuum: %s, Enable Global: %s)" %
(self.get_state(
"input_boolean.control_vacuum_%s_automatic_control" %
entityid),
self.get_state("input_boolean.control_vacuum_enable_global")),
prefix=entityid)
self._set_handle(entityid, "vc_handle", vc_handle)
def _get_vacuumlist(self):
vacuumlist = list()
for k in self._vacuumdict:
self._log_debug(f"vacuumlist: {k}")
if k != "global":
vacuumlist.append(k)
return vacuumlist
def _config_change_global(self, entity, attribute, old, new, kwargs):
#global variable changed
require_reset = ["input_boolean.control_vacuum_enable_global"]
if entity in require_reset:
#disable all handles
for vacuum in self._get_vacuumlist():
self._log_debug("Reset required. Disable all handles")
self._config_change(entity, None, old, new,
{'entityid': vacuum})
def _config_change(self, entity, attribute, old, new, kwargs):
try:
self._lock.acquire(True)
entityid = kwargs.get('entityid', None)
self._log_debug(
f"entityid: {entityid}, entity: {entity}, attribute: {attribute}, old: {old}, new: {new}, kwargs: {kwargs}",
prefix=entityid)
if entityid is not None:
self._cancel_restart_handle(entityid)
else:
#global config has changed
for eid in self._vacuumdict:
# cancel and create new vacuum control handle
self._cancel_restart_handle(eid)
except Exception:
entityid = kwargs.get('entityid', None)
self._log_error(traceback.format_exc(), prefix=entityid)
finally:
self._lock.release()
def _control_vacuum(self, kwargs):
# calculate the next start time per vacuum
try:
self._lock.acquire(True)
entityid = kwargs.get('entityid', None)
self._log_debug("entityid: {}".format(entityid))
# detect current day
# Return the day of the week as an integer,
# where Monday is 1 and Sunday is 7
isoweekdaydict = {
1: "monday",
2: "tuesday",
3: "wednesday",
4: "thursday",
5: "friday",
6: "saturday",
7: "sunday"
}
dtime = datetime.now()
wday = isoweekdaydict.get(dtime.isoweekday(), None)
self._log_debug("Current isoweekday is {}/{}".format(
dtime.isoweekday(), wday))
self._log_debug(
"input_datetime.control_vacuum_{}_start_time_{}".format(
entityid, wday))
if wday is not None:
today = datetime.now().replace(hour=0,
minute=0,
second=0,
microsecond=0)
# Zeit für das starten des Vacuum bestimmen.
vc_start_time = today + timedelta(
hours=self.get_state(
"input_datetime.control_vacuum_{}_start_time_{}".
format(entityid, wday),
attribute="hour"),
minutes=self.get_state(
"input_datetime.control_vacuum_{}_start_time_{}".
format(entityid, wday),
attribute="minute"),
seconds=self.get_state(
"input_datetime.control_vacuum_{}_start_time_{}".
format(entityid, wday),
attribute="second"))
if vc_start_time < datetime.now():
# startzeit ist schon vorbei
# Trigger am nächsten Tag neustarten
dtime = datetime.now().replace(
hour=0, minute=0, second=0, microsecond=0) + timedelta(
days=1, seconds=5)
self._log_info(
"Time to start vacuum has passed nexttrigger: {}".
format(dtime))
self._set_handle(
entityid, "vc_handle",
self.run_at(self._control_vacuum,
dtime,
entityid=entityid))
else:
# start zeit liegt später am Tag
self._log_info(
"Time to start vacuum: {}".format(vc_start_time))
self._set_handle(
entityid, "vc_handle",
self.run_at(self._start_vacuum,
vc_start_time,
entityid=entityid))
else:
self._log_error("Could not detect current weekday!"
"isoweekday: {}".format(dtime.isoweekday()))
except Exception:
entityid = kwargs.get('entityid', None)
self._log_error(traceback.format_exc(), prefix=entityid)
nexttrigger = datetime.now() + timedelta(seconds=5)
self._log_error("Catched Error. Restart at %s" % nexttrigger,
prefix=entityid)
self._set_handle(
entityid, "vc_handle",
self.run_at(self._close_blinds,
datetime.now() + timedelta(seconds=5),
entityid=entityid))
finally:
self._lock.release()
def _start_vacuum(self, kwargs):
try:
self._lock.acquire(True)
entityid = kwargs.get('entityid', None)
self._set_handle(entityid, "vc_handle", None)
self._log_info("start vacuum %s" %
self._get_variable(entityid, "vacuumID"),
prefix=entityid)
self.call_service("vacuum/start",
entity_id=self._get_variable(
entityid, "vacuumID"))
# Trigger neu starten
self._log_debug("nexttrigger %s" %
(datetime.now() + timedelta(minutes=5)),
prefix=entityid)
self._set_handle(
entityid, "vc_handle",
self.run_at(self._control_vacuum,
datetime.now() + timedelta(minutes=5),
entityid=entityid))
except Exception:
entityid = kwargs.get('entityid', None)
self._log_error(traceback.format_exc(), prefix=entityid)
nexttrigger = datetime.now() + timedelta(seconds=5)
self._log_error("Catched Error. Restart in %s" % nexttrigger,
prefix=entityid)
self._set_handle(
entityid, "vc_handle",
self.run_at(self._control_vacuum,
datetime.now() + timedelta(seconds=5),
entityid=entityid))
finally:
self._lock.release()
class ProgressBar:
"""
Implement a progress bar to see the advance of each thread
Attributes:
- barMutex: semaphore to be sure that only one thread update the bar at time
- part: list that contains all parts of the download. Each part represent
a thread.
- toDownload: Total of bytes to download (size of the file).
- downloaded: number of bytes which had been downloaded
- bar: String that contains the bar
Methods:
- update: method to be called for every thread to get update on the progressbar
- updateBar: generate the self.bar attribute with all parts information and progressbar
information
- finalize: complete the bar and print an \n
"""
def __init__(self, lpairs):
#subtract 5 characters for percent, 2 character for '[' and ']' and 10 for download rate
self.columns = get_term_size() - 17
#semaphore to mutex the bar update and downloaded variable
self.barMutex = Semaphore()
self.downloadedMutex = Semaphore()
#initialize all parts and calculate the total bytes to download
columns_part = self.columns / len(lpairs)
self.part = []
self.toDownload = 0
for p in lpairs:
temp = p[1] - p[0]
self.part.append(Part(temp, columns_part))
self.toDownload += temp
#add the rest of bar characters to the last thread
self.part[-1].columns += self.columns % len(lpairs)
self.downloaded = 0
#download rate variables
self.rate = ' 0b/s'
self.oldDownloaded = 0
self.speedometer = Timer(1, self.updateRate)
self.speedometer.start()
self.oldbar = None
self.updateBar()
def update(self, order, bytesAdded):
#first update the part
self.part[order].downloaded += bytesAdded
################################################
self.downloadedMutex.acquire(
) #initiate mutex over the downloaded variable
################################################
self.downloaded += bytesAdded
################################################
self.downloadedMutex.release()
################################################
#update the bar
self.updateBar()
def updateBar(self):
################################################
self.barMutex.acquire(
) #initiate the bar mutex (so many threads updating the bar)
################################################
self.bar = "["
#generating minibars for every thread
for p in self.part:
blackCharacters = int(
(p.downloaded / float(p.toDownload)) * p.columns)
self.bar += "=" * blackCharacters + " " * (p.columns -
blackCharacters)
self.bar += "] "
#making porcentage
porcentage = "%d%%" % int(
(self.downloaded / float(self.toDownload)) * 100)
porcentage = " " * (4 - len(porcentage)) + porcentage
self.bar += porcentage + " "
self.bar += self.rate
self.bar += "\r"
#print it only if change
if not self.oldbar == self.bar:
printStdout(self.bar)
self.oldbar = self.bar
###############################################
self.barMutex.release() #release the bar mutex
###############################################
def updateRate(self):
bytesPsec = self.downloaded - self.oldDownloaded
self.rate = humanize_rate(bytesPsec)
self.rate = " " * (9 - len(self.rate)) + self.rate
self.updateBar()
self.oldDownloaded = self.downloaded
self.speedometer = Timer(1, self.updateRate)
self.speedometer.start()
def finalize(self, interrupted):
# if it was interrupted, we do not need to print the bar again
if (not interrupted):
self.downloaded = self.toDownload
self.updateBar()
print "" # printing the \n
self.speedometer.cancel()
class WechatPublic:
class WechatInfo:
__slots__ = ['pid', 'name', 'cards']
def __init__(self, pid, name, card):
self.pid = pid
self.name = name
self.cards = card
def __init__(self, user_agent=None):
"""
:type user_agent: str
:param:user_agent：浏览器
"""
self.request = requests.Session()
self.headers = dict()
self.semaphore = Semaphore(10) # 任务信号量
self.dataqueue = Queue(20) # 数据队列
self.wait = Semaphore(1)
if user_agent is None:
self.headers['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 ' \
'(KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
else:
self.headers['User-Agent'] = user_agent
def product_url(self,
start: int,
end: int,
seq: int = 1) -> Iterator[WechatUrl]:
"""
产生请求链接
:param start:
:param end:
:param seq:
:return:
"""
if end > 153362:
raise AttributeError("end参数超过范围")
for pid in range(start, end, seq):
url = 'https://www.wxnmh.com/user-{0}.htm'
url = url.format(pid)
yield WechatUrl(pid=pid, url=url)
def get_detail_public_info(self, pid: int, url: str):
"""
获取文章标题和链接
:param pid:公众号数字id
:param url: 首页链接
"""
response = None
soup = None
try:
response = self.request.get(url=url, headers=self.headers)
soup = BeautifulSoup(response.text, "lxml")
except Exception:
print("coonect error")
self.dataqueue.put(1)
time.sleep(3)
return None
keyword = soup.find(name="meta", attrs={
"name": "keywords"
}).get("content")
info: list = keyword.split(",")
name, public_pid = info[0], info[1] # 公众号名字和微信号
try:
nav = soup.find(name="a", attrs={"class": "nav-link active"}).text
except AttributeError:
self.dataqueue.put(1)
return None
articles_total_num = int(re.search("(\\d+)", nav,
re.S).group(1)) # 总文章数
pages = int(articles_total_num / 10.0 + 1) # 文章总页数
''' #文章列表链接：https://www.wxnmh.com/user-pid-第几页文章.htm'''
task_count = pages # 判断该任务是否完成，若为0，则完成
articles = soup.find_all(name="a",
attrs={
"target": "_blank",
"href": re.compile(r"thread")
}) # 该页所有文章
datalist = list()
'''第一页'''
for value in articles:
href = value.get("href") # 链接
title = value.text # 标题
datalist.append({"title": title, "href": href})
wechatinfo = WechatPublic_Info(page=1,
name=name,
public_pid=public_pid,
pid=pid,
articlelist=iter(datalist))
if len(datalist) == 0:
self.dataqueue.put(1)
return None
self.dataqueue.put(wechatinfo)
task_count -= 1 # 任务量减一
''''第2页开始'''
def requests_next(nextpage, pid):
nonlocal task_count
datalist = []
url = 'https://www.wxnmh.com/user-{0}-{1}.htm'.format(
pid, nextpage)
result = self.get_all_article(url)
for item in result:
datalist.append(item)
if len(datalist) > 0:
wechatinfo = WechatPublic_Info(page=nextpage,
name=name,
public_pid=public_pid,
pid=pid,
articlelist=iter(datalist))
# print(wechatinfo)
self.dataqueue.put(wechatinfo)
task_count -= 1 # 任务量减一
self.semaphore.release()
for page in range(2, pages + 1):
self.semaphore.acquire()
Thread(target=requests_next, args=(
page,
pid,
)).start()
while task_count: # f当任务量为0时通知线程可以退出了
continue
self.dataqueue.put(1) # 通知可以释放线程了
def get_all_article(self, url) -> list:
"""
请求并处理文章列表
:param url: 文章列表链接
:return list[dict->{"标题": title, "链接": href}]
"""
response = self.request.get(url=url, headers=self.headers)
soup = BeautifulSoup(response.text, "lxml")
articles = soup.find_all(name="a",
attrs={
"target": "_blank",
"href": re.compile(r"thread")
})
for value in articles:
href = value.get("href")
title = value.text
yield {"title": title, "href": href}
class Game(Scene):
def activate(self):
super().activate()
self.semaphore_chosing_move = Semaphore()
self.start_game_setup()
def on_draw(self):
super().on_draw()
self.draw_grid()
if self.chosing:
arcade.draw_circle_filled(self.position_x, PICK_ROW, RADIUS,
self.color)
def on_mouse_motion(self, x, y, dx, dy):
super().on_mouse_motion(x, y, dx, dy)
self.position_x = x
for column in range(fd.Field.FIELD_LENGTH):
center = (column + 1) * MARGIN + column * 2 * RADIUS + RADIUS
left = center - RADIUS
right = center + RADIUS
if x > left and x < right:
self.position_x = center
def on_mouse_release(self, x, y, dx, dy):
super().on_mouse_release(x, y, dx, dy)
self.move = self.get_pick_from_position()
self.field.makeMove(self.move, self.player_color)
self.chosing = False
self.semaphore_chosing_move.release()
def set_player_functions(self,
red_player_function=None,
yellow_player_function=None):
if not red_player_function == None:
self.player = red_player_function(self)
if not yellow_player_function == None:
self.oplayer = yellow_player_function(self)
def start_game_setup(self):
self.semaphore_chosing_move.acquire(
) # Needs to be acquired, before the start of the game
self.window.set_mouse_visible(False)
self.position_x = 50
self.field = fd.Field()
self.chosing = False
self.color = arcade.color.BLACK
game = cf(self.player, self.oplayer)
t = Thread(target=game.play)
t.daemon = True
t.start()
def draw_grid(self):
field = self.field.getField()
for row in range(fd.Field.FIELD_HEIGHT):
for column in range(fd.Field.FIELD_LENGTH):
if field[fd.Field.FIELD_HEIGHT - row -
1][column] == fd.Field.RED_PLAYER:
color = arcade.color.RED
elif field[fd.Field.FIELD_HEIGHT - row -
1][column] == fd.Field.YELLOW_PLAYER:
color = arcade.color.YELLOW
else:
color = arcade.color.BABY_BLUE
x = (MARGIN + 2 * RADIUS) * column + MARGIN + RADIUS
y = (MARGIN + 2 * RADIUS) * row + MARGIN + RADIUS
arcade.draw_circle_filled(x, y, RADIUS, color)
def get_pick_from_position(self):
column = 0
x = self.position_x
while x > (MARGIN + RADIUS):
column += 1
x -= (2 * RADIUS + MARGIN)
return column
def make_move(self, field, color):
self.field = field
self.player_color = color
if color == fd.Field.RED_PLAYER:
self.color = arcade.color.RED
else:
self.color = arcade.color.YELLOW
self.chosing = True
self.semaphore_chosing_move.acquire()
self.chosing = False
return self.move
def lost(self, field, color):
if color == fd.Field.RED_PLAYER:
color = fd.Field.YELLOW_PLAYER
else:
color = fd.Field.RED_PLAYER
self.game_finished(color, field)
def won(self, field, color):
self.game_finished(color, field)
def draw(self, field, color):
self.game_finished(fd.Field.NO_PLAYER, field)
def game_finished(self, winner, field):
self.window.set_mouse_visible(True)
self.window.switch_to_game_over_scene(field, winner)
class Fader(Thread):
def __init__(self):
self.config = {
'r': {
'pin': get_config('gpio_r', 17),
'freqs': {
'continuous': 500,
'static': 2000
},
'threshold': 0.025
},
'g': {
'pin': get_config('gpio_g', HW_CHANNEL1),
'freqs': {
'continuous': 3000,
'static': 4000
},
'threshold': 0.005
},
'b': {
'pin': get_config('gpio_b', HW_CHANNEL2),
'freqs': {
'continuous': 1500,
'static': 4000
},
'threshold': 0.005
}
}
self.logger = logger
# the semaphore on order to maintain integrity of
# smooth fading despite incoming requests
self.working_on_color = Semaphore()
# color to fade into
self.target_color = [0, 0, 0]
# current color during fading and statically
self.current_color = [0, 0, 0]
# color which is used as the fading starting color. Changes
# to current at that moment.
self.start_color = [0, 0, 0]
# what is the maximum possible color value
# http://abyz.me.uk/rpi/pigpio/python.html#set_PWM_range
# (I think this has really absolutely no effect, it's just
# for convenience so you don't have to convert between ranges
# i guess. If you set it too low you limit the resolution though)
# Make sure to set this to the same value as in the clients code.
# I don't think hardware PWMs are affected by this. They always
# use a value of 1000000 instead.
self.range = 20000
# how fast the client reads their screen
# default to 1 per second
self.set_fading_speed(1)
# how much progress the current fading already has
self.fade_state = 0
self.constant_since = time.time()
self.current_color_mode = None
# setup the PWM
pi.set_PWM_range(self.config['r']['pin'], self.range)
pi.set_PWM_range(self.config['g']['pin'], self.range)
pi.set_PWM_range(self.config['b']['pin'], self.range)
self.set_freq('continuous')
self.set_pwm_dutycycle([0, 0, 0])
logger.info('resolutions of channels in continuous mode:')
logger.info('- r: {}'.format(
pi.get_PWM_real_range(self.config['r']['pin'])))
logger.info('- g: {}'.format(
pi.get_PWM_real_range(self.config['g']['pin'])))
logger.info('- b: {}'.format(
pi.get_PWM_real_range(self.config['b']['pin'])))
self.run_fader = True
# complete thread creation
Thread.__init__(self)
def get_color(self, normalize=255):
return [
self.current_color[c] * normalize / self.range for c in range(3)
]
def set_freq(self, mode):
"""
sets the frequency of the gpio PWM signal.
Higher frequencies result in smaller resolution
for color changes.
https://github.com/fivdi/pigpio/blob/master/doc/gpio.md
mode: one of 'static' or 'continuous'
"""
# don't ask for pi.get_PWM_frequency as that might be different
# from freq depending on the available frequencies
if mode != self.current_color_mode:
pi.set_PWM_frequency(self.config['r']['pin'],
self.config['r']['freqs'][mode])
pi.set_PWM_frequency(self.config['g']['pin'],
self.config['g']['freqs'][mode])
pi.set_PWM_frequency(self.config['b']['pin'],
self.config['b']['freqs'][mode])
self.current_color_mode = mode
self.logger.info('switching to {} mode'.format(mode))
def set_pwm_dutycycle(self, values):
"""
values is an array of floats for [r, g, b]
between 0 and self.range
changes the color of the LEDs instantly
"""
hardware_pins = [12, 18, 13, 19]
for c, color in enumerate(['r', 'g', 'b']):
pin = self.config[color]['pin']
freq = self.config[color]['freqs'][self.current_color_mode]
value = values[c]
if pin in hardware_pins:
# 1000000 is fully on
pi.hardware_PWM(pin, freq, int(1000000 * value / self.range))
else:
pi.set_PWM_dutycycle(pin, max(1, value))
def set_target(self, values, cps=1, mode='static'):
"""
sets a new target to fade into. Takes the current color
and uses it as the new starting point.
values is an array of floats for [r, g, b]
between 0 and self.range
cps is the number of targets that the client will try to
send approximately per second, or rather, how fast the
color should fade.
mode can be 'static' or 'continuous'.
"""
# By how many percent of range does the color need to change
# in order to trigger a change of the LEDs?
# 'continuous': Don't fade when the color delta is not large enough to fade smoothly.
# 'static': Always change the color on a new static-color request
self.working_on_color.acquire()
change_happened = False
for c, color in enumerate(['r', 'g', 'b']):
delta = abs(values[c] - self.target_color[c])
if delta > self.config[color]['threshold']:
self.target_color[c] = max(0, min(self.range, values[c]))
change_happened = True
if change_happened:
self.set_fading_speed(cps)
self.set_freq(mode)
self.constant_since = time.time()
# start where fading has just been
self.start_color = self.current_color.copy()
self.fade_state = 0
else:
self.logger.info('delta color change was below the threshold')
self.working_on_color.release()
def set_fading_speed(self, checks_per_second, fader_frequency=60):
"""
Changes the smoothness and speed of the fading.
Will continue to fade into the current target color.
"""
self.checks_per_second = max(1, checks_per_second)
# how often the fader will iterate in order to
# fade from start to target color
self.checks = max(1, int(fader_frequency / self.checks_per_second) - 1)
# self.fade_state = 0
self.start_color = self.current_color.copy()
def run(self):
"""
This loop iterates like crazy in the specified frequency.
It just takes the members of this object and fades from
start to target colors
"""
while self.run_fader:
start = time.time()
# f will move from 0 to 1
self.working_on_color.acquire()
if self.checks >= 1:
if self.fade_state < 1:
self.fade_state += 1 / self.checks
if self.fade_state > 1:
# the fade_state might have been something like 0.99999
# when it should be 1. Avoid this problem by another check for > 1
# after increasing the fade_state. Also make sure fade_state is in
# consistent state when finished, hence min(1, ...) it
self.fade_state = min(1, self.fade_state)
else:
# Add old and new color together proportionally
# so that a fading effect is created.
# Overwrite globals r, g and b so that when fading restarts,
# that is going to be the new starting color.
self.current_color = [
self.start_color[c] * (1 - self.fade_state) +
self.target_color[c] * self.fade_state
for c in range(3)
]
# print(self.fade_state, self.r_target, self.g_target, self.b_target)
self.set_pwm_dutycycle(self.current_color)
# logger.info('{} {} {}'.format(*self.current_color))
else:
# after 3 minutes increase the LED frequency
# in order to protect the eye
if time.time() - self.constant_since > 180:
# reduces resolution, so the color will
# make a visible jump if it is a dark one
self.set_freq('static')
# the server will take care of setting the
# frequency back to gpio_freq_continuous
self.working_on_color.release()
delta = time.time() - start
time.sleep(
max(0, 1 / self.checks_per_second / (self.checks + 1) - delta))
class BoilerConn(object):
def __init__(self, host="127.0.0.1", port=4545):
self.host = host
self.port = port
self.sock = None
self._heat_flux = 0.0
self._water_flux = 0.0
self._semaphore = Semaphore()
self.open()
def open(self):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def close(self):
self.sock.close()
def _lock(self):
self._semaphore.acquire()
def _unlock(self):
self._semaphore.release()
def _send(self, msg):
return self.sock.sendto(msg, (self.host, self.port))
def _read(self, size=10000):
return self.sock.recv(size)
def _set_cmd(self, cmd, value):
self._lock()
value = str(value).encode()
self._send(cmd + value + b"\r\n")
self._read()
self._unlock()
def _get_msg(self, msg):
self._lock()
self._send(msg)
data = self._read()
self._unlock()
data = data.decode("utf-8")
data = data[3:].replace(",", ".").strip()
try:
data = float(data)
except ValueError:
data = data.split('.')
new_data = ''.join(data[:-1]) + '.' + data[-1]
data = float(new_data)
return data
@property
def heat_flux(self):
return self._heat_flux
@heat_flux.setter
def heat_flux(self, value):
self._heat_flux = value
self._set_cmd(CMD_HEAT_FLUX, value)
@property
def water_flux(self):
return self._water_flux
@water_flux.setter
def water_flux(self, value):
self._water_flux = value
self._set_cmd(CMD_WATER_FLUX, value)
@property
def air_temp(self):
return self._get_msg(MSG_AIR_TEMP)
@property
def water_inside_temp(self):
return self._get_msg(MSG_WATER_INSIDE_TEMP)
@property
def water_in_temp(self):
return self._get_msg(MSG_WATER_IN_TEMP)
@property
def water_out_temp(self):
return self._get_msg(MSG_WATER_OUT_TEMP)
@property
def water_column(self):
return self._get_msg(MSG_WATER_COLUMN)
class BlockCache(object):
def __init__(self):
#This will contain the actual data of each block
self.descriptors_dict = {}
#Structure to store the cache metadata of each block
self.descriptors = []
#Cache statistics
self.get_hits = 0
self.put_hits = 0
self.misses = 0
self.evictions = 0
self.reads = 0
self.writes = 0
self.cache_size_bytes = 0
#Eviction policy
self.policy = "LFU"
#Synchronize shared cache content
self.semaphore = Semaphore()
def access_cache(self,
operation='PUT',
block_id=None,
block_data=None,
etag=None):
result = None
if ENABLE_CACHE:
self.semaphore.acquire()
if operation == 'PUT':
result = self._put(block_id, block_data, etag)
elif operation == 'GET':
result = self._get(block_id)
else:
raise Exception("Unsupported cache operation" + operation)
#Sort descriptors based on eviction policy order
self._sort_descriptors()
self.semaphore.release()
return result
def _put(self, block_id, block_size, etag):
self.writes += 1
to_evict = []
#Check if the cache is full and if the element is new
if CACHE_MAX_SIZE <= (self.cache_size_bytes + block_size
) and block_id not in self.descriptors_dict:
#Evict as many files as necessary until having enough space for new one
while (CACHE_MAX_SIZE <= (self.cache_size_bytes + block_size)):
#Get the last element ordered by the eviction policy
self.descriptors, evicted = self.descriptors[:
-1], self.descriptors[
-1]
#Reduce the size of the cache
self.cache_size_bytes -= evicted.size
#Icrease evictions count and add to
self.evictions += 1
to_evict.append(evicted.block_id)
#Remove from evictions dict
del self.descriptors_dict[evicted.block_id]
if block_id in self.descriptors_dict:
descriptor = self.descriptors_dict[block_id]
self.descriptors_dict[block_id].size = block_size
self.descriptors_dict[block_id].etag = etag
descriptor.put_hit()
self.put_hits += 1
else:
#Add the new element to the cache
descriptor = CacheObjectDescriptor(block_id, block_size, etag)
self.descriptors.append(descriptor)
self.descriptors_dict[block_id] = descriptor
self.cache_size_bytes += block_size
assert len(self.descriptors) == len(self.descriptors_dict.keys()) ==\
len(self.descriptors_dict.keys()), "Unequal length in cache data structures"
return to_evict
def _get(self, block_id):
self.reads += 1
if block_id in self.descriptors_dict:
self.descriptors_dict[block_id].get_hit()
self.get_hits += 1
return block_id, self.descriptors_dict[
block_id].size, self.descriptors_dict[block_id].etag
self.misses += 1
return None, 0, ''
def _sort_descriptors(self):
#Order the descriptor list depending on the policy
if self.policy == "LRU":
self.descriptors.sort(key=lambda desc: desc.last_access,
reverse=True)
elif self.policy == "LFU":
self.descriptors.sort(key=lambda desc: desc.get_hits, reverse=True)
else:
raise Exception("Unsupported caching policy.")
def write_statistics(self, statistics_manager):
if ENABLE_CACHE:
statistics_manager.cache_state(self.get_hits, self.put_hits,
self.misses, self.evictions,
self.reads, self.writes,
self.cache_size_bytes)
def cache_state(self):
print "CACHE GET HITS: ", self.get_hits
print "CACHE PUT HITS: ", self.put_hits
print "CACHE MISSES: ", self.misses
print "CACHE EVICTIONS: ", self.evictions
print "CACHE READS: ", self.reads
print "CACHE WRITES: ", self.writes
print "CACHE SIZE: ", self.cache_size_bytes
for descriptor in self.descriptors:
print "Object: ", descriptor.block_id, descriptor.last_access, descriptor.get_hits, descriptor.put_hits, descriptor.num_accesses, descriptor.size
class AutoPollerThread(Thread):
"""Background thread where the EPG is parsed (unless initiated by the user)."""
def __init__(self):
Thread.__init__(self)
self.__semaphore = Semaphore(0)
self.__queue = deque(maxlen=1)
self.__pump = ePythonMessagePump()
self.__pump.recv_msg.get().append(self.gotThreadMsg)
self.__timer = eTimer()
self.__timer.callback.append(self.timeout)
self.running = False
def timeout(self):
self.__semaphore.release()
def gotThreadMsg(self, msg):
"""Create Notifications if there is anything to display."""
ret = self.__queue.pop()
conflicts = ret[4]
if conflicts and config.plugins.autotimer.notifconflict.value and Standby.inStandby is None:
AddPopup(
#_("%(conflicts)d conflict(s) encountered when trying to add new timers:\n%(timers)s") % { "conflicts":len(conflicts), "timers":'\n'.join( [ _("%(sname)s - %(tname)s: %(name)s at %(begin)s") % {"sname":ServiceReference(x[3]).getServiceName(), "tname":x[4], "name":x[0], "begin":FuzzyTime(x[2])} for x in conflicts ] ) },
_("%d conflict(s) encountered when trying to add new timers:\n%s"
) % (len(conflicts), '\n'.join([
_("%s: %s at %s") %
(x[4], x[0], "('%s', '%s')" % FuzzyTime(x[2]))
for x in conflicts
])),
MessageBox.TYPE_INFO,
config.plugins.autotimer.popup_timeout.value,
NOTIFICATIONID)
similars = ret[5]
if similars and config.plugins.autotimer.notifsimilar.value and Standby.inStandby is None:
AddPopup(
_("%d conflict(s) solved with similar timer(s):\n%s") %
(len(similars), '\n'.join([
_("%s: %s at %s") %
(x[4], x[0], "('%s', '%s')" % FuzzyTime(x[2]))
for x in similars
])), MessageBox.TYPE_INFO,
config.plugins.autotimer.popup_timeout.value,
SIMILARNOTIFICATIONID)
added_timer = ret[1]
if added_timer and config.plugins.autotimer.notiftimers.value and Standby.inStandby is None:
AddPopup(
_("AutoTimer\n%d timer(s) were added.") % (ret[1]),
MessageBox.TYPE_INFO,
config.plugins.autotimer.popup_timeout.value,
TIMERNOTIFICATIONID)
def start(self, initial=True):
if initial:
delay = config.plugins.autotimer.delay.value * 60
if delay == 0:
delay = 30
else:
delay = config.plugins.autotimer.interval.value * 3600
self.__timer.startLongTimer(delay)
if not self.isAlive():
Thread.start(self)
def pause(self):
self.__timer.stop()
def stop(self):
self.__timer.stop()
self.running = False
self.__semaphore.release()
self.__pump.recv_msg.get().remove(self.gotThreadMsg)
self.__timer.callback.remove(self.timeout)
def run(self):
if config.plugins.autotimer.clear_memory.value:
self.clearMemory()
sem = self.__semaphore
queue = self.__queue
pump = self.__pump
timer = self.__timer
self.running = True
while 1:
sem.acquire()
# NOTE: we have to check this here and not using the while to prevent the parser to be started on shutdown
if not self.running: break
if config.plugins.autotimer.skip_during_records.value:
try:
import NavigationInstance
if NavigationInstance.instance.getRecordings():
doLog("[AutoTimer] Skip check during running records")
reactor.callFromThread(
timer.startLongTimer,
config.plugins.autotimer.interval.value * 3600)
continue
except:
pass
try:
if config.plugins.autotimer.onlyinstandby.value and Standby.inStandby is None:
doLog("[AutoTimer] Skip check during live tv")
reactor.callFromThread(
timer.startLongTimer,
config.plugins.autotimer.interval.value * 3600)
continue
except:
pass
if config.plugins.autotimer.skip_during_epgrefresh.value:
try:
from Plugins.Extensions.EPGRefresh.EPGRefresh import epgrefresh
if epgrefresh.isrunning:
doLog("[AutoTimer] Skip check during EPGRefresh")
reactor.callFromThread(
timer.startLongTimer,
config.plugins.autotimer.interval.value * 3600)
continue
except:
pass
from plugin import autotimer
# Ignore any program errors
try:
queue.append(autotimer.parseEPG())
pump.send(0)
except Exception:
# Dump error to stdout
import traceback, sys
traceback.print_exc(file=sys.stdout)
#Keep that eTimer in the mainThread
reactor.callFromThread(
timer.startLongTimer,
config.plugins.autotimer.interval.value * 3600)
def clearMemory(self):
eConsoleAppContainer().execute("sync")
open("/proc/sys/vm/drop_caches", "w").write("3")
if nDancers == 0:
emptyFloor.release()
print('\n Number of dancers on floor are: %d.\n' % nDancers)
floorMutex2.release()
if __name__ == '__main__':
no_of_leaders=int(input('Enter the number of leaders: '))
no_of_followers=int(input('Enter the number of followers: '))
ldrthrd = [Thread(target=leaders, args=[i]) for i in range(no_of_leaders)]
for lt in ldrthrd: lt.start()
flrthrd = [Thread(target=followers, args=[i]) for i in range(no_of_followers)]
for ft in flrthrd: ft.start()
for music in cycle(['waltz', 'tango', 'foxtrot']):
print("** Band leader started playing the music %s **" %(music))
emptyFloor.release()
bandLeaderBarrier.release()
sleep(5)
bandLeaderBarrier.acquire()
emptyFloor.acquire()
sleep(random())
# floorEmpty.release()
print("** Band leader stopped playing the music %s **" %(music))
class BrowserView:
instances = {}
app = AppKit.NSApplication.sharedApplication()
cascade_loc = Foundation.NSMakePoint(100.0, 0.0)
class WindowDelegate(AppKit.NSObject):
def windowShouldClose_(self, window):
i = BrowserView.get_instance('window', window)
quit = localization['global.quit']
cancel = localization['global.cancel']
msg = localization['global.quitConfirmation']
if not i.confirm_close or BrowserView.display_confirmation_dialog(
quit, cancel, msg):
i.closing.set()
return Foundation.YES
else:
return Foundation.NO
def windowWillClose_(self, notification):
# Delete the closed instance from the dict
i = BrowserView.get_instance('window', notification.object())
del BrowserView.instances[i.uid]
if i.pywebview_window in windows:
windows.remove(i.pywebview_window)
i.closed.set()
if BrowserView.instances == {}:
BrowserView.app.stop_(self)
class JSBridge(AppKit.NSObject):
def initWithObject_(self, window):
super(BrowserView.JSBridge, self).init()
self.window = window
return self
def userContentController_didReceiveScriptMessage_(
self, controller, message):
func_name, param, value_id = json.loads(message.body())
if param is WebKit.WebUndefined.undefined():
param = None
js_bridge_call(self.window, func_name, param, value_id)
class BrowserDelegate(AppKit.NSObject):
# Display a JavaScript alert panel containing the specified message
def webView_runJavaScriptAlertPanelWithMessage_initiatedByFrame_completionHandler_(
self, webview, message, frame, handler):
AppKit.NSRunningApplication.currentApplication(
).activateWithOptions_(
AppKit.NSApplicationActivateIgnoringOtherApps)
alert = AppKit.NSAlert.alloc().init()
alert.setInformativeText_(message)
alert.runModal()
if not handler.__block_signature__:
handler.__block_signature__ = BrowserView.pyobjc_method_signature(
b'v@')
handler()
# Display a JavaScript confirm panel containing the specified message
def webView_runJavaScriptConfirmPanelWithMessage_initiatedByFrame_completionHandler_(
self, webview, message, frame, handler):
ok = localization['global.ok']
cancel = localization['global.cancel']
if not handler.__block_signature__:
handler.__block_signature__ = BrowserView.pyobjc_method_signature(
b'v@B')
if BrowserView.display_confirmation_dialog(ok, cancel, message):
handler(Foundation.YES)
else:
handler(Foundation.NO)
# Display an open panel for <input type="file"> element
def webView_runOpenPanelWithParameters_initiatedByFrame_completionHandler_(
self, webview, param, frame, handler):
i = list(BrowserView.instances.values())[0]
files = i.create_file_dialog(OPEN_DIALOG,
'',
param.allowsMultipleSelection(),
'', [],
main_thread=True)
if not handler.__block_signature__:
handler.__block_signature__ = BrowserView.pyobjc_method_signature(
b'v@@')
if files:
urls = [
Foundation.NSURL.fileURLWithPath_(BrowserView.quote(i))
for i in files
]
handler(urls)
else:
handler(nil)
# Open target="_blank" links in external browser
def webView_createWebViewWithConfiguration_forNavigationAction_windowFeatures_(
self, webview, config, action, features):
if action.navigationType() == getattr(
WebKit, 'WKNavigationTypeLinkActivated', 0):
webbrowser.open(action.request().URL().absoluteString(), 2,
True)
return nil
# WKNavigationDelegate method, invoked when a navigation decision needs to be made
def webView_decidePolicyForNavigationAction_decisionHandler_(
self, webview, action, handler):
# The event that might have triggered the navigation
event = AppKit.NSApp.currentEvent()
if not handler.__block_signature__:
handler.__block_signature__ = BrowserView.pyobjc_method_signature(
b'v@i')
""" Disable back navigation on pressing the Delete key: """
# Check if the requested navigation action is Back/Forward
if action.navigationType() == getattr(
WebKit, 'WKNavigationTypeBackForward', 2):
# Check if the event is a Delete key press (keyCode = 51)
if event and event.type(
) == AppKit.NSKeyDown and event.keyCode() == 51:
# If so, ignore the request and return
handler(
getattr(WebKit, 'WKNavigationActionPolicyCancel', 0))
return
# Normal navigation, allow
handler(getattr(WebKit, 'WKNavigationActionPolicyAllow', 1))
# Show the webview when it finishes loading
def webView_didFinishNavigation_(self, webview, nav):
# Add the webview to the window if it's not yet the contentView
i = BrowserView.get_instance('webkit', webview)
if i:
if not webview.window():
i.window.setContentView_(webview)
i.window.makeFirstResponder_(webview)
script = parse_api_js(i.js_bridge.window, 'cocoa')
i.webkit.evaluateJavaScript_completionHandler_(
script, lambda a, b: None)
if not i.text_select:
i.webkit.evaluateJavaScript_completionHandler_(
disable_text_select, lambda a, b: None)
print_hook = 'window.print = function() { window.webkit.messageHandlers.browserDelegate.postMessage("print") };'
i.webkit.evaluateJavaScript_completionHandler_(
print_hook, lambda a, b: None)
i.loaded.set()
# Handle JavaScript window.print()
def userContentController_didReceiveScriptMessage_(
self, controller, message):
if message.body() == 'print':
i = BrowserView.get_instance('_browserDelegate', self)
BrowserView.print_webview(i.webkit)
class FileFilterChooser(AppKit.NSPopUpButton):
def initWithFilter_(self, file_filter):
super(BrowserView.FileFilterChooser, self).init()
self.filter = file_filter
self.addItemsWithTitles_([i[0] for i in self.filter])
self.setAction_('onChange:')
self.setTarget_(self)
return self
def onChange_(self, sender):
option = sender.indexOfSelectedItem()
self.window().setAllowedFileTypes_(self.filter[option][1])
class WebKitHost(WebKit.WKWebView):
def mouseDown_(self, event):
i = BrowserView.get_instance('webkit', self)
window = self.window()
if i.frameless and i.easy_drag:
windowFrame = window.frame()
if windowFrame is None:
raise RuntimeError('Failed to obtain screen')
self.initialLocation = window.convertBaseToScreen_(
event.locationInWindow())
self.initialLocation.x -= windowFrame.origin.x
self.initialLocation.y -= windowFrame.origin.y
super(BrowserView.WebKitHost, self).mouseDown_(event)
def mouseDragged_(self, event):
i = BrowserView.get_instance('webkit', self)
window = self.window()
if i.frameless and i.easy_drag:
screenFrame = AppKit.NSScreen.mainScreen().frame()
if screenFrame is None:
raise RuntimeError('Failed to obtain screen')
windowFrame = window.frame()
if windowFrame is None:
raise RuntimeError('Failed to obtain frame')
currentLocation = window.convertBaseToScreen_(
window.mouseLocationOutsideOfEventStream())
newOrigin = AppKit.NSMakePoint(
(currentLocation.x - self.initialLocation.x),
(currentLocation.y - self.initialLocation.y))
if (newOrigin.y + windowFrame.size.height) > \
(screenFrame.origin.y + screenFrame.size.height):
newOrigin.y = screenFrame.origin.y + \
(screenFrame.size.height + windowFrame.size.height)
window.setFrameOrigin_(newOrigin)
if event.modifierFlags() & getattr(
AppKit, 'NSEventModifierFlagControl', 1 << 18):
i = BrowserView.get_instance('webkit', self)
if not _debug:
return
super(BrowserView.WebKitHost, self).mouseDown_(event)
def rightMouseDown_(self, event):
i = BrowserView.get_instance('webkit', self)
if _debug:
super(BrowserView.WebKitHost, self).rightMouseDown_(event)
def performKeyEquivalent_(self, theEvent):
"""
Handle common hotkey shortcuts as copy/cut/paste/undo/select all/quit
:param theEvent:
:return:
"""
# Fix arrow keys not responding in text inputs
keyCode_ = theEvent.keyCode()
UP, DOWN, LEFT, RIGHT, DELETE, PG_DWN, PG_UP = 126, 125, 123, 124, 117, 121, 116
if keyCode_ in (UP, DOWN, LEFT, RIGHT, DELETE, PG_DWN, PG_UP):
return False
if theEvent.type() == AppKit.NSKeyDown and theEvent.modifierFlags(
) & AppKit.NSCommandKeyMask:
responder = self.window().firstResponder()
keyCode = theEvent.keyCode()
if responder != None:
handled = False
range_ = responder.selectedRange()
hasSelectedText = len(range_) > 0
if keyCode == 7 and hasSelectedText: #cut
responder.cut_(self)
handled = True
elif keyCode == 8 and hasSelectedText: #copy
responder.copy_(self)
handled = True
elif keyCode == 9: # paste
responder.paste_(self)
handled = True
elif keyCode == 0: # select all
responder.selectAll_(self)
handled = True
elif keyCode == 6: # undo
if responder.undoManager().canUndo():
responder.undoManager().undo()
handled = True
elif keyCode == 12: # quit
BrowserView.app.stop_(self)
elif keyCode == 13: # w (close)
self.window().performClose_(theEvent)
handled = True
return handled
return True
def __init__(self, window):
BrowserView.instances[window.uid] = self
self.uid = window.uid
self.pywebview_window = window
self.js_bridge = None
self._file_name = None
self._file_name_semaphore = Semaphore(0)
self._current_url_semaphore = Semaphore(0)
self.closed = window.closed
self.closing = window.closing
self.shown = window.shown
self.loaded = window.loaded
self.confirm_close = window.confirm_close
self.title = window.title
self.text_select = window.text_select
self.is_fullscreen = False
self.hidden = window.hidden
self.minimized = window.minimized
rect = AppKit.NSMakeRect(0.0, 0.0, window.initial_width,
window.initial_height)
window_mask = AppKit.NSTitledWindowMask | AppKit.NSClosableWindowMask | AppKit.NSMiniaturizableWindowMask
if window.resizable:
window_mask = window_mask | AppKit.NSResizableWindowMask
if window.frameless:
window_mask = window_mask | NSFullSizeContentViewWindowMask | AppKit.NSTexturedBackgroundWindowMask
# The allocated resources are retained because we would explicitly delete
# this instance when its window is closed
self.window = AppKit.NSWindow.alloc().\
initWithContentRect_styleMask_backing_defer_(rect, window_mask, AppKit.NSBackingStoreBuffered, False).retain()
self.window.setTitle_(window.title)
self.window.setMinSize_(
AppKit.NSSize(window.min_size[0], window.min_size[1]))
self.window.setAnimationBehavior_(
AppKit.NSWindowAnimationBehaviorDocumentWindow)
BrowserView.cascade_loc = self.window.cascadeTopLeftFromPoint_(
BrowserView.cascade_loc)
frame = self.window.frame()
frame.size.width = window.initial_width
frame.size.height = window.initial_height
self.window.setFrame_display_(frame, True)
self.webkit = BrowserView.WebKitHost.alloc().initWithFrame_(
rect).retain()
user_agent = settings.get('user_agent') or _user_agent
if user_agent:
self.webkit.setCustomUserAgent_(user_agent)
if window.initial_x is not None and window.initial_y is not None:
self.move(window.initial_x, window.initial_y)
else:
self.window.center()
if window.transparent:
self.window.setOpaque_(False)
self.window.setHasShadow_(False)
self.window.setBackgroundColor_(
BrowserView.nscolor_from_hex(window.background_color, 0))
self.webkit.setValue_forKey_(True, 'drawsTransparentBackground')
else:
self.window.setBackgroundColor_(
BrowserView.nscolor_from_hex(window.background_color))
self._browserDelegate = BrowserView.BrowserDelegate.alloc().init(
).retain()
self._windowDelegate = BrowserView.WindowDelegate.alloc().init(
).retain()
self.webkit.setUIDelegate_(self._browserDelegate)
self.webkit.setNavigationDelegate_(self._browserDelegate)
self.window.setDelegate_(self._windowDelegate)
self.frameless = window.frameless
self.easy_drag = window.easy_drag
if window.frameless:
# Make content full size and titlebar transparent
self.window.setTitlebarAppearsTransparent_(True)
self.window.setTitleVisibility_(NSWindowTitleHidden)
self.window.standardWindowButton_(
AppKit.NSWindowCloseButton).setHidden_(True)
self.window.standardWindowButton_(
AppKit.NSWindowMiniaturizeButton).setHidden_(True)
self.window.standardWindowButton_(
AppKit.NSWindowZoomButton).setHidden_(True)
else:
# Set the titlebar color (so that it does not change with the window color)
self.window.contentView().superview().subviews().lastObject(
).setBackgroundColor_(AppKit.NSColor.windowBackgroundColor())
if window.on_top:
self.window.setLevel_(AppKit.NSStatusWindowLevel)
try:
self.webkit.evaluateJavaScript_completionHandler_(
'', lambda a, b: None)
except TypeError:
registerMetaDataForSelector(
b'WKWebView', b'evaluateJavaScript:completionHandler:',
_eval_js_metadata)
config = self.webkit.configuration()
config.userContentController().addScriptMessageHandler_name_(
self._browserDelegate, 'browserDelegate')
try:
config.preferences().setValue_forKey_(
Foundation.NO, 'backspaceKeyNavigationEnabled')
except:
pass
if _debug:
config.preferences().setValue_forKey_(Foundation.YES,
'developerExtrasEnabled')
self.js_bridge = BrowserView.JSBridge.alloc().initWithObject_(window)
config.userContentController().addScriptMessageHandler_name_(
self.js_bridge, 'jsBridge')
if window.real_url:
self.url = window.real_url
self.load_url(window.real_url)
elif window.html:
self.load_html(window.html, '')
else:
self.load_html(default_html, '')
if window.fullscreen:
self.toggle_fullscreen()
self.shown.set()
def first_show(self):
if not self.hidden:
self.window.makeKeyAndOrderFront_(self.window)
else:
self.hidden = False
if self.minimized:
self.minimize()
if not BrowserView.app.isRunning():
# Add the default Cocoa application menu
self._add_app_menu()
self._add_view_menu()
BrowserView.app.activateIgnoringOtherApps_(Foundation.YES)
BrowserView.app.run()
def show(self):
def _show():
self.window.makeKeyAndOrderFront_(self.window)
AppHelper.callAfter(_show)
def hide(self):
def _hide():
self.window.orderOut_(self.window)
AppHelper.callAfter(_hide)
def destroy(self):
AppHelper.callAfter(self.window.close)
def set_title(self, title):
def _set_title():
self.window.setTitle_(title)
AppHelper.callAfter(_set_title)
def toggle_fullscreen(self):
def toggle():
if self.is_fullscreen:
window_behaviour = 1 << 2 # NSWindowCollectionBehaviorManaged
else:
window_behaviour = 1 << 7 # NSWindowCollectionBehaviorFullScreenPrimary
self.window.setCollectionBehavior_(window_behaviour)
self.window.toggleFullScreen_(None)
AppHelper.callAfter(toggle)
self.is_fullscreen = not self.is_fullscreen
def resize(self, width, height):
def _resize():
frame = self.window.frame()
# Keep the top left of the window in the same place
frame.origin.y += frame.size.height
frame.origin.y -= height
frame.size.width = width
frame.size.height = height
self.window.setFrame_display_(frame, True)
AppHelper.callAfter(_resize)
def minimize(self):
self.window.miniaturize_(self)
def restore(self):
self.window.deminiaturize_(self)
def move(self, x, y):
screen_frame = AppKit.NSScreen.mainScreen().frame()
if screen_frame is None:
raise RuntimeError('Failed to obtain screen')
flipped_y = screen_frame.size.height - y
self.window.setFrameTopLeftPoint_(AppKit.NSPoint(x, flipped_y))
def get_current_url(self):
def get():
self._current_url = str(self.webkit.URL())
self._current_url_semaphore.release()
AppHelper.callAfter(get)
self._current_url_semaphore.acquire()
return None if self._current_url == 'about:blank' else self._current_url
def load_url(self, url):
def load(url):
page_url = Foundation.NSURL.URLWithString_(BrowserView.quote(url))
req = Foundation.NSURLRequest.requestWithURL_(page_url)
self.webkit.loadRequest_(req)
self.loaded.clear()
self.url = url
AppHelper.callAfter(load, url)
def load_html(self, content, base_uri):
def load(content, url):
url = Foundation.NSURL.URLWithString_(BrowserView.quote(url))
self.webkit.loadHTMLString_baseURL_(content, url)
self.loaded.clear()
AppHelper.callAfter(load, content, base_uri)
def evaluate_js(self, script):
def eval():
self.webkit.evaluateJavaScript_completionHandler_(script, handler)
def handler(result, error):
JSResult.result = None if result is None or result == 'null' else json.loads(
result)
JSResult.result_semaphore.release()
class JSResult:
result = None
result_semaphore = Semaphore(0)
self.loaded.wait()
AppHelper.callAfter(eval)
JSResult.result_semaphore.acquire()
return JSResult.result
def create_file_dialog(self,
dialog_type,
directory,
allow_multiple,
save_filename,
file_filter,
main_thread=False):
def create_dialog(*args):
dialog_type = args[0]
if dialog_type == SAVE_DIALOG:
save_filename = args[2]
save_dlg = AppKit.NSSavePanel.savePanel()
save_dlg.setTitle_(localization['global.saveFile'])
if directory: # set initial directory
save_dlg.setDirectoryURL_(
Foundation.NSURL.fileURLWithPath_(directory))
if save_filename: # set file name
save_dlg.setNameFieldStringValue_(save_filename)
if save_dlg.runModal() == AppKit.NSFileHandlingPanelOKButton:
self._file_name = save_dlg.filename()
else:
self._file_name = None
else:
allow_multiple = args[1]
open_dlg = AppKit.NSOpenPanel.openPanel()
# Enable the selection of files in the dialog.
open_dlg.setCanChooseFiles_(dialog_type != FOLDER_DIALOG)
# Enable the selection of directories in the dialog.
open_dlg.setCanChooseDirectories_(dialog_type == FOLDER_DIALOG)
# Enable / disable multiple selection
open_dlg.setAllowsMultipleSelection_(allow_multiple)
# Set allowed file extensions
if file_filter:
open_dlg.setAllowedFileTypes_(file_filter[0][1])
# Add a menu to choose between multiple file filters
if len(file_filter) > 1:
filter_chooser = BrowserView.FileFilterChooser.alloc(
).initWithFilter_(file_filter)
open_dlg.setAccessoryView_(filter_chooser)
open_dlg.setAccessoryViewDisclosed_(True)
if directory: # set initial directory
open_dlg.setDirectoryURL_(
Foundation.NSURL.fileURLWithPath_(directory))
if open_dlg.runModal() == AppKit.NSFileHandlingPanelOKButton:
files = open_dlg.filenames()
self._file_name = tuple(files)
else:
self._file_name = None
if not main_thread:
self._file_name_semaphore.release()
if main_thread:
create_dialog(dialog_type, allow_multiple, save_filename)
else:
AppHelper.callAfter(create_dialog, dialog_type, allow_multiple,
save_filename)
self._file_name_semaphore.acquire()
return self._file_name
def _add_app_menu(self):
"""
Create a default Cocoa menu that shows 'Services', 'Hide',
'Hide Others', 'Show All', and 'Quit'. Will append the application name
to some menu items if it's available.
"""
# Set the main menu for the application
mainMenu = AppKit.NSMenu.alloc().init()
self.app.setMainMenu_(mainMenu)
# Create an application menu and make it a submenu of the main menu
mainAppMenuItem = AppKit.NSMenuItem.alloc().init()
mainMenu.addItem_(mainAppMenuItem)
appMenu = AppKit.NSMenu.alloc().init()
mainAppMenuItem.setSubmenu_(appMenu)
appMenu.addItemWithTitle_action_keyEquivalent_(
self._append_app_name(localization["cocoa.menu.about"]),
"orderFrontStandardAboutPanel:", "")
appMenu.addItem_(AppKit.NSMenuItem.separatorItem())
# Set the 'Services' menu for the app and create an app menu item
appServicesMenu = AppKit.NSMenu.alloc().init()
self.app.setServicesMenu_(appServicesMenu)
servicesMenuItem = appMenu.addItemWithTitle_action_keyEquivalent_(
localization["cocoa.menu.services"], nil, "")
servicesMenuItem.setSubmenu_(appServicesMenu)
appMenu.addItem_(AppKit.NSMenuItem.separatorItem())
# Append the 'Hide', 'Hide Others', and 'Show All' menu items
appMenu.addItemWithTitle_action_keyEquivalent_(
self._append_app_name(localization["cocoa.menu.hide"]), "hide:",
"h")
hideOthersMenuItem = appMenu.addItemWithTitle_action_keyEquivalent_(
localization["cocoa.menu.hideOthers"], "hideOtherApplications:",
"h")
hideOthersMenuItem.setKeyEquivalentModifierMask_(
AppKit.NSAlternateKeyMask | AppKit.NSCommandKeyMask)
appMenu.addItemWithTitle_action_keyEquivalent_(
localization["cocoa.menu.showAll"], "unhideAllApplications:", "")
appMenu.addItem_(AppKit.NSMenuItem.separatorItem())
# Append a 'Quit' menu item
appMenu.addItemWithTitle_action_keyEquivalent_(
self._append_app_name(localization["cocoa.menu.quit"]),
"terminate:", "q")
def _add_view_menu(self):
"""
Create a default View menu that shows 'Enter Full Screen'.
"""
mainMenu = self.app.mainMenu()
# Create an View menu and make it a submenu of the main menu
viewMenu = AppKit.NSMenu.alloc().init()
viewMenu.setTitle_(localization["cocoa.menu.view"])
viewMenuItem = AppKit.NSMenuItem.alloc().init()
viewMenuItem.setSubmenu_(viewMenu)
mainMenu.addItem_(viewMenuItem)
# TODO: localization of the Enter fullscreen string has no effect
fullScreenMenuItem = viewMenu.addItemWithTitle_action_keyEquivalent_(
localization["cocoa.menu.fullscreen"], "toggleFullScreen:", "f")
fullScreenMenuItem.setKeyEquivalentModifierMask_(
AppKit.NSControlKeyMask | AppKit.NSCommandKeyMask)
def _append_app_name(self, val):
"""
Append the application name to a string if it's available. If not, the
string is returned unchanged.
:param str val: The string to append to
:return: String with app name appended, or unchanged string
:rtype: str
"""
if "CFBundleName" in info:
val += " {}".format(info["CFBundleName"])
return val
@staticmethod
def nscolor_from_hex(hex_string, alpha=1.0):
"""
Convert given hex color to NSColor.
:hex_string: Hex code of the color as #RGB or #RRGGBB
"""
hex_string = hex_string[1:] # Remove leading hash
if len(hex_string) == 3:
hex_string = ''.join([c * 2
for c in hex_string]) # 3-digit to 6-digit
hex_int = int(hex_string, 16)
rgb = (
(hex_int >> 16) & 0xff, # Red byte
(hex_int >> 8) & 0xff, # Blue byte
(hex_int) & 0xff # Green byte
)
rgb = [i / 255.0 for i in rgb] # Normalize to range(0.0, 1.0)
return AppKit.NSColor.colorWithSRGBRed_green_blue_alpha_(
rgb[0], rgb[1], rgb[2], alpha)
@staticmethod
def get_instance(attr, value):
"""
Return a BrowserView instance by the :value of its given :attribute,
and None if no match is found.
"""
for i in list(BrowserView.instances.values()):
try:
if getattr(i, attr) == value:
return i
except AttributeError:
break
return None
@staticmethod
def display_confirmation_dialog(first_button, second_button, message):
AppKit.NSApplication.sharedApplication()
AppKit.NSRunningApplication.currentApplication().activateWithOptions_(
AppKit.NSApplicationActivateIgnoringOtherApps)
alert = AppKit.NSAlert.alloc().init()
alert.addButtonWithTitle_(first_button)
alert.addButtonWithTitle_(second_button)
alert.setMessageText_(message)
alert.setAlertStyle_(AppKit.NSWarningAlertStyle)
if alert.runModal() == AppKit.NSAlertFirstButtonReturn:
return True
else:
return False
@staticmethod
def print_webview(webview):
info = AppKit.NSPrintInfo.sharedPrintInfo().copy()
# default print settings used by Safari
info.setHorizontalPagination_(AppKit.NSFitPagination)
info.setHorizontallyCentered_(Foundation.NO)
info.setVerticallyCentered_(Foundation.NO)
imageableBounds = info.imageablePageBounds()
paperSize = info.paperSize()
if (Foundation.NSWidth(imageableBounds) > paperSize.width):
imageableBounds.origin.x = 0
imageableBounds.size.width = paperSize.width
if (Foundation.NSHeight(imageableBounds) > paperSize.height):
imageableBounds.origin.y = 0
imageableBounds.size.height = paperSize.height
info.setBottomMargin_(Foundation.NSMinY(imageableBounds))
info.setTopMargin_(paperSize.height -
Foundation.NSMinY(imageableBounds) -
Foundation.NSHeight(imageableBounds))
info.setLeftMargin_(Foundation.NSMinX(imageableBounds))
info.setRightMargin_(paperSize.width -
Foundation.NSMinX(imageableBounds) -
Foundation.NSWidth(imageableBounds))
# show the print panel
print_op = webview._printOperationWithPrintInfo_(info)
print_op.runOperationModalForWindow_delegate_didRunSelector_contextInfo_(
webview.window(), nil, nil, nil)
@staticmethod
def pyobjc_method_signature(signature_str):
"""
Return a PyObjCMethodSignature object for given signature string.
:param signature_str: A byte string containing the type encoding for the method signature
:return: A method signature object, assignable to attributes like __block_signature__
:rtype: <type objc._method_signature>
"""
_objc_so.PyObjCMethodSignature_WithMetaData.restype = ctypes.py_object
return _objc_so.PyObjCMethodSignature_WithMetaData(
ctypes.create_string_buffer(signature_str), None, False)
@staticmethod
def quote(string):
return string.replace(' ', '%20')
def main_params(in_fname, out_fname, opt_diff, min_num_atoms, max_num_atoms,
min_num_components, max_num_components, min_num_mix_components,
max_num_mix_components, mix_fname, descriptors_transformation,
mix_type, opt_mix_ordered, opt_verbose, opt_noH, frag_fname,
per_atom_fragments, self_association_mix, reaction_diff,
quasimix, id_field_name, output_format, ncores):
# define which property will be loaded from external file or from sdf-file
opt_diff_builtin = [v for v in opt_diff if v in builtin_types]
opt_diff_sdf = [v for v in opt_diff if v not in builtin_types]
# load sdf, rdf or rxn file depending on its extension
input_file_extension = in_fname.strip().split(".")[-1].lower()
setup_path = os.path.join(GetWorkDir(in_fname), "setup.txt")
# check all properties are present in setup
if opt_diff_sdf:
not_avail = set(opt_diff_sdf).difference(
files.GetAtomPropertyFromSetup(setup_path))
if not_avail:
for v in not_avail:
print(
"WARNING. Chosen atomic property values (%s) is absent in setup.txt file. "
"Therefore its values will used as categorical variable ('as is') for atom labeling."
% v)
# init pool of workers for calculation of single compounds
ncores = min(cpu_count(), max(ncores, 1))
p = Pool(ncores)
chunksize = 5
semaphore = Semaphore(ncores * chunksize)
if mix_fname is None and not quasimix and input_file_extension == 'sdf':
saver = None
sirms = None
if output_format == "svm":
saver = files.SvmSaver(out_fname)
if output_format == "txt":
sirms = OrderedDict()
frags = files.LoadFragments(frag_fname)
try:
for result in p.imap(
MapCalcMolSingleSirms,
prep_input(in_fname, id_field_name, opt_diff, opt_diff_sdf,
setup_path, min_num_atoms, max_num_atoms,
min_num_components, max_num_components, opt_noH,
opt_verbose, per_atom_fragments, frags,
semaphore),
chunksize=chunksize):
if output_format == "txt":
sirms.update(result)
semaphore.release()
if output_format == "svm":
for mol_name, descr_dict in result.items():
saver.save_mol_descriptors(mol_name, descr_dict)
semaphore.release()
finally:
p.close()
if output_format == "txt":
SaveSimplexes(out_fname, sirms, output_format)
else:
# read input
if input_file_extension == 'sdf':
mols = OrderedDict()
for m in ReadSDF(in_fname, id_field_name, opt_diff_sdf,
setup_path):
mols[m.title] = m
elif input_file_extension == 'rdf':
mols, mix = ReadRDF(in_fname, id_field_name)
elif input_file_extension == 'rxn':
mols, mix = ReadRXN(in_fname, id_field_name, opt_diff_sdf,
setup_path)
else:
print(
"Input file extension should be SDF, RDF or RXN. Current file has %s. Please check it."
% input_file_extension.upper())
return None
# set labels of built-in types
SetLabelsInternal(mols, opt_diff_builtin, setup_path)
# create mix data for sdf (for rdf/rxn mix is created during file loading)
if input_file_extension == 'sdf':
if quasimix:
mix = GenQuasiMix(list(mols.keys()))
elif mix_fname is not None:
mix = files.LoadMixturesTxt(mix_fname, mix_type)
else:
print("Strange error occurred during mix preparation")
exit()
mols_used = set(chain.from_iterable([m['names']
for m in mix.values()]))
sirms = OrderedDict()
try:
for result in p.imap(
MapCalcMolSingleSirms,
prep_input_mix([mols[mol_name]
for mol_name in mols_used], opt_diff, 1,
max_num_atoms, min_num_components,
max_num_components, opt_noH, opt_verbose),
chunksize=chunksize):
sirms.update(result)
finally:
p.close()
# min_num_atoms set to 1 to be able to generate mixtures
# sirms = CalcSingleSirms([mols[mol_name] for mol_name in mols_used], opt_diff, 1,
# max_num_atoms, min_num_components, max_num_components, opt_noH,
# opt_verbose, None)
sirms = CalcMixSirms(single_sirms=sirms,
mix=mix,
atom_labeling=opt_diff,
min_num_atoms=min_num_atoms,
max_num_atoms=max_num_atoms,
min_num_mix_components=min_num_mix_components,
max_num_mix_components=max_num_mix_components,
verbose=opt_verbose,
ordered=opt_mix_ordered,
self_assembly_mix=self_association_mix)
# filter single sirms with number of atoms lower than min_num_atoms
if input_file_extension in ['rdf', 'rxn']:
sirms = concat_reaction_sirms(sirms)
if descriptors_transformation in ['prob', 'both']:
sirms = CalcProbSirms(sirms, descriptors_transformation)
if reaction_diff:
sirms = CalcReactionDiffSirms(sirms)
SaveSimplexes(out_fname, sirms, output_format)
def unlock(self, sem: Semaphore):
self.mutex.acquire()
self.counter -= 1
if self.counter == 0:
sem.release()
self.mutex.release()
class ActivityWorker:
"""Activity worker for a Stepfunctions task."""
def __init__(self,
activity_arn,
activity_fxn,
heartbeat_interval=4,
worker_count=1,
*,
client=None,
**kwargs):
"""Instantiate with an Activity ARN and a callable."""
self.activity_kwargs = {
"activityArn": activity_arn,
"workerName": kwargs.get("worker_name", socket.gethostname()),
}
self.activity_name = activity_arn.split(":")[-1]
self.activity_fxn = activity_fxn
self.heartbeat_interval = heartbeat_interval
self.task_pool = concurrent.futures.ThreadPoolExecutor(
max_workers=worker_count)
self._task_semaphore = Semaphore(worker_count)
self.stepfunctions = client if client else self._default_config()
@staticmethod
def _default_config():
"""Return default stepfunctions client configuration.
StepFunctions GetActivitiyTask opens connections for 60 seconds.
botocores's default read timeout is 60 seconds.
Occaisonally the socket will close before the request completes,
causing an exception to be raised.
"""
config = botocore.config.Config(read_timeout=70)
return boto3.client('stepfunctions', config=config)
def __call__(self):
self.perform_task()
def _poll_for_task(self):
task = dict()
while not task.get("taskToken"):
print("Polling for an activity task.")
task = self.stepfunctions.get_activity_task(**self.activity_kwargs)
print("Recieved a task!")
print(json.dumps(json.loads(task["input"]), indent=4, sort_keys=True))
return task
def perform_task(self, task=None):
"""Listen for and run a Stepfunctions activity task."""
task = self._poll_for_task() if not task else task
heartbeat = Heartbeat(self.heartbeat_interval,
self.stepfunctions.send_task_heartbeat,
args=None,
kwargs={"taskToken": task["taskToken"]})
try:
print(f"Performing {self.activity_name}")
with heartbeat:
task_input = json.loads(task["input"])
output = self.activity_fxn(**task_input)
except (Exception, KeyboardInterrupt) as error:
print(f"{self.activity_name} failed!")
*_, raw_traceback = sys.exc_info()
formatted_traceback = traceback.format_tb(raw_traceback)
self.stepfunctions.send_task_failure(
taskToken=task["taskToken"],
error=str(error)[:256],
cause="\n".join(formatted_traceback),
)
self._task_semaphore.release()
raise
print(f"{self.activity_name} is completed!")
print(json.dumps(output, indent=4, sort_keys=True))
self.stepfunctions.send_task_success(
taskToken=task["taskToken"],
output=json.dumps(output, sort_keys=True),
)
self._task_semaphore.release()
def listen(self):
"""Repeatedly listen & execute tasks associated with this activity."""
print(f"Listening for {self.activity_name}...")
try:
while True:
self._task_semaphore.acquire()
task = self._poll_for_task()
self.task_pool.submit(self.perform_task, task)
except KeyboardInterrupt:
print("\nStopping listener...")
class CachedGBD:
EMPTY = 0xffffffffffffffff
def __init__(self, cache_file, dirty=False, *args, **kargs):
if 'workers' not in kargs:
kargs['workers'] = 16
self.done = False
self.gbd = GBD(*args, **kargs)
self.uuid = self.gbd.uuid
self.block_size = self.gbd.block_size
self.block_count = self.gbd.block_count
self.total_size = self.gbd.total_size
self.cache = open(cache_file, 'r+b')
self.cache_lock = Lock()
self.entry_count = self.calc_entry_count()
self.clean_que = RLUQueue(self.entry_count)
self.dirty_que = RLUQueue(self.entry_count)
self.last_modify = [0] * self.entry_count
self.map = {}
self.rmap = [self.EMPTY] * self.entry_count
self.map_lock = Lock()
self.load_cache(dirty)
self.wb_sem = Semaphore(8)
self.wb_daemon = Thread(target=self.do_writeback)
self.wb_daemon.daemon = True
self.wb_daemon.start()
self.pull_que = TimedPriorityQueue()
self.dque_lock = Lock()
self.pull_delay_que = {}
self.pull_daemon = Thread(target=self.do_pull)
self.pull_daemon.daemon = True
self.pull_daemon.start()
self.done = True
## init
def load_cache(self, dirty=True):
self.cache.seek(0, os.SEEK_SET)
cache_uuid = self.cache.read(len(self.uuid))
if cache_uuid == "\0" * len(self.uuid):
logger.info("The cache file is empty, not loading anything")
for i in range(self.entry_count):
self.clean_que.put(i)
return
if cache_uuid != self.uuid:
raise AssertionError(
"It's not the correct cache device. (uuid mismatch)")
self.cache.seek(len(self.uuid), os.SEEK_SET)
record = self.cache.read(8 * self.entry_count)
for i in range(0, self.entry_count):
entry = struct.unpack("!Q", record[i * 8:i * 8 + 8])[0]
if entry != self.EMPTY:
assert entry < self.block_count and entry not in self.map
self.map[entry] = i
self.rmap[i] = entry
if dirty:
self.dirty_que.put(i)
else:
self.clean_que.put(i)
logger.debug("Map {0} => {1}".format(entry, i))
else:
self.clean_que.put(i)
## interface
def read(self, offset, length, callback=None):
assert 0 <= offset < offset + length <= self.total_size
idxl = offset // self.block_size
idxr = (offset + length - 1) // self.block_size
cv = Condition()
state = [idxr + 1 - idxl, None]
data_list = [None] * state[0]
for idx in range(idxl, idxr + 1):
rngl = max(offset, idx * self.block_size)
rngr = min(offset + length, (idx + 1) * self.block_size)
shift = rngl % self.block_size
to_read = rngr - rngl
def gcb(idx, shift, to_read):
def cb(err, obj, data):
with cv:
if state[1] is not None:
return False
if err:
state[1] = err
if callback:
callback(err, None)
else:
cv.notify()
return False
if to_read == self.block_size:
data_list[idx - idxl] = data
else:
with self.cache_lock:
self.cache.seek(
self.calc_offset(obj) + shift, os.SEEK_SET)
data_list[idx -
idxl] = self.cache.read(to_read)
state[0] = state[0] - 1
if state[0] == 0:
if callback is None:
cv.notify()
else:
callback(None, ''.join(data_list))
return False
return cb
self.pull(idx,
read_data=(to_read == self.block_size),
callback=gcb(idx, shift, to_read))
if callback is None:
cv.acquire()
while state[0] != 0:
cv.wait()
cv.release()
if state[1]:
raise state[1]
else:
assert all(x is not None for x in data_list)
return ''.join(data_list)
def write(self, offset, data, callback=None):
assert 0 <= offset < offset + len(data) <= self.total_size
idxl = offset // self.block_size
idxr = (offset + len(data) - 1) // self.block_size
lock = Lock()
state = [idxr + 1 - idxl, None]
for idx in range(idxl, idxr + 1):
rngl = max(offset, idx * self.block_size)
rngr = min(offset + len(data), (idx + 1) * self.block_size)
ndata = data[rngl - offset:rngr - offset]
shift = rngl % self.block_size
def gcb(shift, ndata):
def cb(err, obj, _):
with lock:
if state[1] is not None:
return False
if err:
state[1] = err
if callback:
callback(err)
return False
with self.cache_lock:
self.cache.seek(
self.calc_offset(obj) + shift, os.SEEK_SET)
self.cache.write(ndata)
state[0] = state[0] - 1
if state[0] == 0 and callback:
callback(None)
return True
return cb
cb = gcb(shift, ndata)
if len(ndata) == self.block_size:
obj = self.pull(idx, pull_data=False, callback=cb)
else:
obj = self.pull(idx, callback=cb)
def save_map(self):
def pack(ull):
return ''.join(chr((ull >> i) % 256) for i in range(56, -1, -8))
logger.info("Saving map...")
with self.cache_lock:
self.cache.seek(0, 0)
self.cache.write(self.uuid)
self.cache.write(''.join(pack(ent) for ent in self.rmap))
self.cache.close()
def sync(self):
logger.info("Flushing all request to gbd...")
while True:
with self.dque_lock:
if self.dirty_que.empty() and self.pull_que.empty() and len(
self.pull_delay_que) == 0:
break
time.sleep(1)
self.gbd.sync()
def end(self, force=False):
if not force:
self.sync()
self.gbd.end(True)
self.save_map()
logger.info("End CachedGBD")
## helper
def calc_entry_count(self):
self.cache.seek(0, os.SEEK_END)
entry_count = (self.cache.tell() -
len(self.uuid)) // (self.block_size + 8)
assert entry_count > 0
return entry_count
def calc_offset(self, idx):
return len(self.uuid) + 8 * self.entry_count + idx * self.block_size
def pull(self, idx, pull_data=True, read_data=False, callback=None):
assert 0 <= idx < self.block_count
assert pull_data or not read_data
self.pull_que.put((idx, pull_data, read_data, callback))
## daemon
def check_delay_pull(self, idx):
with self.dque_lock:
if idx in self.pull_delay_que:
pack = self.pull_delay_que[idx].get()
logging.debug("Put pack {0}".format(pack))
self.pull_que.put(pack, TimedPriorityQueue.PRI_HIGH)
if self.pull_delay_que[idx].empty():
del self.pull_delay_que[idx]
def do_pull(self):
while True:
pack = self.pull_que.get()
data = None
modify = False
idx, pull_data, read_data, callback = pack
with self.map_lock:
if idx in self.map:
new_block = False
obj = self.map[idx]
with self.dque_lock:
cobj = self.clean_que.pop(obj)
dobj = self.dirty_que.pop(obj)
assert cobj is None or dobj is None
if cobj is None and dobj is None:
logging.debug("Delay {0}".format(pack))
if idx not in self.pull_delay_que:
self.pull_delay_que[idx] = Queue()
self.pull_delay_que[idx].put(pack)
continue
else:
new_block = True
obj = self.clean_que.get()
if self.rmap[obj] != self.EMPTY:
del self.map[self.rmap[obj]]
self.rmap[obj] = idx
self.map[idx] = obj
if not new_block:
if read_data:
with self.cache_lock:
self.cache.seek(self.calc_offset(obj), os.SEEK_SET)
data = self.cache.read(self.block_size)
else:
logger.debug("Pull {0} => {1}".format(idx, obj))
if pull_data or read_data:
def gcb(idx, obj, callback):
def cb(err, data):
if err:
logger.error("Pull {0} => {1}: Fail".format(
idx, obj))
raise NotImplementedError(
"Need to propagate pull error")
else:
logger.debug(
"Pull {0} => {1}: Check = {2}".format(
idx, obj,
hashlib.sha1(data).hexdigest()))
with self.cache_lock:
self.cache.seek(self.calc_offset(obj),
os.SEEK_SET)
self.cache.write(data)
if callback and callback(None, obj, data):
self.last_modify[obj] = time.time()
self.dirty_que.put(obj)
else:
self.clean_que.put(obj)
self.check_delay_pull(idx)
logger.debug("Pull {0} => {1}: End".format(
idx, obj))
return cb
self.gbd.read_block(idx, gcb(idx, obj, callback))
continue
else:
modify = True
assert data is None or len(data) == self.block_size
if callback and callback(None, obj, data):
modify = True
if dobj is not None or modify:
if modify:
self.last_modify[obj] = time.time()
self.dirty_que.put(obj)
else:
self.clean_que.put(obj)
self.check_delay_pull(idx)
def do_writeback(self):
delay = 0.5
while True:
self.wb_sem.acquire()
ent = self.dirty_que.get()
to_sleep = self.last_modify[ent] + delay - time.time()
if to_sleep > 0:
self.wb_sem.release()
logging.debug("Sleep wb {0}".format(to_sleep))
self.dirty_que.unget(ent)
time.sleep(to_sleep)
continue
with self.map_lock:
idx = self.rmap[ent]
assert self.map[idx] == ent
logger.debug("Collected {0}".format(ent))
with self.cache_lock:
self.cache.seek(self.calc_offset(ent), os.SEEK_SET)
data = self.cache.read(self.block_size)
logger.debug("Push {0} <= {1}: Check = {2}".format(
idx, ent,
hashlib.sha1(data).hexdigest()))
def gcb(idx, ent):
def cb(err, _):
if err:
logger.warning("Push {0} <= {1}: Fail".format(
idx, ent))
self.dirty_que.put(ent)
else:
logger.debug("Push {0} <= {1}: Success".format(
idx, ent))
self.clean_que.put(ent)
self.check_delay_pull(idx)
self.wb_sem.release()
return cb
self.gbd.write_block(idx, data, gcb(idx, ent),
TimedPriorityQueue.PRI_LOW)
class FizzBuzz:
def __init__(self, n: int):
self.n = n
self.done = False
self.fizz_semaphore = Semaphore(0)
self.buzz_semaphore = Semaphore(0)
self.fizzbuzz_semaphore = Semaphore(0)
self.number_semaphore = Semaphore(1)
def fizz(self, printFizz: 'Callable[[], None]') -> None:
while True:
self.fizz_semaphore.acquire()
if self.done:
break
printFizz()
self.number_semaphore.release()
def buzz(self, printBuzz: 'Callable[[], None]') -> None:
while True:
self.buzz_semaphore.acquire()
if self.done:
break
printBuzz()
self.number_semaphore.release()
def fizzbuzz(self, printFizzBuzz: 'Callable[[], None]') -> None:
while True:
self.fizzbuzz_semaphore.acquire()
if self.done:
break
printFizzBuzz()
self.number_semaphore.release()
# printNumber(x) outputs "x", where x is an integer.
def number(self, printNumber: 'Callable[[int], None]') -> None:
for number in range(1, self.n + 1):
self.number_semaphore.acquire()
if number % 15 == 0:
self.fizzbuzz_semaphore.release()
elif number % 3 == 0:
self.fizz_semaphore.release()
elif number % 5 == 0:
self.buzz_semaphore.release()
else:
printNumber(number)
self.number_semaphore.release()
self.number_semaphore.acquire()
self.done = True
self.fizz_semaphore.release()
self.buzz_semaphore.release()
self.fizzbuzz_semaphore.release()
t=Thread(target=golf,args=[i])
t.start()
rng.seed(i*100)
prng=rng.random()
sleep (prng)
def cart():
global stash
global balls_on_field
mutex.acquire()
print('######################################')
print('stash=', stash, 'gathering balls on field')
stash+=balls_on_field
balls_on_field=0
#signal
stash_sem.release()
print('done stash=', stash, 'return')
print('#####################################')
mutex.release()
while(True):
t=Thread(target=cart)
if(stash<bucket):
mutex.acquire()
t.start()
mutex.release()
rng.seed(50)
prng=rng.random()
sleep (prng)
class BrowserView:
instance = None
class JSBridge:
def __init__(self, api_instance):
self.api = api_instance
self.uid = uuid1().hex[:8]
def call(self, func_name, param):
if param == 'undefined':
param = None
return _js_bridge_call(self.api, func_name, param)
def __init__(self, title, url, width, height, resizable, fullscreen,
min_size, confirm_quit, background_color, debug, js_api,
webview_ready):
BrowserView.instance = self
self.webview_ready = webview_ready
self.is_fullscreen = False
self._js_result_semaphore = Semaphore(0)
self.load_event = Event()
self.js_bridge = None
glib.threads_init()
window = gtk.Window(title=title)
if resizable:
window.set_size_request(min_size[0], min_size[1])
window.resize(width, height)
else:
window.set_size_request(width, height)
window.set_resizable(resizable)
window.set_position(gtk.WindowPosition.CENTER)
# Set window background color
style_provider = gtk.CssProvider()
style_provider.load_from_data(
'GtkWindow {{ background-color: {}; }}'.format(
background_color).encode())
gtk.StyleContext.add_provider_for_screen(
Gdk.Screen.get_default(), style_provider,
gtk.STYLE_PROVIDER_PRIORITY_APPLICATION)
scrolled_window = gtk.ScrolledWindow()
window.add(scrolled_window)
self.window = window
if confirm_quit:
self.window.connect('delete-event', self.on_destroy)
else:
self.window.connect('delete-event', self.close_window)
if js_api:
self.js_bridge = BrowserView.JSBridge(js_api)
self.webview = webkit.WebView()
self.webview.connect('notify::visible', self.on_webview_ready)
self.webview.connect('document-load-finished', self.on_load_finish)
self.webview.connect('status-bar-text-changed', self.on_status_change)
self.webview.props.settings.props.enable_default_context_menu = False
self.webview.props.opacity = 0.0
scrolled_window.add(self.webview)
window.show_all()
if url is not None:
self.webview.load_uri(url)
if fullscreen:
self.toggle_fullscreen()
def close_window(self, *data):
while gtk.events_pending():
gtk.main_iteration()
self.window.destroy()
gtk.main_quit()
self._js_result_semaphore.release()
def on_destroy(self, widget=None, *data):
dialog = gtk.MessageDialog(
parent=self.window,
flags=gtk.DialogFlags.MODAL & gtk.DialogFlags.DESTROY_WITH_PARENT,
type=gtk.MessageType.QUESTION,
buttons=gtk.ButtonsType.OK_CANCEL,
message_format=localization['global.quitConfirmation'])
result = dialog.run()
if result == gtk.ResponseType.OK:
self.close_window()
else:
dialog.destroy()
return True
def on_webview_ready(self, arg1, arg2):
glib.idle_add(self.webview_ready.set)
def on_load_finish(self, webview, webframe):
# Show the webview if it's not already visible
if not webview.props.opacity:
glib.idle_add(webview.set_opacity, 1.0)
if self.js_bridge:
self._set_js_api()
else:
self.load_event.set()
def on_status_change(self, webview, status):
try:
delim = '_' + self.js_bridge.uid + '_'
except AttributeError:
return
# Check if status was updated by a JSBridge call
if status.startswith(delim):
_, func_name, param = status.split(delim)
return_val = self.js_bridge.call(func_name, param)
# Give back the return value to JS as a string
code = 'pywebview._bridge.return_val = "{0}";'.format(
_escape_string(str(return_val)))
webview.execute_script(code)
def show(self):
gtk.main()
def destroy(self):
self.window.emit('delete-event', Gdk.Event())
def toggle_fullscreen(self):
if self.is_fullscreen:
self.window.unfullscreen()
else:
self.window.fullscreen()
self.is_fullscreen = not self.is_fullscreen
def create_file_dialog(self, dialog_type, directory, allow_multiple,
save_filename, file_types):
if dialog_type == FOLDER_DIALOG:
gtk_dialog_type = gtk.FileChooserAction.SELECT_FOLDER
title = localization["linux.openFolder"]
button = gtk.STOCK_OPEN
elif dialog_type == OPEN_DIALOG:
gtk_dialog_type = gtk.FileChooserAction.OPEN
if allow_multiple:
title = localization['linux.openFiles']
else:
title = localization['linux.openFile']
button = gtk.STOCK_OPEN
elif dialog_type == SAVE_DIALOG:
gtk_dialog_type = gtk.FileChooserAction.SAVE
title = localization['global.saveFile']
button = gtk.STOCK_SAVE
dialog = gtk.FileChooserDialog(
title, self.window, gtk_dialog_type,
(gtk.STOCK_CANCEL, gtk.ResponseType.CANCEL, button,
gtk.ResponseType.OK))
dialog.set_select_multiple(allow_multiple)
dialog.set_current_folder(directory)
self._add_file_filters(dialog, file_types)
if dialog_type == SAVE_DIALOG:
dialog.set_current_name(save_filename)
response = dialog.run()
if response == gtk.ResponseType.OK:
file_name = dialog.get_filenames()
else:
file_name = None
dialog.destroy()
return file_name
def _add_file_filters(self, dialog, file_types):
for s in file_types:
description, extensions = _parse_file_type(s)
f = gtk.FileFilter()
f.set_name(description)
for e in extensions.split(';'):
f.add_pattern(e)
dialog.add_filter(f)
def get_current_url(self):
uri = self.webview.get_uri()
return uri
def load_url(self, url):
self.load_event.clear()
self.webview.load_uri(url)
def load_html(self, content, base_uri):
self.load_event.clear()
self.webview.load_string(content, 'text/html', 'utf-8', base_uri)
def evaluate_js(self, script):
def _evaluate_js():
self.webview.execute_script(code)
self._js_result_semaphore.release()
unique_id = uuid1().hex
# Backup the doc title and store the result in it with a custom prefix
code = 'oldTitle{0} = document.title; document.title = eval("{1}");'.format(
unique_id, _escape_string(script))
self.load_event.wait()
glib.idle_add(_evaluate_js)
self._js_result_semaphore.acquire()
if not gtk.main_level():
# Webview has been closed, don't proceed
return None
# Restore document title and return
_js_result = self._parse_js_result(self.webview.get_title())
code = 'document.title = oldTitle{0};'.format(unique_id)
glib.idle_add(self.webview.execute_script, code)
return _js_result
def _parse_js_result(self, result):
try:
return int(result)
except ValueError:
try:
return float(result)
except ValueError:
return result
def _set_js_api(self):
def create_bridge():
# Make the `call` method write the function name and param to the
# `status` attribute of the JS window, delimited by a unique token.
# The return value will be passed back to the `return_val` attribute
# of the bridge by the on_status_change handler.
code = """
window.pywebview._bridge.call = function(funcName, param) {{
window.status = "_{0}_" + funcName + "_{0}_" + param;
return this.return_val;
}};""".format(self.js_bridge.uid)
# Create the `pywebview` JS api object
self.webview.execute_script(_parse_api_js(self.js_bridge.api))
self.webview.execute_script(code)
self.load_event.set()
glib.idle_add(create_bridge)
class IPConnection:
FUNCTION_ENUMERATE = 254
FUNCTION_ADC_CALIBRATE = 251
FUNCTION_GET_ADC_CALIBRATION = 250
FUNCTION_READ_BRICKLET_UID = 249
FUNCTION_WRITE_BRICKLET_UID = 248
FUNCTION_READ_BRICKLET_PLUGIN = 247
FUNCTION_WRITE_BRICKLET_PLUGIN = 246
FUNCTION_DISCONNECT_PROBE = 128
CALLBACK_ENUMERATE = 253
CALLBACK_CONNECTED = 0
CALLBACK_DISCONNECTED = 1
BROADCAST_UID = 0
PLUGIN_CHUNK_SIZE = 32
# enumeration_type parameter to the enumerate callback
ENUMERATION_TYPE_AVAILABLE = 0
ENUMERATION_TYPE_CONNECTED = 1
ENUMERATION_TYPE_DISCONNECTED = 2
# connect_reason parameter to the connected callback
CONNECT_REASON_REQUEST = 0
CONNECT_REASON_AUTO_RECONNECT = 1
# disconnect_reason parameter to the disconnected callback
DISCONNECT_REASON_REQUEST = 0
DISCONNECT_REASON_ERROR = 1
DISCONNECT_REASON_SHUTDOWN = 2
# returned by get_connection_state
CONNECTION_STATE_DISCONNECTED = 0
CONNECTION_STATE_CONNECTED = 1
CONNECTION_STATE_PENDING = 2 # auto-reconnect in process
QUEUE_EXIT = 0
QUEUE_META = 1
QUEUE_PACKET = 2
DISCONNECT_PROBE_INTERVAL = 5
class CallbackContext:
def __init__(self):
self.queue = None
self.thread = None
self.packet_dispatch_allowed = False
self.lock = None
def __init__(self):
"""
Creates an IP Connection object that can be used to enumerate the available
devices. It is also required for the constructor of Bricks and Bricklets.
"""
self.host = None
self.port = None
self.timeout = 2.5
self.auto_reconnect = True
self.auto_reconnect_allowed = False
self.auto_reconnect_pending = False
self.sequence_number_lock = Lock()
self.next_sequence_number = 0 # protected by sequence_number_lock
self.authentication_lock = Lock() # protects authentication handshake
self.next_authentication_nonce = 0 # protected by authentication_lock
self.devices = {}
self.registered_callbacks = {}
self.socket = None # protected by socket_lock
self.socket_id = 0 # protected by socket_lock
self.socket_lock = Lock()
self.socket_send_lock = Lock()
self.receive_flag = False
self.receive_thread = None
self.callback = None
self.disconnect_probe_flag = False
self.disconnect_probe_queue = None
self.disconnect_probe_thread = None
self.waiter = Semaphore()
self.brickd = BrickDaemon("2", self)
def connect(self, host, port):
"""
Creates a TCP/IP connection to the given *host* and *port*. The host
and port can point to a Brick Daemon or to a WIFI/Ethernet Extension.
Devices can only be controlled when the connection was established
successfully.
Blocks until the connection is established and throws an exception if
there is no Brick Daemon or WIFI/Ethernet Extension listening at the
given host and port.
"""
with self.socket_lock:
if self.socket is not None:
raise Error(
Error.ALREADY_CONNECTED,
'Already connected to {0}:{1}'.format(
self.host, self.port))
self.host = host
self.port = port
self.connect_unlocked(False)
def disconnect(self):
"""
Disconnects the TCP/IP connection from the Brick Daemon or the
WIFI/Ethernet Extension.
"""
with self.socket_lock:
self.auto_reconnect_allowed = False
if self.auto_reconnect_pending:
# abort potentially pending auto reconnect
self.auto_reconnect_pending = False
else:
if self.socket is None:
raise Error(Error.NOT_CONNECTED, 'Not connected')
self.disconnect_unlocked()
# end callback thread
callback = self.callback
self.callback = None
# do this outside of socket_lock to allow calling (dis-)connect from
# the callbacks while blocking on the join call here
callback.queue.put((IPConnection.QUEUE_META,
(IPConnection.CALLBACK_DISCONNECTED,
IPConnection.DISCONNECT_REASON_REQUEST, None)))
callback.queue.put((IPConnection.QUEUE_EXIT, None))
if current_thread() is not callback.thread:
callback.thread.join()
def authenticate(self, secret):
"""
Performs an authentication handshake with the connected Brick Daemon or
WIFI/Ethernet Extension. If the handshake succeeds the connection switches
from non-authenticated to authenticated state and communication can
continue as normal. If the handshake fails then the connection gets closed.
Authentication can fail if the wrong secret was used or if authentication
is not enabled at all on the Brick Daemon or the WIFI/Ethernet Extension.
For more information about authentication see
http://www.tinkerforge.com/en/doc/Tutorials/Tutorial_Authentication/Tutorial.html
"""
secret_bytes = secret.encode('ascii')
with self.authentication_lock:
if self.next_authentication_nonce == 0:
try:
self.next_authentication_nonce = struct.unpack(
'<I', os.urandom(4))[0]
except NotImplementedError:
subseconds, seconds = math.modf(time.time())
seconds = int(seconds)
subseconds = int(subseconds * 1000000)
self.next_authentication_nonce = (
(seconds << 26 | seconds >> 6)
& 0xFFFFFFFF) + subseconds + os.getpid()
server_nonce = self.brickd.get_authentication_nonce()
client_nonce = struct.unpack(
'<4B', struct.pack('<I', self.next_authentication_nonce))
self.next_authentication_nonce = (self.next_authentication_nonce +
1) % (1 << 32)
h = hmac.new(secret_bytes, digestmod=hashlib.sha1)
h.update(struct.pack('<4B', *server_nonce))
h.update(struct.pack('<4B', *client_nonce))
digest = struct.unpack('<20B', h.digest())
h = None
self.brickd.authenticate(client_nonce, digest)
def get_connection_state(self):
"""
Can return the following states:
- CONNECTION_STATE_DISCONNECTED: No connection is established.
- CONNECTION_STATE_CONNECTED: A connection to the Brick Daemon or
the WIFI/Ethernet Extension is established.
- CONNECTION_STATE_PENDING: IP Connection is currently trying to
connect.
"""
if self.socket is not None:
return IPConnection.CONNECTION_STATE_CONNECTED
elif self.auto_reconnect_pending:
return IPConnection.CONNECTION_STATE_PENDING
else:
return IPConnection.CONNECTION_STATE_DISCONNECTED
def set_auto_reconnect(self, auto_reconnect):
"""
Enables or disables auto-reconnect. If auto-reconnect is enabled,
the IP Connection will try to reconnect to the previously given
host and port, if the connection is lost.
Default value is *True*.
"""
self.auto_reconnect = bool(auto_reconnect)
if not self.auto_reconnect:
# abort potentially pending auto reconnect
self.auto_reconnect_allowed = False
def get_auto_reconnect(self):
"""
Returns *true* if auto-reconnect is enabled, *false* otherwise.
"""
return self.auto_reconnect
def set_timeout(self, timeout):
"""
Sets the timeout in seconds for getters and for setters for which the
response expected flag is activated.
Default timeout is 2.5.
"""
timeout = float(timeout)
if timeout < 0:
raise ValueError('Timeout cannot be negative')
self.timeout = timeout
def get_timeout(self):
"""
Returns the timeout as set by set_timeout.
"""
return self.timeout
def enumerate(self):
"""
Broadcasts an enumerate request. All devices will respond with an
enumerate callback.
"""
request, _, _ = self.create_packet_header(
None, 8, IPConnection.FUNCTION_ENUMERATE)
self.send(request)
def wait(self):
"""
Stops the current thread until unwait is called.
This is useful if you rely solely on callbacks for events, if you want
to wait for a specific callback or if the IP Connection was created in
a thread.
Wait and unwait act in the same way as "acquire" and "release" of a
semaphore.
"""
self.waiter.acquire()
def unwait(self):
"""
Unwaits the thread previously stopped by wait.
Wait and unwait act in the same way as "acquire" and "release" of
a semaphore.
"""
self.waiter.release()
def register_callback(self, id, callback):
"""
Registers a callback with ID *id* to the function *callback*.
"""
if callback is None:
self.registered_callbacks.pop(id, None)
else:
self.registered_callbacks[id] = callback
def connect_unlocked(self, is_auto_reconnect):
# NOTE: assumes that socket is None and socket_lock is locked
# create callback thread and queue
if self.callback is None:
try:
self.callback = IPConnection.CallbackContext()
self.callback.queue = Queue()
self.callback.packet_dispatch_allowed = False
self.callback.lock = Lock()
self.callback.thread = Thread(name='Callback-Processor',
target=self.callback_loop,
args=(self.callback, ))
self.callback.thread.daemon = True
self.callback.thread.start()
except:
self.callback = None
raise
# create and connect socket
try:
tmp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tmp.settimeout(5)
tmp.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
tmp.connect((self.host, self.port))
tmp.settimeout(None)
except:
def cleanup():
# end callback thread
if not is_auto_reconnect:
self.callback.queue.put((IPConnection.QUEUE_EXIT, None))
if current_thread() is not self.callback.thread:
self.callback.thread.join()
self.callback = None
cleanup()
raise
self.socket = tmp
self.socket_id += 1
# create disconnect probe thread
try:
self.disconnect_probe_flag = True
self.disconnect_probe_queue = Queue()
self.disconnect_probe_thread = Thread(
name='Disconnect-Prober',
target=self.disconnect_probe_loop,
args=(self.disconnect_probe_queue, ))
self.disconnect_probe_thread.daemon = True
self.disconnect_probe_thread.start()
except:
def cleanup():
self.disconnect_probe_thread = None
# close socket
self.socket.close()
self.socket = None
# end callback thread
if not is_auto_reconnect:
self.callback.queue.put((IPConnection.QUEUE_EXIT, None))
if current_thread() is not self.callback.thread:
self.callback.thread.join()
self.callback = None
cleanup()
raise
# create receive thread
self.callback.packet_dispatch_allowed = True
try:
self.receive_flag = True
self.receive_thread = Thread(name='Brickd-Receiver',
target=self.receive_loop,
args=(self.socket_id, ))
self.receive_thread.daemon = True
self.receive_thread.start()
except:
def cleanup():
# close socket
self.disconnect_unlocked()
# end callback thread
if not is_auto_reconnect:
self.callback.queue.put((IPConnection.QUEUE_EXIT, None))
if current_thread() is not self.callback.thread:
self.callback.thread.join()
self.callback = None
cleanup()
raise
self.auto_reconnect_allowed = False
self.auto_reconnect_pending = False
if is_auto_reconnect:
connect_reason = IPConnection.CONNECT_REASON_AUTO_RECONNECT
else:
connect_reason = IPConnection.CONNECT_REASON_REQUEST
self.callback.queue.put(
(IPConnection.QUEUE_META, (IPConnection.CALLBACK_CONNECTED,
connect_reason, None)))
def disconnect_unlocked(self):
# NOTE: assumes that socket is not None and socket_lock is locked
# end disconnect probe thread
self.disconnect_probe_queue.put(True)
self.disconnect_probe_thread.join() # FIXME: use a timeout?
self.disconnect_probe_thread = None
# stop dispatching packet callbacks before ending the receive
# thread to avoid timeout exceptions due to callback functions
# trying to call getters
if current_thread() is not self.callback.thread:
# FIXME: cannot hold callback lock here because this can
# deadlock due to an ordering problem with the socket lock
#with self.callback.lock:
if True:
self.callback.packet_dispatch_allowed = False
else:
self.callback.packet_dispatch_allowed = False
# end receive thread
self.receive_flag = False
try:
self.socket.shutdown(socket.SHUT_RDWR)
except socket.error:
pass
if self.receive_thread is not None:
self.receive_thread.join() # FIXME: use a timeout?
self.receive_thread = None
# close socket
self.socket.close()
self.socket = None
def receive_loop(self, socket_id):
if sys.hexversion < 0x03000000:
pending_data = ''
else:
pending_data = bytes()
while self.receive_flag:
try:
data = self.socket.recv(8192)
except socket.error:
if self.receive_flag:
e = sys.exc_info()[1]
if e.errno == errno.EINTR:
continue
self.handle_disconnect_by_peer(
IPConnection.DISCONNECT_REASON_ERROR, socket_id, False)
break
if len(data) == 0:
if self.receive_flag:
self.handle_disconnect_by_peer(
IPConnection.DISCONNECT_REASON_SHUTDOWN, socket_id,
False)
break
pending_data += data
while self.receive_flag:
if len(pending_data) < 8:
# Wait for complete header
break
length = get_length_from_data(pending_data)
if len(pending_data) < length:
# Wait for complete packet
break
packet = pending_data[0:length]
pending_data = pending_data[length:]
self.handle_response(packet)
def dispatch_meta(self, function_id, parameter, socket_id):
if function_id == IPConnection.CALLBACK_CONNECTED:
if IPConnection.CALLBACK_CONNECTED in self.registered_callbacks:
self.registered_callbacks[IPConnection.CALLBACK_CONNECTED](
parameter)
elif function_id == IPConnection.CALLBACK_DISCONNECTED:
if parameter != IPConnection.DISCONNECT_REASON_REQUEST:
# need to do this here, the receive_loop is not allowed to
# hold the socket_lock because this could cause a deadlock
# with a concurrent call to the (dis-)connect function
with self.socket_lock:
# don't close the socket if it got disconnected or
# reconnected in the meantime
if self.socket is not None and self.socket_id == socket_id:
# end disconnect probe thread
self.disconnect_probe_queue.put(True)
self.disconnect_probe_thread.join(
) # FIXME: use a timeout?
self.disconnect_probe_thread = None
# close socket
self.socket.close()
self.socket = None
# FIXME: wait a moment here, otherwise the next connect
# attempt will succeed, even if there is no open server
# socket. the first receive will then fail directly
time.sleep(0.1)
if IPConnection.CALLBACK_DISCONNECTED in self.registered_callbacks:
self.registered_callbacks[IPConnection.CALLBACK_DISCONNECTED](
parameter)
if parameter != IPConnection.DISCONNECT_REASON_REQUEST and \
self.auto_reconnect and self.auto_reconnect_allowed:
self.auto_reconnect_pending = True
retry = True
# block here until reconnect. this is okay, there is no
# callback to deliver when there is no connection
while retry:
retry = False
with self.socket_lock:
if self.auto_reconnect_allowed and self.socket is None:
try:
self.connect_unlocked(True)
except:
retry = True
else:
self.auto_reconnect_pending = False
if retry:
time.sleep(0.1)
def dispatch_packet(self, packet):
uid = get_uid_from_data(packet)
length = get_length_from_data(packet)
function_id = get_function_id_from_data(packet)
payload = packet[8:]
if function_id == IPConnection.CALLBACK_ENUMERATE and \
IPConnection.CALLBACK_ENUMERATE in self.registered_callbacks:
uid, connected_uid, position, hardware_version, \
firmware_version, device_identifier, enumeration_type = \
self.deserialize_data(payload, '8s 8s c 3B 3B H B')
cb = self.registered_callbacks[IPConnection.CALLBACK_ENUMERATE]
cb(uid, connected_uid, position, hardware_version,
firmware_version, device_identifier, enumeration_type)
return
if uid not in self.devices:
return
device = self.devices[uid]
if function_id in device.registered_callbacks:
cb = device.registered_callbacks[function_id]
form = device.callback_formats[function_id]
if len(form) == 0:
cb()
elif len(form) == 1:
cb(self.deserialize_data(payload, form))
else:
cb(*self.deserialize_data(payload, form))
def callback_loop(self, callback):
while True:
kind, data = callback.queue.get()
# FIXME: cannot hold callback lock here because this can
# deadlock due to an ordering problem with the socket lock
#with callback.lock:
if True:
if kind == IPConnection.QUEUE_EXIT:
break
elif kind == IPConnection.QUEUE_META:
self.dispatch_meta(*data)
elif kind == IPConnection.QUEUE_PACKET:
# don't dispatch callbacks when the receive thread isn't running
if callback.packet_dispatch_allowed:
self.dispatch_packet(data)
# NOTE: the disconnect probe thread is not allowed to hold the socket_lock at any
# time because it is created and joined while the socket_lock is locked
def disconnect_probe_loop(self, disconnect_probe_queue):
request, _, _ = self.create_packet_header(
None, 8, IPConnection.FUNCTION_DISCONNECT_PROBE)
while True:
try:
disconnect_probe_queue.get(
True, IPConnection.DISCONNECT_PROBE_INTERVAL)
break
except Empty:
pass
if self.disconnect_probe_flag:
try:
with self.socket_send_lock:
self.socket.send(request)
except socket.error:
self.handle_disconnect_by_peer(
IPConnection.DISCONNECT_REASON_ERROR, self.socket_id,
False)
break
else:
self.disconnect_probe_flag = True
def deserialize_data(self, data, form):
ret = []
for f in form.split(' '):
f = '<' + f
length = struct.calcsize(f)
x = struct.unpack(f, data[:length])
if len(x) > 1:
if 'c' in f:
x = tuple([self.handle_deserialized_char(c) for c in x])
ret.append(x)
elif 'c' in f:
ret.append(self.handle_deserialized_char(x[0]))
elif 's' in f:
ret.append(self.handle_deserialized_string(x[0]))
else:
ret.append(x[0])
data = data[length:]
if len(ret) == 1:
return ret[0]
else:
return ret
def handle_deserialized_char(self, c):
if sys.hexversion >= 0x03000000:
try:
# c is a bytes object, try to decode it as ASCII. if it is
# not decodable keep it as a bytes object because there is no
# other option for this in Python 3
c = c.decode('ascii')
except:
pass
return c
def handle_deserialized_string(self, s):
nul = b'\x00'
if sys.hexversion >= 0x03000000:
try:
# s is a bytes object, try to decode it as ASCII. if it is
# not decodable keep it as a bytes object because there is no
# other option for this in Python 3
s = s.decode('ascii')
nul = '\x00'
except:
pass
i = s.find(nul)
if i >= 0:
s = s[:i]
return s
def send(self, packet):
with self.socket_lock:
if self.socket is None:
raise Error(Error.NOT_CONNECTED, 'Not connected')
try:
with self.socket_send_lock:
self.socket.send(packet)
except socket.error:
self.handle_disconnect_by_peer(
IPConnection.DISCONNECT_REASON_ERROR, None, True)
raise Error(Error.NOT_CONNECTED, 'Not connected')
self.disconnect_probe_flag = False
def send_request(self, device, function_id, data, form, form_ret):
length = 8 + struct.calcsize('<' + form)
request, response_expected, sequence_number = \
self.create_packet_header(device, length, function_id)
def pack_string(f, d):
if sys.hexversion < 0x03000000:
if isinstance(d, unicode):
f = f.replace('s', 'B').replace('c', 'B')
l = map(ord, d)
p = f.replace('B', '')
if len(p) == 0:
p = '1'
l += [0] * (int(p) - len(l))
return struct.pack('<' + f, *l)
else:
return struct.pack('<' + f, d)
else:
if isinstance(d, str):
return struct.pack('<' + f, bytes(map(ord, d)))
else:
return struct.pack('<' + f, d)
for f, d in zip(form.split(' '), data):
if len(f) > 1 and not 's' in f and not 'c' in f:
request += struct.pack('<' + f, *d)
elif 's' in f:
request += pack_string(f, d)
elif 'c' in f:
if len(f) > 1:
if int(f.replace('c', '')) != len(d):
raise ValueError('Incorrect char list length')
for k in d:
request += pack_string('c', k)
else:
request += pack_string(f, d)
else:
request += struct.pack('<' + f, d)
if response_expected:
with device.request_lock:
device.expected_response_function_id = function_id
device.expected_response_sequence_number = sequence_number
try:
self.send(request)
while True:
response = device.response_queue.get(
True, self.timeout)
if function_id == get_function_id_from_data(response) and \
sequence_number == get_sequence_number_from_data(response):
# ignore old responses that arrived after the timeout expired, but before setting
# expected_response_function_id and expected_response_sequence_number back to None
break
except Empty:
msg = 'Did not receive response for function {0} in time'.format(
function_id)
raise Error(Error.TIMEOUT, msg)
finally:
device.expected_response_function_id = None
device.expected_response_sequence_number = None
error_code = get_error_code_from_data(response)
if error_code == 0:
# no error
pass
elif error_code == 1:
msg = 'Got invalid parameter for function {0}'.format(
function_id)
raise Error(Error.INVALID_PARAMETER, msg)
elif error_code == 2:
msg = 'Function {0} is not supported'.format(function_id)
raise Error(Error.NOT_SUPPORTED, msg)
else:
msg = 'Function {0} returned an unknown error'.format(
function_id)
raise Error(Error.UNKNOWN_ERROR_CODE, msg)
if len(form_ret) > 0:
return self.deserialize_data(response[8:], form_ret)
else:
self.send(request)
def get_next_sequence_number(self):
with self.sequence_number_lock:
sequence_number = self.next_sequence_number + 1
self.next_sequence_number = sequence_number % 15
return sequence_number
def handle_response(self, packet):
self.disconnect_probe_flag = False
function_id = get_function_id_from_data(packet)
sequence_number = get_sequence_number_from_data(packet)
if sequence_number == 0 and function_id == IPConnection.CALLBACK_ENUMERATE:
if IPConnection.CALLBACK_ENUMERATE in self.registered_callbacks:
self.callback.queue.put((IPConnection.QUEUE_PACKET, packet))
return
uid = get_uid_from_data(packet)
if not uid in self.devices:
# Response from an unknown device, ignoring it
return
device = self.devices[uid]
if sequence_number == 0:
if function_id in device.registered_callbacks:
self.callback.queue.put((IPConnection.QUEUE_PACKET, packet))
return
if device.expected_response_function_id == function_id and \
device.expected_response_sequence_number == sequence_number:
device.response_queue.put(packet)
return
# Response seems to be OK, but can't be handled
def handle_disconnect_by_peer(self, disconnect_reason, socket_id,
disconnect_immediately):
# NOTE: assumes that socket_lock is locked if disconnect_immediately is true
self.auto_reconnect_allowed = True
if disconnect_immediately:
self.disconnect_unlocked()
self.callback.queue.put(
(IPConnection.QUEUE_META, (IPConnection.CALLBACK_DISCONNECTED,
disconnect_reason, socket_id)))
def create_packet_header(self, device, length, function_id):
uid = IPConnection.BROADCAST_UID
sequence_number = self.get_next_sequence_number()
r_bit = 0
if device is not None:
uid = device.uid
if device.get_response_expected(function_id):
r_bit = 1
sequence_number_and_options = (sequence_number << 4) | (r_bit << 3)
return (struct.pack('<IBBBB', uid, length, function_id,
sequence_number_and_options,
0), bool(r_bit), sequence_number)
def write_bricklet_plugin(self, device, port, position, plugin_chunk):
self.send_request(device, IPConnection.FUNCTION_WRITE_BRICKLET_PLUGIN,
(port, position, plugin_chunk), 'c B 32B', '')
def read_bricklet_plugin(self, device, port, position):
return self.send_request(device,
IPConnection.FUNCTION_READ_BRICKLET_PLUGIN,
(port, position), 'c B', '32B')
def get_adc_calibration(self, device):
return self.send_request(device,
IPConnection.FUNCTION_GET_ADC_CALIBRATION, (),
'', 'h h')
def adc_calibrate(self, device, port):
self.send_request(device, IPConnection.FUNCTION_ADC_CALIBRATE,
(port, ), 'c', '')
def write_bricklet_uid(self, device, port, uid):
uid_int = base58decode(uid)
self.send_request(device, IPConnection.FUNCTION_WRITE_BRICKLET_UID,
(port, uid_int), 'c I', '')
def read_bricklet_uid(self, device, port):
uid_int = self.send_request(device,
IPConnection.FUNCTION_READ_BRICKLET_UID,
(port, ), 'c', 'I')
return base58encode(uid_int)
class Registry(Thread):
"""Class which transparently provides access to the enumeration."""
_instance_count = 0
def __init__(self, scripts_path=Path(SCRIPTS_ROOT)):
# Required by the thread class.
super().__init__(name="Registry-%i" % (Registry._instance_count), daemon=True)
Registry._instance_count += 1
# Prepare fields.
if isinstance(scripts_path, str):
scripts_path = Path(scripts_path)
elif not isinstance(scripts_path, Path):
raise ValueError("scripts_path must be either a string or a Path.")
self.path = scripts_path
self.lock = Semaphore()
self.state = RegistryStates.waiting
self.root = None
self.observer = RegistryObserver()
# Fire up the thread.
self.start()
def _scan_ahead(self):
"Scans and parses directory structure."
child_list = []
self.root = CategoryEntry(path=self.path)
pending_cats = deque([self.root])
while len(pending_cats) != 0:
current_cat = pending_cats.pop()
current_path = current_cat.path
log.info("Currently scanning: %s" % (current_path.name))
for file in sorted(current_path.iterdir()):
if match_list(file.name, SCRIPTS_IGNORE):
# Ignore case.
log.warning("Ignoring: %s" % (file.name))
elif file.is_dir():
# Directory case.
new_cat = CategoryEntry(parent=current_cat, path=file)
key = file.name
current_cat[key] = new_cat
pending_cats.appendleft(new_cat)
else:
# Executable case.
if os.access(str(file), os.X_OK):
new_child = ChildHandler(path=file, root_cat=current_cat, parent=self)
child_list.append(new_child)
else:
log.warning("Ignoring: %s" % (file.name))
return child_list
def _execute_children(self, children):
"Multi-dispatch children."
# Children waiting execution.
pending = deque(children)
# Children currently under execution.
executing = [None] * SCRIPTS_MAX_MULTI_DISPATCH
while len(pending) != 0 or executing.count(None) != len(executing):
for i, slot in enumerate(executing):
if slot is None and len(pending) != 0:
# Get the next available child.
executing[i] = pending.popleft().start()
elif slot is not None and slot.do_step():
# Child has finished, pick next one.
if len(pending) != 0:
executing[i] = pending.popleft().start()
else:
executing[i] = None
# Finish up.
for c in children:
c.finish_up()
def run(self):
log.info("Registry thread is starting up...")
while True:
if self.lock.acquire(blocking=True) and self.state is RegistryStates.start:
log.debug("A refresh has been requested.")
# Actual refresh starts here.
# First scan the directories.
self.state = RegistryStates.scanning
pending_children = self._scan_ahead()
# Take a break to process events.
GObject.idle_add(make_signal_emitter(self.observer, "scan_complete"))
# Start collecting and parsing commands in steps,
# so the main thread is not getting blocked.
self.state = RegistryStates.collecting
self._execute_children(pending_children)
# Change state and prepare to hand data to main thread.
self.state = RegistryStates.finish
# Inform main thread.
GObject.idle_add(make_signal_emitter(self.observer, "refresh_complete"))
log.debug("Registry refresh has been completed.")
def request_refresh(self):
"This method reloads the enumeration by running all the scripts(async version)."
if self.state is RegistryStates.waiting:
self.state = RegistryStates.start
self.lock.release()
return True
else:
raise RuntimeError("Refresh requested but registry is in an invalid state!")
def get_async_data(self):
"Gets data(an Entry tree) after a refresh request has finished. Else returns false."
if self.state is RegistryStates.finish:
# Reset state to prepare for next refresh.
self.state = RegistryStates.waiting
return self.root
else:
return False
def refresh(self):
"This method reloads the enumeration by running all the scripts(synchronized version). No glib main loop must be running!"
# Registry needs a glib main loop to be running.
self.observer.loop = GObject.MainLoop()
# Register event handlers.
self.observer.connect("refresh_complete", lambda self: self.loop.quit())
# Start refresh.
self.request_refresh()
# Enter main loop.
self.observer.loop.run()
# Cleanup.
del self.observer.loop
# The refresh has already finished, so this resets the state and also returns the root.
return self.get_async_data()
class Parallel:
def __init__(self,
ncpu,
lsfQueue=None,
lsfJobName="job",
asyncLsf=False,
maxThreads=500,
jobDriver=None,
batchSystem="lsf"):
self.returned = Queue()
self.njobs = 0
self.JobDriver = jobDriver
self.lsfJobs = Queue()
self.lsfQueue = lsfQueue
self.lsfJobName = lsfJobName
self.jobId = 0
self.sem = Semaphore()
self.maxThreads = maxThreads
self.asyncLsf = asyncLsf
self.batchSystem = batchSystem
if self.lsfQueue:
self.running = Queue()
else:
self.running = Queue(ncpu)
if not self.JobDriver:
self.JobDriver = BatchRegistry.getRunner(self.batchSystem)
if self.lsfQueue and self.asyncLsf:
self.lsfMon = BatchRegistry.getMonitor(self.batchSystem)(
self.lsfJobs, self.returned)
thread = Thread(None, self.lsfMon)
thread.start()
def run(self, cmd, args, interactive=False, jobName=None):
myargs = [cmd, args, interactive]
if jobName:
myargs.append(jobName)
wrap = Wrap(self, myargs, self.returned, self.running)
if interactive:
return wrap(interactive=True)
while threading.activeCount() > self.maxThreads:
sleep(0.05)
ret = (None, (None, None))
if not (self.lsfQueue and self.asyncLsf):
thread = Thread(None, wrap)
thread.start()
else:
ret = wrap(interactive=True)
self.sem.acquire()
self.njobs += 1
self.sem.release()
return ret
def addJob(self, cmd, args, batchId, jobName=None):
if not self.asyncLsf:
return
job = self.JobDriver(self.lsfQueue, jobName, async=True)
job.setJobId(batchId)
job.cmd = " ".join([cmd] + args)
self.lsfJobs.put(job)
self.sem.acquire()
self.njobs += 1
self.sem.release()
def currJobId(self):
self.sem.acquire()
ret = self.jobId
self.sem.release()
return ret
def setJobId(self, jobId):
self.sem.acquire()
self.jobId = jobId
self.sem.release()
def getJobId(self):
self.sem.acquire()
ret = self.jobId
self.jobId += 1
self.sem.release()
return ret
def __call__(self, cmd, args, interactive, jobName=None, replacesJob=None):
if type(cmd) == str or type(cmd) == unicode:
## print cmd
cmd = "%s %s" % (cmd, " ".join(args))
args = (cmd, )
if self.lsfQueue and not interactive:
if not jobName:
jobName = "%s%d" % (self.lsfJobName, self.getJobId())
cmd = self.JobDriver(self.lsfQueue,
jobName,
async=self.asyncLsf,
replacesJob=replacesJob)
else:
cmd = commands.getstatusoutput
ret = cmd(*args)
if self.lsfQueue and not interactive and self.asyncLsf:
self.lsfJobs.put(cmd)
return cmd, args, ret
def stop(self):
if self.lsfQueue and self.asyncLsf:
self.lsfMon.stop = True
def wait(self, handler=None, printOutput=True, struggleThr=0.):
returns = []
self.sem.acquire()
njobs = int(self.njobs)
self.sem.release()
nleft = njobs
nstruggle = int(floor(nleft * struggleThr))
while nleft > 0:
print ""
print "--- Running jobs: %d. Total jobs: %d (total submissions: %s)" % (
nleft, njobs, self.njobs)
print ""
job, jobargs, ret = self.returned.get()
if printOutput:
try:
print "Job finished: '%s' '%s'" % (job, " ".join(
[str(a) for a in jobargs]))
if not handler:
for line in ret[1].split("\n"):
print line
except:
pass
if handler:
nleft += handler.handleJobOutput(job, jobargs, ret)
else:
returns.append((job, jobargs, ret))
nleft -= 1
if nleft != 0 and nleft == nstruggle and handler:
handler.analyzeStrugglers(self)
nstruggle = int(floor(nstruggle * struggleThr))
self.sem.acquire()
self.njobs -= njobs
self.sem.release()
return returns
